From 11ddeba615b8a73b73425d4b0fb075eb9e38cf56 Mon Sep 17 00:00:00 2001 From: Karl Berry Date: Tue, 9 Apr 2013 17:05:52 +0000 Subject: context (8apr13 18:03) git-svn-id: svn://tug.org/texlive/trunk@29800 c570f23f-e606-0410-a88d-b1316a301751 --- .../linked_scripts/context/stubs/unix/mtxrun | 103 +- Master/bin/win32/ctxtools.exe | Bin 6144 -> 6144 bytes Master/bin/win32/mtxrun.dll | Bin 9216 -> 38400 bytes Master/bin/win32/mtxrun.exe | Bin 6144 -> 34816 bytes Master/bin/win32/mtxrun.lua | 29825 ++++++++++--------- .../context/data/scite/scite-context-readme.pdf | Bin 212111 -> 212029 bytes .../context/data/scite/scite-context-readme.tex | 391 +- Master/texmf-dist/doc/man/man1/context.man1.pdf | Bin 8837 -> 7542 bytes Master/texmf-dist/doc/man/man1/ctxtools.man1.pdf | Bin 6253 -> 5096 bytes Master/texmf-dist/doc/man/man1/luatools.man1.pdf | Bin 5105 -> 5105 bytes Master/texmf-dist/doc/man/man1/mtx-babel.man1.pdf | Bin 3672 -> 3672 bytes Master/texmf-dist/doc/man/man1/mtx-base.man1.pdf | Bin 5124 -> 5124 bytes Master/texmf-dist/doc/man/man1/mtx-cache.man1.pdf | Bin 3707 -> 3707 bytes Master/texmf-dist/doc/man/man1/mtx-chars.man1.pdf | Bin 3566 -> 3566 bytes Master/texmf-dist/doc/man/man1/mtx-check.man1.pdf | Bin 3580 -> 3580 bytes Master/texmf-dist/doc/man/man1/mtx-colors.man1.pdf | Bin 3442 -> 3442 bytes .../texmf-dist/doc/man/man1/mtx-context.man1.pdf | Bin 7542 -> 7542 bytes Master/texmf-dist/doc/man/man1/mtx-epub.man1.pdf | Bin 3535 -> 3535 bytes Master/texmf-dist/doc/man/man1/mtx-fcd.man1.pdf | Bin 3960 -> 3960 bytes Master/texmf-dist/doc/man/man1/mtx-flac.man1.pdf | Bin 3625 -> 3625 bytes Master/texmf-dist/doc/man/man1/mtx-fonts.man1.pdf | Bin 4206 -> 4206 bytes Master/texmf-dist/doc/man/man1/mtx-grep.man1.pdf | Bin 3545 -> 3545 bytes .../texmf-dist/doc/man/man1/mtx-interface.man1.pdf | Bin 4105 -> 4105 bytes .../texmf-dist/doc/man/man1/mtx-metapost.man1.pdf | 16 +- .../texmf-dist/doc/man/man1/mtx-metatex.man1.pdf | Bin 3630 -> 3630 bytes .../texmf-dist/doc/man/man1/mtx-modules.man1.pdf | Bin 3760 -> 3760 bytes .../texmf-dist/doc/man/man1/mtx-package.man1.pdf | Bin 3568 -> 3568 bytes .../texmf-dist/doc/man/man1/mtx-patterns.man1.pdf | Bin 4013 -> 4013 bytes Master/texmf-dist/doc/man/man1/mtx-pdf.man1.pdf | Bin 3661 -> 3661 bytes .../texmf-dist/doc/man/man1/mtx-profile.man1.pdf | 16 +- Master/texmf-dist/doc/man/man1/mtx-rsync.man1.pdf | Bin 3635 -> 3635 bytes Master/texmf-dist/doc/man/man1/mtx-scite.man1.pdf | Bin 3446 -> 3446 bytes Master/texmf-dist/doc/man/man1/mtx-server.man1.pdf | 16 +- .../texmf-dist/doc/man/man1/mtx-texworks.man1.pdf | Bin 3557 -> 3557 bytes Master/texmf-dist/doc/man/man1/mtx-timing.man1.pdf | Bin 3679 -> 3679 bytes Master/texmf-dist/doc/man/man1/mtx-tools.man1.pdf | Bin 3967 -> 3967 bytes Master/texmf-dist/doc/man/man1/mtx-unzip.man1.pdf | 16 +- Master/texmf-dist/doc/man/man1/mtx-update.man1.pdf | Bin 5030 -> 5030 bytes Master/texmf-dist/doc/man/man1/mtx-watch.man1.pdf | Bin 3867 -> 3867 bytes Master/texmf-dist/doc/man/man1/mtxrun.man1.pdf | Bin 7032 -> 7032 bytes Master/texmf-dist/doc/man/man1/pstopdf.man1.pdf | Bin 4945 -> 3557 bytes Master/texmf-dist/doc/man/man1/texexec.man1.pdf | Bin 28831 -> 4713 bytes Master/texmf-dist/doc/man/man1/texmfstart.man1.pdf | Bin 13752 -> 7032 bytes Master/texmf-dist/scripts/context/lua/mtxrun.lua | 103 +- .../scripts/context/stubs/mswin/mtxrun.lua | 103 +- .../texmf-dist/scripts/context/stubs/unix/mtxrun | 103 +- Master/texmf-dist/tex/context/base/chem-str.lua | 80 +- Master/texmf-dist/tex/context/base/cont-new.mkiv | 4 +- .../tex/context/base/context-version.pdf | Bin 4131 -> 4128 bytes .../tex/context/base/context-version.png | Bin 40047 -> 40483 bytes Master/texmf-dist/tex/context/base/context.mkiv | 2 +- Master/texmf-dist/tex/context/base/data-res.lua | 5 +- Master/texmf-dist/tex/context/base/l-lua.lua | 13 +- Master/texmf-dist/tex/context/base/math-ali.mkiv | 12 +- .../texmf-dist/tex/context/base/status-files.pdf | Bin 24764 -> 24752 bytes Master/texmf-dist/tex/context/base/status-lua.pdf | Bin 211760 -> 211622 bytes Master/texmf-dist/tex/context/base/tabl-xtb.lua | 12 +- Master/texmf-dist/tex/context/base/util-lib.lua | 98 +- .../generic/context/luatex/luatex-fonts-merged.lua | 14 +- 59 files changed, 15684 insertions(+), 15248 deletions(-) diff --git a/Build/source/texk/texlive/linked_scripts/context/stubs/unix/mtxrun b/Build/source/texk/texlive/linked_scripts/context/stubs/unix/mtxrun index 00f63a5791d..d337ed75f74 100755 --- a/Build/source/texk/texlive/linked_scripts/context/stubs/unix/mtxrun +++ b/Build/source/texk/texlive/linked_scripts/context/stubs/unix/mtxrun @@ -56,7 +56,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-lua"] = package.loaded["l-lua"] or true --- original size: 10048, stripped down to: 5684 +-- original size: 10352, stripped down to: 5955 if not modules then modules={} end modules ['l-lua']={ version=1.001, @@ -247,7 +247,7 @@ local function loadedbypath(name,rawname,paths,islib,what) end local function notloaded(name) if helpers.trace then - helpers.report("? unable to locate library '%s'",name) + helpers.report("unable to locate library '%s'",name) end end helpers.loadedaslib=loadedaslib @@ -278,6 +278,16 @@ function helpers.loaded(name) end return notloaded(name) end +function helpers.unload(name) + if helpers.trace then + if package.loaded[name] then + helpers.report("unloading library '%s', %s",name,"done") + else + helpers.report("unloading library '%s', %s",name,"not loaded") + end + end + package.loaded[name]=nil +end end -- of closure @@ -12632,7 +12642,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-res"] = package.loaded["data-res"] or true --- original size: 60821, stripped down to: 42503 +-- original size: 60857, stripped down to: 42496 if not modules then modules={} end modules ['data-res']={ version=1.001, @@ -12807,7 +12817,7 @@ local slash=P("/") local pathexpressionpattern=Cs ( Cc("^")*( Cc("%")*S(".-")+slash^2*P(-1)/"/.*" -+slash^2/"/[^/]*/*"+(1-slash)*P(-1)*Cc("/")+P(1) ++slash^2/"/"+(1-slash)*P(-1)*Cc("/")+P(1) )^1*Cc("$") ) local cache={} @@ -15403,7 +15413,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-lib"] = package.loaded["util-lib"] or true --- original size: 8911, stripped down to: 4216 +-- original size: 10762, stripped down to: 5269 if not modules then modules={} end modules ['util-lib']={ version=1.001, @@ -15418,6 +15428,7 @@ local findfile,findfiles=resolvers and resolvers.findfile,resolvers and resolver local loaded=package.loaded local report_swiglib=logs.reporter("swiglib") local trace_swiglib=false trackers.register("resolvers.swiglib",function(v) trace_swiglib=v end) +local done=false local function requireswiglib(required,version) local library=loaded[required] if library==nil then @@ -15426,32 +15437,48 @@ local function requireswiglib(required,version) local required_base=nameonly(required_full) local required_name=required_base.."."..os.libsuffix local version=type(version)=="string" and version~="" and version or false + local engine=environment.ownmain or false + if trace_swiglib and not done then + local list=resolvers.expandedpathlistfromvariable("lib") + for i=1,#list do + report_swiglib("tds path %i: %s",i,list[i]) + end + end + local function found(locate,asked_library,how,...) + if trace_swiglib then + report_swiglib("checking %s: %a",how,asked_library) + end + return locate(asked_library,...) + end local function check(locate,...) - local found_library=nil + local found=nil if version then local asked_library=joinfile(required_path,version,required_name) if trace_swiglib then report_swiglib("checking %s: %a","with version",asked_library) end - found_library=locate(asked_library,...) - if not found_library or found_library==""then - asked_library=joinfile(required_path,required_name) - if trace_swiglib then - report_swiglib("checking %s: %a","without version",asked_library) - end - found_library=locate(asked_library,...) - end - else + found=locate(asked_library,...) + end + if not found or found=="" then local asked_library=joinfile(required_path,required_name) if trace_swiglib then - report_swiglib("checking %s: %a","without version",asked_library) + report_swiglib("checking %s: %a","with version",asked_library) end - found_library=locate(asked_library,...) + found=locate(asked_library,...) end - return found_library and found_library~="" and found_library or false + return found and found~="" and found or false end - local found_library=findfile and check(findfile,"lib") - if findfiles and not found_library then + local function attempt(checkpattern) + if trace_swiglib then + report_swiglib("checking tds lib paths strictly") + end + local found=findfile and check(findfile,"lib") + if found and (not checkpattern or find(found,checkpattern)) then + return found + end + if trace_swiglib then + report_swiglib("checking tds lib paths with wildcard") + end local asked_library=joinfile(required_path,".*",required_name) if trace_swiglib then report_swiglib("checking %s: %a","latest version",asked_library) @@ -15459,18 +15486,38 @@ local function requireswiglib(required,version) local list=findfiles(asked_library,"lib",true) if list and #list>0 then table.sort(list) - found_library=list[#list] + local found=list[#list] + if found and (not checkpattern or find(found,checkpattern)) then + return found + end + end + if trace_swiglib then + report_swiglib("checking clib paths") end - end - if not found_library then package.extraclibpath(environment.ownpath) local paths=package.clibpaths() for i=1,#paths do - local found_library=check(lfs.isfile) - if found_library then - break + local found=check(lfs.isfile) + if found and (not checkpattern or find(found,checkpattern)) then + return found end end + return false + end + local found_library=nil + if engine then + if trace_swiglib then + report_swiglib("attemp 1, engine %a",engine) + end + found_library=attempt("/"..engine.."/") + if not found_library then + if trace_swiglib then + report_swiglib("attemp 2, no engine",asked_library) + end + found_library=attempt() + end + else + found_library=attempt() end if not found_library then if trace_swiglib then @@ -15773,8 +15820,8 @@ end -- of closure -- used libraries : l-lua.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-md5.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-sto.lua util-prs.lua util-fmt.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-mrg.lua util-tpl.lua util-env.lua luat-env.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua -- skipped libraries : - --- original bytes : 658276 --- stripped bytes : 241564 +-- original bytes : 660467 +-- stripped bytes : 242438 -- end library merge diff --git a/Master/bin/win32/ctxtools.exe b/Master/bin/win32/ctxtools.exe index 745eaf22464..2d45f27494d 100755 Binary files a/Master/bin/win32/ctxtools.exe and b/Master/bin/win32/ctxtools.exe differ diff --git a/Master/bin/win32/mtxrun.dll b/Master/bin/win32/mtxrun.dll index 4116c5a24a6..6ba2597d56f 100755 Binary files a/Master/bin/win32/mtxrun.dll and b/Master/bin/win32/mtxrun.dll differ diff --git a/Master/bin/win32/mtxrun.exe b/Master/bin/win32/mtxrun.exe index 745eaf22464..acd99ddbf3b 100755 Binary files a/Master/bin/win32/mtxrun.exe and b/Master/bin/win32/mtxrun.exe differ diff --git a/Master/bin/win32/mtxrun.lua b/Master/bin/win32/mtxrun.lua index 108f2a8a112..d337ed75f74 100755 --- a/Master/bin/win32/mtxrun.lua +++ b/Master/bin/win32/mtxrun.lua @@ -1,5 +1,16 @@ #!/usr/bin/env texlua +-- for k, v in next, _G.string do +-- local tv = type(v) +-- if tv == "table" then +-- for kk, vv in next, v do +-- print(k,kk,vv) +-- end +-- else +-- print(tv,k,v) +-- end +-- end + if not modules then modules = { } end modules ['mtxrun'] = { version = 1.001, comment = "runner, lua replacement for texmfstart.rb", @@ -43,3081 +54,6807 @@ if not modules then modules = { } end modules ['mtxrun'] = { do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['l-string'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local string = string -local sub, gsub, find, match, gmatch, format, char, byte, rep, lower = string.sub, string.gsub, string.find, string.match, string.gmatch, string.format, string.char, string.byte, string.rep, string.lower -local lpegmatch, S, C, Ct = lpeg.match, lpeg.S, lpeg.C, lpeg.Ct - --- some functions may disappear as they are not used anywhere +package.loaded["l-lua"] = package.loaded["l-lua"] or true -if not string.split then +-- original size: 10352, stripped down to: 5955 - -- this will be overloaded by a faster lpeg variant - - function string.split(str,pattern) - local t = { } - if #str > 0 then - local n = 1 - for s in gmatch(str..pattern,"(.-)"..pattern) do - t[n] = s - n = n + 1 - end - end - return t +if not modules then modules={} end modules ['l-lua']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local major,minor=string.match(_VERSION,"^[^%d]+(%d+)%.(%d+).*$") +_MAJORVERSION=tonumber(major) or 5 +_MINORVERSION=tonumber(minor) or 1 +_LUAVERSION=_MAJORVERSION+_MINORVERSION/10 +if not lpeg then + lpeg=require("lpeg") +end +if loadstring then + local loadnormal=load + function load(first,...) + if type(first)=="string" then + return loadstring(first,...) + else + return loadnormal(first,...) end - + end +else + loadstring=load end - -function string.unquoted(str) - return (gsub(str,"^([\"\'])(.*)%1$","%2")) +if not ipairs then + local function iterate(a,i) + i=i+1 + local v=a[i] + if v~=nil then + return i,v + end + end + function ipairs(a) + return iterate,a,0 + end end - - -function string.quoted(str) - return format("%q",str) -- always " +if not pairs then + function pairs(t) + return next,t + end end - -function string.count(str,pattern) -- variant 3 - local n = 0 - for _ in gmatch(str,pattern) do -- not for utf - n = n + 1 +if not table.unpack then + table.unpack=_G.unpack +elseif not unpack then + _G.unpack=table.unpack +end +if not package.loaders then + package.loaders=package.searchers +end +local print,select,tostring=print,select,tostring +local inspectors={} +function setinspector(inspector) + inspectors[#inspectors+1]=inspector +end +function inspect(...) + for s=1,select("#",...) do + local value=select(s,...) + local done=false + for i=1,#inspectors do + done=inspectors[i](value) + if done then + break + end end - return n + if not done then + print(tostring(value)) + end + end end - -function string.limit(str,n,sentinel) -- not utf proof - if #str > n then - sentinel = sentinel or "..." - return sub(str,1,(n-#sentinel)) .. sentinel +local dummy=function() end +function optionalrequire(...) + local ok,result=xpcall(require,dummy,...) + if ok then + return result + end +end +local type=type +local gsub,format=string.gsub,string.format +local package=package +local searchers=package.searchers or package.loaders +local libpaths=nil +local clibpaths=nil +local libhash={} +local clibhash={} +local libextras={} +local clibextras={} +local filejoin=file and file.join or function(path,name) return path.."/"..name end +local isreadable=file and file.is_readable or function(name) local f=io.open(name) if f then f:close() return true end end +local addsuffix=file and file.addsuffix or function(name,suffix) return name.."."..suffix end +local function cleanpath(path) + return path +end +local helpers=package.helpers or { + libpaths=function() return {} end, + clibpaths=function() return {} end, + cleanpath=cleanpath, + trace=false, + report=function(...) print(format(...)) end, +} +package.helpers=helpers +local function getlibpaths() + return libpaths or helpers.libpaths(libhash) +end +local function getclibpaths() + return clibpaths or helpers.clibpaths(clibhash) +end +package.libpaths=getlibpaths +package.clibpaths=getclibpaths +local function addpath(what,paths,extras,hash,...) + local pathlist={... } + local cleanpath=helpers.cleanpath + local trace=helpers.trace + local report=helpers.report + local function add(path) + local path=cleanpath(path) + if not hash[path] then + if trace then + report("extra %s path: %s",what,path) + end + paths [#paths+1]=path + extras[#extras+1]=path + end + end + for p=1,#pathlist do + local path=pathlist[p] + if type(path)=="table" then + for i=1,#path do + add(path[i]) + end + else + add(path) + end + end + return paths,extras +end +function package.extralibpath(...) + libpaths,libextras=addpath("lua",getlibpaths(),libextras,libhash,...) +end +function package.extraclibpath(...) + clibpaths,clibextras=addpath("lib",getclibpaths(),clibextras,clibhash,...) +end +if not searchers[-2] then + searchers[-2]=searchers[2] +end +searchers[2]=function(name) + return helpers.loaded(name) +end +searchers[3]=nil +local function loadedaslib(resolved,rawname) + local init="luaopen_"..gsub(rawname,"%.","_") + if helpers.trace then + helpers.report("calling loadlib with '%s' with init '%s'",resolved,init) + end + return package.loadlib(resolved,init) +end +local function loadedbylua(name) + if helpers.trace then + helpers.report("locating '%s' using normal loader",name) + end + return true,searchers[-2](name) +end +local function loadedbypath(name,rawname,paths,islib,what) + local trace=helpers.trace + local report=helpers.report + if trace then + report("locating '%s' as '%s' on '%s' paths",rawname,name,what) + end + for p=1,#paths do + local path=paths[p] + local resolved=filejoin(path,name) + if trace then + report("checking for '%s' using '%s' path '%s'",name,what,path) + end + if isreadable(resolved) then + if trace then + report("lib '%s' located on '%s'",name,resolved) + end + if islib then + return true,loadedaslib(resolved,rawname) + else + return true,loadfile(resolved) + end + end + end +end +local function notloaded(name) + if helpers.trace then + helpers.report("unable to locate library '%s'",name) + end +end +helpers.loadedaslib=loadedaslib +helpers.loadedbylua=loadedbylua +helpers.loadedbypath=loadedbypath +helpers.notloaded=notloaded +function helpers.loaded(name) + local thename=gsub(name,"%.","/") + local luaname=addsuffix(thename,"lua") + local libname=addsuffix(thename,os.libsuffix or "so") + local libpaths=getlibpaths() + local clibpaths=getclibpaths() + local done,result=loadedbypath(luaname,name,libpaths,false,"lua") + if done then + return result + end + local done,result=loadedbypath(luaname,name,clibpaths,false,"lua") + if done then + return result + end + local done,result=loadedbypath(libname,name,clibpaths,true,"lib") + if done then + return result + end + local done,result=loadedbylua(name) + if done then + return result + end + return notloaded(name) +end +function helpers.unload(name) + if helpers.trace then + if package.loaded[name] then + helpers.report("unloading library '%s', %s",name,"done") else - return str + helpers.report("unloading library '%s', %s",name,"not loaded") end + end + package.loaded[name]=nil end -local space = S(" \t\v\n") -local nospace = 1 - space -local stripper = space^0 * C((space^0 * nospace^1)^0) -- roberto's code -function string.strip(str) - return lpegmatch(stripper,str) or "" -end +end -- of closure -function string.is_empty(str) - return not find(str,"%S") -end +do -- create closure to overcome 200 locals limit -local patterns_escapes = { - ["%"] = "%%", - ["."] = "%.", - ["+"] = "%+", ["-"] = "%-", ["*"] = "%*", - ["["] = "%[", ["]"] = "%]", - ["("] = "%(", [")"] = "%)", - -- ["{"] = "%{", ["}"] = "%}" - -- ["^"] = "%^", ["$"] = "%$", -} +package.loaded["l-lpeg"] = package.loaded["l-lpeg"] or true -local simple_escapes = { - ["-"] = "%-", - ["."] = "%.", - ["?"] = ".", - ["*"] = ".*", -} +-- original size: 26252, stripped down to: 14371 -function string.escapedpattern(str,simple) - return (gsub(str,".",simple and simple_escapes or patterns_escapes)) +if not modules then modules={} end modules ['l-lpeg']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +lpeg=require("lpeg") +local type,next,tostring=type,next,tostring +local byte,char,gmatch,format=string.byte,string.char,string.gmatch,string.format +local floor=math.floor +local P,R,S,V,Ct,C,Cs,Cc,Cp,Cmt=lpeg.P,lpeg.R,lpeg.S,lpeg.V,lpeg.Ct,lpeg.C,lpeg.Cs,lpeg.Cc,lpeg.Cp,lpeg.Cmt +local lpegtype,lpegmatch,lpegprint=lpeg.type,lpeg.match,lpeg.print +setinspector(function(v) if lpegtype(v) then lpegprint(v) return true end end) +lpeg.patterns=lpeg.patterns or {} +local patterns=lpeg.patterns +local anything=P(1) +local endofstring=P(-1) +local alwaysmatched=P(true) +patterns.anything=anything +patterns.endofstring=endofstring +patterns.beginofstring=alwaysmatched +patterns.alwaysmatched=alwaysmatched +local digit,sign=R('09'),S('+-') +local cr,lf,crlf=P("\r"),P("\n"),P("\r\n") +local newline=crlf+S("\r\n") +local escaped=P("\\")*anything +local squote=P("'") +local dquote=P('"') +local space=P(" ") +local utfbom_32_be=P('\000\000\254\255') +local utfbom_32_le=P('\255\254\000\000') +local utfbom_16_be=P('\255\254') +local utfbom_16_le=P('\254\255') +local utfbom_8=P('\239\187\191') +local utfbom=utfbom_32_be+utfbom_32_le+utfbom_16_be+utfbom_16_le+utfbom_8 +local utftype=utfbom_32_be*Cc("utf-32-be")+utfbom_32_le*Cc("utf-32-le")+utfbom_16_be*Cc("utf-16-be")+utfbom_16_le*Cc("utf-16-le")+utfbom_8*Cc("utf-8")+alwaysmatched*Cc("utf-8") +local utfoffset=utfbom_32_be*Cc(4)+utfbom_32_le*Cc(4)+utfbom_16_be*Cc(2)+utfbom_16_le*Cc(2)+utfbom_8*Cc(3)+Cc(0) +local utf8next=R("\128\191") +patterns.utf8one=R("\000\127") +patterns.utf8two=R("\194\223")*utf8next +patterns.utf8three=R("\224\239")*utf8next*utf8next +patterns.utf8four=R("\240\244")*utf8next*utf8next*utf8next +patterns.utfbom=utfbom +patterns.utftype=utftype +patterns.utfoffset=utfoffset +local utf8char=patterns.utf8one+patterns.utf8two+patterns.utf8three+patterns.utf8four +local validutf8char=utf8char^0*endofstring*Cc(true)+Cc(false) +local utf8character=P(1)*R("\128\191")^0 +patterns.utf8=utf8char +patterns.utf8char=utf8char +patterns.utf8character=utf8character +patterns.validutf8=validutf8char +patterns.validutf8char=validutf8char +local eol=S("\n\r") +local spacer=S(" \t\f\v") +local whitespace=eol+spacer +local nonspacer=1-spacer +local nonwhitespace=1-whitespace +patterns.eol=eol +patterns.spacer=spacer +patterns.whitespace=whitespace +patterns.nonspacer=nonspacer +patterns.nonwhitespace=nonwhitespace +local stripper=spacer^0*C((spacer^0*nonspacer^1)^0) +local collapser=Cs(spacer^0/""*nonspacer^0*((spacer^0/" "*nonspacer^1)^0)) +patterns.stripper=stripper +patterns.collapser=collapser +patterns.digit=digit +patterns.sign=sign +patterns.cardinal=sign^0*digit^1 +patterns.integer=sign^0*digit^1 +patterns.unsigned=digit^0*P('.')*digit^1 +patterns.float=sign^0*patterns.unsigned +patterns.cunsigned=digit^0*P(',')*digit^1 +patterns.cfloat=sign^0*patterns.cunsigned +patterns.number=patterns.float+patterns.integer +patterns.cnumber=patterns.cfloat+patterns.integer +patterns.oct=P("0")*R("07")^1 +patterns.octal=patterns.oct +patterns.HEX=P("0x")*R("09","AF")^1 +patterns.hex=P("0x")*R("09","af")^1 +patterns.hexadecimal=P("0x")*R("09","AF","af")^1 +patterns.lowercase=R("az") +patterns.uppercase=R("AZ") +patterns.letter=patterns.lowercase+patterns.uppercase +patterns.space=space +patterns.tab=P("\t") +patterns.spaceortab=patterns.space+patterns.tab +patterns.newline=newline +patterns.emptyline=newline^1 +patterns.equal=P("=") +patterns.comma=P(",") +patterns.commaspacer=P(",")*spacer^0 +patterns.period=P(".") +patterns.colon=P(":") +patterns.semicolon=P(";") +patterns.underscore=P("_") +patterns.escaped=escaped +patterns.squote=squote +patterns.dquote=dquote +patterns.nosquote=(escaped+(1-squote))^0 +patterns.nodquote=(escaped+(1-dquote))^0 +patterns.unsingle=(squote/"")*patterns.nosquote*(squote/"") +patterns.undouble=(dquote/"")*patterns.nodquote*(dquote/"") +patterns.unquoted=patterns.undouble+patterns.unsingle +patterns.unspacer=((patterns.spacer^1)/"")^0 +patterns.singlequoted=squote*patterns.nosquote*squote +patterns.doublequoted=dquote*patterns.nodquote*dquote +patterns.quoted=patterns.doublequoted+patterns.singlequoted +patterns.propername=R("AZ","az","__")*R("09","AZ","az","__")^0*P(-1) +patterns.somecontent=(anything-newline-space)^1 +patterns.beginline=#(1-newline) +patterns.longtostring=Cs(whitespace^0/""*nonwhitespace^0*((whitespace^0/" "*(patterns.quoted+nonwhitespace)^1)^0)) +local function anywhere(pattern) + return P { P(pattern)+1*V(1) } +end +lpeg.anywhere=anywhere +function lpeg.instringchecker(p) + p=anywhere(p) + return function(str) + return lpegmatch(p,str) and true or false + end +end +function lpeg.splitter(pattern,action) + return (((1-P(pattern))^1)/action+1)^0 +end +function lpeg.tsplitter(pattern,action) + return Ct((((1-P(pattern))^1)/action+1)^0) +end +local splitters_s,splitters_m,splitters_t={},{},{} +local function splitat(separator,single) + local splitter=(single and splitters_s[separator]) or splitters_m[separator] + if not splitter then + separator=P(separator) + local other=C((1-separator)^0) + if single then + local any=anything + splitter=other*(separator*C(any^0)+"") + splitters_s[separator]=splitter + else + splitter=other*(separator*other)^0 + splitters_m[separator]=splitter + end + end + return splitter end - -function string.topattern(str,lowercase,strict) - if str == "" then - return ".*" +local function tsplitat(separator) + local splitter=splitters_t[separator] + if not splitter then + splitter=Ct(splitat(separator)) + splitters_t[separator]=splitter + end + return splitter +end +lpeg.splitat=splitat +lpeg.tsplitat=tsplitat +function string.splitup(str,separator) + if not separator then + separator="," + end + return lpegmatch(splitters_m[separator] or splitat(separator),str) +end +local cache={} +function lpeg.split(separator,str) + local c=cache[separator] + if not c then + c=tsplitat(separator) + cache[separator]=c + end + return lpegmatch(c,str) +end +function string.split(str,separator) + if separator then + local c=cache[separator] + if not c then + c=tsplitat(separator) + cache[separator]=c + end + return lpegmatch(c,str) + else + return { str } + end +end +local spacing=patterns.spacer^0*newline +local empty=spacing*Cc("") +local nonempty=Cs((1-spacing)^1)*spacing^-1 +local content=(empty+nonempty)^1 +patterns.textline=content +local linesplitter=tsplitat(newline) +patterns.linesplitter=linesplitter +function string.splitlines(str) + return lpegmatch(linesplitter,str) +end +local cache={} +function lpeg.checkedsplit(separator,str) + local c=cache[separator] + if not c then + separator=P(separator) + local other=C((1-separator)^1) + c=Ct(separator^0*other*(separator^1*other)^0) + cache[separator]=c + end + return lpegmatch(c,str) +end +function string.checkedsplit(str,separator) + local c=cache[separator] + if not c then + separator=P(separator) + local other=C((1-separator)^1) + c=Ct(separator^0*other*(separator^1*other)^0) + cache[separator]=c + end + return lpegmatch(c,str) +end +local function f2(s) local c1,c2=byte(s,1,2) return c1*64+c2-12416 end +local function f3(s) local c1,c2,c3=byte(s,1,3) return (c1*64+c2)*64+c3-925824 end +local function f4(s) local c1,c2,c3,c4=byte(s,1,4) return ((c1*64+c2)*64+c3)*64+c4-63447168 end +local utf8byte=patterns.utf8one/byte+patterns.utf8two/f2+patterns.utf8three/f3+patterns.utf8four/f4 +patterns.utf8byte=utf8byte +local cache={} +function lpeg.stripper(str) + if type(str)=="string" then + local s=cache[str] + if not s then + s=Cs(((S(str)^1)/""+1)^0) + cache[str]=s + end + return s + else + return Cs(((str^1)/""+1)^0) + end +end +local cache={} +function lpeg.keeper(str) + if type(str)=="string" then + local s=cache[str] + if not s then + s=Cs((((1-S(str))^1)/""+1)^0) + cache[str]=s + end + return s + else + return Cs((((1-str)^1)/""+1)^0) + end +end +function lpeg.frontstripper(str) + return (P(str)+P(true))*Cs(anything^0) +end +function lpeg.endstripper(str) + return Cs((1-P(str)*endofstring)^0) +end +function lpeg.replacer(one,two,makefunction,isutf) + local pattern + local u=isutf and utf8char or 1 + if type(one)=="table" then + local no=#one + local p=P(false) + if no==0 then + for k,v in next,one do + p=p+P(k)/v + end + pattern=Cs((p+u)^0) + elseif no==1 then + local o=one[1] + one,two=P(o[1]),o[2] + pattern=Cs((one/two+u)^0) + else + for i=1,no do + local o=one[i] + p=p+P(o[1])/o[2] + end + pattern=Cs((p+u)^0) + end + else + pattern=Cs((P(one)/(two or "")+u)^0) + end + if makefunction then + return function(str) + return lpegmatch(pattern,str) + end + else + return pattern + end +end +function lpeg.finder(lst,makefunction) + local pattern + if type(lst)=="table" then + pattern=P(false) + if #lst==0 then + for k,v in next,lst do + pattern=pattern+P(k) + end + else + for i=1,#lst do + pattern=pattern+P(lst[i]) + end + end + else + pattern=P(lst) + end + pattern=(1-pattern)^0*pattern + if makefunction then + return function(str) + return lpegmatch(pattern,str) + end + else + return pattern + end +end +local splitters_f,splitters_s={},{} +function lpeg.firstofsplit(separator) + local splitter=splitters_f[separator] + if not splitter then + separator=P(separator) + splitter=C((1-separator)^0) + splitters_f[separator]=splitter + end + return splitter +end +function lpeg.secondofsplit(separator) + local splitter=splitters_s[separator] + if not splitter then + separator=P(separator) + splitter=(1-separator)^0*separator*C(anything^0) + splitters_s[separator]=splitter + end + return splitter +end +function lpeg.balancer(left,right) + left,right=P(left),P(right) + return P { left*((1-left-right)+V(1))^0*right } +end +local nany=utf8char/"" +function lpeg.counter(pattern) + pattern=Cs((P(pattern)/" "+nany)^0) + return function(str) + return #lpegmatch(pattern,str) + end +end +utf=utf or (unicode and unicode.utf8) or {} +local utfcharacters=utf and utf.characters or string.utfcharacters +local utfgmatch=utf and utf.gmatch +local utfchar=utf and utf.char +lpeg.UP=lpeg.P +if utfcharacters then + function lpeg.US(str) + local p=P(false) + for uc in utfcharacters(str) do + p=p+P(uc) + end + return p + end +elseif utfgmatch then + function lpeg.US(str) + local p=P(false) + for uc in utfgmatch(str,".") do + p=p+P(uc) + end + return p + end +else + function lpeg.US(str) + local p=P(false) + local f=function(uc) + p=p+P(uc) + end + lpegmatch((utf8char/f)^0,str) + return p + end +end +local range=utf8byte*utf8byte+Cc(false) +function lpeg.UR(str,more) + local first,last + if type(str)=="number" then + first=str + last=more or first + else + first,last=lpegmatch(range,str) + if not last then + return P(str) + end + end + if first==last then + return P(str) + elseif utfchar and (last-first<8) then + local p=P(false) + for i=first,last do + p=p+P(utfchar(i)) + end + return p + else + local f=function(b) + return b>=first and b<=last + end + return utf8byte/f + end +end +function lpeg.is_lpeg(p) + return p and lpegtype(p)=="pattern" +end +function lpeg.oneof(list,...) + if type(list)~="table" then + list={ list,... } + end + local p=P(list[1]) + for l=2,#list do + p=p+P(list[l]) + end + return p +end +local sort=table.sort +local function copyindexed(old) + local new={} + for i=1,#old do + new[i]=old + end + return new +end +local function sortedkeys(tab) + local keys,s={},0 + for key,_ in next,tab do + s=s+1 + keys[s]=key + end + sort(keys) + return keys +end +function lpeg.append(list,pp,delayed,checked) + local p=pp + if #list>0 then + local keys=copyindexed(list) + sort(keys) + for i=#keys,1,-1 do + local k=keys[i] + if p then + p=P(k)+p + else + p=P(k) + end + end + elseif delayed then + local keys=sortedkeys(list) + if p then + for i=1,#keys,1 do + local k=keys[i] + local v=list[k] + p=P(k)/list+p + end else - str = gsub(str,".",simple_escapes) - if lowercase then - str = lower(str) + for i=1,#keys do + local k=keys[i] + local v=list[k] + if p then + p=P(k)+p + else + p=P(k) + end + end + if p then + p=p/list + end + end + elseif checked then + local keys=sortedkeys(list) + for i=1,#keys do + local k=keys[i] + local v=list[k] + if p then + if k==v then + p=P(k)+p + else + p=P(k)/v+p end - if strict then - return "^" .. str .. "$" + else + if k==v then + p=P(k) else - return str + p=P(k)/v end + end end + else + local keys=sortedkeys(list) + for i=1,#keys do + local k=keys[i] + local v=list[k] + if p then + p=P(k)/v+p + else + p=P(k)/v + end + end + end + return p end - --- obsolete names: - -string.quote = string.quoted -string.unquote = string.unquoted +local function make(t) + local p + local keys=sortedkeys(t) + for i=1,#keys do + local k=keys[i] + local v=t[k] + if not p then + if next(v) then + p=P(k)*make(v) + else + p=P(k) + end + else + if next(v) then + p=p+P(k)*make(v) + else + p=p+P(k) + end + end + end + return p +end +function lpeg.utfchartabletopattern(list) + local tree={} + for i=1,#list do + local t=tree + for c in gmatch(list[i],".") do + if not t[c] then + t[c]={} + end + t=t[c] + end + end + return make(tree) +end +patterns.containseol=lpeg.finder(eol) +local function nextstep(n,step,result) + local m=n%step + local d=floor(n/step) + if d>0 then + local v=V(tostring(step)) + local s=result.start + for i=1,d do + if s then + s=v*s + else + s=v + end + end + result.start=s + end + if step>1 and result.start then + local v=V(tostring(step/2)) + result[tostring(step)]=v*v + end + if step>0 then + return nextstep(m,step/2,result) + else + return result + end +end +function lpeg.times(pattern,n) + return P(nextstep(n,2^16,{ "start",["1"]=pattern })) +end +local digit=R("09") +local period=P(".") +local zero=P("0") +local trailingzeros=zero^0*-digit +local case_1=period*trailingzeros/"" +local case_2=period*(digit-trailingzeros)^1*(trailingzeros/"") +local number=digit^1*(case_1+case_2) +local stripper=Cs((number+1)^0) +lpeg.patterns.stripzeros=stripper end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['l-table'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local type, next, tostring, tonumber, ipairs, table, string = type, next, tostring, tonumber, ipairs, table, string -local concat, sort, insert, remove = table.concat, table.sort, table.insert, table.remove -local format, find, gsub, lower, dump, match = string.format, string.find, string.gsub, string.lower, string.dump, string.match -local getmetatable, setmetatable = getmetatable, setmetatable -local getinfo = debug.getinfo +package.loaded["l-function"] = package.loaded["l-function"] or true --- Starting with version 5.2 Lua no longer provide ipairs, which makes --- sense. As we already used the for loop and # in most places the --- impact on ConTeXt was not that large; the remaining ipairs already --- have been replaced. In a similar fashion we also hardly used pairs. --- --- Just in case, we provide the fallbacks as discussed in Programming --- in Lua (http://www.lua.org/pil/7.3.html): +-- original size: 361, stripped down to: 322 -if not ipairs then +if not modules then modules={} end modules ['l-functions']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +functions=functions or {} +function functions.dummy() end - -- for k, v in ipairs(t) do ... end - -- for k=1,#t do local v = t[k] ... end - local function iterate(a,i) - i = i + 1 - local v = a[i] - if v ~= nil then - return i, v --, nil - end - end +end -- of closure - function ipairs(a) - return iterate, a, 0 - end +do -- create closure to overcome 200 locals limit -end +package.loaded["l-string"] = package.loaded["l-string"] or true -if not pairs then +-- original size: 5513, stripped down to: 2708 - -- for k, v in pairs(t) do ... end - -- for k, v in next, t do ... end +if not modules then modules={} end modules ['l-string']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local string=string +local sub,gmatch,format,char,byte,rep,lower=string.sub,string.gmatch,string.format,string.char,string.byte,string.rep,string.lower +local lpegmatch,patterns=lpeg.match,lpeg.patterns +local P,S,C,Ct,Cc,Cs=lpeg.P,lpeg.S,lpeg.C,lpeg.Ct,lpeg.Cc,lpeg.Cs +local unquoted=patterns.squote*C(patterns.nosquote)*patterns.squote+patterns.dquote*C(patterns.nodquote)*patterns.dquote +function string.unquoted(str) + return lpegmatch(unquoted,str) or str +end +function string.quoted(str) + return format("%q",str) +end +function string.count(str,pattern) + local n=0 + for _ in gmatch(str,pattern) do + n=n+1 + end + return n +end +function string.limit(str,n,sentinel) + if #str>n then + sentinel=sentinel or "..." + return sub(str,1,(n-#sentinel))..sentinel + else + return str + end +end +local stripper=patterns.stripper +local collapser=patterns.collapser +local longtostring=patterns.longtostring +function string.strip(str) + return lpegmatch(stripper,str) or "" +end +function string.collapsespaces(str) + return lpegmatch(collapser,str) or "" +end +function string.longtostring(str) + return lpegmatch(longtostring,str) or "" +end +local pattern=P(" ")^0*P(-1) +function string.is_empty(str) + if str=="" then + return true + else + return lpegmatch(pattern,str) and true or false + end +end +local anything=patterns.anything +local allescapes=Cc("%")*S(".-+%?()[]*") +local someescapes=Cc("%")*S(".-+%()[]") +local matchescapes=Cc(".")*S("*?") +local pattern_a=Cs ((allescapes+anything )^0 ) +local pattern_b=Cs ((someescapes+matchescapes+anything )^0 ) +local pattern_c=Cs (Cc("^")*(someescapes+matchescapes+anything )^0*Cc("$") ) +function string.escapedpattern(str,simple) + return lpegmatch(simple and pattern_b or pattern_a,str) +end +function string.topattern(str,lowercase,strict) + if str=="" or type(str)~="string" then + return ".*" + elseif strict then + str=lpegmatch(pattern_c,str) + else + str=lpegmatch(pattern_b,str) + end + if lowercase then + return lower(str) + else + return str + end +end +function string.valid(str,default) + return (type(str)=="string" and str~="" and str) or default or nil +end +string.itself=function(s) return s end +local pattern=Ct(C(1)^0) +function string.totable(str) + return lpegmatch(pattern,str) +end +local replacer=lpeg.replacer("@","%%") +function string.tformat(fmt,...) + return format(lpegmatch(replacer,fmt),...) +end +string.quote=string.quoted +string.unquote=string.unquoted - function pairs(t) - return next, t -- , nil - end -end +end -- of closure --- Also, unpack has been moved to the table table, and for compatiility --- reasons we provide both now. +do -- create closure to overcome 200 locals limit -if not table.unpack then - table.unpack = _G.unpack -elseif not unpack then - _G.unpack = table.unpack -end +package.loaded["l-table"] = package.loaded["l-table"] or true --- extra functions, some might go (when not used) +-- original size: 44643, stripped down to: 19717 +if not modules then modules={} end modules ['l-table']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local type,next,tostring,tonumber,ipairs,select=type,next,tostring,tonumber,ipairs,select +local table,string=table,string +local concat,sort,insert,remove=table.concat,table.sort,table.insert,table.remove +local format,lower,dump=string.format,string.lower,string.dump +local getmetatable,setmetatable=getmetatable,setmetatable +local getinfo=debug.getinfo +local lpegmatch,patterns=lpeg.match,lpeg.patterns +local floor=math.floor +local stripper=patterns.stripper function table.strip(tab) - local lst, l = { }, 0 - for i=1,#tab do - local s = gsub(tab[i],"^%s*(.-)%s*$","%1") - if s == "" then - -- skip this one - else - l = l + 1 - lst[l] = s - end + local lst,l={},0 + for i=1,#tab do + local s=lpegmatch(stripper,tab[i]) or "" + if s=="" then + else + l=l+1 + lst[l]=s end - return lst + end + return lst end - function table.keys(t) - local keys, k = { }, 0 - for key, _ in next, t do - k = k + 1 - keys[k] = key + if t then + local keys,k={},0 + for key,_ in next,t do + k=k+1 + keys[k]=key end return keys + else + return {} + end end - local function compare(a,b) - local ta, tb = type(a), type(b) -- needed, else 11 < 2 - if ta == tb then - return a < b - else - return tostring(a) < tostring(b) - end + local ta,tb=type(a),type(b) + if ta==tb then + return a0 then + local n=0 + for _,v in next,t do + n=n+1 + end + if n==#t then + local tt,nt={},0 + for i=1,#t do + local v=t[i] + local tv=type(v) + if tv=="number" then + nt=nt+1 + if hexify then + tt[nt]=format("0x%04X",v) + else + tt[nt]=tostring(v) + end + elseif tv=="boolean" then + nt=nt+1 + tt[nt]=tostring(v) + elseif tv=="string" then + nt=nt+1 + tt[nt]=format("%q",v) + else + tt=nil + break end + end + return tt end - return tmp + end + return nil end - -local function fastcopy(old,metatabletoo) -- fast one - if old then - local new = { } - for k,v in next, old do - if type(v) == "table" then - new[k] = fastcopy(v,metatabletoo) -- was just table.copy - else - new[k] = v - end +local propername=patterns.propername +local function dummy() end +local function do_serialize(root,name,depth,level,indexed) + if level>0 then + depth=depth.." " + if indexed then + handle(format("%s{",depth)) + else + local tn=type(name) + if tn=="number" then + if hexify then + handle(format("%s[0x%04X]={",depth,name)) + else + handle(format("%s[%s]={",depth,name)) end - if metatabletoo then - -- optional second arg - local mt = getmetatable(old) - if mt then - setmetatable(new,mt) + elseif tn=="string" then + if noquotes and not reserved[name] and lpegmatch(propername,name) then + handle(format("%s%s={",depth,name)) + else + handle(format("%s[%q]={",depth,name)) + end + elseif tn=="boolean" then + handle(format("%s[%s]={",depth,tostring(name))) + else + handle(format("%s{",depth)) + end + end + end + if root and next(root) then + local first,last=nil,0 + if compact then + last=#root + for k=1,last do + if root[k]==nil then + last=k-1 + break + end + end + if last>0 then + first=1 + end + end + local sk=sortedkeys(root) + for i=1,#sk do + local k=sk[i] + local v=root[k] + local t,tk=type(v),type(k) + if compact and first and tk=="number" and k>=first and k<=last then + if t=="number" then + if hexify then + handle(format("%s 0x%04X,",depth,v)) + else + handle(format("%s %s,",depth,v)) + end + elseif t=="string" then + if reduce and tonumber(v) then + handle(format("%s %s,",depth,v)) + else + handle(format("%s %q,",depth,v)) + end + elseif t=="table" then + if not next(v) then + handle(format("%s {},",depth)) + elseif inline then + local st=simple_table(v) + if st then + handle(format("%s { %s },",depth,concat(st,", "))) + else + do_serialize(v,k,depth,level+1,true) + end + else + do_serialize(v,k,depth,level+1,true) + end + elseif t=="boolean" then + handle(format("%s %s,",depth,tostring(v))) + elseif t=="function" then + if functions then + handle(format('%s load(%q),',depth,dump(v))) + else + handle(format('%s "function",',depth)) + end + else + handle(format("%s %q,",depth,tostring(v))) + end + elseif k=="__p__" then + if false then + handle(format("%s __p__=nil,",depth)) + end + elseif t=="number" then + if tk=="number" then + if hexify then + handle(format("%s [0x%04X]=0x%04X,",depth,k,v)) + else + handle(format("%s [%s]=%s,",depth,k,v)) + end + elseif tk=="boolean" then + if hexify then + handle(format("%s [%s]=0x%04X,",depth,tostring(k),v)) + else + handle(format("%s [%s]=%s,",depth,tostring(k),v)) + end + elseif noquotes and not reserved[k] and lpegmatch(propername,k) then + if hexify then + handle(format("%s %s=0x%04X,",depth,k,v)) + else + handle(format("%s %s=%s,",depth,k,v)) + end + else + if hexify then + handle(format("%s [%q]=0x%04X,",depth,k,v)) + else + handle(format("%s [%q]=%s,",depth,k,v)) + end + end + elseif t=="string" then + if reduce and tonumber(v) then + if tk=="number" then + if hexify then + handle(format("%s [0x%04X]=%s,",depth,k,v)) + else + handle(format("%s [%s]=%s,",depth,k,v)) + end + elseif tk=="boolean" then + handle(format("%s [%s]=%s,",depth,tostring(k),v)) + elseif noquotes and not reserved[k] and lpegmatch(propername,k) then + handle(format("%s %s=%s,",depth,k,v)) + else + handle(format("%s [%q]=%s,",depth,k,v)) + end + else + if tk=="number" then + if hexify then + handle(format("%s [0x%04X]=%q,",depth,k,v)) + else + handle(format("%s [%s]=%q,",depth,k,v)) + end + elseif tk=="boolean" then + handle(format("%s [%s]=%q,",depth,tostring(k),v)) + elseif noquotes and not reserved[k] and lpegmatch(propername,k) then + handle(format("%s %s=%q,",depth,k,v)) + else + handle(format("%s [%q]=%q,",depth,k,v)) + end + end + elseif t=="table" then + if not next(v) then + if tk=="number" then + if hexify then + handle(format("%s [0x%04X]={},",depth,k)) + else + handle(format("%s [%s]={},",depth,k)) + end + elseif tk=="boolean" then + handle(format("%s [%s]={},",depth,tostring(k))) + elseif noquotes and not reserved[k] and lpegmatch(propername,k) then + handle(format("%s %s={},",depth,k)) + else + handle(format("%s [%q]={},",depth,k)) + end + elseif inline then + local st=simple_table(v) + if st then + if tk=="number" then + if hexify then + handle(format("%s [0x%04X]={ %s },",depth,k,concat(st,", "))) + else + handle(format("%s [%s]={ %s },",depth,k,concat(st,", "))) + end + elseif tk=="boolean" then + handle(format("%s [%s]={ %s },",depth,tostring(k),concat(st,", "))) + elseif noquotes and not reserved[k] and lpegmatch(propername,k) then + handle(format("%s %s={ %s },",depth,k,concat(st,", "))) + else + handle(format("%s [%q]={ %s },",depth,k,concat(st,", "))) end + else + do_serialize(v,k,depth,level+1) + end + else + do_serialize(v,k,depth,level+1) + end + elseif t=="boolean" then + if tk=="number" then + if hexify then + handle(format("%s [0x%04X]=%s,",depth,k,tostring(v))) + else + handle(format("%s [%s]=%s,",depth,k,tostring(v))) + end + elseif tk=="boolean" then + handle(format("%s [%s]=%s,",depth,tostring(k),tostring(v))) + elseif noquotes and not reserved[k] and lpegmatch(propername,k) then + handle(format("%s %s=%s,",depth,k,tostring(v))) + else + handle(format("%s [%q]=%s,",depth,k,tostring(v))) end - return new + elseif t=="function" then + if functions then + local f=getinfo(v).what=="C" and dump(dummy) or dump(v) + if tk=="number" then + if hexify then + handle(format("%s [0x%04X]=load(%q),",depth,k,f)) + else + handle(format("%s [%s]=load(%q),",depth,k,f)) + end + elseif tk=="boolean" then + handle(format("%s [%s]=load(%q),",depth,tostring(k),f)) + elseif noquotes and not reserved[k] and lpegmatch(propername,k) then + handle(format("%s %s=load(%q),",depth,k,f)) + else + handle(format("%s [%q]=load(%q),",depth,k,f)) + end + end + else + if tk=="number" then + if hexify then + handle(format("%s [0x%04X]=%q,",depth,k,tostring(v))) + else + handle(format("%s [%s]=%q,",depth,k,tostring(v))) + end + elseif tk=="boolean" then + handle(format("%s [%s]=%q,",depth,tostring(k),tostring(v))) + elseif noquotes and not reserved[k] and lpegmatch(propername,k) then + handle(format("%s %s=%q,",depth,k,tostring(v))) + else + handle(format("%s [%q]=%q,",depth,k,tostring(v))) + end + end + end + end + if level>0 then + handle(format("%s},",depth)) + end +end +local function serialize(_handle,root,name,specification) + local tname=type(name) + if type(specification)=="table" then + noquotes=specification.noquotes + hexify=specification.hexify + handle=_handle or specification.handle or print + reduce=specification.reduce or false + functions=specification.functions + compact=specification.compact + inline=specification.inline and compact + if functions==nil then + functions=true + end + if compact==nil then + compact=true + end + if inline==nil then + inline=compact + end + else + noquotes=false + hexify=false + handle=_handle or print + reduce=false + compact=true + inline=true + functions=true + end + if tname=="string" then + if name=="return" then + handle("return {") else - return { } + handle(name.."={") end -end - --- todo : copy without metatable - -local function copy(t, tables) -- taken from lua wiki, slightly adapted - tables = tables or { } - local tcopy = {} - if not tables[t] then - tables[t] = tcopy + elseif tname=="number" then + if hexify then + handle(format("[0x%04X]={",name)) + else + handle("["..name.."]={") end - for i,v in next, t do -- brrr, what happens with sparse indexed - if type(i) == "table" then - if tables[i] then - i = tables[i] - else - i = copy(i, tables) - end - end - if type(v) ~= "table" then - tcopy[i] = v - elseif tables[v] then - tcopy[i] = tables[v] - else - tcopy[i] = copy(v, tables) - end + elseif tname=="boolean" then + if name then + handle("return {") + else + handle("{") end - local mt = getmetatable(t) - if mt then - setmetatable(tcopy,mt) + else + handle("t={") + end + if root then + if getmetatable(root) then + local dummy=root._w_h_a_t_e_v_e_r_ + root._w_h_a_t_e_v_e_r_=nil end - return tcopy -end - -table.fastcopy = fastcopy -table.copy = copy - -function table.derive(parent) - local child = { } - if parent then - setmetatable(child,{ __index = parent }) + if next(root) then + do_serialize(root,name,"",0) end - return child + end + handle("}") end - -function table.tohash(t,value) - local h = { } - if t then - if value == nil then value = true end - for _, v in next, t do -- no ipairs here - h[v] = value - end +function table.serialize(root,name,specification) + local t,n={},0 + local function flush(s) + n=n+1 + t[n]=s + end + serialize(flush,root,name,specification) + return concat(t,"\n") +end +table.tohandle=serialize +local maxtab=2*1024 +function table.tofile(filename,root,name,specification) + local f=io.open(filename,'w') + if f then + if maxtab>1 then + local t,n={},0 + local function flush(s) + n=n+1 + t[n]=s + if n>maxtab then + f:write(concat(t,"\n"),"\n") + t,n={},0 + end + end + serialize(flush,root,name,specification) + f:write(concat(t,"\n"),"\n") + else + local function flush(s) + f:write(s,"\n") + end + serialize(flush,root,name,specification) end - return h + f:close() + io.flush() + end end - -function table.fromhash(t) - local hsh, h = { }, 0 - for k, v in next, t do -- no ipairs here - if v then - h = h + 1 - hsh[h] = k - end - end - return hsh -end - -local noquotes, hexify, handle, reduce, compact, inline, functions - -local reserved = table.tohash { -- intercept a language inconvenience: no reserved words as key - 'and', 'break', 'do', 'else', 'elseif', 'end', 'false', 'for', 'function', 'if', - 'in', 'local', 'nil', 'not', 'or', 'repeat', 'return', 'then', 'true', 'until', 'while', -} - -local function simple_table(t) - if #t > 0 then - local n = 0 - for _,v in next, t do - n = n + 1 - end - if n == #t then - local tt, nt = { }, 0 - for i=1,#t do - local v = t[i] - local tv = type(v) - if tv == "number" then - nt = nt + 1 - if hexify then - tt[nt] = format("0x%04X",v) - else - tt[nt] = tostring(v) -- tostring not needed - end - elseif tv == "boolean" then - nt = nt + 1 - tt[nt] = tostring(v) - elseif tv == "string" then - nt = nt + 1 - tt[nt] = format("%q",v) - else - tt = nil - break - end - end - return tt - end +local function flattened(t,f,depth) + if f==nil then + f={} + depth=0xFFFF + elseif tonumber(f) then + depth=f + f={} + elseif not depth then + depth=0xFFFF + end + for k,v in next,t do + if type(k)~="number" then + if depth>0 and type(v)=="table" then + flattened(v,f,depth-1) + else + f[k]=v + end + end + end + local n=#f + for k=1,#t do + local v=t[k] + if depth>0 and type(v)=="table" then + flattened(v,f,depth-1) + n=#f + else + n=n+1 + f[n]=v + end + end + return f +end +table.flattened=flattened +local function unnest(t,f) + if not f then + f={} + end + for i=1,#t do + local v=t[i] + if type(v)=="table" then + if type(v[1])=="table" then + unnest(v,f) + else + f[#f+1]=v + end + else + f[#f+1]=v + end + end + return f +end +function table.unnest(t) + return unnest(t) +end +local function are_equal(a,b,n,m) + if a and b and #a==#b then + n=n or 1 + m=m or #a + for i=n,m do + local ai,bi=a[i],b[i] + if ai==bi then + elseif type(ai)=="table" and type(bi)=="table" then + if not are_equal(ai,bi) then + return false + end + else + return false + end end - return nil -end - --- Because this is a core function of mkiv I moved some function calls --- inline. --- --- twice as fast in a test: --- --- local propername = lpeg.P(lpeg.R("AZ","az","__") * lpeg.R("09","AZ","az", "__")^0 * lpeg.P(-1) ) - --- problem: there no good number_to_string converter with the best resolution - -local function dummy() end - -local function do_serialize(root,name,depth,level,indexed) - if level > 0 then - depth = depth .. " " - if indexed then - handle(format("%s{",depth)) - else - local tn = type(name) - if tn == "number" then -- or find(k,"^%d+$") then - if hexify then - handle(format("%s[0x%04X]={",depth,name)) - else - handle(format("%s[%s]={",depth,name)) - end - elseif tn == "string" then - if noquotes and not reserved[name] and find(name,"^%a[%w%_]*$") then - handle(format("%s%s={",depth,name)) - else - handle(format("%s[%q]={",depth,name)) - end - elseif tn == "boolean" then - handle(format("%s[%s]={",depth,tostring(name))) - else - handle(format("%s{",depth)) - end - end + return true + else + return false + end +end +local function identical(a,b) + for ka,va in next,a do + local vb=b[ka] + if va==vb then + elseif type(va)=="table" and type(vb)=="table" then + if not identical(va,vb) then + return false + end + else + return false end - -- we could check for k (index) being number (cardinal) - if root and next(root) then - local first, last = nil, 0 -- #root cannot be trusted here (will be ok in 5.2 when ipairs is gone) - if compact then - -- NOT: for k=1,#root do (we need to quit at nil) - for k,v in ipairs(root) do -- can we use next? - if not first then first = k end - last = last + 1 - end - end - local sk = sortedkeys(root) - for i=1,#sk do - local k = sk[i] - local v = root[k] - -- circular - local t, tk = type(v), type(k) - if compact and first and tk == "number" and k >= first and k <= last then - if t == "number" then - if hexify then - handle(format("%s 0x%04X,",depth,v)) - else - handle(format("%s %s,",depth,v)) -- %.99g - end - elseif t == "string" then - if reduce and tonumber(v) then - handle(format("%s %s,",depth,v)) - else - handle(format("%s %q,",depth,v)) - end - elseif t == "table" then - if not next(v) then - handle(format("%s {},",depth)) - elseif inline then -- and #t > 0 - local st = simple_table(v) - if st then - handle(format("%s { %s },",depth,concat(st,", "))) - else - do_serialize(v,k,depth,level+1,true) - end - else - do_serialize(v,k,depth,level+1,true) - end - elseif t == "boolean" then - handle(format("%s %s,",depth,tostring(v))) - elseif t == "function" then - if functions then - handle(format('%s loadstring(%q),',depth,dump(v))) - else - handle(format('%s "function",',depth)) - end - else - handle(format("%s %q,",depth,tostring(v))) - end - elseif k == "__p__" then -- parent - if false then - handle(format("%s __p__=nil,",depth)) - end - elseif t == "number" then - if tk == "number" then -- or find(k,"^%d+$") then - if hexify then - handle(format("%s [0x%04X]=0x%04X,",depth,k,v)) - else - handle(format("%s [%s]=%s,",depth,k,v)) -- %.99g - end - elseif tk == "boolean" then - if hexify then - handle(format("%s [%s]=0x%04X,",depth,tostring(k),v)) - else - handle(format("%s [%s]=%s,",depth,tostring(k),v)) -- %.99g - end - elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then - if hexify then - handle(format("%s %s=0x%04X,",depth,k,v)) - else - handle(format("%s %s=%s,",depth,k,v)) -- %.99g - end - else - if hexify then - handle(format("%s [%q]=0x%04X,",depth,k,v)) - else - handle(format("%s [%q]=%s,",depth,k,v)) -- %.99g - end - end - elseif t == "string" then - if reduce and tonumber(v) then - if tk == "number" then -- or find(k,"^%d+$") then - if hexify then - handle(format("%s [0x%04X]=%s,",depth,k,v)) - else - handle(format("%s [%s]=%s,",depth,k,v)) - end - elseif tk == "boolean" then - handle(format("%s [%s]=%s,",depth,tostring(k),v)) - elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then - handle(format("%s %s=%s,",depth,k,v)) - else - handle(format("%s [%q]=%s,",depth,k,v)) - end - else - if tk == "number" then -- or find(k,"^%d+$") then - if hexify then - handle(format("%s [0x%04X]=%q,",depth,k,v)) - else - handle(format("%s [%s]=%q,",depth,k,v)) - end - elseif tk == "boolean" then - handle(format("%s [%s]=%q,",depth,tostring(k),v)) - elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then - handle(format("%s %s=%q,",depth,k,v)) - else - handle(format("%s [%q]=%q,",depth,k,v)) - end - end - elseif t == "table" then - if not next(v) then - if tk == "number" then -- or find(k,"^%d+$") then - if hexify then - handle(format("%s [0x%04X]={},",depth,k)) - else - handle(format("%s [%s]={},",depth,k)) - end - elseif tk == "boolean" then - handle(format("%s [%s]={},",depth,tostring(k))) - elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then - handle(format("%s %s={},",depth,k)) - else - handle(format("%s [%q]={},",depth,k)) - end - elseif inline then - local st = simple_table(v) - if st then - if tk == "number" then -- or find(k,"^%d+$") then - if hexify then - handle(format("%s [0x%04X]={ %s },",depth,k,concat(st,", "))) - else - handle(format("%s [%s]={ %s },",depth,k,concat(st,", "))) - end - elseif tk == "boolean" then -- or find(k,"^%d+$") then - handle(format("%s [%s]={ %s },",depth,tostring(k),concat(st,", "))) - elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then - handle(format("%s %s={ %s },",depth,k,concat(st,", "))) - else - handle(format("%s [%q]={ %s },",depth,k,concat(st,", "))) - end - else - do_serialize(v,k,depth,level+1) - end - else - do_serialize(v,k,depth,level+1) - end - elseif t == "boolean" then - if tk == "number" then -- or find(k,"^%d+$") then - if hexify then - handle(format("%s [0x%04X]=%s,",depth,k,tostring(v))) - else - handle(format("%s [%s]=%s,",depth,k,tostring(v))) - end - elseif tk == "boolean" then -- or find(k,"^%d+$") then - handle(format("%s [%s]=%s,",depth,tostring(k),tostring(v))) - elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then - handle(format("%s %s=%s,",depth,k,tostring(v))) - else - handle(format("%s [%q]=%s,",depth,k,tostring(v))) - end - elseif t == "function" then - if functions then - local f = getinfo(v).what == "C" and dump(dummy) or dump(v) - -- local f = getinfo(v).what == "C" and dump(function(...) return v(...) end) or dump(v) - if tk == "number" then -- or find(k,"^%d+$") then - if hexify then - handle(format("%s [0x%04X]=loadstring(%q),",depth,k,f)) - else - handle(format("%s [%s]=loadstring(%q),",depth,k,f)) - end - elseif tk == "boolean" then - handle(format("%s [%s]=loadstring(%q),",depth,tostring(k),f)) - elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then - handle(format("%s %s=loadstring(%q),",depth,k,f)) - else - handle(format("%s [%q]=loadstring(%q),",depth,k,f)) - end - end - else - if tk == "number" then -- or find(k,"^%d+$") then - if hexify then - handle(format("%s [0x%04X]=%q,",depth,k,tostring(v))) - else - handle(format("%s [%s]=%q,",depth,k,tostring(v))) - end - elseif tk == "boolean" then -- or find(k,"^%d+$") then - handle(format("%s [%s]=%q,",depth,tostring(k),tostring(v))) - elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then - handle(format("%s %s=%q,",depth,k,tostring(v))) - else - handle(format("%s [%q]=%q,",depth,k,tostring(v))) - end - end - end + end + return true +end +table.identical=identical +table.are_equal=are_equal +function table.compact(t) + if t then + for k,v in next,t do + if not next(v) then + t[k]=nil + end end - if level > 0 then - handle(format("%s},",depth)) + end +end +function table.contains(t,v) + if t then + for i=1,#t do + if t[i]==v then + return i + end end + end + return false end - --- replacing handle by a direct t[#t+1] = ... (plus test) is not much --- faster (0.03 on 1.00 for zapfino.tma) - -local function serialize(_handle,root,name,specification) -- handle wins - local tname = type(name) - if type(specification) == "table" then - noquotes = specification.noquotes - hexify = specification.hexify - handle = _handle or specification.handle or print - reduce = specification.reduce or false - functions = specification.functions - compact = specification.compact - inline = specification.inline and compact - if functions == nil then - functions = true - end - if compact == nil then - compact = true - end - if inline == nil then - inline = compact - end - else - noquotes = false - hexify = false - handle = _handle or print - reduce = false - compact = true - inline = true - functions = true - end - if tname == "string" then - if name == "return" then - handle("return {") - else - handle(name .. "={") - end - elseif tname == "number" then - if hexify then - handle(format("[0x%04X]={",name)) - else - handle("[" .. name .. "]={") - end - elseif tname == "boolean" then - if name then - handle("return {") - else - handle("{") - end - else - handle("t={") - end - if root then - -- The dummy access will initialize a table that has a delayed initialization - -- using a metatable. (maybe explicitly test for metatable) - if getmetatable(root) then -- todo: make this an option, maybe even per subtable - local dummy = root._w_h_a_t_e_v_e_r_ - root._w_h_a_t_e_v_e_r_ = nil - end - -- Let's forget about empty tables. - if next(root) then - do_serialize(root,name,"",0) - end +function table.count(t) + local n=0 + for k,v in next,t do + n=n+1 + end + return n +end +function table.swapped(t,s) + local n={} + if s then + for k,v in next,s do + n[k]=v + end + end + for k,v in next,t do + n[v]=k + end + return n +end +function table.mirrored(t) + local n={} + for k,v in next,t do + n[v]=k + n[k]=v + end + return n +end +function table.reversed(t) + if t then + local tt,tn={},#t + if tn>0 then + local ttn=0 + for i=tn,1,-1 do + ttn=ttn+1 + tt[ttn]=t[i] + end + end + return tt + end +end +function table.reverse(t) + if t then + local n=#t + for i=1,floor(n/2) do + local j=n-i+1 + t[i],t[j]=t[j],t[i] end - handle("}") + return t + end end - - -function table.serialize(root,name,specification) - local t, n = { }, 0 - local function flush(s) - n = n + 1 - t[n] = s +function table.sequenced(t,sep,simple) + if not t then + return "" + end + local n=#t + local s={} + if n>0 then + for i=1,n do + s[i]=tostring(t[i]) + end + else + n=0 + for k,v in sortedhash(t) do + if simple then + if v==true then + n=n+1 + s[n]=k + elseif v and v~="" then + n=n+1 + s[n]=k.."="..tostring(v) + end + else + n=n+1 + s[n]=k.."="..tostring(v) + end + end + end + return concat(s,sep or " | ") +end +function table.print(t,...) + if type(t)~="table" then + print(tostring(t)) + else + serialize(print,t,...) + end +end +setinspector(function(v) if type(v)=="table" then serialize(print,v,"table") return true end end) +function table.sub(t,i,j) + return { unpack(t,i,j) } +end +function table.is_empty(t) + return not t or not next(t) +end +function table.has_one_entry(t) + return t and not next(t,next(t)) +end +function table.loweredkeys(t) + local l={} + for k,v in next,t do + l[lower(k)]=v + end + return l +end +function table.unique(old) + local hash={} + local new={} + local n=0 + for i=1,#old do + local oi=old[i] + if not hash[oi] then + n=n+1 + new[n]=oi + hash[oi]=true end - serialize(flush,root,name,specification) - return concat(t,"\n") + end + return new +end +function table.sorted(t,...) + sort(t,...) + return t end -table.tohandle = serialize - --- sometimes tables are real use (zapfino extra pro is some 85M) in which --- case a stepwise serialization is nice; actually, we could consider: --- --- for line in table.serializer(root,name,reduce,noquotes) do --- ...(line) --- end --- --- so this is on the todo list -local maxtab = 2*1024 +end -- of closure -function table.tofile(filename,root,name,specification) - local f = io.open(filename,'w') - if f then - if maxtab > 1 then - local t, n = { }, 0 - local function flush(s) - n = n + 1 - t[n] = s - if n > maxtab then - f:write(concat(t,"\n"),"\n") -- hm, write(sometable) should be nice - t, n = { }, 0 -- we could recycle t if needed - end - end - serialize(flush,root,name,specification) - f:write(concat(t,"\n"),"\n") - else - local function flush(s) - f:write(s,"\n") - end - serialize(flush,root,name,specification) - end - f:close() - io.flush() - end -end +do -- create closure to overcome 200 locals limit -local function flattened(t,f,depth) - if f == nil then - f = { } - depth = 0xFFFF - elseif tonumber(f) then - -- assume then only two arguments are given - depth = f - f = { } - elseif not depth then - depth = 0xFFFF - end - for k, v in next, t do - if type(k) ~= "number" then - if depth > 0 and type(v) == "table" then - flattened(v,f,depth-1) - else - f[k] = v - end - end - end - local n = #f - for k=1,#t do - local v = t[k] - if depth > 0 and type(v) == "table" then - flattened(v,f,depth-1) - n = #f - else - n = n + 1 - f[n] = v - end - end - return f -end +package.loaded["l-io"] = package.loaded["l-io"] or true -table.flattened = flattened +-- original size: 8799, stripped down to: 6325 -local function unnest(t,f) -- only used in mk, for old times sake - if not f then -- and only relevant for token lists - f = { } - end - for i=1,#t do - local v = t[i] - if type(v) == "table" then - if type(v[1]) == "table" then - unnest(v,f) - else - f[#f+1] = v - end - else - f[#f+1] = v - end - end - return f +if not modules then modules={} end modules ['l-io']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local io=io +local byte,find,gsub,format=string.byte,string.find,string.gsub,string.format +local concat=table.concat +local floor=math.floor +local type=type +if string.find(os.getenv("PATH"),";") then + io.fileseparator,io.pathseparator="\\",";" +else + io.fileseparator,io.pathseparator="/",":" end - -function table.unnest(t) -- bad name - return unnest(t) +local function readall(f) + return f:read("*all") end - -local function are_equal(a,b,n,m) -- indexed - if a and b and #a == #b then - n = n or 1 - m = m or #a - for i=n,m do - local ai, bi = a[i], b[i] - if ai==bi then - -- same - elseif type(ai)=="table" and type(bi)=="table" then - if not are_equal(ai,bi) then - return false - end - else - return false - end - end - return true +local function readall(f) + local size=f:seek("end") + if size==0 then + return "" + elseif size<1024*1024 then + f:seek("set",0) + return f:read('*all') + else + local done=f:seek("set",0) + if size<1024*1024 then + step=1024*1024 + elseif size>16*1024*1024 then + step=16*1024*1024 else - return false + step=floor(size/(1024*1024))*1024*1024/8 end + local data={} + while true do + local r=f:read(step) + if not r then + return concat(data) + else + data[#data+1]=r + end + end + end +end +io.readall=readall +function io.loaddata(filename,textmode) + local f=io.open(filename,(textmode and 'r') or 'rb') + if f then + local data=readall(f) + f:close() + if #data>0 then + return data + end + end end - -local function identical(a,b) -- assumes same structure - for ka, va in next, a do - local vb = b[ka] - if va == vb then - -- same - elseif type(va) == "table" and type(vb) == "table" then - if not identical(va,vb) then - return false - end - else - return false - end +function io.savedata(filename,data,joiner) + local f=io.open(filename,"wb") + if f then + if type(data)=="table" then + f:write(concat(data,joiner or "")) + elseif type(data)=="function" then + data(f) + else + f:write(data or "") end + f:close() + io.flush() return true + else + return false + end +end +function io.loadlines(filename,n) + local f=io.open(filename,'r') + if not f then + elseif n then + local lines={} + for i=1,n do + local line=f:read("*lines") + if line then + lines[#lines+1]=line + else + break + end + end + f:close() + lines=concat(lines,"\n") + if #lines>0 then + return lines + end + else + local line=f:read("*line") or "" + f:close() + if #line>0 then + return line + end + end +end +function io.loadchunk(filename,n) + local f=io.open(filename,'rb') + if f then + local data=f:read(n or 1024) + f:close() + if #data>0 then + return data + end + end end - -table.identical = identical -table.are_equal = are_equal - --- maybe also make a combined one - -function table.compact(t) - if t then - for k,v in next, t do - if not next(v) then - t[k] = nil - end - end - end -end - -function table.contains(t, v) - if t then - for i=1, #t do - if t[i] == v then - return i - end - end - end +function io.exists(filename) + local f=io.open(filename) + if f==nil then return false + else + f:close() + return true + end end - -function table.count(t) - local n = 0 - for k, v in next, t do - n = n + 1 - end - return n +function io.size(filename) + local f=io.open(filename) + if f==nil then + return 0 + else + local s=f:seek("end") + f:close() + return s + end end - -function table.swapped(t,s) -- hash - local n = { } - if s then - for k, v in next, s do - n[k] = v - end +function io.noflines(f) + if type(f)=="string" then + local f=io.open(filename) + if f then + local n=f and io.noflines(f) or 0 + f:close() + return n + else + return 0 end - for k, v in next, t do - n[v] = k + else + local n=0 + for _ in f:lines() do + n=n+1 end + f:seek('set',0) return n + end +end +local nextchar={ + [ 4]=function(f) + return f:read(1,1,1,1) + end, + [ 2]=function(f) + return f:read(1,1) + end, + [ 1]=function(f) + return f:read(1) + end, + [-2]=function(f) + local a,b=f:read(1,1) + return b,a + end, + [-4]=function(f) + local a,b,c,d=f:read(1,1,1,1) + return d,c,b,a + end +} +function io.characters(f,n) + if f then + return nextchar[n or 1],f + end end - -function table.reversed(t) - if t then - local tt, tn = { }, #t - if tn > 0 then - local ttn = 0 - for i=tn,1,-1 do - ttn = ttn + 1 - tt[ttn] = t[i] - end - end - return tt +local nextbyte={ + [4]=function(f) + local a,b,c,d=f:read(1,1,1,1) + if d then + return byte(a),byte(b),byte(c),byte(d) + end + end, + [3]=function(f) + local a,b,c=f:read(1,1,1) + if b then + return byte(a),byte(b),byte(c) + end + end, + [2]=function(f) + local a,b=f:read(1,1) + if b then + return byte(a),byte(b) + end + end, + [1]=function (f) + local a=f:read(1) + if a then + return byte(a) + end + end, + [-2]=function (f) + local a,b=f:read(1,1) + if b then + return byte(b),byte(a) + end + end, + [-3]=function(f) + local a,b,c=f:read(1,1,1) + if b then + return byte(c),byte(b),byte(a) + end + end, + [-4]=function(f) + local a,b,c,d=f:read(1,1,1,1) + if d then + return byte(d),byte(c),byte(b),byte(a) end + end +} +function io.bytes(f,n) + if f then + return nextbyte[n or 1],f + else + return nil,nil + end end - -function table.sequenced(t,sep,simple) -- hash only - local s, n = { }, 0 - for k, v in sortedhash(t) do - if simple then - if v == true then - n = n + 1 - s[n] = k - elseif v and v~= "" then - n = n + 1 - s[n] = k .. "=" .. tostring(v) - end - else - n = n + 1 - s[n] = k .. "=" .. tostring(v) +function io.ask(question,default,options) + while true do + io.write(question) + if options then + io.write(format(" [%s]",concat(options,"|"))) + end + if default then + io.write(format(" [%s]",default)) + end + io.write(format(" ")) + io.flush() + local answer=io.read() + answer=gsub(answer,"^%s*(.*)%s*$","%1") + if answer=="" and default then + return default + elseif not options then + return answer + else + for k=1,#options do + if options[k]==answer then + return answer end + end + local pattern="^"..answer + for k=1,#options do + local v=options[k] + if find(v,pattern) then + return v + end + end end - return concat(s, sep or " | ") + end end - -function table.print(t,...) - if type(t) ~= "table" then - print(tostring(t)) - else - table.tohandle(print,t,...) - end +local function readnumber(f,n,m) + if m then + f:seek("set",n) + n=m + end + if n==1 then + return byte(f:read(1)) + elseif n==2 then + local a,b=byte(f:read(2),1,2) + return 256*a+b + elseif n==3 then + local a,b,c=byte(f:read(3),1,3) + return 256*256*a+256*b+c + elseif n==4 then + local a,b,c,d=byte(f:read(4),1,4) + return 256*256*256*a+256*256*b+256*c+d + elseif n==8 then + local a,b=readnumber(f,4),readnumber(f,4) + return 256*a+b + elseif n==12 then + local a,b,c=readnumber(f,4),readnumber(f,4),readnumber(f,4) + return 256*256*a+256*b+c + elseif n==-2 then + local b,a=byte(f:read(2),1,2) + return 256*a+b + elseif n==-3 then + local c,b,a=byte(f:read(3),1,3) + return 256*256*a+256*b+c + elseif n==-4 then + local d,c,b,a=byte(f:read(4),1,4) + return 256*256*256*a+256*256*b+256*c+d + elseif n==-8 then + local h,g,f,e,d,c,b,a=byte(f:read(8),1,8) + return 256*256*256*256*256*256*256*a+256*256*256*256*256*256*b+256*256*256*256*256*c+256*256*256*256*d+256*256*256*e+256*256*f+256*g+h + else + return 0 + end end - --- -- -- obsolete but we keep them for a while and might comment them later -- -- -- - --- roughly: copy-loop : unpack : sub == 0.9 : 0.4 : 0.45 (so in critical apps, use unpack) - -function table.sub(t,i,j) - return { unpack(t,i,j) } +io.readnumber=readnumber +function io.readstring(f,n,m) + if m then + f:seek("set",n) + n=m + end + local str=gsub(f:read(n),"\000","") + return str end +if not io.i_limiter then function io.i_limiter() end end +if not io.o_limiter then function io.o_limiter() end end --- slower than #t on indexed tables (#t only returns the size of the numerically indexed slice) - -function table.is_empty(t) - return not t or not next(t) -end -function table.has_one_entry(t) - return t and not next(t,next(t)) -end +end -- of closure --- new +do -- create closure to overcome 200 locals limit -function table.loweredkeys(t) -- maybe utf - local l = { } - for k, v in next, t do - l[lower(k)] = v - end - return l -end +package.loaded["l-number"] = package.loaded["l-number"] or true --- new, might move (maybe duplicate) +-- original size: 4939, stripped down to: 2830 -function table.unique(old) - local hash = { } - local new = { } - local n = 0 - for i=1,#old do - local oi = old[i] - if not hash[oi] then - n = n + 1 - new[n] = oi - hash[oi] = true - end +if not modules then modules={} end modules ['l-number']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local tostring,tonumber=tostring,tonumber +local format,floor,match,rep=string.format,math.floor,string.match,string.rep +local concat,insert=table.concat,table.insert +local lpegmatch=lpeg.match +number=number or {} +local number=number +if bit32 then + local btest,bor=bit32.btest,bit32.bor + function number.bit(p) + return 2^(p-1) + end + number.hasbit=btest + number.setbit=bor + function number.setbit(x,p) + return btest(x,p) and x or x+p + end + function number.clearbit(x,p) + return btest(x,p) and x-p or x + end +else + function number.bit(p) + return 2^(p-1) + end + function number.hasbit(x,p) + return x%(p+p)>=p + end + function number.setbit(x,p) + return (x%(p+p)>=p) and x or x+p + end + function number.clearbit(x,p) + return (x%(p+p)>=p) and x-p or x + end +end +if bit32 then + local bextract=bit32.extract + local t={ + "0","0","0","0","0","0","0","0", + "0","0","0","0","0","0","0","0", + "0","0","0","0","0","0","0","0", + "0","0","0","0","0","0","0","0", + } + function number.tobitstring(b,m) + local n=32 + for i=0,31 do + local v=bextract(b,i) + local k=32-i + if v==1 then + n=k + t[k]="1" + else + t[k]="0" + end end - return new + if m then + m=33-m*8 + if m<1 then + m=1 + end + return concat(t,"",m) + elseif n<8 then + return concat(t) + elseif n<16 then + return concat(t,"",9) + elseif n<24 then + return concat(t,"",17) + else + return concat(t,"",25) + end + end +else + function number.tobitstring(n,m) + if n>0 then + local t={} + while n>0 do + insert(t,1,n%2>0 and 1 or 0) + n=floor(n/2) + end + local nn=8-#t%8 + if nn>0 and nn<8 then + for i=1,nn do + insert(t,1,0) + end + end + if m then + m=m*8-#t + if m>0 then + insert(t,1,rep("0",m)) + end + end + return concat(t) + elseif m then + rep("00000000",m) + else + return "00000000" + end + end +end +function number.valid(str,default) + return tonumber(str) or default or nil +end +function number.toevenhex(n) + local s=format("%X",n) + if #s%2==0 then + return s + else + return "0"..s + end +end +local one=lpeg.C(1-lpeg.S('')/tonumber)^1 +function number.toset(n) + return lpegmatch(one,tostring(n)) +end +local function bits(n,i,...) + if n>0 then + local m=n%2 + local n=floor(n/2) + if m>0 then + return bits(n,i+1,i,...) + else + return bits(n,i+1,...) + end + else + return... + end +end +function number.bits(n) + return { bits(n,1) } end - --- function table.sorted(t,...) --- table.sort(t,...) --- return t -- still sorts in-place --- end end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['l-lpeg'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - - --- a new lpeg fails on a #(1-P(":")) test and really needs a + P(-1) - -local lpeg = require("lpeg") - --- tracing (only used when we encounter a problem in integration of lpeg in luatex) - -local report = texio and texio.write_nl or print - --- local lpmatch = lpeg.match --- local lpprint = lpeg.print --- local lpp = lpeg.P --- local lpr = lpeg.R --- local lps = lpeg.S --- local lpc = lpeg.C --- local lpb = lpeg.B --- local lpv = lpeg.V --- local lpcf = lpeg.Cf --- local lpcb = lpeg.Cb --- local lpcg = lpeg.Cg --- local lpct = lpeg.Ct --- local lpcs = lpeg.Cs --- local lpcc = lpeg.Cc --- local lpcmt = lpeg.Cmt --- local lpcarg = lpeg.Carg - --- function lpeg.match(l,...) report("LPEG MATCH") lpprint(l) return lpmatch(l,...) end - --- function lpeg.P (l) local p = lpp (l) report("LPEG P =") lpprint(l) return p end --- function lpeg.R (l) local p = lpr (l) report("LPEG R =") lpprint(l) return p end --- function lpeg.S (l) local p = lps (l) report("LPEG S =") lpprint(l) return p end --- function lpeg.C (l) local p = lpc (l) report("LPEG C =") lpprint(l) return p end --- function lpeg.B (l) local p = lpb (l) report("LPEG B =") lpprint(l) return p end --- function lpeg.V (l) local p = lpv (l) report("LPEG V =") lpprint(l) return p end --- function lpeg.Cf (l) local p = lpcf (l) report("LPEG Cf =") lpprint(l) return p end --- function lpeg.Cb (l) local p = lpcb (l) report("LPEG Cb =") lpprint(l) return p end --- function lpeg.Cg (l) local p = lpcg (l) report("LPEG Cg =") lpprint(l) return p end --- function lpeg.Ct (l) local p = lpct (l) report("LPEG Ct =") lpprint(l) return p end --- function lpeg.Cs (l) local p = lpcs (l) report("LPEG Cs =") lpprint(l) return p end --- function lpeg.Cc (l) local p = lpcc (l) report("LPEG Cc =") lpprint(l) return p end --- function lpeg.Cmt (l) local p = lpcmt (l) report("LPEG Cmt =") lpprint(l) return p end --- function lpeg.Carg (l) local p = lpcarg(l) report("LPEG Carg =") lpprint(l) return p end +package.loaded["l-set"] = package.loaded["l-set"] or true -local type = type -local byte, char, gmatch = string.byte, string.char, string.gmatch - --- Beware, we predefine a bunch of patterns here and one reason for doing so --- is that we get consistent behaviour in some of the visualizers. - -lpeg.patterns = lpeg.patterns or { } -- so that we can share -local patterns = lpeg.patterns - -local P, R, S, V, match = lpeg.P, lpeg.R, lpeg.S, lpeg.V, lpeg.match -local Ct, C, Cs, Cc = lpeg.Ct, lpeg.C, lpeg.Cs, lpeg.Cc -local lpegtype = lpeg.type - -local utfcharacters = string.utfcharacters -local utfgmatch = unicode and unicode.utf8.gmatch - -local anything = P(1) -local endofstring = P(-1) -local alwaysmatched = P(true) - -patterns.anything = anything -patterns.endofstring = endofstring -patterns.beginofstring = alwaysmatched -patterns.alwaysmatched = alwaysmatched - -local digit, sign = R('09'), S('+-') -local cr, lf, crlf = P("\r"), P("\n"), P("\r\n") -local newline = crlf + S("\r\n") -- cr + lf -local escaped = P("\\") * anything -local squote = P("'") -local dquote = P('"') -local space = P(" ") - -local utfbom_32_be = P('\000\000\254\255') -local utfbom_32_le = P('\255\254\000\000') -local utfbom_16_be = P('\255\254') -local utfbom_16_le = P('\254\255') -local utfbom_8 = P('\239\187\191') -local utfbom = utfbom_32_be + utfbom_32_le - + utfbom_16_be + utfbom_16_le - + utfbom_8 -local utftype = utfbom_32_be / "utf-32-be" + utfbom_32_le / "utf-32-le" - + utfbom_16_be / "utf-16-be" + utfbom_16_le / "utf-16-le" - + utfbom_8 / "utf-8" + alwaysmatched / "unknown" - -local utf8next = R("\128\191") - -patterns.utf8one = R("\000\127") -patterns.utf8two = R("\194\223") * utf8next -patterns.utf8three = R("\224\239") * utf8next * utf8next -patterns.utf8four = R("\240\244") * utf8next * utf8next * utf8next -patterns.utfbom = utfbom -patterns.utftype = utftype - -local utf8char = patterns.utf8one + patterns.utf8two + patterns.utf8three + patterns.utf8four -local validutf8char = utf8char^0 * endofstring * Cc(true) + Cc(false) - -patterns.utf8 = utf8char -patterns.utf8char = utf8char -patterns.validutf8 = validutf8char -patterns.validutf8char = validutf8char - -patterns.digit = digit -patterns.sign = sign -patterns.cardinal = sign^0 * digit^1 -patterns.integer = sign^0 * digit^1 -patterns.float = sign^0 * digit^0 * P('.') * digit^1 -patterns.cfloat = sign^0 * digit^0 * P(',') * digit^1 -patterns.number = patterns.float + patterns.integer -patterns.cnumber = patterns.cfloat + patterns.integer -patterns.oct = P("0") * R("07")^1 -patterns.octal = patterns.oct -patterns.HEX = P("0x") * R("09","AF")^1 -patterns.hex = P("0x") * R("09","af")^1 -patterns.hexadecimal = P("0x") * R("09","AF","af")^1 -patterns.lowercase = R("az") -patterns.uppercase = R("AZ") -patterns.letter = patterns.lowercase + patterns.uppercase -patterns.space = space -patterns.tab = P("\t") -patterns.spaceortab = patterns.space + patterns.tab -patterns.eol = S("\n\r") -patterns.spacer = S(" \t\f\v") -- + char(0xc2, 0xa0) if we want utf (cf mail roberto) -patterns.newline = newline -patterns.emptyline = newline^1 -patterns.nonspacer = 1 - patterns.spacer -patterns.whitespace = patterns.eol + patterns.spacer -patterns.nonwhitespace = 1 - patterns.whitespace -patterns.equal = P("=") -patterns.comma = P(",") -patterns.commaspacer = P(",") * patterns.spacer^0 -patterns.period = P(".") -patterns.colon = P(":") -patterns.semicolon = P(";") -patterns.underscore = P("_") -patterns.escaped = escaped -patterns.squote = squote -patterns.dquote = dquote -patterns.nosquote = (escaped + (1-squote))^0 -patterns.nodquote = (escaped + (1-dquote))^0 -patterns.unsingle = (squote/"") * patterns.nosquote * (squote/"") -patterns.undouble = (dquote/"") * patterns.nodquote * (dquote/"") -patterns.unquoted = patterns.undouble + patterns.unsingle -- more often undouble -patterns.unspacer = ((patterns.spacer^1)/"")^0 - -patterns.somecontent = (anything - newline - space)^1 -- (utf8char - newline - space)^1 -patterns.beginline = #(1-newline) - --- print(string.unquoted("test")) --- print(string.unquoted([["t\"est"]])) --- print(string.unquoted([["t\"est"x]])) --- print(string.unquoted("\'test\'")) --- print(string.unquoted('"test"')) --- print(string.unquoted('"test"')) - -function lpeg.anywhere(pattern) --slightly adapted from website - return P { P(pattern) + 1 * V(1) } -- why so complex? -end - -function lpeg.splitter(pattern, action) - return (((1-P(pattern))^1)/action+1)^0 -end - -function lpeg.tsplitter(pattern, action) - return Ct((((1-P(pattern))^1)/action+1)^0) -end - --- probleem: separator can be lpeg and that does not hash too well, but --- it's quite okay as the key is then not garbage collected - -local splitters_s, splitters_m, splitters_t = { }, { }, { } - -local function splitat(separator,single) - local splitter = (single and splitters_s[separator]) or splitters_m[separator] - if not splitter then - separator = P(separator) - local other = C((1 - separator)^0) - if single then - local any = anything - splitter = other * (separator * C(any^0) + "") -- ? - splitters_s[separator] = splitter - else - splitter = other * (separator * other)^0 - splitters_m[separator] = splitter - end - end - return splitter -end +-- original size: 1923, stripped down to: 1133 -local function tsplitat(separator) - local splitter = splitters_t[separator] - if not splitter then - splitter = Ct(splitat(separator)) - splitters_t[separator] = splitter +if not modules then modules={} end modules ['l-set']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +set=set or {} +local nums={} +local tabs={} +local concat=table.concat +local next,type=next,type +set.create=table.tohash +function set.tonumber(t) + if next(t) then + local s="" + for k,v in next,t do + if v then + s=s.." "..k + end + end + local n=nums[s] + if not n then + n=#tabs+1 + tabs[n]=t + nums[s]=n end - return splitter + return n + else + return 0 + end end - -lpeg.splitat = splitat -lpeg.tsplitat = tsplitat - -function string.splitup(str,separator) - if not separator then - separator = "," - end - return match(splitters_m[separator] or splitat(separator),str) +function set.totable(n) + if n==0 then + return {} + else + return tabs[n] or {} + end end - - -local cache = { } - -function lpeg.split(separator,str) - local c = cache[separator] - if not c then - c = tsplitat(separator) - cache[separator] = c +function set.tolist(n) + if n==0 or not tabs[n] then + return "" + else + local t,n={},0 + for k,v in next,tabs[n] do + if v then + n=n+1 + t[n]=k + end end - return match(c,str) + return concat(t," ") + end end - -function string.split(str,separator) - local c = cache[separator] - if not c then - c = tsplitat(separator) - cache[separator] = c - end - return match(c,str) +function set.contains(n,s) + if type(n)=="table" then + return n[s] + elseif n==0 then + return false + else + local t=tabs[n] + return t and t[s] + end end -local spacing = patterns.spacer^0 * newline -- sort of strip -local empty = spacing * Cc("") -local nonempty = Cs((1-spacing)^1) * spacing^-1 -local content = (empty + nonempty)^1 -patterns.textline = content +end -- of closure +do -- create closure to overcome 200 locals limit -local linesplitter = tsplitat(newline) +package.loaded["l-os"] = package.loaded["l-os"] or true -patterns.linesplitter = linesplitter +-- original size: 13692, stripped down to: 8406 -function string.splitlines(str) - return match(linesplitter,str) +if not modules then modules={} end modules ['l-os']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local os=os +local date,time=os.date,os.time +local find,format,gsub,upper,gmatch=string.find,string.format,string.gsub,string.upper,string.gmatch +local concat=table.concat +local random,ceil,randomseed=math.random,math.ceil,math.randomseed +local rawget,rawset,type,getmetatable,setmetatable,tonumber,tostring=rawget,rawset,type,getmetatable,setmetatable,tonumber,tostring +math.initialseed=tonumber(string.sub(string.reverse(tostring(ceil(socket and socket.gettime()*10000 or time()))),1,6)) +randomseed(math.initialseed) +if not os.__getenv__ then + os.__getenv__=os.getenv + os.__setenv__=os.setenv + if os.env then + local osgetenv=os.getenv + local ossetenv=os.setenv + local osenv=os.env local _=osenv.PATH + function os.setenv(k,v) + if v==nil then + v="" + end + local K=upper(k) + osenv[K]=v + if type(v)=="table" then + v=concat(v,";") + end + ossetenv(K,v) + end + function os.getenv(k) + local K=upper(k) + local v=osenv[K] or osenv[k] or osgetenv(K) or osgetenv(k) + if v=="" then + return nil + else + return v + end + end + else + local ossetenv=os.setenv + local osgetenv=os.getenv + local osenv={} + function os.setenv(k,v) + if v==nil then + v="" + end + local K=upper(k) + osenv[K]=v + end + function os.getenv(k) + local K=upper(k) + local v=osenv[K] or osgetenv(K) or osgetenv(k) + if v=="" then + return nil + else + return v + end + end + local function __index(t,k) + return os.getenv(k) + end + local function __newindex(t,k,v) + os.setenv(k,v) + end + os.env={} + setmetatable(os.env,{ __index=__index,__newindex=__newindex } ) + end end - -local utflinesplitter = utfbom^-1 * tsplitat(newline) - -patterns.utflinesplitter = utflinesplitter - -function string.utfsplitlines(str) - return match(utflinesplitter,str or "") +local execute,spawn,exec,iopopen,ioflush=os.execute,os.spawn or os.execute,os.exec or os.execute,io.popen,io.flush +function os.execute(...) ioflush() return execute(...) end +function os.spawn (...) ioflush() return spawn (...) end +function os.exec (...) ioflush() return exec (...) end +function io.popen (...) ioflush() return iopopen(...) end +function os.resultof(command) + local handle=io.popen(command,"r") + return handle and handle:read("*all") or "" end - - -local cache = { } - -function lpeg.checkedsplit(separator,str) - local c = cache[separator] - if not c then - separator = P(separator) - local other = C((1 - separator)^1) - c = Ct(separator^0 * other * (separator^1 * other)^0) - cache[separator] = c +if not io.fileseparator then + if find(os.getenv("PATH"),";") then + io.fileseparator,io.pathseparator,os.type="\\",";",os.type or "mswin" + else + io.fileseparator,io.pathseparator,os.type="/",":",os.type or "unix" + end +end +os.type=os.type or (io.pathseparator==";" and "windows") or "unix" +os.name=os.name or (os.type=="windows" and "mswin" ) or "linux" +if os.type=="windows" then + os.libsuffix,os.binsuffix,os.binsuffixes='dll','exe',{ 'exe','cmd','bat' } +else + os.libsuffix,os.binsuffix,os.binsuffixes='so','',{ '' } +end +local launchers={ + windows="start %s", + macosx="open %s", + unix="$BROWSER %s &> /dev/null &", +} +function os.launch(str) + os.execute(format(launchers[os.name] or launchers.unix,str)) +end +if not os.times then + function os.times() + return { + utime=os.gettimeofday(), + stime=0, + cutime=0, + cstime=0, + } + end +end +os.gettimeofday=os.gettimeofday or os.clock +local startuptime=os.gettimeofday() +function os.runtime() + return os.gettimeofday()-startuptime +end +os.resolvers=os.resolvers or {} +local resolvers=os.resolvers +setmetatable(os,{ __index=function(t,k) + local r=resolvers[k] + return r and r(t,k) or nil +end }) +local name,platform=os.name or "linux",os.getenv("MTX_PLATFORM") or "" +local function guess() + local architecture=os.resultof("uname -m") or "" + if architecture~="" then + return architecture + end + architecture=os.getenv("HOSTTYPE") or "" + if architecture~="" then + return architecture + end + return os.resultof("echo $HOSTTYPE") or "" +end +if platform~="" then + os.platform=platform +elseif os.type=="windows" then + function os.resolvers.platform(t,k) + local platform,architecture="",os.getenv("PROCESSOR_ARCHITECTURE") or "" + if find(architecture,"AMD64") then + platform="mswin-64" + else + platform="mswin" + end + os.setenv("MTX_PLATFORM",platform) + os.platform=platform + return platform + end +elseif name=="linux" then + function os.resolvers.platform(t,k) + local platform,architecture="",os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" + if find(architecture,"x86_64") then + platform="linux-64" + elseif find(architecture,"ppc") then + platform="linux-ppc" + else + platform="linux" + end + os.setenv("MTX_PLATFORM",platform) + os.platform=platform + return platform + end +elseif name=="macosx" then + function os.resolvers.platform(t,k) + local platform,architecture="",os.resultof("echo $HOSTTYPE") or "" + if architecture=="" then + platform="osx-intel" + elseif find(architecture,"i386") then + platform="osx-intel" + elseif find(architecture,"x86_64") then + platform="osx-64" + else + platform="osx-ppc" + end + os.setenv("MTX_PLATFORM",platform) + os.platform=platform + return platform + end +elseif name=="sunos" then + function os.resolvers.platform(t,k) + local platform,architecture="",os.resultof("uname -m") or "" + if find(architecture,"sparc") then + platform="solaris-sparc" + else + platform="solaris-intel" + end + os.setenv("MTX_PLATFORM",platform) + os.platform=platform + return platform + end +elseif name=="freebsd" then + function os.resolvers.platform(t,k) + local platform,architecture="",os.resultof("uname -m") or "" + if find(architecture,"amd64") then + platform="freebsd-amd64" + else + platform="freebsd" + end + os.setenv("MTX_PLATFORM",platform) + os.platform=platform + return platform + end +elseif name=="kfreebsd" then + function os.resolvers.platform(t,k) + local platform,architecture="",os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" + if find(architecture,"x86_64") then + platform="kfreebsd-amd64" + else + platform="kfreebsd-i386" end - return match(c,str) + os.setenv("MTX_PLATFORM",platform) + os.platform=platform + return platform + end +else + function os.resolvers.platform(t,k) + local platform="linux" + os.setenv("MTX_PLATFORM",platform) + os.platform=platform + return platform + end +end +local t={ 8,9,"a","b" } +function os.uuid() + return format("%04x%04x-4%03x-%s%03x-%04x-%04x%04x%04x", + random(0xFFFF),random(0xFFFF), + random(0x0FFF), + t[ceil(random(4))] or 8,random(0x0FFF), + random(0xFFFF), + random(0xFFFF),random(0xFFFF),random(0xFFFF) + ) end - -function string.checkedsplit(str,separator) - local c = cache[separator] - if not c then - separator = P(separator) - local other = C((1 - separator)^1) - c = Ct(separator^0 * other * (separator^1 * other)^0) - cache[separator] = c +local d +function os.timezone(delta) + d=d or tonumber(tonumber(date("%H")-date("!%H"))) + if delta then + if d>0 then + return format("+%02i:00",d) + else + return format("-%02i:00",-d) + end + else + return 1 + end +end +local timeformat=format("%%s%s",os.timezone(true)) +local dateformat="!%Y-%m-%d %H:%M:%S" +function os.fulltime(t,default) + t=tonumber(t) or 0 + if t>0 then + elseif default then + return default + else + t=nil + end + return format(timeformat,date(dateformat,t)) +end +local dateformat="%Y-%m-%d %H:%M:%S" +function os.localtime(t,default) + t=tonumber(t) or 0 + if t>0 then + elseif default then + return default + else + t=nil + end + return date(dateformat,t) +end +function os.converttime(t,default) + local t=tonumber(t) + if t and t>0 then + return date(dateformat,t) + else + return default or "-" + end +end +local memory={} +local function which(filename) + local fullname=memory[filename] + if fullname==nil then + local suffix=file.suffix(filename) + local suffixes=suffix=="" and os.binsuffixes or { suffix } + for directory in gmatch(os.getenv("PATH"),"[^"..io.pathseparator.."]+") do + local df=file.join(directory,filename) + for i=1,#suffixes do + local dfs=file.addsuffix(df,suffixes[i]) + if io.exists(dfs) then + fullname=dfs + break + end + end + end + if not fullname then + fullname=false end - return match(c,str) + memory[filename]=fullname + end + return fullname +end +os.which=which +os.where=which +function os.today() + return date("!*t") +end +function os.now() + return date("!%Y-%m-%d %H:%M:%S") +end +if not os.sleep and socket then + os.sleep=socket.sleep end -local function f2(s) local c1, c2 = byte(s,1,2) return c1 * 64 + c2 - 12416 end -local function f3(s) local c1, c2, c3 = byte(s,1,3) return (c1 * 64 + c2) * 64 + c3 - 925824 end -local function f4(s) local c1, c2, c3, c4 = byte(s,1,4) return ((c1 * 64 + c2) * 64 + c3) * 64 + c4 - 63447168 end - -local utf8byte = patterns.utf8one/byte + patterns.utf8two/f2 + patterns.utf8three/f3 + patterns.utf8four/f4 - -patterns.utf8byte = utf8byte +end -- of closure +do -- create closure to overcome 200 locals limit +package.loaded["l-file"] = package.loaded["l-file"] or true -local cache = { } +-- original size: 16648, stripped down to: 9051 -function lpeg.stripper(str) - if type(str) == "string" then - local s = cache[str] - if not s then - s = Cs(((S(str)^1)/"" + 1)^0) - cache[str] = s - end - return s - else - return Cs(((str^1)/"" + 1)^0) - end +if not modules then modules={} end modules ['l-file']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +file=file or {} +local file=file +if not lfs then + lfs=optionalrequire("lfs") +end +if not lfs then + lfs={ + getcurrentdir=function() + return "." + end, + attributes=function() + return nil + end, + isfile=function(name) + local f=io.open(name,'rb') + if f then + f:close() + return true + end + end, + isdir=function(name) + print("you need to load lfs") + return false + end + } +elseif not lfs.isfile then + local attributes=lfs.attributes + function lfs.isdir(name) + return attributes(name,"mode")=="directory" + end + function lfs.isfile(name) + return attributes(name,"mode")=="file" + end +end +local insert,concat=table.insert,table.concat +local match=string.match +local lpegmatch=lpeg.match +local getcurrentdir,attributes=lfs.currentdir,lfs.attributes +local checkedsplit=string.checkedsplit +local P,R,S,C,Cs,Cp,Cc,Ct=lpeg.P,lpeg.R,lpeg.S,lpeg.C,lpeg.Cs,lpeg.Cp,lpeg.Cc,lpeg.Ct +local colon=P(":") +local period=P(".") +local periods=P("..") +local fwslash=P("/") +local bwslash=P("\\") +local slashes=S("\\/") +local noperiod=1-period +local noslashes=1-slashes +local name=noperiod^1 +local suffix=period/""*(1-period-slashes)^1*-1 +local pattern=C((1-(slashes^1*noslashes^1*-1))^1)*P(1) +local function pathpart(name,default) + return name and lpegmatch(pattern,name) or default or "" +end +local pattern=(noslashes^0*slashes)^1*C(noslashes^1)*-1 +local function basename(name) + return name and lpegmatch(pattern,name) or name end - -local cache = { } - -function lpeg.keeper(str) - if type(str) == "string" then - local s = cache[str] - if not s then - s = Cs((((1-S(str))^1)/"" + 1)^0) - cache[str] = s - end - return s +local pattern=(noslashes^0*slashes^1)^0*Cs((1-suffix)^1)*suffix^0 +local function nameonly(name) + return name and lpegmatch(pattern,name) or name +end +local pattern=(noslashes^0*slashes)^0*(noperiod^1*period)^1*C(noperiod^1)*-1 +local function suffixonly(name) + return name and lpegmatch(pattern,name) or "" +end +file.pathpart=pathpart +file.basename=basename +file.nameonly=nameonly +file.suffixonly=suffixonly +file.suffix=suffixonly +file.dirname=pathpart +file.extname=suffixonly +local drive=C(R("az","AZ"))*colon +local path=C((noslashes^0*slashes)^0) +local suffix=period*C(P(1-period)^0*P(-1)) +local base=C((1-suffix)^0) +local rest=C(P(1)^0) +drive=drive+Cc("") +path=path+Cc("") +base=base+Cc("") +suffix=suffix+Cc("") +local pattern_a=drive*path*base*suffix +local pattern_b=path*base*suffix +local pattern_c=C(drive*path)*C(base*suffix) +local pattern_d=path*rest +function file.splitname(str,splitdrive) + if not str then + elseif splitdrive then + return lpegmatch(pattern_a,str) + else + return lpegmatch(pattern_b,str) + end +end +function file.splitbase(str) + return str and lpegmatch(pattern_d,str) +end +function file.nametotable(str,splitdrive) + if str then + local path,drive,subpath,name,base,suffix=lpegmatch(pattern_c,str) + if splitdrive then + return { + path=path, + drive=drive, + subpath=subpath, + name=name, + base=base, + suffix=suffix, + } + else + return { + path=path, + name=name, + base=base, + suffix=suffix, + } + end + end +end +local pattern=Cs(((period*(1-period-slashes)^1*-1)/""+1)^1) +function file.removesuffix(name) + return name and lpegmatch(pattern,name) +end +local suffix=period/""*(1-period-slashes)^1*-1 +local pattern=Cs((noslashes^0*slashes^1)^0*((1-suffix)^1))*Cs(suffix) +function file.addsuffix(filename,suffix,criterium) + if not filename or not suffix or suffix=="" then + return filename + elseif criterium==true then + return filename.."."..suffix + elseif not criterium then + local n,s=lpegmatch(pattern,filename) + if not s or s=="" then + return filename.."."..suffix else - return Cs((((1-str)^1)/"" + 1)^0) + return filename + end + else + local n,s=lpegmatch(pattern,filename) + if s and s~="" then + local t=type(criterium) + if t=="table" then + for i=1,#criterium do + if s==criterium[i] then + return filename + end + end + elseif t=="string" then + if s==criterium then + return filename + end + end end + return (n or filename).."."..suffix + end end - -function lpeg.frontstripper(str) -- or pattern (yet undocumented) - return (P(str) + P(true)) * Cs(P(1)^0) +local suffix=period*(1-period-slashes)^1*-1 +local pattern=Cs((1-suffix)^0) +function file.replacesuffix(name,suffix) + if name and suffix and suffix~="" then + return lpegmatch(pattern,name).."."..suffix + else + return name + end end - -function lpeg.endstripper(str) -- or pattern (yet undocumented) - return Cs((1 - P(str) * P(-1))^0) +local reslasher=lpeg.replacer(P("\\"),"/") +function file.reslash(str) + return str and lpegmatch(reslasher,str) end - --- Just for fun I looked at the used bytecode and --- p = (p and p + pp) or pp gets one more (testset). - -function lpeg.replacer(one,two) - if type(one) == "table" then - local no = #one - if no > 0 then - local p - for i=1,no do - local o = one[i] - local pp = P(o[1]) / o[2] - if p then - p = p + pp - else - p = pp - end - end - return Cs((p + 1)^0) - end - else - two = two or "" - return Cs((P(one)/two + 1)^0) +function file.is_writable(name) + if not name then + elseif lfs.isdir(name) then + name=name.."/m_t_x_t_e_s_t.tmp" + local f=io.open(name,"wb") + if f then + f:close() + os.remove(name) + return true end -end - -local splitters_f, splitters_s = { }, { } - -function lpeg.firstofsplit(separator) -- always return value - local splitter = splitters_f[separator] - if not splitter then - separator = P(separator) - splitter = C((1 - separator)^0) - splitters_f[separator] = splitter + elseif lfs.isfile(name) then + local f=io.open(name,"ab") + if f then + f:close() + return true + end + else + local f=io.open(name,"ab") + if f then + f:close() + os.remove(name) + return true end - return splitter + end + return false end - -function lpeg.secondofsplit(separator) -- nil if not split - local splitter = splitters_s[separator] - if not splitter then - separator = P(separator) - splitter = (1 - separator)^0 * separator * C(anything^0) - splitters_s[separator] = splitter +local readable=P("r")*Cc(true) +function file.is_readable(name) + if name then + local a=attributes(name) + return a and lpegmatch(readable,a.permissions) or false + else + return false + end +end +file.isreadable=file.is_readable +file.iswritable=file.is_writable +function file.size(name) + if name then + local a=attributes(name) + return a and a.size or 0 + else + return 0 + end +end +function file.splitpath(str,separator) + return str and checkedsplit(lpegmatch(reslasher,str),separator or io.pathseparator) +end +function file.joinpath(tab,separator) + return tab and concat(tab,separator or io.pathseparator) +end +local stripper=Cs(P(fwslash)^0/""*reslasher) +local isnetwork=fwslash*fwslash*(1-fwslash)+(1-fwslash-colon)^1*colon +local isroot=fwslash^1*-1 +local hasroot=fwslash^1 +local deslasher=lpeg.replacer(S("\\/")^1,"/") +function file.join(...) + local lst={... } + local one=lst[1] + if lpegmatch(isnetwork,one) then + local two=lpegmatch(deslasher,concat(lst,"/",2)) + return one.."/"..two + elseif lpegmatch(isroot,one) then + local two=lpegmatch(deslasher,concat(lst,"/",2)) + if lpegmatch(hasroot,two) then + return two + else + return "/"..two + end + elseif one=="" then + return lpegmatch(stripper,concat(lst,"/",2)) + else + return lpegmatch(deslasher,concat(lst,"/")) + end +end +local drivespec=R("az","AZ")^1*colon +local anchors=fwslash+drivespec +local untouched=periods+(1-period)^1*P(-1) +local splitstarter=(Cs(drivespec*(bwslash/"/"+fwslash)^0)+Cc(false))*Ct(lpeg.splitat(S("/\\")^1)) +local absolute=fwslash +function file.collapsepath(str,anchor) + if not str then + return + end + if anchor and not lpegmatch(anchors,str) then + str=getcurrentdir().."/"..str + end + if str=="" or str=="." then + return "." + elseif lpegmatch(untouched,str) then + return lpegmatch(reslasher,str) + end + local starter,oldelements=lpegmatch(splitstarter,str) + local newelements={} + local i=#oldelements + while i>0 do + local element=oldelements[i] + if element=='.' then + elseif element=='..' then + local n=i-1 + while n>0 do + local element=oldelements[n] + if element~='..' and element~='.' then + oldelements[n]='.' + break + else + n=n-1 + end + end + if n<1 then + insert(newelements,1,'..') + end + elseif element~="" then + insert(newelements,1,element) + end + i=i-1 + end + if #newelements==0 then + return starter or "." + elseif starter then + return starter..concat(newelements,'/') + elseif lpegmatch(absolute,str) then + return "/"..concat(newelements,'/') + else + return concat(newelements,'/') + end +end +local validchars=R("az","09","AZ","--","..") +local pattern_a=lpeg.replacer(1-validchars) +local pattern_a=Cs((validchars+P(1)/"-")^1) +local whatever=P("-")^0/"" +local pattern_b=Cs(whatever*(1-whatever*-1)^1) +function file.robustname(str,strict) + if str then + str=lpegmatch(pattern_a,str) or str + if strict then + return lpegmatch(pattern_b,str) or str + else + return str end - return splitter + end end - -function lpeg.balancer(left,right) - left, right = P(left), P(right) - return P { left * ((1 - left - right) + V(1))^0 * right } +file.readdata=io.loaddata +file.savedata=io.savedata +function file.copy(oldname,newname) + if oldname and newname then + local data=io.loaddata(oldname) + if data and data~="" then + file.savedata(newname,data) + end + end +end +local letter=R("az","AZ")+S("_-+") +local separator=P("://") +local qualified=period^0*fwslash+letter*colon+letter^1*separator+letter^1*fwslash +local rootbased=fwslash+letter*colon +lpeg.patterns.qualified=qualified +lpeg.patterns.rootbased=rootbased +function file.is_qualified_path(filename) + return filename and lpegmatch(qualified,filename)~=nil +end +function file.is_rootbased_path(filename) + return filename and lpegmatch(rootbased,filename)~=nil end +function file.strip(name,dir) + if name then + local b,a=match(name,"^(.-)"..dir.."(.*)$") + return a~="" and a or name + end +end + + +end -- of closure +do -- create closure to overcome 200 locals limit +package.loaded["l-md5"] = package.loaded["l-md5"] or true -local nany = utf8char/"" +-- original size: 3760, stripped down to: 2088 -function lpeg.counter(pattern) - pattern = Cs((P(pattern)/" " + nany)^0) - return function(str) - return #match(pattern,str) +if not modules then modules={} end modules ['l-md5']={ + version=1.001, + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +if not md5 then + md5=optionalrequire("md5") +end +if not md5 then + md5={ + sum=function(str) print("error: md5 is not loaded (sum ignored)") return str end, + sumhexa=function(str) print("error: md5 is not loaded (sumhexa ignored)") return str end, + } +end +local md5,file=md5,file +local gsub,format,byte=string.gsub,string.format,string.byte +local md5sum=md5.sum +local function convert(str,fmt) + return (gsub(md5sum(str),".",function(chr) return format(fmt,byte(chr)) end)) +end +if not md5.HEX then function md5.HEX(str) return convert(str,"%02X") end end +if not md5.hex then function md5.hex(str) return convert(str,"%02x") end end +if not md5.dec then function md5.dec(str) return convert(str,"%03i") end end +function file.needsupdating(oldname,newname,threshold) + local oldtime=lfs.attributes(oldname,"modification") + if oldtime then + local newtime=lfs.attributes(newname,"modification") + if not newtime then + return true + elseif newtime>=oldtime then + return false + elseif oldtime-newtime<(threshold or 1) then + return false + else + return true end + else + return false + end end - -if utfgmatch then - - function lpeg.count(str,what) -- replaces string.count - if type(what) == "string" then - local n = 0 - for _ in utfgmatch(str,what) do - n = n + 1 - end - return n - else -- 4 times slower but still faster than / function - return #match(Cs((P(what)/" " + nany)^0),str) - end +file.needs_updating=file.needsupdating +function file.syncmtimes(oldname,newname) + local oldtime=lfs.attributes(oldname,"modification") + if oldtime and lfs.isfile(newname) then + lfs.touch(newname,oldtime,oldtime) + end +end +function file.checksum(name) + if md5 then + local data=io.loaddata(name) + if data then + return md5.HEX(data) end + end + return nil +end +function file.loadchecksum(name) + if md5 then + local data=io.loaddata(name..".md5") + return data and (gsub(data,"%s","")) + end + return nil +end +function file.savechecksum(name,checksum) + if not checksum then checksum=file.checksum(name) end + if checksum then + io.savedata(name..".md5",checksum) + return checksum + end + return nil +end -else - local cache = { } +end -- of closure - function lpeg.count(str,what) -- replaces string.count - if type(what) == "string" then - local p = cache[what] - if not p then - p = Cs((P(what)/" " + nany)^0) - cache[p] = p - end - return #match(p,str) - else -- 4 times slower but still faster than / function - return #match(Cs((P(what)/" " + nany)^0),str) - end - end +do -- create closure to overcome 200 locals limit -end +package.loaded["l-url"] = package.loaded["l-url"] or true -local patterns_escapes = { -- also defines in l-string - ["%"] = "%%", - ["."] = "%.", - ["+"] = "%+", ["-"] = "%-", ["*"] = "%*", - ["["] = "%[", ["]"] = "%]", - ["("] = "%)", [")"] = "%)", - -- ["{"] = "%{", ["}"] = "%}" - -- ["^"] = "%^", ["$"] = "%$", -} +-- original size: 11806, stripped down to: 5417 -local simple_escapes = { -- also defines in l-string - ["-"] = "%-", - ["."] = "%.", - ["?"] = ".", - ["*"] = ".*", +if not modules then modules={} end modules ['l-url']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" } - -local p = Cs((S("-.+*%()[]") / patterns_escapes + anything)^0) -local s = Cs((S("-.+*%()[]") / simple_escapes + anything)^0) - -function string.escapedpattern(str,simple) - return match(simple and s or p,str) +local char,format,byte=string.char,string.format,string.byte +local concat=table.concat +local tonumber,type=tonumber,type +local P,C,R,S,Cs,Cc,Ct,Cf,Cg,V=lpeg.P,lpeg.C,lpeg.R,lpeg.S,lpeg.Cs,lpeg.Cc,lpeg.Ct,lpeg.Cf,lpeg.Cg,lpeg.V +local lpegmatch,lpegpatterns,replacer=lpeg.match,lpeg.patterns,lpeg.replacer +url=url or {} +local url=url +local tochar=function(s) return char(tonumber(s,16)) end +local colon=P(":") +local qmark=P("?") +local hash=P("#") +local slash=P("/") +local percent=P("%") +local endofstring=P(-1) +local hexdigit=R("09","AF","af") +local plus=P("+") +local nothing=Cc("") +local escapedchar=(percent*C(hexdigit*hexdigit))/tochar +local escaped=(plus/" ")+escapedchar +local noslash=P("/")/"" +local schemestr=Cs((escaped+(1-colon-slash-qmark-hash))^2) +local authoritystr=Cs((escaped+(1- slash-qmark-hash))^0) +local pathstr=Cs((escaped+(1- qmark-hash))^0) +local querystr=Cs(((1- hash))^0) +local fragmentstr=Cs((escaped+(1- endofstring))^0) +local scheme=schemestr*colon+nothing +local authority=slash*slash*authoritystr+nothing +local path=slash*pathstr+nothing +local query=qmark*querystr+nothing +local fragment=hash*fragmentstr+nothing +local validurl=scheme*authority*path*query*fragment +local parser=Ct(validurl) +lpegpatterns.url=validurl +lpegpatterns.urlsplitter=parser +local escapes={} +setmetatable(escapes,{ __index=function(t,k) + local v=format("%%%02X",byte(k)) + t[k]=v + return v +end }) +local escaper=Cs((R("09","AZ","az")^1+P(" ")/"%%20"+S("-./_")^1+P(1)/escapes)^0) +local unescaper=Cs((escapedchar+1)^0) +lpegpatterns.urlunescaped=escapedchar +lpegpatterns.urlescaper=escaper +lpegpatterns.urlunescaper=unescaper +local function split(str) + return (type(str)=="string" and lpegmatch(parser,str)) or str +end +local isscheme=schemestr*colon*slash*slash +local function hasscheme(str) + if str then + local scheme=lpegmatch(isscheme,str) + return scheme~="" and scheme or false + else + return false + end +end +local rootletter=R("az","AZ")+S("_-+") +local separator=P("://") +local qualified=P(".")^0*P("/")+rootletter*P(":")+rootletter^1*separator+rootletter^1*P("/") +local rootbased=P("/")+rootletter*P(":") +local barswapper=replacer("|",":") +local backslashswapper=replacer("\\","/") +local equal=P("=") +local amp=P("&") +local key=Cs(((escapedchar+1)-equal )^0) +local value=Cs(((escapedchar+1)-amp -endofstring)^0) +local splitquery=Cf (Ct("")*P { "sequence", + sequence=V("pair")*(amp*V("pair"))^0, + pair=Cg(key*equal*value), +},rawset) +local function hashed(str) + if str=="" then + return { + scheme="invalid", + original=str, + } + end + local s=split(str) + local rawscheme=s[1] + local rawquery=s[4] + local somescheme=rawscheme~="" + local somequery=rawquery~="" + if not somescheme and not somequery then + s={ + scheme="file", + authority="", + path=str, + query="", + fragment="", + original=str, + noscheme=true, + filename=str, + } + else + local authority,path,filename=s[2],s[3] + if authority=="" then + filename=path + elseif path=="" then + filename="" + else + filename=authority.."/"..path + end + s={ + scheme=rawscheme, + authority=authority, + path=path, + query=lpegmatch(unescaper,rawquery), + queries=lpegmatch(splitquery,rawquery), + fragment=s[5], + original=str, + noscheme=false, + filename=filename, + } + end + return s +end +url.split=split +url.hasscheme=hasscheme +url.hashed=hashed +function url.addscheme(str,scheme) + if hasscheme(str) then + return str + elseif not scheme then + return "file:///"..str + else + return scheme..":///"..str + end +end +function url.construct(hash) + local fullurl,f={},0 + local scheme,authority,path,query,fragment=hash.scheme,hash.authority,hash.path,hash.query,hash.fragment + if scheme and scheme~="" then + f=f+1;fullurl[f]=scheme.."://" + end + if authority and authority~="" then + f=f+1;fullurl[f]=authority + end + if path and path~="" then + f=f+1;fullurl[f]="/"..path + end + if query and query~="" then + f=f+1;fullurl[f]="?"..query + end + if fragment and fragment~="" then + f=f+1;fullurl[f]="#"..fragment + end + return lpegmatch(escaper,concat(fullurl)) +end +local pattern=Cs(noslash*R("az","AZ")*(S(":|")/":")*noslash*P(1)^0) +function url.filename(filename) + local spec=hashed(filename) + local path=spec.path + return (spec.scheme=="file" and path and lpegmatch(pattern,path)) or filename +end +local function escapestring(str) + return lpegmatch(escaper,str) +end +url.escape=escapestring +function url.query(str) + if type(str)=="string" then + return lpegmatch(splitquery,str) or "" + else + return str + end +end +function url.toquery(data) + local td=type(data) + if td=="string" then + return #str and escape(data) or nil + elseif td=="table" then + if next(data) then + local t={} + for k,v in next,data do + t[#t+1]=format("%s=%s",k,escapestring(v)) + end + return concat(t,"&") + end + else + end +end +local pattern=Cs(noslash^0*(1-noslash*P(-1))^0) +function url.barepath(path) + if not path or path=="" then + return "" + else + return lpegmatch(pattern,path) + end end --- utf extensies - -lpeg.UP = lpeg.P -if utfcharacters then +end -- of closure - function lpeg.US(str) - local p - for uc in utfcharacters(str) do - if p then - p = p + P(uc) - else - p = P(uc) - end - end - return p - end +do -- create closure to overcome 200 locals limit +package.loaded["l-dir"] = package.loaded["l-dir"] or true -elseif utfgmatch then +-- original size: 13139, stripped down to: 8196 - function lpeg.US(str) - local p - for uc in utfgmatch(str,".") do - if p then - p = p + P(uc) - else - p = P(uc) - end +if not modules then modules={} end modules ['l-dir']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local type,select=type,select +local find,gmatch,match,gsub=string.find,string.gmatch,string.match,string.gsub +local concat,insert,remove=table.concat,table.insert,table.remove +local lpegmatch=lpeg.match +local P,S,R,C,Cc,Cs,Ct,Cv,V=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.Cc,lpeg.Cs,lpeg.Ct,lpeg.Cv,lpeg.V +dir=dir or {} +local dir=dir +local lfs=lfs +local attributes=lfs.attributes +local walkdir=lfs.dir +local isdir=lfs.isdir +local isfile=lfs.isfile +local currentdir=lfs.currentdir +local chdir=lfs.chdir +if not isdir then + function isdir(name) + local a=attributes(name) + return a and a.mode=="directory" + end + lfs.isdir=isdir +end +if not isfile then + function isfile(name) + local a=attributes(name) + return a and a.mode=="file" + end + lfs.isfile=isfile +end +function dir.current() + return (gsub(currentdir(),"\\","/")) +end +local lfsisdir=isdir +local function isdir(path) + path=gsub(path,"[/\\]+$","") + return lfsisdir(path) +end +lfs.isdir=isdir +local function globpattern(path,patt,recurse,action) + if path=="/" then + path=path.."." + elseif not find(path,"/$") then + path=path..'/' + end + if isdir(path) then + for name in walkdir(path) do + local full=path..name + local mode=attributes(full,'mode') + if mode=='file' then + if find(full,patt) then + action(full) + end + elseif recurse and (mode=="directory") and (name~='.') and (name~="..") then + globpattern(full,patt,recurse,action) + end + end + end +end +dir.globpattern=globpattern +local function collectpattern(path,patt,recurse,result) + local ok,scanner + result=result or {} + if path=="/" then + ok,scanner,first=xpcall(function() return walkdir(path..".") end,function() end) + else + ok,scanner,first=xpcall(function() return walkdir(path) end,function() end) + end + if ok and type(scanner)=="function" then + if not find(path,"/$") then path=path..'/' end + for name in scanner,first do + local full=path..name + local attr=attributes(full) + local mode=attr.mode + if mode=='file' then + if find(full,patt) then + result[name]=attr + end + elseif recurse and (mode=="directory") and (name~='.') and (name~="..") then + attr.list=collectpattern(full,patt,recurse) + result[name]=attr + end + end + end + return result +end +dir.collectpattern=collectpattern +local pattern=Ct { + [1]=(C(P(".")+P("/")^1)+C(R("az","AZ")*P(":")*P("/")^0)+Cc("./"))*V(2)*V(3), + [2]=C(((1-S("*?/"))^0*P("/"))^0), + [3]=C(P(1)^0) +} +local filter=Cs (( + P("**")/".*"+P("*")/"[^/]*"+P("?")/"[^/]"+P(".")/"%%."+P("+")/"%%+"+P("-")/"%%-"+P(1) +)^0 ) +local function glob(str,t) + if type(t)=="function" then + if type(str)=="table" then + for s=1,#str do + glob(str[s],t) + end + elseif isfile(str) then + t(str) + else + local split=lpegmatch(pattern,str) + if split then + local root,path,base=split[1],split[2],split[3] + local recurse=find(base,"%*%*") + local start=root..path + local result=lpegmatch(filter,start..base) + globpattern(start,result,recurse,t) + end + end + else + if type(str)=="table" then + local t=t or {} + for s=1,#str do + glob(str[s],t) + end + return t + elseif isfile(str) then + if t then + t[#t+1]=str + return t + else + return { str } + end + else + local split=lpegmatch(pattern,str) + if split then + local t=t or {} + local action=action or function(name) t[#t+1]=name end + local root,path,base=split[1],split[2],split[3] + local recurse=find(base,"%*%*") + local start=root..path + local result=lpegmatch(filter,start..base) + globpattern(start,result,recurse,action) + return t + else + return {} + end + end + end +end +dir.glob=glob +local function globfiles(path,recurse,func,files) + if type(func)=="string" then + local s=func + func=function(name) return find(name,s) end + end + files=files or {} + local noffiles=#files + for name in walkdir(path) do + if find(name,"^%.") then + else + local mode=attributes(name,'mode') + if mode=="directory" then + if recurse then + globfiles(path.."/"..name,recurse,func,files) + end + elseif mode=="file" then + if not func or func(name) then + noffiles=noffiles+1 + files[noffiles]=path.."/"..name end - return p + end end - -else - - function lpeg.US(str) - local p - local f = function(uc) - if p then - p = p + P(uc) - else - p = P(uc) - end - end - match((utf8char/f)^0,str) - return p - end - + end + return files end - -local range = Cs(utf8byte) * (Cs(utf8byte) + Cc(false)) - -local utfchar = unicode and unicode.utf8 and unicode.utf8.char - -function lpeg.UR(str,more) - local first, last - if type(str) == "number" then - first = str - last = more or first +dir.globfiles=globfiles +function dir.ls(pattern) + return concat(glob(pattern),"\n") +end +local make_indeed=true +local onwindows=os.type=="windows" or find(os.getenv("PATH"),";") +if onwindows then + function dir.mkdirs(...) + local str,pth="","" + for i=1,select("#",...) do + local s=select(i,...) + if s=="" then + elseif str=="" then + str=s + else + str=str.."/"..s + end + end + local first,middle,last + local drive=false + first,middle,last=match(str,"^(//)(//*)(.*)$") + if first then else - first, last = match(range,str) - if not last then - return P(str) + first,last=match(str,"^(//)/*(.-)$") + if first then + middle,last=match(str,"([^/]+)/+(.-)$") + if middle then + pth="//"..middle + else + pth="//"..last + last="" + end + else + first,middle,last=match(str,"^([a-zA-Z]:)(/*)(.-)$") + if first then + pth,drive=first..middle,true + else + middle,last=match(str,"^(/*)(.-)$") + if not middle then + last=str + end + end + end + end + for s in gmatch(last,"[^/]+") do + if pth=="" then + pth=s + elseif drive then + pth,drive=pth..s,false + else + pth=pth.."/"..s + end + if make_indeed and not isdir(pth) then + lfs.mkdir(pth) + end + end + return pth,(isdir(pth)==true) + end +else + function dir.mkdirs(...) + local str,pth="","" + for i=1,select("#",...) do + local s=select(i,...) + if s and s~="" then + if str~="" then + str=str.."/"..s + else + str=s end + end end - if first == last then - return P(str) - elseif utfchar and last - first < 8 then -- a somewhat arbitrary criterium - local p - for i=first,last do - if p then - p = p + P(utfchar(i)) - else - p = P(utfchar(i)) - end + str=gsub(str,"/+","/") + if find(str,"^/") then + pth="/" + for s in gmatch(str,"[^/]+") do + local first=(pth=="/") + if first then + pth=pth..s + else + pth=pth.."/"..s + end + if make_indeed and not first and not isdir(pth) then + lfs.mkdir(pth) end - return p -- nil when invalid range + end else - local f = function(b) - return b >= first and b <= last + pth="." + for s in gmatch(str,"[^/]+") do + pth=pth.."/"..s + if make_indeed and not isdir(pth) then + lfs.mkdir(pth) end - return utf8byte / f -- nil when invalid range + end end + return pth,(isdir(pth)==true) + end end - - - -function lpeg.oneof(list,...) -- lpeg.oneof("elseif","else","if","then") - if type(list) ~= "table" then - list = { list, ... } +dir.makedirs=dir.mkdirs +if onwindows then + function dir.expandname(str) + local first,nothing,last=match(str,"^(//)(//*)(.*)$") + if first then + first=dir.current().."/" + end + if not first then + first,last=match(str,"^(//)/*(.*)$") + end + if not first then + first,last=match(str,"^([a-zA-Z]:)(.*)$") + if first and not find(last,"^/") then + local d=currentdir() + if chdir(first) then + first=dir.current() + end + chdir(d) + end + end + if not first then + first,last=dir.current(),str + end + last=gsub(last,"//","/") + last=gsub(last,"/%./","/") + last=gsub(last,"^/*","") + first=gsub(first,"/*$","") + if last=="" or last=="." then + return first + else + return first.."/"..last end - -- sort(list) -- longest match first - local p = P(list[1]) - for l=2,#list do - p = p + P(list[l]) + end +else + function dir.expandname(str) + if not find(str,"^/") then + str=currentdir().."/"..str end - return p + str=gsub(str,"//","/") + str=gsub(str,"/%./","/") + str=gsub(str,"(.)/%.$","%1") + return str + end end - -function lpeg.is_lpeg(p) - return p and lpegtype(p) == "pattern" +file.expandname=dir.expandname +local stack={} +function dir.push(newdir) + insert(stack,currentdir()) + if newdir and newdir~="" then + chdir(newdir) + end +end +function dir.pop() + local d=remove(stack) + if d then + chdir(d) + end + return d end --- For the moment here, but it might move to utilities. Beware, we need to --- have the longest keyword first, so 'aaa' comes beforte 'aa' which is why we --- loop back from the end cq. prepend. - -local sort, fastcopy, sortedkeys = table.sort, table.fastcopy, table.sortedkeys -- dependency! -function lpeg.append(list,pp,delayed,checked) - local p = pp - if #list > 0 then - local keys = fastcopy(list) - sort(keys) - for i=#keys,1,-1 do - local k = keys[i] - if p then - p = P(k) + p - else - p = P(k) - end - end - elseif delayed then -- hm, it looks like the lpeg parser resolves anyway - local keys = sortedkeys(list) - if p then - for i=1,#keys,1 do - local k = keys[i] - local v = list[k] - p = P(k)/list + p - end - else - for i=1,#keys do - local k = keys[i] - local v = list[k] - if p then - p = P(k) + p - else - p = P(k) - end - end - if p then - p = p / list - end - end - elseif checked then - -- problem: substitution gives a capture - local keys = sortedkeys(list) - for i=1,#keys do - local k = keys[i] - local v = list[k] - if p then - if k == v then - p = P(k) + p - else - p = P(k)/v + p - end - else - if k == v then - p = P(k) - else - p = P(k)/v - end - end - end - else - local keys = sortedkeys(list) - for i=1,#keys do - local k = keys[i] - local v = list[k] - if p then - p = P(k)/v + p - else - p = P(k)/v - end - end - end - return p -end +end -- of closure --- inspect(lpeg.append({ a = "1", aa = "1", aaa = "1" } ,nil,true)) --- inspect(lpeg.append({ ["degree celsius"] = "1", celsius = "1", degree = "1" } ,nil,true)) +do -- create closure to overcome 200 locals limit --- function lpeg.exact_match(words,case_insensitive) --- local pattern = concat(words) --- if case_insensitive then --- local pattern = S(upper(characters)) + S(lower(characters)) --- local list = { } --- for i=1,#words do --- list[lower(words[i])] = true --- end --- return Cmt(pattern^1, function(_,i,s) --- return list[lower(s)] and i --- end) --- else --- local pattern = S(concat(words)) --- local list = { } --- for i=1,#words do --- list[words[i]] = true --- end --- return Cmt(pattern^1, function(_,i,s) --- return list[s] and i --- end) --- end --- end +package.loaded["l-boolean"] = package.loaded["l-boolean"] or true --- experiment: +-- original size: 1781, stripped down to: 1503 -local function make(t) - local p --- for k, v in next, t do - for k, v in table.sortedhash(t) do - if not p then - if next(v) then - p = P(k) * make(v) - else - p = P(k) - end - else - if next(v) then - p = p + P(k) * make(v) - else - p = p + P(k) - end - end - end - return p +if not modules then modules={} end modules ['l-boolean']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local type,tonumber=type,tonumber +boolean=boolean or {} +local boolean=boolean +function boolean.tonumber(b) + if b then return 1 else return 0 end end - -function lpeg.utfchartabletopattern(list) - local tree = { } - for i=1,#list do - local t = tree - for c in gmatch(list[i],".") do - if not t[c] then - t[c] = { } - end - t = t[c] - end +function toboolean(str,tolerant) + if str==nil then + return false + elseif str==false then + return false + elseif str==true then + return true + elseif str=="true" then + return true + elseif str=="false" then + return false + elseif not tolerant then + return false + elseif str==0 then + return false + elseif (tonumber(str) or 0)>0 then + return true + else + return str=="yes" or str=="on" or str=="t" + end +end +string.toboolean=toboolean +function string.booleanstring(str) + if str=="0" then + return false + elseif str=="1" then + return true + elseif str=="" then + return false + elseif str=="false" then + return false + elseif str=="true" then + return true + elseif (tonumber(str) or 0)>0 then + return true + else + return str=="yes" or str=="on" or str=="t" + end +end +function string.is_boolean(str,default) + if type(str)=="string" then + if str=="true" or str=="yes" or str=="on" or str=="t" then + return true + elseif str=="false" or str=="no" or str=="off" or str=="f" then + return false end - return make(tree) + end + return default end --- inspect ( lpeg.utfchartabletopattern { --- utfchar(0x00A0), -- nbsp --- utfchar(0x2000), -- enquad --- utfchar(0x2001), -- emquad --- utfchar(0x2002), -- enspace --- utfchar(0x2003), -- emspace --- utfchar(0x2004), -- threeperemspace --- utfchar(0x2005), -- fourperemspace --- utfchar(0x2006), -- sixperemspace --- utfchar(0x2007), -- figurespace --- utfchar(0x2008), -- punctuationspace --- utfchar(0x2009), -- breakablethinspace --- utfchar(0x200A), -- hairspace --- utfchar(0x200B), -- zerowidthspace --- utfchar(0x202F), -- narrownobreakspace --- utfchar(0x205F), -- math thinspace --- } ) - end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['l-io'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local io = io -local byte, find, gsub, format = string.byte, string.find, string.gsub, string.format -local concat = table.concat -local type = type +package.loaded["l-unicode"] = package.loaded["l-unicode"] or true -if string.find(os.getenv("PATH"),";") then - io.fileseparator, io.pathseparator = "\\", ";" -else - io.fileseparator, io.pathseparator = "/" , ":" -end +-- original size: 26810, stripped down to: 11943 -function io.loaddata(filename,textmode) - local f = io.open(filename,(textmode and 'r') or 'rb') - if f then - local data = f:read('*all') - f:close() - return data +if not modules then modules={} end modules ['l-unicode']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +utf=utf or (unicode and unicode.utf8) or {} +utf.characters=utf.characters or string.utfcharacters +utf.values=utf.values or string.utfvalues +local type=type +local char,byte,format,sub=string.char,string.byte,string.format,string.sub +local concat=table.concat +local P,C,R,Cs,Ct,Cmt,Cc,Carg,Cp=lpeg.P,lpeg.C,lpeg.R,lpeg.Cs,lpeg.Ct,lpeg.Cmt,lpeg.Cc,lpeg.Carg,lpeg.Cp +local lpegmatch,patterns=lpeg.match,lpeg.patterns +local bytepairs=string.bytepairs +local finder=lpeg.finder +local replacer=lpeg.replacer +local utfvalues=utf.values +local utfgmatch=utf.gmatch +local p_utftype=patterns.utftype +local p_utfoffset=patterns.utfoffset +local p_utf8char=patterns.utf8char +local p_utf8byte=patterns.utf8byte +local p_utfbom=patterns.utfbom +local p_newline=patterns.newline +local p_whitespace=patterns.whitespace +if not unicode then + unicode={ utf=utf } +end +if not utf.char then + local floor,char=math.floor,string.char + function utf.char(n) + if n<0x80 then + return char(n) + elseif n<0x800 then + return char( + 0xC0+floor(n/0x40), + 0x80+(n%0x40) + ) + elseif n<0x10000 then + return char( + 0xE0+floor(n/0x1000), + 0x80+(floor(n/0x40)%0x40), + 0x80+(n%0x40) + ) + elseif n<0x200000 then + return char( + 0xF0+floor(n/0x40000), + 0x80+(floor(n/0x1000)%0x40), + 0x80+(floor(n/0x40)%0x40), + 0x80+(n%0x40) + ) else - return nil + return "" + end + end +end +if not utf.byte then + local utf8byte=patterns.utf8byte + function utf.byte(c) + return lpegmatch(utf8byte,c) + end +end +local utfchar,utfbyte=utf.char,utf.byte +function utf.filetype(data) + return data and lpegmatch(p_utftype,data) or "unknown" +end +local toentities=Cs ( + ( + patterns.utf8one+( + patterns.utf8two+patterns.utf8three+patterns.utf8four + )/function(s) local b=utfbyte(s) if b<127 then return s else return format("&#%X;",b) end end + )^0 +) +patterns.toentities=toentities +function utf.toentities(str) + return lpegmatch(toentities,str) +end +local one=P(1) +local two=C(1)*C(1) +local four=C(R(utfchar(0xD8),utfchar(0xFF)))*C(1)*C(1)*C(1) +local pattern=P("\254\255")*Cs(( + four/function(a,b,c,d) + local ab=0xFF*byte(a)+byte(b) + local cd=0xFF*byte(c)+byte(d) + return utfchar((ab-0xD800)*0x400+(cd-0xDC00)+0x10000) + end+two/function(a,b) + return utfchar(byte(a)*256+byte(b)) + end+one + )^1 )+P("\255\254")*Cs(( + four/function(b,a,d,c) + local ab=0xFF*byte(a)+byte(b) + local cd=0xFF*byte(c)+byte(d) + return utfchar((ab-0xD800)*0x400+(cd-0xDC00)+0x10000) + end+two/function(b,a) + return utfchar(byte(a)*256+byte(b)) + end+one + )^1 ) +function string.toutf(s) + return lpegmatch(pattern,s) or s +end +local validatedutf=Cs ( + ( + patterns.utf8one+patterns.utf8two+patterns.utf8three+patterns.utf8four+P(1)/"�" + )^0 +) +patterns.validatedutf=validatedutf +function utf.is_valid(str) + return type(str)=="string" and lpegmatch(validatedutf,str) or false +end +if not utf.len then + local n,f=0,1 + local utfcharcounter=patterns.utfbom^-1*Cmt ( + Cc(1)*patterns.utf8one^1+Cc(2)*patterns.utf8two^1+Cc(3)*patterns.utf8three^1+Cc(4)*patterns.utf8four^1, + function(_,t,d) + n=n+(t-f)/d + f=t + return true + end + )^0 + function utf.len(str) + n,f=0,1 + lpegmatch(utfcharcounter,str or "") + return n + end +end +utf.length=utf.len +if not utf.sub then + local utflength=utf.length + local b,e,n,first,last=0,0,0,0,0 + local function slide_zero(s,p) + n=n+1 + if n>=last then + e=p-1 + else + return p end -end - -function io.savedata(filename,data,joiner) - local f = io.open(filename,"wb") - if f then - if type(data) == "table" then - f:write(concat(data,joiner or "")) - elseif type(data) == "function" then - data(f) + end + local function slide_one(s,p) + n=n+1 + if n==first then + b=p + end + if n>=last then + e=p-1 + else + return p + end + end + local function slide_two(s,p) + n=n+1 + if n==first then + b=p + else + return true + end + end + local pattern_zero=Cmt(p_utf8char,slide_zero)^0 + local pattern_one=Cmt(p_utf8char,slide_one )^0 + local pattern_two=Cmt(p_utf8char,slide_two )^0 + function utf.sub(str,start,stop) + if not start then + return str + end + if start==0 then + start=1 + end + if not stop then + if start<0 then + local l=utflength(str) + start=l+start + else + start=start-1 + end + b,n,first=0,0,start + lpegmatch(pattern_two,str) + if n>=first then + return sub(str,b) + else + return "" + end + end + if start<0 or stop<0 then + local l=utf.length(str) + if start<0 then + start=l+start + if start<=0 then + start=1 else - f:write(data or "") + start=start+1 end - f:close() - io.flush() - return true + end + if stop<0 then + stop=l+stop + if stop==0 then + stop=1 + else + stop=stop+1 + end + end + end + if start>stop then + return "" + elseif start>1 then + b,e,n,first,last=0,0,0,start-1,stop + lpegmatch(pattern_one,str) + if n>=first and e==0 then + e=#str + end + return sub(str,b,e) else - return false - end -end - -function io.exists(filename) - local f = io.open(filename) - if f == nil then - return false + b,e,n,last=1,0,0,stop + lpegmatch(pattern_zero,str) + if e==0 then + e=#str + end + return sub(str,b,e) + end + end +end +function utf.remapper(mapping) + local pattern=Cs((p_utf8char/mapping)^0) + return function(str) + if not str or str=="" then + return "" else - assert(f:close()) - return true - end -end - -function io.size(filename) - local f = io.open(filename) - if f == nil then - return 0 + return lpegmatch(pattern,str) + end + end,pattern +end +function utf.replacer(t) + local r=replacer(t,false,false,true) + return function(str) + return lpegmatch(r,str) + end +end +function utf.subtituter(t) + local f=finder (t) + local r=replacer(t,false,false,true) + return function(str) + local i=lpegmatch(f,str) + if not i then + return str + elseif i>#str then + return str else - local s = f:seek("end") - assert(f:close()) - return s - end + return lpegmatch(r,str) + end + end +end +local utflinesplitter=p_utfbom^-1*lpeg.tsplitat(p_newline) +local utfcharsplitter_ows=p_utfbom^-1*Ct(C(p_utf8char)^0) +local utfcharsplitter_iws=p_utfbom^-1*Ct((p_whitespace^1+C(p_utf8char))^0) +local utfcharsplitter_raw=Ct(C(p_utf8char)^0) +patterns.utflinesplitter=utflinesplitter +function utf.splitlines(str) + return lpegmatch(utflinesplitter,str or "") +end +function utf.split(str,ignorewhitespace) + if ignorewhitespace then + return lpegmatch(utfcharsplitter_iws,str or "") + else + return lpegmatch(utfcharsplitter_ows,str or "") + end +end +function utf.totable(str) + return lpegmatch(utfcharsplitter_raw,str) +end +function utf.magic(f) + local str=f:read(4) or "" + local off=lpegmatch(p_utfoffset,str) + if off<4 then + f:seek('set',off) + end + return lpegmatch(p_utftype,str) end - -function io.noflines(f) - if type(f) == "string" then - local f = io.open(filename) - local n = f and io.noflines(f) or 0 - assert(f:close()) - return n - else - local n = 0 - for _ in f:lines() do - n = n + 1 +local function utf16_to_utf8_be(t) + if type(t)=="string" then + t=lpegmatch(utflinesplitter,t) + end + local result={} + for i=1,#t do + local r,more=0,0 + for left,right in bytepairs(t[i]) do + if right then + local now=256*left+right + if more>0 then + now=(more-0xD800)*0x400+(now-0xDC00)+0x10000 + more=0 + r=r+1 + result[r]=utfchar(now) + elseif now>=0xD800 and now<=0xDBFF then + more=now + else + r=r+1 + result[r]=utfchar(now) end - f:seek('set',0) - return n + end end + t[i]=concat(result,"",1,r) + end + return t end - -local nextchar = { - [ 4] = function(f) - return f:read(1,1,1,1) - end, - [ 2] = function(f) - return f:read(1,1) - end, - [ 1] = function(f) - return f:read(1) - end, - [-2] = function(f) - local a, b = f:read(1,1) - return b, a - end, - [-4] = function(f) - local a, b, c, d = f:read(1,1,1,1) - return d, c, b, a - end -} - -function io.characters(f,n) - if f then - return nextchar[n or 1], f +local function utf16_to_utf8_le(t) + if type(t)=="string" then + t=lpegmatch(utflinesplitter,t) + end + local result={} + for i=1,#t do + local r,more=0,0 + for left,right in bytepairs(t[i]) do + if right then + local now=256*right+left + if more>0 then + now=(more-0xD800)*0x400+(now-0xDC00)+0x10000 + more=0 + r=r+1 + result[r]=utfchar(now) + elseif now>=0xD800 and now<=0xDBFF then + more=now + else + r=r+1 + result[r]=utfchar(now) + end + end end + t[i]=concat(result,"",1,r) + end + return t end - -local nextbyte = { - [4] = function(f) - local a, b, c, d = f:read(1,1,1,1) - if d then - return byte(a), byte(b), byte(c), byte(d) - end - end, - [3] = function(f) - local a, b, c = f:read(1,1,1) - if b then - return byte(a), byte(b), byte(c) - end - end, - [2] = function(f) - local a, b = f:read(1,1) - if b then - return byte(a), byte(b) - end - end, - [1] = function (f) - local a = f:read(1) - if a then - return byte(a) - end - end, - [-2] = function (f) - local a, b = f:read(1,1) - if b then - return byte(b), byte(a) - end - end, - [-3] = function(f) - local a, b, c = f:read(1,1,1) - if b then - return byte(c), byte(b), byte(a) - end - end, - [-4] = function(f) - local a, b, c, d = f:read(1,1,1,1) - if d then - return byte(d), byte(c), byte(b), byte(a) +local function utf32_to_utf8_be(t) + if type(t)=="string" then + t=lpegmatch(utflinesplitter,t) + end + local result={} + for i=1,#t do + local r,more=0,-1 + for a,b in bytepairs(t[i]) do + if a and b then + if more<0 then + more=256*256*256*a+256*256*b + else + r=r+1 + result[t]=utfchar(more+256*a+b) + more=-1 end + else + break + end end -} - -function io.bytes(f,n) - if f then - return nextbyte[n or 1], f - else - return nil, nil - end + t[i]=concat(result,"",1,r) + end + return t end - -function io.ask(question,default,options) - while true do - io.write(question) - if options then - io.write(format(" [%s]",concat(options,"|"))) - end - if default then - io.write(format(" [%s]",default)) - end - io.write(format(" ")) - io.flush() - local answer = io.read() - answer = gsub(answer,"^%s*(.*)%s*$","%1") - if answer == "" and default then - return default - elseif not options then - return answer +local function utf32_to_utf8_le(t) + if type(t)=="string" then + t=lpegmatch(utflinesplitter,t) + end + local result={} + for i=1,#t do + local r,more=0,-1 + for a,b in bytepairs(t[i]) do + if a and b then + if more<0 then + more=256*b+a else - for k=1,#options do - if options[k] == answer then - return answer - end - end - local pattern = "^" .. answer - for k=1,#options do - local v = options[k] - if find(v,pattern) then - return v - end - end + r=r+1 + result[t]=utfchar(more+256*256*256*b+256*256*a) + more=-1 end + else + break + end end + t[i]=concat(result,"",1,r) + end + return t end - -local function readnumber(f,n,m) - if m then - f:seek("set",n) - n = m - end - if n == 1 then - return byte(f:read(1)) - elseif n == 2 then - local a, b = byte(f:read(2),1,2) - return 256 * a + b - elseif n == 3 then - local a, b, c = byte(f:read(3),1,3) - return 256*256 * a + 256 * b + c - elseif n == 4 then - local a, b, c, d = byte(f:read(4),1,4) - return 256*256*256 * a + 256*256 * b + 256 * c + d - elseif n == 8 then - local a, b = readnumber(f,4), readnumber(f,4) - return 256 * a + b - elseif n == 12 then - local a, b, c = readnumber(f,4), readnumber(f,4), readnumber(f,4) - return 256*256 * a + 256 * b + c - elseif n == -2 then - local b, a = byte(f:read(2),1,2) - return 256*a + b - elseif n == -3 then - local c, b, a = byte(f:read(3),1,3) - return 256*256 * a + 256 * b + c - elseif n == -4 then - local d, c, b, a = byte(f:read(4),1,4) - return 256*256*256 * a + 256*256 * b + 256*c + d - elseif n == -8 then - local h, g, f, e, d, c, b, a = byte(f:read(8),1,8) - return 256*256*256*256*256*256*256 * a + - 256*256*256*256*256*256 * b + - 256*256*256*256*256 * c + - 256*256*256*256 * d + - 256*256*256 * e + - 256*256 * f + - 256 * g + - h - else - return 0 - end +utf.utf32_to_utf8_be=utf32_to_utf8_be +utf.utf32_to_utf8_le=utf32_to_utf8_le +utf.utf16_to_utf8_be=utf16_to_utf8_be +utf.utf16_to_utf8_le=utf16_to_utf8_le +function utf.utf8_to_utf8(t) + return type(t)=="string" and lpegmatch(utflinesplitter,t) or t end - -io.readnumber = readnumber - -function io.readstring(f,n,m) - if m then - f:seek("set",n) - n = m +function utf.utf16_to_utf8(t,endian) + return endian and utf16_to_utf8_be(t) or utf16_to_utf8_le(t) or t +end +function utf.utf32_to_utf8(t,endian) + return endian and utf32_to_utf8_be(t) or utf32_to_utf8_le(t) or t +end +local function little(c) + local b=byte(c) + if b<0x10000 then + return char(b%256,b/256) + else + b=b-0x10000 + local b1,b2=b/1024+0xD800,b%1024+0xDC00 + return char(b1%256,b1/256,b2%256,b2/256) + end +end +local function big(c) + local b=byte(c) + if b<0x10000 then + return char(b/256,b%256) + else + b=b-0x10000 + local b1,b2=b/1024+0xD800,b%1024+0xDC00 + return char(b1/256,b1%256,b2/256,b2%256) + end +end +local _,l_remap=utf.remapper(little) +local _,b_remap=utf.remapper(big) +function utf.utf8_to_utf16(str,littleendian) + if littleendian then + return char(255,254)..lpegmatch(l_remap,str) + else + return char(254,255)..lpegmatch(b_remap,str) + end +end +local pattern=Cs ( + (p_utf8byte/function(unicode ) return format("0x%04X",unicode) end)*(p_utf8byte*Carg(1)/function(unicode,separator) return format("%s0x%04X",separator,unicode) end)^0 +) +function utf.tocodes(str,separator) + return lpegmatch(pattern,str,1,separator or " ") +end +function utf.ustring(s) + return format("U+%05X",type(s)=="number" and s or utfbyte(s)) +end +function utf.xstring(s) + return format("0x%05X",type(s)=="number" and s or utfbyte(s)) +end +local p_nany=p_utf8char/"" +if utfgmatch then + function utf.count(str,what) + if type(what)=="string" then + local n=0 + for _ in utfgmatch(str,what) do + n=n+1 + end + return n + else + return #lpegmatch(Cs((P(what)/" "+p_nany)^0),str) + end + end +else + local cache={} + function utf.count(str,what) + if type(what)=="string" then + local p=cache[what] + if not p then + p=Cs((P(what)/" "+p_nany)^0) + cache[p]=p + end + return #lpegmatch(p,str) + else + return #lpegmatch(Cs((P(what)/" "+p_nany)^0),str) + end + end +end +if not utf.characters then + function utf.characters(str) + return gmatch(str,".[\128-\191]*") + end + string.utfcharacters=utf.characters +end +if not utf.values then + local find=string.find + local dummy=function() + end + function utf.values(str) + local n=#str + if n==0 then + return dummy + elseif n==1 then + return function() return utfbyte(str) end + else + local p=1 + return function() + local b,e=find(str,".[\128-\191]*",p) + if b then + p=e+1 + return utfbyte(sub(str,b,e)) + end + end end - local str = gsub(f:read(n),"%z","") - return str + end + string.utfvalues=utf.values end --- - -if not io.i_limiter then function io.i_limiter() end end -- dummy so we can test safely -if not io.o_limiter then function io.o_limiter() end end -- dummy so we can test safely - end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['l-number'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- this module will be replaced when we have the bit library - -local tostring = tostring -local format, floor, match, rep = string.format, math.floor, string.match, string.rep -local concat, insert = table.concat, table.insert -local lpegmatch = lpeg.match - -number = number or { } -local number = number +package.loaded["l-math"] = package.loaded["l-math"] or true --- a,b,c,d,e,f = number.toset(100101) +-- original size: 915, stripped down to: 836 -function number.toset(n) - return match(tostring(n),"(.?)(.?)(.?)(.?)(.?)(.?)(.?)(.?)") +if not modules then modules={} end modules ['l-math']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local floor,sin,cos,tan=math.floor,math.sin,math.cos,math.tan +if not math.round then + function math.round(x) return floor(x+0.5) end end - -function number.toevenhex(n) - local s = format("%X",n) - if #s % 2 == 0 then - return s - else - return "0" .. s - end +if not math.div then + function math.div(n,m) return floor(n/m) end +end +if not math.mod then + function math.mod(n,m) return n%m end +end +local pipi=2*math.pi/360 +if not math.sind then + function math.sind(d) return sin(d*pipi) end + function math.cosd(d) return cos(d*pipi) end + function math.tand(d) return tan(d*pipi) end +end +if not math.odd then + function math.odd (n) return n%2~=0 end + function math.even(n) return n%2==0 end end --- the lpeg way is slower on 8 digits, but faster on 4 digits, some 7.5% --- on --- --- for i=1,1000000 do --- local a,b,c,d,e,f,g,h = number.toset(12345678) --- local a,b,c,d = number.toset(1234) --- local a,b,c = number.toset(123) --- end --- --- of course dedicated "(.)(.)(.)(.)" matches are even faster -local one = lpeg.C(1-lpeg.S(''))^1 +end -- of closure -function number.toset(n) - return lpegmatch(one,tostring(n)) -end - -function number.bits(n,zero) - local t, i = { }, (zero and 0) or 1 - while n > 0 do - local m = n % 2 - if m > 0 then - insert(t,1,i) - end - n = floor(n/2) - i = i + 1 - end - return t -end +do -- create closure to overcome 200 locals limit +package.loaded["util-str"] = package.loaded["util-str"] or true -function number.bit(p) - return 2 ^ (p - 1) -- 1-based indexing -end +-- original size: 22834, stripped down to: 12570 -function number.hasbit(x, p) -- typical call: if hasbit(x, bit(3)) then ... - return x % (p + p) >= p +if not modules then modules={} end modules ['util-str']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +utilities=utilities or {} +utilities.strings=utilities.strings or {} +local strings=utilities.strings +local format,gsub,rep,sub=string.format,string.gsub,string.rep,string.sub +local load,dump=load,string.dump +local tonumber,type,tostring=tonumber,type,tostring +local unpack,concat=table.unpack,table.concat +local P,V,C,S,R,Ct,Cs,Cp,Carg,Cc=lpeg.P,lpeg.V,lpeg.C,lpeg.S,lpeg.R,lpeg.Ct,lpeg.Cs,lpeg.Cp,lpeg.Carg,lpeg.Cc +local patterns,lpegmatch=lpeg.patterns,lpeg.match +local utfchar,utfbyte=utf.char,utf.byte +local loadstripped=_LUAVERSION<5.2 and load or function(str) + return load(dump(load(str),true)) +end +if not number then number={} end +local stripper=patterns.stripzeros +local function points(n) + return (not n or n==0) and "0pt" or lpegmatch(stripper,format("%.5fpt",n/65536)) +end +local function basepoints(n) + return (not n or n==0) and "0bp" or lpegmatch(stripper,format("%.5fbp",n*(7200/7227)/65536)) +end +number.points=points +number.basepoints=basepoints +local rubish=patterns.spaceortab^0*patterns.newline +local anyrubish=patterns.spaceortab+patterns.newline +local anything=patterns.anything +local stripped=(patterns.spaceortab^1/"")*patterns.newline +local leading=rubish^0/"" +local trailing=(anyrubish^1*patterns.endofstring)/"" +local redundant=rubish^3/"\n" +local pattern=Cs(leading*(trailing+redundant+stripped+anything)^0) +function strings.collapsecrlf(str) + return lpegmatch(pattern,str) +end +local repeaters={} +function strings.newrepeater(str,offset) + offset=offset or 0 + local s=repeaters[str] + if not s then + s={} + repeaters[str]=s + end + local t=s[offset] + if t then + return t + end + t={} + setmetatable(t,{ __index=function(t,k) + if not k then + return "" + end + local n=k+offset + local s=n>0 and rep(str,n) or "" + t[k]=s + return s + end }) + s[offset]=t + return t +end +local extra,tab,start=0,0,4,0 +local nspaces=strings.newrepeater(" ") +string.nspaces=nspaces +local pattern=Carg(1)/function(t) + extra,tab,start=0,t or 7,1 + end*Cs(( + Cp()*patterns.tab/function(position) + local current=(position-start+1)+extra + local spaces=tab-(current-1)%tab + if spaces>0 then + extra=extra+spaces-1 + return nspaces[spaces] + else + return "" + end + end+patterns.newline*Cp()/function(position) + extra,start=0,position + end+patterns.anything + )^1) +function strings.tabtospace(str,tab) + return lpegmatch(pattern,str,1,tab or 7) +end +function strings.striplong(str) + str=gsub(str,"^%s*","") + str=gsub(str,"[\n\r]+ *","\n") + return str +end +function strings.nice(str) + str=gsub(str,"[:%-+_]+"," ") + return str +end +local n=0 +local sequenced=table.sequenced +function string.autodouble(s,sep) + if s==nil then + return '""' + end + local t=type(s) + if t=="number" then + return tostring(s) + end + if t=="table" then + return ('"'..sequenced(s,sep or ",")..'"') + end + return ('"'..tostring(s)..'"') +end +function string.autosingle(s,sep) + if s==nil then + return "''" + end + local t=type(s) + if t=="number" then + return tostring(s) + end + if t=="table" then + return ("'"..sequenced(s,sep or ",").."'") + end + return ("'"..tostring(s).."'") +end +local tracedchars={} +string.tracedchars=tracedchars +strings.tracers=tracedchars +function string.tracedchar(b) + if type(b)=="number" then + return tracedchars[b] or (utfchar(b).." (U+"..format('%05X',b)..")") + else + local c=utfbyte(b) + return tracedchars[c] or (b.." (U+"..format('%05X',c)..")") + end +end +function number.signed(i) + if i>0 then + return "+",i + else + return "-",-i + end +end +local preamble=[[ +local type = type +local tostring = tostring +local tonumber = tonumber +local format = string.format +local concat = table.concat +local signed = number.signed +local points = number.points +local basepoints = number.basepoints +local utfchar = utf.char +local utfbyte = utf.byte +local lpegmatch = lpeg.match +local nspaces = string.nspaces +local tracedchar = string.tracedchar +local autosingle = string.autosingle +local autodouble = string.autodouble +local sequenced = table.sequenced +]] +local template=[[ +%s +%s +return function(%s) return %s end +]] +local arguments={ "a1" } +setmetatable(arguments,{ __index=function(t,k) + local v=t[k-1]..",a"..k + t[k]=v + return v + end +}) +local prefix_any=C((S("+- .")+R("09"))^0) +local prefix_tab=C((1-R("az","AZ","09","%%"))^0) +local format_s=function(f) + n=n+1 + if f and f~="" then + return format("format('%%%ss',a%s)",f,n) + else + return format("(a%s or '')",n) + end +end +local format_S=function(f) + n=n+1 + if f and f~="" then + return format("format('%%%ss',tostring(a%s))",f,n) + else + return format("tostring(a%s)",n) + end +end +local format_q=function() + n=n+1 + return format("(a%s and format('%%q',a%s) or '')",n,n) +end +local format_Q=function() + n=n+1 + return format("format('%%q',tostring(a%s))",n) +end +local format_i=function(f) + n=n+1 + if f and f~="" then + return format("format('%%%si',a%s)",f,n) + else + return format("a%s",n) + end +end +local format_d=format_i +local format_I=function(f) + n=n+1 + return format("format('%%s%%%si',signed(a%s))",f,n) +end +local format_f=function(f) + n=n+1 + return format("format('%%%sf',a%s)",f,n) +end +local format_g=function(f) + n=n+1 + return format("format('%%%sg',a%s)",f,n) +end +local format_G=function(f) + n=n+1 + return format("format('%%%sG',a%s)",f,n) +end +local format_e=function(f) + n=n+1 + return format("format('%%%se',a%s)",f,n) +end +local format_E=function(f) + n=n+1 + return format("format('%%%sE',a%s)",f,n) +end +local format_x=function(f) + n=n+1 + return format("format('%%%sx',a%s)",f,n) +end +local format_X=function(f) + n=n+1 + return format("format('%%%sX',a%s)",f,n) +end +local format_o=function(f) + n=n+1 + return format("format('%%%so',a%s)",f,n) +end +local format_c=function() + n=n+1 + return format("utfchar(a%s)",n) +end +local format_C=function() + n=n+1 + return format("tracedchar(a%s)",n) +end +local format_r=function(f) + n=n+1 + return format("format('%%%s.0f',a%s)",f,n) +end +local format_h=function(f) + n=n+1 + if f=="-" then + f=sub(f,2) + return format("format('%%%sx',type(a%s) == 'number' and a%s or utfbyte(a%s))",f=="" and "05" or f,n,n,n) + else + return format("format('0x%%%sx',type(a%s) == 'number' and a%s or utfbyte(a%s))",f=="" and "05" or f,n,n,n) + end +end +local format_H=function(f) + n=n+1 + if f=="-" then + f=sub(f,2) + return format("format('%%%sX',type(a%s) == 'number' and a%s or utfbyte(a%s))",f=="" and "05" or f,n,n,n) + else + return format("format('0x%%%sX',type(a%s) == 'number' and a%s or utfbyte(a%s))",f=="" and "05" or f,n,n,n) + end +end +local format_u=function(f) + n=n+1 + if f=="-" then + f=sub(f,2) + return format("format('%%%sx',type(a%s) == 'number' and a%s or utfbyte(a%s))",f=="" and "05" or f,n,n,n) + else + return format("format('u+%%%sx',type(a%s) == 'number' and a%s or utfbyte(a%s))",f=="" and "05" or f,n,n,n) + end +end +local format_U=function(f) + n=n+1 + if f=="-" then + f=sub(f,2) + return format("format('%%%sX',type(a%s) == 'number' and a%s or utfbyte(a%s))",f=="" and "05" or f,n,n,n) + else + return format("format('U+%%%sX',type(a%s) == 'number' and a%s or utfbyte(a%s))",f=="" and "05" or f,n,n,n) + end +end +local format_p=function() + n=n+1 + return format("points(a%s)",n) +end +local format_b=function() + n=n+1 + return format("basepoints(a%s)",n) +end +local format_t=function(f) + n=n+1 + if f and f~="" then + return format("concat(a%s,%q)",n,f) + else + return format("concat(a%s)",n) + end +end +local format_T=function(f) + n=n+1 + if f and f~="" then + return format("sequenced(a%s,%q)",n,f) + else + return format("sequenced(a%s)",n) + end +end +local format_l=function() + n=n+1 + return format("(a%s and 'true' or 'false')",n) +end +local format_L=function() + n=n+1 + return format("(a%s and 'TRUE' or 'FALSE')",n) +end +local format_N=function() + n=n+1 + return format("tostring(tonumber(a%s) or a%s)",n,n) +end +local format_a=function(f) + n=n+1 + if f and f~="" then + return format("autosingle(a%s,%q)",n,f) + else + return format("autosingle(a%s)",n) + end +end +local format_A=function(f) + n=n+1 + if f and f~="" then + return format("autodouble(a%s,%q)",n,f) + else + return format("autodouble(a%s)",n) + end +end +local format_w=function(f) + n=n+1 + f=tonumber(f) + if f then + return format("nspaces[%s+a%s]",f,n) + else + return format("nspaces[a%s]",n) + end +end +local format_W=function(f) + return format("nspaces[%s]",tonumber(f) or 0) +end +local format_rest=function(s) + return format("%q",s) +end +local format_extension=function(extensions,f,name) + local extension=extensions[name] or "tostring(%s)" + local f=tonumber(f) or 1 + if f==0 then + return extension + elseif f==1 then + n=n+1 + local a="a"..n + return format(extension,a,a) + elseif f<0 then + local a="a"..(n+f+1) + return format(extension,a,a) + else + local t={} + for i=1,f do + n=n+1 + t[#t+1]="a"..n + end + return format(extension,unpack(t)) + end +end +local builder=Cs { "start", + start=( + ( + P("%")/""*( + V("!") ++V("s")+V("q")+V("i")+V("d")+V("f")+V("g")+V("G")+V("e")+V("E")+V("x")+V("X")+V("o") ++V("c")+V("C")+V("S") ++V("Q") ++V("N") ++V("r")+V("h")+V("H")+V("u")+V("U")+V("p")+V("b")+V("t")+V("T")+V("l")+V("L")+V("I")+V("h") ++V("w") ++V("W") ++V("a") ++V("A") ++V("*") + )+V("*") + )*(P(-1)+Carg(1)) + )^0, + ["s"]=(prefix_any*P("s"))/format_s, + ["q"]=(prefix_any*P("q"))/format_q, + ["i"]=(prefix_any*P("i"))/format_i, + ["d"]=(prefix_any*P("d"))/format_d, + ["f"]=(prefix_any*P("f"))/format_f, + ["g"]=(prefix_any*P("g"))/format_g, + ["G"]=(prefix_any*P("G"))/format_G, + ["e"]=(prefix_any*P("e"))/format_e, + ["E"]=(prefix_any*P("E"))/format_E, + ["x"]=(prefix_any*P("x"))/format_x, + ["X"]=(prefix_any*P("X"))/format_X, + ["o"]=(prefix_any*P("o"))/format_o, + ["S"]=(prefix_any*P("S"))/format_S, + ["Q"]=(prefix_any*P("Q"))/format_S, + ["N"]=(prefix_any*P("N"))/format_N, + ["c"]=(prefix_any*P("c"))/format_c, + ["C"]=(prefix_any*P("C"))/format_C, + ["r"]=(prefix_any*P("r"))/format_r, + ["h"]=(prefix_any*P("h"))/format_h, + ["H"]=(prefix_any*P("H"))/format_H, + ["u"]=(prefix_any*P("u"))/format_u, + ["U"]=(prefix_any*P("U"))/format_U, + ["p"]=(prefix_any*P("p"))/format_p, + ["b"]=(prefix_any*P("b"))/format_b, + ["t"]=(prefix_tab*P("t"))/format_t, + ["T"]=(prefix_tab*P("T"))/format_T, + ["l"]=(prefix_tab*P("l"))/format_l, + ["L"]=(prefix_tab*P("L"))/format_L, + ["I"]=(prefix_any*P("I"))/format_I, + ["w"]=(prefix_any*P("w"))/format_w, + ["W"]=(prefix_any*P("W"))/format_W, + ["a"]=(prefix_any*P("a"))/format_a, + ["A"]=(prefix_any*P("A"))/format_A, + ["*"]=Cs(((1-P("%"))^1+P("%%")/"%%%%")^1)/format_rest, + ["!"]=Carg(2)*prefix_any*P("!")*C((1-P("!"))^1)*P("!")/format_extension, +} +local direct=Cs ( + P("%")/""*Cc([[local format = string.format return function(str) return format("%]])*(S("+- .")+R("09"))^0*S("sqidfgGeExXo")*Cc([[",str) end]])*P(-1) + ) +local function make(t,str) + local f + local p + local p=lpegmatch(direct,str) + if p then + f=loadstripped(p)() + else + n=0 + p=lpegmatch(builder,str,1,"..",t._extensions_) + if n>0 then + p=format(template,preamble,t._preamble_,arguments[n],p) + f=loadstripped(p)() + else + f=function() return str end + end + end + t[str]=f + return f end - -function number.setbit(x, p) - return hasbit(x, p) and x or x + p +local function use(t,fmt,...) + return t[fmt](...) end - -function number.clearbit(x, p) - return hasbit(x, p) and x - p or x +strings.formatters={} +function strings.formatters.new() + local t={ _extensions_={},_preamble_="",_type_="formatter" } + setmetatable(t,{ __index=make,__call=use }) + return t end - - -function number.tobitstring(n,m) - if n == 0 then - if m then - rep("00000000",m) - else - return "00000000" - end - else - local t = { } - while n > 0 do - insert(t,1,n % 2 > 0 and 1 or 0) - n = floor(n/2) - end - local nn = 8 - #t % 8 - if nn > 0 and nn < 8 then - for i=1,nn do - insert(t,1,0) - end - end - if m then - m = m * 8 - #t - if m > 0 then - insert(t,1,rep("0",m)) - end - end - return concat(t) +local formatters=strings.formatters.new() +string.formatters=formatters +string.formatter=function(str,...) return formatters[str](...) end +local function add(t,name,template,preamble) + if type(t)=="table" and t._type_=="formatter" then + t._extensions_[name]=template or "%s" + if preamble then + t._preamble_=preamble.."\n"..t._preamble_ end + end end - +strings.formatters.add=add +lpeg.patterns.xmlescape=Cs((P("<")/"<"+P(">")/">"+P("&")/"&"+P('"')/"""+P(1))^0) +lpeg.patterns.texescape=Cs((C(S("#$%\\{}"))/"\\%1"+P(1))^0) +add(formatters,"xml",[[lpegmatch(xmlescape,%s)]],[[local xmlescape = lpeg.patterns.xmlescape]]) +add(formatters,"tex",[[lpegmatch(texescape,%s)]],[[local texescape = lpeg.patterns.texescape]]) end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['l-set'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- This will become obsolete when we have the bitset library embedded. - -set = set or { } - -local nums = { } -local tabs = { } -local concat = table.concat -local next, type = next, type +package.loaded["util-tab"] = package.loaded["util-tab"] or true -set.create = table.tohash +-- original size: 14491, stripped down to: 8512 -function set.tonumber(t) - if next(t) then - local s = "" - -- we could save mem by sorting, but it slows down - for k, v in next, t do - if v then - -- why bother about the leading space - s = s .. " " .. k - end - end - local n = nums[s] - if not n then - n = #tabs + 1 - tabs[n] = t - nums[s] = n - end - return n +if not modules then modules={} end modules ['util-tab']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +utilities=utilities or {} +utilities.tables=utilities.tables or {} +local tables=utilities.tables +local format,gmatch,gsub=string.format,string.gmatch,string.gsub +local concat,insert,remove=table.concat,table.insert,table.remove +local setmetatable,getmetatable,tonumber,tostring=setmetatable,getmetatable,tonumber,tostring +local type,next,rawset,tonumber,tostring,load,select=type,next,rawset,tonumber,tostring,load,select +local lpegmatch,P,Cs,Cc=lpeg.match,lpeg.P,lpeg.Cs,lpeg.Cc +local serialize,sortedkeys,sortedpairs=table.serialize,table.sortedkeys,table.sortedpairs +local formatters=string.formatters +local splitter=lpeg.tsplitat(".") +function tables.definetable(target,nofirst,nolast) + local composed,shortcut,t=nil,nil,{} + local snippets=lpegmatch(splitter,target) + for i=1,#snippets-(nolast and 1 or 0) do + local name=snippets[i] + if composed then + composed=shortcut.."."..name + shortcut=shortcut.."_"..name + t[#t+1]=formatters["local %s = %s if not %s then %s = { } %s = %s end"](shortcut,composed,shortcut,shortcut,composed,shortcut) else - return 0 + composed=name + shortcut=name + if not nofirst then + t[#t+1]=formatters["%s = %s or { }"](composed,composed) + end + end + end + if nolast then + composed=shortcut.."."..snippets[#snippets] + end + return concat(t,"\n"),composed +end +function tables.definedtable(...) + local t=_G + for i=1,select("#",...) do + local li=select(i,...) + local tl=t[li] + if not tl then + tl={} + t[li]=tl + end + t=tl + end + return t +end +function tables.accesstable(target,root) + local t=root or _G + for name in gmatch(target,"([^%.]+)") do + t=t[name] + if not t then + return end + end + return t end - -function set.totable(n) - if n == 0 then - return { } - else - return tabs[n] or { } +function tables.migratetable(target,v,root) + local t=root or _G + local names=string.split(target,".") + for i=1,#names-1 do + local name=names[i] + t[name]=t[name] or {} + t=t[name] + if not t then + return end + end + t[names[#names]]=v end - -function set.tolist(n) - if n == 0 or not tabs[n] then - return "" - else - local t, n = { }, 0 - for k, v in next, tabs[n] do - if v then - n = n + 1 - t[n] = k - end +function tables.removevalue(t,value) + if value then + for i=1,#t do + if t[i]==value then + remove(t,i) + end + end + end +end +function tables.insertbeforevalue(t,value,extra) + for i=1,#t do + if t[i]==extra then + remove(t,i) + end + end + for i=1,#t do + if t[i]==value then + insert(t,i,extra) + return + end + end + insert(t,1,extra) +end +function tables.insertaftervalue(t,value,extra) + for i=1,#t do + if t[i]==extra then + remove(t,i) + end + end + for i=1,#t do + if t[i]==value then + insert(t,i+1,extra) + return + end + end + insert(t,#t+1,extra) +end +local escape=Cs(Cc('"')*((P('"')/'""'+P(1))^0)*Cc('"')) +function table.tocsv(t,specification) + if t and #t>0 then + local result={} + local r={} + specification=specification or {} + local fields=specification.fields + if type(fields)~="string" then + fields=sortedkeys(t[1]) + end + local separator=specification.separator or "," + if specification.preamble==true then + for f=1,#fields do + r[f]=lpegmatch(escape,tostring(fields[f])) + end + result[1]=concat(r,separator) + end + for i=1,#t do + local ti=t[i] + for f=1,#fields do + local field=ti[fields[f]] + if type(field)=="string" then + r[f]=lpegmatch(escape,field) + else + r[f]=tostring(field) end - return concat(t," ") + end + result[#result+1]=concat(r,separator) end + return concat(result,"\n") + else + return "" + end end - -function set.contains(n,s) - if type(n) == "table" then - return n[s] - elseif n == 0 then - return false +local nspaces=utilities.strings.newrepeater(" ") +local function toxml(t,d,result,step) + for k,v in sortedpairs(t) do + local s=nspaces[d] + local tk=type(k) + local tv=type(v) + if tv=="table" then + if tk=="number" then + result[#result+1]=formatters["%s"](s,k) + toxml(v,d+step,result,step) + result[#result+1]=formatters["%s"](s,k) + else + result[#result+1]=formatters["%s<%s>"](s,k) + toxml(v,d+step,result,step) + result[#result+1]=formatters["%s"](s,k) + end + elseif tv=="string" then + if tk=="number" then + result[#result+1]=formatters["%s%!xml!"](s,k,v,k) + else + result[#result+1]=formatters["%s<%s>%!xml!"](s,k,v,k) + end + elseif tk=="number" then + result[#result+1]=formatters["%s%S"](s,k,v,k) + else + result[#result+1]=formatters["%s<%s>%S"](s,k,v,k) + end + end +end +function table.toxml(t,specification) + specification=specification or {} + local name=specification.name + local noroot=name==false + local result=(specification.nobanner or noroot) and {} or { "" } + local indent=specification.indent or 0 + local spaces=specification.spaces or 1 + if noroot then + toxml(t,indent,result,spaces) + else + toxml({ [name or "data"]=t },indent,result,spaces) + end + return concat(result,"\n") +end +function tables.encapsulate(core,capsule,protect) + if type(capsule)~="table" then + protect=true + capsule={} + end + for key,value in next,core do + if capsule[key] then + print(formatters["\ninvalid %s %a in %a"]("inheritance",key,core)) + os.exit() else - local t = tabs[n] - return t and t[s] + capsule[key]=value end + end + if protect then + for key,value in next,core do + core[key]=nil + end + setmetatable(core,{ + __index=capsule, + __newindex=function(t,key,value) + if capsule[key] then + print(formatters["\ninvalid %s %a' in %a"]("overload",key,core)) + os.exit() + else + rawset(t,key,value) + end + end + } ) + end +end +local function fastserialize(t,r,outer) + r[#r+1]="{" + local n=#t + if n>0 then + for i=1,n do + local v=t[i] + local tv=type(v) + if tv=="string" then + r[#r+1]=formatters["%q,"](v) + elseif tv=="number" then + r[#r+1]=formatters["%s,"](v) + elseif tv=="table" then + fastserialize(v,r) + elseif tv=="boolean" then + r[#r+1]=formatters["%S,"](v) + end + end + else + for k,v in next,t do + local tv=type(v) + if tv=="string" then + r[#r+1]=formatters["[%q]=%q,"](k,v) + elseif tv=="number" then + r[#r+1]=formatters["[%q]=%s,"](k,v) + elseif tv=="table" then + r[#r+1]=formatters["[%q]="](k) + fastserialize(v,r) + elseif tv=="boolean" then + r[#r+1]=formatters["[%q]=%S,"](k,v) + end + end + end + if outer then + r[#r+1]="}" + else + r[#r+1]="}," + end + return r +end +function table.fastserialize(t,prefix) + return concat(fastserialize(t,{ prefix or "return" },true)) +end +function table.deserialize(str) + if not str or str=="" then + return + end + local code=load(str) + if not code then + return + end + code=code() + if not code then + return + end + return code +end +function table.load(filename) + if filename then + local t=io.loaddata(filename) + if t and t~="" then + t=load(t) + if type(t)=="function" then + t=t() + if type(t)=="table" then + return t + end + end + end + end +end +function table.save(filename,t,n,...) + io.savedata(filename,serialize(t,n==nil and true or n,...)) +end +local function slowdrop(t) + local r={} + local l={} + for i=1,#t do + local ti=t[i] + local j=0 + for k,v in next,ti do + j=j+1 + l[j]=formatters["%s=%q"](k,v) + end + r[i]=formatters[" {%t},\n"](l) + end + return formatters["return {\n%st}"](r) +end +local function fastdrop(t) + local r={ "return {\n" } + for i=1,#t do + local ti=t[i] + r[#r+1]=" {" + for k,v in next,ti do + r[#r+1]=formatters["%s=%q"](k,v) + end + r[#r+1]="},\n" + end + r[#r+1]="}" + return concat(r) +end +function table.drop(t,slow) + if #t==0 then + return "return { }" + elseif slow==true then + return slowdrop(t) + else + return fastdrop(t) + end +end +function table.autokey(t,k) + local v={} + t[k]=v + return v +end +local selfmapper={ __index=function(t,k) t[k]=k return k end } +function table.twowaymapper(t) + if not t then + t={} + else + for i=0,#t do + local ti=t[i] + if ti then + local i=tostring(i) + t[i]=ti + t[ti]=i + end + end + t[""]=t[0] or "" + end + setmetatable(t,selfmapper) + return t end - - end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['l-os'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- This file deals with some operating system issues. Please don't bother me --- with the pros and cons of operating systems as they all have their flaws --- and benefits. Bashing one of them won't help solving problems and fixing --- bugs faster and is a waste of time and energy. --- --- path separators: / or \ ... we can use / everywhere --- suffixes : dll so exe ... no big deal --- quotes : we can use "" in most cases --- expansion : unless "" are used * might give side effects --- piping/threads : somewhat different for each os --- locations : specific user file locations and settings can change over time --- --- os.type : windows | unix (new, we already guessed os.platform) --- os.name : windows | msdos | linux | macosx | solaris | .. | generic (new) --- os.platform : extended os.name with architecture - --- maybe build io.flush in os.execute +package.loaded["util-sto"] = package.loaded["util-sto"] or true -local os = os -local find, format, gsub, upper, gmatch = string.find, string.format, string.gsub, string.upper, string.gmatch -local concat = table.concat -local random, ceil = math.random, math.ceil -local rawget, rawset, type, getmetatable, setmetatable, tonumber = rawget, rawset, type, getmetatable, setmetatable, tonumber - --- The following code permits traversing the environment table, at least --- in luatex. Internally all environment names are uppercase. +-- original size: 4432, stripped down to: 3123 -if not os.__getenv__ then +if not modules then modules={} end modules ['util-sto']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local setmetatable,getmetatable,type=setmetatable,getmetatable,type +utilities=utilities or {} +utilities.storage=utilities.storage or {} +local storage=utilities.storage +function storage.mark(t) + if not t then + print("\nfatal error: storage cannot be marked\n") + os.exit() + return + end + local m=getmetatable(t) + if not m then + m={} + setmetatable(t,m) + end + m.__storage__=true + return t +end +function storage.allocate(t) + t=t or {} + local m=getmetatable(t) + if not m then + m={} + setmetatable(t,m) + end + m.__storage__=true + return t +end +function storage.marked(t) + local m=getmetatable(t) + return m and m.__storage__ +end +function storage.checked(t) + if not t then + report("\nfatal error: storage has not been allocated\n") + os.exit() + return + end + return t +end +function storage.setinitializer(data,initialize) + local m=getmetatable(data) or {} + m.__index=function(data,k) + m.__index=nil + initialize() + return data[k] + end + setmetatable(data,m) +end +local keyisvalue={ __index=function(t,k) + t[k]=k + return k +end } +function storage.sparse(t) + t=t or {} + setmetatable(t,keyisvalue) + return t +end +local function f_empty () return "" end +local function f_self (t,k) t[k]=k return k end +local function f_table (t,k) local v={} t[k]=v return v end +local function f_ignore() end +local t_empty={ __index=f_empty } +local t_self={ __index=f_self } +local t_table={ __index=f_table } +local t_ignore={ __newindex=f_ignore } +function table.setmetatableindex(t,f) + if type(t)~="table" then + f,t=t,{} + end + local m=getmetatable(t) + if m then + if f=="empty" then + m.__index=f_empty + elseif f=="key" then + m.__index=f_self + elseif f=="table" then + m.__index=f_table + else + m.__index=f + end + else + if f=="empty" then + setmetatable(t,t_empty) + elseif f=="key" then + setmetatable(t,t_self) + elseif f=="table" then + setmetatable(t,t_table) + else + setmetatable(t,{ __index=f }) + end + end + return t +end +function table.setmetatablenewindex(t,f) + if type(t)~="table" then + f,t=t,{} + end + local m=getmetatable(t) + if m then + if f=="ignore" then + m.__newindex=f_ignore + else + m.__newindex=f + end + else + if f=="ignore" then + setmetatable(t,t_ignore) + else + setmetatable(t,{ __newindex=f }) + end + end + return t +end +function table.setmetatablecall(t,f) + if type(t)~="table" then + f,t=t,{} + end + local m=getmetatable(t) + if m then + m.__call=f + else + setmetatable(t,{ __call=f }) + end + return t +end +function table.setmetatablekey(t,key,value) + local m=getmetatable(t) + if not m then + m={} + setmetatable(t,m) + end + m[key]=value + return t +end +function table.getmetatablekey(t,key,value) + local m=getmetatable(t) + return m and m[key] +end - os.__getenv__ = os.getenv - os.__setenv__ = os.setenv - if os.env then +end -- of closure - local osgetenv = os.getenv - local ossetenv = os.setenv - local osenv = os.env local _ = osenv.PATH -- initialize the table +do -- create closure to overcome 200 locals limit - function os.setenv(k,v) - if v == nil then - v = "" - end - local K = upper(k) - osenv[K] = v - if type(v) == "table" then - v = concat(v,";") -- path - end - ossetenv(K,v) - end +package.loaded["util-prs"] = package.loaded["util-prs"] or true - function os.getenv(k) - local K = upper(k) - local v = osenv[K] or osenv[k] or osgetenv(K) or osgetenv(k) - if v == "" then - return nil - else - return v - end - end +-- original size: 16976, stripped down to: 12143 +if not modules then modules={} end modules ['util-prs']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local lpeg,table,string=lpeg,table,string +local P,R,V,S,C,Ct,Cs,Carg,Cc,Cg,Cf,Cp=lpeg.P,lpeg.R,lpeg.V,lpeg.S,lpeg.C,lpeg.Ct,lpeg.Cs,lpeg.Carg,lpeg.Cc,lpeg.Cg,lpeg.Cf,lpeg.Cp +local lpegmatch,lpegpatterns=lpeg.match,lpeg.patterns +local concat,format,gmatch,find=table.concat,string.format,string.gmatch,string.find +local tostring,type,next,rawset=tostring,type,next,rawset +utilities=utilities or {} +local parsers=utilities.parsers or {} +utilities.parsers=parsers +local patterns=parsers.patterns or {} +parsers.patterns=patterns +local setmetatableindex=table.setmetatableindex +local sortedhash=table.sortedhash +local digit=R("09") +local space=P(' ') +local equal=P("=") +local comma=P(",") +local lbrace=P("{") +local rbrace=P("}") +local lparent=P("(") +local rparent=P(")") +local period=S(".") +local punctuation=S(".,:;") +local spacer=lpegpatterns.spacer +local whitespace=lpegpatterns.whitespace +local newline=lpegpatterns.newline +local anything=lpegpatterns.anything +local endofstring=lpegpatterns.endofstring +local nobrace=1-(lbrace+rbrace ) +local noparent=1-(lparent+rparent) +local escape,left,right=P("\\"),P('{'),P('}') +lpegpatterns.balanced=P { + [1]=((escape*(left+right))+(1-(left+right))+V(2))^0, + [2]=left*V(1)*right +} +local nestedbraces=P { lbrace*(nobrace+V(1))^0*rbrace } +local nestedparents=P { lparent*(noparent+V(1))^0*rparent } +local spaces=space^0 +local argument=Cs((lbrace/"")*((nobrace+nestedbraces)^0)*(rbrace/"")) +local content=(1-endofstring)^0 +lpegpatterns.nestedbraces=nestedbraces +lpegpatterns.nestedparents=nestedparents +lpegpatterns.nested=nestedbraces +lpegpatterns.argument=argument +lpegpatterns.content=content +local value=P(lbrace*C((nobrace+nestedbraces)^0)*rbrace)+C((nestedbraces+(1-comma))^0) +local key=C((1-equal-comma)^1) +local pattern_a=(space+comma)^0*(key*equal*value+key*C("")) +local pattern_c=(space+comma)^0*(key*equal*value) +local key=C((1-space-equal-comma)^1) +local pattern_b=spaces*comma^0*spaces*(key*((spaces*equal*spaces*value)+C(""))) +local hash={} +local function set(key,value) + hash[key]=value +end +local pattern_a_s=(pattern_a/set)^1 +local pattern_b_s=(pattern_b/set)^1 +local pattern_c_s=(pattern_c/set)^1 +patterns.settings_to_hash_a=pattern_a_s +patterns.settings_to_hash_b=pattern_b_s +patterns.settings_to_hash_c=pattern_c_s +function parsers.make_settings_to_hash_pattern(set,how) + if how=="strict" then + return (pattern_c/set)^1 + elseif how=="tolerant" then + return (pattern_b/set)^1 + else + return (pattern_a/set)^1 + end +end +function parsers.settings_to_hash(str,existing) + if str and str~="" then + hash=existing or {} + lpegmatch(pattern_a_s,str) + return hash + else + return {} + end +end +function parsers.settings_to_hash_tolerant(str,existing) + if str and str~="" then + hash=existing or {} + lpegmatch(pattern_b_s,str) + return hash + else + return {} + end +end +function parsers.settings_to_hash_strict(str,existing) + if str and str~="" then + hash=existing or {} + lpegmatch(pattern_c_s,str) + return next(hash) and hash + else + return nil + end +end +local separator=comma*space^0 +local value=P(lbrace*C((nobrace+nestedbraces)^0)*rbrace)+C((nestedbraces+(1-comma))^0) +local pattern=spaces*Ct(value*(separator*value)^0) +patterns.settings_to_array=pattern +function parsers.settings_to_array(str,strict) + if not str or str=="" then + return {} + elseif strict then + if find(str,"{") then + return lpegmatch(pattern,str) else - - local ossetenv = os.setenv - local osgetenv = os.getenv - local osenv = { } - - function os.setenv(k,v) - if v == nil then - v = "" - end - local K = upper(k) - osenv[K] = v - end - - function os.getenv(k) - local K = upper(k) - local v = osenv[K] or osgetenv(K) or osgetenv(k) - if v == "" then - return nil - else - return v - end - end - - local function __index(t,k) - return os.getenv(k) - end - local function __newindex(t,k,v) - os.setenv(k,v) + return { str } + end + else + return lpegmatch(pattern,str) + end +end +local function set(t,v) + t[#t+1]=v +end +local value=P(Carg(1)*value)/set +local pattern=value*(separator*value)^0*Carg(1) +function parsers.add_settings_to_array(t,str) + return lpegmatch(pattern,str,nil,t) +end +function parsers.hash_to_string(h,separator,yes,no,strict,omit) + if h then + local t,tn,s={},0,table.sortedkeys(h) + omit=omit and table.tohash(omit) + for i=1,#s do + local key=s[i] + if not omit or not omit[key] then + local value=h[key] + if type(value)=="boolean" then + if yes and no then + if value then + tn=tn+1 + t[tn]=key..'='..yes + elseif not strict then + tn=tn+1 + t[tn]=key..'='..no + end + elseif value or not strict then + tn=tn+1 + t[tn]=key..'='..tostring(value) + end + else + tn=tn+1 + t[tn]=key..'='..value end - - os.env = { } - - setmetatable(os.env, { __index = __index, __newindex = __newindex } ) - + end end - + return concat(t,separator or ",") + else + return "" + end end - --- end of environment hack - -local execute, spawn, exec, iopopen, ioflush = os.execute, os.spawn or os.execute, os.exec or os.execute, io.popen, io.flush - -function os.execute(...) ioflush() return execute(...) end -function os.spawn (...) ioflush() return spawn (...) end -function os.exec (...) ioflush() return exec (...) end -function io.popen (...) ioflush() return iopopen(...) end - -function os.resultof(command) - local handle = io.popen(command,"r") - return handle and handle:read("*all") or "" +function parsers.array_to_string(a,separator) + if a then + return concat(a,separator or ",") + else + return "" + end +end +function parsers.settings_to_set(str,t) + t=t or {} + for s in gmatch(str,"[^, ]+") do + t[s]=true + end + return t +end +function parsers.simple_hash_to_string(h,separator) + local t,tn={},0 + for k,v in sortedhash(h) do + if v then + tn=tn+1 + t[tn]=k + end + end + return concat(t,separator or ",") end - -if not io.fileseparator then - if find(os.getenv("PATH"),";") then - io.fileseparator, io.pathseparator, os.type = "\\", ";", os.type or "mswin" +local value=P(lbrace*C((nobrace+nestedbraces)^0)*rbrace)+C(digit^1*lparent*(noparent+nestedparents)^1*rparent)+C((nestedbraces+(1-comma))^1) +local pattern_a=spaces*Ct(value*(separator*value)^0) +local function repeater(n,str) + if not n then + return str + else + local s=lpegmatch(pattern_a,str) + if n==1 then + return unpack(s) else - io.fileseparator, io.pathseparator, os.type = "/" , ":", os.type or "unix" - end + local t,tn={},0 + for i=1,n do + for j=1,#s do + tn=tn+1 + t[tn]=s[j] + end + end + return unpack(t) + end + end +end +local value=P(lbrace*C((nobrace+nestedbraces)^0)*rbrace)+(C(digit^1)/tonumber*lparent*Cs((noparent+nestedparents)^1)*rparent)/repeater+C((nestedbraces+(1-comma))^1) +local pattern_b=spaces*Ct(value*(separator*value)^0) +function parsers.settings_to_array_with_repeat(str,expand) + if expand then + return lpegmatch(pattern_b,str) or {} + else + return lpegmatch(pattern_a,str) or {} + end +end +local value=lbrace*C((nobrace+nestedbraces)^0)*rbrace +local pattern=Ct((space+value)^0) +function parsers.arguments_to_table(str) + return lpegmatch(pattern,str) end - -os.type = os.type or (io.pathseparator == ";" and "windows") or "unix" -os.name = os.name or (os.type == "windows" and "mswin" ) or "linux" - -if os.type == "windows" then - os.libsuffix, os.binsuffix, os.binsuffixes = 'dll', 'exe', { 'exe', 'cmd', 'bat' } -else - os.libsuffix, os.binsuffix, os.binsuffixes = 'so', '', { '' } +function parsers.getparameters(self,class,parentclass,settings) + local sc=self[class] + if not sc then + sc={} + self[class]=sc + if parentclass then + local sp=self[parentclass] + if not sp then + sp={} + self[parentclass]=sp + end + setmetatableindex(sc,sp) + end + end + parsers.settings_to_hash(settings,sc) end - -function os.launch(str) - if os.type == "windows" then - os.execute("start " .. str) -- os.spawn ? +function parsers.listitem(str) + return gmatch(str,"[^, ]+") +end +local pattern=Cs { "start", + start=V("one")+V("two")+V("three"), + rest=(Cc(",")*V("thousand"))^0*(P(".")+endofstring)*anything^0, + thousand=digit*digit*digit, + one=digit*V("rest"), + two=digit*digit*V("rest"), + three=V("thousand")*V("rest"), +} +lpegpatterns.splitthousands=pattern +function parsers.splitthousands(str) + return lpegmatch(pattern,str) or str +end +local optionalwhitespace=whitespace^0 +lpegpatterns.words=Ct((Cs((1-punctuation-whitespace)^1)+anything)^1) +lpegpatterns.sentences=Ct((optionalwhitespace*Cs((1-period)^0*period))^1) +lpegpatterns.paragraphs=Ct((optionalwhitespace*Cs((whitespace^1*endofstring/""+1-(spacer^0*newline*newline))^1))^1) +local dquote=P('"') +local equal=P('=') +local escape=P('\\') +local separator=S(' ,') +local key=C((1-equal)^1) +local value=dquote*C((1-dquote-escape*dquote)^0)*dquote +local pattern=Cf(Ct("")*Cg(key*equal*value)*separator^0,rawset)^0*P(-1) +patterns.keq_to_hash_c=pattern +function parsers.keq_to_hash(str) + if str and str~="" then + return lpegmatch(pattern,str) + else + return {} + end +end +local defaultspecification={ separator=",",quote='"' } +function parsers.csvsplitter(specification) + specification=specification and table.setmetatableindex(specification,defaultspecification) or defaultspecification + local separator=specification.separator + local quotechar=specification.quote + local separator=S(separator~="" and separator or ",") + local whatever=C((1-separator-newline)^0) + if quotechar and quotechar~="" then + local quotedata=nil + for chr in gmatch(quotechar,".") do + local quotechar=P(chr) + local quoteword=quotechar*C((1-quotechar)^0)*quotechar + if quotedata then + quotedata=quotedata+quoteword + else + quotedata=quoteword + end + end + whatever=quotedata+whatever + end + local parser=Ct((Ct(whatever*(separator*whatever)^0)*S("\n\r"))^0 ) + return function(data) + return lpegmatch(parser,data) + end +end +function parsers.rfc4180splitter(specification) + specification=specification and table.setmetatableindex(specification,defaultspecification) or defaultspecification + local separator=specification.separator + local quotechar=P(specification.quote) + local dquotechar=quotechar*quotechar +/specification.quote + local separator=S(separator~="" and separator or ",") + local escaped=quotechar*Cs((dquotechar+(1-quotechar))^0)*quotechar + local non_escaped=C((1-quotechar-newline-separator)^1) + local field=escaped+non_escaped + local record=Ct((field*separator^-1)^1) + local headerline=record*Cp() + local wholeblob=Ct((newline^-1*record)^0) + return function(data,getheader) + if getheader then + local header,position=lpegmatch(headerline,data) + local data=lpegmatch(wholeblob,data,position) + return data,header else - os.execute(str .. " &") -- os.spawn ? + return lpegmatch(wholeblob,data) + end + end +end +local function ranger(first,last,n,action) + if not first then + elseif last==true then + for i=first,n or first do + action(i) + end + elseif last then + for i=first,last do + action(i) + end + else + action(first) + end +end +local cardinal=lpegpatterns.cardinal/tonumber +local spacers=lpegpatterns.spacer^0 +local endofstring=lpegpatterns.endofstring +local stepper=spacers*(C(cardinal)*(spacers*S(":-")*spacers*(C(cardinal)+Cc(true) )+Cc(false) )*Carg(1)*Carg(2)/ranger*S(", ")^0 )^1 +local stepper=spacers*(C(cardinal)*(spacers*S(":-")*spacers*(C(cardinal)+(P("*")+endofstring)*Cc(true) )+Cc(false) )*Carg(1)*Carg(2)/ranger*S(", ")^0 )^1*endofstring +function parsers.stepper(str,n,action) + if type(n)=="function" then + lpegmatch(stepper,str,1,false,n or print) + else + lpegmatch(stepper,str,1,n,action or print) + end +end +local pattern_math=Cs((P("%")/"\\percent "+P("^")*Cc("{")*lpegpatterns.integer*Cc("}")+P(1))^0) +local pattern_text=Cs((P("%")/"\\percent "+(P("^")/"\\high")*Cc("{")*lpegpatterns.integer*Cc("}")+P(1))^0) +patterns.unittotex=pattern +function parsers.unittotex(str,textmode) + return lpegmatch(textmode and pattern_text or pattern_math,str) +end +local pattern=Cs((P("^")/""*lpegpatterns.integer*Cc("")+P(1))^0) +function parsers.unittoxml(str) + return lpegmatch(pattern,str) +end +local cache={} +local spaces=lpeg.patterns.space^0 +local dummy=function() end +table.setmetatableindex(cache,function(t,k) + local separator=P(k) + local value=(1-separator)^0 + local pattern=spaces*C(value)*separator^0*Cp() + t[k]=pattern + return pattern +end) +local commalistiterator=cache[","] +function utilities.parsers.iterator(str,separator) + local n=#str + if n==0 then + return dummy + else + local pattern=separator and cache[separator] or commalistiterator + local p=1 + return function() + if p<=n then + local s,e=lpegmatch(pattern,str,p) + if e then + p=e + return s + end + end end + end end - -if not os.times then - -- utime = user time - -- stime = system time - -- cutime = children user time - -- cstime = children system time - function os.times() - return { - utime = os.gettimeofday(), -- user - stime = 0, -- system - cutime = 0, -- children user - cstime = 0, -- children system - } +local function initialize(t,name) + local source=t[name] + if source then + local result={} + for k,v in next,t[name] do + result[k]=v end + return result + else + return {} + end end - -os.gettimeofday = os.gettimeofday or os.clock - -local startuptime = os.gettimeofday() - -function os.runtime() - return os.gettimeofday() - startuptime +local function fetch(t,name) + return t[name] or {} +end +function process(result,more) + for k,v in next,more do + result[k]=v + end + return result +end +local name=C((1-S(", "))^1) +local parser=(Carg(1)*name/initialize)*(S(", ")^1*(Carg(1)*name/fetch))^0 +local merge=Cf(parser,process) +function utilities.parsers.mergehashes(hash,list) + return lpegmatch(merge,list,1,hash) end --- no need for function anymore as we have more clever code and helpers now --- this metatable trickery might as well disappear +end -- of closure -os.resolvers = os.resolvers or { } -- will become private +do -- create closure to overcome 200 locals limit -local resolvers = os.resolvers +package.loaded["util-fmt"] = package.loaded["util-fmt"] or true -local osmt = getmetatable(os) or { __index = function(t,k) t[k] = "unset" return "unset" end } -- maybe nil -local osix = osmt.__index +-- original size: 2274, stripped down to: 1781 -osmt.__index = function(t,k) - return (resolvers[k] or osix)(t,k) +if not modules then modules={} end modules ['util-fmt']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +utilities=utilities or {} +utilities.formatters=utilities.formatters or {} +local formatters=utilities.formatters +local concat,format=table.concat,string.format +local tostring,type=tostring,type +local strip=string.strip +local lpegmatch=lpeg.match +local stripper=lpeg.patterns.stripzeros +function formatters.stripzeros(str) + return lpegmatch(stripper,str) end - -setmetatable(os,osmt) - --- we can use HOSTTYPE on some platforms - -local name, platform = os.name or "linux", os.getenv("MTX_PLATFORM") or "" - -local function guess() - local architecture = os.resultof("uname -m") or "" - if architecture ~= "" then - return architecture +function formatters.formatcolumns(result,between) + if result and #result>0 then + between=between or " " + local widths,numbers={},{} + local first=result[1] + local n=#first + for i=1,n do + widths[i]=0 + end + for i=1,#result do + local r=result[i] + for j=1,n do + local rj=r[j] + local tj=type(rj) + if tj=="number" then + numbers[j]=true + end + if tj~="string" then + rj=tostring(rj) + r[j]=rj + end + local w=#rj + if w>widths[j] then + widths[j]=w + end + end + end + for i=1,n do + local w=widths[i] + if numbers[i] then + if w>80 then + widths[i]="%s"..between + else + widths[i]="%0"..w.."i"..between + end + else + if w>80 then + widths[i]="%s"..between + elseif w>0 then + widths[i]="%-"..w.."s"..between + else + widths[i]="%s" + end + end end - architecture = os.getenv("HOSTTYPE") or "" - if architecture ~= "" then - return architecture + local template=strip(concat(widths)) + for i=1,#result do + local str=format(template,unpack(result[i])) + result[i]=strip(str) end - return os.resultof("echo $HOSTTYPE") or "" + end + return result end -if platform ~= "" then - - os.platform = platform -elseif os.type == "windows" then +end -- of closure - -- we could set the variable directly, no function needed here +do -- create closure to overcome 200 locals limit - function os.resolvers.platform(t,k) - local platform, architecture = "", os.getenv("PROCESSOR_ARCHITECTURE") or "" - if find(architecture,"AMD64") then - platform = "mswin-64" - else - platform = "mswin" - end - os.setenv("MTX_PLATFORM",platform) - os.platform = platform - return platform - end +package.loaded["trac-set"] = package.loaded["trac-set"] or true -elseif name == "linux" then +-- original size: 12365, stripped down to: 8799 - function os.resolvers.platform(t,k) - -- we sometimes have HOSTTYPE set so let's check that first - local platform, architecture = "", os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" - if find(architecture,"x86_64") then - platform = "linux-64" - elseif find(architecture,"ppc") then - platform = "linux-ppc" - else - platform = "linux" - end - os.setenv("MTX_PLATFORM",platform) - os.platform = platform - return platform - end - -elseif name == "macosx" then - - --[[ - Identifying the architecture of OSX is quite a mess and this - is the best we can come up with. For some reason $HOSTTYPE is - a kind of pseudo environment variable, not known to the current - environment. And yes, uname cannot be trusted either, so there - is a change that you end up with a 32 bit run on a 64 bit system. - Also, some proper 64 bit intel macs are too cheap (low-end) and - therefore not permitted to run the 64 bit kernel. - ]]-- - - function os.resolvers.platform(t,k) - -- local platform, architecture = "", os.getenv("HOSTTYPE") or "" - -- if architecture == "" then - -- architecture = os.resultof("echo $HOSTTYPE") or "" - -- end - local platform, architecture = "", os.resultof("echo $HOSTTYPE") or "" - if architecture == "" then - -- print("\nI have no clue what kind of OSX you're running so let's assume an 32 bit intel.\n") - platform = "osx-intel" - elseif find(architecture,"i386") then - platform = "osx-intel" - elseif find(architecture,"x86_64") then - platform = "osx-64" +if not modules then modules={} end modules ['trac-set']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local type,next,tostring=type,next,tostring +local concat=table.concat +local format,find,lower,gsub,topattern=string.format,string.find,string.lower,string.gsub,string.topattern +local is_boolean=string.is_boolean +local settings_to_hash=utilities.parsers.settings_to_hash +local allocate=utilities.storage.allocate +utilities=utilities or {} +local utilities=utilities +local setters=utilities.setters or {} +utilities.setters=setters +local data={} +local trace_initialize=false +function setters.initialize(filename,name,values) + local setter=data[name] + if setter then + frozen=true + local data=setter.data + if data then + for key,newvalue in next,values do + local newvalue=is_boolean(newvalue,newvalue) + local functions=data[key] + if functions then + local oldvalue=functions.value + if functions.frozen then + if trace_initialize then + setter.report("%s: %a is %s to %a",filename,key,"frozen",oldvalue) + end + elseif #functions>0 and not oldvalue then + if trace_initialize then + setter.report("%s: %a is %s to %a",filename,key,"set",newvalue) + end + for i=1,#functions do + functions[i](newvalue) + end + functions.value=newvalue + functions.frozen=functions.frozen or frozen + else + if trace_initialize then + setter.report("%s: %a is %s as %a",filename,key,"kept",oldvalue) + end + end else - platform = "osx-ppc" + functions={ default=newvalue,frozen=frozen } + data[key]=functions + if trace_initialize then + setter.report("%s: %a is %s to %a",filename,key,"defaulted",newvalue) + end end - os.setenv("MTX_PLATFORM",platform) - os.platform = platform - return platform + end + return true end - -elseif name == "sunos" then - - function os.resolvers.platform(t,k) - local platform, architecture = "", os.resultof("uname -m") or "" - if find(architecture,"sparc") then - platform = "solaris-sparc" - else -- if architecture == 'i86pc' - platform = "solaris-intel" - end - os.setenv("MTX_PLATFORM",platform) - os.platform = platform - return platform + end +end +local function set(t,what,newvalue) + local data=t.data + if not data.frozen then + local done=t.done + if type(what)=="string" then + what=settings_to_hash(what) + end + if type(what)~="table" then + return + end + if not done then + done={} + t.done=done + end + for w,value in next,what do + if value=="" then + value=newvalue + elseif not value then + value=false + else + value=is_boolean(value,value) + end + w=topattern(w,true,true) + for name,functions in next,data do + if done[name] then + elseif find(name,w) then + done[name]=true + for i=1,#functions do + functions[i](value) + end + functions.value=value + end + end + end + end +end +local function reset(t) + local data=t.data + if not data.frozen then + for name,functions in next,data do + for i=1,#functions do + functions[i](false) + end + functions.value=false end - -elseif name == "freebsd" then - - function os.resolvers.platform(t,k) - local platform, architecture = "", os.resultof("uname -m") or "" - if find(architecture,"amd64") then - platform = "freebsd-amd64" - else - platform = "freebsd" - end - os.setenv("MTX_PLATFORM",platform) - os.platform = platform - return platform - end - -elseif name == "kfreebsd" then - - function os.resolvers.platform(t,k) - -- we sometimes have HOSTTYPE set so let's check that first - local platform, architecture = "", os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" - if find(architecture,"x86_64") then - platform = "kfreebsd-amd64" - else - platform = "kfreebsd-i386" - end - os.setenv("MTX_PLATFORM",platform) - os.platform = platform - return platform - end - -else - - -- platform = "linux" - -- os.setenv("MTX_PLATFORM",platform) - -- os.platform = platform - - function os.resolvers.platform(t,k) - local platform = "linux" - os.setenv("MTX_PLATFORM",platform) - os.platform = platform - return platform - end - + end end - --- beware, we set the randomseed - --- from wikipedia: Version 4 UUIDs use a scheme relying only on random numbers. This algorithm sets the --- version number as well as two reserved bits. All other bits are set using a random or pseudorandom --- data source. Version 4 UUIDs have the form xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx with hexadecimal --- digits x and hexadecimal digits 8, 9, A, or B for y. e.g. f47ac10b-58cc-4372-a567-0e02b2c3d479. --- --- as we don't call this function too often there is not so much risk on repetition - -local t = { 8, 9, "a", "b" } - -function os.uuid() - return format("%04x%04x-4%03x-%s%03x-%04x-%04x%04x%04x", - random(0xFFFF),random(0xFFFF), - random(0x0FFF), - t[ceil(random(4))] or 8,random(0x0FFF), - random(0xFFFF), - random(0xFFFF),random(0xFFFF),random(0xFFFF) - ) +local function enable(t,what) + set(t,what,true) end - -local d - -function os.timezone(delta) - d = d or tonumber(tonumber(os.date("%H")-os.date("!%H"))) - if delta then - if d > 0 then - return format("+%02i:00",d) - else - return format("-%02i:00",-d) - end +local function disable(t,what) + local data=t.data + if not what or what=="" then + t.done={} + reset(t) + else + set(t,what,false) + end +end +function setters.register(t,what,...) + local data=t.data + what=lower(what) + local functions=data[what] + if not functions then + functions={} + data[what]=functions + if trace_initialize then + t.report("defining %a",what) + end + end + local default=functions.default + for i=1,select("#",...) do + local fnc=select(i,...) + local typ=type(fnc) + if typ=="string" then + if trace_initialize then + t.report("coupling %a to %a",what,fnc) + end + local s=fnc + fnc=function(value) set(t,s,value) end + elseif typ~="function" then + fnc=nil + end + if fnc then + functions[#functions+1]=fnc + local value=functions.value or default + if value~=nil then + fnc(value) + functions.value=value + end + end + end + return false +end +function setters.enable(t,what) + local e=t.enable + t.enable,t.done=enable,{} + enable(t,what) + t.enable,t.done=e,{} +end +function setters.disable(t,what) + local e=t.disable + t.disable,t.done=disable,{} + disable(t,what) + t.disable,t.done=e,{} +end +function setters.reset(t) + t.done={} + reset(t) +end +function setters.list(t) + local list=table.sortedkeys(t.data) + local user,system={},{} + for l=1,#list do + local what=list[l] + if find(what,"^%*") then + system[#system+1]=what else - return 1 + user[#user+1]=what end + end + return user,system end - -local memory = { } - -local function which(filename) - local fullname = memory[filename] - if fullname == nil then - local suffix = file.suffix(filename) - local suffixes = suffix == "" and os.binsuffixes or { suffix } - for directory in gmatch(os.getenv("PATH"),"[^" .. io.pathseparator .."]+") do - local df = file.join(directory,filename) - for i=1,#suffixes do - local dfs = file.addsuffix(df,suffixes[i]) - if io.exists(dfs) then - fullname = dfs - break - end - end - end - if not fullname then - fullname = false - end - memory[filename] = fullname +function setters.show(t) + local category=t.name + local list=setters.list(t) + t.report() + for k=1,#list do + local name=list[k] + local functions=t.data[name] + if functions then + local value,default,modules=functions.value,functions.default,#functions + value=value==nil and "unset" or tostring(value) + default=default==nil and "unset" or tostring(default) + t.report("%-50s modules: %2i default: %-12s value: %-12s",name,modules,default,value) + end + end + t.report() +end +local enable,disable,register,list,show=setters.enable,setters.disable,setters.register,setters.list,setters.show +function setters.report(setter,...) + print(format("%-15s : %s\n",setter.name,format(...))) +end +local function default(setter,name) + local d=setter.data[name] + return d and d.default +end +local function value(setter,name) + local d=setter.data[name] + return d and (d.value or d.default) +end +function setters.new(name) + local setter + setter={ + data=allocate(), + name=name, + report=function(...) setters.report (setter,...) end, + enable=function(...) enable (setter,...) end, + disable=function(...) disable (setter,...) end, + register=function(...) register(setter,...) end, + list=function(...) list (setter,...) end, + show=function(...) show (setter,...) end, + default=function(...) return default (setter,...) end, + value=function(...) return value (setter,...) end, + } + data[name]=setter + return setter +end +trackers=setters.new("trackers") +directives=setters.new("directives") +experiments=setters.new("experiments") +local t_enable,t_disable=trackers .enable,trackers .disable +local d_enable,d_disable=directives .enable,directives .disable +local e_enable,e_disable=experiments.enable,experiments.disable +local trace_directives=false local trace_directives=false trackers.register("system.directives",function(v) trace_directives=v end) +local trace_experiments=false local trace_experiments=false trackers.register("system.experiments",function(v) trace_experiments=v end) +function directives.enable(...) + if trace_directives then + directives.report("enabling: % t",{...}) + end + d_enable(...) +end +function directives.disable(...) + if trace_directives then + directives.report("disabling: % t",{...}) + end + d_disable(...) +end +function experiments.enable(...) + if trace_experiments then + experiments.report("enabling: % t",{...}) + end + e_enable(...) +end +function experiments.disable(...) + if trace_experiments then + experiments.report("disabling: % t",{...}) + end + e_disable(...) +end +directives.register("system.nostatistics",function(v) + if statistics then + statistics.enable=not v + else + end +end) +directives.register("system.nolibraries",function(v) + if libraries then + libraries=nil + else + end +end) +if environment then + local engineflags=environment.engineflags + if engineflags then + local list=engineflags["c:trackers"] or engineflags["trackers"] + if type(list)=="string" then + setters.initialize("commandline flags","trackers",settings_to_hash(list)) end - return fullname + local list=engineflags["c:directives"] or engineflags["directives"] + if type(list)=="string" then + setters.initialize("commandline flags","directives",settings_to_hash(list)) + end + end +end +if texconfig then + local function set(k,v) + v=tonumber(v) + if v then + texconfig[k]=v + end + end + directives.register("luatex.expanddepth",function(v) set("expand_depth",v) end) + directives.register("luatex.hashextra",function(v) set("hash_extra",v) end) + directives.register("luatex.nestsize",function(v) set("nest_size",v) end) + directives.register("luatex.maxinopen",function(v) set("max_in_open",v) end) + directives.register("luatex.maxprintline",function(v) set("max_print_line",v) end) + directives.register("luatex.maxstrings",function(v) set("max_strings",v) end) + directives.register("luatex.paramsize",function(v) set("param_size",v) end) + directives.register("luatex.savesize",function(v) set("save_size",v) end) + directives.register("luatex.stacksize",function(v) set("stack_size",v) end) end - -os.which = which -os.where = which - --- print(os.which("inkscape.exe")) --- print(os.which("inkscape")) --- print(os.which("gs.exe")) --- print(os.which("ps2pdf")) end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['l-file'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- needs a cleanup - -file = file or { } -local file = file - -local insert, concat = table.insert, table.concat -local find, gmatch, match, gsub, sub, char, lower = string.find, string.gmatch, string.match, string.gsub, string.sub, string.char, string.lower -local lpegmatch = lpeg.match -local getcurrentdir, attributes = lfs.currentdir, lfs.attributes - -local P, R, S, C, Cs, Cp, Cc = lpeg.P, lpeg.R, lpeg.S, lpeg.C, lpeg.Cs, lpeg.Cp, lpeg.Cc - -local function dirname(name,default) - return match(name,"^(.+)[/\\].-$") or (default or "") -end - -local function basename(name) - return match(name,"^.+[/\\](.-)$") or name -end - --- local function nameonly(name) --- return (gsub(match(name,"^.+[/\\](.-)$") or name,"%..*$","")) --- end - -local function nameonly(name) - return (gsub(match(name,"^.+[/\\](.-)$") or name,"%.[%a%d]+$","")) -end - -local function extname(name,default) - return match(name,"^.+%.([^/\\]-)$") or default or "" -end - -local function splitname(name) - local n, s = match(name,"^(.+)%.([^/\\]-)$") - return n or name, s or "" -end - -file.basename = basename -file.dirname = dirname -file.nameonly = nameonly -file.extname = extname -file.suffix = extname +package.loaded["trac-log"] = package.loaded["trac-log"] or true -function file.removesuffix(filename) - return (gsub(filename,"%.[%a%d]+$","")) -end +-- original size: 21795, stripped down to: 14194 -function file.addsuffix(filename, suffix, criterium) - if not suffix or suffix == "" then - return filename - elseif criterium == true then - return filename .. "." .. suffix - elseif not criterium then - local n, s = splitname(filename) - if not s or s == "" then - return filename .. "." .. suffix - else - return filename +if not modules then modules={} end modules ['trac-log']={ + version=1.001, + comment="companion to trac-log.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local write_nl,write=texio and texio.write_nl or print,texio and texio.write or io.write +local format,gmatch,find=string.format,string.gmatch,string.find +local concat,insert,remove=table.concat,table.insert,table.remove +local topattern=string.topattern +local texcount=tex and tex.count +local next,type,select=next,type,select +local utfchar=utf.char +local setmetatableindex=table.setmetatableindex +local formatters=string.formatters +logs=logs or {} +local logs=logs +local moreinfo=[[ +More information about ConTeXt and the tools that come with it can be found at: +]].."\n"..[[ +maillist : ntg-context@ntg.nl / http://www.ntg.nl/mailman/listinfo/ntg-context +webpage : http://www.pragma-ade.nl / http://tex.aanhet.net +wiki : http://contextgarden.net +]] +utilities.strings.formatters.add ( + formatters,"unichr", + [["U+" .. format("%%05X",%s) .. " (" .. utfchar(%s) .. ")"]] +) +utilities.strings.formatters.add ( + formatters,"chruni", + [[utfchar(%s) .. " (U+" .. format("%%05X",%s) .. ")"]] +) +local function ignore() end +setmetatableindex(logs,function(t,k) t[k]=ignore;return ignore end) +local report,subreport,status,settarget,setformats,settranslations +local direct,subdirect,writer,pushtarget,poptarget +if tex and (tex.jobname or tex.formatname) then + local valueiskey={ __index=function(t,k) t[k]=k return k end } + local target="term and log" + logs.flush=io.flush + local formats={} setmetatable(formats,valueiskey) + local translations={} setmetatable(translations,valueiskey) + writer=function(...) + write_nl(target,...) + end + newline=function() + write_nl(target,"\n") + end + local f_one=formatters["%-15s > %s\n"] + local f_two=formatters["%-15s >\n"] + report=function(a,b,c,...) + if c then + write_nl(target,f_one(translations[a],formatters[formats[b]](c,...))) + elseif b then + write_nl(target,f_one(translations[a],formats[b])) + elseif a then + write_nl(target,f_two(translations[a])) + else + write_nl(target,"\n") + end + end + local f_one=formatters["%-15s > %s"] + local f_two=formatters["%-15s >"] + direct=function(a,b,c,...) + if c then + return f_one(translations[a],formatters[formats[b]](c,...)) + elseif b then + return f_one(translations[a],formats[b]) + elseif a then + return f_two(translations[a]) + else + return "" + end + end + local f_one=formatters["%-15s > %s > %s\n"] + local f_two=formatters["%-15s > %s >\n"] + subreport=function(a,s,b,c,...) + if c then + write_nl(target,f_one(translations[a],translations[s],formatters[formats[b]](c,...))) + elseif b then + write_nl(target,f_one(translations[a],translations[s],formats[b])) + elseif a then + write_nl(target,f_two(translations[a],translations[s])) + else + write_nl(target,"\n") + end + end + local f_one=formatters["%-15s > %s > %s"] + local f_two=formatters["%-15s > %s >"] + subdirect=function(a,s,b,c,...) + if c then + return f_one(translations[a],translations[s],formatters[formats[b]](c,...)) + elseif b then + return f_one(translations[a],translations[s],formats[b]) + elseif a then + return f_two(translations[a],translations[s]) + else + return "" + end + end + local f_one=formatters["%-15s : %s\n"] + local f_two=formatters["%-15s :\n"] + status=function(a,b,c,...) + if c then + write_nl(target,f_one(translations[a],formatters[formats[b]](c,...))) + elseif b then + write_nl(target,f_one(translations[a],formats[b])) + elseif a then + write_nl(target,f_two(translations[a])) + else + write_nl(target,"\n") + end + end + local targets={ + logfile="log", + log="log", + file="log", + console="term", + terminal="term", + both="term and log", + } + settarget=function(whereto) + target=targets[whereto or "both"] or targets.both + if target=="term" or target=="term and log" then + logs.flush=io.flush + else + logs.flush=ignore + end + end + local stack={} + pushtarget=function(newtarget) + insert(stack,target) + settarget(newtarget) + end + poptarget=function() + if #stack>0 then + settarget(remove(stack)) + end + end + setformats=function(f) + formats=f + end + settranslations=function(t) + translations=t + end +else + logs.flush=ignore + writer=write_nl + newline=function() + write_nl("\n") + end + local f_one=formatters["%-15s | %s"] + local f_two=formatters["%-15s |"] + report=function(a,b,c,...) + if c then + write_nl(f_one(a,formatters[b](c,...))) + elseif b then + write_nl(f_one(a,b)) + elseif a then + write_nl(f_two(a)) + else + write_nl("") + end + end + local f_one=formatters["%-15s | %s | %s"] + local f_two=formatters["%-15s | %s |"] + subreport=function(a,sub,b,c,...) + if c then + write_nl(f_one(a,sub,formatters[b](c,...))) + elseif b then + write_nl(f_one(a,sub,b)) + elseif a then + write_nl(f_two(a,sub)) + else + write_nl("") + end + end + local f_one=formatters["%-15s : %s\n"] + local f_two=formatters["%-15s :\n"] + status=function(a,b,c,...) + if c then + write_nl(f_one(a,formatters[b](c,...))) + elseif b then + write_nl(f_one(a,b)) + elseif a then + write_nl(f_two(a)) + else + write_nl("\n") + end + end + direct=ignore + subdirect=ignore + settarget=ignore + pushtarget=ignore + poptarget=ignore + setformats=ignore + settranslations=ignore +end +logs.report=report +logs.subreport=subreport +logs.status=status +logs.settarget=settarget +logs.pushtarget=pushtarget +logs.poptarget=poptarget +logs.setformats=setformats +logs.settranslations=settranslations +logs.direct=direct +logs.subdirect=subdirect +logs.writer=writer +logs.newline=newline +local data,states={},nil +function logs.reporter(category,subcategory) + local logger=data[category] + if not logger then + local state=false + if states==true then + state=true + elseif type(states)=="table" then + for c,_ in next,states do + if find(category,c) then + state=true + break + end + end + end + logger={ + reporters={}, + state=state, + } + data[category]=logger + end + local reporter=logger.reporters[subcategory or "default"] + if not reporter then + if subcategory then + reporter=function(...) + if not logger.state then + subreport(category,subcategory,...) end + end + logger.reporters[subcategory]=reporter else - local n, s = splitname(filename) - if s and s ~= "" then - local t = type(criterium) - if t == "table" then - -- keep if in criterium - for i=1,#criterium do - if s == criterium[i] then - return filename - end - end - elseif t == "string" then - -- keep if criterium - if s == criterium then - return filename - end - end + local tag=category + reporter=function(...) + if not logger.state then + report(category,...) end - return n .. "." .. suffix + end + logger.reporters.default=reporter end + end + return reporter end - - -function file.replacesuffix(filename, suffix) - return (gsub(filename,"%.[%a%d]+$","")) .. "." .. suffix +logs.new=logs.reporter +local ctxreport=logs.writer +function logs.setmessenger(m) + ctxreport=m end - - -local trick_1 = char(1) -local trick_2 = "^" .. trick_1 .. "/+" - -function file.join(...) -- rather dirty - local lst = { ... } - local a, b = lst[1], lst[2] - if not a or a == "" then -- not a added - lst[1] = trick_1 - elseif b and find(a,"^/+$") and find(b,"^/") then - lst[1] = "" - lst[2] = gsub(b,"^/+","") - end - local pth = concat(lst,"/") - pth = gsub(pth,"\\","/") - local a, b = match(pth,"^(.*://)(.*)$") - if a and b then - return a .. gsub(b,"//+","/") +function logs.messenger(category,subcategory) + if subcategory then + return function(...) + ctxreport(subdirect(category,subcategory,...)) end - a, b = match(pth,"^(//)(.*)$") - if a and b then - return a .. gsub(b,"//+","/") + else + return function(...) + ctxreport(direct(category,...)) end - pth = gsub(pth,trick_2,"") - return (gsub(pth,"//+","/")) + end end - - --- We should be able to use: --- --- function file.is_writable(name) --- local a = attributes(name) or attributes(dirname(name,".")) --- return a and sub(a.permissions,2,2) == "w" --- end --- --- But after some testing Taco and I came up with: - -function file.is_writable(name) - if lfs.isdir(name) then - name = name .. "/m_t_x_t_e_s_t.tmp" - local f = io.open(name,"wb") - if f then - f:close() - os.remove(name) - return true - end - elseif lfs.isfile(name) then - local f = io.open(name,"ab") - if f then - f:close() - return true - end - else - local f = io.open(name,"ab") - if f then - f:close() - os.remove(name) - return true - end - end - return false +local function setblocked(category,value) + if category==true then + category,value="*",true + elseif category==false then + category,value="*",false + elseif value==nil then + value=true + end + if category=="*" then + states=value + for k,v in next,data do + v.state=value + end + else + states=utilities.parsers.settings_to_hash(category) + for c,_ in next,states do + if data[c] then + v.state=value + else + c=topattern(c,true,true) + for k,v in next,data do + if find(k,c) then + v.state=value + end + end + end + end + end end - -function file.is_readable(name) - local a = attributes(name) - return a and sub(a.permissions,1,1) == "r" +function logs.disable(category,value) + setblocked(category,value==nil and true or value) end - -file.isreadable = file.is_readable -- depricated -file.iswritable = file.is_writable -- depricated - --- todo: lpeg \\ / .. does not save much - -local checkedsplit = string.checkedsplit - -function file.splitpath(str,separator) -- string - str = gsub(str,"\\","/") - return checkedsplit(str,separator or io.pathseparator) +function logs.enable(category) + setblocked(category,false) end - -function file.joinpath(tab,separator) -- table - return concat(tab,separator or io.pathseparator) -- can have trailing // +function logs.categories() + return table.sortedkeys(data) end - --- we can hash them weakly - - -function file.collapsepath(str,anchor) - if anchor and not find(str,"^/") and not find(str,"^%a:") then - str = getcurrentdir() .. "/" .. str - end - if str == "" or str =="." then - return "." - elseif find(str,"^%.%.") then - str = gsub(str,"\\","/") - return str - elseif not find(str,"%.") then - str = gsub(str,"\\","/") - return str - end - str = gsub(str,"\\","/") - local starter, rest = match(str,"^(%a+:/*)(.-)$") - if starter then - str = rest - end - local oldelements = checkedsplit(str,"/") - local newelements = { } - local i = #oldelements - while i > 0 do - local element = oldelements[i] - if element == '.' then - -- do nothing - elseif element == '..' then - local n = i - 1 - while n > 0 do - local element = oldelements[n] - if element ~= '..' and element ~= '.' then - oldelements[n] = '.' - break - else - n = n - 1 - end - end - if n < 1 then - insert(newelements,1,'..') - end - elseif element ~= "" then - insert(newelements,1,element) - end - i = i - 1 - end - if #newelements == 0 then - return starter or "." - elseif starter then - return starter .. concat(newelements, '/') - elseif find(str,"^/") then - return "/" .. concat(newelements,'/') +function logs.show() + local n,c,s,max=0,0,0,0 + for category,v in table.sortedpairs(data) do + n=n+1 + local state=v.state + local reporters=v.reporters + local nc=#category + if nc>c then + c=nc + end + for subcategory,_ in next,reporters do + local ns=#subcategory + if ns>c then + s=ns + end + local m=nc+ns + if m>max then + max=m + end + end + local subcategories=concat(table.sortedkeys(reporters),", ") + if state==true then + state="disabled" + elseif state==false then + state="enabled" else - return concat(newelements, '/') + state="unknown" end + report("logging","category %a, subcategories %a, state %a",category,subcategories,state) + end + report("logging","categories: %s, max category: %s, max subcategory: %s, max combined: %s",n,c,s,max) end - - -function file.robustname(str,strict) - str = gsub(str,"[^%a%d%/%-%.\\]+","-") - if strict then - return lower(gsub(str,"^%-*(.-)%-*$","%1")) +local delayed_reporters={} +setmetatableindex(delayed_reporters,function(t,k) + local v=logs.reporter(k.name) + t[k]=v + return v +end) +function utilities.setters.report(setter,...) + delayed_reporters[setter](...) +end +directives.register("logs.blocked",function(v) + setblocked(v,true) +end) +directives.register("logs.target",function(v) + settarget(v) +end) +local report_pages=logs.reporter("pages") +local real,user,sub +function logs.start_page_number() + real,user,sub=texcount.realpageno,texcount.userpageno,texcount.subpageno +end +local timing=false +local starttime=nil +local lasttime=nil +trackers.register("pages.timing",function(v) + starttime=os.clock() + timing=true +end) +function logs.stop_page_number() + if timing then + local elapsed,average + local stoptime=os.clock() + if not lasttime or real<2 then + elapsed=stoptime + average=stoptime + starttime=stoptime + else + elapsed=stoptime-lasttime + average=(stoptime-starttime)/(real-1) + end + lasttime=stoptime + if real<=0 then + report_pages("flushing page, time %0.04f / %0.04f",elapsed,average) + elseif user<=0 then + report_pages("flushing realpage %s, time %0.04f / %0.04f",real,elapsed,average) + elseif sub<=0 then + report_pages("flushing realpage %s, userpage %s, time %0.04f / %0.04f",real,user,elapsed,average) else - return str + report_pages("flushing realpage %s, userpage %s, subpage %s, time %0.04f / %0.04f",real,user,sub,elapsed,average) + end + else + if real<=0 then + report_pages("flushing page") + elseif user<=0 then + report_pages("flushing realpage %s",real) + elseif sub<=0 then + report_pages("flushing realpage %s, userpage %s",real,user) + else + report_pages("flushing realpage %s, userpage %s, subpage %s",real,user,sub) end + end + logs.flush() end - -file.readdata = io.loaddata -file.savedata = io.savedata - -function file.copy(oldname,newname) - file.savedata(newname,io.loaddata(oldname)) +local report_files=logs.reporter("files") +local nesting=0 +local verbose=false +local hasscheme=url.hasscheme +function logs.show_open(name) end - --- lpeg variants, slightly faster, not always - - - - - - - - - - - - - - - - - - - --- also rewrite previous - -local letter = R("az","AZ") + S("_-+") -local separator = P("://") - -local qualified = P(".")^0 * P("/") + letter*P(":") + letter^1*separator + letter^1 * P("/") -local rootbased = P("/") + letter*P(":") - -lpeg.patterns.qualified = qualified -lpeg.patterns.rootbased = rootbased - --- ./name ../name /name c: :// name/name - -function file.is_qualified_path(filename) - return lpegmatch(qualified,filename) ~= nil +function logs.show_close(name) end - -function file.is_rootbased_path(filename) - return lpegmatch(rootbased,filename) ~= nil +function logs.show_load(name) end - --- actually these are schemes - -local slash = S("\\/") -local period = P(".") -local drive = C(R("az","AZ")) * P(":") -local path = C(((1-slash)^0 * slash)^0) -local suffix = period * C(P(1-period)^0 * P(-1)) -local base = C((1-suffix)^0) - -drive = drive + Cc("") -path = path + Cc("") -base = base + Cc("") -suffix = suffix + Cc("") - -local pattern_a = drive * path * base * suffix -local pattern_b = path * base * suffix -local pattern_c = C(drive * path) * C(base * suffix) - -function file.splitname(str,splitdrive) - if splitdrive then - return lpegmatch(pattern_a,str) -- returns drive, path, base, suffix - else - return lpegmatch(pattern_b,str) -- returns path, base, suffix +local simple=logs.reporter("comment") +logs.simple=simple +logs.simpleline=simple +function logs.setprogram () end +function logs.extendbanner() end +function logs.reportlines () end +function logs.reportbanner() end +function logs.reportline () end +function logs.simplelines () end +function logs.help () end +local Carg,C,lpegmatch=lpeg.Carg,lpeg.C,lpeg.match +local p_newline=lpeg.patterns.newline +local linewise=( + Carg(1)*C((1-p_newline)^1)/function(t,s) t.report(s) end+Carg(1)*p_newline^2/function(t) t.report() end+p_newline +)^1 +local function reportlines(t,str) + if str then + lpegmatch(linewise,str,1,t) + end +end +local function reportbanner(t) + local banner=t.banner + if banner then + t.report(banner) + t.report() + end +end +local function reportversion(t) + local banner=t.banner + if banner then + t.report(banner) + end +end +local function reporthelp(t,...) + local helpinfo=t.helpinfo + if type(helpinfo)=="string" then + reportlines(t,helpinfo) + elseif type(helpinfo)=="table" then + for i=1,select("#",...) do + reportlines(t,t.helpinfo[select(i,...)]) + if i %s => %s => %s\r"](os.date("%d/%m/%y %H:%m:%S"),process,jobname,category,format(...)) + for i=1,10 do + local f=io.open(whereto,"a") + if f then + f:write(message) + f:close() + break else - return { - path = path, - name = name, - base = base, - suffix = suffix, - } + sleep(0.1) end + end end - --- function test(t) for k, v in next, t do print(v, "=>", file.splitname(v)) end end --- --- test { "c:", "c:/aa", "c:/aa/bb", "c:/aa/bb/cc", "c:/aa/bb/cc.dd", "c:/aa/bb/cc.dd.ee" } --- test { "c:", "c:aa", "c:aa/bb", "c:aa/bb/cc", "c:aa/bb/cc.dd", "c:aa/bb/cc.dd.ee" } --- test { "/aa", "/aa/bb", "/aa/bb/cc", "/aa/bb/cc.dd", "/aa/bb/cc.dd.ee" } --- test { "aa", "aa/bb", "aa/bb/cc", "aa/bb/cc.dd", "aa/bb/cc.dd.ee" } - - --- for myself: - -function file.strip(name,dir) - local b, a = match(name,"^(.-)" .. dir .. "(.*)$") - return a ~= "" and a or name +local report_system=logs.reporter("system","logs") +function logs.obsolete(old,new) + local o=loadstring("return "..new)() + if type(o)=="function" then + return function(...) + report_system("function %a is obsolete, use %a",old,new) + loadstring(old.."="..new.." return "..old)()(...) + end + elseif type(o)=="table" then + local t,m={},{} + m.__index=function(t,k) + report_system("table %a is obsolete, use %a",old,new) + m.__index,m.__newindex=o,o + return o[k] + end + m.__newindex=function(t,k,v) + report_system("table %a is obsolete, use %a",old,new) + m.__index,m.__newindex=o,o + o[k]=v + end + if libraries then + libraries.obsolete[old]=t + end + setmetatable(t,m) + return t + end +end +if utilities then + utilities.report=report_system +end +if tex and tex.error then + function logs.texerrormessage(...) + tex.error(format(...),{}) + end +else + function logs.texerrormessage(...) + print(format(...)) + end end +io.stdout:setvbuf('no') +io.stderr:setvbuf('no') end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['l-md5'] = { - version = 1.001, - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- This also provides file checksums and checkers. +package.loaded["trac-inf"] = package.loaded["trac-inf"] or true -local md5, file = md5, file -local gsub, format, byte = string.gsub, string.format, string.byte +-- original size: 5791, stripped down to: 4540 -local function convert(str,fmt) - return (gsub(md5.sum(str),".",function(chr) return format(fmt,byte(chr)) end)) +if not modules then modules={} end modules ['trac-inf']={ + version=1.001, + comment="companion to trac-inf.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local type,tonumber=type,tonumber +local format,lower=string.format,string.lower +local concat=table.concat +local clock=os.gettimeofday or os.clock +statistics=statistics or {} +local statistics=statistics +statistics.enable=true +statistics.threshold=0.01 +local statusinfo,n,registered,timers={},0,{},{} +table.setmetatableindex(timers,function(t,k) + local v={ timing=0,loadtime=0 } + t[k]=v + return v +end) +local function hastiming(instance) + return instance and timers[instance] end - -if not md5.HEX then function md5.HEX(str) return convert(str,"%02X") end end -if not md5.hex then function md5.hex(str) return convert(str,"%02x") end end -if not md5.dec then function md5.dec(str) return convert(str,"%03i") end end - - -function file.needs_updating(oldname,newname,threshold) -- size modification access change - local oldtime = lfs.attributes(oldname, modification) - local newtime = lfs.attributes(newname, modification) - if newtime >= oldtime then - return false - elseif oldtime - newtime < (threshold or 1) then - return false - else - return true - end +local function resettiming(instance) + timers[instance or "notimer"]={ timing=0,loadtime=0 } end - -function file.checksum(name) - if md5 then - local data = io.loaddata(name) - if data then - return md5.HEX(data) - end - end - return nil +local function starttiming(instance) + local timer=timers[instance or "notimer"] + local it=timer.timing or 0 + if it==0 then + timer.starttime=clock() + if not timer.loadtime then + timer.loadtime=0 + end + end + timer.timing=it+1 +end +local function stoptiming(instance,report) + local timer=timers[instance or "notimer"] + local it=timer.timing + if it>1 then + timer.timing=it-1 + else + local starttime=timer.starttime + if starttime then + local stoptime=clock() + local loadtime=stoptime-starttime + timer.stoptime=stoptime + timer.loadtime=timer.loadtime+loadtime + if report then + statistics.report("load time %0.3f",loadtime) + end + timer.timing=0 + return loadtime + end + end + return 0 +end +local function elapsed(instance) + if type(instance)=="number" then + return instance or 0 + else + local timer=timers[instance or "notimer"] + return timer and timer.loadtime or 0 + end end - -function file.loadchecksum(name) - if md5 then - local data = io.loaddata(name .. ".md5") - return data and (gsub(data,"%s","")) - end - return nil +local function elapsedtime(instance) + return format("%0.3f",elapsed(instance)) end - -function file.savechecksum(name, checksum) - if not checksum then checksum = file.checksum(name) end - if checksum then - io.savedata(name .. ".md5",checksum) - return checksum +local function elapsedindeed(instance) + return elapsed(instance)>statistics.threshold +end +local function elapsedseconds(instance,rest) + if elapsedindeed(instance) then + return format("%0.3f seconds %s",elapsed(instance),rest or "") + end +end +statistics.hastiming=hastiming +statistics.resettiming=resettiming +statistics.starttiming=starttiming +statistics.stoptiming=stoptiming +statistics.elapsed=elapsed +statistics.elapsedtime=elapsedtime +statistics.elapsedindeed=elapsedindeed +statistics.elapsedseconds=elapsedseconds +function statistics.register(tag,fnc) + if statistics.enable and type(fnc)=="function" then + local rt=registered[tag] or (#statusinfo+1) + statusinfo[rt]={ tag,fnc } + registered[tag]=rt + if #tag>n then n=#tag end + end +end +local report=logs.reporter("mkiv lua stats") +function statistics.show() + if statistics.enable then + local register=statistics.register + register("luatex banner",function() + return lower(status.banner) + end) + register("control sequences",function() + return format("%s of %s + %s",status.cs_count,status.hash_size,status.hash_extra) + end) + register("callbacks",function() + local total,indirect=status.callbacks or 0,status.indirect_callbacks or 0 + return format("%s direct, %s indirect, %s total",total-indirect,indirect,total) + end) + if jit then + local status={ jit.status() } + if status[1] then + register("luajit status",function() + return concat(status," ",2) + end) + end end - return nil + register("current memory usage",statistics.memused) + register("runtime",statistics.runtime) + logs.newline() + for i=1,#statusinfo do + local s=statusinfo[i] + local r=s[2]() + if r then + report("%s: %s",s[1],r) + end + end + statistics.enable=false + end +end +function statistics.memused() + local round=math.round or math.floor + return format("%s MB (ctx: %s MB)",round(collectgarbage("count")/1000),round(status.luastate_bytes/1000000)) +end +starttiming(statistics) +function statistics.formatruntime(runtime) + return format("%s seconds",runtime) +end +function statistics.runtime() + stoptiming(statistics) + return statistics.formatruntime(elapsedtime(statistics)) +end +local report=logs.reporter("system") +function statistics.timed(action) + starttiming("run") + action() + stoptiming("run") + report("total runtime: %s",elapsedtime("run")) +end +commands=commands or {} +function commands.resettimer(name) + resettiming(name or "whatever") + starttiming(name or "whatever") +end +function commands.elapsedtime(name) + stoptiming(name or "whatever") + context(elapsedtime(name or "whatever")) end @@ -3125,1225 +6862,2145 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['l-url'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local char, gmatch, gsub, format, byte, find = string.char, string.gmatch, string.gsub, string.format, string.byte, string.find -local concat = table.concat -local tonumber, type = tonumber, type -local P, C, R, S, Cs, Cc, Ct = lpeg.P, lpeg.C, lpeg.R, lpeg.S, lpeg.Cs, lpeg.Cc, lpeg.Ct -local lpegmatch, lpegpatterns, replacer = lpeg.match, lpeg.patterns, lpeg.replacer +package.loaded["trac-pro"] = package.loaded["trac-pro"] or true --- from wikipedia: --- --- foo://username:password@example.com:8042/over/there/index.dtb?type=animal;name=narwhal#nose --- \_/ \_______________/ \_________/ \__/ \___/ \_/ \______________________/ \__/ --- | | | | | | | | --- | userinfo hostname port | | query fragment --- | \________________________________/\_____________|____|/ --- scheme | | | | --- | authority path | | --- | | | --- | path interpretable as filename --- | ___________|____________ | --- / \ / \ | --- urn:example:animal:ferret:nose interpretable as extension - -url = url or { } -local url = url - -local tochar = function(s) return char(tonumber(s,16)) end - -local colon = P(":") -local qmark = P("?") -local hash = P("#") -local slash = P("/") -local percent = P("%") -local endofstring = P(-1) - -local hexdigit = R("09","AF","af") -local plus = P("+") -local nothing = Cc("") -local escaped = (plus / " ") + (percent * C(hexdigit * hexdigit) / tochar) - --- we assume schemes with more than 1 character (in order to avoid problems with windows disks) --- we also assume that when we have a scheme, we also have an authority - -local schemestr = Cs((escaped+(1-colon-slash-qmark-hash))^2) -local authoritystr = Cs((escaped+(1- slash-qmark-hash))^0) -local pathstr = Cs((escaped+(1- qmark-hash))^0) -local querystr = Cs((escaped+(1- hash))^0) -local fragmentstr = Cs((escaped+(1- endofstring))^0) - -local scheme = schemestr * colon + nothing -local authority = slash * slash * authoritystr + nothing -local path = slash * pathstr + nothing -local query = qmark * querystr + nothing -local fragment = hash * fragmentstr + nothing - -local validurl = scheme * authority * path * query * fragment -local parser = Ct(validurl) - -lpegpatterns.url = validurl -lpegpatterns.urlsplitter = parser - -local escapes = { } ; for i=0,255 do escapes[i] = format("%%%02X",i) end - -local escaper = Cs((R("09","AZ","az") + S("-./_") + P(1) / escapes)^0) - -lpegpatterns.urlescaper = escaper - --- todo: reconsider Ct as we can as well have five return values (saves a table) --- so we can have two parsers, one with and one without +-- original size: 5773, stripped down to: 3453 -local function split(str) - return (type(str) == "string" and lpegmatch(parser,str)) or str +if not modules then modules={} end modules ['trac-pro']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local getmetatable,setmetatable,rawset,type=getmetatable,setmetatable,rawset,type +local trace_namespaces=false trackers.register("system.namespaces",function(v) trace_namespaces=v end) +local report_system=logs.reporter("system","protection") +namespaces=namespaces or {} +local namespaces=namespaces +local registered={} +local function report_index(k,name) + if trace_namespaces then + report_system("reference to %a in protected namespace %a: %s",k,name,debug.traceback()) + else + report_system("reference to %a in protected namespace %a",k,name) + end end - -local isscheme = schemestr * colon * slash * slash -- this test also assumes authority - -local function hasscheme(str) - local scheme = lpegmatch(isscheme,str) -- at least one character - return scheme ~= "" and scheme or false -end - - --- todo: cache them - -local rootletter = R("az","AZ") - + S("_-+") -local separator = P("://") -local qualified = P(".")^0 * P("/") - + rootletter * P(":") - + rootletter^1 * separator - + rootletter^1 * P("/") -local rootbased = P("/") - + rootletter * P(":") - -local barswapper = replacer("|",":") -local backslashswapper = replacer("\\","/") - -local function hashed(str) -- not yet ok (/test?test) - local s = split(str) - local somescheme = s[1] ~= "" - local somequery = s[4] ~= "" - if not somescheme and not somequery then - s = { - scheme = "file", - authority = "", - path = str, - query = "", - fragment = "", - original = str, - noscheme = true, - filename = str, - } - else -- not always a filename but handy anyway - local authority, path, filename = s[2], s[3] - if authority == "" then - filename = path - else - filename = authority .. "/" .. path - end - s = { - scheme = s[1], - authority = authority, - path = path, - query = s[4], - fragment = s[5], - original = str, - noscheme = false, - filename = filename, - } +local function report_newindex(k,name) + if trace_namespaces then + report_system("assignment to %a in protected namespace %a: %s",k,name,debug.traceback()) + else + report_system("assignment to %a in protected namespace %a",k,name) + end +end +local function register(name) + local data=name=="global" and _G or _G[name] + if not data then + return + end + registered[name]=data + local m=getmetatable(data) + if not m then + m={} + setmetatable(data,m) + end + local index,newindex={},{} + m.__saved__index=m.__index + m.__no__index=function(t,k) + if not index[k] then + index[k]=true + report_index(k,name) end - return s + return nil + end + m.__saved__newindex=m.__newindex + m.__no__newindex=function(t,k,v) + if not newindex[k] then + newindex[k]=true + report_newindex(k,name) + end + rawset(t,k,v) + end + m.__protection__depth=0 +end +local function private(name) + local data=registered[name] + if not data then + data=_G[name] + if not data then + data={} + _G[name]=data + end + register(name) + end + return data end +local function protect(name) + local data=registered[name] + if not data then + return + end + local m=getmetatable(data) + local pd=m.__protection__depth + if pd>0 then + m.__protection__depth=pd+1 + else + m.__save_d_index,m.__saved__newindex=m.__index,m.__newindex + m.__index,m.__newindex=m.__no__index,m.__no__newindex + m.__protection__depth=1 + end +end +local function unprotect(name) + local data=registered[name] + if not data then + return + end + local m=getmetatable(data) + local pd=m.__protection__depth + if pd>1 then + m.__protection__depth=pd-1 + else + m.__index,m.__newindex=m.__saved__index,m.__saved__newindex + m.__protection__depth=0 + end +end +local function protectall() + for name,_ in next,registered do + if name~="global" then + protect(name) + end + end +end +local function unprotectall() + for name,_ in next,registered do + if name~="global" then + unprotect(name) + end + end +end +namespaces.register=register +namespaces.private=private +namespaces.protect=protect +namespaces.unprotect=unprotect +namespaces.protectall=protectall +namespaces.unprotectall=unprotectall +namespaces.private("namespaces") registered={} register("global") +directives.register("system.protect",function(v) + if v then + protectall() + else + unprotectall() + end +end) +directives.register("system.checkglobals",function(v) + if v then + report_system("enabling global namespace guard") + protect("global") + else + report_system("disabling global namespace guard") + unprotect("global") + end +end) --- Here we assume: --- --- files: /// = relative --- files: //// = absolute (!) +end -- of closure +do -- create closure to overcome 200 locals limit -url.split = split -url.hasscheme = hasscheme -url.hashed = hashed +package.loaded["util-lua"] = package.loaded["util-lua"] or true -function url.addscheme(str,scheme) -- no authority - if hasscheme(str) then - return str - elseif not scheme then - return "file:///" .. str - else - return scheme .. ":///" .. str - end -end +-- original size: 12575, stripped down to: 8700 -function url.construct(hash) -- dodo: we need to escape ! - local fullurl, f = { }, 0 - local scheme, authority, path, query, fragment = hash.scheme, hash.authority, hash.path, hash.query, hash.fragment - if scheme and scheme ~= "" then - f = f + 1 ; fullurl[f] = scheme .. "://" - end - if authority and authority ~= "" then - f = f + 1 ; fullurl[f] = authority - end - if path and path ~= "" then - f = f + 1 ; fullurl[f] = "/" .. path +if not modules then modules={} end modules ['util-lua']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + comment="the strip code is written by Peter Cawley", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local rep,sub,byte,dump,format=string.rep,string.sub,string.byte,string.dump,string.format +local load,loadfile,type=load,loadfile,type +utilities=utilities or {} +utilities.lua=utilities.lua or {} +local luautilities=utilities.lua +local report_lua=logs.reporter("system","lua") +local tracestripping=false +local forcestupidcompile=true +luautilities.stripcode=true +luautilities.alwaysstripcode=false +luautilities.nofstrippedchunks=0 +luautilities.nofstrippedbytes=0 +local strippedchunks={} +luautilities.strippedchunks=strippedchunks +luautilities.suffixes={ + tma="tma", + tmc=jit and "tmb" or "tmc", + lua="lua", + luc=jit and "lub" or "luc", + lui="lui", + luv="luv", + luj="luj", + tua="tua", + tuc="tuc", +} +if jit or status.luatex_version>=74 then + local function register(name) + if tracestripping then + report_lua("stripped bytecode from %a",name or "unknown") + end + strippedchunks[#strippedchunks+1]=name + luautilities.nofstrippedchunks=luautilities.nofstrippedchunks+1 + end + local function stupidcompile(luafile,lucfile,strip) + local code=io.loaddata(luafile) + if code and code~="" then + code=load(code) + if code then + code=dump(code,strip and luautilities.stripcode or luautilities.alwaysstripcode) + if code and code~="" then + register(name) + io.savedata(lucfile,code) + return true,0 + end + else + report_lua("fatal error %a in file %a",1,luafile) + end + else + report_lua("fatal error %a in file %a",2,luafile) + end + return false,0 + end + function luautilities.loadedluacode(fullname,forcestrip,name) + name=name or fullname + local code=environment.loadpreprocessedfile and environment.loadpreprocessedfile(fullname) or loadfile(fullname) + if code then + code() + end + if forcestrip and luautilities.stripcode then + if type(forcestrip)=="function" then + forcestrip=forcestrip(fullname) + end + if forcestrip or luautilities.alwaysstripcode then + register(name) + return load(dump(code,true)),0 + else + return code,0 + end + elseif luautilities.alwaysstripcode then + register(name) + return load(dump(code,true)),0 + else + return code,0 + end + end + function luautilities.strippedloadstring(code,forcestrip,name) + if forcestrip and luautilities.stripcode or luautilities.alwaysstripcode then + code=load(code) + if not code then + report_lua("fatal error %a in file %a",3,name) + end + register(name) + code=dump(code,true) + end + return load(code),0 + end + function luautilities.compile(luafile,lucfile,cleanup,strip,fallback) + report_lua("compiling %a into %a",luafile,lucfile) + os.remove(lucfile) + local done=stupidcompile(luafile,lucfile,strip~=false) + if done then + report_lua("dumping %a into %a stripped",luafile,lucfile) + if cleanup==true and lfs.isfile(lucfile) and lfs.isfile(luafile) then + report_lua("removing %a",luafile) + os.remove(luafile) + end end - if query and query ~= "" then - f = f + 1 ; fullurl[f] = "?".. query + return done + end + function luautilities.loadstripped(...) + local l=load(...) + if l then + return load(dump(l,true)) end - if fragment and fragment ~= "" then - f = f + 1 ; fullurl[f] = "#".. fragment + end +else + local function register(name,before,after) + local delta=before-after + if tracestripping then + report_lua("bytecodes stripped from %a, # before %s, # after %s, delta %s",name,before,after,delta) + end + strippedchunks[#strippedchunks+1]=name + luautilities.nofstrippedchunks=luautilities.nofstrippedchunks+1 + luautilities.nofstrippedbytes=luautilities.nofstrippedbytes+delta + return delta + end + local strip_code_pc + if _MAJORVERSION==5 and _MINORVERSION==1 then + strip_code_pc=function(dump,name) + local before=#dump + local version,format,endian,int,size,ins,num=byte(dump,5,11) + local subint + if endian==1 then + subint=function(dump,i,l) + local val=0 + for n=l,1,-1 do + val=val*256+byte(dump,i+n-1) + end + return val,i+l + end + else + subint=function(dump,i,l) + local val=0 + for n=1,l,1 do + val=val*256+byte(dump,i+n-1) + end + return val,i+l + end + end + local strip_function + strip_function=function(dump) + local count,offset=subint(dump,1,size) + local stripped,dirty=rep("\0",size),offset+count + offset=offset+count+int*2+4 + offset=offset+int+subint(dump,offset,int)*ins + count,offset=subint(dump,offset,int) + for n=1,count do + local t + t,offset=subint(dump,offset,1) + if t==1 then + offset=offset+1 + elseif t==4 then + offset=offset+size+subint(dump,offset,size) + elseif t==3 then + offset=offset+num + end + end + count,offset=subint(dump,offset,int) + stripped=stripped..sub(dump,dirty,offset-1) + for n=1,count do + local proto,off=strip_function(sub(dump,offset,-1)) + stripped,offset=stripped..proto,offset+off-1 + end + offset=offset+subint(dump,offset,int)*int+int + count,offset=subint(dump,offset,int) + for n=1,count do + offset=offset+subint(dump,offset,size)+size+int*2 + end + count,offset=subint(dump,offset,int) + for n=1,count do + offset=offset+subint(dump,offset,size)+size + end + stripped=stripped..rep("\0",int*3) + return stripped,offset + end + dump=sub(dump,1,12)..strip_function(sub(dump,13,-1)) + local after=#dump + local delta=register(name,before,after) + return dump,delta + end + else + strip_code_pc=function(dump,name) + return dump,0 + end + end + function luautilities.loadedluacode(fullname,forcestrip,name) + local code=environment.loadpreprocessedfile and environment.preprocessedloadfile(fullname) or loadfile(fullname) + if code then + code() + end + if forcestrip and luautilities.stripcode then + if type(forcestrip)=="function" then + forcestrip=forcestrip(fullname) + end + if forcestrip then + local code,n=strip_code_pc(dump(code),name) + return load(code),n + elseif luautilities.alwaysstripcode then + return load(strip_code_pc(dump(code),name)) + else + return code,0 + end + elseif luautilities.alwaysstripcode then + return load(strip_code_pc(dump(code),name)) + else + return code,0 + end + end + function luautilities.strippedloadstring(code,forcestrip,name) + local n=0 + if (forcestrip and luautilities.stripcode) or luautilities.alwaysstripcode then + code=load(code) + if not code then + report_lua("fatal error in file %a",name) + end + code,n=strip_code_pc(dump(code),name) + end + return load(code),n + end + local function stupidcompile(luafile,lucfile,strip) + local code=io.loaddata(luafile) + local n=0 + if code and code~="" then + code=load(code) + if not code then + report_lua("fatal error in file %a",luafile) + end + code=dump(code) + if strip then + code,n=strip_code_pc(code,luautilities.stripcode or luautilities.alwaysstripcode,luafile) + end + if code and code~="" then + io.savedata(lucfile,code) + end end - return lpegmatch(escaper,concat(fullurl)) -end - -function url.filename(filename) - local t = hashed(filename) - return (t.scheme == "file" and (gsub(t.path,"^/([a-zA-Z])([:|])/)","%1:"))) or filename -end - -function url.query(str) - if type(str) == "string" then - local t = { } - for k, v in gmatch(str,"([^&=]*)=([^&=]*)") do - t[k] = v - end - return t + return n + end + local luac_normal="texluac -o %q %q" + local luac_strip="texluac -s -o %q %q" + function luautilities.compile(luafile,lucfile,cleanup,strip,fallback) + report_lua("compiling %a into %a",luafile,lucfile) + os.remove(lucfile) + local done=false + if strip~=false then + strip=true + end + if forcestupidcompile then + fallback=true + elseif strip then + done=os.spawn(format(luac_strip,lucfile,luafile))==0 else - return str + done=os.spawn(format(luac_normal,lucfile,luafile))==0 + end + if not done and fallback then + local n=stupidcompile(luafile,lucfile,strip) + if n>0 then + report_lua("%a dumped into %a (%i bytes stripped)",luafile,lucfile,n) + else + report_lua("%a dumped into %a (unstripped)",luafile,lucfile) + end + cleanup=false + done=true + end + if done and cleanup==true and lfs.isfile(lucfile) and lfs.isfile(luafile) then + report_lua("removing %a",luafile) + os.remove(luafile) end + return done + end + luautilities.loadstripped=loadstring end +end -- of closure +do -- create closure to overcome 200 locals limit +package.loaded["util-deb"] = package.loaded["util-deb"] or true +-- original size: 3708, stripped down to: 2568 - - +if not modules then modules={} end modules ['util-deb']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local debug=require "debug" +local getinfo=debug.getinfo +local type,next,tostring=type,next,tostring +local format,find=string.format,string.find +local is_boolean=string.is_boolean +utilities=utilities or {} +local debugger=utilities.debugger or {} +utilities.debugger=debugger +local counters={} +local names={} +local report=logs.reporter("debugger") +local function hook() + local f=getinfo(2) + if f then + local n="unknown" + if f.what=="C" then + n=f.name or '' + if not names[n] then + names[n]=format("%42s",n) + end + else + n=f.name or f.namewhat or f.what + if not n or n=="" then + n="?" + end + if not names[n] then + names[n]=format("%42s : % 5i : %s",n,f.linedefined or 0,f.short_src or "unknown source") + end + end + counters[n]=(counters[n] or 0)+1 + end +end +function debugger.showstats(printer,threshold) + printer=printer or report + threshold=threshold or 0 + local total,grandtotal,functions=0,0,0 + local dataset={} + for name,count in next,counters do + dataset[#dataset+1]={ name,count } + end + table.sort(dataset,function(a,b) return a[2]==b[2] and b[1]>a[1] or a[2]>b[2] end) + for i=1,#dataset do + local d=dataset[i] + local name=d[1] + local count=d[2] + if count>threshold and not find(name,"for generator") then + printer(format("%8i %s\n",count,names[name])) + total=total+count + end + grandtotal=grandtotal+count + functions=functions+1 + end + printer("\n") + printer(format("functions : % 10i\n",functions)) + printer(format("total : % 10i\n",total)) + printer(format("grand total: % 10i\n",grandtotal)) + printer(format("threshold : % 10i\n",threshold)) +end +function debugger.savestats(filename,threshold) + local f=io.open(filename,'w') + if f then + debugger.showstats(function(str) f:write(str) end,threshold) + f:close() + end +end +function debugger.enable() + debug.sethook(hook,"c") +end +function debugger.disable() + debug.sethook() +end +function traceback() + local level=1 + while true do + local info=debug.getinfo(level,"Sl") + if not info then + break + elseif info.what=="C" then + print(format("%3i : C function",level)) + else + print(format("%3i : [%s]:%d",level,info.short_src,info.currentline)) + end + level=level+1 + end +end end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['l-dir'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} +package.loaded["util-mrg"] = package.loaded["util-mrg"] or true --- dir.expandname will be merged with cleanpath and collapsepath +-- original size: 7294, stripped down to: 5798 -local type = type -local find, gmatch, match, gsub = string.find, string.gmatch, string.match, string.gsub -local concat, insert, remove = table.concat, table.insert, table.remove -local lpegmatch = lpeg.match +if not modules then modules={} end modules ['util-mrg']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local gsub,format=string.gsub,string.format +local concat=table.concat +local type,next=type,next +local P,R,S,V,Ct,C,Cs,Cc,Cp,Cmt,Cb,Cg=lpeg.P,lpeg.R,lpeg.S,lpeg.V,lpeg.Ct,lpeg.C,lpeg.Cs,lpeg.Cc,lpeg.Cp,lpeg.Cmt,lpeg.Cb,lpeg.Cg +local lpegmatch,patterns=lpeg.match,lpeg.patterns +utilities=utilities or {} +local merger=utilities.merger or {} +utilities.merger=merger +merger.strip_comment=true +local report=logs.reporter("system","merge") +utilities.report=report +local m_begin_merge="begin library merge" +local m_end_merge="end library merge" +local m_begin_closure="do -- create closure to overcome 200 locals limit" +local m_end_closure="end -- of closure" +local m_pattern="%c+".."%-%-%s+"..m_begin_merge.."%c+(.-)%c+".."%-%-%s+"..m_end_merge.."%c+" +local m_format="\n\n-- "..m_begin_merge.."\n%s\n".."-- "..m_end_merge.."\n\n" +local m_faked="-- ".."created merged file".."\n\n".."-- "..m_begin_merge.."\n\n".."-- "..m_end_merge.."\n\n" +local m_report=[[ +-- used libraries : %s +-- skipped libraries : %s +-- original bytes : %s +-- stripped bytes : %s +]] +local m_preloaded=[[package.loaded[%q] = package.loaded[%q] or true]] +local function self_fake() + return m_faked +end +local function self_nothing() + return "" +end +local function self_load(name) + local data=io.loaddata(name) or "" + if data=="" then + report("unknown file %a",name) + else + report("inserting file %a",name) + end + return data or "" +end +local space=patterns.space +local eol=patterns.newline +local equals=P("=")^0 +local open=P("[")*Cg(equals,"init")*P("[")*P("\n")^-1 +local close=P("]")*C(equals)*P("]") +local closeeq=Cmt(close*Cb("init"),function(s,i,a,b) return a==b end) +local longstring=open*(1-closeeq)^0*close +local quoted=patterns.quoted +local emptyline=space^0*eol +local operator1=P("<=")+P(">=")+P("~=")+P("..")+S("/^<>=*+%%") +local operator2=S("*+/") +local operator3=S("-") +local separator=S(",;") +local ignore=(P("]")*space^1*P("=")*space^1*P("]"))/"]=["+(P("=")*space^1*P("{"))/"={"+(P("(")*space^1)/"("+(P("{")*(space+eol)^1*P("}"))/"{}" +local strings=quoted +local longcmt=(emptyline^0*P("--")*longstring*emptyline^0)/"" +local longstr=longstring +local comment=emptyline^0*P("--")*P("-")^0*(1-eol)^0*emptyline^1/"\n" +local pack=((eol+space)^0/"")*operator1*((eol+space)^0/"")+((eol+space)^0/"")*operator2*((space)^0/"")+((eol+space)^1/"")*operator3*((space)^1/"")+((space)^0/"")*separator*((space)^0/"") +local lines=emptyline^2/"\n" +local spaces=(space*space)/" " +local compact=Cs (( + ignore+strings+longcmt+longstr+comment+pack+lines+spaces+1 +)^1 ) +local strip=Cs((emptyline^2/"\n"+1)^0) +local stripreturn=Cs((1-P("return")*space^1*P(1-space-eol)^1*(space+eol)^0*P(-1))^1) +function merger.compact(data) + return lpegmatch(strip,lpegmatch(compact,data)) +end +local function self_compact(data) + local delta=0 + if merger.strip_comment then + local before=#data + data=lpegmatch(compact,data) + data=lpegmatch(strip,data) + local after=#data + delta=before-after + report("original size %s, compacted to %s, stripped %s",before,after,delta) + data=format("-- original size: %s, stripped down to: %s\n\n%s",before,after,data) + end + return lpegmatch(stripreturn,data) or data,delta +end +local function self_save(name,data) + if data~="" then + io.savedata(name,data) + report("saving %s with size %s",name,#data) + end +end +local function self_swap(data,code) + return data~="" and (gsub(data,m_pattern,function() return format(m_format,code) end,1)) or "" +end +local function self_libs(libs,list) + local result,f,frozen,foundpath={},nil,false,nil + result[#result+1]="\n" + if type(libs)=='string' then libs={ libs } end + if type(list)=='string' then list={ list } end + for i=1,#libs do + local lib=libs[i] + for j=1,#list do + local pth=gsub(list[j],"\\","/") + report("checking library path %a",pth) + local name=pth.."/"..lib + if lfs.isfile(name) then + foundpath=pth + end + end + if foundpath then break end + end + if foundpath then + report("using library path %a",foundpath) + local right,wrong,original,stripped={},{},0,0 + for i=1,#libs do + local lib=libs[i] + local fullname=foundpath.."/"..lib + if lfs.isfile(fullname) then + report("using library %a",fullname) + local preloaded=file.nameonly(lib) + local data=io.loaddata(fullname,true) + original=original+#data + local data,delta=self_compact(data) + right[#right+1]=lib + result[#result+1]=m_begin_closure + result[#result+1]=format(m_preloaded,preloaded,preloaded) + result[#result+1]=data + result[#result+1]=m_end_closure + stripped=stripped+delta + else + report("skipping library %a",fullname) + wrong[#wrong+1]=lib + end + end + right=#right>0 and concat(right," ") or "-" + wrong=#wrong>0 and concat(wrong," ") or "-" + report("used libraries: %a",right) + report("skipped libraries: %a",wrong) + report("original bytes: %a",original) + report("stripped bytes: %a",stripped) + result[#result+1]=format(m_report,right,wrong,original,stripped) + else + report("no valid library path found") + end + return concat(result,"\n\n") +end +function merger.selfcreate(libs,list,target) + if target then + self_save(target,self_swap(self_fake(),self_libs(libs,list))) + end +end +function merger.selfmerge(name,libs,list,target) + self_save(target or name,self_swap(self_load(name),self_libs(libs,list))) +end +function merger.selfclean(name) + self_save(name,self_swap(self_load(name),self_nothing())) +end -local P, S, R, C, Cc, Cs, Ct, Cv, V = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.Cc, lpeg.Cs, lpeg.Ct, lpeg.Cv, lpeg.V -dir = dir or { } -local dir = dir -local lfs = lfs +end -- of closure -local attributes = lfs.attributes -local walkdir = lfs.dir -local isdir = lfs.isdir -local isfile = lfs.isfile -local currentdir = lfs.currentdir +do -- create closure to overcome 200 locals limit --- handy +package.loaded["util-tpl"] = package.loaded["util-tpl"] or true -function dir.current() - return (gsub(currentdir(),"\\","/")) +-- original size: 5655, stripped down to: 3242 + +if not modules then modules={} end modules ['util-tpl']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +utilities.templates=utilities.templates or {} +local templates=utilities.templates +local trace_template=false trackers.register("templates.trace",function(v) trace_template=v end) +local report_template=logs.reporter("template") +local tostring=tostring +local format,sub=string.format,string.sub +local P,C,Cs,Carg,lpegmatch=lpeg.P,lpeg.C,lpeg.Cs,lpeg.Carg,lpeg.match +local replacer +local function replacekey(k,t,how,recursive) + local v=t[k] + if not v then + if trace_template then + report_template("unknown key %a",k) + end + return "" + else + v=tostring(v) + if trace_template then + report_template("setting key %a to value %a",k,v) + end + if recursive then + return lpegmatch(replacer,v,1,t,how,recursive) + else + return v + end + end +end +local sqlescape=lpeg.replacer { + { "'","''" }, + { "\\","\\\\" }, + { "\r\n","\\n" }, + { "\r","\\n" }, +} +local sqlquotedescape=lpeg.Cs(lpeg.Cc("'")*sqlescape*lpeg.Cc("'")) +local escapers={ + lua=function(s) + return sub(format("%q",s),2,-2) + end, + sql=function(s) + return lpegmatch(sqlescape,s) + end, +} +local quotedescapers={ + lua=function(s) + return format("%q",s) + end, + sql=function(s) + return lpegmatch(sqlquotedescape,s) + end, +} +lpeg.patterns.sqlescape=sqlescape +lpeg.patterns.sqlescape=sqlquotedescape +local luaescaper=escapers.lua +local quotedluaescaper=quotedescapers.lua +local function replacekeyunquoted(s,t,how,recurse) + local escaper=how and escapers[how] or luaescaper + return escaper(replacekey(s,t,how,recurse)) +end +local function replacekeyquoted(s,t,how,recurse) + local escaper=how and quotedescapers[how] or quotedluaescaper + return escaper(replacekey(s,t,how,recurse)) +end +local single=P("%") +local double=P("%%") +local lquoted=P("%[") +local rquoted=P("]%") +local lquotedq=P("%(") +local rquotedq=P(")%") +local escape=double/'%%' +local nosingle=single/'' +local nodouble=double/'' +local nolquoted=lquoted/'' +local norquoted=rquoted/'' +local nolquotedq=lquotedq/'' +local norquotedq=rquotedq/'' +local key=nosingle*((C((1-nosingle )^1)*Carg(1)*Carg(2)*Carg(3))/replacekey )*nosingle +local quoted=nolquotedq*((C((1-norquotedq)^1)*Carg(1)*Carg(2)*Carg(3))/replacekeyquoted )*norquotedq +local unquoted=nolquoted*((C((1-norquoted )^1)*Carg(1)*Carg(2)*Carg(3))/replacekeyunquoted)*norquoted +local any=P(1) + replacer=Cs((unquoted+quoted+escape+key+any)^0) +local function replace(str,mapping,how,recurse) + if mapping and str then + return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str + else + return str + end +end +templates.replace=replace +function templates.load(filename,mapping,how,recurse) + local data=io.loaddata(filename) or "" + if mapping and next(mapping) then + return replace(data,mapping,how,recurse) + else + return data + end +end +function templates.resolve(t,mapping,how,recurse) + if not mapping then + mapping=t + end + for k,v in next,t do + t[k]=replace(v,mapping,how,recurse) + end + return t end --- optimizing for no find (*) does not save time +end -- of closure -local lfsisdir = isdir +do -- create closure to overcome 200 locals limit -local function isdir(path) - path = gsub(path,"[/\\]+$","") - return lfsisdir(path) -end +package.loaded["util-env"] = package.loaded["util-env"] or true -lfs.isdir = isdir +-- original size: 7702, stripped down to: 4701 -local function globpattern(path,patt,recurse,action) - if path == "/" then - path = path .. "." - elseif not find(path,"/$") then - path = path .. '/' - end - if isdir(path) then -- lfs.isdir does not like trailing / - for name in walkdir(path) do -- lfs.dir accepts trailing / - local full = path .. name - local mode = attributes(full,'mode') - if mode == 'file' then - if find(full,patt) then - action(full) - end - elseif recurse and (mode == "directory") and (name ~= '.') and (name ~= "..") then - globpattern(full,patt,recurse,action) - end +if not modules then modules={} end modules ['util-env']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local allocate,mark=utilities.storage.allocate,utilities.storage.mark +local format,sub,match,gsub,find=string.format,string.sub,string.match,string.gsub,string.find +local unquoted,quoted=string.unquoted,string.quoted +local concat,insert,remove=table.concat,table.insert,table.remove +environment=environment or {} +local environment=environment +os.setlocale(nil,nil) +function os.setlocale() +end +local validengines=allocate { + ["luatex"]=true, + ["luajittex"]=true, +} +local basicengines=allocate { + ["luatex"]="luatex", + ["texlua"]="luatex", + ["texluac"]="luatex", + ["luajittex"]="luajittex", + ["texluajit"]="luajittex", +} +local luaengines=allocate { + ["lua"]=true, + ["luajit"]=true, +} +environment.validengines=validengines +environment.basicengines=basicengines +if not arg then +elseif luaengines[file.removesuffix(arg[-1])] then +elseif validengines[file.removesuffix(arg[0])] then + if arg[1]=="--luaonly" then + arg[-1]=arg[0] + arg[ 0]=arg[2] + for k=3,#arg do + arg[k-2]=arg[k] + end + remove(arg) + remove(arg) + else + end + local originalzero=file.basename(arg[0]) + local specialmapping={ luatools=="base" } + if originalzero~="mtxrun" and originalzero~="mtxrun.lua" then + arg[0]=specialmapping[originalzero] or originalzero + insert(arg,0,"--script") + insert(arg,0,"mtxrun") + end +end +environment.arguments=allocate() +environment.files=allocate() +environment.sortedflags=nil +function environment.initializearguments(arg) + local arguments,files={},{} + environment.arguments,environment.files,environment.sortedflags=arguments,files,nil + for index=1,#arg do + local argument=arg[index] + if index>0 then + local flag,value=match(argument,"^%-+(.-)=(.-)$") + if flag then + flag=gsub(flag,"^c:","") + arguments[flag]=unquoted(value or "") + else + flag=match(argument,"^%-+(.+)") + if flag then + flag=gsub(flag,"^c:","") + arguments[flag]=true + else + files[#files+1]=argument end + end end + end + environment.ownname=file.reslash(environment.ownname or arg[0] or 'unknown.lua') end - -dir.globpattern = globpattern - -local function collectpattern(path,patt,recurse,result) - local ok, scanner - result = result or { } - if path == "/" then - ok, scanner, first = xpcall(function() return walkdir(path..".") end, function() end) -- kepler safe +function environment.setargument(name,value) + environment.arguments[name]=value +end +function environment.getargument(name,partial) + local arguments,sortedflags=environment.arguments,environment.sortedflags + if arguments[name] then + return arguments[name] + elseif partial then + if not sortedflags then + sortedflags=allocate(table.sortedkeys(arguments)) + for k=1,#sortedflags do + sortedflags[k]="^"..sortedflags[k] + end + environment.sortedflags=sortedflags + end + for k=1,#sortedflags do + local v=sortedflags[k] + if find(name,v) then + return arguments[sub(v,2,#v)] + end + end + end + return nil +end +environment.argument=environment.getargument +function environment.splitarguments(separator) + local done,before,after=false,{},{} + local originalarguments=environment.originalarguments + for k=1,#originalarguments do + local v=originalarguments[k] + if not done and v==separator then + done=true + elseif done then + after[#after+1]=v else - ok, scanner, first = xpcall(function() return walkdir(path) end, function() end) -- kepler safe - end - if ok and type(scanner) == "function" then - if not find(path,"/$") then path = path .. '/' end - for name in scanner, first do - local full = path .. name - local attr = attributes(full) - local mode = attr.mode - if mode == 'file' then - if find(full,patt) then - result[name] = attr - end - elseif recurse and (mode == "directory") and (name ~= '.') and (name ~= "..") then - attr.list = collectpattern(full,patt,recurse) - result[name] = attr - end - end + before[#before+1]=v end - return result + end + return before,after end - -dir.collectpattern = collectpattern - -local pattern = Ct { - [1] = (C(P(".") + P("/")^1) + C(R("az","AZ") * P(":") * P("/")^0) + Cc("./")) * V(2) * V(3), - [2] = C(((1-S("*?/"))^0 * P("/"))^0), - [3] = C(P(1)^0) -} - -local filter = Cs ( ( - P("**") / ".*" + - P("*") / "[^/]*" + - P("?") / "[^/]" + - P(".") / "%%." + - P("+") / "%%+" + - P("-") / "%%-" + - P(1) -)^0 ) - -local function glob(str,t) - if type(t) == "function" then - if type(str) == "table" then - for s=1,#str do - glob(str[s],t) - end - elseif isfile(str) then - t(str) - else - local split = lpegmatch(pattern,str) -- we could use the file splitter - if split then - local root, path, base = split[1], split[2], split[3] - local recurse = find(base,"%*%*") - local start = root .. path - local result = lpegmatch(filter,start .. base) - globpattern(start,result,recurse,t) - end - end +function environment.reconstructcommandline(arg,noquote) + arg=arg or environment.originalarguments + if noquote and #arg==1 then + local a=arg[1] + a=resolvers.resolve(a) + a=unquoted(a) + return a + elseif #arg>0 then + local result={} + for i=1,#arg do + local a=arg[i] + a=resolvers.resolve(a) + a=unquoted(a) + a=gsub(a,'"','\\"') + if find(a," ") then + result[#result+1]=quoted(a) + else + result[#result+1]=a + end + end + return concat(result," ") + else + return "" + end +end +if arg then + local newarg,instring={},false + for index=1,#arg do + local argument=arg[index] + if find(argument,"^\"") then + newarg[#newarg+1]=gsub(argument,"^\"","") + if not find(argument,"\"$") then + instring=true + end + elseif find(argument,"\"$") then + newarg[#newarg]=newarg[#newarg].." "..gsub(argument,"\"$","") + instring=false + elseif instring then + newarg[#newarg]=newarg[#newarg].." "..argument else - if type(str) == "table" then - local t = t or { } - for s=1,#str do - glob(str[s],t) - end - return t - elseif isfile(str) then - if t then - t[#t+1] = str - return t - else - return { str } - end - else - local split = lpegmatch(pattern,str) -- we could use the file splitter - if split then - local t = t or { } - local action = action or function(name) t[#t+1] = name end - local root, path, base = split[1], split[2], split[3] - local recurse = find(base,"%*%*") - local start = root .. path - local result = lpegmatch(filter,start .. base) - globpattern(start,result,recurse,action) - return t - else - return { } - end - end + newarg[#newarg+1]=argument end + end + for i=1,-5,-1 do + newarg[i]=arg[i] + end + environment.initializearguments(newarg) + environment.originalarguments=mark(newarg) + environment.rawarguments=mark(arg) + arg={} end -dir.glob = glob +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["luat-env"] = package.loaded["luat-env"] or true + +-- original size: 5874, stripped down to: 4184 -local function globfiles(path,recurse,func,files) -- func == pattern or function - if type(func) == "string" then - local s = func - func = function(name) return find(name,s) end + if not modules then modules={} end modules ['luat-env']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local rawset,rawget,loadfile,assert=rawset,rawget,loadfile,assert +local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) +local report_lua=logs.reporter("resolvers","lua") +local luautilities=utilities.lua +local luasuffixes=luautilities.suffixes +environment=environment or {} +local environment=environment +local mt={ + __index=function(_,k) + if k=="version" then + local version=tex.toks and tex.toks.contextversiontoks + if version and version~="" then + rawset(environment,"version",version) + return version + else + return "unknown" + end + elseif k=="kind" then + local kind=tex.toks and tex.toks.contextkindtoks + if kind and kind~="" then + rawset(environment,"kind",kind) + return kind + else + return "unknown" + end + elseif k=="jobname" or k=="formatname" then + local name=tex and tex[k] + if name or name=="" then + rawset(environment,k,name) + return name + else + return "unknown" + end + elseif k=="outputfilename" then + local name=environment.jobname + rawset(environment,k,name) + return name + end + end +} +setmetatable(environment,mt) +function environment.texfile(filename) + return resolvers.findfile(filename,'tex') +end +function environment.luafile(filename) + local resolved=resolvers.findfile(filename,'tex') or "" + if resolved~="" then + return resolved + end + resolved=resolvers.findfile(filename,'texmfscripts') or "" + if resolved~="" then + return resolved + end + return resolvers.findfile(filename,'luatexlibs') or "" +end +local stripindeed=false directives.register("system.compile.strip",function(v) stripindeed=v end) +local function strippable(filename) + if stripindeed then + local modu=modules[file.nameonly(filename)] + return modu and modu.dataonly + else + return false + end +end +function environment.luafilechunk(filename,silent) + filename=file.replacesuffix(filename,"lua") + local fullname=environment.luafile(filename) + if fullname and fullname~="" then + local data=luautilities.loadedluacode(fullname,strippable,filename) + if trace_locating then + report_lua("loading file %a %s",fullname,not data and "failed" or "succeeded") + elseif not silent then + texio.write("<",data and "+ " or "- ",fullname,">") end - files = files or { } - local noffiles = #files - for name in walkdir(path) do - if find(name,"^%.") then - --- skip - else - local mode = attributes(name,'mode') - if mode == "directory" then - if recurse then - globfiles(path .. "/" .. name,recurse,func,files) - end - elseif mode == "file" then - if not func or func(name) then - noffiles = noffiles + 1 - files[noffiles] = path .. "/" .. name - end - end + return data + else + if trace_locating then + report_lua("unknown file %a",filename) + end + return nil + end +end +function environment.loadluafile(filename,version) + local lucname,luaname,chunk + local basename=file.removesuffix(filename) + if basename==filename then + luaname=file.addsuffix(basename,luasuffixes.lua) + lucname=file.addsuffix(basename,luasuffixes.luc) + else + luaname=basename + lucname=nil + end + local fullname=(lucname and environment.luafile(lucname)) or "" + if fullname~="" then + if trace_locating then + report_lua("loading %a",fullname) + end + chunk=loadfile(fullname) + end + if chunk then + assert(chunk)() + if version then + local v=version + if modules and modules[filename] then + v=modules[filename].version + elseif versions and versions[filename] then + v=versions[filename] + end + if v==version then + return true + else + if trace_locating then + report_lua("version mismatch for %a, lua version %a, luc version %a",filename,v,version) end + environment.loadluafile(filename) + end + else + return true + end + end + fullname=(luaname and environment.luafile(luaname)) or "" + if fullname~="" then + if trace_locating then + report_lua("loading %a",fullname) + end + chunk=loadfile(fullname) + if not chunk then + if trace_locating then + report_lua("unknown file %a",filename) + end + else + assert(chunk)() + return true end - return files + end + return false end -dir.globfiles = globfiles - --- t = dir.glob("c:/data/develop/context/sources/**/????-*.tex") --- t = dir.glob("c:/data/develop/tex/texmf/**/*.tex") --- t = dir.glob("c:/data/develop/context/texmf/**/*.tex") --- t = dir.glob("f:/minimal/tex/**/*") --- print(dir.ls("f:/minimal/tex/**/*")) --- print(dir.ls("*.tex")) - -function dir.ls(pattern) - return concat(glob(pattern),"\n") -end +end -- of closure -local make_indeed = true -- false +do -- create closure to overcome 200 locals limit -local onwindows = os.type == "windows" or find(os.getenv("PATH"),";") +package.loaded["lxml-tab"] = package.loaded["lxml-tab"] or true -if onwindows then +-- original size: 42495, stripped down to: 26647 - function dir.mkdirs(...) - local str, pth, t = "", "", { ... } - for i=1,#t do - local s = t[i] - if s ~= "" then - if str ~= "" then - str = str .. "/" .. s - else - str = s - end - end - end - local first, middle, last - local drive = false - first, middle, last = match(str,"^(//)(//*)(.*)$") - if first then - -- empty network path == local path - else - first, last = match(str,"^(//)/*(.-)$") - if first then - middle, last = match(str,"([^/]+)/+(.-)$") - if middle then - pth = "//" .. middle - else - pth = "//" .. last - last = "" - end - else - first, middle, last = match(str,"^([a-zA-Z]:)(/*)(.-)$") - if first then - pth, drive = first .. middle, true - else - middle, last = match(str,"^(/*)(.-)$") - if not middle then - last = str - end - end - end - end - for s in gmatch(last,"[^/]+") do - if pth == "" then - pth = s - elseif drive then - pth, drive = pth .. s, false - else - pth = pth .. "/" .. s - end - if make_indeed and not isdir(pth) then - lfs.mkdir(pth) - end - end - return pth, (isdir(pth) == true) - end - - -else - - function dir.mkdirs(...) - local str, pth, t = "", "", { ... } - for i=1,#t do - local s = t[i] - if s and s ~= "" then -- we catch nil and false - if str ~= "" then - str = str .. "/" .. s - else - str = s - end - end - end - str = gsub(str,"/+","/") - if find(str,"^/") then - pth = "/" - for s in gmatch(str,"[^/]+") do - local first = (pth == "/") - if first then - pth = pth .. s - else - pth = pth .. "/" .. s - end - if make_indeed and not first and not isdir(pth) then - lfs.mkdir(pth) - end - end - else - pth = "." - for s in gmatch(str,"[^/]+") do - pth = pth .. "/" .. s - if make_indeed and not isdir(pth) then - lfs.mkdir(pth) - end - end - end - return pth, (isdir(pth) == true) - end - - +if not modules then modules={} end modules ['lxml-tab']={ + version=1.001, + comment="this module is the basis for the lxml-* ones", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local trace_entities=false trackers.register("xml.entities",function(v) trace_entities=v end) +local report_xml=logs and logs.reporter("xml","core") or function(...) print(string.format(...)) end +xml=xml or {} +local xml=xml +local concat,remove,insert=table.concat,table.remove,table.insert +local type,next,setmetatable,getmetatable,tonumber=type,next,setmetatable,getmetatable,tonumber +local lower,find,match,gsub=string.lower,string.find,string.match,string.gsub +local utfchar=utf.char +local lpegmatch=lpeg.match +local P,S,R,C,V,C,Cs=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.V,lpeg.C,lpeg.Cs +local formatters=string.formatters +xml.xmlns=xml.xmlns or {} +local check=P(false) +local parse=check +function xml.registerns(namespace,pattern) + check=check+C(P(lower(pattern)))/namespace + parse=P { P(check)+1*V(1) } end - -dir.makedirs = dir.mkdirs - --- we can only define it here as it uses dir.current - -if onwindows then - - function dir.expandname(str) -- will be merged with cleanpath and collapsepath - local first, nothing, last = match(str,"^(//)(//*)(.*)$") - if first then - first = dir.current() .. "/" - end - if not first then - first, last = match(str,"^(//)/*(.*)$") - end - if not first then - first, last = match(str,"^([a-zA-Z]:)(.*)$") - if first and not find(last,"^/") then - local d = currentdir() - if lfs.chdir(first) then - first = dir.current() - end - lfs.chdir(d) - end - end - if not first then - first, last = dir.current(), str - end - last = gsub(last,"//","/") - last = gsub(last,"/%./","/") - last = gsub(last,"^/*","") - first = gsub(first,"/*$","") - if last == "" or last == "." then - return first - else - return first .. "/" .. last - end - end - -else - - function dir.expandname(str) -- will be merged with cleanpath and collapsepath - if not find(str,"^/") then - str = currentdir() .. "/" .. str - end - str = gsub(str,"//","/") - str = gsub(str,"/%./","/") - str = gsub(str,"(.)/%.$","%1") - return str - end - +function xml.checkns(namespace,url) + local ns=lpegmatch(parse,lower(url)) + if ns and namespace~=ns then + xml.xmlns[namespace]=ns + end end - -file.expandname = dir.expandname -- for convenience - -local stack = { } - -function dir.push(newdir) - insert(stack,lfs.currentdir()) +function xml.resolvens(url) + return lpegmatch(parse,lower(url)) or "" +end +local nsremap,resolvens=xml.xmlns,xml.resolvens +local stack={} +local top={} +local dt={} +local at={} +local xmlns={} +local errorstr=nil +local entities={} +local strip=false +local cleanup=false +local utfize=false +local resolve_predefined=false +local unify_predefined=false +local dcache={} +local hcache={} +local acache={} +local mt={} +local function initialize_mt(root) + mt={ __index=root } end - -function dir.pop() - local d = remove(stack) - if d then - lfs.chdir(d) - end - return d +function xml.setproperty(root,k,v) + getmetatable(root).__index[k]=v end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['l-boolean'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" +function xml.checkerror(top,toclose) + return "" +end +local function add_attribute(namespace,tag,value) + if cleanup and #value>0 then + value=cleanup(value) + end + if tag=="xmlns" then + xmlns[#xmlns+1]=resolvens(value) + at[tag]=value + elseif namespace=="" then + at[tag]=value + elseif namespace=="xmlns" then + xml.checkns(tag,value) + at["xmlns:"..tag]=value + else + at[namespace..":"..tag]=value + end +end +local function add_empty(spacing,namespace,tag) + if #spacing>0 then + dt[#dt+1]=spacing + end + local resolved=namespace=="" and xmlns[#xmlns] or nsremap[namespace] or namespace + top=stack[#stack] + dt=top.dt + local t={ ns=namespace or "",rn=resolved,tg=tag,at=at,dt={},__p__=top } + dt[#dt+1]=t + setmetatable(t,mt) + if at.xmlns then + remove(xmlns) + end + at={} +end +local function add_begin(spacing,namespace,tag) + if #spacing>0 then + dt[#dt+1]=spacing + end + local resolved=namespace=="" and xmlns[#xmlns] or nsremap[namespace] or namespace + top={ ns=namespace or "",rn=resolved,tg=tag,at=at,dt={},__p__=stack[#stack] } + setmetatable(top,mt) + dt=top.dt + stack[#stack+1]=top + at={} +end +local function add_end(spacing,namespace,tag) + if #spacing>0 then + dt[#dt+1]=spacing + end + local toclose=remove(stack) + top=stack[#stack] + if #stack<1 then + errorstr=formatters["unable to close %s %s"](tag,xml.checkerror(top,toclose) or "") + elseif toclose.tg~=tag then + errorstr=formatters["unable to close %s with %s %s"](toclose.tg,tag,xml.checkerror(top,toclose) or "") + end + dt=top.dt + dt[#dt+1]=toclose + if toclose.at.xmlns then + remove(xmlns) + end +end +local function add_text(text) + if cleanup and #text>0 then + dt[#dt+1]=cleanup(text) + else + dt[#dt+1]=text + end +end +local function add_special(what,spacing,text) + if #spacing>0 then + dt[#dt+1]=spacing + end + if strip and (what=="@cm@" or what=="@dt@") then + else + dt[#dt+1]={ special=true,ns="",tg=what,dt={ text } } + end +end +local function set_message(txt) + errorstr="garbage at the end of the file: "..gsub(txt,"([ \n\r\t]*)","") +end +local reported_attribute_errors={} +local function attribute_value_error(str) + if not reported_attribute_errors[str] then + report_xml("invalid attribute value %a",str) + reported_attribute_errors[str]=true + at._error_=str + end + return str +end +local function attribute_specification_error(str) + if not reported_attribute_errors[str] then + report_xml("invalid attribute specification %a",str) + reported_attribute_errors[str]=true + at._error_=str + end + return str +end +xml.placeholders={ + unknown_dec_entity=function(str) return str=="" and "&error;" or formatters["&%s;"](str) end, + unknown_hex_entity=function(str) return formatters["&#x%s;"](str) end, + unknown_any_entity=function(str) return formatters["&#x%s;"](str) end, } - -local type, tonumber = type, tonumber - -boolean = boolean or { } -local boolean = boolean - -function boolean.tonumber(b) - if b then return 1 else return 0 end -- test and return or return +local placeholders=xml.placeholders +local function fromhex(s) + local n=tonumber(s,16) + if n then + return utfchar(n) + else + return formatters["h:%s"](s),true + end end - -function toboolean(str,tolerant) - if tolerant then - local tstr = type(str) - if tstr == "string" then - return str == "true" or str == "yes" or str == "on" or str == "1" or str == "t" - elseif tstr == "number" then - return tonumber(str) ~= 0 - elseif tstr == "nil" then - return false - else - return str - end - elseif str == "true" then - return true - elseif str == "false" then - return false +local function fromdec(s) + local n=tonumber(s) + if n then + return utfchar(n) + else + return formatters["d:%s"](s),true + end +end +local rest=(1-P(";"))^0 +local many=P(1)^0 +local parsedentity=P("&")*(P("#x")*(rest/fromhex)+P("#")*(rest/fromdec))*P(";")*P(-1)+(P("#x")*(many/fromhex)+P("#")*(many/fromdec)) +local predefined_unified={ + [38]="&", + [42]=""", + [47]="'", + [74]="<", + [76]=">", +} +local predefined_simplified={ + [38]="&",amp="&", + [42]='"',quot='"', + [47]="'",apos="'", + [74]="<",lt="<", + [76]=">",gt=">", +} +local nofprivates=0xF0000 +local privates_u={ + [ [[&]] ]="&", + [ [["]] ]=""", + [ [[']] ]="'", + [ [[<]] ]="<", + [ [[>]] ]=">", +} +local privates_p={} +local privates_n={ +} +local escaped=utf.remapper(privates_u) +local function unescaped(s) + local p=privates_n[s] + if not p then + nofprivates=nofprivates+1 + p=utfchar(nofprivates) + privates_n[s]=p + s="&"..s..";" + privates_u[p]=s + privates_p[p]=s + end + return p +end +local unprivatized=utf.remapper(privates_p) +xml.privatetoken=unescaped +xml.unprivatized=unprivatized +xml.privatecodes=privates_n +local function handle_hex_entity(str) + local h=hcache[str] + if not h then + local n=tonumber(str,16) + h=unify_predefined and predefined_unified[n] + if h then + if trace_entities then + report_xml("utfize, converting hex entity &#x%s; into %a",str,h) + end + elseif utfize then + h=(n and utfchar(n)) or xml.unknown_hex_entity(str) or "" + if not n then + report_xml("utfize, ignoring hex entity &#x%s;",str) + elseif trace_entities then + report_xml("utfize, converting hex entity &#x%s; into %a",str,h) + end else - return str + if trace_entities then + report_xml("found entity &#x%s;",str) + end + h="&#x"..str..";" end + hcache[str]=h + end + return h end - -string.toboolean = toboolean - -function string.is_boolean(str,default) - if type(str) == "string" then - if str == "true" or str == "yes" or str == "on" or str == "t" then - return true - elseif str == "false" or str == "no" or str == "off" or str == "f" then - return false - end +local function handle_dec_entity(str) + local d=dcache[str] + if not d then + local n=tonumber(str) + d=unify_predefined and predefined_unified[n] + if d then + if trace_entities then + report_xml("utfize, converting dec entity &#%s; into %a",str,d) + end + elseif utfize then + d=(n and utfchar(n)) or placeholders.unknown_dec_entity(str) or "" + if not n then + report_xml("utfize, ignoring dec entity &#%s;",str) + elseif trace_entities then + report_xml("utfize, converting dec entity &#%s; into %a",str,d) + end + else + if trace_entities then + report_xml("found entity &#%s;",str) + end + d="&#"..str..";" end - return default + dcache[str]=d + end + return d end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['l-unicode'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -if not unicode then - - unicode = { utf8 = { } } - - local floor, char = math.floor, string.char - - function unicode.utf8.utfchar(n) - if n < 0x80 then - return char(n) - elseif n < 0x800 then - return char( - 0xC0 + floor(n/0x40), - 0x80 + (n % 0x40) - ) - elseif n < 0x10000 then - return char( - 0xE0 + floor(n/0x1000), - 0x80 + (floor(n/0x40) % 0x40), - 0x80 + (n % 0x40) - ) - elseif n < 0x40000 then - return char( - 0xF0 + floor(n/0x40000), - 0x80 + floor(n/0x1000), - 0x80 + (floor(n/0x40) % 0x40), - 0x80 + (n % 0x40) - ) +xml.parsedentitylpeg=parsedentity +local function handle_any_entity(str) + if resolve then + local a=acache[str] + if not a then + a=resolve_predefined and predefined_simplified[str] + if a then + if trace_entities then + report_xml("resolving entity &%s; to predefined %a",str,a) + end + else + if type(resolve)=="function" then + a=resolve(str) or entities[str] else - -- return char( - -- 0xF1 + floor(n/0x1000000), - -- 0x80 + floor(n/0x40000), - -- 0x80 + floor(n/0x1000), - -- 0x80 + (floor(n/0x40) % 0x40), - -- 0x80 + (n % 0x40) - -- ) - return "?" + a=entities[str] end - end - + if a then + if type(a)=="function" then + if trace_entities then + report_xml("expanding entity &%s; to function call",str) + end + a=a(str) or "" + end + a=lpegmatch(parsedentity,a) or a + if trace_entities then + report_xml("resolving entity &%s; to internal %a",str,a) + end + else + local unknown_any_entity=placeholders.unknown_any_entity + if unknown_any_entity then + a=unknown_any_entity(str) or "" + end + if a then + if trace_entities then + report_xml("resolving entity &%s; to external %s",str,a) + end + else + if trace_entities then + report_xml("keeping entity &%s;",str) + end + if str=="" then + a="&error;" + else + a="&"..str..";" + end + end + end + end + acache[str]=a + elseif trace_entities then + if not acache[str] then + report_xml("converting entity &%s; to %a",str,a) + acache[str]=a + end + end + return a + else + local a=acache[str] + if not a then + a=resolve_predefined and predefined_simplified[str] + if a then + acache[str]=a + if trace_entities then + report_xml("entity &%s; becomes %a",str,a) + end + elseif str=="" then + if trace_entities then + report_xml("invalid entity &%s;",str) + end + a="&error;" + acache[str]=a + else + if trace_entities then + report_xml("entity &%s; is made private",str) + end + a=unescaped(str) + acache[str]=a + end + end + return a + end end - -local unicode = unicode - -utf = utf or unicode.utf8 - -local concat = table.concat -local utfchar, utfbyte, utfgsub = utf.char, utf.byte, utf.gsub -local char, byte, find, bytepairs, utfvalues, format = string.char, string.byte, string.find, string.bytepairs, string.utfvalues, string.format -local type = type - -local utfsplitlines = string.utfsplitlines - --- 0 EF BB BF UTF-8 --- 1 FF FE UTF-16-little-endian --- 2 FE FF UTF-16-big-endian --- 3 FF FE 00 00 UTF-32-little-endian --- 4 00 00 FE FF UTF-32-big-endian - -unicode.utfname = { - [0] = 'utf-8', - [1] = 'utf-16-le', - [2] = 'utf-16-be', - [3] = 'utf-32-le', - [4] = 'utf-32-be' +local function handle_end_entity(chr) + report_xml("error in entity, %a found instead of %a",chr,";") +end +local space=S(' \r\n\t') +local open=P('<') +local close=P('>') +local squote=S("'") +local dquote=S('"') +local equal=P('=') +local slash=P('/') +local colon=P(':') +local semicolon=P(';') +local ampersand=P('&') +local valid=R('az','AZ','09')+S('_-.') +local name_yes=C(valid^1)*colon*C(valid^1) +local name_nop=C(P(true))*C(valid^1) +local name=name_yes+name_nop +local utfbom=lpeg.patterns.utfbom +local spacing=C(space^0) +local anyentitycontent=(1-open-semicolon-space-close)^0 +local hexentitycontent=R("AF","af","09")^0 +local decentitycontent=R("09")^0 +local parsedentity=P("#")/""*( + P("x")/""*(hexentitycontent/handle_hex_entity)+(decentitycontent/handle_dec_entity) + )+(anyentitycontent/handle_any_entity) +local entity=ampersand/""*parsedentity*((semicolon/"")+#(P(1)/handle_end_entity)) +local text_unparsed=C((1-open)^1) +local text_parsed=Cs(((1-open-ampersand)^1+entity)^1) +local somespace=space^1 +local optionalspace=space^0 +local value=(squote*Cs((entity+(1-squote))^0)*squote)+(dquote*Cs((entity+(1-dquote))^0)*dquote) +local endofattributes=slash*close+close +local whatever=space*name*optionalspace*equal +local wrongvalue=Cs(P(entity+(1-space-endofattributes))^1)/attribute_value_error +local attributevalue=value+wrongvalue +local attribute=(somespace*name*optionalspace*equal*optionalspace*attributevalue)/add_attribute +local attributes=(attribute+somespace^-1*(((1-endofattributes)^1)/attribute_specification_error))^0 +local parsedtext=text_parsed/add_text +local unparsedtext=text_unparsed/add_text +local balanced=P { "["*((1-S"[]")+V(1))^0*"]" } +local emptyelement=(spacing*open*name*attributes*optionalspace*slash*close)/add_empty +local beginelement=(spacing*open*name*attributes*optionalspace*close)/add_begin +local endelement=(spacing*open*slash*name*optionalspace*close)/add_end +local begincomment=open*P("!--") +local endcomment=P("--")*close +local begininstruction=open*P("?") +local endinstruction=P("?")*close +local begincdata=open*P("![CDATA[") +local endcdata=P("]]")*close +local someinstruction=C((1-endinstruction)^0) +local somecomment=C((1-endcomment )^0) +local somecdata=C((1-endcdata )^0) +local function normalentity(k,v ) entities[k]=v end +local function systementity(k,v,n) entities[k]=v end +local function publicentity(k,v,n) entities[k]=v end +local begindoctype=open*P("!DOCTYPE") +local enddoctype=close +local beginset=P("[") +local endset=P("]") +local doctypename=C((1-somespace-close)^0) +local elementdoctype=optionalspace*P("=5.1 where %z is depricated - -function unicode.utftype(f) - local str = f:read(4) - if not str then - f:seek('set') - return 0 - -- elseif find(str,"^%z%z\254\255") then -- depricated - -- elseif find(str,"^\000\000\254\255") then -- not permitted and bugged - elseif find(str,"\000\000\254\255",1,true) then -- seems to work okay (TH) - return 4 - -- elseif find(str,"^\255\254%z%z") then -- depricated - -- elseif find(str,"^\255\254\000\000") then -- not permitted and bugged - elseif find(str,"\255\254\000\000",1,true) then -- seems to work okay (TH) - return 3 - elseif find(str,"^\254\255") then - f:seek('set',2) - return 2 - elseif find(str,"^\255\254") then - f:seek('set',2) - return 1 - elseif find(str,"^\239\187\191") then - f:seek('set',3) - return 0 +local grammar_unparsed_text=P { "preamble", + preamble=utfbom^0*instruction^0*(doctype+comment+instruction)^0*V("parent")*trailer, + parent=beginelement*V("children")^0*endelement, + children=unparsedtext+V("parent")+emptyelement+comment+cdata+instruction, +} +local function _xmlconvert_(data,settings) + settings=settings or {} + strip=settings.strip_cm_and_dt + utfize=settings.utfize_entities + resolve=settings.resolve_entities + resolve_predefined=settings.resolve_predefined_entities + unify_predefined=settings.unify_predefined_entities + cleanup=settings.text_cleanup + entities=settings.entities or {} + if utfize==nil then + settings.utfize_entities=true + utfize=true + end + if resolve_predefined==nil then + settings.resolve_predefined_entities=true + resolve_predefined=true + end + stack,top,at,xmlns,errorstr={},{},{},{},nil + acache,hcache,dcache={},{},{} + reported_attribute_errors={} + if settings.parent_root then + mt=getmetatable(settings.parent_root) + else + initialize_mt(top) + end + stack[#stack+1]=top + top.dt={} + dt=top.dt + if not data or data=="" then + errorstr="empty xml file" + elseif utfize or resolve then + if lpegmatch(grammar_parsed_text,data) then + errorstr="" else - f:seek('set') - return 0 + errorstr="invalid xml file - parsed text" end + elseif type(data)=="string" then + if lpegmatch(grammar_unparsed_text,data) then + errorstr="" + else + errorstr="invalid xml file - unparsed text" + end + else + errorstr="invalid xml file - no text at all" + end + local result + if errorstr and errorstr~="" then + result={ dt={ { ns="",tg="error",dt={ errorstr },at={},er=true } } } + setmetatable(stack,mt) + local errorhandler=settings.error_handler + if errorhandler==false then + else + errorhandler=errorhandler or xml.errorhandler + if errorhandler then + local currentresource=settings.currentresource + if currentresource and currentresource~="" then + xml.errorhandler(formatters["load error in [%s]: %s"](currentresource,errorstr)) + else + xml.errorhandler(formatters["load error: %s"](errorstr)) + end + end + end + else + result=stack[1] + end + if not settings.no_root then + result={ special=true,ns="",tg='@rt@',dt=result.dt,at={},entities=entities,settings=settings } + setmetatable(result,mt) + local rdt=result.dt + for k=1,#rdt do + local v=rdt[k] + if type(v)=="table" and not v.special then + result.ri=k + v.__p__=result + break + end + end + end + if errorstr and errorstr~="" then + result.error=true + end + result.statistics={ + entities={ + decimals=dcache, + hexadecimals=hcache, + names=acache, + } + } + strip,utfize,resolve,resolve_predefined=nil,nil,nil,nil + unify_predefined,cleanup,entities=nil,nil,nil + stack,top,at,xmlns,errorstr=nil,nil,nil,nil,nil + acache,hcache,dcache=nil,nil,nil + reported_attribute_errors,mt,errorhandler=nil,nil,nil + return result end - - - -local function utf16_to_utf8_be(t) - if type(t) == "string" then - t = utfsplitlines(str) - end - local result = { } -- we reuse result - for i=1,#t do - local r, more = 0, 0 - for left, right in bytepairs(t[i]) do - if right then - local now = 256*left + right - if more > 0 then - now = (more-0xD800)*0x400 + (now-0xDC00) + 0x10000 -- the 0x10000 smells wrong - more = 0 - r = r + 1 - result[r] = utfchar(now) - elseif now >= 0xD800 and now <= 0xDBFF then - more = now - else - r = r + 1 - result[r] = utfchar(now) - end - end - end - t[i] = concat(result,"",1,r) -- we reused tmp, hence t - end - return t +function xmlconvert(data,settings) + local ok,result=pcall(function() return _xmlconvert_(data,settings) end) + if ok then + return result + else + return _xmlconvert_("",settings) + end +end +xml.convert=xmlconvert +function xml.inheritedconvert(data,xmldata) + local settings=xmldata.settings + if settings then + settings.parent_root=xmldata + end + local xc=xmlconvert(data,settings) + return xc end - -local function utf16_to_utf8_le(t) - if type(t) == "string" then - t = utfsplitlines(str) - end - local result = { } -- we reuse result - for i=1,#t do - local r, more = 0, 0 - for left, right in bytepairs(t[i]) do - if right then - local now = 256*right + left - if more > 0 then - now = (more-0xD800)*0x400 + (now-0xDC00) + 0x10000 -- the 0x10000 smells wrong - more = 0 - r = r + 1 - result[r] = utfchar(now) - elseif now >= 0xD800 and now <= 0xDBFF then - more = now - else - r = r + 1 - result[r] = utfchar(now) - end - end - end - t[i] = concat(result,"",1,r) -- we reused tmp, hence t - end - return t +function xml.is_valid(root) + return root and root.dt and root.dt[1] and type(root.dt[1])=="table" and not root.dt[1].er end - -local function utf32_to_utf8_be(t) - if type(t) == "string" then - t = utfsplitlines(t) +function xml.package(tag,attributes,data) + local ns,tg=match(tag,"^(.-):?([^:]+)$") + local t={ ns=ns,tg=tg,dt=data or "",at=attributes or {} } + setmetatable(t,mt) + return t +end +function xml.is_valid(root) + return root and not root.error +end +xml.errorhandler=report_xml +function xml.load(filename,settings) + local data="" + if type(filename)=="string" then + local f=io.open(filename,'r') + if f then + data=f:read("*all") + f:close() + end + elseif filename then + data=filename:read("*all") + end + if settings then + settings.currentresource=filename + local result=xmlconvert(data,settings) + settings.currentresource=nil + return result + else + return xmlconvert(data,{ currentresource=filename }) + end +end +local no_root={ no_root=true } +function xml.toxml(data) + if type(data)=="string" then + local root={ xmlconvert(data,no_root) } + return (#root>1 and root) or root[1] + else + return data + end +end +local function copy(old,tables) + if old then + tables=tables or {} + local new={} + if not tables[old] then + tables[old]=new end - local result = { } -- we reuse result - for i=1,#t do - local r, more = 0, -1 - for a,b in bytepairs(t[i]) do - if a and b then - if more < 0 then - more = 256*256*256*a + 256*256*b - else - r = r + 1 - result[t] = utfchar(more + 256*a + b) - more = -1 - end - else - break - end - end - t[i] = concat(result,"",1,r) + for k,v in next,old do + new[k]=(type(v)=="table" and (tables[v] or copy(v,tables))) or v end - return t -end - -local function utf32_to_utf8_le(t) - if type(t) == "string" then - t = utfsplitlines(t) + local mt=getmetatable(old) + if mt then + setmetatable(new,mt) end - local result = { } -- we reuse result - for i=1,#t do - local r, more = 0, -1 - for a,b in bytepairs(t[i]) do - if a and b then - if more < 0 then - more = 256*b + a - else - r = r + 1 - result[t] = utfchar(more + 256*256*256*b + 256*256*a) - more = -1 - end - else - break - end + return new + else + return {} + end +end +xml.copy=copy +function xml.checkbom(root) + if root.ri then + local dt=root.dt + for k=1,#dt do + local v=dt[k] + if type(v)=="table" and v.special and v.tg=="@pi@" and find(v.dt[1],"xml.*version=") then + return + end + end + insert(dt,1,{ special=true,ns="",tg="@pi@",dt={ "xml version='1.0' standalone='yes'" } } ) + insert(dt,2,"\n" ) + end +end +local function verbose_element(e,handlers) + local handle=handlers.handle + local serialize=handlers.serialize + local ens,etg,eat,edt,ern=e.ns,e.tg,e.at,e.dt,e.rn + local ats=eat and next(eat) and {} + if ats then + for k,v in next,eat do + ats[#ats+1]=formatters['%s=%q'](k,escaped(v)) + end + end + if ern and trace_entities and ern~=ens then + ens=ern + end + if ens~="" then + if edt and #edt>0 then + if ats then + handle("<",ens,":",etg," ",concat(ats," "),">") + else + handle("<",ens,":",etg,">") + end + for i=1,#edt do + local e=edt[i] + if type(e)=="string" then + handle(escaped(e)) + else + serialize(e,handlers) + end + end + handle("") + else + if ats then + handle("<",ens,":",etg," ",concat(ats," "),"/>") + else + handle("<",ens,":",etg,"/>") + end + end + else + if edt and #edt>0 then + if ats then + handle("<",etg," ",concat(ats," "),">") + else + handle("<",etg,">") + end + for i=1,#edt do + local e=edt[i] + if type(e)=="string" then + handle(escaped(e)) + else + serialize(e,handlers) end - t[i] = concat(result,"",1,r) + end + handle("") + else + if ats then + handle("<",etg," ",concat(ats," "),"/>") + else + handle("<",etg,"/>") + end end - return t + end end - -unicode.utf32_to_utf8_be = utf32_to_utf8_be -unicode.utf32_to_utf8_le = utf32_to_utf8_le -unicode.utf16_to_utf8_be = utf16_to_utf8_be -unicode.utf16_to_utf8_le = utf16_to_utf8_le - -function unicode.utf8_to_utf8(t) - return type(t) == "string" and utfsplitlines(t) or t +local function verbose_pi(e,handlers) + handlers.handle("") end - -function unicode.utf16_to_utf8(t,endian) - return endian and utf16_to_utf8_be(t) or utf16_to_utf8_le(t) or t +local function verbose_comment(e,handlers) + handlers.handle("") end - -function unicode.utf32_to_utf8(t,endian) - return endian and utf32_to_utf8_be(t) or utf32_to_utf8_le(t) or t +local function verbose_cdata(e,handlers) + handlers.handle("") end - -local function little(c) - local b = byte(c) - if b < 0x10000 then - return char(b%256,b/256) - else - b = b - 0x10000 - local b1, b2 = b/1024 + 0xD800, b%1024 + 0xDC00 - return char(b1%256,b1/256,b2%256,b2/256) - end +local function verbose_doctype(e,handlers) + handlers.handle("") end - -local function big(c) - local b = byte(c) - if b < 0x10000 then - return char(b/256,b%256) - else - b = b - 0x10000 - local b1, b2 = b/1024 + 0xD800, b%1024 + 0xDC00 - return char(b1/256,b1%256,b2/256,b2%256) - end +local function verbose_root(e,handlers) + handlers.serialize(e.dt,handlers) end - -function unicode.utf8_to_utf16(str,littleendian) - if littleendian then - return char(255,254) .. utfgsub(str,".",little) +local function verbose_text(e,handlers) + handlers.handle(escaped(e)) +end +local function verbose_document(e,handlers) + local serialize=handlers.serialize + local functions=handlers.functions + for i=1,#e do + local ei=e[i] + if type(ei)=="string" then + functions["@tx@"](ei,handlers) else - return char(254,255) .. utfgsub(str,".",big) + serialize(ei,handlers) end + end end - -function unicode.utfcodes(str) - local t, n = { }, 0 - for u in utfvalues(str) do - n = n + 1 - t[n] = format("0x%04X",u) - end - return concat(t,separator or " ") +local function serialize(e,handlers,...) + local initialize=handlers.initialize + local finalize=handlers.finalize + local functions=handlers.functions + if initialize then + local state=initialize(...) + if not state==true then + return state + end + end + local etg=e.tg + if etg then + (functions[etg] or functions["@el@"])(e,handlers) + else + functions["@dc@"](e,handlers) + end + if finalize then + return finalize() + end end - -function unicode.ustring(s) - return format("U+%05X",type(s) == "number" and s or utfbyte(s)) +local function xserialize(e,handlers) + local functions=handlers.functions + local etg=e.tg + if etg then + (functions[etg] or functions["@el@"])(e,handlers) + else + functions["@dc@"](e,handlers) + end +end +local handlers={} +local function newhandlers(settings) + local t=table.copy(handlers[settings and settings.parent or "verbose"] or {}) + if settings then + for k,v in next,settings do + if type(v)=="table" then + local tk=t[k] if not tk then tk={} t[k]=tk end + for kk,vv in next,v do + tk[kk]=vv + end + else + t[k]=v + end + end + if settings.name then + handlers[settings.name]=t + end + end + utilities.storage.mark(t) + return t +end +local nofunction=function() end +function xml.sethandlersfunction(handler,name,fnc) + handler.functions[name]=fnc or nofunction end - -function unicode.xstring(s) - return format("0x%05X",type(s) == "number" and s or utfbyte(s)) +function xml.gethandlersfunction(handler,name) + return handler.functions[name] end - - -local lpegmatch = lpeg.match -local patterns = lpeg.patterns -local utftype = patterns.utftype - -function unicode.filetype(data) - return data and lpegmatch(utftype,data) or "unknown" -end - -local toentities = lpeg.Cs ( - ( - patterns.utf8one - + ( - patterns.utf8two - + patterns.utf8three - + patterns.utf8four - ) / function(s) local b = utfbyte(s) if b < 127 then return s else return format("&#%X;",b) end end - )^0 -) - -patterns.toentities = toentities - -function utf.toentities(str) - return lpegmatch(toentities,str) -end - - - - -local P, C, R, Cs = lpeg.P, lpeg.C, lpeg.R, lpeg.Cs - -local one = P(1) -local two = C(1) * C(1) -local four = C(R(utfchar(0xD8),utfchar(0xFF))) * C(1) * C(1) * C(1) - --- actually one of them is already utf ... sort of useless this one - -local pattern = P("\254\255") * Cs( ( - four / function(a,b,c,d) - local ab = 0xFF * byte(a) + byte(b) - local cd = 0xFF * byte(c) + byte(d) - return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000) - end - + two / function(a,b) - return utfchar(byte(a)*256 + byte(b)) - end - + one - )^1 ) - + P("\255\254") * Cs( ( - four / function(b,a,d,c) - local ab = 0xFF * byte(a) + byte(b) - local cd = 0xFF * byte(c) + byte(d) - return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000) - end - + two / function(b,a) - return utfchar(byte(a)*256 + byte(b)) - end - + one - )^1 ) - -function string.toutf(s) - return lpegmatch(pattern,s) or s -- todo: utf32 -end - -local validatedutf = Cs ( - ( - patterns.utf8one - + patterns.utf8two - + patterns.utf8three - + patterns.utf8four - + P(1) / "�" - )^0 -) - -patterns.validatedutf = validatedutf - -function string.validutf(str) - return lpegmatch(validatedutf,str) +function xml.gethandlers(name) + return handlers[name] end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['l-math'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" +newhandlers { + name="verbose", + initialize=false, + finalize=false, + serialize=xserialize, + handle=print, + functions={ + ["@dc@"]=verbose_document, + ["@dt@"]=verbose_doctype, + ["@rt@"]=verbose_root, + ["@el@"]=verbose_element, + ["@pi@"]=verbose_pi, + ["@cm@"]=verbose_comment, + ["@cd@"]=verbose_cdata, + ["@tx@"]=verbose_text, + } } - -local floor, sin, cos, tan = math.floor, math.sin, math.cos, math.tan - -if not math.round then - function math.round(x) return floor(x + 0.5) end -end - -if not math.div then - function math.div(n,m) return floor(n/m) end +local result +local xmlfilehandler=newhandlers { + name="file", + initialize=function(name) + result=io.open(name,"wb") + return result + end, + finalize=function() + result:close() + return true + end, + handle=function(...) + result:write(...) + end, +} +function xml.save(root,name) + serialize(root,xmlfilehandler,name) end - -if not math.mod then - function math.mod(n,m) return n % m end +local result +local xmlstringhandler=newhandlers { + name="string", + initialize=function() + result={} + return result + end, + finalize=function() + return concat(result) + end, + handle=function(...) + result[#result+1]=concat {... } + end, +} +local function xmltostring(root) + if not root then + return "" + elseif type(root)=="string" then + return root + else + return serialize(root,xmlstringhandler) or "" + end end - -local pipi = 2*math.pi/360 - -if not math.sind then - function math.sind(d) return sin(d*pipi) end - function math.cosd(d) return cos(d*pipi) end - function math.tand(d) return tan(d*pipi) end +local function __tostring(root) + return (root and xmltostring(root)) or "" end - -if not math.odd then - function math.odd (n) return n % 2 ~= 0 end - function math.even(n) return n % 2 == 0 end +initialize_mt=function(root) + mt={ __tostring=__tostring,__index=root } end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['util-tab'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -utilities = utilities or {} -utilities.tables = utilities.tables or { } -local tables = utilities.tables - -local format, gmatch, rep = string.format, string.gmatch, string.rep -local concat, insert, remove = table.concat, table.insert, table.remove -local setmetatable, getmetatable, tonumber, tostring = setmetatable, getmetatable, tonumber, tostring -local type, next, rawset, tonumber = type, next, rawset, tonumber - -function tables.definetable(target) -- defines undefined tables - local composed, t, n = nil, { }, 0 - for name in gmatch(target,"([^%.]+)") do - n = n + 1 - if composed then - composed = composed .. "." .. name - else - composed = name - end - t[n] = format("%s = %s or { }",composed,composed) +xml.defaulthandlers=handlers +xml.newhandlers=newhandlers +xml.serialize=serialize +xml.tostring=xmltostring +local function xmlstring(e,handle) + if not handle or (e.special and e.tg~="@rt@") then + elseif e.tg then + local edt=e.dt + if edt then + for i=1,#edt do + xmlstring(edt[i],handle) + end end - return concat(t,"\n") + else + handle(e) + end end - -function tables.accesstable(target,root) - local t = root or _G - for name in gmatch(target,"([^%.]+)") do - t = t[name] - if not t then - return - end +xml.string=xmlstring +function xml.settings(e) + while e do + local s=e.settings + if s then + return s + else + e=e.__p__ end - return t + end + return nil end - -function tables.migratetable(target,v,root) - local t = root or _G - local names = string.split(target,".") - for i=1,#names-1 do - local name = names[i] - t[name] = t[name] or { } - t = t[name] - if not t then - return - end +function xml.root(e) + local r=e + while e do + e=e.__p__ + if e then + r=e end - t[names[#names]] = v + end + return r end - -function tables.removevalue(t,value) -- todo: n - if value then - for i=1,#t do - if t[i] == value then - remove(t,i) - -- remove all, so no: return - end - end - end +function xml.parent(root) + return root.__p__ end - -function tables.insertbeforevalue(t,value,extra) - for i=1,#t do - if t[i] == extra then - remove(t,i) - end - end - for i=1,#t do - if t[i] == value then - insert(t,i,extra) - return - end - end - insert(t,1,extra) +function xml.body(root) + return root.ri and root.dt[root.ri] or root end - -function tables.insertaftervalue(t,value,extra) - for i=1,#t do - if t[i] == extra then - remove(t,i) - end - end - for i=1,#t do - if t[i] == value then - insert(t,i+1,extra) - return - end - end - insert(t,#t+1,extra) +function xml.name(root) + if not root then + return "" + end + local ns=root.ns + local tg=root.tg + if ns=="" then + return tg + else + return ns..":"..tg + end end - --- experimental - -local function toxml(t,d,result,step) - for k, v in table.sortedpairs(t) do - if type(v) == "table" then - if type(k) == "number" then - result[#result+1] = format("%s",d,k) - toxml(v,d..step,result,step) - result[#result+1] = format("%s",d,k) - else - result[#result+1] = format("%s<%s>",d,k) - toxml(v,d..step,result,step) - result[#result+1] = format("%s",d,k) - end - elseif type(k) == "number" then - result[#result+1] = format("%s%s",d,k,v,k) - else - result[#result+1] = format("%s<%s>%s",d,k,tostring(v),k) - end - end +function xml.erase(dt,k) + if dt then + if k then + dt[k]="" + else for k=1,#dt do + dt[1]={ "" } + end end + end end - -function table.toxml(t,name,nobanner,indent,spaces) - local noroot = name == false - local result = (nobanner or noroot) and { } or { "" } - local indent = rep(" ",indent or 0) - local spaces = rep(" ",spaces or 1) - if noroot then - toxml( t, inndent, result, spaces) - else - toxml( { [name or "root"] = t }, indent, result, spaces) - end - return concat(result,"\n") +function xml.assign(dt,k,root) + if dt and k then + dt[k]=type(root)=="table" and xml.body(root) or root + return dt[k] + else + return xml.body(root) + end +end +function xml.tocdata(e,wrapper) + local whatever=type(e)=="table" and xmltostring(e.dt) or e or "" + if wrapper then + whatever=formatters["<%s>%s"](wrapper,whatever,wrapper) + end + local t={ special=true,ns="",tg="@cd@",at={},rn="",dt={ whatever },__p__=e } + setmetatable(t,getmetatable(e)) + e.dt={ t } end - --- also experimental - --- encapsulate(table,utilities.tables) --- encapsulate(table,utilities.tables,true) --- encapsulate(table,true) - -function tables.encapsulate(core,capsule,protect) - if type(capsule) ~= "table" then - protect = true - capsule = { } - end - for key, value in next, core do - if capsule[key] then - print(format("\ninvalid inheritance '%s' in '%s': %s",key,tostring(core))) - os.exit() - else - capsule[key] = value - end - end - if protect then - for key, value in next, core do - core[key] = nil +function xml.makestandalone(root) + if root.ri then + local dt=root.dt + for k=1,#dt do + local v=dt[k] + if type(v)=="table" and v.special and v.tg=="@pi@" then + local txt=v.dt[1] + if find(txt,"xml.*version=") then + v.dt[1]=txt.." standalone='yes'" + break end - setmetatable(core, { - __index = capsule, - __newindex = function(t,key,value) - if capsule[key] then - print(format("\ninvalid overload '%s' in '%s'",key,tostring(core))) - os.exit() - else - rawset(t,key,value) - end - end - } ) + end end + end + return root +end +function xml.kind(e) + local dt=e and e.dt + if dt then + local n=#dt + if n==1 then + local d=dt[1] + if d.special then + local tg=d.tg + if tg=="@cd@" then + return "cdata" + elseif tg=="@cm" then + return "comment" + elseif tg=="@pi@" then + return "instruction" + elseif tg=="@dt@" then + return "declaration" + end + elseif type(d)=="string" then + return "text" + end + return "element" + elseif n>0 then + return "mixed" + end + end + return "empty" end @@ -4351,9885 +9008,3514 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['util-sto'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local setmetatable, getmetatable = setmetatable, getmetatable +package.loaded["lxml-lpt"] = package.loaded["lxml-lpt"] or true -utilities = utilities or { } -utilities.storage = utilities.storage or { } -local storage = utilities.storage +-- original size: 48956, stripped down to: 30516 -function storage.mark(t) - if not t then - texio.write_nl("fatal error: storage cannot be marked") - return -- os.exit() +if not modules then modules={} end modules ['lxml-lpt']={ + version=1.001, + comment="this module is the basis for the lxml-* ones", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local concat,remove,insert=table.concat,table.remove,table.insert +local type,next,tonumber,tostring,setmetatable,load,select=type,next,tonumber,tostring,setmetatable,load,select +local format,upper,lower,gmatch,gsub,find,rep=string.format,string.upper,string.lower,string.gmatch,string.gsub,string.find,string.rep +local lpegmatch,lpegpatterns=lpeg.match,lpeg.patterns +local setmetatableindex=table.setmetatableindex +local formatters=string.formatters +local trace_lpath=false if trackers then trackers.register("xml.path",function(v) trace_lpath=v end) end +local trace_lparse=false if trackers then trackers.register("xml.parse",function(v) trace_lparse=v end) end +local trace_lprofile=false if trackers then trackers.register("xml.profile",function(v) trace_lpath=v trace_lparse=v trace_lprofile=v end) end +local report_lpath=logs.reporter("xml","lpath") +local xml=xml +local lpathcalls=0 function xml.lpathcalls () return lpathcalls end +local lpathcached=0 function xml.lpathcached() return lpathcached end +xml.functions=xml.functions or {} +local functions=xml.functions +xml.expressions=xml.expressions or {} +local expressions=xml.expressions +xml.finalizers=xml.finalizers or {} +local finalizers=xml.finalizers +xml.specialhandler=xml.specialhandler or {} +local specialhandler=xml.specialhandler +lpegpatterns.xml=lpegpatterns.xml or {} +local xmlpatterns=lpegpatterns.xml +finalizers.xml=finalizers.xml or {} +finalizers.tex=finalizers.tex or {} +local function fallback (t,name) + local fn=finalizers[name] + if fn then + t[name]=fn + else + report_lpath("unknown sub finalizer %a",name) + fn=function() end + end + return fn +end +setmetatableindex(finalizers.xml,fallback) +setmetatableindex(finalizers.tex,fallback) +xml.defaultprotocol="xml" +local apply_axis={} +apply_axis['root']=function(list) + local collected={} + for l=1,#list do + local ll=list[l] + local rt=ll + while ll do + ll=ll.__p__ + if ll then + rt=ll + end + end + collected[l]=rt + end + return collected +end +apply_axis['self']=function(list) + return list +end +apply_axis['child']=function(list) + local collected,c={},0 + for l=1,#list do + local ll=list[l] + local dt=ll.dt + if dt then + local en=0 + for k=1,#dt do + local dk=dt[k] + if dk.tg then + c=c+1 + collected[c]=dk + dk.ni=k + en=en+1 + dk.ei=en + end + end + ll.en=en + end + end + return collected +end +local function collect(list,collected,c) + local dt=list.dt + if dt then + local en=0 + for k=1,#dt do + local dk=dt[k] + if dk.tg then + c=c+1 + collected[c]=dk + dk.ni=k + en=en+1 + dk.ei=en + c=collect(dk,collected,c) + end + end + list.en=en + end + return c +end +apply_axis['descendant']=function(list) + local collected,c={},0 + for l=1,#list do + c=collect(list[l],collected,c) + end + return collected +end +local function collect(list,collected,c) + local dt=list.dt + if dt then + local en=0 + for k=1,#dt do + local dk=dt[k] + if dk.tg then + c=c+1 + collected[c]=dk + dk.ni=k + en=en+1 + dk.ei=en + c=collect(dk,collected,c) + end + end + list.en=en + end + return c +end +apply_axis['descendant-or-self']=function(list) + local collected,c={},0 + for l=1,#list do + local ll=list[l] + if ll.special~=true then + c=c+1 + collected[c]=ll + end + c=collect(ll,collected,c) + end + return collected +end +apply_axis['ancestor']=function(list) + local collected,c={},0 + for l=1,#list do + local ll=list[l] + while ll do + ll=ll.__p__ + if ll then + c=c+1 + collected[c]=ll + end + end + end + return collected +end +apply_axis['ancestor-or-self']=function(list) + local collected,c={},0 + for l=1,#list do + local ll=list[l] + c=c+1 + collected[c]=ll + while ll do + ll=ll.__p__ + if ll then + c=c+1 + collected[c]=ll + end + end + end + return collected +end +apply_axis['parent']=function(list) + local collected,c={},0 + for l=1,#list do + local pl=list[l].__p__ + if pl then + c=c+1 + collected[c]=pl + end + end + return collected +end +apply_axis['attribute']=function(list) + return {} +end +apply_axis['namespace']=function(list) + return {} +end +apply_axis['following']=function(list) + return {} +end +apply_axis['preceding']=function(list) + return {} +end +apply_axis['following-sibling']=function(list) + local collected,c={},0 + for l=1,#list do + local ll=list[l] + local p=ll.__p__ + local d=p.dt + for i=ll.ni+1,#d do + local di=d[i] + if type(di)=="table" then + c=c+1 + collected[c]=di + end + end + end + return collected +end +apply_axis['preceding-sibling']=function(list) + local collected,c={},0 + for l=1,#list do + local ll=list[l] + local p=ll.__p__ + local d=p.dt + for i=1,ll.ni-1 do + local di=d[i] + if type(di)=="table" then + c=c+1 + collected[c]=di + end + end + end + return collected +end +apply_axis['reverse-sibling']=function(list) + local collected,c={},0 + for l=1,#list do + local ll=list[l] + local p=ll.__p__ + local d=p.dt + for i=ll.ni-1,1,-1 do + local di=d[i] + if type(di)=="table" then + c=c+1 + collected[c]=di + end + end + end + return collected +end +apply_axis['auto-descendant-or-self']=apply_axis['descendant-or-self'] +apply_axis['auto-descendant']=apply_axis['descendant'] +apply_axis['auto-child']=apply_axis['child'] +apply_axis['auto-self']=apply_axis['self'] +apply_axis['initial-child']=apply_axis['child'] +local function apply_nodes(list,directive,nodes) + local maxn=#nodes + if maxn==3 then + local nns,ntg=nodes[2],nodes[3] + if not nns and not ntg then + if directive then + return list + else + return {} + end + else + local collected,c,m,p={},0,0,nil + if not nns then + for l=1,#list do + local ll=list[l] + local ltg=ll.tg + if ltg then + if directive then + if ntg==ltg then + local llp=ll.__p__;if llp~=p then p,m=llp,1 else m=m+1 end + c=c+1 + collected[c],ll.mi=ll,m + end + elseif ntg~=ltg then + local llp=ll.__p__;if llp~=p then p,m=llp,1 else m=m+1 end + c=c+1 + collected[c],ll.mi=ll,m + end + end + end + elseif not ntg then + for l=1,#list do + local ll=list[l] + local lns=ll.rn or ll.ns + if lns then + if directive then + if lns==nns then + local llp=ll.__p__;if llp~=p then p,m=llp,1 else m=m+1 end + c=c+1 + collected[c],ll.mi=ll,m + end + elseif lns~=nns then + local llp=ll.__p__;if llp~=p then p,m=llp,1 else m=m+1 end + c=c+1 + collected[c],ll.mi=ll,m + end + end + end + else + for l=1,#list do + local ll=list[l] + local ltg=ll.tg + if ltg then + local lns=ll.rn or ll.ns + local ok=ltg==ntg and lns==nns + if directive then + if ok then + local llp=ll.__p__;if llp~=p then p,m=llp,1 else m=m+1 end + c=c+1 + collected[c],ll.mi=ll,m + end + elseif not ok then + local llp=ll.__p__;if llp~=p then p,m=llp,1 else m=m+1 end + c=c+1 + collected[c],ll.mi=ll,m + end + end + end + end + return collected + end + else + local collected,c,m,p={},0,0,nil + for l=1,#list do + local ll=list[l] + local ltg=ll.tg + if ltg then + local lns=ll.rn or ll.ns + local ok=false + for n=1,maxn,3 do + local nns,ntg=nodes[n+1],nodes[n+2] + ok=(not ntg or ltg==ntg) and (not nns or lns==nns) + if ok then + break + end + end + if directive then + if ok then + local llp=ll.__p__;if llp~=p then p,m=llp,1 else m=m+1 end + c=c+1 + collected[c],ll.mi=ll,m + end + elseif not ok then + local llp=ll.__p__;if llp~=p then p,m=llp,1 else m=m+1 end + c=c+1 + collected[c],ll.mi=ll,m + end + end end - local m = getmetatable(t) - if not m then - m = { } - setmetatable(t,m) + return collected + end +end +local quit_expression=false +local function apply_expression(list,expression,order) + local collected,c={},0 + quit_expression=false + for l=1,#list do + local ll=list[l] + if expression(list,ll,l,order) then + c=c+1 + collected[c]=ll + end + if quit_expression then + break + end + end + return collected +end +local P,V,C,Cs,Cc,Ct,R,S,Cg,Cb=lpeg.P,lpeg.V,lpeg.C,lpeg.Cs,lpeg.Cc,lpeg.Ct,lpeg.R,lpeg.S,lpeg.Cg,lpeg.Cb +local spaces=S(" \n\r\t\f")^0 +local lp_space=S(" \n\r\t\f") +local lp_any=P(1) +local lp_noequal=P("!=")/"~="+P("<=")+P(">=")+P("==") +local lp_doequal=P("=")/"==" +local lp_or=P("|")/" or " +local lp_and=P("&")/" and " +local lp_builtin=P ( + P("text")/"(ll.dt[1] or '')"+ + P("content")/"ll.dt"+ + P("name")/"((ll.ns~='' and ll.ns..':'..ll.tg) or ll.tg)"+P("tag")/"ll.tg"+P("position")/"l"+ + P("firstindex")/"1"+P("lastindex")/"(#ll.__p__.dt or 1)"+P("firstelement")/"1"+P("lastelement")/"(ll.__p__.en or 1)"+P("first")/"1"+P("last")/"#list"+P("rootposition")/"order"+P("order")/"order"+P("element")/"(ll.ei or 1)"+P("index")/"(ll.ni or 1)"+P("match")/"(ll.mi or 1)"+ + P("ns")/"ll.ns" + )*((spaces*P("(")*spaces*P(")"))/"") +local lp_attribute=(P("@")+P("attribute::"))/""*Cc("(ll.at and ll.at['")*((R("az","AZ")+S("-_:"))^1)*Cc("'])") +lp_fastpos_p=P("+")^0*R("09")^1*P(-1)/"l==%0" +lp_fastpos_n=P("-")*R("09")^1*P(-1)/"(%0<0 and (#list+%0==l))" +local lp_fastpos=lp_fastpos_n+lp_fastpos_p +local lp_reserved=C("and")+C("or")+C("not")+C("div")+C("mod")+C("true")+C("false") +local lp_lua_function=Cs((R("az","AZ","__")^1*(P(".")*R("az","AZ","__")^1)^1)*("("))/"%0" +local lp_function=C(R("az","AZ","__")^1)*P("(")/function(t) + if expressions[t] then + return "expr."..t.."(" + else + return "expr.error(" + end +end +local lparent=P("(") +local rparent=P(")") +local noparent=1-(lparent+rparent) +local nested=P{lparent*(noparent+V(1))^0*rparent} +local value=P(lparent*C((noparent+nested)^0)*rparent) +local lp_child=Cc("expr.child(ll,'")*R("az","AZ","--","__")^1*Cc("')") +local lp_number=S("+-")*R("09")^1 +local lp_string=Cc("'")*R("az","AZ","--","__")^1*Cc("'") +local lp_content=(P("'")*(1-P("'"))^0*P("'")+P('"')*(1-P('"'))^0*P('"')) +local cleaner +local lp_special=(C(P("name")+P("text")+P("tag")+P("count")+P("child")))*value/function(t,s) + if expressions[t] then + s=s and s~="" and lpegmatch(cleaner,s) + if s and s~="" then + return "expr."..t.."(ll,"..s..")" + else + return "expr."..t.."(ll)" end - m.__storage__ = true - return t + else + return "expr.error("..t..")" + end end - -function storage.allocate(t) - t = t or { } - local m = getmetatable(t) - if not m then - m = { } - setmetatable(t,m) +local content=lp_builtin+lp_attribute+lp_special+lp_noequal+lp_doequal+lp_or+lp_and+lp_reserved+lp_lua_function+lp_function+lp_content+ + lp_child+lp_any +local converter=Cs ( + lp_fastpos+(P { lparent*(V(1))^0*rparent+content } )^0 +) +cleaner=Cs (( + lp_reserved+lp_number+lp_string+1 )^1 ) +local template_e=[[ + local expr = xml.expressions + return function(list,ll,l,order) + return %s end - m.__storage__ = true - return t +]] +local template_f_y=[[ + local finalizer = xml.finalizers['%s']['%s'] + return function(collection) + return finalizer(collection,%s) + end +]] +local template_f_n=[[ + return xml.finalizers['%s']['%s'] +]] +local register_self={ kind="axis",axis="self" } +local register_parent={ kind="axis",axis="parent" } +local register_descendant={ kind="axis",axis="descendant" } +local register_child={ kind="axis",axis="child" } +local register_descendant_or_self={ kind="axis",axis="descendant-or-self" } +local register_root={ kind="axis",axis="root" } +local register_ancestor={ kind="axis",axis="ancestor" } +local register_ancestor_or_self={ kind="axis",axis="ancestor-or-self" } +local register_attribute={ kind="axis",axis="attribute" } +local register_namespace={ kind="axis",axis="namespace" } +local register_following={ kind="axis",axis="following" } +local register_following_sibling={ kind="axis",axis="following-sibling" } +local register_preceding={ kind="axis",axis="preceding" } +local register_preceding_sibling={ kind="axis",axis="preceding-sibling" } +local register_reverse_sibling={ kind="axis",axis="reverse-sibling" } +local register_auto_descendant_or_self={ kind="axis",axis="auto-descendant-or-self" } +local register_auto_descendant={ kind="axis",axis="auto-descendant" } +local register_auto_self={ kind="axis",axis="auto-self" } +local register_auto_child={ kind="axis",axis="auto-child" } +local register_initial_child={ kind="axis",axis="initial-child" } +local register_all_nodes={ kind="nodes",nodetest=true,nodes={ true,false,false } } +local skip={} +local function errorrunner_e(str,cnv) + if not skip[str] then + report_lpath("error in expression: %s => %s",str,cnv) + skip[str]=cnv or str + end + return false end - -function storage.marked(t) - local m = getmetatable(t) - return m and m.__storage__ +local function errorrunner_f(str,arg) + report_lpath("error in finalizer: %s(%s)",str,arg or "") + return false end - -function storage.checked(t) - if not t then - texio.write_nl("fatal error: storage has not been allocated") - return -- os.exit() - end - return t +local function register_nodes(nodetest,nodes) + return { kind="nodes",nodetest=nodetest,nodes=nodes } end - - -function storage.setinitializer(data,initialize) - local m = getmetatable(data) or { } - m.__index = function(data,k) - m.__index = nil -- so that we can access the entries during initializing - initialize() - return data[k] - end - setmetatable(data, m) +local function register_expression(expression) + local converted=lpegmatch(converter,expression) + local runner=load(format(template_e,converted)) + runner=(runner and runner()) or function() errorrunner_e(expression,converted) end + return { kind="expression",expression=expression,converted=converted,evaluator=runner } end - -local keyisvalue = { __index = function(t,k) - t[k] = k - return k -end } - -function storage.sparse(t) - t = t or { } - setmetatable(t,keyisvalue) - return t +local function register_finalizer(protocol,name,arguments) + local runner + if arguments and arguments~="" then + runner=load(format(template_f_y,protocol or xml.defaultprotocol,name,arguments)) + else + runner=load(format(template_f_n,protocol or xml.defaultprotocol,name)) + end + runner=(runner and runner()) or function() errorrunner_f(name,arguments) end + return { kind="finalizer",name=name,arguments=arguments,finalizer=runner } +end +local expression=P { "ex", + ex="["*C((V("sq")+V("dq")+(1-S("[]"))+V("ex"))^0)*"]", + sq="'"*(1-S("'"))^0*"'", + dq='"'*(1-S('"'))^0*'"', +} +local arguments=P { "ar", + ar="("*Cs((V("sq")+V("dq")+V("nq")+P(1-P(")")))^0)*")", + nq=((1-S("),'\""))^1)/function(s) return format("%q",s) end, + sq=P("'")*(1-P("'"))^0*P("'"), + dq=P('"')*(1-P('"'))^0*P('"'), +} +local function register_error(str) + return { kind="error",error=format("unparsed: %s",str) } +end +local special_1=P("*")*Cc(register_auto_descendant)*Cc(register_all_nodes) +local special_2=P("/")*Cc(register_auto_self) +local special_3=P("")*Cc(register_auto_self) +local no_nextcolon=P(-1)+#(1-P(":")) +local no_nextlparent=P(-1)+#(1-P("(")) +local pathparser=Ct { "patterns", + patterns=spaces*V("protocol")*spaces*( + (V("special")*spaces*P(-1) )+(V("initial")*spaces*V("step")*spaces*(P("/")*spaces*V("step")*spaces)^0 ) + ), + protocol=Cg(V("letters"),"protocol")*P("://")+Cg(Cc(nil),"protocol"), + step=((V("shortcuts")+P("/")+V("axis"))*spaces*V("nodes")^0+V("error"))*spaces*V("expressions")^0*spaces*V("finalizer")^0, + axis=V("descendant")+V("child")+V("parent")+V("self")+V("root")+V("ancestor")+V("descendant_or_self")+V("following_sibling")+V("following")+V("reverse_sibling")+V("preceding_sibling")+V("preceding")+V("ancestor_or_self")+#(1-P(-1))*Cc(register_auto_child), + special=special_1+special_2+special_3, + initial=(P("/")*spaces*Cc(register_initial_child))^-1, + error=(P(1)^1)/register_error, + shortcuts_a=V("s_descendant_or_self")+V("s_descendant")+V("s_child")+V("s_parent")+V("s_self")+V("s_root")+V("s_ancestor"), + shortcuts=V("shortcuts_a")*(spaces*"/"*spaces*V("shortcuts_a"))^0, + s_descendant_or_self=(P("***/")+P("/"))*Cc(register_descendant_or_self), + s_descendant=P("**")*Cc(register_descendant), + s_child=P("*")*no_nextcolon*Cc(register_child ), + s_parent=P("..")*Cc(register_parent ), + s_self=P("." )*Cc(register_self ), + s_root=P("^^")*Cc(register_root ), + s_ancestor=P("^")*Cc(register_ancestor ), + descendant=P("descendant::")*Cc(register_descendant ), + child=P("child::")*Cc(register_child ), + parent=P("parent::")*Cc(register_parent ), + self=P("self::")*Cc(register_self ), + root=P('root::')*Cc(register_root ), + ancestor=P('ancestor::')*Cc(register_ancestor ), + descendant_or_self=P('descendant-or-self::')*Cc(register_descendant_or_self ), + ancestor_or_self=P('ancestor-or-self::')*Cc(register_ancestor_or_self ), + following=P('following::')*Cc(register_following ), + following_sibling=P('following-sibling::')*Cc(register_following_sibling ), + preceding=P('preceding::')*Cc(register_preceding ), + preceding_sibling=P('preceding-sibling::')*Cc(register_preceding_sibling ), + reverse_sibling=P('reverse-sibling::')*Cc(register_reverse_sibling ), + nodes=(V("nodefunction")*spaces*P("(")*V("nodeset")*P(")")+V("nodetest")*V("nodeset"))/register_nodes, + expressions=expression/register_expression, + letters=R("az")^1, + name=(1-S("/[]()|:*!"))^1, + negate=P("!")*Cc(false), + nodefunction=V("negate")+P("not")*Cc(false)+Cc(true), + nodetest=V("negate")+Cc(true), + nodename=(V("negate")+Cc(true))*spaces*((V("wildnodename")*P(":")*V("wildnodename"))+(Cc(false)*V("wildnodename"))), + wildnodename=(C(V("name"))+P("*")*Cc(false))*no_nextlparent, + nodeset=spaces*Ct(V("nodename")*(spaces*P("|")*spaces*V("nodename"))^0)*spaces, + finalizer=(Cb("protocol")*P("/")^-1*C(V("name"))*arguments*P(-1))/register_finalizer, +} +xmlpatterns.pathparser=pathparser +local cache={} +local function nodesettostring(set,nodetest) + local t={} + for i=1,#set,3 do + local directive,ns,tg=set[i],set[i+1],set[i+2] + if not ns or ns=="" then ns="*" end + if not tg or tg=="" then tg="*" end + tg=(tg=="@rt@" and "[root]") or format("%s:%s",ns,tg) + t[i]=(directive and tg) or format("not(%s)",tg) + end + if nodetest==false then + return format("not(%s)",concat(t,"|")) + else + return concat(t,"|") + end end - --- table namespace ? - -local function f_empty () return "" end -- t,k -local function f_self (t,k) t[k] = k return k end -local function f_ignore() end -- t,k,v - -local t_empty = { __index = f_empty } -local t_self = { __index = f_self } -local t_ignore = { __newindex = f_ignore } - -function table.setmetatableindex(t,f) - local m = getmetatable(t) - if m then - if f == "empty" then - m.__index = f_empty - elseif f == "key" then - m.__index = f_self - else - m.__index = f - end - else - if f == "empty" then - setmetatable(t, t_empty) - elseif f == "key" then - setmetatable(t, t_self) - else - setmetatable(t,{ __index = f }) - end - end - return t +local function tagstostring(list) + if #list==0 then + return "no elements" + else + local t={} + for i=1,#list do + local li=list[i] + local ns,tg=li.ns,li.tg + if not ns or ns=="" then ns="*" end + if not tg or tg=="" then tg="*" end + t[i]=(tg=="@rt@" and "[root]") or format("%s:%s",ns,tg) + end + return concat(t," ") + end +end +xml.nodesettostring=nodesettostring +local lpath +local lshowoptions={ functions=false } +local function lshow(parsed) + if type(parsed)=="string" then + parsed=lpath(parsed) + end + report_lpath("%s://%s => %s",parsed.protocol or xml.defaultprotocol,parsed.pattern, + table.serialize(parsed,false,lshowoptions)) end - -function table.setmetatablenewindex(t,f) - local m = getmetatable(t) - if m then - if f == "ignore" then - m.__newindex = f_ignore - else - m.__newindex = f - end +xml.lshow=lshow +local function add_comment(p,str) + local pc=p.comment + if not pc then + p.comment={ str } + else + pc[#pc+1]=str + end +end +lpath=function (pattern) + lpathcalls=lpathcalls+1 + if type(pattern)=="table" then + return pattern + else + local parsed=cache[pattern] + if parsed then + lpathcached=lpathcached+1 else - if f == "ignore" then - setmetatable(t, t_ignore) + parsed=lpegmatch(pathparser,pattern) + if parsed then + parsed.pattern=pattern + local np=#parsed + if np==0 then + parsed={ pattern=pattern,register_self,state="parsing error" } + report_lpath("parsing error in pattern: %s",pattern) + lshow(parsed) else - setmetatable(t,{ __newindex = f }) - end + local pi=parsed[1] + if pi.axis=="auto-child" then + if false then + add_comment(parsed,"auto-child replaced by auto-descendant-or-self") + parsed[1]=register_auto_descendant_or_self + else + add_comment(parsed,"auto-child replaced by auto-descendant") + parsed[1]=register_auto_descendant + end + elseif pi.axis=="initial-child" and np>1 and parsed[2].axis then + add_comment(parsed,"initial-child removed") + remove(parsed,1) + end + local np=#parsed + if np>1 then + local pnp=parsed[np] + if pnp.kind=="nodes" and pnp.nodetest==true then + local nodes=pnp.nodes + if nodes[1]==true and nodes[2]==false and nodes[3]==false then + add_comment(parsed,"redundant final wildcard filter removed") + remove(parsed,np) + end + end + end + end + else + parsed={ pattern=pattern } + end + cache[pattern]=parsed + if trace_lparse and not trace_lprofile then + lshow(parsed) + end end - return t + return parsed + end end - -function table.setmetatablecall(t,f) - local m = getmetatable(t) - if m then - m.__call = f - else - setmetatable(t,{ __call = f }) +xml.lpath=lpath +local profiled={} xml.profiled=profiled +local function profiled_apply(list,parsed,nofparsed,order) + local p=profiled[parsed.pattern] + if p then + p.tested=p.tested+1 + else + p={ tested=1,matched=0,finalized=0 } + profiled[parsed.pattern]=p + end + local collected=list + for i=1,nofparsed do + local pi=parsed[i] + local kind=pi.kind + if kind=="axis" then + collected=apply_axis[pi.axis](collected) + elseif kind=="nodes" then + collected=apply_nodes(collected,pi.nodetest,pi.nodes) + elseif kind=="expression" then + collected=apply_expression(collected,pi.evaluator,order) + elseif kind=="finalizer" then + collected=pi.finalizer(collected) + p.matched=p.matched+1 + p.finalized=p.finalized+1 + return collected + end + if not collected or #collected==0 then + local pn=i %s",(collected and #collected) or 0,pi.expression,pi.converted) + elseif kind=="finalizer" then + collected=pi.finalizer(collected) + report_lpath("% 10i : fi : %s : %s(%s)",(type(collected)=="table" and #collected) or 0,parsed.protocol or xml.defaultprotocol,pi.name,pi.arguments or "") + return collected + end + if not collected or #collected==0 then + local pn=i 0 then - utilities.report("merge: used libraries: %s",concat(right," ")) - end - if #wrong > 0 then - utilities.report("merge: skipped libraries: %s",concat(wrong," ")) - end - else - utilities.report("merge: no valid library path found") - end - return concat(result, "\n\n") -end - -function merger.selfcreate(libs,list,target) - if target then - self_save(target,self_swap(self_fake(),self_libs(libs,list))) - end -end - -function merger.selfmerge(name,libs,list,target) - self_save(target or name,self_swap(self_load(name),self_libs(libs,list))) -end - -function merger.selfclean(name) - self_save(name,self_swap(self_load(name),self_nothing())) -end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['util-lua'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -utilities = utilities or {} -utilities.lua = utilities.lua or { } -utilities.report = logs and logs.reporter("system") or print - -local function stupidcompile(luafile,lucfile) - local data = io.loaddata(luafile) - if data and data ~= "" then - data = string.dump(data) - if data and data ~= "" then - io.savedata(lucfile,data) - end - end -end - -function utilities.lua.compile(luafile,lucfile,cleanup,strip,fallback) -- defaults: cleanup=false strip=true - utilities.report("lua: compiling %s into %s",luafile,lucfile) - os.remove(lucfile) - local command = "-o " .. string.quoted(lucfile) .. " " .. string.quoted(luafile) - if strip ~= false then - command = "-s " .. command - end - local done = os.spawn("texluac " .. command) == 0 -- or os.spawn("luac " .. command) == 0 - if not done and fallback then - utilities.report("lua: dumping %s into %s (unstripped)",luafile,lucfile) - stupidcompile(luafile,lucfile) -- maybe use the stripper we have elsewhere - cleanup = false -- better see how worse it is - end - if done and cleanup == true and lfs.isfile(lucfile) and lfs.isfile(luafile) then - utilities.report("lua: removing %s",luafile) - os.remove(luafile) - end - return done -end - - - - - - - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['util-prs'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local P, R, V, C, Ct, Cs, Carg = lpeg.P, lpeg.R, lpeg.V, lpeg.C, lpeg.Ct, lpeg.Cs, lpeg.Carg -local lpegmatch = lpeg.match -local concat, format, gmatch, find = table.concat, string.format, string.gmatch, string.find -local tostring, type, next = tostring, type, next - -utilities = utilities or {} -utilities.parsers = utilities.parsers or { } -local parsers = utilities.parsers -parsers.patterns = parsers.patterns or { } - -local setmetatableindex = table.setmetatableindex -local sortedhash = table.sortedhash - --- we could use a Cf Cg construct - -local escape, left, right = P("\\"), P('{'), P('}') - -lpeg.patterns.balanced = P { - [1] = ((escape * (left+right)) + (1 - (left+right)) + V(2))^0, - [2] = left * V(1) * right -} - -local space = P(' ') -local equal = P("=") -local comma = P(",") -local lbrace = P("{") -local rbrace = P("}") -local nobrace = 1 - (lbrace+rbrace) -local nested = P { lbrace * (nobrace + V(1))^0 * rbrace } -local spaces = space^0 -local argument = Cs((lbrace/"") * ((nobrace + nested)^0) * (rbrace/"")) -local content = (1-P(-1))^0 - -lpeg.patterns.nested = nested -- no capture -lpeg.patterns.argument = argument -- argument after e.g. = -lpeg.patterns.content = content -- rest after e.g = - -local value = P(lbrace * C((nobrace + nested)^0) * rbrace) + C((nested + (1-comma))^0) - -local key = C((1-equal-comma)^1) -local pattern_a = (space+comma)^0 * (key * equal * value + key * C("")) -local pattern_c = (space+comma)^0 * (key * equal * value) - -local key = C((1-space-equal-comma)^1) -local pattern_b = spaces * comma^0 * spaces * (key * ((spaces * equal * spaces * value) + C(""))) - --- "a=1, b=2, c=3, d={a{b,c}d}, e=12345, f=xx{a{b,c}d}xx, g={}" : outer {} removes, leading spaces ignored - -local hash = { } - -local function set(key,value) - hash[key] = value -end - -local function set(key,value) - hash[key] = value -end - -local pattern_a_s = (pattern_a/set)^1 -local pattern_b_s = (pattern_b/set)^1 -local pattern_c_s = (pattern_c/set)^1 - -parsers.patterns.settings_to_hash_a = pattern_a_s -parsers.patterns.settings_to_hash_b = pattern_b_s -parsers.patterns.settings_to_hash_c = pattern_c_s - -function parsers.make_settings_to_hash_pattern(set,how) - if how == "strict" then - return (pattern_c/set)^1 - elseif how == "tolerant" then - return (pattern_b/set)^1 - else - return (pattern_a/set)^1 - end -end - -function parsers.settings_to_hash(str,existing) - if str and str ~= "" then - hash = existing or { } - lpegmatch(pattern_a_s,str) - return hash - else - return { } - end -end - -function parsers.settings_to_hash_tolerant(str,existing) - if str and str ~= "" then - hash = existing or { } - lpegmatch(pattern_b_s,str) - return hash - else - return { } - end -end - -function parsers.settings_to_hash_strict(str,existing) - if str and str ~= "" then - hash = existing or { } - lpegmatch(pattern_c_s,str) - return next(hash) and hash - else - return nil - end -end - -local separator = comma * space^0 -local value = P(lbrace * C((nobrace + nested)^0) * rbrace) + C((nested + (1-comma))^0) -local pattern = Ct(value*(separator*value)^0) - --- "aap, {noot}, mies" : outer {} removes, leading spaces ignored - -parsers.patterns.settings_to_array = pattern - --- we could use a weak table as cache - -function parsers.settings_to_array(str,strict) - if not str or str == "" then - return { } - elseif strict then - if find(str,"{") then - return lpegmatch(pattern,str) - else - return { str } - end - else - return lpegmatch(pattern,str) - end -end - -local function set(t,v) - t[#t+1] = v -end - -local value = P(Carg(1)*value) / set -local pattern = value*(separator*value)^0 * Carg(1) - -function parsers.add_settings_to_array(t,str) - return lpegmatch(pattern,str,nil,t) -end - -function parsers.hash_to_string(h,separator,yes,no,strict,omit) - if h then - local t, tn, s = { }, 0, table.sortedkeys(h) - omit = omit and table.tohash(omit) - for i=1,#s do - local key = s[i] - if not omit or not omit[key] then - local value = h[key] - if type(value) == "boolean" then - if yes and no then - if value then - tn = tn + 1 - t[tn] = key .. '=' .. yes - elseif not strict then - tn = tn + 1 - t[tn] = key .. '=' .. no - end - elseif value or not strict then - tn = tn + 1 - t[tn] = key .. '=' .. tostring(value) - end - else - tn = tn + 1 - t[tn] = key .. '=' .. value - end - end - end - return concat(t,separator or ",") - else - return "" - end -end - -function parsers.array_to_string(a,separator) - if a then - return concat(a,separator or ",") - else - return "" - end -end - -function parsers.settings_to_set(str,t) -- tohash? -- todo: lpeg -- duplicate anyway - t = t or { } --- for s in gmatch(str,"%s*([^, ]+)") do -- space added - for s in gmatch(str,"[^, ]+") do -- space added - t[s] = true - end - return t -end - -function parsers.simple_hash_to_string(h, separator) - local t, tn = { }, 0 - for k, v in sortedhash(h) do - if v then - tn = tn + 1 - t[tn] = k - end - end - return concat(t,separator or ",") -end - -local value = lbrace * C((nobrace + nested)^0) * rbrace -local pattern = Ct((space + value)^0) - -function parsers.arguments_to_table(str) - return lpegmatch(pattern,str) -end - --- temporary here (unoptimized) - -function parsers.getparameters(self,class,parentclass,settings) - local sc = self[class] - if not sc then - sc = { } - self[class] = sc - if parentclass then - local sp = self[parentclass] - if not sp then - sp = { } - self[parentclass] = sp - end - setmetatableindex(sc,sp) - end - end - parsers.settings_to_hash(settings,sc) -end - -function parsers.listitem(str) - return gmatch(str,"[^, ]+") -end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['util-fmt'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -utilities = utilities or { } -utilities.formatters = utilities.formatters or { } -local formatters = utilities.formatters - -local concat, format = table.concat, string.format -local tostring, type = tostring, type -local strip = string.strip - -local P, R, Cs = lpeg.P, lpeg.R, lpeg.Cs -local lpegmatch = lpeg.match - --- temporary here - -local digit = R("09") -local period = P(".") -local zero = P("0") -local trailingzeros = zero^0 * -digit -- suggested by Roberto R -local case_1 = period * trailingzeros / "" -local case_2 = period * (digit - trailingzeros)^1 * (trailingzeros / "") -local number = digit^1 * (case_1 + case_2) -local stripper = Cs((number + 1)^0) - - -lpeg.patterns.stripzeros = stripper - -function formatters.stripzeros(str) - return lpegmatch(stripper,str) -end - -function formatters.formatcolumns(result,between) - if result and #result > 0 then - between = between or " " - local widths, numbers = { }, { } - local first = result[1] - local n = #first - for i=1,n do - widths[i] = 0 - end - for i=1,#result do - local r = result[i] - for j=1,n do - local rj = r[j] - local tj = type(rj) - if tj == "number" then - numbers[j] = true - end - if tj ~= "string" then - rj = tostring(rj) - r[j] = rj - end - local w = #rj - if w > widths[j] then - widths[j] = w - end - end - end - for i=1,n do - local w = widths[i] - if numbers[i] then - if w > 80 then - widths[i] = "%s" .. between - else - widths[i] = "%0" .. w .. "i" .. between - end - else - if w > 80 then - widths[i] = "%s" .. between - elseif w > 0 then - widths[i] = "%-" .. w .. "s" .. between - else - widths[i] = "%s" - end - end - end - local template = strip(concat(widths)) - for i=1,#result do - local str = format(template,unpack(result[i])) - result[i] = strip(str) - end - end - return result -end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['util.deb'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- the tag is kind of generic and used for functions that are not --- bound to a variable, like node.new, node.copy etc (contrary to for instance --- node.has_attribute which is bound to a has_attribute local variable in mkiv) - -local debug = require "debug" - -local getinfo = debug.getinfo -local type, next, tostring = type, next, tostring -local format, find = string.format, string.find -local is_boolean = string.is_boolean - -utilities = utilities or { } -utilities.debugger = utilities.debugger or { } -local debugger = utilities.debugger - -local counters = { } -local names = { } - --- one - -local function hook() - local f = getinfo(2) -- "nS" - if f then - local n = "unknown" - if f.what == "C" then - n = f.name or '' - if not names[n] then - names[n] = format("%42s",n) - end - else - -- source short_src linedefined what name namewhat nups func - n = f.name or f.namewhat or f.what - if not n or n == "" then - n = "?" - end - if not names[n] then - names[n] = format("%42s : % 5i : %s",n,f.linedefined or 0,f.short_src or "unknown source") - end - end - counters[n] = (counters[n] or 0) + 1 - end -end - -function debugger.showstats(printer,threshold) -- hm, something has changed, rubish now - printer = printer or texio.write or print - threshold = threshold or 0 - local total, grandtotal, functions = 0, 0, 0 - local dataset = { } - for name, count in next, counters do - dataset[#dataset+1] = { name, count } - end - table.sort(dataset,function(a,b) return a[2] == b[2] and b[1] > a[1] or a[2] > b[2] end) - for i=1,#dataset do - local d = dataset[i] - local name = d[1] - local count = d[2] - if count > threshold and not find(name,"for generator") then -- move up - printer(format("%8i %s\n", count, names[name])) - total = total + count - end - grandtotal = grandtotal + count - functions = functions + 1 - end - printer("\n") - printer(format("functions : % 10i\n", functions)) - printer(format("total : % 10i\n", total)) - printer(format("grand total: % 10i\n", grandtotal)) - printer(format("threshold : % 10i\n", threshold)) -end - -function debugger.savestats(filename,threshold) - local f = io.open(filename,'w') - if f then - debugger.showstats(function(str) f:write(str) end,threshold) - f:close() - end -end - -function debugger.enable() - debug.sethook(hook,"c") -end - -function debugger.disable() - debug.sethook() -end - - - - - -local is_node = node and node.is_node -local is_lpeg = lpeg and lpeg.type - -function inspect(i) -- global function - local ti = type(i) - if ti == "table" then - table.print(i,"table") - elseif is_node and is_node(i) then - table.print(nodes.astable(i),tostring(i)) - elseif is_lpeg and is_lpeg(i) then - lpeg.print(i) - else - print(tostring(i)) - end -end - --- from the lua book: - -function traceback() - local level = 1 - while true do - local info = debug.getinfo(level, "Sl") - if not info then - break - elseif info.what == "C" then - print(format("%3i : C function",level)) - else - print(format("%3i : [%s]:%d",level,info.short_src,info.currentline)) - end - level = level + 1 - end -end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['trac-inf'] = { - version = 1.001, - comment = "companion to trac-inf.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- As we want to protect the global tables, we no longer store the timing --- in the tables themselves but in a hidden timers table so that we don't --- get warnings about assignments. This is more efficient than using rawset --- and rawget. - -local format, lower = string.format, string.lower -local clock = os.gettimeofday or os.clock -- should go in environment -local write_nl = texio.write_nl - -statistics = statistics or { } -local statistics = statistics - -statistics.enable = true -statistics.threshold = 0.05 - -local statusinfo, n, registered, timers = { }, 0, { }, { } - -local function hastiming(instance) - return instance and timers[instance] -end - -local function resettiming(instance) - timers[instance or "notimer"] = { timing = 0, loadtime = 0 } -end - -local function starttiming(instance) - local timer = timers[instance or "notimer"] - if not timer then - timer = { } - timers[instance or "notimer"] = timer - end - local it = timer.timing - if not it then - it = 0 - end - if it == 0 then - timer.starttime = clock() - if not timer.loadtime then - timer.loadtime = 0 - end - end - timer.timing = it + 1 -end - -local function stoptiming(instance, report) - local timer = timers[instance or "notimer"] - local it = timer.timing - if it > 1 then - timer.timing = it - 1 - else - local starttime = timer.starttime - if starttime then - local stoptime = clock() - local loadtime = stoptime - starttime - timer.stoptime = stoptime - timer.loadtime = timer.loadtime + loadtime - if report then - statistics.report("load time %0.3f",loadtime) - end - timer.timing = 0 - return loadtime - end - end - return 0 -end - -local function elapsedtime(instance) - local timer = timers[instance or "notimer"] - return format("%0.3f",timer and timer.loadtime or 0) -end - -local function elapsedindeed(instance) - local timer = timers[instance or "notimer"] - return (timer and timer.loadtime or 0) > statistics.threshold -end - -local function elapsedseconds(instance,rest) -- returns nil if 0 seconds - if elapsedindeed(instance) then - return format("%s seconds %s", elapsedtime(instance),rest or "") - end -end - -statistics.hastiming = hastiming -statistics.resettiming = resettiming -statistics.starttiming = starttiming -statistics.stoptiming = stoptiming -statistics.elapsedtime = elapsedtime -statistics.elapsedindeed = elapsedindeed -statistics.elapsedseconds = elapsedseconds - --- general function - -function statistics.register(tag,fnc) - if statistics.enable and type(fnc) == "function" then - local rt = registered[tag] or (#statusinfo + 1) - statusinfo[rt] = { tag, fnc } - registered[tag] = rt - if #tag > n then n = #tag end - end -end - -function statistics.show(reporter) - if statistics.enable then - if not reporter then reporter = function(tag,data,n) write_nl(tag .. " " .. data) end end - -- this code will move - local register = statistics.register - register("luatex banner", function() - return lower(status.banner) - end) - register("control sequences", function() - return format("%s of %s + %s", status.cs_count, status.hash_size,status.hash_extra) - end) - register("callbacks", function() - local total, indirect = status.callbacks or 0, status.indirect_callbacks or 0 - return format("%s direct, %s indirect, %s total", total-indirect, indirect, total) - end) - collectgarbage("collect") - register("current memory usage", statistics.memused) - register("runtime",statistics.runtime) - for i=1,#statusinfo do - local s = statusinfo[i] - local r = s[2]() - if r then - reporter(s[1],r,n) - end - end - write_nl("") -- final newline - statistics.enable = false - end -end - -local template, report_statistics, nn = nil, nil, 0 -- we only calcute it once - -function statistics.showjobstat(tag,data,n) - if not logs then - -- sorry - elseif type(data) == "table" then - for i=1,#data do - statistics.showjobstat(tag,data[i],n) - end - else - if not template or n > nn then - template, n = format("%%-%ss - %%s",n), nn - report_statistics = logs.reporter("mkiv lua stats") - end - report_statistics(format(template,tag,data)) - end -end - -function statistics.memused() -- no math.round yet -) - local round = math.round or math.floor - return format("%s MB (ctx: %s MB)",round(collectgarbage("count")/1000), round(status.luastate_bytes/1000000)) -end - -starttiming(statistics) - -function statistics.formatruntime(runtime) -- indirect so it can be overloaded and - return format("%s seconds", runtime) -- indeed that happens in cure-uti.lua -end - -function statistics.runtime() - stoptiming(statistics) - return statistics.formatruntime(elapsedtime(statistics)) -end - -function statistics.timed(action,report) - report = report or logs.reporter("system") - starttiming("run") - action() - stoptiming("run") - report("total runtime: %s",elapsedtime("run")) -end - --- where, not really the best spot for this: - -commands = commands or { } - -function commands.resettimer(name) - resettiming(name or "whatever") - starttiming(name or "whatever") -end - -function commands.elapsedtime(name) - stoptiming(name or "whatever") - context(elapsedtime(name or "whatever")) -end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['trac-set'] = { -- might become util-set.lua - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local type, next, tostring = type, next, tostring -local concat = table.concat -local format, find, lower, gsub, escapedpattern = string.format, string.find, string.lower, string.gsub, string.escapedpattern -local is_boolean = string.is_boolean -local settings_to_hash = utilities.parsers.settings_to_hash -local allocate = utilities.storage.allocate - -utilities = utilities or { } -local utilities = utilities -utilities.setters = utilities.setters or { } -local setters = utilities.setters - -local data = { } -- maybe just local - --- We can initialize from the cnf file. This is sort of tricky as --- later defined setters also need to be initialized then. If set --- this way, we need to ensure that they are not reset later on. - -local trace_initialize = false -- only for testing during development - -function setters.initialize(filename,name,values) -- filename only for diagnostics - local setter = data[name] - if setter then - local data = setter.data - if data then - for key, value in next, values do - -- key = gsub(key,"_",".") - value = is_boolean(value,value) - local functions = data[key] - if functions then - if #functions > 0 and not functions.value then - if trace_initialize then - setter.report("executing %s (%s -> %s)",key,filename,tostring(value)) - end - for i=1,#functions do - functions[i](value) - end - functions.value = value - else - if trace_initialize then - setter.report("skipping %s (%s -> %s)",key,filename,tostring(value)) - end - end - else - -- we do a simple preregistration i.e. not in the - -- list as it might be an obsolete entry - functions = { default = value } - data[key] = functions - if trace_initialize then - setter.report("storing %s (%s -> %s)",key,filename,tostring(value)) - end - end - end - return true - end - end -end - --- user interface code - -local function set(t,what,newvalue) - local data, done = t.data, t.done - if type(what) == "string" then - what = settings_to_hash(what) -- inefficient but ok - end - if type(what) ~= "table" then - return - end - if not done then -- catch ... why not set? - done = { } - t.done = done - end - for w, value in next, what do - if value == "" then - value = newvalue - elseif not value then - value = false -- catch nil - else - value = is_boolean(value,value) - end - w = "^" .. escapedpattern(w,true) .. "$" -- new: anchored - for name, functions in next, data do - if done[name] then - -- prevent recursion due to wildcards - elseif find(name,w) then - done[name] = true - for i=1,#functions do - functions[i](value) - end - functions.value = value - end - end - end -end - -local function reset(t) - for name, functions in next, t.data do - for i=1,#functions do - functions[i](false) - end - functions.value = false - end -end - -local function enable(t,what) - set(t,what,true) -end - -local function disable(t,what) - local data = t.data - if not what or what == "" then - t.done = { } - reset(t) - else - set(t,what,false) - end -end - -function setters.register(t,what,...) - local data = t.data - what = lower(what) - local functions = data[what] - if not functions then - functions = { } - data[what] = functions - if trace_initialize then - t.report("defining %s",what) - end - end - local default = functions.default -- can be set from cnf file - for _, fnc in next, { ... } do - local typ = type(fnc) - if typ == "string" then - if trace_initialize then - t.report("coupling %s to %s",what,fnc) - end - local s = fnc -- else wrong reference - fnc = function(value) set(t,s,value) end - elseif typ ~= "function" then - fnc = nil - end - if fnc then - functions[#functions+1] = fnc - -- default: set at command line or in cnf file - -- value : set in tex run (needed when loading runtime) - local value = functions.value or default - if value ~= nil then - fnc(value) - functions.value = value - end - end - end - return false -- so we can use it in an assignment -end - -function setters.enable(t,what) - local e = t.enable - t.enable, t.done = enable, { } - enable(t,what) - t.enable, t.done = e, { } -end - -function setters.disable(t,what) - local e = t.disable - t.disable, t.done = disable, { } - disable(t,what) - t.disable, t.done = e, { } -end - -function setters.reset(t) - t.done = { } - reset(t) -end - -function setters.list(t) -- pattern - local list = table.sortedkeys(t.data) - local user, system = { }, { } - for l=1,#list do - local what = list[l] - if find(what,"^%*") then - system[#system+1] = what - else - user[#user+1] = what - end - end - return user, system -end - -function setters.show(t) - local category = t.name - local list = setters.list(t) - t.report() - for k=1,#list do - local name = list[k] - local functions = t.data[name] - if functions then - local value, default, modules = functions.value, functions.default, #functions - value = value == nil and "unset" or tostring(value) - default = default == nil and "unset" or tostring(default) - t.report("%-30s modules: %2i default: %6s value: %6s",name,modules,default,value) - end - end - t.report() -end - --- we could have used a bit of oo and the trackers:enable syntax but --- there is already a lot of code around using the singular tracker - --- we could make this into a module but we also want the rest avaliable - -local enable, disable, register, list, show = setters.enable, setters.disable, setters.register, setters.list, setters.show - -local function report(setter,...) - local report = logs and logs.report - if report then - report(setter.name,...) - else -- fallback, as this module is loaded before the logger - write_nl(format("%-15s : %s\n",setter.name,format(...))) - end -end - -function setters.new(name) - local setter -- we need to access it in setter itself - setter = { - data = allocate(), -- indexed, but also default and value fields - name = name, - report = function(...) report (setter,...) end, - enable = function(...) enable (setter,...) end, - disable = function(...) disable (setter,...) end, - register = function(...) register(setter,...) end, - list = function(...) list (setter,...) end, - show = function(...) show (setter,...) end, - } - data[name] = setter - return setter -end - -trackers = setters.new("trackers") -directives = setters.new("directives") -experiments = setters.new("experiments") - -local t_enable, t_disable, t_report = trackers .enable, trackers .disable, trackers .report -local d_enable, d_disable, d_report = directives .enable, directives .disable, directives .report -local e_enable, e_disable, e_report = experiments.enable, experiments.disable, experiments.report - --- nice trick: we overload two of the directives related functions with variants that --- do tracing (itself using a tracker) .. proof of concept - -local trace_directives = false local trace_directives = false trackers.register("system.directives", function(v) trace_directives = v end) -local trace_experiments = false local trace_experiments = false trackers.register("system.experiments", function(v) trace_experiments = v end) - -function directives.enable(...) - if trace_directives then - d_report("enabling: %s",concat({...}," ")) - end - d_enable(...) -end - -function directives.disable(...) - if trace_directives then - d_report("disabling: %s",concat({...}," ")) - end - d_disable(...) -end - -function experiments.enable(...) - if trace_experiments then - e_report("enabling: %s",concat({...}," ")) - end - e_enable(...) -end - -function experiments.disable(...) - if trace_experiments then - e_report("disabling: %s",concat({...}," ")) - end - e_disable(...) -end - --- a useful example - -directives.register("system.nostatistics", function(v) - statistics.enable = not v -end) - -directives.register("system.nolibraries", function(v) - libraries = nil -- we discard this tracing for security -end) - --- experiment - -local flags = environment and environment.engineflags - -if flags then - if trackers and flags.trackers then - setters.initialize("flags","trackers", settings_to_hash(flags.trackers)) - -- t_enable(flags.trackers) - end - if directives and flags.directives then - setters.initialize("flags","directives", settings_to_hash(flags.directives)) - -- d_enable(flags.directives) - end -end - --- here - -if texconfig then - - -- this happens too late in ini mode but that is no problem - - local function set(k,v) - v = tonumber(v) - if v then - texconfig[k] = v - end - end - - directives.register("luatex.expanddepth", function(v) set("expand_depth",v) end) - directives.register("luatex.hashextra", function(v) set("hash_extra",v) end) - directives.register("luatex.nestsize", function(v) set("nest_size",v) end) - directives.register("luatex.maxinopen", function(v) set("max_in_open",v) end) - directives.register("luatex.maxprintline", function(v) set("max_print_line",v) end) - directives.register("luatex.maxstrings", function(v) set("max_strings",v) end) - directives.register("luatex.paramsize", function(v) set("param_size",v) end) - directives.register("luatex.savesize", function(v) set("save_size",v) end) - directives.register("luatex.stacksize", function(v) set("stack_size",v) end) - -end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['trac-log'] = { - version = 1.001, - comment = "companion to trac-log.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- todo: less categories, more subcategories (e.g. nodes) - - -local write_nl, write = texio and texio.write_nl or print, texio and texio.write or io.write -local format, gmatch, find = string.format, string.gmatch, string.find -local concat, insert, remove = table.concat, table.insert, table.remove -local escapedpattern = string.escapedpattern -local texcount = tex and tex.count -local next, type = next, type - -local setmetatableindex = table.setmetatableindex - ---[[ldx-- -

This is a prelude to a more extensive logging module. We no longer -provide based logging a sparsing is relatively easy anyway.

---ldx]]-- - -logs = logs or { } -local logs = logs - -local moreinfo = [[ -More information about ConTeXt and the tools that come with it can be found at: - -maillist : ntg-context@ntg.nl / http://www.ntg.nl/mailman/listinfo/ntg-context -webpage : http://www.pragma-ade.nl / http://tex.aanhet.net -wiki : http://contextgarden.net -]] - --- basic loggers - -local function ignore() end - -setmetatableindex(logs, function(t,k) t[k] = ignore ; return ignore end) - -local report, subreport, status, settarget, setformats, settranslations - -local direct, subdirect, writer, pushtarget, poptarget - -if tex and (tex.jobname or tex.formatname) then - - local valueiskey = { __index = function(t,k) t[k] = k return k end } -- will be helper - - local target = "term and log" - - logs.flush = io.flush - - local formats = { } setmetatable(formats, valueiskey) - local translations = { } setmetatable(translations,valueiskey) - - writer = function(...) - write_nl(target,...) - end - - newline = function() - write_nl(target,"\n") - end - - report = function(a,b,c,...) - if c then - write_nl(target,format("%-15s > %s\n",translations[a],format(formats[b],c,...))) - elseif b then - write_nl(target,format("%-15s > %s\n",translations[a],formats[b])) - elseif a then - write_nl(target,format("%-15s >\n", translations[a])) - else - write_nl(target,"\n") - end - end - - direct = function(a,b,c,...) - if c then - return format("%-15s > %s",translations[a],format(formats[b],c,...)) - elseif b then - return format("%-15s > %s",translations[a],formats[b]) - elseif a then - return format("%-15s >", translations[a]) - else - return "" - end - end - - subreport = function(a,s,b,c,...) - if c then - write_nl(target,format("%-15s > %s > %s\n",translations[a],translations[s],format(formats[b],c,...))) - elseif b then - write_nl(target,format("%-15s > %s > %s\n",translations[a],translations[s],formats[b])) - elseif a then - write_nl(target,format("%-15s > %s >\n", translations[a],translations[s])) - else - write_nl(target,"\n") - end - end - - subdirect = function(a,s,b,c,...) - if c then - return format("%-15s > %s > %s",translations[a],translations[s],format(formats[b],c,...)) - elseif b then - return format("%-15s > %s > %s",translations[a],translations[s],formats[b]) - elseif a then - return format("%-15s > %s >", translations[a],translations[s]) - else - return "" - end - end - - status = function(a,b,c,...) - if c then - write_nl(target,format("%-15s : %s\n",translations[a],format(formats[b],c,...))) - elseif b then - write_nl(target,format("%-15s : %s\n",translations[a],formats[b])) - elseif a then - write_nl(target,format("%-15s :\n", translations[a])) - else - write_nl(target,"\n") - end - end - - local targets = { - logfile = "log", - log = "log", - file = "log", - console = "term", - terminal = "term", - both = "term and log", - } - - settarget = function(whereto) - target = targets[whereto or "both"] or targets.both - if target == "term" or target == "term and log" then - logs.flush = io.flush - else - logs.flush = ignore - end - end - - local stack = { } - - pushtarget = function(newtarget) - insert(stack,target) - settarget(newtarget) - end - - poptarget = function() - if #stack > 0 then - settarget(remove(stack)) - end - end - - setformats = function(f) - formats = f - end - - settranslations = function(t) - translations = t - end - -else - - logs.flush = ignore - - writer = write_nl - - newline = function() - write_nl("\n") - end - - report = function(a,b,c,...) - if c then - write_nl(format("%-15s | %s",a,format(b,c,...))) - elseif b then - write_nl(format("%-15s | %s",a,b)) - elseif a then - write_nl(format("%-15s |", a)) - else - write_nl("") - end - end - - subreport = function(a,sub,b,c,...) - if c then - write_nl(format("%-15s | %s | %s",a,sub,format(b,c,...))) - elseif b then - write_nl(format("%-15s | %s | %s",a,sub,b)) - elseif a then - write_nl(format("%-15s | %s |", a,sub)) - else - write_nl("") - end - end - - status = function(a,b,c,...) -- not to be used in lua anyway - if c then - write_nl(format("%-15s : %s\n",a,format(b,c,...))) - elseif b then - write_nl(format("%-15s : %s\n",a,b)) -- b can have %'s - elseif a then - write_nl(format("%-15s :\n", a)) - else - write_nl("\n") - end - end - - direct = ignore - subdirect = ignore - - settarget = ignore - pushtarget = ignore - poptarget = ignore - setformats = ignore - settranslations = ignore - -end - -logs.report = report -logs.subreport = subreport -logs.status = status -logs.settarget = settarget -logs.pushtarget = pushtarget -logs.poptarget = poptarget -logs.setformats = setformats -logs.settranslations = settranslations - -logs.direct = direct -logs.subdirect = subdirect -logs.writer = writer -logs.newline = newline - --- installer - --- todo: renew (un) locks when a new one is added and wildcard - -local data, states = { }, nil - -function logs.reporter(category,subcategory) - local logger = data[category] - if not logger then - local state = false - if states == true then - state = true - elseif type(states) == "table" then - for c, _ in next, states do - if find(category,c) then - state = true - break - end - end - end - logger = { - reporters = { }, - state = state, - } - data[category] = logger - end - local reporter = logger.reporters[subcategory or "default"] - if not reporter then - if subcategory then - reporter = function(...) - if not logger.state then - subreport(category,subcategory,...) - end - end - logger.reporters[subcategory] = reporter - else - local tag = category - reporter = function(...) - if not logger.state then - report(category,...) - end - end - logger.reporters.default = reporter - end - end - return reporter -end - -logs.new = logs.reporter -- for old times sake - --- context specicific: this ends up in the macro stream - -local ctxreport = logs.writer - -function logs.setmessenger(m) - ctxreport = m -end - -function logs.messenger(category,subcategory) - -- we need to avoid catcode mess (todo: fast context) - if subcategory then - return function(...) - ctxreport(subdirect(category,subcategory,...)) - end - else - return function(...) - ctxreport(direct(category,...)) - end - end -end - --- so far - -local function setblocked(category,value) - if category == true then - -- lock all - category, value = "*", true - elseif category == false then - -- unlock all - category, value = "*", false - elseif value == nil then - -- lock selective - value = true - end - if category == "*" then - states = value - for k, v in next, data do - v.state = value - end - else - states = utilities.parsers.settings_to_hash(category) - for c, _ in next, states do - if data[c] then - v.state = value - else - c = escapedpattern(c,true) - for k, v in next, data do - if find(k,c) then - v.state = value - end - end - end - end - end -end - -function logs.disable(category,value) - setblocked(category,value == nil and true or value) -end - -function logs.enable(category) - setblocked(category,false) -end - -function logs.categories() - return table.sortedkeys(data) -end - -function logs.show() - local n, c, s, max = 0, 0, 0, 0 - for category, v in table.sortedpairs(data) do - n = n + 1 - local state = v.state - local reporters = v.reporters - local nc = #category - if nc > c then - c = nc - end - for subcategory, _ in next, reporters do - local ns = #subcategory - if ns > c then - s = ns - end - local m = nc + ns - if m > max then - max = m - end - end - local subcategories = concat(table.sortedkeys(reporters),", ") - if state == true then - state = "disabled" - elseif state == false then - state = "enabled" - else - state = "unknown" - end - -- no new here - report("logging","category: '%s', subcategories: '%s', state: '%s'",category,subcategories,state) - end - report("logging","categories: %s, max category: %s, max subcategory: %s, max combined: %s",n,c,s,max) -end - -directives.register("logs.blocked", function(v) - setblocked(v,true) -end) - -directives.register("logs.target", function(v) - settarget(v) -end) - --- tex specific loggers (might move elsewhere) - -local report_pages = logs.reporter("pages") -- not needed but saves checking when we grep for it - -local real, user, sub - -function logs.start_page_number() - real, user, sub = texcount.realpageno, texcount.userpageno, texcount.subpageno --- real, user, sub = 0, 0, 0 -end - -local timing = false -local starttime = nil -local lasttime = nil - -trackers.register("pages.timing", function(v) -- only for myself (diagnostics) - starttime = os.clock() - timing = true -end) - -function logs.stop_page_number() -- the first page can includes the initialization so we omit this in average - if timing then - local elapsed, average - local stoptime = os.clock() - if not lasttime or real < 2 then - elapsed = stoptime - average = stoptime - starttime = stoptime - else - elapsed = stoptime - lasttime - average = (stoptime - starttime) / (real - 1) - end - lasttime = stoptime - if real > 0 then - if user > 0 then - if sub > 0 then - report_pages("flushing realpage %s, userpage %s, subpage %s, time %0.04f / %0.04f",real,user,sub,elapsed,average) - else - report_pages("flushing realpage %s, userpage %s, time %0.04f / %0.04f",real,user,elapsed,average) - end - else - report_pages("flushing realpage %s, time %0.04f / %0.04f",real,elapsed,average) - end - else - report_pages("flushing page, time %0.04f / %0.04f",elapsed,average) - end - else - if real > 0 then - if user > 0 then - if sub > 0 then - report_pages("flushing realpage %s, userpage %s, subpage %s",real,user,sub) - else - report_pages("flushing realpage %s, userpage %s",real,user) - end - else - report_pages("flushing realpage %s",real) - end - else - report_pages("flushing page") - end - end - logs.flush() -end - -logs.report_job_stat = statistics and statistics.showjobstat - -local report_files = logs.reporter("files") - -local nesting = 0 -local verbose = false -local hasscheme = url.hasscheme - --- we don't have show_open and show_close callbacks yet - -function logs.show_open(name) - -- if hasscheme(name) ~= "virtual" then - -- if verbose then - -- nesting = nesting + 1 - -- report_files("level %s, opening %s",nesting,name) - -- else - -- write(format("(%s",name)) -- tex adds a space - -- end - -- end -end - -function logs.show_close(name) - -- if hasscheme(name) ~= "virtual" then - -- if verbose then - -- report_files("level %s, closing %s",nesting,name) - -- nesting = nesting - 1 - -- else - -- write(")") -- tex adds a space - -- end - -- end -end - -function logs.show_load(name) - -- if hasscheme(name) ~= "virtual" then - -- if verbose then - -- report_files("level %s, loading %s",nesting+1,name) - -- else - -- write(format("(%s)",name)) - -- end - -- end -end - --- there may be scripts out there using this: - -local simple = logs.reporter("comment") - -logs.simple = simple -logs.simpleline = simple - --- obsolete - -function logs.setprogram () end -- obsolete -function logs.extendbanner() end -- obsolete -function logs.reportlines () end -- obsolete -function logs.reportbanner() end -- obsolete -function logs.reportline () end -- obsolete -function logs.simplelines () end -- obsolete -function logs.help () end -- obsolete - --- applications - -local function reportlines(t,str) - if str then - for line in gmatch(str,"(.-)[\n\r]") do - t.report(line) - end - end -end - -local function reportbanner(t) - local banner = t.banner - if banner then - t.report(banner) - t.report() - end -end - -local function reportversion(t) - local banner = t.banner - if banner then - t.report(banner) - end -end - -local function reporthelp(t,...) - local helpinfo = t.helpinfo - if type(helpinfo) == "string" then - reportlines(t,helpinfo) - elseif type(helpinfo) == "table" then - local tags = { ... } - for i=1,#tags do - reportlines(t,t.helpinfo[tags[i]]) - if i < #tags then - t.report() - end - end - end -end - -local function reportinfo(t) - t.report() - reportlines(t,moreinfo) -end - -function logs.application(t) - t.name = t.name or "unknown" - t.banner = t.banner - t.report = logs.reporter(t.name) - t.help = function(...) reportbanner(t) ; reporthelp(t,...) ; reportinfo(t) end - t.identify = function() reportbanner(t) end - t.version = function() reportversion(t) end - return t -end - --- somewhat special - --- logging to a file - - -function logs.system(whereto,process,jobname,category,...) - local message = format("%s %s => %s => %s => %s\r",os.date("%d/%m/%y %H:%m:%S"),process,jobname,category,format(...)) - for i=1,10 do - local f = io.open(whereto,"a") -- we can consider keepint the file open - if f then - f:write(message) - f:close() - break - else - sleep(0.1) - end - end -end - -local report_system = logs.reporter("system","logs") - -function logs.obsolete(old,new) - local o = loadstring("return " .. new)() - if type(o) == "function" then - return function(...) - report_system("function %s is obsolete, use %s",old,new) - loadstring(old .. "=" .. new .. " return ".. old)()(...) - end - elseif type(o) == "table" then - local t, m = { }, { } - m.__index = function(t,k) - report_system("table %s is obsolete, use %s",old,new) - m.__index, m.__newindex = o, o - return o[k] - end - m.__newindex = function(t,k,v) - report_system("table %s is obsolete, use %s",old,new) - m.__index, m.__newindex = o, o - o[k] = v - end - if libraries then - libraries.obsolete[old] = t -- true - end - setmetatable(t,m) - return t - end -end - -if utilities then - utilities.report = report_system -end - -if tex and tex.error then - function logs.texerrormessage(...) -- for the moment we put this function here - tex.error(format(...), { }) - end -else - function logs.texerrormessage(...) - print(format(...)) - end -end - --- do we still need io.flush then? - -io.stdout:setvbuf('no') -io.stderr:setvbuf('no') - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['trac-pro'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local getmetatable, setmetatable, rawset, type = getmetatable, setmetatable, rawset, type - --- The protection implemented here is probably not that tight but good enough to catch --- problems due to naive usage. --- --- There's a more extensive version (trac-xxx.lua) that supports nesting. --- --- This will change when we have _ENV in lua 5.2+ - -local trace_namespaces = false trackers.register("system.namespaces", function(v) trace_namespaces = v end) - -local report_system = logs.reporter("system","protection") - -namespaces = namespaces or { } -local namespaces = namespaces - -local registered = { } - -local function report_index(k,name) - if trace_namespaces then - report_system("reference to '%s' in protected namespace '%s', %s",k,name,debug.traceback()) - else - report_system("reference to '%s' in protected namespace '%s'",k,name) - end -end - -local function report_newindex(k,name) - if trace_namespaces then - report_system("assignment to '%s' in protected namespace '%s', %s",k,name,debug.traceback()) - else - report_system("assignment to '%s' in protected namespace '%s'",k,name) - end -end - -local function register(name) - local data = name == "global" and _G or _G[name] - if not data then - return -- error - end - registered[name] = data - local m = getmetatable(data) - if not m then - m = { } - setmetatable(data,m) - end - local index, newindex = { }, { } - m.__saved__index = m.__index - m.__no__index = function(t,k) - if not index[k] then - index[k] = true - report_index(k,name) - end - return nil - end - m.__saved__newindex = m.__newindex - m.__no__newindex = function(t,k,v) - if not newindex[k] then - newindex[k] = true - report_newindex(k,name) - end - rawset(t,k,v) - end - m.__protection__depth = 0 -end - -local function private(name) -- maybe save name - local data = registered[name] - if not data then - data = _G[name] - if not data then - data = { } - _G[name] = data - end - register(name) - end - return data -end - -local function protect(name) - local data = registered[name] - if not data then - return - end - local m = getmetatable(data) - local pd = m.__protection__depth - if pd > 0 then - m.__protection__depth = pd + 1 - else - m.__save_d_index, m.__saved__newindex = m.__index, m.__newindex - m.__index, m.__newindex = m.__no__index, m.__no__newindex - m.__protection__depth = 1 - end -end - -local function unprotect(name) - local data = registered[name] - if not data then - return - end - local m = getmetatable(data) - local pd = m.__protection__depth - if pd > 1 then - m.__protection__depth = pd - 1 - else - m.__index, m.__newindex = m.__saved__index, m.__saved__newindex - m.__protection__depth = 0 - end -end - -local function protectall() - for name, _ in next, registered do - if name ~= "global" then - protect(name) - end - end -end - -local function unprotectall() - for name, _ in next, registered do - if name ~= "global" then - unprotect(name) - end - end -end - -namespaces.register = register -- register when defined -namespaces.private = private -- allocate and register if needed -namespaces.protect = protect -namespaces.unprotect = unprotect -namespaces.protectall = protectall -namespaces.unprotectall = unprotectall - -namespaces.private("namespaces") registered = { } register("global") -- unreachable - -directives.register("system.protect", function(v) - if v then - protectall() - else - unprotectall() - end -end) - -directives.register("system.checkglobals", function(v) - if v then - report_system("enabling global namespace guard") - protect("global") - else - report_system("disabling global namespace guard") - unprotect("global") - end -end) - --- dummy section (will go to luat-dum.lua) - - - - - - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['luat-env'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- A former version provided functionality for non embeded core --- scripts i.e. runtime library loading. Given the amount of --- Lua code we use now, this no longer makes sense. Much of this --- evolved before bytecode arrays were available and so a lot of --- code has disappeared already. - -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) - -local report_lua = logs.reporter("resolvers","lua") - -local allocate, mark = utilities.storage.allocate, utilities.storage.mark - -local format, sub, match, gsub, find = string.format, string.sub, string.match, string.gsub, string.find -local unquoted, quoted = string.unquoted, string.quoted -local concat = table.concat - --- precautions - -os.setlocale(nil,nil) -- useless feature and even dangerous in luatex - -function os.setlocale() - -- no way you can mess with it -end - --- dirty tricks - -if arg and (arg[0] == 'luatex' or arg[0] == 'luatex.exe') and arg[1] == "--luaonly" then - arg[-1] = arg[0] - arg[ 0] = arg[2] - for k=3,#arg do - arg[k-2] = arg[k] - end - arg[#arg] = nil -- last - arg[#arg] = nil -- pre-last -end - --- environment - -environment = environment or { } -local environment = environment - -environment.arguments = allocate() -environment.files = allocate() -environment.sortedflags = nil - -local mt = { - __index = function(_,k) - if k == "version" then - local version = tex.toks and tex.toks.contextversiontoks - if version and version ~= "" then - rawset(environment,"version",version) - return version - else - return "unknown" - end - elseif k == "jobname" or k == "formatname" then - local name = tex and tex[k] - if name or name== "" then - rawset(environment,k,name) - return name - else - return "unknown" - end - elseif k == "outputfilename" then - local name = environment.jobname - rawset(environment,k,name) - return name - end - end -} - -setmetatable(environment,mt) - -function environment.initializearguments(arg) - local arguments, files = { }, { } - environment.arguments, environment.files, environment.sortedflags = arguments, files, nil - for index=1,#arg do - local argument = arg[index] - if index > 0 then - local flag, value = match(argument,"^%-+(.-)=(.-)$") - if flag then - arguments[flag] = unquoted(value or "") - else - flag = match(argument,"^%-+(.+)") - if flag then - arguments[flag] = true - else - files[#files+1] = argument - end - end - end - end - environment.ownname = environment.ownname or arg[0] or 'unknown.lua' -end - -function environment.setargument(name,value) - environment.arguments[name] = value -end - --- todo: defaults, better checks e.g on type (boolean versus string) --- --- tricky: too many hits when we support partials unless we add --- a registration of arguments so from now on we have 'partial' - -function environment.argument(name,partial) - local arguments, sortedflags = environment.arguments, environment.sortedflags - if arguments[name] then - return arguments[name] - elseif partial then - if not sortedflags then - sortedflags = allocate(table.sortedkeys(arguments)) - for k=1,#sortedflags do - sortedflags[k] = "^" .. sortedflags[k] - end - environment.sortedflags = sortedflags - end - -- example of potential clash: ^mode ^modefile - for k=1,#sortedflags do - local v = sortedflags[k] - if find(name,v) then - return arguments[sub(v,2,#v)] - end - end - end - return nil -end - -function environment.splitarguments(separator) -- rather special, cut-off before separator - local done, before, after = false, { }, { } - local originalarguments = environment.originalarguments - for k=1,#originalarguments do - local v = originalarguments[k] - if not done and v == separator then - done = true - elseif done then - after[#after+1] = v - else - before[#before+1] = v - end - end - return before, after -end - -function environment.reconstructcommandline(arg,noquote) - arg = arg or environment.originalarguments - if noquote and #arg == 1 then - -- we could just do: return unquoted(resolvers.resolve(arg[i])) - local a = arg[1] - a = resolvers.resolve(a) - a = unquoted(a) - return a - elseif #arg > 0 then - local result = { } - for i=1,#arg do - -- we could just do: result[#result+1] = format("%q",unquoted(resolvers.resolve(arg[i]))) - local a = arg[i] - a = resolvers.resolve(a) - a = unquoted(a) - a = gsub(a,'"','\\"') -- tricky - if find(a," ") then - result[#result+1] = quoted(a) - else - result[#result+1] = a - end - end - return concat(result," ") - else - return "" - end -end - - -if arg then - - -- new, reconstruct quoted snippets (maybe better just remove the " then and add them later) - local newarg, instring = { }, false - - for index=1,#arg do - local argument = arg[index] - if find(argument,"^\"") then - newarg[#newarg+1] = gsub(argument,"^\"","") - if not find(argument,"\"$") then - instring = true - end - elseif find(argument,"\"$") then - newarg[#newarg] = newarg[#newarg] .. " " .. gsub(argument,"\"$","") - instring = false - elseif instring then - newarg[#newarg] = newarg[#newarg] .. " " .. argument - else - newarg[#newarg+1] = argument - end - end - for i=1,-5,-1 do - newarg[i] = arg[i] - end - - environment.initializearguments(newarg) - - environment.originalarguments = mark(newarg) - environment.rawarguments = mark(arg) - - arg = { } -- prevent duplicate handling - -end - --- weird place ... depends on a not yet loaded module - -function environment.texfile(filename) - return resolvers.findfile(filename,'tex') -end - -function environment.luafile(filename) - local resolved = resolvers.findfile(filename,'tex') or "" - if resolved ~= "" then - return resolved - end - resolved = resolvers.findfile(filename,'texmfscripts') or "" - if resolved ~= "" then - return resolved - end - return resolvers.findfile(filename,'luatexlibs') or "" -end - -environment.loadedluacode = loadfile -- can be overloaded - -function environment.luafilechunk(filename,silent) -- used for loading lua bytecode in the format - filename = file.replacesuffix(filename, "lua") - local fullname = environment.luafile(filename) - if fullname and fullname ~= "" then - local data = environment.loadedluacode(fullname) - if trace_locating then - report_lua("loading file %s%s", fullname, not data and " failed" or "") - elseif not silent then - texio.write("<",data and "+ " or "- ",fullname,">") - end - return data - else - if trace_locating then - report_lua("unknown file %s", filename) - end - return nil - end -end - --- the next ones can use the previous ones / combine - -function environment.loadluafile(filename, version) - local lucname, luaname, chunk - local basename = file.removesuffix(filename) - if basename == filename then - lucname, luaname = basename .. ".luc", basename .. ".lua" - else - lucname, luaname = nil, basename -- forced suffix - end - -- when not overloaded by explicit suffix we look for a luc file first - local fullname = (lucname and environment.luafile(lucname)) or "" - if fullname ~= "" then - if trace_locating then - report_lua("loading %s", fullname) - end - chunk = loadfile(fullname) -- this way we don't need a file exists check - end - if chunk then - assert(chunk)() - if version then - -- we check of the version number of this chunk matches - local v = version -- can be nil - if modules and modules[filename] then - v = modules[filename].version -- new method - elseif versions and versions[filename] then - v = versions[filename] -- old method - end - if v == version then - return true - else - if trace_locating then - report_lua("version mismatch for %s: lua=%s, luc=%s", filename, v, version) - end - environment.loadluafile(filename) - end - else - return true - end - end - fullname = (luaname and environment.luafile(luaname)) or "" - if fullname ~= "" then - if trace_locating then - report_lua("loading %s", fullname) - end - chunk = loadfile(fullname) -- this way we don't need a file exists check - if not chunk then - if trace_locating then - report_lua("unknown file %s", filename) - end - else - assert(chunk)() - return true - end - end - return false -end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['lxml-tab'] = { - version = 1.001, - comment = "this module is the basis for the lxml-* ones", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- this module needs a cleanup: check latest lpeg, passing args, (sub)grammar, etc etc --- stripping spaces from e.g. cont-en.xml saves .2 sec runtime so it's not worth the --- trouble - --- todo: when serializing optionally remap named entities to hex (if known in char-ent.lua) --- maybe when letter -> utf, else name .. then we need an option to the serializer .. a bit --- of work so we delay this till we cleanup - -local trace_entities = false trackers.register("xml.entities", function(v) trace_entities = v end) - -local report_xml = logs and logs.reporter("xml","core") or function(...) print(format(...)) end - ---[[ldx-- -

The parser used here is inspired by the variant discussed in the lua book, but -handles comment and processing instructions, has a different structure, provides -parent access; a first version used different trickery but was less optimized to we -went this route. First we had a find based parser, now we have an based one. -The find based parser can be found in l-xml-edu.lua along with other older code.

- -

Beware, the interface may change. For instance at, ns, tg, dt may get more -verbose names. Once the code is stable we will also remove some tracing and -optimize the code.

- -

I might even decide to reimplement the parser using the latest trickery -as the current variant was written when showed up and it's easier now to -build tables in one go.

---ldx]]-- - -xml = xml or { } -local xml = xml - - -local utf = unicode.utf8 -local concat, remove, insert = table.concat, table.remove, table.insert -local type, next, setmetatable, getmetatable, tonumber = type, next, setmetatable, getmetatable, tonumber -local format, lower, find, match, gsub = string.format, string.lower, string.find, string.match, string.gsub -local utfchar, utffind, utfgsub = utf.char, utf.find, utf.gsub -local lpegmatch = lpeg.match -local P, S, R, C, V, C, Cs = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.C, lpeg.Cs - ---[[ldx-- -

First a hack to enable namespace resolving. A namespace is characterized by -a . The following function associates a namespace prefix with a -pattern. We use , which in this case is more than twice as fast as a -find based solution where we loop over an array of patterns. Less code and -much cleaner.

---ldx]]-- - -xml.xmlns = xml.xmlns or { } - -local check = P(false) -local parse = check - ---[[ldx-- -

The next function associates a namespace prefix with an . This -normally happens independent of parsing.

- - -xml.registerns("mml","mathml") - ---ldx]]-- - -function xml.registerns(namespace, pattern) -- pattern can be an lpeg - check = check + C(P(lower(pattern))) / namespace - parse = P { P(check) + 1 * V(1) } -end - ---[[ldx-- -

The next function also registers a namespace, but this time we map a -given namespace prefix onto a registered one, using the given -. This used for attributes like xmlns:m.

- - -xml.checkns("m","http://www.w3.org/mathml") - ---ldx]]-- - -function xml.checkns(namespace,url) - local ns = lpegmatch(parse,lower(url)) - if ns and namespace ~= ns then - xml.xmlns[namespace] = ns - end -end - ---[[ldx-- -

Next we provide a way to turn an into a registered -namespace. This used for the xmlns attribute.

- - -resolvedns = xml.resolvens("http://www.w3.org/mathml") - - -This returns mml. ---ldx]]-- - -function xml.resolvens(url) - return lpegmatch(parse,lower(url)) or "" -end - ---[[ldx-- -

A namespace in an element can be remapped onto the registered -one efficiently by using the xml.xmlns table.

---ldx]]-- - ---[[ldx-- -

This version uses . We follow the same approach as before, stack and top and -such. This version is about twice as fast which is mostly due to the fact that -we don't have to prepare the stream for cdata, doctype etc etc. This variant is -is dedicated to Luigi Scarso, who challenged me with 40 megabyte files that -took 12.5 seconds to load (1.5 for file io and the rest for tree building). With -the implementation we got that down to less 7.3 seconds. Loading the 14 - interface definition files (2.6 meg) went down from 1.05 seconds to 0.55.

- -

Next comes the parser. The rather messy doctype definition comes in many -disguises so it is no surprice that later on have to dedicate quite some - code to it.

- - - - - - - - - - -

The code may look a bit complex but this is mostly due to the fact that we -resolve namespaces and attach metatables. There is only one public function:

- - -local x = xml.convert(somestring) - - -

An optional second boolean argument tells this function not to create a root -element.

- -

Valid entities are:

- - - - - - ---ldx]]-- - --- not just one big nested table capture (lpeg overflow) - -local nsremap, resolvens = xml.xmlns, xml.resolvens - -local stack = { } -local top = { } -local dt = { } -local at = { } -local xmlns = { } -local errorstr = nil -local entities = { } -local strip = false -local cleanup = false -local utfize = false -local resolve_predefined = false -local unify_predefined = false - -local dcache = { } -local hcache = { } -local acache = { } - -local mt = { } - -local function initialize_mt(root) - mt = { __index = root } -- will be redefined later -end - -function xml.setproperty(root,k,v) - getmetatable(root).__index[k] = v -end - -function xml.checkerror(top,toclose) - return "" -- can be set -end - -local function add_attribute(namespace,tag,value) - if cleanup and #value > 0 then - value = cleanup(value) -- new - end - if tag == "xmlns" then - xmlns[#xmlns+1] = resolvens(value) - at[tag] = value - elseif namespace == "" then - at[tag] = value - elseif namespace == "xmlns" then - xml.checkns(tag,value) - at["xmlns:" .. tag] = value - else - -- for the moment this way: - at[namespace .. ":" .. tag] = value - end -end - -local function add_empty(spacing, namespace, tag) - if #spacing > 0 then - dt[#dt+1] = spacing - end - local resolved = (namespace == "" and xmlns[#xmlns]) or nsremap[namespace] or namespace - top = stack[#stack] - dt = top.dt - local t = { ns=namespace or "", rn=resolved, tg=tag, at=at, dt={}, __p__ = top } - dt[#dt+1] = t - setmetatable(t, mt) - if at.xmlns then - remove(xmlns) - end - at = { } -end - -local function add_begin(spacing, namespace, tag) - if #spacing > 0 then - dt[#dt+1] = spacing - end - local resolved = (namespace == "" and xmlns[#xmlns]) or nsremap[namespace] or namespace - top = { ns=namespace or "", rn=resolved, tg=tag, at=at, dt={}, __p__ = stack[#stack] } - setmetatable(top, mt) - dt = top.dt - stack[#stack+1] = top - at = { } -end - -local function add_end(spacing, namespace, tag) - if #spacing > 0 then - dt[#dt+1] = spacing - end - local toclose = remove(stack) - top = stack[#stack] - if #stack < 1 then - errorstr = format("nothing to close with %s %s", tag, xml.checkerror(top,toclose) or "") - elseif toclose.tg ~= tag then -- no namespace check - errorstr = format("unable to close %s with %s %s", toclose.tg, tag, xml.checkerror(top,toclose) or "") - end - dt = top.dt - dt[#dt+1] = toclose - -- dt[0] = top -- nasty circular reference when serializing table - if toclose.at.xmlns then - remove(xmlns) - end -end - -local function add_text(text) - if cleanup and #text > 0 then - dt[#dt+1] = cleanup(text) - else - dt[#dt+1] = text - end -end - -local function add_special(what, spacing, text) - if #spacing > 0 then - dt[#dt+1] = spacing - end - if strip and (what == "@cm@" or what == "@dt@") then - -- forget it - else - dt[#dt+1] = { special=true, ns="", tg=what, dt={ text } } - end -end - -local function set_message(txt) - errorstr = "garbage at the end of the file: " .. gsub(txt,"([ \n\r\t]*)","") -end - -local reported_attribute_errors = { } - -local function attribute_value_error(str) - if not reported_attribute_errors[str] then - report_xml("invalid attribute value: %q",str) - reported_attribute_errors[str] = true - at._error_ = str - end - return str -end - -local function attribute_specification_error(str) - if not reported_attribute_errors[str] then - report_xml("invalid attribute specification: %q",str) - reported_attribute_errors[str] = true - at._error_ = str - end - return str -end - -xml.placeholders = { - unknown_dec_entity = function(str) return (str == "" and "&error;") or format("&%s;",str) end, - unknown_hex_entity = function(str) return format("&#x%s;",str) end, - unknown_any_entity = function(str) return format("&#x%s;",str) end, -} - -local placeholders = xml.placeholders - -local function fromhex(s) - local n = tonumber(s,16) - if n then - return utfchar(n) - else - return format("h:%s",s), true - end -end - -local function fromdec(s) - local n = tonumber(s) - if n then - return utfchar(n) - else - return format("d:%s",s), true - end -end - --- one level expansion (simple case), no checking done - -local rest = (1-P(";"))^0 -local many = P(1)^0 - -local parsedentity = - P("&") * (P("#x")*(rest/fromhex) + P("#")*(rest/fromdec)) * P(";") * P(-1) + - (P("#x")*(many/fromhex) + P("#")*(many/fromdec)) - --- parsing in the xml file - -local predefined_unified = { - [38] = "&", - [42] = """, - [47] = "'", - [74] = "<", - [76] = ">", -} - -local predefined_simplified = { - [38] = "&", amp = "&", - [42] = '"', quot = '"', - [47] = "'", apos = "'", - [74] = "<", lt = "<", - [76] = ">", gt = ">", -} - -local nofprivates = 0xF0000 -- shared but seldom used - -local privates_u = { -- unescaped - [ [[&]] ] = "&", - [ [["]] ] = """, - [ [[']] ] = "'", - [ [[<]] ] = "<", - [ [[>]] ] = ">", -} - -local privates_p = { -} - -local privates_n = { - -- keeps track of defined ones -} - -local function escaped(s) - if s == "" then - return "" - else -- if utffind(s,privates_u) then - return (utfgsub(s,".",privates_u)) - -- else - -- return s - end -end - -local function unescaped(s) - local p = privates_n[s] - if not p then - nofprivates = nofprivates + 1 - p = utfchar(nofprivates) - privates_n[s] = p - s = "&" .. s .. ";" -- todo: use char-ent to map to hex - privates_u[p] = s - privates_p[p] = s - end - return p -end - -local function unprivatized(s,resolve) - if s == "" then - return "" - else - return (utfgsub(s,".",privates_p)) - end -end - -xml.privatetoken = unescaped -xml.unprivatized = unprivatized -xml.privatecodes = privates_n - -local function handle_hex_entity(str) - local h = hcache[str] - if not h then - local n = tonumber(str,16) - h = unify_predefined and predefined_unified[n] - if h then - if trace_entities then - report_xml("utfize, converting hex entity &#x%s; into %s",str,h) - end - elseif utfize then - h = (n and utfchar(n)) or xml.unknown_hex_entity(str) or "" - if not n then - report_xml("utfize, ignoring hex entity &#x%s;",str) - elseif trace_entities then - report_xml("utfize, converting hex entity &#x%s; into %s",str,h) - end - else - if trace_entities then - report_xml("found entity &#x%s;",str) - end - h = "&#x" .. str .. ";" - end - hcache[str] = h - end - return h -end - -local function handle_dec_entity(str) - local d = dcache[str] - if not d then - local n = tonumber(str) - d = unify_predefined and predefined_unified[n] - if d then - if trace_entities then - report_xml("utfize, converting dec entity &#%s; into %s",str,d) - end - elseif utfize then - d = (n and utfchar(n)) or placeholders.unknown_dec_entity(str) or "" - if not n then - report_xml("utfize, ignoring dec entity &#%s;",str) - elseif trace_entities then - report_xml("utfize, converting dec entity &#%s; into %s",str,d) - end - else - if trace_entities then - report_xml("found entity &#%s;",str) - end - d = "&#" .. str .. ";" - end - dcache[str] = d - end - return d -end - -xml.parsedentitylpeg = parsedentity - -local function handle_any_entity(str) - if resolve then - local a = acache[str] -- per instance ! todo - if not a then - a = resolve_predefined and predefined_simplified[str] - if a then - if trace_entities then - report_xml("resolved entity &%s; -> %s (predefined)",str,a) - end - else - if type(resolve) == "function" then - a = resolve(str) or entities[str] - else - a = entities[str] - end - if a then - if type(a) == "function" then - if trace_entities then - report_xml("expanding entity &%s; (function)",str) - end - a = a(str) or "" - end - a = lpegmatch(parsedentity,a) or a -- for nested - if trace_entities then - report_xml("resolved entity &%s; -> %s (internal)",str,a) - end - else - local unknown_any_entity = placeholders.unknown_any_entity - if unknown_any_entity then - a = unknown_any_entity(str) or "" - end - if a then - if trace_entities then - report_xml("resolved entity &%s; -> %s (external)",str,a) - end - else - if trace_entities then - report_xml("keeping entity &%s;",str) - end - if str == "" then - a = "&error;" - else - a = "&" .. str .. ";" - end - end - end - end - acache[str] = a - elseif trace_entities then - if not acache[str] then - report_xml("converting entity &%s; into %s",str,a) - acache[str] = a - end - end - return a - else - local a = acache[str] - if not a then - a = resolve_predefined and predefined_simplified[str] - if a then - -- one of the predefined - acache[str] = a - if trace_entities then - report_xml("entity &%s; becomes %s",str,tostring(a)) - end - elseif str == "" then - if trace_entities then - report_xml("invalid entity &%s;",str) - end - a = "&error;" - acache[str] = a - else - if trace_entities then - report_xml("entity &%s; is made private",str) - end - -- a = "&" .. str .. ";" - a = unescaped(str) - acache[str] = a - end - end - return a - end -end - -local function handle_end_entity(chr) - report_xml("error in entity, %q found instead of ';'",chr) -end - -local space = S(' \r\n\t') -local open = P('<') -local close = P('>') -local squote = S("'") -local dquote = S('"') -local equal = P('=') -local slash = P('/') -local colon = P(':') -local semicolon = P(';') -local ampersand = P('&') -local valid = R('az', 'AZ', '09') + S('_-.') -local name_yes = C(valid^1) * colon * C(valid^1) -local name_nop = C(P(true)) * C(valid^1) -local name = name_yes + name_nop -local utfbom = lpeg.patterns.utfbom -- no capture -local spacing = C(space^0) - ------ entitycontent = (1-open-semicolon)^0 -local anyentitycontent = (1-open-semicolon-space-close)^0 -local hexentitycontent = R("AF","af","09")^0 -local decentitycontent = R("09")^0 -local parsedentity = P("#")/"" * ( - P("x")/"" * (hexentitycontent/handle_hex_entity) + - (decentitycontent/handle_dec_entity) - ) + (anyentitycontent/handle_any_entity) -local entity = ampersand/"" * parsedentity * ( (semicolon/"") + #(P(1)/handle_end_entity)) - -local text_unparsed = C((1-open)^1) -local text_parsed = Cs(((1-open-ampersand)^1 + entity)^1) - -local somespace = space^1 -local optionalspace = space^0 - ------ value = (squote * C((1 - squote)^0) * squote) + (dquote * C((1 - dquote)^0) * dquote) -- ampersand and < also invalid in value -local value = (squote * Cs((entity + (1 - squote))^0) * squote) + (dquote * Cs((entity + (1 - dquote))^0) * dquote) -- ampersand and < also invalid in value - -local endofattributes = slash * close + close -- recovery of flacky html -local whatever = space * name * optionalspace * equal ------ wrongvalue = C(P(1-whatever-close)^1 + P(1-close)^1) / attribute_value_error ------ wrongvalue = C(P(1-whatever-endofattributes)^1 + P(1-endofattributes)^1) / attribute_value_error ------ wrongvalue = C(P(1-space-endofattributes)^1) / attribute_value_error -local wrongvalue = Cs(P(entity + (1-space-endofattributes))^1) / attribute_value_error - -local attributevalue = value + wrongvalue - -local attribute = (somespace * name * optionalspace * equal * optionalspace * attributevalue) / add_attribute ------ attributes = (attribute)^0 - -local attributes = (attribute + somespace^-1 * (((1-endofattributes)^1)/attribute_specification_error))^0 - -local parsedtext = text_parsed / add_text -local unparsedtext = text_unparsed / add_text -local balanced = P { "[" * ((1 - S"[]") + V(1))^0 * "]" } -- taken from lpeg manual, () example - -local emptyelement = (spacing * open * name * attributes * optionalspace * slash * close) / add_empty -local beginelement = (spacing * open * name * attributes * optionalspace * close) / add_begin -local endelement = (spacing * open * slash * name * optionalspace * close) / add_end - -local begincomment = open * P("!--") -local endcomment = P("--") * close -local begininstruction = open * P("?") -local endinstruction = P("?") * close -local begincdata = open * P("![CDATA[") -local endcdata = P("]]") * close - -local someinstruction = C((1 - endinstruction)^0) -local somecomment = C((1 - endcomment )^0) -local somecdata = C((1 - endcdata )^0) - -local function normalentity(k,v ) entities[k] = v end -local function systementity(k,v,n) entities[k] = v end -local function publicentity(k,v,n) entities[k] = v end - --- todo: separate dtd parser - -local begindoctype = open * P("!DOCTYPE") -local enddoctype = close -local beginset = P("[") -local endset = P("]") -local doctypename = C((1-somespace-close)^0) -local elementdoctype = optionalspace * P(" & - cleanup = settings.text_cleanup - entities = settings.entities or { } - -- - if utfize == nil then - settings.utfize_entities = true - utfize = true - end - if resolve_predefined == nil then - settings.resolve_predefined_entities = true - resolve_predefined = true - end - -- - -- - stack, top, at, xmlns, errorstr = { }, { }, { }, { }, nil - acache, hcache, dcache = { }, { }, { } -- not stored - reported_attribute_errors = { } - if settings.parent_root then - mt = getmetatable(settings.parent_root) - else - initialize_mt(top) - end - stack[#stack+1] = top - top.dt = { } - dt = top.dt - if not data or data == "" then - errorstr = "empty xml file" - elseif utfize or resolve then - if lpegmatch(grammar_parsed_text,data) then - errorstr = "" - else - errorstr = "invalid xml file - parsed text" - end - elseif type(data) == "string" then - if lpegmatch(grammar_unparsed_text,data) then - errorstr = "" - else - errorstr = "invalid xml file - unparsed text" - end - else - errorstr = "invalid xml file - no text at all" - end - local result - if errorstr and errorstr ~= "" then - result = { dt = { { ns = "", tg = "error", dt = { errorstr }, at={ }, er = true } } } - setmetatable(stack, mt) - local errorhandler = settings.error_handler - if errorhandler == false then - -- no error message - else - errorhandler = errorhandler or xml.errorhandler - if errorhandler then - xml.errorhandler(format("load error: %s",errorstr)) - end - end - else - result = stack[1] - end - if not settings.no_root then - result = { special = true, ns = "", tg = '@rt@', dt = result.dt, at={ }, entities = entities, settings = settings } - setmetatable(result, mt) - local rdt = result.dt - for k=1,#rdt do - local v = rdt[k] - if type(v) == "table" and not v.special then -- always table -) - result.ri = k -- rootindex - v.__p__ = result -- new, experiment, else we cannot go back to settings, we need to test this ! - break - end - end - end - if errorstr and errorstr ~= "" then - result.error = true - end - result.statistics = { - entities = { - decimals = dcache, - hexadecimals = hcache, - names = acache, - } - } - strip, utfize, resolve, resolve_predefined = nil, nil, nil, nil - unify_predefined, cleanup, entities = nil, nil, nil - stack, top, at, xmlns, errorstr = nil, nil, nil, nil, nil - acache, hcache, dcache = nil, nil, nil - reported_attribute_errors, mt, errorhandler = nil, nil, nil - return result -end - --- Because we can have a crash (stack issues) with faulty xml, we wrap this one --- in a protector: - -function xmlconvert(data,settings) - local ok, result = pcall(function() return _xmlconvert_(data,settings) end) - if ok then - return result - else - return _xmlconvert_("") - end -end - -xml.convert = xmlconvert - -function xml.inheritedconvert(data,xmldata) -- xmldata is parent - local settings = xmldata.settings - if settings then - settings.parent_root = xmldata -- to be tested - end - -- settings.no_root = true - local xc = xmlconvert(data,settings) -- hm, we might need to locate settings - -- xc.settings = nil - -- xc.entities = nil - -- xc.special = nil - -- xc.ri = nil - -- print(xc.tg) - return xc -end - ---[[ldx-- -

Packaging data in an xml like table is done with the following -function. Maybe it will go away (when not used).

---ldx]]-- - -function xml.is_valid(root) - return root and root.dt and root.dt[1] and type(root.dt[1]) == "table" and not root.dt[1].er -end - -function xml.package(tag,attributes,data) - local ns, tg = match(tag,"^(.-):?([^:]+)$") - local t = { ns = ns, tg = tg, dt = data or "", at = attributes or {} } - setmetatable(t, mt) - return t -end - -function xml.is_valid(root) - return root and not root.error -end - -xml.errorhandler = report_xml - ---[[ldx-- -

We cannot load an from a filehandle so we need to load -the whole file first. The function accepts a string representing -a filename or a file handle.

---ldx]]-- - -function xml.load(filename,settings) - local data = "" - if type(filename) == "string" then - -- local data = io.loaddata(filename) - -todo: check type in io.loaddata - local f = io.open(filename,'r') - if f then - data = f:read("*all") - f:close() - end - elseif filename then -- filehandle - data = filename:read("*all") - end - return xmlconvert(data,settings) -end - ---[[ldx-- -

When we inject new elements, we need to convert strings to -valid trees, which is what the next function does.

---ldx]]-- - -local no_root = { no_root = true } - -function xml.toxml(data) - if type(data) == "string" then - local root = { xmlconvert(data,no_root) } - return (#root > 1 and root) or root[1] - else - return data - end -end - ---[[ldx-- -

For copying a tree we use a dedicated function instead of the -generic table copier. Since we know what we're dealing with we -can speed up things a bit. The second argument is not to be used!

---ldx]]-- - -local function copy(old,tables) - if old then - tables = tables or { } - local new = { } - if not tables[old] then - tables[old] = new - end - for k,v in next, old do - new[k] = (type(v) == "table" and (tables[v] or copy(v, tables))) or v - end - local mt = getmetatable(old) - if mt then - setmetatable(new,mt) - end - return new - else - return { } - end -end - -xml.copy = copy - ---[[ldx-- -

In serializing the tree or parts of the tree is a major -actitivity which is why the following function is pretty optimized resulting -in a few more lines of code than needed. The variant that uses the formatting -function for all components is about 15% slower than the concatinating -alternative.

---ldx]]-- - --- todo: add when not present - -function xml.checkbom(root) -- can be made faster - if root.ri then - local dt = root.dt - for k=1,#dt do - local v = dt[k] - if type(v) == "table" and v.special and v.tg == "@pi@" and find(v.dt[1],"xml.*version=") then - return - end - end - insert(dt, 1, { special=true, ns="", tg="@pi@", dt = { "xml version='1.0' standalone='yes'"} } ) - insert(dt, 2, "\n" ) - end -end - ---[[ldx-- -

At the cost of some 25% runtime overhead you can first convert the tree to a string -and then handle the lot.

---ldx]]-- - --- new experimental reorganized serialize - -local function verbose_element(e,handlers) -- options - local handle = handlers.handle - local serialize = handlers.serialize - local ens, etg, eat, edt, ern = e.ns, e.tg, e.at, e.dt, e.rn - local ats = eat and next(eat) and { } - if ats then - for k,v in next, eat do - ats[#ats+1] = format('%s=%q',k,escaped(v)) - end - end - if ern and trace_entities and ern ~= ens then - ens = ern - end - if ens ~= "" then - if edt and #edt > 0 then - if ats then - handle("<",ens,":",etg," ",concat(ats," "),">") - else - handle("<",ens,":",etg,">") - end - for i=1,#edt do - local e = edt[i] - if type(e) == "string" then - handle(escaped(e)) - else - serialize(e,handlers) - end - end - handle("") - else - if ats then - handle("<",ens,":",etg," ",concat(ats," "),"/>") - else - handle("<",ens,":",etg,"/>") - end - end - else - if edt and #edt > 0 then - if ats then - handle("<",etg," ",concat(ats," "),">") - else - handle("<",etg,">") - end - for i=1,#edt do - local e = edt[i] - if type(e) == "string" then - handle(escaped(e)) -- option: hexify escaped entities - else - serialize(e,handlers) - end - end - handle("") - else - if ats then - handle("<",etg," ",concat(ats," "),"/>") - else - handle("<",etg,"/>") - end - end - end -end - -local function verbose_pi(e,handlers) - handlers.handle("") -end - -local function verbose_comment(e,handlers) - handlers.handle("") -end - -local function verbose_cdata(e,handlers) - handlers.handle("") -end - -local function verbose_doctype(e,handlers) - handlers.handle("") -end - -local function verbose_root(e,handlers) - handlers.serialize(e.dt,handlers) -end - -local function verbose_text(e,handlers) - handlers.handle(escaped(e)) -end - -local function verbose_document(e,handlers) - local serialize = handlers.serialize - local functions = handlers.functions - for i=1,#e do - local ei = e[i] - if type(ei) == "string" then - functions["@tx@"](ei,handlers) - else - serialize(ei,handlers) - end - end -end - -local function serialize(e,handlers,...) - local initialize = handlers.initialize - local finalize = handlers.finalize - local functions = handlers.functions - if initialize then - local state = initialize(...) - if not state == true then - return state - end - end - local etg = e.tg - if etg then - (functions[etg] or functions["@el@"])(e,handlers) - -- elseif type(e) == "string" then - -- functions["@tx@"](e,handlers) - else - functions["@dc@"](e,handlers) -- dc ? - end - if finalize then - return finalize() - end -end - -local function xserialize(e,handlers) - local functions = handlers.functions - local etg = e.tg - if etg then - (functions[etg] or functions["@el@"])(e,handlers) - -- elseif type(e) == "string" then - -- functions["@tx@"](e,handlers) - else - functions["@dc@"](e,handlers) - end -end - -local handlers = { } - -local function newhandlers(settings) - local t = table.copy(handlers.verbose or { }) -- merge - if settings then - for k,v in next, settings do - if type(v) == "table" then - local tk = t[k] if not tk then tk = { } t[k] = tk end - for kk,vv in next, v do - tk[kk] = vv - end - else - t[k] = v - end - end - if settings.name then - handlers[settings.name] = t - end - end - utilities.storage.mark(t) - return t -end - -local nofunction = function() end - -function xml.sethandlersfunction(handler,name,fnc) - handler.functions[name] = fnc or nofunction -end - -function xml.gethandlersfunction(handler,name) - return handler.functions[name] -end - -function xml.gethandlers(name) - return handlers[name] -end - -newhandlers { - name = "verbose", - initialize = false, -- faster than nil and mt lookup - finalize = false, -- faster than nil and mt lookup - serialize = xserialize, - handle = print, - functions = { - ["@dc@"] = verbose_document, - ["@dt@"] = verbose_doctype, - ["@rt@"] = verbose_root, - ["@el@"] = verbose_element, - ["@pi@"] = verbose_pi, - ["@cm@"] = verbose_comment, - ["@cd@"] = verbose_cdata, - ["@tx@"] = verbose_text, - } -} - ---[[ldx-- -

How you deal with saving data depends on your preferences. For a 40 MB database -file the timing on a 2.3 Core Duo are as follows (time in seconds):

- - -1.3 : load data from file to string -6.1 : convert string into tree -5.3 : saving in file using xmlsave -6.8 : converting to string using xml.tostring -3.6 : saving converted string in file - - -

Beware, these were timing with the old routine but measurements will not be that -much different I guess.

---ldx]]-- - --- maybe this will move to lxml-xml - -local result - -local xmlfilehandler = newhandlers { - name = "file", - initialize = function(name) - result = io.open(name,"wb") - return result - end, - finalize = function() - result:close() - return true - end, - handle = function(...) - result:write(...) - end, -} - --- no checking on writeability here but not faster either --- --- local xmlfilehandler = newhandlers { --- initialize = function(name) --- io.output(name,"wb") --- return true --- end, --- finalize = function() --- io.close() --- return true --- end, --- handle = io.write, --- } - -function xml.save(root,name) - serialize(root,xmlfilehandler,name) -end - -local result - -local xmlstringhandler = newhandlers { - name = "string", - initialize = function() - result = { } - return result - end, - finalize = function() - return concat(result) - end, - handle = function(...) - result[#result+1] = concat { ... } - end, -} - -local function xmltostring(root) -- 25% overhead due to collecting - if not root then - return "" - elseif type(root) == 'string' then - return root - else -- if next(root) then -- next is faster than type (and >0 test) - return serialize(root,xmlstringhandler) or "" - end -end - -local function __tostring(root) -- inline - return (root and xmltostring(root)) or "" -end - -initialize_mt = function(root) -- redefinition - mt = { __tostring = __tostring, __index = root } -end - -xml.defaulthandlers = handlers -xml.newhandlers = newhandlers -xml.serialize = serialize -xml.tostring = xmltostring - ---[[ldx-- -

The next function operated on the content only and needs a handle function -that accepts a string.

---ldx]]-- - -local function xmlstring(e,handle) - if not handle or (e.special and e.tg ~= "@rt@") then - -- nothing - elseif e.tg then - local edt = e.dt - if edt then - for i=1,#edt do - xmlstring(edt[i],handle) - end - end - else - handle(e) - end -end - -xml.string = xmlstring - ---[[ldx-- -

A few helpers:

---ldx]]-- - - -function xml.settings(e) - while e do - local s = e.settings - if s then - return s - else - e = e.__p__ - end - end - return nil -end - -function xml.root(e) - local r = e - while e do - e = e.__p__ - if e then - r = e - end - end - return r -end - -function xml.parent(root) - return root.__p__ -end - -function xml.body(root) - return (root.ri and root.dt[root.ri]) or root -- not ok yet -end - -function xml.name(root) - if not root then - return "" - elseif root.ns == "" then - return root.tg - else - return root.ns .. ":" .. root.tg - end -end - ---[[ldx-- -

The next helper erases an element but keeps the table as it is, -and since empty strings are not serialized (effectively) it does -not harm. Copying the table would take more time. Usage:

---ldx]]-- - -function xml.erase(dt,k) - if dt then - if k then - dt[k] = "" - else for k=1,#dt do - dt[1] = { "" } - end end - end -end - ---[[ldx-- -

The next helper assigns a tree (or string). Usage:

- - -dt[k] = xml.assign(root) or xml.assign(dt,k,root) - ---ldx]]-- - -function xml.assign(dt,k,root) - if dt and k then - dt[k] = (type(root) == "table" and xml.body(root)) or root - return dt[k] - else - return xml.body(root) - end -end - --- the following helpers may move - ---[[ldx-- -

The next helper assigns a tree (or string). Usage:

- -xml.tocdata(e) -xml.tocdata(e,"error") - ---ldx]]-- - -function xml.tocdata(e,wrapper) -- a few more in the aux module - local whatever = type(e) == "table" and xmltostring(e.dt) or e or "" - if wrapper then - whatever = format("<%s>%s",wrapper,whatever,wrapper) - end - local t = { special = true, ns = "", tg = "@cd@", at = {}, rn = "", dt = { whatever }, __p__ = e } - setmetatable(t,getmetatable(e)) - e.dt = { t } -end - -function xml.makestandalone(root) - if root.ri then - local dt = root.dt - for k=1,#dt do - local v = dt[k] - if type(v) == "table" and v.special and v.tg == "@pi@" then - local txt = v.dt[1] - if find(txt,"xml.*version=") then - v.dt[1] = txt .. " standalone='yes'" - break - end - end - end - end - return root -end - -function xml.kind(e) - local dt = e and e.dt - if dt then - local n = #dt - if n == 1 then - local d = dt[1] - if d.special then - local tg = d.tg - if tg == "@cd@" then - return "cdata" - elseif tg == "@cm" then - return "comment" - elseif tg == "@pi@" then - return "instruction" - elseif tg == "@dt@" then - return "declaration" - end - elseif type(d) == "string" then - return "text" - end - return "element" - elseif n > 0 then - return "mixed" - end - end - return "empty" -end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['lxml-pth'] = { - version = 1.001, - comment = "this module is the basis for the lxml-* ones", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- e.ni is only valid after a filter run --- todo: B/C/[get first match] - -local concat, remove, insert = table.concat, table.remove, table.insert -local type, next, tonumber, tostring, setmetatable, loadstring = type, next, tonumber, tostring, setmetatable, loadstring -local format, upper, lower, gmatch, gsub, find, rep = string.format, string.upper, string.lower, string.gmatch, string.gsub, string.find, string.rep -local lpegmatch, lpegpatterns = lpeg.match, lpeg.patterns - -local setmetatableindex = table.setmetatableindex - --- beware, this is not xpath ... e.g. position is different (currently) and --- we have reverse-sibling as reversed preceding sibling - ---[[ldx-- -

This module can be used stand alone but also inside in -which case it hooks into the tracker code. Therefore we provide a few -functions that set the tracers. Here we overload a previously defined -function.

-

If I can get in the mood I will make a variant that is XSLT compliant -but I wonder if it makes sense.

---ldx]]-- - ---[[ldx-- -

Expecially the lpath code is experimental, we will support some of xpath, but -only things that make sense for us; as compensation it is possible to hook in your -own functions. Apart from preprocessing content for we also need -this module for process management, like handling and -files.

- - -a/b/c /*/c -a/b/c/first() a/b/c/last() a/b/c/index(n) a/b/c/index(-n) -a/b/c/text() a/b/c/text(1) a/b/c/text(-1) a/b/c/text(n) - ---ldx]]-- - -local trace_lpath = false if trackers then trackers.register("xml.path", function(v) trace_lpath = v end) end -local trace_lparse = false if trackers then trackers.register("xml.parse", function(v) trace_lparse = v end) end -local trace_lprofile = false if trackers then trackers.register("xml.profile", function(v) trace_lpath = v trace_lparse = v trace_lprofile = v end) end - -local report_lpath = logs.reporter("xml","lpath") - ---[[ldx-- -

We've now arrived at an interesting part: accessing the tree using a subset -of and since we're not compatible we call it . We -will explain more about its usage in other documents.

---ldx]]-- - -local xml = xml - -local lpathcalls = 0 function xml.lpathcalls () return lpathcalls end -local lpathcached = 0 function xml.lpathcached() return lpathcached end - -xml.functions = xml.functions or { } -- internal -local functions = xml.functions - -xml.expressions = xml.expressions or { } -- in expressions -local expressions = xml.expressions - -xml.finalizers = xml.finalizers or { } -- fast do-with ... (with return value other than collection) -local finalizers = xml.finalizers - -xml.specialhandler = xml.specialhandler or { } -local specialhandler = xml.specialhandler - -lpegpatterns.xml = lpegpatterns.xml or { } -local xmlpatterns = lpegpatterns.xml - -finalizers.xml = finalizers.xml or { } -finalizers.tex = finalizers.tex or { } - -local function fallback (t, name) - local fn = finalizers[name] - if fn then - t[name] = fn - else - report_lpath("unknown sub finalizer '%s'",tostring(name)) - fn = function() end - end - return fn -end - -setmetatableindex(finalizers.xml, fallback) -setmetatableindex(finalizers.tex, fallback) - -xml.defaultprotocol = "xml" - --- as xsl does not follow xpath completely here we will also --- be more liberal especially with regards to the use of | and --- the rootpath: --- --- test : all 'test' under current --- /test : 'test' relative to current --- a|b|c : set of names --- (a|b|c) : idem --- ! : not --- --- after all, we're not doing transformations but filtering. in --- addition we provide filter functions (last bit) --- --- todo: optimizer --- --- .. : parent --- * : all kids --- / : anchor here --- // : /**/ --- ** : all in between --- --- so far we had (more practical as we don't transform) --- --- {/test} : kids 'test' under current node --- {test} : any kid with tag 'test' --- {//test} : same as above - --- evaluator (needs to be redone, for the moment copied) - --- todo: apply_axis(list,notable) and collection vs single - -local apply_axis = { } - -apply_axis['root'] = function(list) - local collected = { } - for l=1,#list do - local ll = list[l] - local rt = ll - while ll do - ll = ll.__p__ - if ll then - rt = ll - end - end - collected[l] = rt - end - return collected -end - -apply_axis['self'] = function(list) - return list -end - -apply_axis['child'] = function(list) - local collected, c = { }, 0 - for l=1,#list do - local ll = list[l] - local dt = ll.dt - if dt then -- weird that this is needed - local en = 0 - for k=1,#dt do - local dk = dt[k] - if dk.tg then - c = c + 1 - collected[c] = dk - dk.ni = k -- refresh - en = en + 1 - dk.ei = en - end - end - ll.en = en - end - end - return collected -end - -local function collect(list,collected,c) - local dt = list.dt - if dt then - local en = 0 - for k=1,#dt do - local dk = dt[k] - if dk.tg then - c = c + 1 - collected[c] = dk - dk.ni = k -- refresh - en = en + 1 - dk.ei = en - c = collect(dk,collected,c) - end - end - list.en = en - end - return c -end - -apply_axis['descendant'] = function(list) - local collected, c = { }, 0 - for l=1,#list do - c = collect(list[l],collected,c) - end - return collected -end - -local function collect(list,collected,c) - local dt = list.dt - if dt then - local en = 0 - for k=1,#dt do - local dk = dt[k] - if dk.tg then - c = c + 1 - collected[c] = dk - dk.ni = k -- refresh - en = en + 1 - dk.ei = en - c = collect(dk,collected,c) - end - end - list.en = en - end - return c -end -apply_axis['descendant-or-self'] = function(list) - local collected, c = { }, 0 - for l=1,#list do - local ll = list[l] - if ll.special ~= true then -- catch double root - c = c + 1 - collected[c] = ll - end - c = collect(ll,collected,c) - end - return collected -end - -apply_axis['ancestor'] = function(list) - local collected, c = { }, 0 - for l=1,#list do - local ll = list[l] - while ll do - ll = ll.__p__ - if ll then - c = c + 1 - collected[c] = ll - end - end - end - return collected -end - -apply_axis['ancestor-or-self'] = function(list) - local collected, c = { }, 0 - for l=1,#list do - local ll = list[l] - c = c + 1 - collected[c] = ll - while ll do - ll = ll.__p__ - if ll then - c = c + 1 - collected[c] = ll - end - end - end - return collected -end - -apply_axis['parent'] = function(list) - local collected, c = { }, 0 - for l=1,#list do - local pl = list[l].__p__ - if pl then - c = c + 1 - collected[c] = pl - end - end - return collected -end - -apply_axis['attribute'] = function(list) - return { } -end - -apply_axis['namespace'] = function(list) - return { } -end - -apply_axis['following'] = function(list) -- incomplete - return { } -end - -apply_axis['preceding'] = function(list) -- incomplete - return { } -end - -apply_axis['following-sibling'] = function(list) - local collected, c = { }, 0 - for l=1,#list do - local ll = list[l] - local p = ll.__p__ - local d = p.dt - for i=ll.ni+1,#d do - local di = d[i] - if type(di) == "table" then - c = c + 1 - collected[c] = di - end - end - end - return collected -end - -apply_axis['preceding-sibling'] = function(list) - local collected, c = { }, 0 - for l=1,#list do - local ll = list[l] - local p = ll.__p__ - local d = p.dt - for i=1,ll.ni-1 do - local di = d[i] - if type(di) == "table" then - c = c + 1 - collected[c] = di - end - end - end - return collected -end - -apply_axis['reverse-sibling'] = function(list) -- reverse preceding - local collected, c = { }, 0 - for l=1,#list do - local ll = list[l] - local p = ll.__p__ - local d = p.dt - for i=ll.ni-1,1,-1 do - local di = d[i] - if type(di) == "table" then - c = c + 1 - collected[c] = di - end - end - end - return collected -end - -apply_axis['auto-descendant-or-self'] = apply_axis['descendant-or-self'] -apply_axis['auto-descendant'] = apply_axis['descendant'] -apply_axis['auto-child'] = apply_axis['child'] -apply_axis['auto-self'] = apply_axis['self'] -apply_axis['initial-child'] = apply_axis['child'] - -local function apply_nodes(list,directive,nodes) - -- todo: nodes[1] etc ... negated node name in set ... when needed - -- ... currently ignored - local maxn = #nodes - if maxn == 3 then --optimized loop - local nns, ntg = nodes[2], nodes[3] - if not nns and not ntg then -- wildcard - if directive then - return list - else - return { } - end - else - local collected, c, m, p = { }, 0, 0, nil - if not nns then -- only check tag - for l=1,#list do - local ll = list[l] - local ltg = ll.tg - if ltg then - if directive then - if ntg == ltg then - local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end - c = c + 1 - collected[c], ll.mi = ll, m - end - elseif ntg ~= ltg then - local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end - c = c + 1 - collected[c], ll.mi = ll, m - end - end - end - elseif not ntg then -- only check namespace - for l=1,#list do - local ll = list[l] - local lns = ll.rn or ll.ns - if lns then - if directive then - if lns == nns then - local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end - c = c + 1 - collected[c], ll.mi = ll, m - end - elseif lns ~= nns then - local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end - c = c + 1 - collected[c], ll.mi = ll, m - end - end - end - else -- check both - for l=1,#list do - local ll = list[l] - local ltg = ll.tg - if ltg then - local lns = ll.rn or ll.ns - local ok = ltg == ntg and lns == nns - if directive then - if ok then - local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end - c = c + 1 - collected[c], ll.mi = ll, m - end - elseif not ok then - local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end - c = c + 1 - collected[c], ll.mi = ll, m - end - end - end - end - return collected - end - else - local collected, c, m, p = { }, 0, 0, nil - for l=1,#list do - local ll = list[l] - local ltg = ll.tg - if ltg then - local lns = ll.rn or ll.ns - local ok = false - for n=1,maxn,3 do - local nns, ntg = nodes[n+1], nodes[n+2] - ok = (not ntg or ltg == ntg) and (not nns or lns == nns) - if ok then - break - end - end - if directive then - if ok then - local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end - c = c + 1 - collected[c], ll.mi = ll, m - end - elseif not ok then - local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end - c = c + 1 - collected[c], ll.mi = ll, m - end - end - end - return collected - end -end - -local quit_expression = false - -local function apply_expression(list,expression,order) - local collected, c = { }, 0 - quit_expression = false - for l=1,#list do - local ll = list[l] - if expression(list,ll,l,order) then -- nasty, order alleen valid als n=1 - c = c + 1 - collected[c] = ll - end - if quit_expression then - break - end - end - return collected -end - -local P, V, C, Cs, Cc, Ct, R, S, Cg, Cb = lpeg.P, lpeg.V, lpeg.C, lpeg.Cs, lpeg.Cc, lpeg.Ct, lpeg.R, lpeg.S, lpeg.Cg, lpeg.Cb - -local spaces = S(" \n\r\t\f")^0 -local lp_space = S(" \n\r\t\f") -local lp_any = P(1) -local lp_noequal = P("!=") / "~=" + P("<=") + P(">=") + P("==") -local lp_doequal = P("=") / "==" -local lp_or = P("|") / " or " -local lp_and = P("&") / " and " - -local lp_builtin = P ( - P("text") / "(ll.dt[1] or '')" + -- fragile - P("content") / "ll.dt" + - -- P("name") / "(ll.ns~='' and ll.ns..':'..ll.tg)" + - P("name") / "((ll.ns~='' and ll.ns..':'..ll.tg) or ll.tg)" + - P("tag") / "ll.tg" + - P("position") / "l" + -- is element in finalizer - P("firstindex") / "1" + - P("lastindex") / "(#ll.__p__.dt or 1)" + - P("firstelement") / "1" + - P("lastelement") / "(ll.__p__.en or 1)" + - P("first") / "1" + - P("last") / "#list" + - P("rootposition") / "order" + - P("order") / "order" + - P("element") / "(ll.ei or 1)" + - P("index") / "(ll.ni or 1)" + - P("match") / "(ll.mi or 1)" + - -- P("namespace") / "ll.ns" + - P("ns") / "ll.ns" - ) * ((spaces * P("(") * spaces * P(")"))/"") - --- for the moment we keep namespaces with attributes - -local lp_attribute = (P("@") + P("attribute::")) / "" * Cc("(ll.at and ll.at['") * ((R("az","AZ") + S("-_:"))^1) * Cc("'])") -local lp_fastpos_p = ((P("+")^0 * R("09")^1 * P(-1)) / function(s) return "l==" .. s end) -local lp_fastpos_n = ((P("-") * R("09")^1 * P(-1)) / function(s) return "(" .. s .. "<0 and (#list+".. s .. "==l))" end) -local lp_fastpos = lp_fastpos_n + lp_fastpos_p -local lp_reserved = C("and") + C("or") + C("not") + C("div") + C("mod") + C("true") + C("false") - -local lp_lua_function = C(R("az","AZ","__")^1 * (P(".") * R("az","AZ","__")^1)^1) * ("(") / function(t) -- todo: better . handling - return t .. "(" -end - -local lp_function = C(R("az","AZ","__")^1) * P("(") / function(t) -- todo: better . handling - if expressions[t] then - return "expr." .. t .. "(" - else - return "expr.error(" - end -end - -local lparent = P("(") -local rparent = P(")") -local noparent = 1 - (lparent+rparent) -local nested = P{lparent * (noparent + V(1))^0 * rparent} -local value = P(lparent * C((noparent + nested)^0) * rparent) -- P{"("*C(((1-S("()"))+V(1))^0)*")"} - -local lp_child = Cc("expr.child(ll,'") * R("az","AZ","--","__")^1 * Cc("')") -local lp_number = S("+-") * R("09")^1 -local lp_string = Cc("'") * R("az","AZ","--","__")^1 * Cc("'") -local lp_content = (P("'") * (1-P("'"))^0 * P("'") + P('"') * (1-P('"'))^0 * P('"')) - -local cleaner - -local lp_special = (C(P("name")+P("text")+P("tag")+P("count")+P("child"))) * value / function(t,s) - if expressions[t] then - s = s and s ~= "" and lpegmatch(cleaner,s) - if s and s ~= "" then - return "expr." .. t .. "(ll," .. s ..")" - else - return "expr." .. t .. "(ll)" - end - else - return "expr.error(" .. t .. ")" - end -end - -local content = - lp_builtin + - lp_attribute + - lp_special + - lp_noequal + lp_doequal + - lp_or + lp_and + - lp_reserved + - lp_lua_function + lp_function + - lp_content + -- too fragile - lp_child + - lp_any - -local converter = Cs ( - lp_fastpos + (P { lparent * (V(1))^0 * rparent + content } )^0 -) - -cleaner = Cs ( ( - lp_reserved + - lp_number + - lp_string + -1 )^1 ) - - - -local template_e = [[ - local expr = xml.expressions - return function(list,ll,l,order) - return %s - end -]] - -local template_f_y = [[ - local finalizer = xml.finalizers['%s']['%s'] - return function(collection) - return finalizer(collection,%s) - end -]] - -local template_f_n = [[ - return xml.finalizers['%s']['%s'] -]] - --- - -local register_self = { kind = "axis", axis = "self" } -- , apply = apply_axis["self"] } -local register_parent = { kind = "axis", axis = "parent" } -- , apply = apply_axis["parent"] } -local register_descendant = { kind = "axis", axis = "descendant" } -- , apply = apply_axis["descendant"] } -local register_child = { kind = "axis", axis = "child" } -- , apply = apply_axis["child"] } -local register_descendant_or_self = { kind = "axis", axis = "descendant-or-self" } -- , apply = apply_axis["descendant-or-self"] } -local register_root = { kind = "axis", axis = "root" } -- , apply = apply_axis["root"] } -local register_ancestor = { kind = "axis", axis = "ancestor" } -- , apply = apply_axis["ancestor"] } -local register_ancestor_or_self = { kind = "axis", axis = "ancestor-or-self" } -- , apply = apply_axis["ancestor-or-self"] } -local register_attribute = { kind = "axis", axis = "attribute" } -- , apply = apply_axis["attribute"] } -local register_namespace = { kind = "axis", axis = "namespace" } -- , apply = apply_axis["namespace"] } -local register_following = { kind = "axis", axis = "following" } -- , apply = apply_axis["following"] } -local register_following_sibling = { kind = "axis", axis = "following-sibling" } -- , apply = apply_axis["following-sibling"] } -local register_preceding = { kind = "axis", axis = "preceding" } -- , apply = apply_axis["preceding"] } -local register_preceding_sibling = { kind = "axis", axis = "preceding-sibling" } -- , apply = apply_axis["preceding-sibling"] } -local register_reverse_sibling = { kind = "axis", axis = "reverse-sibling" } -- , apply = apply_axis["reverse-sibling"] } - -local register_auto_descendant_or_self = { kind = "axis", axis = "auto-descendant-or-self" } -- , apply = apply_axis["auto-descendant-or-self"] } -local register_auto_descendant = { kind = "axis", axis = "auto-descendant" } -- , apply = apply_axis["auto-descendant"] } -local register_auto_self = { kind = "axis", axis = "auto-self" } -- , apply = apply_axis["auto-self"] } -local register_auto_child = { kind = "axis", axis = "auto-child" } -- , apply = apply_axis["auto-child"] } - -local register_initial_child = { kind = "axis", axis = "initial-child" } -- , apply = apply_axis["initial-child"] } - -local register_all_nodes = { kind = "nodes", nodetest = true, nodes = { true, false, false } } - -local skip = { } - -local function errorrunner_e(str,cnv) - if not skip[str] then - report_lpath("error in expression: %s => %s",str,cnv) - skip[str] = cnv or str - end - return false -end -local function errorrunner_f(str,arg) - report_lpath("error in finalizer: %s(%s)",str,arg or "") - return false -end - -local function register_nodes(nodetest,nodes) - return { kind = "nodes", nodetest = nodetest, nodes = nodes } -end - -local function register_expression(expression) - local converted = lpegmatch(converter,expression) - local runner = loadstring(format(template_e,converted)) - runner = (runner and runner()) or function() errorrunner_e(expression,converted) end - return { kind = "expression", expression = expression, converted = converted, evaluator = runner } -end - -local function register_finalizer(protocol,name,arguments) - local runner - if arguments and arguments ~= "" then - runner = loadstring(format(template_f_y,protocol or xml.defaultprotocol,name,arguments)) - else - runner = loadstring(format(template_f_n,protocol or xml.defaultprotocol,name)) - end - runner = (runner and runner()) or function() errorrunner_f(name,arguments) end - return { kind = "finalizer", name = name, arguments = arguments, finalizer = runner } -end - -local expression = P { "ex", - ex = "[" * C((V("sq") + V("dq") + (1 - S("[]")) + V("ex"))^0) * "]", - sq = "'" * (1 - S("'"))^0 * "'", - dq = '"' * (1 - S('"'))^0 * '"', -} - -local arguments = P { "ar", - ar = "(" * Cs((V("sq") + V("dq") + V("nq") + P(1-P(")")))^0) * ")", - nq = ((1 - S("),'\""))^1) / function(s) return format("%q",s) end, - sq = P("'") * (1 - P("'"))^0 * P("'"), - dq = P('"') * (1 - P('"'))^0 * P('"'), -} - --- todo: better arg parser - -local function register_error(str) - return { kind = "error", error = format("unparsed: %s",str) } -end - --- there is a difference in * and /*/ and so we need to catch a few special cases - -local special_1 = P("*") * Cc(register_auto_descendant) * Cc(register_all_nodes) -- last one not needed -local special_2 = P("/") * Cc(register_auto_self) -local special_3 = P("") * Cc(register_auto_self) - -local no_nextcolon = P(-1) + #(1-P(":")) -- newer lpeg needs the P(-1) -local no_nextlparent = P(-1) + #(1-P("(")) -- newer lpeg needs the P(-1) - -local pathparser = Ct { "patterns", -- can be made a bit faster by moving some patterns outside - - patterns = spaces * V("protocol") * spaces * ( - ( V("special") * spaces * P(-1) ) + - ( V("initial") * spaces * V("step") * spaces * (P("/") * spaces * V("step") * spaces)^0 ) - ), - - protocol = Cg(V("letters"),"protocol") * P("://") + Cg(Cc(nil),"protocol"), - - -- the / is needed for // as descendant or self is somewhat special - -- step = (V("shortcuts") + V("axis") * spaces * V("nodes")^0 + V("error")) * spaces * V("expressions")^0 * spaces * V("finalizer")^0, - step = ((V("shortcuts") + P("/") + V("axis")) * spaces * V("nodes")^0 + V("error")) * spaces * V("expressions")^0 * spaces * V("finalizer")^0, - - axis = V("descendant") + V("child") + V("parent") + V("self") + V("root") + V("ancestor") + - V("descendant_or_self") + V("following_sibling") + V("following") + - V("reverse_sibling") + V("preceding_sibling") + V("preceding") + V("ancestor_or_self") + - #(1-P(-1)) * Cc(register_auto_child), - - special = special_1 + special_2 + special_3, - - initial = (P("/") * spaces * Cc(register_initial_child))^-1, - - error = (P(1)^1) / register_error, - - shortcuts_a = V("s_descendant_or_self") + V("s_descendant") + V("s_child") + V("s_parent") + V("s_self") + V("s_root") + V("s_ancestor"), - - shortcuts = V("shortcuts_a") * (spaces * "/" * spaces * V("shortcuts_a"))^0, - - s_descendant_or_self = (P("***/") + P("/")) * Cc(register_descendant_or_self), --- *** is a bonus - s_descendant = P("**") * Cc(register_descendant), - s_child = P("*") * no_nextcolon * Cc(register_child ), - s_parent = P("..") * Cc(register_parent ), - s_self = P("." ) * Cc(register_self ), - s_root = P("^^") * Cc(register_root ), - s_ancestor = P("^") * Cc(register_ancestor ), - - descendant = P("descendant::") * Cc(register_descendant ), - child = P("child::") * Cc(register_child ), - parent = P("parent::") * Cc(register_parent ), - self = P("self::") * Cc(register_self ), - root = P('root::') * Cc(register_root ), - ancestor = P('ancestor::') * Cc(register_ancestor ), - descendant_or_self = P('descendant-or-self::') * Cc(register_descendant_or_self ), - ancestor_or_self = P('ancestor-or-self::') * Cc(register_ancestor_or_self ), - -- attribute = P('attribute::') * Cc(register_attribute ), - -- namespace = P('namespace::') * Cc(register_namespace ), - following = P('following::') * Cc(register_following ), - following_sibling = P('following-sibling::') * Cc(register_following_sibling ), - preceding = P('preceding::') * Cc(register_preceding ), - preceding_sibling = P('preceding-sibling::') * Cc(register_preceding_sibling ), - reverse_sibling = P('reverse-sibling::') * Cc(register_reverse_sibling ), - - nodes = (V("nodefunction") * spaces * P("(") * V("nodeset") * P(")") + V("nodetest") * V("nodeset")) / register_nodes, - - expressions = expression / register_expression, - - letters = R("az")^1, - name = (1-S("/[]()|:*!"))^1, -- make inline - negate = P("!") * Cc(false), - - nodefunction = V("negate") + P("not") * Cc(false) + Cc(true), - nodetest = V("negate") + Cc(true), - nodename = (V("negate") + Cc(true)) * spaces * ((V("wildnodename") * P(":") * V("wildnodename")) + (Cc(false) * V("wildnodename"))), - wildnodename = (C(V("name")) + P("*") * Cc(false)) * no_nextlparent, - nodeset = spaces * Ct(V("nodename") * (spaces * P("|") * spaces * V("nodename"))^0) * spaces, - - finalizer = (Cb("protocol") * P("/")^-1 * C(V("name")) * arguments * P(-1)) / register_finalizer, - -} - -xmlpatterns.pathparser = pathparser - -local cache = { } - -local function nodesettostring(set,nodetest) - local t = { } - for i=1,#set,3 do - local directive, ns, tg = set[i], set[i+1], set[i+2] - if not ns or ns == "" then ns = "*" end - if not tg or tg == "" then tg = "*" end - tg = (tg == "@rt@" and "[root]") or format("%s:%s",ns,tg) - t[i] = (directive and tg) or format("not(%s)",tg) - end - if nodetest == false then - return format("not(%s)",concat(t,"|")) - else - return concat(t,"|") - end -end - -local function tagstostring(list) - if #list == 0 then - return "no elements" - else - local t = { } - for i=1, #list do - local li = list[i] - local ns, tg = li.ns, li.tg - if not ns or ns == "" then ns = "*" end - if not tg or tg == "" then tg = "*" end - t[i] = (tg == "@rt@" and "[root]") or format("%s:%s",ns,tg) - end - return concat(t," ") - end -end - -xml.nodesettostring = nodesettostring - -local lpath -- we have a harmless kind of circular reference - -local lshowoptions = { functions = false } - -local function lshow(parsed) - if type(parsed) == "string" then - parsed = lpath(parsed) - end - report_lpath("%s://%s => %s",parsed.protocol or xml.defaultprotocol,parsed.pattern, - table.serialize(parsed,false,lshowoptions)) -end - -xml.lshow = lshow - -local function add_comment(p,str) - local pc = p.comment - if not pc then - p.comment = { str } - else - pc[#pc+1] = str - end -end - -lpath = function (pattern) -- the gain of caching is rather minimal - lpathcalls = lpathcalls + 1 - if type(pattern) == "table" then - return pattern - else - local parsed = cache[pattern] - if parsed then - lpathcached = lpathcached + 1 - else - parsed = lpegmatch(pathparser,pattern) - if parsed then - parsed.pattern = pattern - local np = #parsed - if np == 0 then - parsed = { pattern = pattern, register_self, state = "parsing error" } - report_lpath("parsing error in '%s'",pattern) - lshow(parsed) - else - -- we could have done this with a more complex parser but this - -- is cleaner - local pi = parsed[1] - if pi.axis == "auto-child" then - if false then - add_comment(parsed, "auto-child replaced by auto-descendant-or-self") - parsed[1] = register_auto_descendant_or_self - else - add_comment(parsed, "auto-child replaced by auto-descendant") - parsed[1] = register_auto_descendant - end - elseif pi.axis == "initial-child" and np > 1 and parsed[2].axis then - add_comment(parsed, "initial-child removed") -- we could also make it a auto-self - remove(parsed,1) - end - local np = #parsed -- can have changed - if np > 1 then - local pnp = parsed[np] - if pnp.kind == "nodes" and pnp.nodetest == true then - local nodes = pnp.nodes - if nodes[1] == true and nodes[2] == false and nodes[3] == false then - add_comment(parsed, "redundant final wildcard filter removed") - remove(parsed,np) - end - end - end - end - else - parsed = { pattern = pattern } - end - cache[pattern] = parsed - if trace_lparse and not trace_lprofile then - lshow(parsed) - end - end - return parsed - end -end - -xml.lpath = lpath - --- we can move all calls inline and then merge the trace back --- technically we can combine axis and the next nodes which is --- what we did before but this a bit cleaner (but slower too) --- but interesting is that it's not that much faster when we --- go inline --- --- beware: we need to return a collection even when we filter --- else the (simple) cache gets messed up - --- caching found lookups saves not that much (max .1 sec on a 8 sec run) --- and it also messes up finalizers - --- watch out: when there is a finalizer, it's always called as there --- can be cases that a finalizer returns (or does) something in case --- there is no match; an example of this is count() - -local profiled = { } xml.profiled = profiled - -local function profiled_apply(list,parsed,nofparsed,order) - local p = profiled[parsed.pattern] - if p then - p.tested = p.tested + 1 - else - p = { tested = 1, matched = 0, finalized = 0 } - profiled[parsed.pattern] = p - end - local collected = list - for i=1,nofparsed do - local pi = parsed[i] - local kind = pi.kind - if kind == "axis" then - collected = apply_axis[pi.axis](collected) - elseif kind == "nodes" then - collected = apply_nodes(collected,pi.nodetest,pi.nodes) - elseif kind == "expression" then - collected = apply_expression(collected,pi.evaluator,order) - elseif kind == "finalizer" then - collected = pi.finalizer(collected) -- no check on # here - p.matched = p.matched + 1 - p.finalized = p.finalized + 1 - return collected - end - if not collected or #collected == 0 then - local pn = i < nofparsed and parsed[nofparsed] - if pn and pn.kind == "finalizer" then - collected = pn.finalizer(collected) - p.finalized = p.finalized + 1 - return collected - end - return nil - end - end - if collected then - p.matched = p.matched + 1 - end - return collected -end - -local function traced_apply(list,parsed,nofparsed,order) - if trace_lparse then - lshow(parsed) - end - report_lpath("collecting: %s",parsed.pattern) - report_lpath("root tags : %s",tagstostring(list)) - report_lpath("order : %s",order or "unset") - local collected = list - for i=1,nofparsed do - local pi = parsed[i] - local kind = pi.kind - if kind == "axis" then - collected = apply_axis[pi.axis](collected) - report_lpath("% 10i : ax : %s",(collected and #collected) or 0,pi.axis) - elseif kind == "nodes" then - collected = apply_nodes(collected,pi.nodetest,pi.nodes) - report_lpath("% 10i : ns : %s",(collected and #collected) or 0,nodesettostring(pi.nodes,pi.nodetest)) - elseif kind == "expression" then - collected = apply_expression(collected,pi.evaluator,order) - report_lpath("% 10i : ex : %s -> %s",(collected and #collected) or 0,pi.expression,pi.converted) - elseif kind == "finalizer" then - collected = pi.finalizer(collected) - report_lpath("% 10i : fi : %s : %s(%s)",(type(collected) == "table" and #collected) or 0,parsed.protocol or xml.defaultprotocol,pi.name,pi.arguments or "") - return collected - end - if not collected or #collected == 0 then - local pn = i < nofparsed and parsed[nofparsed] - if pn and pn.kind == "finalizer" then - collected = pn.finalizer(collected) - report_lpath("% 10i : fi : %s : %s(%s)",(type(collected) == "table" and #collected) or 0,parsed.protocol or xml.defaultprotocol,pn.name,pn.arguments or "") - return collected - end - return nil - end - end - return collected -end - -local function normal_apply(list,parsed,nofparsed,order) - local collected = list - for i=1,nofparsed do - local pi = parsed[i] - local kind = pi.kind - if kind == "axis" then - local axis = pi.axis - if axis ~= "self" then - collected = apply_axis[axis](collected) - end - elseif kind == "nodes" then - collected = apply_nodes(collected,pi.nodetest,pi.nodes) - elseif kind == "expression" then - collected = apply_expression(collected,pi.evaluator,order) - elseif kind == "finalizer" then - return pi.finalizer(collected) - end - if not collected or #collected == 0 then - local pf = i < nofparsed and parsed[nofparsed].finalizer - if pf then - return pf(collected) -- can be anything - end - return nil - end - end - return collected -end - - -local function applylpath(list,pattern) - if not list then - return - end - local parsed = cache[pattern] - if parsed then - lpathcalls = lpathcalls + 1 - lpathcached = lpathcached + 1 - elseif type(pattern) == "table" then - lpathcalls = lpathcalls + 1 - parsed = pattern - else - parsed = lpath(pattern) or pattern - end - if not parsed then - return - end - local nofparsed = #parsed - if nofparsed == 0 then - return -- something is wrong - end - if not trace_lpath then - return normal_apply ({ list },parsed,nofparsed,list.mi) - elseif trace_lprofile then - return profiled_apply({ list },parsed,nofparsed,list.mi) - else - return traced_apply ({ list },parsed,nofparsed,list.mi) - end -end - -xml.applylpath = applylpath -- takes a table as first argment, which is what xml.filter will do - ---[[ldx-- -

This is the main filter function. It returns whatever is asked for.

---ldx]]-- - -function xml.filter(root,pattern) -- no longer funny attribute handling here - return applylpath(root,pattern) -end - --- internal (parsed) - -expressions.child = function(e,pattern) - return applylpath(e,pattern) -- todo: cache -end -expressions.count = function(e,pattern) -- what if pattern == empty or nil - local collected = applylpath(e,pattern) -- todo: cache - return pattern and (collected and #collected) or 0 -end - --- external - -expressions.oneof = function(s,...) -- slow - local t = {...} for i=1,#t do if s == t[i] then return true end end return false -end -expressions.error = function(str) - xml.errorhandler(format("unknown function in lpath expression: %s",tostring(str or "?"))) - return false -end -expressions.undefined = function(s) - return s == nil -end - -expressions.quit = function(s) - if s or s == nil then - quit_expression = true - end - return true -end - -expressions.print = function(...) - print(...) - return true -end - -expressions.contains = find -expressions.find = find -expressions.upper = upper -expressions.lower = lower -expressions.number = tonumber -expressions.boolean = toboolean - -function expressions.contains(str,pattern) - local t = type(str) - if t == "string" then - if find(str,pattern) then - return true - end - elseif t == "table" then - for i=1,#str do - local d = str[i] - if type(d) == "string" and find(d,pattern) then - return true - end - end - end - return false -end - --- user interface - -local function traverse(root,pattern,handle) - report_lpath("use 'xml.selection' instead for '%s'",pattern) - local collected = applylpath(root,pattern) - if collected then - for c=1,#collected do - local e = collected[c] - local r = e.__p__ - handle(r,r.dt,e.ni) - end - end -end - -local function selection(root,pattern,handle) - local collected = applylpath(root,pattern) - if collected then - if handle then - for c=1,#collected do - handle(collected[c]) - end - else - return collected - end - end -end - -xml.traverse = traverse -- old method, r, d, k -xml.selection = selection -- new method, simple handle - - --- generic function finalizer (independant namespace) - -local function dofunction(collected,fnc) - if collected then - local f = functions[fnc] - if f then - for c=1,#collected do - f(collected[c]) - end - else - report_lpath("unknown function '%s'",fnc) - end - end -end - -finalizers.xml["function"] = dofunction -finalizers.tex["function"] = dofunction - --- functions - -expressions.text = function(e,n) - local rdt = e.__p__.dt - return (rdt and rdt[n]) or "" -end - -expressions.name = function(e,n) -- ns + tg - local found = false - n = tonumber(n) or 0 - if n == 0 then - found = type(e) == "table" and e - elseif n < 0 then - local d, k = e.__p__.dt, e.ni - for i=k-1,1,-1 do - local di = d[i] - if type(di) == "table" then - if n == -1 then - found = di - break - else - n = n + 1 - end - end - end - else - local d, k = e.__p__.dt, e.ni - for i=k+1,#d,1 do - local di = d[i] - if type(di) == "table" then - if n == 1 then - found = di - break - else - n = n - 1 - end - end - end - end - if found then - local ns, tg = found.rn or found.ns or "", found.tg - if ns ~= "" then - return ns .. ":" .. tg - else - return tg - end - else - return "" - end -end - -expressions.tag = function(e,n) -- only tg - if not e then - return "" - else - local found = false - n = tonumber(n) or 0 - if n == 0 then - found = (type(e) == "table") and e -- seems to fail - elseif n < 0 then - local d, k = e.__p__.dt, e.ni - for i=k-1,1,-1 do - local di = d[i] - if type(di) == "table" then - if n == -1 then - found = di - break - else - n = n + 1 - end - end - end - else - local d, k = e.__p__.dt, e.ni - for i=k+1,#d,1 do - local di = d[i] - if type(di) == "table" then - if n == 1 then - found = di - break - else - n = n - 1 - end - end - end - end - return (found and found.tg) or "" - end -end - ---[[ldx-- -

Often using an iterators looks nicer in the code than passing handler -functions. The book describes how to use coroutines for that -purpose (). This permits -code like:

- - -for r, d, k in xml.elements(xml.load('text.xml'),"title") do - print(d[k]) -- old method -end -for e in xml.collected(xml.load('text.xml'),"title") do - print(e) -- new one -end - ---ldx]]-- - -local wrap, yield = coroutine.wrap, coroutine.yield - -function xml.elements(root,pattern,reverse) -- r, d, k - local collected = applylpath(root,pattern) - if collected then - if reverse then - return wrap(function() for c=#collected,1,-1 do - local e = collected[c] local r = e.__p__ yield(r,r.dt,e.ni) - end end) - else - return wrap(function() for c=1,#collected do - local e = collected[c] local r = e.__p__ yield(r,r.dt,e.ni) - end end) - end - end - return wrap(function() end) -end - -function xml.collected(root,pattern,reverse) -- e - local collected = applylpath(root,pattern) - if collected then - if reverse then - return wrap(function() for c=#collected,1,-1 do yield(collected[c]) end end) - else - return wrap(function() for c=1,#collected do yield(collected[c]) end end) - end - end - return wrap(function() end) -end - --- handy - -function xml.inspect(collection,pattern) - pattern = pattern or "." - for e in xml.collected(collection,pattern or ".") do - report_lpath("pattern %q\n\n%s\n",pattern,xml.tostring(e)) - end -end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['lxml-mis'] = { - version = 1.001, - comment = "this module is the basis for the lxml-* ones", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local xml, lpeg, string = xml, lpeg, string - -local concat = table.concat -local type, next, tonumber, tostring, setmetatable, loadstring = type, next, tonumber, tostring, setmetatable, loadstring -local format, gsub, match = string.format, string.gsub, string.match -local lpegmatch, lpegpatterns = lpeg.match, lpeg.patterns -local P, S, R, C, V, Cc, Cs = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc, lpeg.Cs - -lpegpatterns.xml = lpegpatterns.xml or { } -local xmlpatterns = lpegpatterns.xml - ---[[ldx-- -

The following helper functions best belong to the lxml-ini -module. Some are here because we need then in the mk -document and other manuals, others came up when playing with -this module. Since this module is also used in we've -put them here instead of loading mode modules there then needed.

---ldx]]-- - -local function xmlgsub(t,old,new) -- will be replaced - local dt = t.dt - if dt then - for k=1,#dt do - local v = dt[k] - if type(v) == "string" then - dt[k] = gsub(v,old,new) - else - xmlgsub(v,old,new) - end - end - end -end - - -function xml.stripleadingspaces(dk,d,k) -- cosmetic, for manual - if d and k then - local dkm = d[k-1] - if dkm and type(dkm) == "string" then - local s = match(dkm,"\n(%s+)") - xmlgsub(dk,"\n"..rep(" ",#s),"\n") - end - end -end - - - --- 100 * 2500 * "oeps< oeps> oeps&" : gsub:lpeg|lpeg|lpeg --- --- 1021:0335:0287:0247 - --- 10 * 1000 * "oeps< oeps> oeps& asfjhalskfjh alskfjh alskfjh alskfjh ;al J;LSFDJ" --- --- 1559:0257:0288:0190 (last one suggested by roberto) - --- escaped = Cs((S("<&>") / xml.escapes + 1)^0) --- escaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0) -local normal = (1 - S("<&>"))^0 -local special = P("<")/"<" + P(">")/">" + P("&")/"&" -local escaped = Cs(normal * (special * normal)^0) - --- 100 * 1000 * "oeps< oeps> oeps&" : gsub:lpeg == 0153:0280:0151:0080 (last one by roberto) - -local normal = (1 - S"&")^0 -local special = P("<")/"<" + P(">")/">" + P("&")/"&" -local unescaped = Cs(normal * (special * normal)^0) - --- 100 * 5000 * "oeps oeps oeps " : gsub:lpeg == 623:501 msec (short tags, less difference) - -local cleansed = Cs(((P("<") * (1-P(">"))^0 * P(">"))/"" + 1)^0) - -xmlpatterns.escaped = escaped -xmlpatterns.unescaped = unescaped -xmlpatterns.cleansed = cleansed - -function xml.escaped (str) return lpegmatch(escaped,str) end -function xml.unescaped(str) return lpegmatch(unescaped,str) end -function xml.cleansed (str) return lpegmatch(cleansed,str) end - --- this might move - -function xml.fillin(root,pattern,str,check) - local e = xml.first(root,pattern) - if e then - local n = #e.dt - if not check or n == 0 or (n == 1 and e.dt[1] == "") then - e.dt = { str } - end - end -end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['lxml-aux'] = { - version = 1.001, - comment = "this module is the basis for the lxml-* ones", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- not all functions here make sense anymore vbut we keep them for --- compatibility reasons - -local trace_manipulations = false trackers.register("lxml.manipulations", function(v) trace_manipulations = v end) - -local report_xml = logs.reporter("xml") - -local xml = xml - -local xmlconvert, xmlcopy, xmlname = xml.convert, xml.copy, xml.name -local xmlinheritedconvert = xml.inheritedconvert -local xmlapplylpath = xml.applylpath -local xmlfilter = xml.filter - -local type, setmetatable, getmetatable = type, setmetatable, getmetatable -local insert, remove, fastcopy, concat = table.insert, table.remove, table.fastcopy, table.concat -local gmatch, gsub, format, find, strip = string.gmatch, string.gsub, string.format, string.find, string.strip -local utfbyte = utf.byte - -local function report(what,pattern,c,e) - report_xml("%s element '%s' (root: '%s', position: %s, index: %s, pattern: %s)",what,xmlname(e),xmlname(e.__p__),c,e.ni,pattern) -end - -local function withelements(e,handle,depth) - if e and handle then - local edt = e.dt - if edt then - depth = depth or 0 - for i=1,#edt do - local e = edt[i] - if type(e) == "table" then - handle(e,depth) - withelements(e,handle,depth+1) - end - end - end - end -end - -xml.withelements = withelements - -function xml.withelement(e,n,handle) -- slow - if e and n ~= 0 and handle then - local edt = e.dt - if edt then - if n > 0 then - for i=1,#edt do - local ei = edt[i] - if type(ei) == "table" then - if n == 1 then - handle(ei) - return - else - n = n - 1 - end - end - end - elseif n < 0 then - for i=#edt,1,-1 do - local ei = edt[i] - if type(ei) == "table" then - if n == -1 then - handle(ei) - return - else - n = n + 1 - end - end - end - end - end - end -end - -function xml.each(root,pattern,handle,reverse) - local collected = xmlapplylpath(root,pattern) - if collected then - if reverse then - for c=#collected,1,-1 do - handle(collected[c]) - end - else - for c=1,#collected do - handle(collected[c]) - end - end - return collected - end -end - -function xml.processattributes(root,pattern,handle) - local collected = xmlapplylpath(root,pattern) - if collected and handle then - for c=1,#collected do - handle(collected[c].at) - end - end - return collected -end - ---[[ldx-- -

The following functions collect elements and texts.

---ldx]]-- - --- are these still needed -> lxml-cmp.lua - -function xml.collect(root, pattern) - return xmlapplylpath(root,pattern) -end - -function xml.collecttexts(root, pattern, flatten) -- todo: variant with handle - local collected = xmlapplylpath(root,pattern) - if collected and flatten then - local xmltostring = xml.tostring - for c=1,#collected do - collected[c] = xmltostring(collected[c].dt) - end - end - return collected or { } -end - -function xml.collect_tags(root, pattern, nonamespace) - local collected = xmlapplylpath(root,pattern) - if collected then - local t, n = { }, 0 - for c=1,#collected do - local e = collected[c] - local ns, tg = e.ns, e.tg - n = n + 1 - if nonamespace then - t[n] = tg - elseif ns == "" then - t[n] = tg - else - t[n] = ns .. ":" .. tg - end - end - return t - end -end - ---[[ldx-- -

We've now arrived at the functions that manipulate the tree.

---ldx]]-- - -local no_root = { no_root = true } - -local function redo_ni(d) - for k=1,#d do - local dk = d[k] - if type(dk) == "table" then - dk.ni = k - end - end -end - -local function xmltoelement(whatever,root) - if not whatever then - return nil - end - local element - if type(whatever) == "string" then - element = xmlinheritedconvert(whatever,root) -- beware, not really a root - else - element = whatever -- we assume a table - end - if element.error then - return whatever -- string - end - if element then - end - return element -end - -xml.toelement = xmltoelement - -local function copiedelement(element,newparent) - if type(element) == "string" then - return element - else - element = xmlcopy(element).dt - if newparent and type(element) == "table" then - element.__p__ = newparent - end - return element - end -end - -function xml.delete(root,pattern) - if not pattern or pattern == "" then - local p = root.__p__ - if p then - if trace_manipulations then - report('deleting',"--",c,root) - end - local d = p.dt - remove(d,root.ni) - redo_ni(d) -- can be made faster and inlined - end - else - local collected = xmlapplylpath(root,pattern) - if collected then - for c=1,#collected do - local e = collected[c] - local p = e.__p__ - if p then - if trace_manipulations then - report('deleting',pattern,c,e) - end - local d = p.dt - remove(d,e.ni) - redo_ni(d) -- can be made faster and inlined - end - end - end - end -end - -function xml.replace(root,pattern,whatever) - local element = root and xmltoelement(whatever,root) - local collected = element and xmlapplylpath(root,pattern) - if collected then - for c=1,#collected do - local e = collected[c] - local p = e.__p__ - if p then - if trace_manipulations then - report('replacing',pattern,c,e) - end - local d = p.dt - d[e.ni] = copiedelement(element,p) - redo_ni(d) -- probably not needed - end - end - end -end - -local function wrap(e,wrapper) - local t = { - rn = e.rn, - tg = e.tg, - ns = e.ns, - at = e.at, - dt = e.dt, - __p__ = e, - } - setmetatable(t,getmetatable(e)) - e.rn = wrapper.rn or e.rn or "" - e.tg = wrapper.tg or e.tg or "" - e.ns = wrapper.ns or e.ns or "" - e.at = fastcopy(wrapper.at) - e.dt = { t } -end - -function xml.wrap(root,pattern,whatever) - if whatever then - local wrapper = xmltoelement(whatever,root) - local collected = xmlapplylpath(root,pattern) - if collected then - for c=1,#collected do - local e = collected[c] - if trace_manipulations then - report('wrapping',pattern,c,e) - end - wrap(e,wrapper) - end - end - else - wrap(root,xmltoelement(pattern)) - end -end - -local function inject_element(root,pattern,whatever,prepend) - local element = root and xmltoelement(whatever,root) - local collected = element and xmlapplylpath(root,pattern) - local function inject_e(e) - local r = e.__p__ - local d, k, rri = r.dt, e.ni, r.ri - local edt = (rri and d[rri].dt) or (d and d[k] and d[k].dt) - if edt then - local be, af - local cp = copiedelement(element,e) - if prepend then - be, af = cp, edt - else - be, af = edt, cp - end - local bn = #be - for i=1,#af do - bn = bn + 1 - be[bn] = af[i] - end - if rri then - r.dt[rri].dt = be - else - d[k].dt = be - end - redo_ni(d) - end - end - if not collected then - -- nothing - elseif collected.tg then - -- first or so - inject_e(collected) - else - for c=1,#collected do - inject_e(collected[c]) - end - end -end - -local function insert_element(root,pattern,whatever,before) -- todo: element als functie - local element = root and xmltoelement(whatever,root) - local collected = element and xmlapplylpath(root,pattern) - local function insert_e(e) - local r = e.__p__ - local d, k = r.dt, e.ni - if not before then - k = k + 1 - end - insert(d,k,copiedelement(element,r)) - redo_ni(d) - end - if not collected then - -- nothing - elseif collected.tg then - -- first or so - insert_e(collected) - else - for c=1,#collected do - insert_e(collected[c]) - end - end -end - -xml.insert_element = insert_element -xml.insertafter = insert_element -xml.insertbefore = function(r,p,e) insert_element(r,p,e,true) end -xml.injectafter = inject_element -xml.injectbefore = function(r,p,e) inject_element(r,p,e,true) end - -local function include(xmldata,pattern,attribute,recursive,loaddata) - -- parse="text" (default: xml), encoding="" (todo) - -- attribute = attribute or 'href' - pattern = pattern or 'include' - loaddata = loaddata or io.loaddata - local collected = xmlapplylpath(xmldata,pattern) - if collected then - for c=1,#collected do - local ek = collected[c] - local name = nil - local ekdt = ek.dt - local ekat = ek.at - local epdt = ek.__p__.dt - if not attribute or attribute == "" then - name = (type(ekdt) == "table" and ekdt[1]) or ekdt -- check, probably always tab or str - end - if not name then - for a in gmatch(attribute or "href","([^|]+)") do - name = ekat[a] - if name then break end - end - end - local data = (name and name ~= "" and loaddata(name)) or "" - if data == "" then - epdt[ek.ni] = "" -- xml.empty(d,k) - elseif ekat["parse"] == "text" then - -- for the moment hard coded - epdt[ek.ni] = xml.escaped(data) -- d[k] = xml.escaped(data) - else - local xi = xmlinheritedconvert(data,xmldata) - if not xi then - epdt[ek.ni] = "" -- xml.empty(d,k) - else - if recursive then - include(xi,pattern,attribute,recursive,loaddata) - end - epdt[ek.ni] = xml.body(xi) -- xml.assign(d,k,xi) - end - end - end - end -end - -xml.include = include - -local function stripelement(e,nolines,anywhere) - local edt = e.dt - if edt then - if anywhere then - local t, n = { }, 0 - for e=1,#edt do - local str = edt[e] - if type(str) ~= "string" then - n = n + 1 - t[n] = str - elseif str ~= "" then - -- todo: lpeg for each case - if nolines then - str = gsub(str,"%s+"," ") - end - str = gsub(str,"^%s*(.-)%s*$","%1") - if str ~= "" then - n = n + 1 - t[n] = str - end - end - end - e.dt = t - else - -- we can assume a regular sparse xml table with no successive strings - -- otherwise we should use a while loop - if #edt > 0 then - -- strip front - local str = edt[1] - if type(str) ~= "string" then - -- nothing - elseif str == "" then - remove(edt,1) - else - if nolines then - str = gsub(str,"%s+"," ") - end - str = gsub(str,"^%s+","") - if str == "" then - remove(edt,1) - else - edt[1] = str - end - end - end - local nedt = #edt - if nedt > 0 then - -- strip end - local str = edt[nedt] - if type(str) ~= "string" then - -- nothing - elseif str == "" then - remove(edt) - else - if nolines then - str = gsub(str,"%s+"," ") - end - str = gsub(str,"%s+$","") - if str == "" then - remove(edt) - else - edt[nedt] = str - end - end - end - end - end - return e -- convenient -end - -xml.stripelement = stripelement - -function xml.strip(root,pattern,nolines,anywhere) -- strips all leading and trailing spacing - local collected = xmlapplylpath(root,pattern) -- beware, indices no longer are valid now - if collected then - for i=1,#collected do - stripelement(collected[i],nolines,anywhere) - end - end -end - -local function renamespace(root, oldspace, newspace) -- fast variant - local ndt = #root.dt - for i=1,ndt or 0 do - local e = root[i] - if type(e) == "table" then - if e.ns == oldspace then - e.ns = newspace - if e.rn then - e.rn = newspace - end - end - local edt = e.dt - if edt then - renamespace(edt, oldspace, newspace) - end - end - end -end - -xml.renamespace = renamespace - -function xml.remaptag(root, pattern, newtg) - local collected = xmlapplylpath(root,pattern) - if collected then - for c=1,#collected do - collected[c].tg = newtg - end - end -end - -function xml.remapnamespace(root, pattern, newns) - local collected = xmlapplylpath(root,pattern) - if collected then - for c=1,#collected do - collected[c].ns = newns - end - end -end - -function xml.checknamespace(root, pattern, newns) - local collected = xmlapplylpath(root,pattern) - if collected then - for c=1,#collected do - local e = collected[c] - if (not e.rn or e.rn == "") and e.ns == "" then - e.rn = newns - end - end - end -end - -function xml.remapname(root, pattern, newtg, newns, newrn) - local collected = xmlapplylpath(root,pattern) - if collected then - for c=1,#collected do - local e = collected[c] - e.tg, e.ns, e.rn = newtg, newns, newrn - end - end -end - ---[[ldx-- -

Helper (for q2p).

---ldx]]-- - -function xml.cdatatotext(e) - local dt = e.dt - if #dt == 1 then - local first = dt[1] - if first.tg == "@cd@" then - e.dt = first.dt - end - else - -- maybe option - end -end - --- local x = xml.convert("123") --- xml.texttocdata(xml.first(x,"a")) --- print(x) -- 23]]> - -function xml.texttocdata(e) -- could be a finalizer - local dt = e.dt - local s = xml.tostring(dt) -- no shortcut? - e.tg = "@cd@" - e.special = true - e.ns = "" - e.rn = "" - e.dt = { s } - e.at = nil -end - --- local x = xml.convert("123") --- xml.tocdata(xml.first(x,"a")) --- print(x) -- 123]]> - -function xml.elementtocdata(e) -- could be a finalizer - local dt = e.dt - local s = xml.tostring(e) -- no shortcut? - e.tg = "@cd@" - e.special = true - e.ns = "" - e.rn = "" - e.dt = { s } - e.at = nil -end - -xml.builtinentities = table.tohash { "amp", "quot", "apos", "lt", "gt" } -- used often so share - -local entities = characters and characters.entities or nil -local builtinentities = xml.builtinentities - -function xml.addentitiesdoctype(root,option) -- we could also have a 'resolve' i.e. inline hex - if not entities then - require("char-ent") - entities = characters.entities - end - if entities and root and root.tg == "@rt@" and root.statistics then - local list = { } - local hexify = option == "hexadecimal" - for k, v in table.sortedhash(root.statistics.entities.names) do - if not builtinentities[k] then - local e = entities[k] - if not e then - e = format("[%s]",k) - elseif hexify then - e = format("&#%05X;",utfbyte(k)) - end - list[#list+1] = format(" ",k,e) - end - end - local dt = root.dt - local n = dt[1].tg == "@pi@" and 2 or 1 - if #list > 0 then - insert(dt, n, { "\n" }) - insert(dt, n, { - tg = "@dt@", -- beware, doctype is unparsed - dt = { format("Something [\n%s\n] ",concat(list)) }, - ns = "", - special = true, - }) - insert(dt, n, { "\n\n" }) - else - -- insert(dt, n, { table.serialize(root.statistics) }) - end - end -end - --- local str = [==[ --- --- --- test   test { test --- --- --- ]==] --- --- local x = xml.convert(str) --- xml.addentitiesdoctype(x,"hexadecimal") --- print(x) - ---[[ldx-- -

Here are a few synonyms.

---ldx]]-- - -xml.all = xml.each -xml.insert = xml.insertafter -xml.inject = xml.injectafter -xml.after = xml.insertafter -xml.before = xml.insertbefore -xml.process = xml.each - --- obsolete - -xml.obsolete = xml.obsolete or { } -local obsolete = xml.obsolete - -xml.strip_whitespace = xml.strip obsolete.strip_whitespace = xml.strip -xml.collect_elements = xml.collect obsolete.collect_elements = xml.collect -xml.delete_element = xml.delete obsolete.delete_element = xml.delete -xml.replace_element = xml.replace obsolete.replace_element = xml.replacet -xml.each_element = xml.each obsolete.each_element = xml.each -xml.process_elements = xml.process obsolete.process_elements = xml.process -xml.insert_element_after = xml.insertafter obsolete.insert_element_after = xml.insertafter -xml.insert_element_before = xml.insertbefore obsolete.insert_element_before = xml.insertbefore -xml.inject_element_after = xml.injectafter obsolete.inject_element_after = xml.injectafter -xml.inject_element_before = xml.injectbefore obsolete.inject_element_before = xml.injectbefore -xml.process_attributes = xml.processattributes obsolete.process_attributes = xml.processattributes -xml.collect_texts = xml.collecttexts obsolete.collect_texts = xml.collecttexts -xml.inject_element = xml.inject obsolete.inject_element = xml.inject -xml.remap_tag = xml.remaptag obsolete.remap_tag = xml.remaptag -xml.remap_name = xml.remapname obsolete.remap_name = xml.remapname -xml.remap_namespace = xml.remapnamespace obsolete.remap_namespace = xml.remapnamespace - --- new (probably ok) - -function xml.cdata(e) - if e then - local dt = e.dt - if dt and #dt == 1 then - local first = dt[1] - return first.tg == "@cd@" and first.dt[1] or "" - end - end - return "" -end - -function xml.finalizers.xml.cdata(collected) - if collected then - local e = collected[1] - if e then - local dt = e.dt - if dt and #dt == 1 then - local first = dt[1] - return first.tg == "@cd@" and first.dt[1] or "" - end - end - end - return "" -end - -function xml.insertcomment(e,str,n) -- also insertcdata - table.insert(e.dt,n or 1,{ - tg = "@cm@", - ns = "", - special = true, - at = { }, - dt = { str }, - }) -end - -function xml.setcdata(e,str) -- also setcomment - e.dt = { { - tg = "@cd@", - ns = "", - special = true, - at = { }, - dt = { str }, - } } -end - --- maybe helpers like this will move to an autoloader - -function xml.separate(x,pattern) - local collected = xmlapplylpath(x,pattern) - if collected then - for c=1,#collected do - local e = collected[c] - local d = e.dt - if d == x then - report_xml("warning: xml.separate changes root") - x = d - end - local t, n = { "\n" }, 1 - local i, nd = 1, #d - while i <= nd do - while i <= nd do - local di = d[i] - if type(di) == "string" then - if di == "\n" or find(di,"^%s+$") then -- first test is speedup - i = i + 1 - else - d[i] = strip(di) - break - end - else - break - end - end - if i > nd then - break - end - t[n+1] = "\n" - t[n+2] = d[i] - t[n+3] = "\n" - n = n + 3 - i = i + 1 - end - t[n+1] = "\n" - setmetatable(t,getmetatable(d)) - e.dt = t - end - end - return x -end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['lxml-xml'] = { - version = 1.001, - comment = "this module is the basis for the lxml-* ones", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local concat = table.concat -local find, lower, upper = string.find, string.lower, string.upper - -local xml = xml - -local finalizers = xml.finalizers.xml -local xmlfilter = xml.filter -- we could inline this one for speed -local xmltostring = xml.tostring -local xmlserialize = xml.serialize -local xmlcollected = xml.collected -local xmlnewhandlers = xml.newhandlers - -local function first(collected) -- wrong ? - return collected and collected[1] -end - -local function last(collected) - return collected and collected[#collected] -end - -local function all(collected) - return collected -end - --- local function reverse(collected) --- if collected then --- local nc = #collected --- if nc > 0 then --- local reversed, r = { }, 0 --- for c=nc,1,-1 do --- r = r + 1 --- reversed[r] = collected[c] --- end --- return reversed --- else --- return collected --- end --- end --- end - -local reverse = table.reversed - -local function attribute(collected,name) - if collected and #collected > 0 then - local at = collected[1].at - return at and at[name] - end -end - -local function att(id,name) - local at = id.at - return at and at[name] -end - -local function count(collected) - return collected and #collected or 0 -end - -local function position(collected,n) - if not collected then - return 0 - end - local nc = #collected - if nc == 0 then - return 0 - end - n = tonumber(n) or 0 - if n < 0 then - return collected[nc + n + 1] - elseif n > 0 then - return collected[n] - else - return collected[1].mi or 0 - end -end - -local function match(collected) - return collected and #collected > 0 and collected[1].mi or 0 -- match -end - -local function index(collected) - return collected and #collected > 0 and collected[1].ni or 0 -- 0 is new -end - -local function attributes(collected,arguments) - if collected and #collected > 0 then - local at = collected[1].at - if arguments then - return at[arguments] - elseif next(at) then - return at -- all of them - end - end -end - -local function chainattribute(collected,arguments) -- todo: optional levels - if collected and #collected > 0 then - local e = collected[1] - while e do - local at = e.at - if at then - local a = at[arguments] - if a then - return a - end - else - break -- error - end - e = e.__p__ - end - end - return "" -end - -local function raw(collected) -- hybrid (not much different from text so it might go) - if collected and #collected > 0 then - local e = collected[1] or collected - return e and xmltostring(e) or "" -- only first as we cannot concat function - else - return "" - end -end - --- - -local xmltexthandler = xmlnewhandlers { - name = "string", - initialize = function() - result = { } - return result - end, - finalize = function() - return concat(result) - end, - handle = function(...) - result[#result+1] = concat { ... } - end, - escape = false, -} - -local function xmltotext(root) - local dt = root.dt - if not dt then - return "" - end - local nt = #dt -- string or table - if nt == 0 then - return "" - elseif nt == 1 and type(dt[1]) == "string" then - return dt[1] -- no escaping of " ' < > & - else - return xmlserialize(root,xmltexthandler) or "" - end -end - --- - -local function text(collected) -- hybrid - if collected then -- no # test here ! - local e = collected[1] or collected -- why fallback to element, how about cdata - return e and xmltotext(e) or "" - else - return "" - end -end - -local function texts(collected) - if not collected then - return { } -- why no nil - end - local nc = #collected - if nc == 0 then - return { } -- why no nil - end - local t, n = { }, 0 - for c=1,nc do - local e = collected[c] - if e and e.dt then - n = n + 1 - t[n] = e.dt - end - end - return t -end - -local function tag(collected,n) - if not collected then - return - end - local nc = #collected - if nc == 0 then - return - end - local c - if n == 0 or not n then - c = collected[1] - elseif n > 1 then - c = collected[n] - else - c = collected[nc-n+1] - end - return c and c.tg -end - -local function name(collected,n) - if not collected then - return - end - local nc = #collected - if nc == 0 then - return - end - local c - if n == 0 or not n then - c = collected[1] - elseif n > 1 then - c = collected[n] - else - c = collected[nc-n+1] - end - if not c then - -- sorry - elseif c.ns == "" then - return c.tg - else - return c.ns .. ":" .. c.tg - end -end - -local function tags(collected,nonamespace) - if not collected then - return - end - local nc = #collected - if nc == 0 then - return - end - local t, n = { }, 0 - for c=1,nc do - local e = collected[c] - local ns, tg = e.ns, e.tg - n = n + 1 - if nonamespace or ns == "" then - t[n] = tg - else - t[n] = ns .. ":" .. tg - end - end - return t -end - -local function empty(collected,spacesonly) - if not collected then - return true - end - local nc = #collected - if nc == 0 then - return true - end - for c=1,nc do - local e = collected[c] - if e then - local edt = e.dt - if edt then - local n = #edt - if n == 1 then - local edk = edt[1] - local typ = type(edk) - if typ == "table" then - return false - elseif edk ~= "" then - return false - elseif spacesonly and not find(edk,"%S") then - return false - end - elseif n > 1 then - return false - end - end - end - end - return true -end - -finalizers.first = first -finalizers.last = last -finalizers.all = all -finalizers.reverse = reverse -finalizers.elements = all -finalizers.default = all -finalizers.attribute = attribute -finalizers.att = att -finalizers.count = count -finalizers.position = position -finalizers.match = match -finalizers.index = index -finalizers.attributes = attributes -finalizers.chainattribute = chainattribute -finalizers.text = text -finalizers.texts = texts -finalizers.tag = tag -finalizers.name = name -finalizers.tags = tags -finalizers.empty = empty - --- shortcuts -- we could support xmlfilter(id,pattern,first) - -function xml.first(id,pattern) - return first(xmlfilter(id,pattern)) -end - -function xml.last(id,pattern) - return last(xmlfilter(id,pattern)) -end - -function xml.count(id,pattern) - return count(xmlfilter(id,pattern)) -end - -function xml.attribute(id,pattern,a,default) - return attribute(xmlfilter(id,pattern),a,default) -end - -function xml.raw(id,pattern) - if pattern then - return raw(xmlfilter(id,pattern)) - else - return raw(id) - end -end - -function xml.text(id,pattern) -- brrr either content or element (when cdata) - if pattern then - -- return text(xmlfilter(id,pattern)) - local collected = xmlfilter(id,pattern) - return collected and #collected > 0 and xmltotext(collected[1]) or "" - elseif id then - -- return text(id) - return xmltotext(id) or "" - else - return "" - end -end - -xml.content = text - --- - -function xml.position(id,pattern,n) -- element - return position(xmlfilter(id,pattern),n) -end - -function xml.match(id,pattern) -- number - return match(xmlfilter(id,pattern)) -end - -function xml.empty(id,pattern,spacesonly) - return empty(xmlfilter(id,pattern),spacesonly) -end - -xml.all = xml.filter -xml.index = xml.position -xml.found = xml.filter - --- a nice one: - -local function totable(x) - local t = { } - for e in xmlcollected(x[1] or x,"/*") do - t[e.tg] = xmltostring(e.dt) or "" - end - return next(t) and t or nil -end - -xml.table = totable -finalizers.table = totable - -local function textonly(e,t) - if e then - local edt = e.dt - if edt then - for i=1,#edt do - local e = edt[i] - if type(e) == "table" then - textonly(e,t) - else - t[#t+1] = e - end - end - end - end - return t -end - -function xml.textonly(e) -- no pattern - return concat(textonly(e,{})) -end - --- - --- local x = xml.convert("123") --- xml.filter(x,"**/lowerall()") print(x) --- xml.filter(x,"**/upperall()") print(x) - -function finalizers.lowerall(collected) - for c=1,#collected do - local e = collected[c] - if not e.special then - e.tg = lower(e.tg) - local eat = e.at - if eat then - local t = { } - for k,v in next, eat do - t[lower(k)] = v - end - e.at = t - end - end - end -end - -function finalizers.upperall(collected) - for c=1,#collected do - local e = collected[c] - if not e.special then - e.tg = upper(e.tg) - local eat = e.at - if eat then - local t = { } - for k,v in next, eat do - t[upper(k)] = v - end - e.at = t - end - end - end -end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['data-ini'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files", -} - -local gsub, find, gmatch, char = string.gsub, string.find, string.gmatch, string.char -local concat = table.concat -local next, type = next, type - -local filedirname, filebasename, fileextname, filejoin = file.dirname, file.basename, file.extname, file.join - -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) -local trace_detail = false trackers.register("resolvers.details", function(v) trace_detail = v end) -local trace_expansions = false trackers.register("resolvers.expansions", function(v) trace_expansions = v end) - -local report_initialization = logs.reporter("resolvers","initialization") - -local ostype, osname, ossetenv, osgetenv = os.type, os.name, os.setenv, os.getenv - --- The code here used to be part of a data-res but for convenience --- we now split it over multiple files. As this file is now the --- starting point we introduce resolvers here. - -resolvers = resolvers or { } -local resolvers = resolvers - --- We don't want the kpse library to kick in. Also, we want to be able to --- execute programs. Control over execution is implemented later. - -texconfig.kpse_init = false -texconfig.shell_escape = 't' - -if kpse and kpse.default_texmfcnf then - local default_texmfcnf = kpse.default_texmfcnf() - -- looks more like context: - default_texmfcnf = gsub(default_texmfcnf,"$SELFAUTOLOC","selfautoloc:") - default_texmfcnf = gsub(default_texmfcnf,"$SELFAUTODIR","selfautodir:") - default_texmfcnf = gsub(default_texmfcnf,"$SELFAUTOPARENT","selfautoparent:") - default_texmfcnf = gsub(default_texmfcnf,"$HOME","home:") - -- - environment.default_texmfcnf = default_texmfcnf -end - -kpse = { original = kpse } - -setmetatable(kpse, { - __index = function(kp,name) - report_initialization("fatal error: kpse library is accessed (key: %s)",name) - os.exit() - end -} ) - --- First we check a couple of environment variables. Some might be --- set already but we need then later on. We start with the system --- font path. - -do - - local osfontdir = osgetenv("OSFONTDIR") - - if osfontdir and osfontdir ~= "" then - -- ok - elseif osname == "windows" then - ossetenv("OSFONTDIR","c:/windows/fonts//") - elseif osname == "macosx" then - ossetenv("OSFONTDIR","$HOME/Library/Fonts//;/Library/Fonts//;/System/Library/Fonts//") - end - -end - --- Next comes the user's home path. We need this as later on we have --- to replace ~ with its value. - -do - - local homedir = osgetenv(ostype == "windows" and 'USERPROFILE' or 'HOME') or '' - - if not homedir or homedir == "" then - homedir = char(127) -- we need a value, later we wil trigger on it - end - - homedir = file.collapsepath(homedir) - - ossetenv("HOME", homedir) -- can be used in unix cnf files - ossetenv("USERPROFILE",homedir) -- can be used in windows cnf files - - environment.homedir = homedir - -end - --- The following code sets the name of the own binary and its --- path. This is fallback code as we have os.selfdir now. - -do - - local args = environment.originalarguments or arg -- this needs a cleanup - - local ownbin = environment.ownbin or args[-2] or arg[-2] or args[-1] or arg[-1] or arg[0] or "luatex" - local ownpath = environment.ownpath or os.selfdir - - ownbin = file.collapsepath(ownbin) - ownpath = file.collapsepath(ownpath) - - if not ownpath or ownpath == "" or ownpath == "unset" then - ownpath = args[-1] or arg[-1] - ownpath = ownpath and filedirname(gsub(ownpath,"\\","/")) - if not ownpath or ownpath == "" then - ownpath = args[-0] or arg[-0] - ownpath = ownpath and filedirname(gsub(ownpath,"\\","/")) - end - local binary = ownbin - if not ownpath or ownpath == "" then - ownpath = ownpath and filedirname(binary) - end - if not ownpath or ownpath == "" then - if os.binsuffix ~= "" then - binary = file.replacesuffix(binary,os.binsuffix) - end - local path = osgetenv("PATH") - if path then - for p in gmatch(path,"[^"..io.pathseparator.."]+") do - local b = filejoin(p,binary) - if lfs.isfile(b) then - -- we assume that after changing to the path the currentdir function - -- resolves to the real location and use this side effect here; this - -- trick is needed because on the mac installations use symlinks in the - -- path instead of real locations - local olddir = lfs.currentdir() - if lfs.chdir(p) then - local pp = lfs.currentdir() - if trace_locating and p ~= pp then - report_initialization("following symlink '%s' to '%s'",p,pp) - end - ownpath = pp - lfs.chdir(olddir) - else - if trace_locating then - report_initialization("unable to check path '%s'",p) - end - ownpath = p - end - break - end - end - end - end - if not ownpath or ownpath == "" then - ownpath = "." - report_initialization("forcing fallback ownpath .") - elseif trace_locating then - report_initialization("using ownpath '%s'",ownpath) - end - end - - environment.ownbin = ownbin - environment.ownpath = ownpath - -end - -resolvers.ownpath = environment.ownpath - -function resolvers.getownpath() - return environment.ownpath -end - --- The self variables permit us to use only a few (or even no) --- environment variables. - -do - - local ownpath = environment.ownpath or dir.current() - - if ownpath then - ossetenv('SELFAUTOLOC', file.collapsepath(ownpath)) - ossetenv('SELFAUTODIR', file.collapsepath(ownpath .. "/..")) - ossetenv('SELFAUTOPARENT', file.collapsepath(ownpath .. "/../..")) - else - report_initialization("error: unable to locate ownpath") - os.exit() - end - -end - --- The running os: - --- todo: check is context sits here os.platform is more trustworthy --- that the bin check as mtx-update runs from another path - -local texos = environment.texos or osgetenv("TEXOS") -local texmfos = environment.texmfos or osgetenv('SELFAUTODIR') - -if not texos or texos == "" then - texos = file.basename(texmfos) -end - -ossetenv('TEXMFOS', texmfos) -- full bin path -ossetenv('TEXOS', texos) -- partial bin parent -ossetenv('SELFAUTOSYSTEM',os.platform) -- bonus - -environment.texos = texos -environment.texmfos = texmfos - --- The current root: - -local texroot = environment.texroot or osgetenv("TEXROOT") - -if not texroot or texroot == "" then - texroot = osgetenv('SELFAUTOPARENT') - ossetenv('TEXROOT',texroot) -end - -environment.texroot = file.collapsepath(texroot) - --- Tracing. Todo ... - -function resolvers.settrace(n) -- no longer number but: 'locating' or 'detail' - if n then - trackers.disable("resolvers.*") - trackers.enable("resolvers."..n) - end -end - -resolvers.settrace(osgetenv("MTX_INPUT_TRACE")) - --- todo: - --- if profiler and osgetenv("MTX_PROFILE_RUN") == "YES" then --- profiler.start("luatex-profile.log") --- end - --- a forward definition - -if not resolvers.resolve then - function resolvers.resolve (s) return s end - function resolvers.unresolve(s) return s end - function resolvers.repath (s) return s end -end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['data-exp'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files", -} - -local format, find, gmatch, lower, char, sub = string.format, string.find, string.gmatch, string.lower, string.char, string.sub -local concat, sort = table.concat, table.sort -local lpegmatch, lpegpatterns = lpeg.match, lpeg.patterns -local Ct, Cs, Cc, P, C, S = lpeg.Ct, lpeg.Cs, lpeg.Cc, lpeg.P, lpeg.C, lpeg.S -local type, next = type, next - -local ostype = os.type -local collapsepath = file.collapsepath - -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) -local trace_expansions = false trackers.register("resolvers.expansions", function(v) trace_expansions = v end) - -local report_expansions = logs.reporter("resolvers","expansions") - -local resolvers = resolvers - --- As this bit of code is somewhat special it gets its own module. After --- all, when working on the main resolver code, I don't want to scroll --- past this every time. See data-obs.lua for the gsub variant. - -local function f_first(a,b) - local t, n = { }, 0 - for s in gmatch(b,"[^,]+") do - n = n + 1 ; t[n] = a .. s - end - return concat(t,",") -end - -local function f_second(a,b) - local t, n = { }, 0 - for s in gmatch(a,"[^,]+") do - n = n + 1 ; t[n] = s .. b - end - return concat(t,",") -end - --- kpsewhich --expand-braces '{a,b}{c,d}' --- ac:bc:ad:bd - --- old {a,b}{c,d} => ac ad bc bd --- --- local function f_both(a,b) --- local t, n = { }, 0 --- for sa in gmatch(a,"[^,]+") do --- for sb in gmatch(b,"[^,]+") do --- n = n + 1 ; t[n] = sa .. sb --- end --- end --- return concat(t,",") --- end --- --- new {a,b}{c,d} => ac bc ad bd - -local function f_both(a,b) - local t, n = { }, 0 - for sb in gmatch(b,"[^,]+") do -- and not sa - for sa in gmatch(a,"[^,]+") do -- sb - n = n + 1 ; t[n] = sa .. sb - end - end - return concat(t,",") -end - -local left = P("{") -local right = P("}") -local var = P((1 - S("{}" ))^0) -local set = P((1 - S("{},"))^0) -local other = P(1) - -local l_first = Cs( ( Cc("{") * (C(set) * left * C(var) * right / f_first) * Cc("}") + other )^0 ) -local l_second = Cs( ( Cc("{") * (left * C(var) * right * C(set) / f_second) * Cc("}") + other )^0 ) -local l_both = Cs( ( Cc("{") * (left * C(var) * right * left * C(var) * right / f_both) * Cc("}") + other )^0 ) -local l_rest = Cs( ( left * var * (left/"") * var * (right/"") * var * right + other )^0 ) - -local stripper_1 = lpeg.stripper ("{}@") -local replacer_1 = lpeg.replacer { { ",}", ",@}" }, { "{,", "{@," }, } - -local function splitpathexpr(str, newlist, validate) -- I couldn't resist lpegging it (nice exercise). - if trace_expansions then - report_expansions("expanding variable '%s'",str) - end - local t, ok, done = newlist or { }, false, false - local n = #t - str = lpegmatch(replacer_1,str) - repeat - local old = str - repeat - local old = str - str = lpegmatch(l_first, str) - until old == str - repeat - local old = str - str = lpegmatch(l_second,str) - until old == str - repeat - local old = str - str = lpegmatch(l_both, str) - until old == str - repeat - local old = str - str = lpegmatch(l_rest, str) - until old == str - until old == str -- or not find(str,"{") - str = lpegmatch(stripper_1,str) - if validate then - for s in gmatch(str,"[^,]+") do - s = validate(s) - if s then - n = n + 1 ; t[n] = s - end - end - else - for s in gmatch(str,"[^,]+") do - n = n + 1 ; t[n] = s - end - end - if trace_expansions then - for k=1,#t do - report_expansions("% 4i: %s",k,t[k]) - end - end - return t -end - --- We could make the previous one public. - -local function validate(s) - s = collapsepath(s) -- already keeps the // - return s ~= "" and not find(s,"^!*unset/*$") and s -end - -resolvers.validatedpath = validate -- keeps the trailing // - -function resolvers.expandedpathfromlist(pathlist) - local newlist = { } - for k=1,#pathlist do - splitpathexpr(pathlist[k],newlist,validate) - end - return newlist -end - --- {a,b,c,d} --- a,b,c/{p,q,r},d --- a,b,c/{p,q,r}/d/{x,y,z}// --- a,b,c/{p,q/{x,y,z},r},d/{p,q,r} --- a,b,c/{p,q/{x,y,z},r},d/{p,q,r} --- a{b,c}{d,e}f --- {a,b,c,d} --- {a,b,c/{p,q,r},d} --- {a,b,c/{p,q,r}/d/{x,y,z}//} --- {a,b,c/{p,q/{x,y,z}},d/{p,q,r}} --- {a,b,c/{p,q/{x,y,z},w}v,d/{p,q,r}} --- {$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c} - -local cleanup = lpeg.replacer { - { "!" , "" }, - { "\\" , "/" }, -} - -function resolvers.cleanpath(str) -- tricky, maybe only simple paths - local doslashes = (P("\\")/"/" + 1)^0 - local donegation = (P("!") /"" )^0 - local homedir = lpegmatch(Cs(donegation * doslashes),environment.homedir or "") - if homedir == "~" or homedir == "" or not lfs.isdir(homedir) then - if trace_expansions then - report_expansions("no home dir set, ignoring dependent paths") - end - function resolvers.cleanpath(str) - if not str or find(str,"~") then - return "" -- special case - else - return lpegmatch(cleanup,str) - end - end - else - local dohome = ((P("~")+P("$HOME"))/homedir)^0 - local cleanup = Cs(donegation * dohome * doslashes) - function resolvers.cleanpath(str) - return str and lpegmatch(cleanup,str) or "" - end - end - return resolvers.cleanpath(str) -end - --- print(resolvers.cleanpath("")) --- print(resolvers.cleanpath("!")) --- print(resolvers.cleanpath("~")) --- print(resolvers.cleanpath("~/test")) --- print(resolvers.cleanpath("!~/test")) --- print(resolvers.cleanpath("~/test~test")) - --- This one strips quotes and funny tokens. - -local expandhome = P("~") / "$HOME" -- environment.homedir - -local dodouble = P('"')/"" * (expandhome + (1 - P('"')))^0 * P('"')/"" -local dosingle = P("'")/"" * (expandhome + (1 - P("'")))^0 * P("'")/"" -local dostring = (expandhome + 1 )^0 - -local stripper = Cs( - lpegpatterns.unspacer * (dosingle + dodouble + dostring) * lpegpatterns.unspacer -) - -function resolvers.checkedvariable(str) -- assumes str is a string - return type(str) == "string" and lpegmatch(stripper,str) or str -end - --- The path splitter: - --- A config (optionally) has the paths split in tables. Internally --- we join them and split them after the expansion has taken place. This --- is more convenient. - -local cache = { } - ------ splitter = lpeg.tsplitat(S(ostype == "windows" and ";" or ":;")) -- maybe add , -local splitter = lpeg.tsplitat(";") -- as we move towards urls, prefixes and use tables we no longer do : - -local backslashswapper = lpeg.replacer("\\","/") - -local function splitconfigurationpath(str) -- beware, this can be either a path or a { specification } - if str then - local found = cache[str] - if not found then - if str == "" then - found = { } - else - local split = lpegmatch(splitter,lpegmatch(backslashswapper,str)) -- can be combined - found = { } - local noffound = 0 - for i=1,#split do - local s = split[i] - if not find(s,"^{*unset}*") then - noffound = noffound + 1 - found[noffound] = s - end - end - if trace_expansions then - report_expansions("splitting path specification '%s'",str) - for k=1,noffound do - report_expansions("% 4i: %s",k,found[k]) - end - end - cache[str] = found - end - end - return found - end -end - -resolvers.splitconfigurationpath = splitconfigurationpath - -function resolvers.splitpath(str) - if type(str) == 'table' then - return str - else - return splitconfigurationpath(str) - end -end - -function resolvers.joinpath(str) - if type(str) == 'table' then - return file.joinpath(str) - else - return str - end -end - --- The next function scans directories and returns a hash where the --- entries are either strings or tables. - --- starting with . or .. etc or funny char - - - - --- a lot of this caching can be stripped away when we have ssd's everywhere --- --- we could cache all the (sub)paths here if needed - -local attributes, directory = lfs.attributes, lfs.dir - -local weird = P(".")^1 + lpeg.anywhere(S("~`!#$%^&*()={}[]:;\"\'||<>,?\n\r\t")) -local timer = { } -local scanned = { } -local nofscans = 0 -local scancache = { } - -local function scan(files,spec,path,n,m,r) - local full = (path == "" and spec) or (spec .. path .. '/') - local dirs = { } - local nofdirs = 0 - for name in directory(full) do - if not lpegmatch(weird,name) then - local mode = attributes(full..name,'mode') - if mode == 'file' then - n = n + 1 - local f = files[name] - if f then - if type(f) == 'string' then - files[name] = { f, path } - else - f[#f+1] = path - end - else -- probably unique anyway - files[name] = path - local lower = lower(name) - if name ~= lower then - files["remap:"..lower] = name - r = r + 1 - end - end - elseif mode == 'directory' then - m = m + 1 - nofdirs = nofdirs + 1 - if path ~= "" then - dirs[nofdirs] = path..'/'..name - else - dirs[nofdirs] = name - end - end - end - end - if nofdirs > 0 then - sort(dirs) - for i=1,nofdirs do - files, n, m, r = scan(files,spec,dirs[i],n,m,r) - end - end - scancache[sub(full,1,-2)] = files - return files, n, m, r -end - -local fullcache = { } - -function resolvers.scanfiles(path,branch,usecache) - statistics.starttiming(timer) - local realpath = resolvers.resolve(path) -- no shortcut - if usecache then - local files = fullcache[realpath] - if files then - if trace_locating then - report_expansions("using caches scan of path '%s', branch '%s'",path,branch or path) - end - return files - end - end - if trace_locating then - report_expansions("scanning path '%s', branch '%s'",path,branch or path) - end - local files, n, m, r = scan({ },realpath .. '/',"",0,0,0) - files.__path__ = path -- can be selfautoparent:texmf-whatever - files.__files__ = n - files.__directories__ = m - files.__remappings__ = r - if trace_locating then - report_expansions("%s files found on %s directories with %s uppercase remappings",n,m,r) - end - if usecache then - scanned[#scanned+1] = realpath - fullcache[realpath] = files - end - nofscans = nofscans + 1 - statistics.stoptiming(timer) - return files -end - -local function simplescan(files,spec,path) -- first match only, no map and such - local full = (path == "" and spec) or (spec .. path .. '/') - local dirs = { } - local nofdirs = 0 - for name in directory(full) do - if not lpegmatch(weird,name) then - local mode = attributes(full..name,'mode') - if mode == 'file' then - if not files[name] then - -- only first match - files[name] = path - end - elseif mode == 'directory' then - nofdirs = nofdirs + 1 - if path ~= "" then - dirs[nofdirs] = path..'/'..name - else - dirs[nofdirs] = name - end - end - end - end - if nofdirs > 0 then - sort(dirs) - for i=1,nofdirs do - files = simplescan(files,spec,dirs[i]) - end - end - return files -end - -local simplecache = { } -local nofsharedscans = 0 - -function resolvers.simplescanfiles(path,branch,usecache) - statistics.starttiming(timer) - local realpath = resolvers.resolve(path) -- no shortcut - if usecache then - local files = simplecache[realpath] - if not files then - files = scancache[realpath] - if files then - nofsharedscans = nofsharedscans + 1 - end - end - if files then - if trace_locating then - report_expansions("using caches scan of path '%s', branch '%s'",path,branch or path) - end - return files - end - end - if trace_locating then - report_expansions("scanning path '%s', branch '%s'",path,branch or path) - end - local files = simplescan({ },realpath .. '/',"") - if trace_locating then - report_expansions("%s files found",table.count(files)) - end - if usecache then - scanned[#scanned+1] = realpath - simplecache[realpath] = files - end - nofscans = nofscans + 1 - statistics.stoptiming(timer) - return files -end - -function resolvers.scandata() - table.sort(scanned) - return { - n = nofscans, - shared = nofsharedscans, - time = statistics.elapsedtime(timer), - paths = scanned, - } -end - - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['data-env'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files", -} - -local lower, gsub = string.lower, string.gsub - -local resolvers = resolvers - -local allocate = utilities.storage.allocate -local setmetatableindex = table.setmetatableindex -local fileextname = file.extname - -local formats = allocate() -local suffixes = allocate() -local dangerous = allocate() -local suffixmap = allocate() - -resolvers.formats = formats -resolvers.suffixes = suffixes -resolvers.dangerous = dangerous -resolvers.suffixmap = suffixmap - -local relations = allocate { -- todo: handlers also here - core = { - ofm = { -- will become obsolete - names = { "ofm", "omega font metric", "omega font metrics" }, - variable = 'OFMFONTS', - suffixes = { 'ofm', 'tfm' }, - }, - ovf = { -- will become obsolete - names = { "ovf", "omega virtual font", "omega virtual fonts" }, - variable = 'OVFFONTS', - suffixes = { 'ovf', 'vf' }, - }, - tfm = { - names = { "tfm", "tex font metric", "tex font metrics" }, - variable = 'TFMFONTS', - suffixes = { 'tfm' }, - }, - vf = { - names = { "vf", "virtual font", "virtual fonts" }, - variable = 'VFFONTS', - suffixes = { 'vf' }, - }, - otf = { - names = { "otf", "opentype", "opentype font", "opentype fonts"}, - variable = 'OPENTYPEFONTS', - suffixes = { 'otf' }, - }, - ttf = { - names = { "ttf", "truetype", "truetype font", "truetype fonts", "truetype collection", "truetype collections", "truetype dictionary", "truetype dictionaries" }, - variable = 'TTFONTS', - suffixes = { 'ttf', 'ttc', 'dfont' }, - }, - afm = { - names = { "afm", "adobe font metric", "adobe font metrics" }, - variable = "AFMFONTS", - suffixes = { "afm" }, - }, - pfb = { - names = { "pfb", "type1", "type 1", "type1 font", "type 1 font", "type1 fonts", "type 1 fonts" }, - variable = 'T1FONTS', - suffixes = { 'pfb', 'pfa' }, - }, - fea = { - names = { "fea", "font feature", "font features", "font feature file", "font feature files" }, - variable = 'FONTFEATURES', - suffixes = { 'fea' }, - }, - cid = { - names = { "cid", "cid map", "cid maps", "cid file", "cid files" }, - variable = 'FONTCIDMAPS', - suffixes = { 'cid', 'cidmap' }, - }, - fmt = { - names = { "fmt", "format", "tex format" }, - variable = 'TEXFORMATS', - suffixes = { 'fmt' }, - }, - mem = { -- will become obsolete - names = { 'mem', "metapost format" }, - variable = 'MPMEMS', - suffixes = { 'mem' }, - }, - mp = { - names = { "mp" }, - variable = 'MPINPUTS', - suffixes = { 'mp', 'mpvi', 'mpiv', 'mpii' }, - }, - tex = { - names = { "tex" }, - variable = 'TEXINPUTS', - suffixes = { 'tex', "mkvi", "mkiv", "mkii" }, - }, - icc = { - names = { "icc", "icc profile", "icc profiles" }, - variable = 'ICCPROFILES', - suffixes = { 'icc' }, - }, - texmfscripts = { - names = { "texmfscript", "texmfscripts", "script", "scripts" }, - variable = 'TEXMFSCRIPTS', - suffixes = { 'rb', 'pl', 'py' }, - }, - lua = { - names = { "lua" }, - variable = 'LUAINPUTS', - suffixes = { 'lua', 'luc', 'tma', 'tmc' }, - }, - lib = { - names = { "lib" }, - variable = 'CLUAINPUTS', - suffixes = os.libsuffix and { os.libsuffix } or { 'dll', 'so' }, - }, - bib = { - names = { 'bib' }, - suffixes = { 'bib' }, - }, - bst = { - names = { 'bst' }, - suffixes = { 'bst' }, - }, - fontconfig = { - names = { 'fontconfig', 'fontconfig file', 'fontconfig files' }, - variable = 'FONTCONFIG_PATH', - }, - }, - obsolete = { - enc = { - names = { "enc", "enc files", "enc file", "encoding files", "encoding file" }, - variable = 'ENCFONTS', - suffixes = { 'enc' }, - }, - map = { - names = { "map", "map files", "map file" }, - variable = 'TEXFONTMAPS', - suffixes = { 'map' }, - }, - lig = { - names = { "lig files", "lig file", "ligature file", "ligature files" }, - variable = 'LIGFONTS', - suffixes = { 'lig' }, - }, - opl = { - names = { "opl" }, - variable = 'OPLFONTS', - suffixes = { 'opl' }, - }, - ovp = { - names = { "ovp" }, - variable = 'OVPFONTS', - suffixes = { 'ovp' }, - }, - }, - kpse = { -- subset - base = { - names = { 'base', "metafont format" }, - variable = 'MFBASES', - suffixes = { 'base', 'bas' }, - }, - cmap = { - names = { 'cmap', 'cmap files', 'cmap file' }, - variable = 'CMAPFONTS', - suffixes = { 'cmap' }, - }, - cnf = { - names = { 'cnf' }, - suffixes = { 'cnf' }, - }, - web = { - names = { 'web' }, - suffixes = { 'web', 'ch' } - }, - cweb = { - names = { 'cweb' }, - suffixes = { 'w', 'web', 'ch' }, - }, - gf = { - names = { 'gf' }, - suffixes = { 'gf' }, - }, - mf = { - names = { 'mf' }, - variable = 'MFINPUTS', - suffixes = { 'mf' }, - }, - mft = { - names = { 'mft' }, - suffixes = { 'mft' }, - }, - pk = { - names = { 'pk' }, - suffixes = { 'pk' }, - }, - }, -} - -resolvers.relations = relations - --- formats: maps a format onto a variable - -function resolvers.updaterelations() - for category, categories in next, relations do - for name, relation in next, categories do - local rn = relation.names - local rv = relation.variable - local rs = relation.suffixes - if rn and rv then - for i=1,#rn do - local rni = lower(gsub(rn[i]," ","")) - formats[rni] = rv - if rs then - suffixes[rni] = rs - for i=1,#rs do - local rsi = rs[i] - suffixmap[rsi] = rni - end - end - end - end - if rs then - end - end - end -end - -resolvers.updaterelations() -- push this in the metatable -> newindex - -local function simplified(t,k) - return k and rawget(t,lower(gsub(k," ",""))) or nil -end - -setmetatableindex(formats, simplified) -setmetatableindex(suffixes, simplified) -setmetatableindex(suffixmap, simplified) - --- A few accessors, mostly for command line tool. - -function resolvers.suffixofformat(str) - local s = suffixes[str] - return s and s[1] or "" -end - -function resolvers.suffixofformat(str) - return suffixes[str] or { } -end - -for name, format in next, formats do - dangerous[name] = true -- still needed ? -end - --- because vf searching is somewhat dangerous, we want to prevent --- too liberal searching esp because we do a lookup on the current --- path anyway; only tex (or any) is safe - -dangerous.tex = nil - - --- more helpers - -function resolvers.formatofvariable(str) - return formats[str] or '' -end - -function resolvers.formatofsuffix(str) -- of file - return suffixmap[fileextname(str)] or 'tex' -- so many map onto tex (like mkiv, cld etc) -end - -function resolvers.variableofformat(str) - return formats[str] or '' -end - -function resolvers.variableofformatorsuffix(str) - local v = formats[str] - if v then - return v - end - v = suffixmap[fileextname(str)] - if v then - return formats[v] - end - return '' -end - - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['data-tmp'] = { - version = 1.100, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - ---[[ldx-- -

This module deals with caching data. It sets up the paths and -implements loaders and savers for tables. Best is to set the -following variable. When not set, the usual paths will be -checked. Personally I prefer the (users) temporary path.

- - -TEXMFCACHE=$TMP;$TEMP;$TMPDIR;$TEMPDIR;$HOME;$TEXMFVAR;$VARTEXMF;. - - -

Currently we do no locking when we write files. This is no real -problem because most caching involves fonts and the chance of them -being written at the same time is small. We also need to extend -luatools with a recache feature.

---ldx]]-- - -local format, lower, gsub, concat = string.format, string.lower, string.gsub, table.concat -local serialize, serializetofile = table.serialize, table.tofile -local mkdirs, isdir = dir.mkdirs, lfs.isdir - -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) -local trace_cache = false trackers.register("resolvers.cache", function(v) trace_cache = v end) - -local report_caches = logs.reporter("resolvers","caches") -local report_resolvers = logs.reporter("resolvers","caching") - -local resolvers = resolvers - --- intermezzo - -local directive_cleanup = false directives.register("system.compile.cleanup", function(v) directive_cleanup = v end) -local directive_strip = true directives.register("system.compile.strip", function(v) directive_strip = v end) - -local compile = utilities.lua.compile - -function utilities.lua.compile(luafile,lucfile,cleanup,strip) - if cleanup == nil then cleanup = directive_cleanup end - if strip == nil then strip = directive_strip end - return compile(luafile,lucfile,cleanup,strip) -end - --- end of intermezzo - -caches = caches or { } -local caches = caches - -caches.base = caches.base or "luatex-cache" -caches.more = caches.more or "context" -caches.direct = false -- true is faster but may need huge amounts of memory -caches.tree = false -caches.force = true -caches.ask = false -caches.relocate = false -caches.defaults = { "TMPDIR", "TEMPDIR", "TMP", "TEMP", "HOME", "HOMEPATH" } - -local writable, readables, usedreadables = nil, { }, { } - --- we could use a metatable for writable and readable but not yet - -local function identify() - -- Combining the loops makes it messy. First we check the format cache path - -- and when the last component is not present we try to create it. - local texmfcaches = resolvers.cleanpathlist("TEXMFCACHE") - if texmfcaches then - for k=1,#texmfcaches do - local cachepath = texmfcaches[k] - if cachepath ~= "" then - cachepath = resolvers.resolve(cachepath) - cachepath = resolvers.cleanpath(cachepath) - cachepath = file.collapsepath(cachepath) - local valid = isdir(cachepath) - if valid then - if file.is_readable(cachepath) then - readables[#readables+1] = cachepath - if not writable and file.is_writable(cachepath) then - writable = cachepath - end - end - elseif not writable and caches.force then - local cacheparent = file.dirname(cachepath) - if file.is_writable(cacheparent) and true then -- we go on anyway (needed for mojca's kind of paths) - if not caches.ask or io.ask(format("\nShould I create the cache path %s?",cachepath), "no", { "yes", "no" }) == "yes" then - mkdirs(cachepath) - if isdir(cachepath) and file.is_writable(cachepath) then - report_caches("created: %s",cachepath) - writable = cachepath - readables[#readables+1] = cachepath - end - end - end - end - end - end - end - -- As a last resort we check some temporary paths but this time we don't - -- create them. - local texmfcaches = caches.defaults - if texmfcaches then - for k=1,#texmfcaches do - local cachepath = texmfcaches[k] - cachepath = resolvers.expansion(cachepath) -- was getenv - if cachepath ~= "" then - cachepath = resolvers.resolve(cachepath) - cachepath = resolvers.cleanpath(cachepath) - local valid = isdir(cachepath) - if valid and file.is_readable(cachepath) then - if not writable and file.is_writable(cachepath) then - readables[#readables+1] = cachepath - writable = cachepath - break - end - end - end - end - end - -- Some extra checking. If we have no writable or readable path then we simply - -- quit. - if not writable then - report_caches("fatal error: there is no valid writable cache path defined") - os.exit() - elseif #readables == 0 then - report_caches("fatal error: there is no valid readable cache path defined") - os.exit() - end - -- why here - writable = dir.expandname(resolvers.cleanpath(writable)) -- just in case - -- moved here - local base, more, tree = caches.base, caches.more, caches.tree or caches.treehash() -- we have only one writable tree - if tree then - caches.tree = tree - writable = mkdirs(writable,base,more,tree) - for i=1,#readables do - readables[i] = file.join(readables[i],base,more,tree) - end - else - writable = mkdirs(writable,base,more) - for i=1,#readables do - readables[i] = file.join(readables[i],base,more) - end - end - -- end - if trace_cache then - for i=1,#readables do - report_caches("using readable path '%s' (order %s)",readables[i],i) - end - report_caches("using writable path '%s'",writable) - end - identify = function() - return writable, readables - end - return writable, readables -end - -function caches.usedpaths() - local writable, readables = identify() - if #readables > 1 then - local result = { } - for i=1,#readables do - local readable = readables[i] - if usedreadables[i] or readable == writable then - result[#result+1] = format("readable: '%s' (order %s)",readable,i) - end - end - result[#result+1] = format("writable: '%s'",writable) - return result - else - return writable - end -end - -function caches.configfiles() - return concat(resolvers.instance.specification,";") -end - -function caches.hashed(tree) - tree = gsub(tree,"\\$","/") - tree = gsub(tree,"/+$","") - tree = lower(tree) - local hash = md5.hex(tree) - if trace_cache or trace_locating then - report_caches("hashing tree %s, hash %s",tree,hash) - end - return hash -end - -function caches.treehash() - local tree = caches.configfiles() - if not tree or tree == "" then - return false - else - return caches.hashed(tree) - end -end - -local r_cache, w_cache = { }, { } -- normally w in in r but who cares - -local function getreadablepaths(...) -- we can optimize this as we have at most 2 tags - local tags = { ... } - local hash = concat(tags,"/") - local done = r_cache[hash] - if not done then - local writable, readables = identify() -- exit if not found - if #tags > 0 then - done = { } - for i=1,#readables do - done[i] = file.join(readables[i],...) - end - else - done = readables - end - r_cache[hash] = done - end - return done -end - -local function getwritablepath(...) - local tags = { ... } - local hash = concat(tags,"/") - local done = w_cache[hash] - if not done then - local writable, readables = identify() -- exit if not found - if #tags > 0 then - done = mkdirs(writable,...) - else - done = writable - end - w_cache[hash] = done - end - return done -end - -caches.getreadablepaths = getreadablepaths -caches.getwritablepath = getwritablepath - -function caches.getfirstreadablefile(filename,...) - local rd = getreadablepaths(...) - for i=1,#rd do - local path = rd[i] - local fullname = file.join(path,filename) - if file.is_readable(fullname) then - usedreadables[i] = true - return fullname, path - end - end - return caches.setfirstwritablefile(filename,...) -end - -function caches.setfirstwritablefile(filename,...) - local wr = getwritablepath(...) - local fullname = file.join(wr,filename) - return fullname, wr -end - -function caches.define(category,subcategory) -- for old times sake - return function() - return getwritablepath(category,subcategory) - end -end - -function caches.setluanames(path,name) - return path .. "/" .. name .. ".tma", path .. "/" .. name .. ".tmc" -end - -function caches.loaddata(readables,name) - if type(readables) == "string" then - readables = { readables } - end - for i=1,#readables do - local path = readables[i] - local tmaname, tmcname = caches.setluanames(path,name) - local loader = loadfile(tmcname) or loadfile(tmaname) - if loader then - loader = loader() - collectgarbage("step") - return loader - end - end - return false -end - -function caches.is_writable(filepath,filename) - local tmaname, tmcname = caches.setluanames(filepath,filename) - return file.is_writable(tmaname) -end - -local saveoptions = { compact = true } - -function caches.savedata(filepath,filename,data,raw) - local tmaname, tmcname = caches.setluanames(filepath,filename) - local reduce, simplify = true, true - if raw then - reduce, simplify = false, false - end - data.cache_uuid = os.uuid() - if caches.direct then - file.savedata(tmaname,serialize(data,true,saveoptions)) - else - serializetofile(tmaname,data,true,saveoptions) - end - utilities.lua.compile(tmaname,tmcname) -end - --- moved from data-res: - -local content_state = { } - -function caches.contentstate() - return content_state or { } -end - -function caches.loadcontent(cachename,dataname) - local name = caches.hashed(cachename) - local full, path = caches.getfirstreadablefile(name ..".lua","trees") - local filename = file.join(path,name) - local blob = loadfile(filename .. ".luc") or loadfile(filename .. ".lua") - if blob then - local data = blob() - if data and data.content then - if data.type == dataname then - if data.version == resolvers.cacheversion then - content_state[#content_state+1] = data.uuid - if trace_locating then - report_resolvers("loading '%s' for '%s' from '%s'",dataname,cachename,filename) - end - return data.content - else - report_resolvers("skipping '%s' for '%s' from '%s' (version mismatch)",dataname,cachename,filename) - end - else - report_resolvers("skipping '%s' for '%s' from '%s' (datatype mismatch)",dataname,cachename,filename) - end - elseif trace_locating then - report_resolvers("skipping '%s' for '%s' from '%s' (no content)",dataname,cachename,filename) - end - elseif trace_locating then - report_resolvers("skipping '%s' for '%s' from '%s' (invalid file)",dataname,cachename,filename) - end -end - -function caches.collapsecontent(content) - for k, v in next, content do - if type(v) == "table" and #v == 1 then - content[k] = v[1] - end - end -end - -function caches.savecontent(cachename,dataname,content) - local name = caches.hashed(cachename) - local full, path = caches.setfirstwritablefile(name ..".lua","trees") - local filename = file.join(path,name) -- is full - local luaname, lucname = filename .. ".lua", filename .. ".luc" - if trace_locating then - report_resolvers("preparing '%s' for '%s'",dataname,cachename) - end - local data = { - type = dataname, - root = cachename, - version = resolvers.cacheversion, - date = os.date("%Y-%m-%d"), - time = os.date("%H:%M:%S"), - content = content, - uuid = os.uuid(), - } - local ok = io.savedata(luaname,serialize(data,true)) - if ok then - if trace_locating then - report_resolvers("category '%s', cachename '%s' saved in '%s'",dataname,cachename,luaname) - end - if utilities.lua.compile(luaname,lucname) then - if trace_locating then - report_resolvers("'%s' compiled to '%s'",dataname,lucname) - end - return true - else - if trace_locating then - report_resolvers("compiling failed for '%s', deleting file '%s'",dataname,lucname) - end - os.remove(lucname) - end - elseif trace_locating then - report_resolvers("unable to save '%s' in '%s' (access error)",dataname,luaname) - end -end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['data-met'] = { - version = 1.100, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local find, format = string.find, string.format -local sequenced = table.sequenced -local addurlscheme, urlhashed = url.addscheme, url.hashed - -local trace_locating = false - -trackers.register("resolvers.locating", function(v) trace_methods = v end) -trackers.register("resolvers.methods", function(v) trace_methods = v end) - - -local report_methods = logs.reporter("resolvers","methods") - -local allocate = utilities.storage.allocate - -local resolvers = resolvers - -local registered = { } - -local function splitmethod(filename) -- todo: filetype in specification - if not filename then - return { scheme = "unknown", original = filename } - end - if type(filename) == "table" then - return filename -- already split - end - filename = file.collapsepath(filename) - if not find(filename,"://") then - return { scheme = "file", path = filename, original = filename, filename = filename } - end - local specification = url.hashed(filename) - if not specification.scheme or specification.scheme == "" then - return { scheme = "file", path = filename, original = filename, filename = filename } - else - return specification - end -end - -resolvers.splitmethod = splitmethod -- bad name but ok - --- the second argument is always analyzed (saves time later on) and the original --- gets passed as original but also as argument - -local function methodhandler(what,first,...) -- filename can be nil or false - local method = registered[what] - if method then - local how, namespace = method.how, method.namespace - if how == "uri" or how == "url" then - local specification = splitmethod(first) - local scheme = specification.scheme - local resolver = namespace and namespace[scheme] - if resolver then - if trace_methods then - report_methods("resolver: method=%s, how=%s, scheme=%s, argument=%s",what,how,scheme,first) - end - return resolver(specification,...) - else - resolver = namespace.default or namespace.file - if resolver then - if trace_methods then - report_methods("resolver: method=%s, how=%s, default, argument=%s",what,how,first) - end - return resolver(specification,...) - elseif trace_methods then - report_methods("resolver: method=%s, how=%s, no handler",what,how) - end - end - elseif how == "tag" then - local resolver = namespace and namespace[first] - if resolver then - if trace_methods then - report_methods("resolver: method=%s, how=%s, tag=%s",what,how,first) - end - return resolver(...) - else - resolver = namespace.default or namespace.file - if resolver then - if trace_methods then - report_methods("resolver: method=%s, how=%s, default",what,how) - end - return resolver(...) - elseif trace_methods then - report_methods("resolver: method=%s, how=%s, unknown",what,how) - end - end - end - else - report_methods("resolver: method=%s, unknown",what) - end -end - -resolvers.methodhandler = methodhandler - -function resolvers.registermethod(name,namespace,how) - registered[name] = { how = how or "tag", namespace = namespace } - namespace["byscheme"] = function(scheme,filename,...) - if scheme == "file" then - return methodhandler(name,filename,...) - else - return methodhandler(name,addurlscheme(filename,scheme),...) - end - end -end - -local concatinators = allocate { notfound = file.join } -- concatinate paths -local locators = allocate { notfound = function() end } -- locate databases -local hashers = allocate { notfound = function() end } -- load databases -local generators = allocate { notfound = function() end } -- generate databases - -resolvers.concatinators = concatinators -resolvers.locators = locators -resolvers.hashers = hashers -resolvers.generators = generators - -local registermethod = resolvers.registermethod - -registermethod("concatinators",concatinators,"tag") -registermethod("locators", locators, "uri") -registermethod("hashers", hashers, "uri") -registermethod("generators", generators, "uri") - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['data-res'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files", -} - --- In practice we will work within one tds tree, but i want to keep --- the option open to build tools that look at multiple trees, which is --- why we keep the tree specific data in a table. We used to pass the --- instance but for practical purposes we now avoid this and use a --- instance variable. We always have one instance active (sort of global). - --- todo: cache:/// home:/// selfautoparent:/// (sometime end 2012) - -local format, gsub, find, lower, upper, match, gmatch = string.format, string.gsub, string.find, string.lower, string.upper, string.match, string.gmatch -local concat, insert, sortedkeys = table.concat, table.insert, table.sortedkeys -local next, type, rawget = next, type, rawget -local os = os - -local P, S, R, C, Cc, Cs, Ct, Carg = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.Cc, lpeg.Cs, lpeg.Ct, lpeg.Carg -local lpegmatch, lpegpatterns = lpeg.match, lpeg.patterns - -local filedirname = file.dirname -local filebasename = file.basename -local fileextname = file.extname -local filejoin = file.join -local collapsepath = file.collapsepath -local joinpath = file.joinpath -local allocate = utilities.storage.allocate -local setmetatableindex = table.setmetatableindex - -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) -local trace_detail = false trackers.register("resolvers.details", function(v) trace_detail = v end) -local trace_expansions = false trackers.register("resolvers.expansions", function(v) trace_expansions = v end) - -local report_resolving = logs.reporter("resolvers","resolving") - -local resolvers = resolvers - -local expandedpathfromlist = resolvers.expandedpathfromlist -local checkedvariable = resolvers.checkedvariable -local splitconfigurationpath = resolvers.splitconfigurationpath -local methodhandler = resolvers.methodhandler - -local initializesetter = utilities.setters.initialize - -local ostype, osname, osenv, ossetenv, osgetenv = os.type, os.name, os.env, os.setenv, os.getenv - -resolvers.cacheversion = '1.0.1' -resolvers.configbanner = '' -resolvers.homedir = environment.homedir -resolvers.criticalvars = allocate { "SELFAUTOLOC", "SELFAUTODIR", "SELFAUTOPARENT", "TEXMFCNF", "TEXMF", "TEXOS" } -resolvers.luacnfname = 'texmfcnf.lua' -resolvers.luacnfstate = "unknown" - --- The web2c tex binaries as well as kpse have built in paths for the configuration --- files and there can be a depressing truckload of them. This is actually the weak --- spot of a distribution. So we don't want: --- --- resolvers.luacnfspec = '{$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,}/web2c}' --- --- but instead use: --- --- resolvers.luacnfspec = 'selfautoparent:{/texmf{-local,}{,/web2c}}' --- --- which does not make texlive happy as there is a texmf-local tree one level up --- (sigh), so we need this. We can assume web2c as mkiv does not run on older --- texlives anyway. --- --- texlive: --- --- selfautodir: --- selfautoparent: --- selfautodir:share/texmf-local/web2c --- selfautodir:share/texmf/web2c --- selfautodir:texmf-local/web2c --- selfautodir:texmf/web2c --- selfautoparent:share/texmf-local/web2c --- selfautoparent:share/texmf/web2c --- selfautoparent:texmf-local/web2c --- selfautoparent:texmf/web2c --- --- minimals: --- --- home:texmf/web2c --- selfautoparent:texmf-local/web2c --- selfautoparent:texmf-context/web2c --- selfautoparent:texmf/web2c - -if environment.default_texmfcnf then - -- unfortunately we now have quite some overkill in the spec (not so nice on a network) - resolvers.luacnfspec = environment.default_texmfcnf -else - -- resolvers.luacnfspec = "selfautoparent:texmf{-local,-context,}/web2c" - resolvers.luacnfspec = "{selfautoloc:,selfautodir:,selfautoparent:}{,/texmf{-local,}/web2c}" -end - -resolvers.luacnfspec = 'home:texmf/web2c;' .. resolvers.luacnfspec - --- which (as we want users to use the web2c path) be can be simplified to this: --- --- if environment and environment.ownpath and string.find(environment.ownpath,"[\\/]texlive[\\/]") then --- resolvers.luacnfspec = 'selfautodir:/texmf-local/web2c,selfautoparent:/texmf-local/web2c,selfautoparent:/texmf/web2c' --- else --- resolvers.luacnfspec = 'selfautoparent:/texmf-local/web2c,selfautoparent:/texmf/web2c' --- end - - - -local unset_variable = "unset" - -local formats = resolvers.formats -local suffixes = resolvers.suffixes -local dangerous = resolvers.dangerous -local suffixmap = resolvers.suffixmap - -resolvers.defaultsuffixes = { "tex" } -- "mkiv", "cld" -- too tricky - -resolvers.instance = resolvers.instance or nil -- the current one (slow access) -local instance = resolvers.instance or nil -- the current one (fast access) - --- An instance has an environment (coming from the outside, kept raw), variables --- (coming from the configuration file), and expansions (variables with nested --- variables replaced). One can push something into the outer environment and --- its internal copy, but only the later one will be the raw unprefixed variant. - -function resolvers.setenv(key,value,raw) - if instance then - -- this one will be consulted first when we stay inside - -- the current environment (prefixes are not resolved here) - instance.environment[key] = value - -- we feed back into the environment, and as this is used - -- by other applications (via os.execute) we need to make - -- sure that prefixes are resolve - ossetenv(key,raw and value or resolvers.resolve(value)) - end -end - --- Beware we don't want empty here as this one can be called early on --- and therefore we use rawget. - -local function getenv(key) - local value = rawget(instance.environment,key) - if value and value ~= "" then - return value - else - local e = osgetenv(key) - return e ~= nil and e ~= "" and checkedvariable(e) or "" - end -end - -resolvers.getenv = getenv -resolvers.env = getenv - --- We are going to use some metatable trickery where we backtrack from --- expansion to variable to environment. - -local function resolve(k) - return instance.expansions[k] -end - -local dollarstripper = lpeg.stripper("$") -local inhibitstripper = P("!")^0 * Cs(P(1)^0) -local backslashswapper = lpeg.replacer("\\","/") - -local somevariable = P("$") / "" -local somekey = C(R("az","AZ","09","__","--")^1) -local somethingelse = P(";") * ((1-S("!{}/\\"))^1 * P(";") / "") - + P(";") * (P(";") / "") - + P(1) -local variableexpander = Cs( (somevariable * (somekey/resolve) + somethingelse)^1 ) - -local cleaner = P("\\") / "/" + P(";") * S("!{}/\\")^0 * P(";")^1 / ";" -local variablecleaner = Cs((cleaner + P(1))^0) - -local somevariable = R("az","AZ","09","__","--")^1 / resolve -local variable = (P("$")/"") * (somevariable + (P("{")/"") * somevariable * (P("}")/"")) -local variableresolver = Cs((variable + P(1))^0) - -local function expandedvariable(var) - return lpegmatch(variableexpander,var) or var -end - -function resolvers.newinstance() -- todo: all vars will become lowercase and alphanum only - - if trace_locating then - report_resolving("creating instance") - end - - local environment, variables, expansions, order = allocate(), allocate(), allocate(), allocate() - - local newinstance = { - environment = environment, - variables = variables, - expansions = expansions, - order = order, - files = allocate(), - setups = allocate(), - found = allocate(), - foundintrees = allocate(), - hashes = allocate(), - hashed = allocate(), - specification = allocate(), - lists = allocate(), - data = allocate(), -- only for loading - fakepaths = allocate(), - remember = true, - diskcache = true, - renewcache = false, - renewtree = false, - loaderror = false, - savelists = true, - pattern = nil, -- lists - force_suffixes = true, - } - - setmetatableindex(variables,function(t,k) - local v - for i=1,#order do - v = order[i][k] - if v ~= nil then - t[k] = v - return v - end - end - if v == nil then - v = "" - end - t[k] = v - return v - end) - - setmetatableindex(environment, function(t,k) - local v = osgetenv(k) - if v == nil then - v = variables[k] - end - if v ~= nil then - v = checkedvariable(v) or "" - end - v = resolvers.repath(v) -- for taco who has a : separated osfontdir - t[k] = v - return v - end) - - setmetatableindex(expansions, function(t,k) - local v = environment[k] - if type(v) == "string" then - v = lpegmatch(variableresolver,v) - v = lpegmatch(variablecleaner,v) - end - t[k] = v - return v - end) - - return newinstance - -end - -function resolvers.setinstance(someinstance) -- only one instance is active - instance = someinstance - resolvers.instance = someinstance - return someinstance -end - -function resolvers.reset() - return resolvers.setinstance(resolvers.newinstance()) -end - -local function reset_hashes() - instance.lists = { } - instance.found = { } -end - -local slash = P("/") - -local pathexpressionpattern = Cs ( - Cc("^") * ( - Cc("%") * S(".-") - + slash^2 * P(-1) / "/.*" - + slash^2 / "/.-/" - + (1-slash) * P(-1) * Cc("/") - + P(1) - )^1 * Cc("$") -- yes or no $ -) - -local cache = { } - -local function makepathexpression(str) - if str == "." then - return "^%./$" - else - local c = cache[str] - if not c then - c = lpegmatch(pathexpressionpattern,str) - cache[str] = c - end - return c - end -end - -local function reportcriticalvariables(cnfspec) - if trace_locating then - for i=1,#resolvers.criticalvars do - local k = resolvers.criticalvars[i] - local v = resolvers.getenv(k) or "unknown" -- this one will not resolve ! - report_resolving("variable '%s' set to '%s'",k,v) - end - report_resolving() - if cnfspec then - if type(cnfspec) == "table" then - report_resolving("using configuration specification '%s'",concat(cnfspec,",")) - else - report_resolving("using configuration specification '%s'",cnfspec) - end - end - report_resolving() - end - reportcriticalvariables = function() end -end - -local function identify_configuration_files() - local specification = instance.specification - if #specification == 0 then - local cnfspec = getenv('TEXMFCNF') - if cnfspec == "" then - cnfspec = resolvers.luacnfspec - resolvers.luacnfstate = "default" - else - resolvers.luacnfstate = "environment" - end - reportcriticalvariables(cnfspec) - local cnfpaths = expandedpathfromlist(resolvers.splitpath(cnfspec)) - local luacnfname = resolvers.luacnfname - for i=1,#cnfpaths do - local filename = collapsepath(filejoin(cnfpaths[i],luacnfname)) - local realname = resolvers.resolve(filename) - if lfs.isfile(realname) then - specification[#specification+1] = filename - if trace_locating then - report_resolving("found configuration file '%s'",realname) - end - elseif trace_locating then - report_resolving("unknown configuration file '%s'",realname) - end - end - if trace_locating then - report_resolving() - end - elseif trace_locating then - report_resolving("configuration files already identified") - end -end - -local function load_configuration_files() - local specification = instance.specification - if #specification > 0 then - local luacnfname = resolvers.luacnfname - for i=1,#specification do - local filename = specification[i] - local pathname = filedirname(filename) - local filename = filejoin(pathname,luacnfname) - local realname = resolvers.resolve(filename) -- no shortcut - local blob = loadfile(realname) - if blob then - local setups = instance.setups - local data = blob() - local parent = data and data.parent - if parent then - local filename = filejoin(pathname,parent) - local realname = resolvers.resolve(filename) -- no shortcut - local blob = loadfile(realname) - if blob then - local parentdata = blob() - if parentdata then - report_resolving("loading configuration file '%s'",filename) - data = table.merged(parentdata,data) - end - end - end - data = data and data.content - if data then - if trace_locating then - report_resolving("loading configuration file '%s'",filename) - report_resolving() - end - local variables = data.variables or { } - local warning = false - for k, v in next, data do - local variant = type(v) - if variant == "table" then - initializesetter(filename,k,v) - elseif variables[k] == nil then - if trace_locating and not warning then - report_resolving("variables like '%s' in configuration file '%s' should move to the 'variables' subtable", - k,resolvers.resolve(filename)) - warning = true - end - variables[k] = v - end - end - setups[pathname] = variables - if resolvers.luacnfstate == "default" then - -- the following code is not tested - local cnfspec = variables["TEXMFCNF"] - if cnfspec then - if trace_locating then - report_resolving("reloading configuration due to TEXMF redefinition") - end - -- we push the value into the main environment (osenv) so - -- that it takes precedence over the default one and therefore - -- also over following definitions - resolvers.setenv('TEXMFCNF',cnfspec) -- resolves prefixes - -- we now identify and load the specified configuration files - instance.specification = { } - identify_configuration_files() - load_configuration_files() - -- we prevent further overload of the configuration variable - resolvers.luacnfstate = "configuration" - -- we quit the outer loop - break - end - end - - else - if trace_locating then - report_resolving("skipping configuration file '%s' (no content)",filename) - end - setups[pathname] = { } - instance.loaderror = true - end - elseif trace_locating then - report_resolving("skipping configuration file '%s' (no valid format)",filename) - end - instance.order[#instance.order+1] = instance.setups[pathname] - if instance.loaderror then - break - end - end - elseif trace_locating then - report_resolving("warning: no lua configuration files found") - end -end - --- scheme magic ... database loading - -local function load_file_databases() - instance.loaderror, instance.files = false, allocate() - if not instance.renewcache then - local hashes = instance.hashes - for k=1,#hashes do - local hash = hashes[k] - resolvers.hashers.byscheme(hash.type,hash.name) - if instance.loaderror then break end - end - end -end - -local function locate_file_databases() - -- todo: cache:// and tree:// (runtime) - local texmfpaths = resolvers.expandedpathlist('TEXMF') - if #texmfpaths > 0 then - for i=1,#texmfpaths do - local path = collapsepath(texmfpaths[i]) - local stripped = lpegmatch(inhibitstripper,path) -- the !! thing - if stripped ~= "" then - local runtime = stripped == path - path = resolvers.cleanpath(path) - local spec = resolvers.splitmethod(stripped) - if runtime and (spec.noscheme or spec.scheme == "file") then - stripped = "tree:///" .. stripped - elseif spec.scheme == "cache" or spec.scheme == "file" then - stripped = spec.path - end - if trace_locating then - if runtime then - report_resolving("locating list of '%s' (runtime) (%s)",path,stripped) - else - report_resolving("locating list of '%s' (cached)",path) - end - end - methodhandler('locators',stripped) - end - end - if trace_locating then - report_resolving() - end - elseif trace_locating then - report_resolving("no texmf paths are defined (using TEXMF)") - end -end - -local function generate_file_databases() - local hashes = instance.hashes - for k=1,#hashes do - local hash = hashes[k] - methodhandler('generators',hash.name) - end - if trace_locating then - report_resolving() - end -end - -local function save_file_databases() -- will become cachers - for i=1,#instance.hashes do - local hash = instance.hashes[i] - local cachename = hash.name - if hash.cache then - local content = instance.files[cachename] - caches.collapsecontent(content) - if trace_locating then - report_resolving("saving tree '%s'",cachename) - end - caches.savecontent(cachename,"files",content) - elseif trace_locating then - report_resolving("not saving runtime tree '%s'",cachename) - end - end +local function normal_apply(list,parsed,nofparsed,order) + local collected=list + for i=1,nofparsed do + local pi=parsed[i] + local kind=pi.kind + if kind=="axis" then + local axis=pi.axis + if axis~="self" then + collected=apply_axis[axis](collected) + end + elseif kind=="nodes" then + collected=apply_nodes(collected,pi.nodetest,pi.nodes) + elseif kind=="expression" then + collected=apply_expression(collected,pi.evaluator,order) + elseif kind=="finalizer" then + return pi.finalizer(collected) + end + if not collected or #collected==0 then + local pf=i 1 then - ie[k] = t - else - ie[k] = t[1] +expressions.tag=function(e,n) + if not e then + return "" + else + local found=false + n=tonumber(n) or 0 + if n==0 then + found=(type(e)=="table") and e + elseif n<0 then + local d,k=e.__p__.dt,e.ni + for i=k-1,1,-1 do + local di=d[i] + if type(di)=="table" then + if n==-1 then + found=di + break + else + n=n+1 + end end + end + else + local d,k=e.__p__.dt,e.ni + for i=k+1,#d,1 do + local di=d[i] + if type(di)=="table" then + if n==1 then + found=di + break + else + n=n-1 + end + end + end + end + return (found and found.tg) or "" + end +end +local dummy=function() end +function xml.elements(root,pattern,reverse) + local collected=applylpath(root,pattern) + if not collected then + return dummy + elseif reverse then + local c=#collected+1 + return function() + if c>1 then + c=c-1 + local e=collected[c] + local r=e.__p__ + return r,r.dt,e.ni + end + end + else + local n,c=#collected,0 + return function() + if c1 then + c=c-1 + return collected[c] + end + end + else + local n,c=#collected,0 + return function() + if c"))^0 +local special=P("<")/"<"+P(">")/">"+P("&")/"&" +local escaped=Cs(normal*(special*normal)^0) +local normal=(1-S"&")^0 +local special=P("<")/"<"+P(">")/">"+P("&")/"&" +local unescaped=Cs(normal*(special*normal)^0) +local cleansed=Cs(((P("<")*(1-P(">"))^0*P(">"))/""+1)^0) +xmlpatterns.escaped=escaped +xmlpatterns.unescaped=unescaped +xmlpatterns.cleansed=cleansed +function xml.escaped (str) return lpegmatch(escaped,str) end +function xml.unescaped(str) return lpegmatch(unescaped,str) end +function xml.cleansed (str) return lpegmatch(cleansed,str) end +function xml.fillin(root,pattern,str,check) + local e=xml.first(root,pattern) + if e then + local n=#e.dt + if not check or n==0 or (n==1 and e.dt[1]=="") then + e.dt={ str } + end + end end -function resolvers.unexpandedpath(str) - return joinpath(resolvers.unexpandedpathlist(str)) -end -local done = { } +end -- of closure -function resolvers.resetextrapath() - local ep = instance.extra_paths - if not ep then - ep, done = { }, { } - instance.extra_paths = ep - elseif #ep > 0 then - instance.lists, done = { }, { } - end -end +do -- create closure to overcome 200 locals limit -function resolvers.registerextrapath(paths,subpaths) - local ep = instance.extra_paths or { } - local oldn = #ep - local newn = oldn - if paths and paths ~= "" then - if subpaths and subpaths ~= "" then - for p in gmatch(paths,"[^,]+") do - -- we gmatch each step again, not that fast, but used seldom - for s in gmatch(subpaths,"[^,]+") do - local ps = p .. "/" .. s - if not done[ps] then - newn = newn + 1 - ep[newn] = resolvers.cleanpath(ps) - done[ps] = true - end - end - end - else - for p in gmatch(paths,"[^,]+") do - if not done[p] then - newn = newn + 1 - ep[newn] = resolvers.cleanpath(p) - done[p] = true - end - end - end - elseif subpaths and subpaths ~= "" then - for i=1,oldn do - -- we gmatch each step again, not that fast, but used seldom - for s in gmatch(subpaths,"[^,]+") do - local ps = ep[i] .. "/" .. s - if not done[ps] then - newn = newn + 1 - ep[newn] = resolvers.cleanpath(ps) - done[ps] = true - end - end +package.loaded["lxml-aux"] = package.loaded["lxml-aux"] or true + +-- original size: 23804, stripped down to: 16817 + +if not modules then modules={} end modules ['lxml-aux']={ + version=1.001, + comment="this module is the basis for the lxml-* ones", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local trace_manipulations=false trackers.register("lxml.manipulations",function(v) trace_manipulations=v end) +local report_xml=logs.reporter("xml") +local xml=xml +local xmlconvert,xmlcopy,xmlname=xml.convert,xml.copy,xml.name +local xmlinheritedconvert=xml.inheritedconvert +local xmlapplylpath=xml.applylpath +local xmlfilter=xml.filter +local type,setmetatable,getmetatable=type,setmetatable,getmetatable +local insert,remove,fastcopy,concat=table.insert,table.remove,table.fastcopy,table.concat +local gmatch,gsub,format,find,strip=string.gmatch,string.gsub,string.format,string.find,string.strip +local utfbyte=utf.byte +local function report(what,pattern,c,e) + report_xml("%s element %a, root %a, position %a, index %a, pattern %a",what,xmlname(e),xmlname(e.__p__),c,e.ni,pattern) +end +local function withelements(e,handle,depth) + if e and handle then + local edt=e.dt + if edt then + depth=depth or 0 + for i=1,#edt do + local e=edt[i] + if type(e)=="table" then + handle(e,depth) + withelements(e,handle,depth+1) end + end end - if newn > 0 then - instance.extra_paths = ep -- register paths - end - if newn > oldn then - instance.lists = { } -- erase the cache - end + end end - -local function made_list(instance,list) - local ep = instance.extra_paths - if not ep or #ep == 0 then - return list - else - local done, new, newn = { }, { }, 0 - -- honour . .. ../.. but only when at the start - for k=1,#list do - local v = list[k] - if not done[v] then - if find(v,"^[%.%/]$") then - done[v] = true - newn = newn + 1 - new[newn] = v - else - break - end - end - end - -- first the extra paths - for k=1,#ep do - local v = ep[k] - if not done[v] then - done[v] = true - newn = newn + 1 - new[newn] = v +xml.withelements=withelements +function xml.withelement(e,n,handle) + if e and n~=0 and handle then + local edt=e.dt + if edt then + if n>0 then + for i=1,#edt do + local ei=edt[i] + if type(ei)=="table" then + if n==1 then + handle(ei) + return + else + n=n-1 end + end end - -- next the formal paths - for k=1,#list do - local v = list[k] - if not done[v] then - done[v] = true - newn = newn + 1 - new[newn] = v + elseif n<0 then + for i=#edt,1,-1 do + local ei=edt[i] + if type(ei)=="table" then + if n==-1 then + handle(ei) + return + else + n=n+1 end + end end - return new - end -end - -function resolvers.cleanpathlist(str) - local t = resolvers.expandedpathlist(str) - if t then - for i=1,#t do - t[i] = collapsepath(resolvers.cleanpath(t[i])) - end + end end - return t -end - -function resolvers.expandpath(str) - return joinpath(resolvers.expandedpathlist(str)) + end end - -function resolvers.expandedpathlist(str) - if not str then - return { } - elseif instance.savelists then - str = lpegmatch(dollarstripper,str) - if not instance.lists[str] then -- cached - local lst = made_list(instance,resolvers.splitpath(resolvers.expansion(str))) - instance.lists[str] = expandedpathfromlist(lst) - end - return instance.lists[str] +function xml.each(root,pattern,handle,reverse) + local collected=xmlapplylpath(root,pattern) + if collected then + if reverse then + for c=#collected,1,-1 do + handle(collected[c]) + end else - local lst = resolvers.splitpath(resolvers.expansion(str)) - return made_list(instance,expandedpathfromlist(lst)) + for c=1,#collected do + handle(collected[c]) + end end + return collected + end end - -function resolvers.expandedpathlistfromvariable(str) -- brrr - str = lpegmatch(dollarstripper,str) - local tmp = resolvers.variableofformatorsuffix(str) - return resolvers.expandedpathlist(tmp ~= "" and tmp or str) -end - -function resolvers.expandpathfromvariable(str) - return joinpath(resolvers.expandedpathlistfromvariable(str)) +function xml.processattributes(root,pattern,handle) + local collected=xmlapplylpath(root,pattern) + if collected and handle then + for c=1,#collected do + handle(collected[c].at) + end + end + return collected end - -function resolvers.expandbraces(str) -- output variable and brace expansion of STRING --- local ori = resolvers.variable(str) --- if ori == "" then - local ori = str --- end - local pth = expandedpathfromlist(resolvers.splitpath(ori)) - return joinpath(pth) +function xml.collect(root,pattern) + return xmlapplylpath(root,pattern) end - -function resolvers.registerfilehash(name,content,someerror) - if content then - instance.files[name] = content - else - instance.files[name] = { } - if somerror == true then -- can be unset - instance.loaderror = someerror - end +function xml.collecttexts(root,pattern,flatten) + local collected=xmlapplylpath(root,pattern) + if collected and flatten then + local xmltostring=xml.tostring + for c=1,#collected do + collected[c]=xmltostring(collected[c].dt) end + end + return collected or {} end - -local function isreadable(name) - local readable = lfs.isfile(name) -- not file.is_readable(name) asit can be a dir - if trace_detail then - if readable then - report_resolving("file '%s' is readable",name) - else - report_resolving("file '%s' is not readable", name) - end +function xml.collect_tags(root,pattern,nonamespace) + local collected=xmlapplylpath(root,pattern) + if collected then + local t,n={},0 + for c=1,#collected do + local e=collected[c] + local ns,tg=e.ns,e.tg + n=n+1 + if nonamespace then + t[n]=tg + elseif ns=="" then + t[n]=tg + else + t[n]=ns..":"..tg + end end - return readable + return t + end end - --- name --- name/name - -local function collect_files(names) - local filelist, noffiles = { }, 0 - for k=1,#names do - local fname = names[k] - if trace_detail then - report_resolving("checking name '%s'",fname) - end - local bname = filebasename(fname) - local dname = filedirname(fname) - if dname == "" or find(dname,"^%.") then - dname = false - else -dname = gsub(dname,"*","%.*") - dname = "/" .. dname .. "$" - end - local hashes = instance.hashes - for h=1,#hashes do - local hash = hashes[h] - local blobpath = hash.name - local files = blobpath and instance.files[blobpath] - if files then - if trace_detail then - report_resolving("deep checking '%s' (%s)",blobpath,bname) - end - local blobfile = files[bname] - if not blobfile then - local rname = "remap:"..bname - blobfile = files[rname] - if blobfile then - bname = files[rname] - blobfile = files[bname] - end - end - if blobfile then - local blobroot = files.__path__ or blobpath - if type(blobfile) == 'string' then - if not dname or find(blobfile,dname) then - local variant = hash.type - -- local search = filejoin(blobpath,blobfile,bname) - local search = filejoin(blobroot,blobfile,bname) - local result = methodhandler('concatinators',hash.type,blobroot,blobfile,bname) - if trace_detail then - report_resolving("match: variant '%s', search '%s', result '%s'",variant,search,result) - end - noffiles = noffiles + 1 - filelist[noffiles] = { variant, search, result } - end - else - for kk=1,#blobfile do - local vv = blobfile[kk] - if not dname or find(vv,dname) then - local variant = hash.type - -- local search = filejoin(blobpath,vv,bname) - local search = filejoin(blobroot,vv,bname) - local result = methodhandler('concatinators',hash.type,blobroot,vv,bname) - if trace_detail then - report_resolving("match: variant '%s', search '%s', result '%s'",variant,search,result) - end - noffiles = noffiles + 1 - filelist[noffiles] = { variant, search, result } - end - end - end - end - elseif trace_locating then - report_resolving("no match in '%s' (%s)",blobpath,bname) - end - end +local no_root={ no_root=true } +local function redo_ni(d) + for k=1,#d do + local dk=d[k] + if type(dk)=="table" then + dk.ni=k end - return noffiles > 0 and filelist or nil + end end - -local fit = { } - -function resolvers.registerintrees(filename,format,filetype,usedmethod,foundname) - local foundintrees = instance.foundintrees - if usedmethod == "direct" and filename == foundname and fit[foundname] then - -- just an extra lookup after a test on presence - else - local t = { - filename = filename, - format = format ~= "" and format or nil, - filetype = filetype ~= "" and filetype or nil, - usedmethod = usedmethod, - foundname = foundname, - } - fit[foundname] = t - foundintrees[#foundintrees+1] = t +local function xmltoelement(whatever,root) + if not whatever then + return nil + end + local element + if type(whatever)=="string" then + element=xmlinheritedconvert(whatever,root) + else + element=whatever + end + if element.error then + return whatever + end + if element then + end + return element +end +xml.toelement=xmltoelement +local function copiedelement(element,newparent) + if type(element)=="string" then + return element + else + element=xmlcopy(element).dt + if newparent and type(element)=="table" then + element.__p__=newparent end + return element + end end - --- split the next one up for readability (but this module needs a cleanup anyway) - -local function can_be_dir(name) -- can become local - local fakepaths = instance.fakepaths - if not fakepaths[name] then - if lfs.isdir(name) then - fakepaths[name] = 1 -- directory - else - fakepaths[name] = 2 -- no directory +function xml.delete(root,pattern) + if not pattern or pattern=="" then + local p=root.__p__ + if p then + if trace_manipulations then + report('deleting',"--",c,root) + end + local d=p.dt + remove(d,root.ni) + redo_ni(d) + end + else + local collected=xmlapplylpath(root,pattern) + if collected then + for c=1,#collected do + local e=collected[c] + local p=e.__p__ + if p then + if trace_manipulations then + report('deleting',pattern,c,e) + end + local d=p.dt + remove(d,e.ni) + redo_ni(d) end + end end - return fakepaths[name] == 1 + end end - -local preparetreepattern = Cs((P(".")/"%%." + P("-")/"%%-" + P(1))^0 * Cc("$")) - --- -- -- begin of main file search routing -- -- -- needs checking as previous has been patched - -local collect_instance_files - -local function find_analyze(filename,askedformat,allresults) - local filetype, wantedfiles, ext = '', { }, fileextname(filename) - -- too tricky as filename can be bla.1.2.3: - -- - -- if not suffixmap[ext] then - -- wantedfiles[#wantedfiles+1] = filename - -- end - wantedfiles[#wantedfiles+1] = filename - if askedformat == "" then - if ext == "" or not suffixmap[ext] then - local defaultsuffixes = resolvers.defaultsuffixes - for i=1,#defaultsuffixes do - local forcedname = filename .. '.' .. defaultsuffixes[i] - wantedfiles[#wantedfiles+1] = forcedname - filetype = resolvers.formatofsuffix(forcedname) - if trace_locating then - report_resolving("forcing filetype '%s'",filetype) - end - end - else - filetype = resolvers.formatofsuffix(filename) - if trace_locating then - report_resolving("using suffix based filetype '%s'",filetype) - end - end - else - if ext == "" or not suffixmap[ext] then - local format_suffixes = suffixes[askedformat] - if format_suffixes then - for i=1,#format_suffixes do - wantedfiles[#wantedfiles+1] = filename .. "." .. format_suffixes[i] - end - end - end - filetype = askedformat - if trace_locating then - report_resolving("using given filetype '%s'",filetype) +function xml.replace(root,pattern,whatever) + local element=root and xmltoelement(whatever,root) + local collected=element and xmlapplylpath(root,pattern) + if collected then + for c=1,#collected do + local e=collected[c] + local p=e.__p__ + if p then + if trace_manipulations then + report('replacing',pattern,c,e) end + local d=p.dt + d[e.ni]=copiedelement(element,p) + redo_ni(d) + end end - return filetype, wantedfiles + end end - -local function find_direct(filename,allresults) - if not dangerous[askedformat] and isreadable(filename) then - if trace_detail then - report_resolving("file '%s' found directly",filename) - end - return "direct", { filename } - end +local function wrap(e,wrapper) + local t={ + rn=e.rn, + tg=e.tg, + ns=e.ns, + at=e.at, + dt=e.dt, + __p__=e, + } + setmetatable(t,getmetatable(e)) + e.rn=wrapper.rn or e.rn or "" + e.tg=wrapper.tg or e.tg or "" + e.ns=wrapper.ns or e.ns or "" + e.at=fastcopy(wrapper.at) + e.dt={ t } end - -local function find_wildcard(filename,allresults) - if find(filename,'%*') then - if trace_locating then - report_resolving("checking wildcard '%s'", filename) - end - local method, result = resolvers.findwildcardfiles(filename) - if result then - return "wildcard", result +function xml.wrap(root,pattern,whatever) + if whatever then + local wrapper=xmltoelement(whatever,root) + local collected=xmlapplylpath(root,pattern) + if collected then + for c=1,#collected do + local e=collected[c] + if trace_manipulations then + report('wrapping',pattern,c,e) end + wrap(e,wrapper) + end end + else + wrap(root,xmltoelement(pattern)) + end end - -local function find_qualified(filename,allresults) -- this one will be split too - if not file.is_qualified_path(filename) then - return - end - if trace_locating then - report_resolving("checking qualified name '%s'", filename) - end - if isreadable(filename) then - if trace_detail then - report_resolving("qualified file '%s' found", filename) - end - return "qualified", { filename } - end - if trace_detail then - report_resolving("locating qualified file '%s'", filename) - end - local forcedname, suffix = "", fileextname(filename) - if suffix == "" then -- why - local format_suffixes = askedformat == "" and resolvers.defaultsuffixes or suffixes[askedformat] - if format_suffixes then - for i=1,#format_suffixes do - local s = format_suffixes[i] - forcedname = filename .. "." .. s - if isreadable(forcedname) then - if trace_locating then - report_resolving("no suffix, forcing format filetype '%s'", s) - end - return "qualified", { forcedname } - end - end - end - end - if suffix and suffix ~= "" then - -- try to find in tree (no suffix manipulation), here we search for the - -- matching last part of the name - local basename = filebasename(filename) - local pattern = lpegmatch(preparetreepattern,filename) - -- messy .. to be sorted out - local savedformat = askedformat - local format = savedformat or "" - if format == "" then - askedformat = resolvers.formatofsuffix(suffix) - end - if not format then - askedformat = "othertextfiles" -- kind of everything, maybe all - end - -- - if basename ~= filename then - local resolved = collect_instance_files(basename,askedformat,allresults) - if #resolved == 0 then - local lowered = lower(basename) - if filename ~= lowered then - resolved = collect_instance_files(lowered,askedformat,allresults) - end - end - resolvers.format = savedformat - -- - if #resolved > 0 then - local result = { } - for r=1,#resolved do - local rr = resolved[r] - if find(rr,pattern) then - result[#result+1] = rr - end - end - if #result > 0 then - return "qualified", result - end - end - end - -- a real wildcard: - -- - -- local filelist = collect_files({basename}) - -- result = { } - -- for f=1,#filelist do - -- local ff = filelist[f][3] or "" - -- if find(ff,pattern) then - -- result[#result+1], ok = ff, true - -- end - -- end - -- if #result > 0 then - -- return "qualified", result - -- end +local function inject_element(root,pattern,whatever,prepend) + local element=root and xmltoelement(whatever,root) + local collected=element and xmlapplylpath(root,pattern) + local function inject_e(e) + local r=e.__p__ + local d,k,rri=r.dt,e.ni,r.ri + local edt=(rri and d[rri].dt) or (d and d[k] and d[k].dt) + if edt then + local be,af + local cp=copiedelement(element,e) + if prepend then + be,af=cp,edt + else + be,af=edt,cp + end + local bn=#be + for i=1,#af do + bn=bn+1 + be[bn]=af[i] + end + if rri then + r.dt[rri].dt=be + else + d[k].dt=be + end + redo_ni(d) + end + end + if not collected then + elseif collected.tg then + inject_e(collected) + else + for c=1,#collected do + inject_e(collected[c]) + end + end +end +local function insert_element(root,pattern,whatever,before) + local element=root and xmltoelement(whatever,root) + local collected=element and xmlapplylpath(root,pattern) + local function insert_e(e) + local r=e.__p__ + local d,k=r.dt,e.ni + if not before then + k=k+1 + end + insert(d,k,copiedelement(element,r)) + redo_ni(d) + end + if not collected then + elseif collected.tg then + insert_e(collected) + else + for c=1,#collected do + insert_e(collected[c]) end + end end - -local function check_subpath(fname) - if isreadable(fname) then - if trace_detail then - report_resolving("found '%s' by deep scanning",fname) +xml.insert_element=insert_element +xml.insertafter=insert_element +xml.insertbefore=function(r,p,e) insert_element(r,p,e,true) end +xml.injectafter=inject_element +xml.injectbefore=function(r,p,e) inject_element(r,p,e,true) end +local function include(xmldata,pattern,attribute,recursive,loaddata) + pattern=pattern or 'include' + loaddata=loaddata or io.loaddata + local collected=xmlapplylpath(xmldata,pattern) + if collected then + for c=1,#collected do + local ek=collected[c] + local name=nil + local ekdt=ek.dt + local ekat=ek.at + local epdt=ek.__p__.dt + if not attribute or attribute=="" then + name=(type(ekdt)=="table" and ekdt[1]) or ekdt + end + if not name then + for a in gmatch(attribute or "href","([^|]+)") do + name=ekat[a] + if name then break end + end + end + local data=(name and name~="" and loaddata(name)) or "" + if data=="" then + epdt[ek.ni]="" + elseif ekat["parse"]=="text" then + epdt[ek.ni]=xml.escaped(data) + else + local xi=xmlinheritedconvert(data,xmldata) + if not xi then + epdt[ek.ni]="" + else + if recursive then + include(xi,pattern,attribute,recursive,loaddata) + end + epdt[ek.ni]=xml.body(xi) end - return fname + end end + end end - -local function find_intree(filename,filetype,wantedfiles,allresults) - local typespec = resolvers.variableofformat(filetype) - local pathlist = resolvers.expandedpathlist(typespec) - local method = "intree" - if pathlist and #pathlist > 0 then - -- list search - local filelist = collect_files(wantedfiles) - local dirlist = { } - if filelist then - for i=1,#filelist do - dirlist[i] = filedirname(filelist[i][3]) .. "/" -- was [2] .. gamble - end - end - if trace_detail then - report_resolving("checking filename '%s'",filename) - end - local result = { } - for k=1,#pathlist do - local path = pathlist[k] - local pathname = lpegmatch(inhibitstripper,path) - local doscan = path == pathname -- no ^!! - if not find (pathname,'//$') then - doscan = false -- we check directly on the path - end - local done = false - -- using file list - if filelist then -- database - -- compare list entries with permitted pattern -- /xx /xx// - local expression = makepathexpression(pathname) - if trace_detail then - report_resolving("using pattern '%s' for path '%s'",expression,pathname) - end - for k=1,#filelist do - local fl = filelist[k] - local f = fl[2] - local d = dirlist[k] - if find(d,expression) then - -- todo, test for readable - result[#result+1] = resolvers.resolve(fl[3]) -- no shortcut - done = true - if allresults then - if trace_detail then - report_resolving("match to '%s' in hash for file '%s' and path '%s', continue scanning",expression,f,d) - end - else - if trace_detail then - report_resolving("match to '%s' in hash for file '%s' and path '%s', quit scanning",expression,f,d) - end - break - end - elseif trace_detail then - report_resolving("no match to '%s' in hash for file '%s' and path '%s'",expression,f,d) - end - end - end - if done then - method = "database" - else - method = "filesystem" -- bonus, even when !! is specified - pathname = gsub(pathname,"/+$","") - pathname = resolvers.resolve(pathname) - local scheme = url.hasscheme(pathname) - if not scheme or scheme == "file" then - local pname = gsub(pathname,"%.%*$",'') - if not find(pname,"%*") then - if can_be_dir(pname) then - -- quick root scan first - for k=1,#wantedfiles do - local w = wantedfiles[k] - local fname = check_subpath(filejoin(pname,w)) - if fname then - result[#result+1] = fname - done = true - if not allresults then - break - end - end - end - if not done and doscan then - -- collect files in path (and cache the result) - local files = resolvers.simplescanfiles(pname,false,true) - for k=1,#wantedfiles do - local w = wantedfiles[k] - local subpath = files[w] - if not subpath or subpath == "" then - -- rootscan already done - elseif type(subpath) == "string" then - local fname = check_subpath(filejoin(pname,subpath,w)) - if fname then - result[#result+1] = fname - done = true - if not allresults then - break - end - end - else - for i=1,#subpath do - local sp = subpath[i] - if sp == "" then - -- roottest already done - else - local fname = check_subpath(filejoin(pname,sp,w)) - if fname then - result[#result+1] = fname - done = true - if not allresults then - break - end - end - end - end - if done and not allresults then - break - end - end - end - end - end - else - -- no access needed for non existing path, speedup (esp in large tree with lots of fake) - end - end - end - -- todo recursive scanning - if done and not allresults then - break - end - end - if #result > 0 then - return method, result - end +xml.include=include +local function stripelement(e,nolines,anywhere) + local edt=e.dt + if edt then + if anywhere then + local t,n={},0 + for e=1,#edt do + local str=edt[e] + if type(str)~="string" then + n=n+1 + t[n]=str + elseif str~="" then + if nolines then + str=gsub(str,"%s+"," ") + end + str=gsub(str,"^%s*(.-)%s*$","%1") + if str~="" then + n=n+1 + t[n]=str + end + end + end + e.dt=t + else + if #edt>0 then + local str=edt[1] + if type(str)~="string" then + elseif str=="" then + remove(edt,1) + else + if nolines then + str=gsub(str,"%s+"," ") + end + str=gsub(str,"^%s+","") + if str=="" then + remove(edt,1) + else + edt[1]=str + end + end + end + local nedt=#edt + if nedt>0 then + local str=edt[nedt] + if type(str)~="string" then + elseif str=="" then + remove(edt) + else + if nolines then + str=gsub(str,"%s+"," ") + end + str=gsub(str,"%s+$","") + if str=="" then + remove(edt) + else + edt[nedt]=str + end + end + end + end + end + return e +end +xml.stripelement=stripelement +function xml.strip(root,pattern,nolines,anywhere) + local collected=xmlapplylpath(root,pattern) + if collected then + for i=1,#collected do + stripelement(collected[i],nolines,anywhere) + end + end +end +local function renamespace(root,oldspace,newspace) + local ndt=#root.dt + for i=1,ndt or 0 do + local e=root[i] + if type(e)=="table" then + if e.ns==oldspace then + e.ns=newspace + if e.rn then + e.rn=newspace + end + end + local edt=e.dt + if edt then + renamespace(edt,oldspace,newspace) + end + end + end +end +xml.renamespace=renamespace +function xml.remaptag(root,pattern,newtg) + local collected=xmlapplylpath(root,pattern) + if collected then + for c=1,#collected do + collected[c].tg=newtg end + end end - -local function find_onpath(filename,filetype,wantedfiles,allresults) - if trace_detail then - report_resolving("checking filename '%s', filetype '%s', wanted files '%s'",filename, filetype or '?',concat(wantedfiles," | ")) - end - local result = { } - for k=1,#wantedfiles do - local fname = wantedfiles[k] - if fname and isreadable(fname) then - filename = fname - result[#result+1] = filejoin('.',fname) - if not allresults then - break - end - end +function xml.remapnamespace(root,pattern,newns) + local collected=xmlapplylpath(root,pattern) + if collected then + for c=1,#collected do + collected[c].ns=newns end - if #result > 0 then - return "onpath", result - end -end - -local function find_otherwise(filename,filetype,wantedfiles,allresults) -- other text files | any | whatever - local filelist = collect_files(wantedfiles) - local fl = filelist and filelist[1] - if fl then - return "otherwise", { resolvers.resolve(fl[3]) } -- filename - end -end - --- we could have a loop over the 6 functions but then we'd have to --- always analyze - -collect_instance_files = function(filename,askedformat,allresults) -- uses nested - askedformat = askedformat or "" - filename = collapsepath(filename) - if allresults then - -- no need for caching, only used for tracing - local filetype, wantedfiles = find_analyze(filename,askedformat) - local results = { - { find_direct (filename,true) }, - { find_wildcard (filename,true) }, - { find_qualified(filename,true) }, - { find_intree (filename,filetype,wantedfiles,true) }, - { find_onpath (filename,filetype,wantedfiles,true) }, - { find_otherwise(filename,filetype,wantedfiles,true) }, - } - local result, status, done = { }, { }, { } - for k, r in next, results do - local method, list = r[1], r[2] - if method and list then - for i=1,#list do - local c = collapsepath(list[i]) - if not done[c] then - result[#result+1] = c - done[c] = true - end - status[#status+1] = format("%-10s: %s",method,c) - end - end - end - if trace_detail then - report_resolving("lookup status: %s",table.serialize(status,filename)) - end - return result, status + end +end +function xml.checknamespace(root,pattern,newns) + local collected=xmlapplylpath(root,pattern) + if collected then + for c=1,#collected do + local e=collected[c] + if (not e.rn or e.rn=="") and e.ns=="" then + e.rn=newns + end + end + end +end +function xml.remapname(root,pattern,newtg,newns,newrn) + local collected=xmlapplylpath(root,pattern) + if collected then + for c=1,#collected do + local e=collected[c] + e.tg,e.ns,e.rn=newtg,newns,newrn + end + end +end +function xml.cdatatotext(e) + local dt=e.dt + if #dt==1 then + local first=dt[1] + if first.tg=="@cd@" then + e.dt=first.dt + end + else + end +end +function xml.texttocdata(e) + local dt=e.dt + local s=xml.tostring(dt) + e.tg="@cd@" + e.special=true + e.ns="" + e.rn="" + e.dt={ s } + e.at=nil +end +function xml.elementtocdata(e) + local dt=e.dt + local s=xml.tostring(e) + e.tg="@cd@" + e.special=true + e.ns="" + e.rn="" + e.dt={ s } + e.at=nil +end +xml.builtinentities=table.tohash { "amp","quot","apos","lt","gt" } +local entities=characters and characters.entities or nil +local builtinentities=xml.builtinentities +function xml.addentitiesdoctype(root,option) + if not entities then + require("char-ent") + entities=characters.entities + end + if entities and root and root.tg=="@rt@" and root.statistics then + local list={} + local hexify=option=="hexadecimal" + for k,v in table.sortedhash(root.statistics.entities.names) do + if not builtinentities[k] then + local e=entities[k] + if not e then + e=format("[%s]",k) + elseif hexify then + e=format("&#%05X;",utfbyte(k)) + end + list[#list+1]=format(" ",k,e) + end + end + local dt=root.dt + local n=dt[1].tg=="@pi@" and 2 or 1 + if #list>0 then + insert(dt,n,{ "\n" }) + insert(dt,n,{ + tg="@dt@", + dt={ format("Something [\n%s\n] ",concat(list)) }, + ns="", + special=true, + }) + insert(dt,n,{ "\n\n" }) else - local method, result, stamp, filetype, wantedfiles - if instance.remember then - stamp = format("%s--%s", filename, askedformat) - result = stamp and instance.found[stamp] - if result then - if trace_locating then - report_resolving("remembered file '%s'",filename) - end - return result - end - end - method, result = find_direct(filename) - if not result then - method, result = find_wildcard(filename) - if not result then - method, result = find_qualified(filename) - if not result then - filetype, wantedfiles = find_analyze(filename,askedformat) - method, result = find_intree(filename,filetype,wantedfiles) - if not result then - method, result = find_onpath(filename,filetype,wantedfiles) - if not result then - method, result = find_otherwise(filename,filetype,wantedfiles) - end - end - end - end - end - if result and #result > 0 then - local foundname = collapsepath(result[1]) - resolvers.registerintrees(filename,askedformat,filetype,method,foundname) - result = { foundname } - else - result = { } -- maybe false - end - if stamp then - if trace_locating then - report_resolving("remembering file '%s'",filename) - end - instance.found[stamp] = result - end - return result end + end +end +xml.all=xml.each +xml.insert=xml.insertafter +xml.inject=xml.injectafter +xml.after=xml.insertafter +xml.before=xml.insertbefore +xml.process=xml.each +xml.obsolete=xml.obsolete or {} +local obsolete=xml.obsolete +xml.strip_whitespace=xml.strip obsolete.strip_whitespace=xml.strip +xml.collect_elements=xml.collect obsolete.collect_elements=xml.collect +xml.delete_element=xml.delete obsolete.delete_element=xml.delete +xml.replace_element=xml.replace obsolete.replace_element=xml.replacet +xml.each_element=xml.each obsolete.each_element=xml.each +xml.process_elements=xml.process obsolete.process_elements=xml.process +xml.insert_element_after=xml.insertafter obsolete.insert_element_after=xml.insertafter +xml.insert_element_before=xml.insertbefore obsolete.insert_element_before=xml.insertbefore +xml.inject_element_after=xml.injectafter obsolete.inject_element_after=xml.injectafter +xml.inject_element_before=xml.injectbefore obsolete.inject_element_before=xml.injectbefore +xml.process_attributes=xml.processattributes obsolete.process_attributes=xml.processattributes +xml.collect_texts=xml.collecttexts obsolete.collect_texts=xml.collecttexts +xml.inject_element=xml.inject obsolete.inject_element=xml.inject +xml.remap_tag=xml.remaptag obsolete.remap_tag=xml.remaptag +xml.remap_name=xml.remapname obsolete.remap_name=xml.remapname +xml.remap_namespace=xml.remapnamespace obsolete.remap_namespace=xml.remapnamespace +function xml.cdata(e) + if e then + local dt=e.dt + if dt and #dt==1 then + local first=dt[1] + return first.tg=="@cd@" and first.dt[1] or "" + end + end + return "" +end +function xml.finalizers.xml.cdata(collected) + if collected then + local e=collected[1] + if e then + local dt=e.dt + if dt and #dt==1 then + local first=dt[1] + return first.tg=="@cd@" and first.dt[1] or "" + end + end + end + return "" +end +function xml.insertcomment(e,str,n) + table.insert(e.dt,n or 1,{ + tg="@cm@", + ns="", + special=true, + at={}, + dt={ str }, + }) +end +function xml.setcdata(e,str) + e.dt={ { + tg="@cd@", + ns="", + special=true, + at={}, + dt={ str }, + } } +end +function xml.separate(x,pattern) + local collected=xmlapplylpath(x,pattern) + if collected then + for c=1,#collected do + local e=collected[c] + local d=e.dt + if d==x then + report_xml("warning: xml.separate changes root") + x=d + end + local t,n={ "\n" },1 + local i,nd=1,#d + while i<=nd do + while i<=nd do + local di=d[i] + if type(di)=="string" then + if di=="\n" or find(di,"^%s+$") then + i=i+1 + else + d[i]=strip(di) + break + end + else + break + end + end + if i>nd then + break + end + t[n+1]="\n" + t[n+2]=d[i] + t[n+3]="\n" + n=n+3 + i=i+1 + end + t[n+1]="\n" + setmetatable(t,getmetatable(d)) + e.dt=t + end + end + return x +end +local helpers=xml.helpers or {} +xml.helpers=helpers +local function normal(e,action) + local edt=e.dt + if edt then + for i=1,#edt do + local str=edt[i] + if type(str)=="string" and str~="" then + edt[i]=action(str) + end + end + end +end +local function recurse(e,action) + local edt=e.dt + if edt then + for i=1,#edt do + local str=edt[i] + if type(str)~="string" then + recurse(str,action,recursive) + elseif str~="" then + edt[i]=action(str) + end + end + end +end +function helpers.recursetext(collected,action,recursive) + if recursive then + for i=1,#collected do + recurse(collected[i],action) + end + else + for i=1,#collected do + normal(collected[i],action) + end + end end --- -- -- end of main file search routing -- -- -- +end -- of closure -local function findfiles(filename,filetype,allresults) - local result, status = collect_instance_files(filename,filetype or "",allresults) - if not result or #result == 0 then - local lowered = lower(filename) - if filename ~= lowered then - result, status = collect_instance_files(lowered,filetype or "",allresults) - end - end - return result or { }, status -end +do -- create closure to overcome 200 locals limit -function resolvers.findfiles(filename,filetype) - return findfiles(filename,filetype,true) -end +package.loaded["lxml-xml"] = package.loaded["lxml-xml"] or true -function resolvers.findfile(filename,filetype) - return findfiles(filename,filetype,false)[1] or "" -end +-- original size: 10274, stripped down to: 7538 -function resolvers.findpath(filename,filetype) - return filedirname(findfiles(filename,filetype,false)[1] or "") +if not modules then modules={} end modules ['lxml-xml']={ + version=1.001, + comment="this module is the basis for the lxml-* ones", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local concat=table.concat +local find,lower,upper=string.find,string.lower,string.upper +local xml=xml +local finalizers=xml.finalizers.xml +local xmlfilter=xml.filter +local xmltostring=xml.tostring +local xmlserialize=xml.serialize +local xmlcollected=xml.collected +local xmlnewhandlers=xml.newhandlers +local function first(collected) + return collected and collected[1] end - -local function findgivenfiles(filename,allresults) - local bname, result = filebasename(filename), { } - local hashes = instance.hashes - local noffound = 0 - for k=1,#hashes do - local hash = hashes[k] - local files = instance.files[hash.name] or { } - local blist = files[bname] - if not blist then - local rname = "remap:"..bname - blist = files[rname] - if blist then - bname = files[rname] - blist = files[bname] - end - end - if blist then - if type(blist) == 'string' then - local found = methodhandler('concatinators',hash.type,hash.name,blist,bname) or "" - if found ~= "" then - noffound = noffound + 1 - result[noffound] = resolvers.resolve(found) - if not allresults then break end - end - else - for kk=1,#blist do - local vv = blist[kk] - local found = methodhandler('concatinators',hash.type,hash.name,vv,bname) or "" - if found ~= "" then - noffound = noffound + 1 - result[noffound] = resolvers.resolve(found) - if not allresults then break end - end - end - end +local function last(collected) + return collected and collected[#collected] +end +local function all(collected) + return collected +end +local reverse=table.reversed +local function attribute(collected,name) + if collected and #collected>0 then + local at=collected[1].at + return at and at[name] + end +end +local function att(id,name) + local at=id.at + return at and at[name] +end +local function count(collected) + return collected and #collected or 0 +end +local function position(collected,n) + if not collected then + return 0 + end + local nc=#collected + if nc==0 then + return 0 + end + n=tonumber(n) or 0 + if n<0 then + return collected[nc+n+1] + elseif n>0 then + return collected[n] + else + return collected[1].mi or 0 + end +end +local function match(collected) + return collected and #collected>0 and collected[1].mi or 0 +end +local function index(collected) + return collected and #collected>0 and collected[1].ni or 0 +end +local function attributes(collected,arguments) + if collected and #collected>0 then + local at=collected[1].at + if arguments then + return at[arguments] + elseif next(at) then + return at + end + end +end +local function chainattribute(collected,arguments) + if collected and #collected>0 then + local e=collected[1] + while e do + local at=e.at + if at then + local a=at[arguments] + if a then + return a end + else + break + end + e=e.__p__ end + end + return "" +end +local function raw(collected) + if collected and #collected>0 then + local e=collected[1] or collected + return e and xmltostring(e) or "" + else + return "" + end +end +local xmltexthandler=xmlnewhandlers { + name="string", + initialize=function() + result={} return result + end, + finalize=function() + return concat(result) + end, + handle=function(...) + result[#result+1]=concat {... } + end, + escape=false, +} +local function xmltotext(root) + local dt=root.dt + if not dt then + return "" + end + local nt=#dt + if nt==0 then + return "" + elseif nt==1 and type(dt[1])=="string" then + return dt[1] + else + return xmlserialize(root,xmltexthandler) or "" + end +end +local function text(collected) + if collected then + local e=collected[1] or collected + return e and xmltotext(e) or "" + else + return "" + end end - -function resolvers.findgivenfiles(filename) - return findgivenfiles(filename,true) +local function texts(collected) + if not collected then + return {} + end + local nc=#collected + if nc==0 then + return {} + end + local t,n={},0 + for c=1,nc do + local e=collected[c] + if e and e.dt then + n=n+1 + t[n]=e.dt + end + end + return t end - -function resolvers.findgivenfile(filename) - return findgivenfiles(filename,false)[1] or "" +local function tag(collected,n) + if not collected then + return + end + local nc=#collected + if nc==0 then + return + end + local c + if n==0 or not n then + c=collected[1] + elseif n>1 then + c=collected[n] + else + c=collected[nc-n+1] + end + return c and c.tg end - -local function doit(path,blist,bname,tag,variant,result,allresults) - local done = false - if blist and variant then - local resolve = resolvers.resolve -- added - if type(blist) == 'string' then - -- make function and share code - if find(lower(blist),path) then - local full = methodhandler('concatinators',variant,tag,blist,bname) or "" - result[#result+1] = resolve(full) - done = true - end +local function name(collected,n) + if not collected then + return + end + local nc=#collected + if nc==0 then + return + end + local c + if n==0 or not n then + c=collected[1] + elseif n>1 then + c=collected[n] + else + c=collected[nc-n+1] + end + if not c then + elseif c.ns=="" then + return c.tg + else + return c.ns..":"..c.tg + end +end +local function tags(collected,nonamespace) + if not collected then + return + end + local nc=#collected + if nc==0 then + return + end + local t,n={},0 + for c=1,nc do + local e=collected[c] + local ns,tg=e.ns,e.tg + n=n+1 + if nonamespace or ns=="" then + t[n]=tg + else + t[n]=ns..":"..tg + end + end + return t +end +local function empty(collected,spacesonly) + if not collected then + return true + end + local nc=#collected + if nc==0 then + return true + end + for c=1,nc do + local e=collected[c] + if e then + local edt=e.dt + if edt then + local n=#edt + if n==1 then + local edk=edt[1] + local typ=type(edk) + if typ=="table" then + return false + elseif edk~="" then + return false + elseif spacesonly and not find(edk,"%S") then + return false + end + elseif n>1 then + return false + end + end + end + end + return true +end +finalizers.first=first +finalizers.last=last +finalizers.all=all +finalizers.reverse=reverse +finalizers.elements=all +finalizers.default=all +finalizers.attribute=attribute +finalizers.att=att +finalizers.count=count +finalizers.position=position +finalizers.match=match +finalizers.index=index +finalizers.attributes=attributes +finalizers.chainattribute=chainattribute +finalizers.text=text +finalizers.texts=texts +finalizers.tag=tag +finalizers.name=name +finalizers.tags=tags +finalizers.empty=empty +function xml.first(id,pattern) + return first(xmlfilter(id,pattern)) +end +function xml.last(id,pattern) + return last(xmlfilter(id,pattern)) +end +function xml.count(id,pattern) + return count(xmlfilter(id,pattern)) +end +function xml.attribute(id,pattern,a,default) + return attribute(xmlfilter(id,pattern),a,default) +end +function xml.raw(id,pattern) + if pattern then + return raw(xmlfilter(id,pattern)) + else + return raw(id) + end +end +function xml.text(id,pattern) + if pattern then + local collected=xmlfilter(id,pattern) + return collected and #collected>0 and xmltotext(collected[1]) or "" + elseif id then + return xmltotext(id) or "" + else + return "" + end +end +xml.content=text +function xml.position(id,pattern,n) + return position(xmlfilter(id,pattern),n) +end +function xml.match(id,pattern) + return match(xmlfilter(id,pattern)) +end +function xml.empty(id,pattern,spacesonly) + return empty(xmlfilter(id,pattern),spacesonly) +end +xml.all=xml.filter +xml.index=xml.position +xml.found=xml.filter +local function totable(x) + local t={} + for e in xmlcollected(x[1] or x,"/*") do + t[e.tg]=xmltostring(e.dt) or "" + end + return next(t) and t or nil +end +xml.table=totable +finalizers.table=totable +local function textonly(e,t) + if e then + local edt=e.dt + if edt then + for i=1,#edt do + local e=edt[i] + if type(e)=="table" then + textonly(e,t) else - for kk=1,#blist do - local vv = blist[kk] - if find(lower(vv),path) then - local full = methodhandler('concatinators',variant,tag,vv,bname) or "" - result[#result+1] = resolve(full) - done = true - if not allresults then break end - end - end + t[#t+1]=e end + end end - return done + end + return t end - - -local makewildcard = Cs( - (P("^")^0 * P("/") * P(-1) + P(-1)) /".*" - + (P("^")^0 * P("/") / "")^0 * (P("*")/".*" + P("-")/"%%-" + P(".")/"%%." + P("?")/"."+ P("\\")/"/" + P(1))^0 -) - -function resolvers.wildcardpattern(pattern) - return lpegmatch(makewildcard,pattern) or pattern -end - -local function findwildcardfiles(filename,allresults,result) -- todo: remap: and lpeg - result = result or { } - local base = filebasename(filename) - local dirn = filedirname(filename) - local path = lower(lpegmatch(makewildcard,dirn) or dirn) - local name = lower(lpegmatch(makewildcard,base) or base) - local files, done = instance.files, false - if find(name,"%*") then - local hashes = instance.hashes - for k=1,#hashes do - local hash = hashes[k] - local hashname, hashtype = hash.name, hash.type - for kk, hh in next, files[hashname] do - if not find(kk,"^remap:") then - if find(lower(kk),name) then - if doit(path,hh,kk,hashname,hashtype,result,allresults) then done = true end - if done and not allresults then break end - end - end - end +function xml.textonly(e) + return concat(textonly(e,{})) +end +function finalizers.lowerall(collected) + for c=1,#collected do + local e=collected[c] + if not e.special then + e.tg=lower(e.tg) + local eat=e.at + if eat then + local t={} + for k,v in next,eat do + t[lower(k)]=v end - else - local hashes = instance.hashes - for k=1,#hashes do - local hash = hashes[k] - local hashname, hashtype = hash.name, hash.type - if doit(path,files[hashname][bname],bname,hashname,hashtype,result,allresults) then done = true end - if done and not allresults then break end + e.at=t + end + end + end +end +function finalizers.upperall(collected) + for c=1,#collected do + local e=collected[c] + if not e.special then + e.tg=upper(e.tg) + local eat=e.at + if eat then + local t={} + for k,v in next,eat do + t[upper(k)]=v end + e.at=t + end end - -- we can consider also searching the paths not in the database, but then - -- we end up with a messy search (all // in all path specs) - return result + end end -function resolvers.findwildcardfiles(filename,result) - return findwildcardfiles(filename,true,result) -end -function resolvers.findwildcardfile(filename) - return findwildcardfiles(filename,false)[1] or "" -end +end -- of closure --- main user functions +do -- create closure to overcome 200 locals limit -function resolvers.automount() - -- implemented later -end +package.loaded["trac-xml"] = package.loaded["trac-xml"] or true -function resolvers.load(option) - statistics.starttiming(instance) - identify_configuration_files() - load_configuration_files() - if option ~= "nofiles" then - load_databases() - resolvers.automount() - end - statistics.stoptiming(instance) - local files = instance.files - return files and next(files) and true -end +-- original size: 6351, stripped down to: 4919 -function resolvers.loadtime() - return statistics.elapsedtime(instance) +if not modules then modules={} end modules ['trac-xml']={ + version=1.001, + comment="companion to trac-log.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local formatters=string.formatters +local reporters=logs.reporters +local xmlserialize=xml.serialize +local xmlcollected=xml.collected +local xmltext=xml.text +local xmlfirst=xml.first +local function showhelp(specification,...) + local root=xml.convert(specification.helpinfo or "") + if not root then + return + end + local xs=xml.gethandlers("string") + xml.sethandlersfunction(xs,"short",function(e,handler) xmlserialize(e.dt,handler) end) + xml.sethandlersfunction(xs,"ref",function(e,handler) handler.handle("--"..e.at.name) end) + local wantedcategories=select("#",...)==0 and true or table.tohash {... } + local nofcategories=xml.count(root,"/application/flags/category") + local report=specification.report + for category in xmlcollected(root,"/application/flags/category") do + local categoryname=category.at.name or "" + if wantedcategories==true or wantedcategories[categoryname] then + if nofcategories>1 then + report("%s options:",categoryname) + report() + end + for subcategory in xmlcollected(category,"/subcategory") do + for flag in xmlcollected(subcategory,"/flag") do + local name=flag.at.name + local value=flag.at.value + local short=xmltext(xmlfirst(flag,"/short")) + if value then + report("--%-20s %s",formatters["%s=%s"](name,value),short) + else + report("--%-20s %s",name,short) + end + end + report() + end + end + end + for category in xmlcollected(root,"/application/examples/category") do + local title=xmltext(xmlfirst(category,"/title")) + if title and title~="" then + report() + report(title) + report() + end + for subcategory in xmlcollected(category,"/subcategory") do + for example in xmlcollected(subcategory,"/example") do + local command=xmltext(xmlfirst(example,"/command")) + local comment=xmltext(xmlfirst(example,"/comment")) + report(command) + end + report() + end + end + for comment in xmlcollected(root,"/application/comments/comment") do + local comment=xmltext(comment) + report() + report(comment) + report() + end end - -local function report(str) - if trace_locating then - report_resolving(str) -- has already verbose +local reporthelp=reporters.help +local exporthelp=reporters.export +local function xmlfound(t) + local helpinfo=t.helpinfo + if type(helpinfo)=="table" then + return false + end + if type(helpinfo)~="string" then + helpinfo="Warning: no helpinfo found." + t.helpinfo=helpinfo + return false + end + if string.find(helpinfo,".xml$") then + local ownscript=environment.ownscript + local helpdata=false + if ownscript then + local helpfile=file.join(file.pathpart(ownscript),helpinfo) + helpdata=io.loaddata(helpfile) + if helpdata=="" then + helpdata=false + end + end + if not helpdata then + local helpfile=resolvers.findfile(helpinfo,"tex") + helpdata=helpfile and io.loaddata(helpfile) + end + if helpdata and helpdata~="" then + helpinfo=helpdata + else + helpinfo=formatters["Warning: help file %a is not found."](helpinfo) + end + end + t.helpinfo=helpinfo + return string.find(t.helpinfo,"^<%?xml") and true or false +end +function reporters.help(t,...) + if xmlfound(t) then + showhelp(t,...) + else + reporthelp(t,...) + end +end +function reporters.export(t,methods,filename) + if not xmlfound(t) then + return exporthelp(t) + end + if not methods or methods=="" then + methods=environment.arguments["exporthelp"] + end + if not filename or filename=="" then + filename=environment.files[1] + end + dofile(resolvers.findfile("trac-exp.lua","tex")) + local exporters=logs.exporters + if not exporters or not methods then + return exporthelp(t) + end + if methods=="all" then + methods=table.keys(exporters) + elseif type(methods)=="string" then + methods=utilities.parsers.settings_to_array(methods) + else + return exporthelp(t) + end + if type(filename)~="string" or filename=="" then + filename=false + elseif file.pathpart(filename)=="" then + t.report("export file %a will not be saved on the current path (safeguard)",filename) + return + end + for i=1,#methods do + local method=methods[i] + local exporter=exporters[method] + if exporter then + local result=exporter(t,method) + if result and result~="" then + if filename then + local fullname=file.replacesuffix(filename,method) + t.report("saving export in %a",fullname) + io.savedata(fullname,result) + else + reporters.lines(t,result) + end + else + t.report("no output from exporter %a",method) + end else - print(str) + t.report("unknown exporter %a",method) end + end end -function resolvers.dowithfilesandreport(command, files, ...) -- will move - if files and #files > 0 then - if trace_locating then - report('') -- ? - end - if type(files) == "string" then - files = { files } - end - for f=1,#files do - local file = files[f] - local result = command(file,...) - if type(result) == 'string' then - report(result) + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["data-ini"] = package.loaded["data-ini"] or true + +-- original size: 7898, stripped down to: 5501 + +if not modules then modules={} end modules ['data-ini']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files", +} +local gsub,find,gmatch,char=string.gsub,string.find,string.gmatch,string.char +local next,type=next,type +local filedirname,filebasename,filejoin=file.dirname,file.basename,file.join +local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) +local trace_detail=false trackers.register("resolvers.details",function(v) trace_detail=v end) +local trace_expansions=false trackers.register("resolvers.expansions",function(v) trace_expansions=v end) +local report_initialization=logs.reporter("resolvers","initialization") +local ostype,osname,ossetenv,osgetenv=os.type,os.name,os.setenv,os.getenv +resolvers=resolvers or {} +local resolvers=resolvers +texconfig.kpse_init=false +texconfig.shell_escape='t' +if not (environment and environment.default_texmfcnf) and kpse and kpse.default_texmfcnf then + local default_texmfcnf=kpse.default_texmfcnf() + default_texmfcnf=gsub(default_texmfcnf,"$SELFAUTOLOC","selfautoloc:") + default_texmfcnf=gsub(default_texmfcnf,"$SELFAUTODIR","selfautodir:") + default_texmfcnf=gsub(default_texmfcnf,"$SELFAUTOPARENT","selfautoparent:") + default_texmfcnf=gsub(default_texmfcnf,"$HOME","home:") + environment.default_texmfcnf=default_texmfcnf +end +kpse={ original=kpse } +setmetatable(kpse,{ + __index=function(kp,name) + report_initialization("fatal error: kpse library is accessed (key: %s)",name) + os.exit() + end +} ) +do + local osfontdir=osgetenv("OSFONTDIR") + if osfontdir and osfontdir~="" then + elseif osname=="windows" then + ossetenv("OSFONTDIR","c:/windows/fonts//") + elseif osname=="macosx" then + ossetenv("OSFONTDIR","$HOME/Library/Fonts//;/Library/Fonts//;/System/Library/Fonts//") + end +end +do + local homedir=osgetenv(ostype=="windows" and 'USERPROFILE' or 'HOME') or '' + if not homedir or homedir=="" then + homedir=char(127) + end + homedir=file.collapsepath(homedir) + ossetenv("HOME",homedir) + ossetenv("USERPROFILE",homedir) + environment.homedir=homedir +end +do + local args=environment.originalarguments or arg + if not environment.ownmain then + environment.ownmain=status and string.match(string.lower(status.banner),"this is ([%a]+)") or "luatex" + end + local ownbin=environment.ownbin or args[-2] or arg[-2] or args[-1] or arg[-1] or arg[0] or "luatex" + local ownpath=environment.ownpath or os.selfdir + ownbin=file.collapsepath(ownbin) + ownpath=file.collapsepath(ownpath) + if not ownpath or ownpath=="" or ownpath=="unset" then + ownpath=args[-1] or arg[-1] + ownpath=ownpath and filedirname(gsub(ownpath,"\\","/")) + if not ownpath or ownpath=="" then + ownpath=args[-0] or arg[-0] + ownpath=ownpath and filedirname(gsub(ownpath,"\\","/")) + end + local binary=ownbin + if not ownpath or ownpath=="" then + ownpath=ownpath and filedirname(binary) + end + if not ownpath or ownpath=="" then + if os.binsuffix~="" then + binary=file.replacesuffix(binary,os.binsuffix) + end + local path=osgetenv("PATH") + if path then + for p in gmatch(path,"[^"..io.pathseparator.."]+") do + local b=filejoin(p,binary) + if lfs.isfile(b) then + local olddir=lfs.currentdir() + if lfs.chdir(p) then + local pp=lfs.currentdir() + if trace_locating and p~=pp then + report_initialization("following symlink %a to %a",p,pp) + end + ownpath=pp + lfs.chdir(olddir) else - for i=1,#result do - report(result[i]) -- could be unpack - end + if trace_locating then + report_initialization("unable to check path %a",p) + end + ownpath=p end + break + end end + end + end + if not ownpath or ownpath=="" then + ownpath="." + report_initialization("forcing fallback to ownpath %a",ownpath) + elseif trace_locating then + report_initialization("using ownpath %a",ownpath) end + end + environment.ownbin=ownbin + environment.ownpath=ownpath +end +resolvers.ownpath=environment.ownpath +function resolvers.getownpath() + return environment.ownpath +end +do + local ownpath=environment.ownpath or dir.current() + if ownpath then + ossetenv('SELFAUTOLOC',file.collapsepath(ownpath)) + ossetenv('SELFAUTODIR',file.collapsepath(ownpath.."/..")) + ossetenv('SELFAUTOPARENT',file.collapsepath(ownpath.."/../..")) + else + report_initialization("error: unable to locate ownpath") + os.exit() + end +end +local texos=environment.texos or osgetenv("TEXOS") +local texmfos=environment.texmfos or osgetenv('SELFAUTODIR') +if not texos or texos=="" then + texos=file.basename(texmfos) +end +ossetenv('TEXMFOS',texmfos) +ossetenv('TEXOS',texos) +ossetenv('SELFAUTOSYSTEM',os.platform) +environment.texos=texos +environment.texmfos=texmfos +local texroot=environment.texroot or osgetenv("TEXROOT") +if not texroot or texroot=="" then + texroot=osgetenv('SELFAUTOPARENT') + ossetenv('TEXROOT',texroot) +end +environment.texroot=file.collapsepath(texroot) +if profiler then + directives.register("system.profile",function() + profiler.start("luatex-profile.log") + end) +end +if not resolvers.resolve then + function resolvers.resolve (s) return s end + function resolvers.unresolve(s) return s end + function resolvers.repath (s) return s end end --- obsolete --- resolvers.varvalue = resolvers.variable -- output the value of variable $STRING. --- resolvers.expandvar = resolvers.expansion -- output variable expansion of STRING. +end -- of closure -function resolvers.showpath(str) -- output search path for file type NAME - return joinpath(resolvers.expandedpathlist(resolvers.formatofvariable(str))) -end +do -- create closure to overcome 200 locals limit -function resolvers.registerfile(files, name, path) - if files[name] then - if type(files[name]) == 'string' then - files[name] = { files[name], path } - else - files[name] = path - end - else - files[name] = path - end -end +package.loaded["data-exp"] = package.loaded["data-exp"] or true -function resolvers.dowithpath(name,func) - local pathlist = resolvers.expandedpathlist(name) - for i=1,#pathlist do - func("^"..resolvers.cleanpath(pathlist[i])) +-- original size: 14643, stripped down to: 9517 + +if not modules then modules={} end modules ['data-exp']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files", +} +local format,find,gmatch,lower,char,sub=string.format,string.find,string.gmatch,string.lower,string.char,string.sub +local concat,sort=table.concat,table.sort +local lpegmatch,lpegpatterns=lpeg.match,lpeg.patterns +local Ct,Cs,Cc,P,C,S=lpeg.Ct,lpeg.Cs,lpeg.Cc,lpeg.P,lpeg.C,lpeg.S +local type,next=type,next +local ostype=os.type +local collapsepath=file.collapsepath +local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) +local trace_expansions=false trackers.register("resolvers.expansions",function(v) trace_expansions=v end) +local report_expansions=logs.reporter("resolvers","expansions") +local resolvers=resolvers +local function f_first(a,b) + local t,n={},0 + for s in gmatch(b,"[^,]+") do + n=n+1;t[n]=a..s + end + return concat(t,",") +end +local function f_second(a,b) + local t,n={},0 + for s in gmatch(a,"[^,]+") do + n=n+1;t[n]=s..b + end + return concat(t,",") +end +local function f_both(a,b) + local t,n={},0 + for sb in gmatch(b,"[^,]+") do + for sa in gmatch(a,"[^,]+") do + n=n+1;t[n]=sa..sb + end + end + return concat(t,",") +end +local left=P("{") +local right=P("}") +local var=P((1-S("{}" ))^0) +local set=P((1-S("{},"))^0) +local other=P(1) +local l_first=Cs((Cc("{")*(C(set)*left*C(var)*right/f_first)*Cc("}")+other )^0 ) +local l_second=Cs((Cc("{")*(left*C(var)*right*C(set)/f_second)*Cc("}")+other )^0 ) +local l_both=Cs((Cc("{")*(left*C(var)*right*left*C(var)*right/f_both)*Cc("}")+other )^0 ) +local l_rest=Cs((left*var*(left/"")*var*(right/"")*var*right+other )^0 ) +local stripper_1=lpeg.stripper ("{}@") +local replacer_1=lpeg.replacer { { ",}",",@}" },{ "{,","{@," },} +local function splitpathexpr(str,newlist,validate) + if trace_expansions then + report_expansions("expanding variable %a",str) + end + local t,ok,done=newlist or {},false,false + local n=#t + str=lpegmatch(replacer_1,str) + repeat + local old=str + repeat + local old=str + str=lpegmatch(l_first,str) + until old==str + repeat + local old=str + str=lpegmatch(l_second,str) + until old==str + repeat + local old=str + str=lpegmatch(l_both,str) + until old==str + repeat + local old=str + str=lpegmatch(l_rest,str) + until old==str + until old==str + str=lpegmatch(stripper_1,str) + if validate then + for s in gmatch(str,"[^,]+") do + s=validate(s) + if s then + n=n+1 + t[n]=s + end + end + else + for s in gmatch(str,"[^,]+") do + n=n+1 + t[n]=s + end + end + if trace_expansions then + for k=1,#t do + report_expansions("% 4i: %s",k,t[k]) end + end + return t end - -function resolvers.dowithvariable(name,func) - func(expandedvariable(name)) +local function validate(s) + s=collapsepath(s) + return s~="" and not find(s,"^!*unset/*$") and s end - -function resolvers.locateformat(name) - local barename = file.removesuffix(name) -- gsub(name,"%.%a+$","") - local fmtname = caches.getfirstreadablefile(barename..".fmt","formats") or "" - if fmtname == "" then - fmtname = resolvers.findfile(barename..".fmt") - fmtname = resolvers.cleanpath(fmtname) - end - if fmtname ~= "" then - local barename = file.removesuffix(fmtname) - local luaname, lucname, luiname = barename .. ".lua", barename .. ".luc", barename .. ".lui" - if lfs.isfile(luiname) then - return barename, luiname - elseif lfs.isfile(lucname) then - return barename, lucname - elseif lfs.isfile(luaname) then - return barename, luaname +resolvers.validatedpath=validate +function resolvers.expandedpathfromlist(pathlist) + local newlist={} + for k=1,#pathlist do + splitpathexpr(pathlist[k],newlist,validate) + end + return newlist +end +local cleanup=lpeg.replacer { + { "!","" }, + { "\\","/" }, +} +function resolvers.cleanpath(str) + local doslashes=(P("\\")/"/"+1)^0 + local donegation=(P("!")/"" )^0 + local homedir=lpegmatch(Cs(donegation*doslashes),environment.homedir or "") + if homedir=="~" or homedir=="" or not lfs.isdir(homedir) then + if trace_expansions then + report_expansions("no home dir set, ignoring dependent paths") + end + function resolvers.cleanpath(str) + if not str or find(str,"~") then + return "" + else + return lpegmatch(cleanup,str) + end + end + else + local dohome=((P("~")+P("$HOME"))/homedir)^0 + local cleanup=Cs(donegation*dohome*doslashes) + function resolvers.cleanpath(str) + return str and lpegmatch(cleanup,str) or "" + end + end + return resolvers.cleanpath(str) +end +local expandhome=P("~")/"$HOME" +local dodouble=P('"')/""*(expandhome+(1-P('"')))^0*P('"')/"" +local dosingle=P("'")/""*(expandhome+(1-P("'")))^0*P("'")/"" +local dostring=(expandhome+1 )^0 +local stripper=Cs( + lpegpatterns.unspacer*(dosingle+dodouble+dostring)*lpegpatterns.unspacer +) +function resolvers.checkedvariable(str) + return type(str)=="string" and lpegmatch(stripper,str) or str +end +local cache={} +local splitter=lpeg.tsplitat(";") +local backslashswapper=lpeg.replacer("\\","/") +local function splitconfigurationpath(str) + if str then + local found=cache[str] + if not found then + if str=="" then + found={} + else + local split=lpegmatch(splitter,lpegmatch(backslashswapper,str)) + found={} + local noffound=0 + for i=1,#split do + local s=split[i] + if not find(s,"^{*unset}*") then + noffound=noffound+1 + found[noffound]=s + end + end + if trace_expansions then + report_expansions("splitting path specification %a",str) + for k=1,noffound do + report_expansions("% 4i: %s",k,found[k]) + end end + cache[str]=found + end end - return nil, nil + return found + end end - -function resolvers.booleanvariable(str,default) - local b = resolvers.expansion(str) - if b == "" then - return default - else - b = toboolean(b) - return (b == nil and default) or b +resolvers.splitconfigurationpath=splitconfigurationpath +function resolvers.splitpath(str) + if type(str)=='table' then + return str + else + return splitconfigurationpath(str) + end +end +function resolvers.joinpath(str) + if type(str)=='table' then + return file.joinpath(str) + else + return str + end +end +local attributes,directory=lfs.attributes,lfs.dir +local weird=P(".")^1+lpeg.anywhere(S("~`!#$%^&*()={}[]:;\"\'||<>,?\n\r\t")) +local timer={} +local scanned={} +local nofscans=0 +local scancache={} +local function scan(files,spec,path,n,m,r) + local full=(path=="" and spec) or (spec..path..'/') + local dirs={} + local nofdirs=0 + for name in directory(full) do + if not lpegmatch(weird,name) then + local mode=attributes(full..name,'mode') + if mode=='file' then + n=n+1 + local f=files[name] + if f then + if type(f)=='string' then + files[name]={ f,path } + else + f[#f+1]=path + end + else + files[name]=path + local lower=lower(name) + if name~=lower then + files["remap:"..lower]=name + r=r+1 + end + end + elseif mode=='directory' then + m=m+1 + nofdirs=nofdirs+1 + if path~="" then + dirs[nofdirs]=path..'/'..name + else + dirs[nofdirs]=name + end + end + end + end + if nofdirs>0 then + sort(dirs) + for i=1,nofdirs do + files,n,m,r=scan(files,spec,dirs[i],n,m,r) end + end + scancache[sub(full,1,-2)]=files + return files,n,m,r end - -function resolvers.dowithfilesintree(pattern,handle,before,after) -- will move, can be a nice iterator instead - local instance = resolvers.instance - local hashes = instance.hashes - for i=1,#hashes do - local hash = hashes[i] - local blobtype = hash.type - local blobpath = hash.name - if blobpath then - if before then - before(blobtype,blobpath,pattern) - end - local files = instance.files[blobpath] - local total, checked, done = 0, 0, 0 - if files then - for k,v in next, files do - total = total + 1 - if find(k,"^remap:") then - k = files[k] - v = k -- files[k] -- chained - end - if find(k,pattern) then - if type(v) == "string" then - checked = checked + 1 - if handle(blobtype,blobpath,v,k) then - done = done + 1 - end - else - checked = checked + #v - for i=1,#v do - if handle(blobtype,blobpath,v[i],k) then - done = done + 1 - end - end - end - end - end - end - if after then - after(blobtype,blobpath,pattern,total,checked,done) - end +local fullcache={} +function resolvers.scanfiles(path,branch,usecache) + statistics.starttiming(timer) + local realpath=resolvers.resolve(path) + if usecache then + local files=fullcache[realpath] + if files then + if trace_locating then + report_expansions("using caches scan of path %a, branch %a",path,branch or path) + end + return files + end + end + if trace_locating then + report_expansions("scanning path %a, branch %a",path,branch or path) + end + local files,n,m,r=scan({},realpath..'/',"",0,0,0) + files.__path__=path + files.__files__=n + files.__directories__=m + files.__remappings__=r + if trace_locating then + report_expansions("%s files found on %s directories with %s uppercase remappings",n,m,r) + end + if usecache then + scanned[#scanned+1]=realpath + fullcache[realpath]=files + end + nofscans=nofscans+1 + statistics.stoptiming(timer) + return files +end +local function simplescan(files,spec,path) + local full=(path=="" and spec) or (spec..path..'/') + local dirs={} + local nofdirs=0 + for name in directory(full) do + if not lpegmatch(weird,name) then + local mode=attributes(full..name,'mode') + if mode=='file' then + if not files[name] then + files[name]=path + end + elseif mode=='directory' then + nofdirs=nofdirs+1 + if path~="" then + dirs[nofdirs]=path..'/'..name + else + dirs[nofdirs]=name end + end + end + end + if nofdirs>0 then + sort(dirs) + for i=1,nofdirs do + files=simplescan(files,spec,dirs[i]) end + end + return files +end +local simplecache={} +local nofsharedscans=0 +function resolvers.simplescanfiles(path,branch,usecache) + statistics.starttiming(timer) + local realpath=resolvers.resolve(path) + if usecache then + local files=simplecache[realpath] + if not files then + files=scancache[realpath] + if files then + nofsharedscans=nofsharedscans+1 + end + end + if files then + if trace_locating then + report_expansions("using caches scan of path %a, branch %a",path,branch or path) + end + return files + end + end + if trace_locating then + report_expansions("scanning path %a, branch %a",path,branch or path) + end + local files=simplescan({},realpath..'/',"") + if trace_locating then + report_expansions("%s files found",table.count(files)) + end + if usecache then + scanned[#scanned+1]=realpath + simplecache[realpath]=files + end + nofscans=nofscans+1 + statistics.stoptiming(timer) + return files +end +function resolvers.scandata() + table.sort(scanned) + return { + n=nofscans, + shared=nofsharedscans, + time=statistics.elapsedtime(timer), + paths=scanned, + } end - -resolvers.obsolete = resolvers.obsolete or { } -local obsolete = resolvers.obsolete - -resolvers.find_file = resolvers.findfile obsolete.find_file = resolvers.findfile -resolvers.find_files = resolvers.findfiles obsolete.find_files = resolvers.findfiles end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-pre'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- It could be interesting to hook the resolver in the file --- opener so that unresolved prefixes travel around and we --- get more abstraction. - --- As we use this beforehand we will move this up in the chain --- of loading. - +package.loaded["data-env"] = package.loaded["data-env"] or true -local resolvers = resolvers -local prefixes = utilities.storage.allocate() -resolvers.prefixes = prefixes +-- original size: 8762, stripped down to: 6484 -local gsub = string.gsub -local cleanpath, findgivenfile, expansion = resolvers.cleanpath, resolvers.findgivenfile, resolvers.expansion -local getenv = resolvers.getenv -- we can probably also use resolvers.expansion -local P, Cs, lpegmatch = lpeg.P, lpeg.Cs, lpeg.match - --- getenv = function(...) return resolvers.getenv(...) end -- needs checking (definitions changes later on) - -prefixes.environment = function(str) - return cleanpath(expansion(str)) +if not modules then modules={} end modules ['data-env']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files", +} +local lower,gsub=string.lower,string.gsub +local resolvers=resolvers +local allocate=utilities.storage.allocate +local setmetatableindex=table.setmetatableindex +local suffixonly=file.suffixonly +local formats=allocate() +local suffixes=allocate() +local dangerous=allocate() +local suffixmap=allocate() +resolvers.formats=formats +resolvers.suffixes=suffixes +resolvers.dangerous=dangerous +resolvers.suffixmap=suffixmap +local luasuffixes=utilities.lua.suffixes +local relations=allocate { + core={ + ofm={ + names={ "ofm","omega font metric","omega font metrics" }, + variable='OFMFONTS', + suffixes={ 'ofm','tfm' }, + }, + ovf={ + names={ "ovf","omega virtual font","omega virtual fonts" }, + variable='OVFFONTS', + suffixes={ 'ovf','vf' }, + }, + tfm={ + names={ "tfm","tex font metric","tex font metrics" }, + variable='TFMFONTS', + suffixes={ 'tfm' }, + }, + vf={ + names={ "vf","virtual font","virtual fonts" }, + variable='VFFONTS', + suffixes={ 'vf' }, + }, + otf={ + names={ "otf","opentype","opentype font","opentype fonts"}, + variable='OPENTYPEFONTS', + suffixes={ 'otf' }, + }, + ttf={ + names={ "ttf","truetype","truetype font","truetype fonts","truetype collection","truetype collections","truetype dictionary","truetype dictionaries" }, + variable='TTFONTS', + suffixes={ 'ttf','ttc','dfont' }, + }, + afm={ + names={ "afm","adobe font metric","adobe font metrics" }, + variable="AFMFONTS", + suffixes={ "afm" }, + }, + pfb={ + names={ "pfb","type1","type 1","type1 font","type 1 font","type1 fonts","type 1 fonts" }, + variable='T1FONTS', + suffixes={ 'pfb','pfa' }, + }, + fea={ + names={ "fea","font feature","font features","font feature file","font feature files" }, + variable='FONTFEATURES', + suffixes={ 'fea' }, + }, + cid={ + names={ "cid","cid map","cid maps","cid file","cid files" }, + variable='FONTCIDMAPS', + suffixes={ 'cid','cidmap' }, + }, + fmt={ + names={ "fmt","format","tex format" }, + variable='TEXFORMATS', + suffixes={ 'fmt' }, + }, + mem={ + names={ 'mem',"metapost format" }, + variable='MPMEMS', + suffixes={ 'mem' }, + }, + mp={ + names={ "mp" }, + variable='MPINPUTS', + suffixes={ 'mp','mpvi','mpiv','mpii' }, + }, + tex={ + names={ "tex" }, + variable='TEXINPUTS', + suffixes={ 'tex',"mkvi","mkiv","mkii" }, + }, + icc={ + names={ "icc","icc profile","icc profiles" }, + variable='ICCPROFILES', + suffixes={ 'icc' }, + }, + texmfscripts={ + names={ "texmfscript","texmfscripts","script","scripts" }, + variable='TEXMFSCRIPTS', + suffixes={ 'rb','pl','py' }, + }, + lua={ + names={ "lua" }, + variable='LUAINPUTS', + suffixes={ luasuffixes.lua,luasuffixes.luc,luasuffixes.tma,luasuffixes.tmc }, + }, + lib={ + names={ "lib" }, + variable='CLUAINPUTS', + suffixes=os.libsuffix and { os.libsuffix } or { 'dll','so' }, + }, + bib={ + names={ 'bib' }, + suffixes={ 'bib' }, + }, + bst={ + names={ 'bst' }, + suffixes={ 'bst' }, + }, + fontconfig={ + names={ 'fontconfig','fontconfig file','fontconfig files' }, + variable='FONTCONFIG_PATH', + }, + }, + obsolete={ + enc={ + names={ "enc","enc files","enc file","encoding files","encoding file" }, + variable='ENCFONTS', + suffixes={ 'enc' }, + }, + map={ + names={ "map","map files","map file" }, + variable='TEXFONTMAPS', + suffixes={ 'map' }, + }, + lig={ + names={ "lig files","lig file","ligature file","ligature files" }, + variable='LIGFONTS', + suffixes={ 'lig' }, + }, + opl={ + names={ "opl" }, + variable='OPLFONTS', + suffixes={ 'opl' }, + }, + ovp={ + names={ "ovp" }, + variable='OVPFONTS', + suffixes={ 'ovp' }, + }, + }, + kpse={ + base={ + names={ 'base',"metafont format" }, + variable='MFBASES', + suffixes={ 'base','bas' }, + }, + cmap={ + names={ 'cmap','cmap files','cmap file' }, + variable='CMAPFONTS', + suffixes={ 'cmap' }, + }, + cnf={ + names={ 'cnf' }, + suffixes={ 'cnf' }, + }, + web={ + names={ 'web' }, + suffixes={ 'web','ch' } + }, + cweb={ + names={ 'cweb' }, + suffixes={ 'w','web','ch' }, + }, + gf={ + names={ 'gf' }, + suffixes={ 'gf' }, + }, + mf={ + names={ 'mf' }, + variable='MFINPUTS', + suffixes={ 'mf' }, + }, + mft={ + names={ 'mft' }, + suffixes={ 'mft' }, + }, + pk={ + names={ 'pk' }, + suffixes={ 'pk' }, + }, + }, +} +resolvers.relations=relations +function resolvers.updaterelations() + for category,categories in next,relations do + for name,relation in next,categories do + local rn=relation.names + local rv=relation.variable + local rs=relation.suffixes + if rn and rv then + for i=1,#rn do + local rni=lower(gsub(rn[i]," ","")) + formats[rni]=rv + if rs then + suffixes[rni]=rs + for i=1,#rs do + local rsi=rs[i] + suffixmap[rsi]=rni + end + end + end + end + if rs then + end + end + end +end +resolvers.updaterelations() +local function simplified(t,k) + return k and rawget(t,lower(gsub(k," ",""))) or nil end - -prefixes.relative = function(str,n) -- lfs.isfile - if io.exists(str) then - -- nothing - elseif io.exists("./" .. str) then - str = "./" .. str - else - local p = "../" - for i=1,n or 2 do - if io.exists(p .. str) then - str = p .. str - break - else - p = p .. "../" - end - end - end - return cleanpath(str) +setmetatableindex(formats,simplified) +setmetatableindex(suffixes,simplified) +setmetatableindex(suffixmap,simplified) +function resolvers.suffixofformat(str) + local s=suffixes[str] + return s and s[1] or "" end - -prefixes.auto = function(str) - local fullname = prefixes.relative(str) - if not lfs.isfile(fullname) then - fullname = prefixes.locate(str) - end - return fullname +function resolvers.suffixofformat(str) + return suffixes[str] or {} end - -prefixes.locate = function(str) - local fullname = findgivenfile(str) or "" - return cleanpath((fullname ~= "" and fullname) or str) +for name,format in next,formats do + dangerous[name]=true end - -prefixes.filename = function(str) - local fullname = findgivenfile(str) or "" - return cleanpath(file.basename((fullname ~= "" and fullname) or str)) -- no cleanpath needed here +dangerous.tex=nil +function resolvers.formatofvariable(str) + return formats[str] or '' end - -prefixes.pathname = function(str) - local fullname = findgivenfile(str) or "" - return cleanpath(file.dirname((fullname ~= "" and fullname) or str)) +function resolvers.formatofsuffix(str) + return suffixmap[suffixonly(str)] or 'tex' end - -prefixes.selfautoloc = function(str) - return cleanpath(file.join(getenv('SELFAUTOLOC'),str)) +function resolvers.variableofformat(str) + return formats[str] or '' end - -prefixes.selfautoparent = function(str) - return cleanpath(file.join(getenv('SELFAUTOPARENT'),str)) +function resolvers.variableofformatorsuffix(str) + local v=formats[str] + if v then + return v + end + v=suffixmap[suffixonly(str)] + if v then + return formats[v] + end + return '' end -prefixes.selfautodir = function(str) - return cleanpath(file.join(getenv('SELFAUTODIR'),str)) -end -prefixes.home = function(str) - return cleanpath(file.join(getenv('HOME'),str)) -end +end -- of closure + +do -- create closure to overcome 200 locals limit -prefixes.env = prefixes.environment -prefixes.rel = prefixes.relative -prefixes.loc = prefixes.locate -prefixes.kpse = prefixes.locate -prefixes.full = prefixes.locate -prefixes.file = prefixes.filename -prefixes.path = prefixes.pathname +package.loaded["data-tmp"] = package.loaded["data-tmp"] or true -function resolvers.allprefixes(separator) - local all = table.sortedkeys(prefixes) - if separator then - for i=1,#all do - all[i] = all[i] .. ":" +-- original size: 14308, stripped down to: 10956 + +if not modules then modules={} end modules ['data-tmp']={ + version=1.100, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local format,lower,gsub,concat=string.format,string.lower,string.gsub,table.concat +local serialize,serializetofile=table.serialize,table.tofile +local mkdirs,isdir,isfile=dir.mkdirs,lfs.isdir,lfs.isfile +local addsuffix,is_writable,is_readable=file.addsuffix,file.is_writable,file.is_readable +local formatters=string.formatters +local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) +local trace_cache=false trackers.register("resolvers.cache",function(v) trace_cache=v end) +local report_caches=logs.reporter("resolvers","caches") +local report_resolvers=logs.reporter("resolvers","caching") +local resolvers=resolvers +local directive_cleanup=false directives.register("system.compile.cleanup",function(v) directive_cleanup=v end) +local directive_strip=false directives.register("system.compile.strip",function(v) directive_strip=v end) +local compile=utilities.lua.compile +function utilities.lua.compile(luafile,lucfile,cleanup,strip) + if cleanup==nil then cleanup=directive_cleanup end + if strip==nil then strip=directive_strip end + return compile(luafile,lucfile,cleanup,strip) +end +caches=caches or {} +local caches=caches +local luasuffixes=utilities.lua.suffixes +caches.base=caches.base or "luatex-cache" +caches.more=caches.more or "context" +caches.direct=false +caches.tree=false +caches.force=true +caches.ask=false +caches.relocate=false +caches.defaults={ "TMPDIR","TEMPDIR","TMP","TEMP","HOME","HOMEPATH" } +local writable,readables,usedreadables=nil,{},{} +local function identify() + local texmfcaches=resolvers.cleanpathlist("TEXMFCACHE") + if texmfcaches then + for k=1,#texmfcaches do + local cachepath=texmfcaches[k] + if cachepath~="" then + cachepath=resolvers.resolve(cachepath) + cachepath=resolvers.cleanpath(cachepath) + cachepath=file.collapsepath(cachepath) + local valid=isdir(cachepath) + if valid then + if is_readable(cachepath) then + readables[#readables+1]=cachepath + if not writable and is_writable(cachepath) then + writable=cachepath + end + end + elseif not writable and caches.force then + local cacheparent=file.dirname(cachepath) + if is_writable(cacheparent) and true then + if not caches.ask or io.ask(format("\nShould I create the cache path %s?",cachepath),"no",{ "yes","no" })=="yes" then + mkdirs(cachepath) + if isdir(cachepath) and is_writable(cachepath) then + report_caches("path %a created",cachepath) + writable=cachepath + readables[#readables+1]=cachepath + end + end + end + end + end + end + end + local texmfcaches=caches.defaults + if texmfcaches then + for k=1,#texmfcaches do + local cachepath=texmfcaches[k] + cachepath=resolvers.expansion(cachepath) + if cachepath~="" then + cachepath=resolvers.resolve(cachepath) + cachepath=resolvers.cleanpath(cachepath) + local valid=isdir(cachepath) + if valid and is_readable(cachepath) then + if not writable and is_writable(cachepath) then + readables[#readables+1]=cachepath + writable=cachepath + break + end end + end + end + end + if not writable then + report_caches("fatal error: there is no valid writable cache path defined") + os.exit() + elseif #readables==0 then + report_caches("fatal error: there is no valid readable cache path defined") + os.exit() + end + writable=dir.expandname(resolvers.cleanpath(writable)) + local base,more,tree=caches.base,caches.more,caches.tree or caches.treehash() + if tree then + caches.tree=tree + writable=mkdirs(writable,base,more,tree) + for i=1,#readables do + readables[i]=file.join(readables[i],base,more,tree) + end + else + writable=mkdirs(writable,base,more) + for i=1,#readables do + readables[i]=file.join(readables[i],base,more) + end + end + if trace_cache then + for i=1,#readables do + report_caches("using readable path %a (order %s)",readables[i],i) end - return all + report_caches("using writable path %a",writable) + end + identify=function() + return writable,readables + end + return writable,readables end - -local function _resolve_(method,target) - local action = prefixes[method] - if action then - return action(target) - else - return method .. ":" .. target +function caches.usedpaths() + local writable,readables=identify() + if #readables>1 then + local result={} + for i=1,#readables do + local readable=readables[i] + if usedreadables[i] or readable==writable then + result[#result+1]=formatters["readable: %a (order %s)"](readable,i) + end end + result[#result+1]=formatters["writable: %a"](writable) + return result + else + return writable + end end - -local resolved, abstract = { }, { } - -function resolvers.resetresolve(str) - resolved, abstract = { }, { } +function caches.configfiles() + return concat(resolvers.instance.specification,";") end - -local function resolve(str) -- use schemes, this one is then for the commandline only - if type(str) == "table" then - local t = { } - for i=1,#str do - t[i] = resolve(str[i]) - end - return t +function caches.hashed(tree) + tree=gsub(tree,"[\\/]+$","") + tree=lower(tree) + local hash=md5.hex(tree) + if trace_cache or trace_locating then + report_caches("hashing tree %a, hash %a",tree,hash) + end + return hash +end +function caches.treehash() + local tree=caches.configfiles() + if not tree or tree=="" then + return false + else + return caches.hashed(tree) + end +end +local r_cache,w_cache={},{} +local function getreadablepaths(...) + local tags={... } + local hash=concat(tags,"/") + local done=r_cache[hash] + if not done then + local writable,readables=identify() + if #tags>0 then + done={} + for i=1,#readables do + done[i]=file.join(readables[i],...) + end else - local res = resolved[str] - if not res then - res = gsub(str,"([a-z][a-z]+):([^ \"\';,]*)",_resolve_) -- home:xx;selfautoparent:xx; etc (comma added) - resolved[str] = res - abstract[res] = str - end - return res + done=readables end + r_cache[hash]=done + end + return done end - -local function unresolve(str) - return abstract[str] or str +local function getwritablepath(...) + local tags={... } + local hash=concat(tags,"/") + local done=w_cache[hash] + if not done then + local writable,readables=identify() + if #tags>0 then + done=mkdirs(writable,...) + else + done=writable + end + w_cache[hash]=done + end + return done end - -resolvers.resolve = resolve -resolvers.unresolve = unresolve - -if os.uname then - - for k, v in next, os.uname() do - if not prefixes[k] then - prefixes[k] = function() return v end - end +caches.getreadablepaths=getreadablepaths +caches.getwritablepath=getwritablepath +function caches.getfirstreadablefile(filename,...) + local rd=getreadablepaths(...) + for i=1,#rd do + local path=rd[i] + local fullname=file.join(path,filename) + if is_readable(fullname) then + usedreadables[i]=true + return fullname,path end - + end + return caches.setfirstwritablefile(filename,...) end - -if os.type == "unix" then - - local pattern - - local function makepattern(t,k,v) - local colon = P(":") - local p - for k, v in table.sortedpairs(prefixes) do - if p then - p = P(k) + p - else - p = P(k) - end - end - pattern = Cs((p * colon + colon/";" + P(1))^0) - if t then - t[k] = v +function caches.setfirstwritablefile(filename,...) + local wr=getwritablepath(...) + local fullname=file.join(wr,filename) + return fullname,wr +end +function caches.define(category,subcategory) + return function() + return getwritablepath(category,subcategory) + end +end +function caches.setluanames(path,name) + return format("%s/%s.%s",path,name,luasuffixes.tma),format("%s/%s.%s",path,name,luasuffixes.tmc) +end +function caches.loaddata(readables,name) + if type(readables)=="string" then + readables={ readables } + end + for i=1,#readables do + local path=readables[i] + local tmaname,tmcname=caches.setluanames(path,name) + local loader=false + if isfile(tmcname) then + loader=loadfile(tmcname) + end + if not loader and isfile(tmaname) then + utilities.lua.compile(tmaname,tmcname) + if isfile(tmcname) then + loader=loadfile(tmcname) + end + if not loader then + loader=loadfile(tmaname) + end + end + if loader then + loader=loader() + collectgarbage("step") + return loader + end + end + return false +end +function caches.is_writable(filepath,filename) + local tmaname,tmcname=caches.setluanames(filepath,filename) + return is_writable(tmaname) +end +local saveoptions={ compact=true } +function caches.savedata(filepath,filename,data,raw) + local tmaname,tmcname=caches.setluanames(filepath,filename) + local reduce,simplify=true,true + if raw then + reduce,simplify=false,false + end + data.cache_uuid=os.uuid() + if caches.direct then + file.savedata(tmaname,serialize(data,true,saveoptions)) + else + serializetofile(tmaname,data,true,saveoptions) + end + utilities.lua.compile(tmaname,tmcname) +end +local content_state={} +function caches.contentstate() + return content_state or {} +end +function caches.loadcontent(cachename,dataname) + local name=caches.hashed(cachename) + local full,path=caches.getfirstreadablefile(addsuffix(name,luasuffixes.lua),"trees") + local filename=file.join(path,name) + local blob=loadfile(addsuffix(filename,luasuffixes.luc)) or loadfile(addsuffix(filename,luasuffixes.lua)) + if blob then + local data=blob() + if data and data.content then + if data.type==dataname then + if data.version==resolvers.cacheversion then + content_state[#content_state+1]=data.uuid + if trace_locating then + report_resolvers("loading %a for %a from %a",dataname,cachename,filename) + end + return data.content + else + report_resolvers("skipping %a for %a from %a (version mismatch)",dataname,cachename,filename) end + else + report_resolvers("skipping %a for %a from %a (datatype mismatch)",dataname,cachename,filename) + end + elseif trace_locating then + report_resolvers("skipping %a for %a from %a (no content)",dataname,cachename,filename) end - - makepattern() - - getmetatable(prefixes).__newindex = makepattern - - function resolvers.repath(str) - return lpegmatch(pattern,str) + elseif trace_locating then + report_resolvers("skipping %a for %a from %a (invalid file)",dataname,cachename,filename) + end +end +function caches.collapsecontent(content) + for k,v in next,content do + if type(v)=="table" and #v==1 then + content[k]=v[1] end - -else -- already the default: - - function resolvers.repath(str) - return str + end +end +function caches.savecontent(cachename,dataname,content) + local name=caches.hashed(cachename) + local full,path=caches.setfirstwritablefile(addsuffix(name,luasuffixes.lua),"trees") + local filename=file.join(path,name) + local luaname=addsuffix(filename,luasuffixes.lua) + local lucname=addsuffix(filename,luasuffixes.luc) + if trace_locating then + report_resolvers("preparing %a for %a",dataname,cachename) + end + local data={ + type=dataname, + root=cachename, + version=resolvers.cacheversion, + date=os.date("%Y-%m-%d"), + time=os.date("%H:%M:%S"), + content=content, + uuid=os.uuid(), + } + local ok=io.savedata(luaname,serialize(data,true)) + if ok then + if trace_locating then + report_resolvers("category %a, cachename %a saved in %a",dataname,cachename,luaname) end - + if utilities.lua.compile(luaname,lucname) then + if trace_locating then + report_resolvers("%a compiled to %a",dataname,lucname) + end + return true + else + if trace_locating then + report_resolvers("compiling failed for %a, deleting file %a",dataname,lucname) + end + os.remove(lucname) + end + elseif trace_locating then + report_resolvers("unable to save %a in %a (access error)",dataname,luaname) + end end @@ -14237,1074 +12523,2555 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-inp'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local allocate = utilities.storage.allocate -local resolvers = resolvers - -local methodhandler = resolvers.methodhandler -local registermethod = resolvers.registermethod - -local finders = allocate { helpers = { }, notfound = function() end } -local openers = allocate { helpers = { }, notfound = function() end } -local loaders = allocate { helpers = { }, notfound = function() return false, nil, 0 end } - -registermethod("finders", finders, "uri") -registermethod("openers", openers, "uri") -registermethod("loaders", loaders, "uri") - -resolvers.finders = finders -resolvers.openers = openers -resolvers.loaders = loaders - - -end -- of closure +package.loaded["data-met"] = package.loaded["data-met"] or true -do -- create closure to overcome 200 locals limit +-- original size: 4915, stripped down to: 3942 -if not modules then modules = { } end modules ['data-out'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" +if not modules then modules={} end modules ['data-met']={ + version=1.100, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" } - -local allocate = utilities.storage.allocate -local resolvers = resolvers - -local registermethod = resolvers.registermethod - -local savers = allocate { helpers = { } } - -resolvers.savers = savers - -registermethod("savers", savers, "uri") +local find,format=string.find,string.format +local sequenced=table.sequenced +local addurlscheme,urlhashed=url.addscheme,url.hashed +local trace_locating=false +trackers.register("resolvers.locating",function(v) trace_methods=v end) +trackers.register("resolvers.methods",function(v) trace_methods=v end) +local report_methods=logs.reporter("resolvers","methods") +local allocate=utilities.storage.allocate +local resolvers=resolvers +local registered={} +local function splitmethod(filename) + if not filename then + return { scheme="unknown",original=filename } + end + if type(filename)=="table" then + return filename + end + filename=file.collapsepath(filename) + if not find(filename,"://") then + return { scheme="file",path=filename,original=filename,filename=filename } + end + local specification=url.hashed(filename) + if not specification.scheme or specification.scheme=="" then + return { scheme="file",path=filename,original=filename,filename=filename } + else + return specification + end +end +resolvers.splitmethod=splitmethod +local function methodhandler(what,first,...) + local method=registered[what] + if method then + local how,namespace=method.how,method.namespace + if how=="uri" or how=="url" then + local specification=splitmethod(first) + local scheme=specification.scheme + local resolver=namespace and namespace[scheme] + if resolver then + if trace_methods then + report_methods("resolving, method %a, how %a, handler %a, argument %a",what,how,scheme,first) + end + return resolver(specification,...) + else + resolver=namespace.default or namespace.file + if resolver then + if trace_methods then + report_methods("resolving, method %a, how %a, handler %a, argument %a",what,how,"default",first) + end + return resolver(specification,...) + elseif trace_methods then + report_methods("resolving, method %a, how %a, handler %a, argument %a",what,how,"unset") + end + end + elseif how=="tag" then + local resolver=namespace and namespace[first] + if resolver then + if trace_methods then + report_methods("resolving, method %a, how %a, tag %a",what,how,first) + end + return resolver(...) + else + resolver=namespace.default or namespace.file + if resolver then + if trace_methods then + report_methods("resolving, method %a, how %a, tag %a",what,how,"default") + end + return resolver(...) + elseif trace_methods then + report_methods("resolving, method %a, how %a, tag %a",what,how,"unset") + end + end + end + else + report_methods("resolving, invalid method %a") + end +end +resolvers.methodhandler=methodhandler +function resolvers.registermethod(name,namespace,how) + registered[name]={ how=how or "tag",namespace=namespace } + namespace["byscheme"]=function(scheme,filename,...) + if scheme=="file" then + return methodhandler(name,filename,...) + else + return methodhandler(name,addurlscheme(filename,scheme),...) + end + end +end +local concatinators=allocate { notfound=file.join } +local locators=allocate { notfound=function() end } +local hashers=allocate { notfound=function() end } +local generators=allocate { notfound=function() end } +resolvers.concatinators=concatinators +resolvers.locators=locators +resolvers.hashers=hashers +resolvers.generators=generators +local registermethod=resolvers.registermethod +registermethod("concatinators",concatinators,"tag") +registermethod("locators",locators,"uri") +registermethod("hashers",hashers,"uri") +registermethod("generators",generators,"uri") end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-fil'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) - -local report_files = logs.reporter("resolvers","files") +package.loaded["data-res"] = package.loaded["data-res"] or true -local resolvers = resolvers +-- original size: 60857, stripped down to: 42496 -local finders, openers, loaders, savers = resolvers.finders, resolvers.openers, resolvers.loaders, resolvers.savers -local locators, hashers, generators, concatinators = resolvers.locators, resolvers.hashers, resolvers.generators, resolvers.concatinators - -local checkgarbage = utilities.garbagecollector and utilities.garbagecollector.check - -function locators.file(specification) - local name = specification.filename - local realname = resolvers.resolve(name) -- no shortcut - if realname and realname ~= '' and lfs.isdir(realname) then +if not modules then modules={} end modules ['data-res']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files", +} +local gsub,find,lower,upper,match,gmatch=string.gsub,string.find,string.lower,string.upper,string.match,string.gmatch +local concat,insert,sortedkeys=table.concat,table.insert,table.sortedkeys +local next,type,rawget=next,type,rawget +local os=os +local P,S,R,C,Cc,Cs,Ct,Carg=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.Cc,lpeg.Cs,lpeg.Ct,lpeg.Carg +local lpegmatch,lpegpatterns=lpeg.match,lpeg.patterns +local formatters=string.formatters +local filedirname=file.dirname +local filebasename=file.basename +local suffixonly=file.suffixonly +local filejoin=file.join +local collapsepath=file.collapsepath +local joinpath=file.joinpath +local allocate=utilities.storage.allocate +local settings_to_array=utilities.parsers.settings_to_array +local setmetatableindex=table.setmetatableindex +local luasuffixes=utilities.lua.suffixes +local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) +local trace_detail=false trackers.register("resolvers.details",function(v) trace_detail=v end) +local trace_expansions=false trackers.register("resolvers.expansions",function(v) trace_expansions=v end) +local report_resolving=logs.reporter("resolvers","resolving") +local resolvers=resolvers +local expandedpathfromlist=resolvers.expandedpathfromlist +local checkedvariable=resolvers.checkedvariable +local splitconfigurationpath=resolvers.splitconfigurationpath +local methodhandler=resolvers.methodhandler +local initializesetter=utilities.setters.initialize +local ostype,osname,osenv,ossetenv,osgetenv=os.type,os.name,os.env,os.setenv,os.getenv +resolvers.cacheversion='1.0.1' +resolvers.configbanner='' +resolvers.homedir=environment.homedir +resolvers.criticalvars=allocate { "SELFAUTOLOC","SELFAUTODIR","SELFAUTOPARENT","TEXMFCNF","TEXMF","TEXOS" } +resolvers.luacnfname="texmfcnf.lua" +resolvers.luacnfstate="unknown" +if environment.default_texmfcnf then + resolvers.luacnfspec=environment.default_texmfcnf +else + resolvers.luacnfspec="{selfautoloc:,selfautodir:,selfautoparent:}{,/texmf{-local,}/web2c}" +end +resolvers.luacnfspec='home:texmf/web2c;'..resolvers.luacnfspec +local unset_variable="unset" +local formats=resolvers.formats +local suffixes=resolvers.suffixes +local dangerous=resolvers.dangerous +local suffixmap=resolvers.suffixmap +resolvers.defaultsuffixes={ "tex" } +resolvers.instance=resolvers.instance or nil +local instance=resolvers.instance or nil +function resolvers.setenv(key,value,raw) + if instance then + instance.environment[key]=value + ossetenv(key,raw and value or resolvers.resolve(value)) + end +end +local function getenv(key) + local value=rawget(instance.environment,key) + if value and value~="" then + return value + else + local e=osgetenv(key) + return e~=nil and e~="" and checkedvariable(e) or "" + end +end +resolvers.getenv=getenv +resolvers.env=getenv +local function resolve(k) + return instance.expansions[k] +end +local dollarstripper=lpeg.stripper("$") +local inhibitstripper=P("!")^0*Cs(P(1)^0) +local backslashswapper=lpeg.replacer("\\","/") +local somevariable=P("$")/"" +local somekey=C(R("az","AZ","09","__","--")^1) +local somethingelse=P(";")*((1-S("!{}/\\"))^1*P(";")/"")+P(";")*(P(";")/"")+P(1) +local variableexpander=Cs((somevariable*(somekey/resolve)+somethingelse)^1 ) +local cleaner=P("\\")/"/"+P(";")*S("!{}/\\")^0*P(";")^1/";" +local variablecleaner=Cs((cleaner+P(1))^0) +local somevariable=R("az","AZ","09","__","--")^1/resolve +local variable=(P("$")/"")*(somevariable+(P("{")/"")*somevariable*(P("}")/"")) +local variableresolver=Cs((variable+P(1))^0) +local function expandedvariable(var) + return lpegmatch(variableexpander,var) or var +end +function resolvers.newinstance() + if trace_locating then + report_resolving("creating instance") + end + local environment,variables,expansions,order=allocate(),allocate(),allocate(),allocate() + local newinstance={ + environment=environment, + variables=variables, + expansions=expansions, + order=order, + files=allocate(), + setups=allocate(), + found=allocate(), + foundintrees=allocate(), + hashes=allocate(), + hashed=allocate(), + specification=allocate(), + lists=allocate(), + data=allocate(), + fakepaths=allocate(), + remember=true, + diskcache=true, + renewcache=false, + renewtree=false, + loaderror=false, + savelists=true, + pattern=nil, + force_suffixes=true, + } + setmetatableindex(variables,function(t,k) + local v + for i=1,#order do + v=order[i][k] + if v~=nil then + t[k]=v + return v + end + end + if v==nil then + v="" + end + t[k]=v + return v + end) + setmetatableindex(environment,function(t,k) + local v=osgetenv(k) + if v==nil then + v=variables[k] + end + if v~=nil then + v=checkedvariable(v) or "" + end + v=resolvers.repath(v) + t[k]=v + return v + end) + setmetatableindex(expansions,function(t,k) + local v=environment[k] + if type(v)=="string" then + v=lpegmatch(variableresolver,v) + v=lpegmatch(variablecleaner,v) + end + t[k]=v + return v + end) + return newinstance +end +function resolvers.setinstance(someinstance) + instance=someinstance + resolvers.instance=someinstance + return someinstance +end +function resolvers.reset() + return resolvers.setinstance(resolvers.newinstance()) +end +local function reset_hashes() + instance.lists={} + instance.found={} +end +local slash=P("/") +local pathexpressionpattern=Cs ( + Cc("^")*( + Cc("%")*S(".-")+slash^2*P(-1)/"/.*" ++slash^2/"/"+(1-slash)*P(-1)*Cc("/")+P(1) + )^1*Cc("$") +) +local cache={} +local function makepathexpression(str) + if str=="." then + return "^%./$" + else + local c=cache[str] + if not c then + c=lpegmatch(pathexpressionpattern,str) + cache[str]=c + end + return c + end +end +local function reportcriticalvariables(cnfspec) + if trace_locating then + for i=1,#resolvers.criticalvars do + local k=resolvers.criticalvars[i] + local v=resolvers.getenv(k) or "unknown" + report_resolving("variable %a set to %a",k,v) + end + report_resolving() + if cnfspec then + report_resolving("using configuration specification %a",type(cnfspec)=="table" and concat(cnfspec,",") or cnfspec) + end + report_resolving() + end + reportcriticalvariables=function() end +end +local function identify_configuration_files() + local specification=instance.specification + if #specification==0 then + local cnfspec=getenv("TEXMFCNF") + if cnfspec=="" then + cnfspec=resolvers.luacnfspec + resolvers.luacnfstate="default" + else + resolvers.luacnfstate="environment" + end + reportcriticalvariables(cnfspec) + local cnfpaths=expandedpathfromlist(resolvers.splitpath(cnfspec)) + local luacnfname=resolvers.luacnfname + for i=1,#cnfpaths do + local filename=collapsepath(filejoin(cnfpaths[i],luacnfname)) + local realname=resolvers.resolve(filename) + if lfs.isfile(realname) then + specification[#specification+1]=filename + if trace_locating then + report_resolving("found configuration file %a",realname) + end + elseif trace_locating then + report_resolving("unknown configuration file %a",realname) + end + end + if trace_locating then + report_resolving() + end + elseif trace_locating then + report_resolving("configuration files already identified") + end +end +local function load_configuration_files() + local specification=instance.specification + if #specification>0 then + local luacnfname=resolvers.luacnfname + for i=1,#specification do + local filename=specification[i] + local pathname=filedirname(filename) + local filename=filejoin(pathname,luacnfname) + local realname=resolvers.resolve(filename) + local blob=loadfile(realname) + if blob then + local setups=instance.setups + local data=blob() + local parent=data and data.parent + if parent then + local filename=filejoin(pathname,parent) + local realname=resolvers.resolve(filename) + local blob=loadfile(realname) + if blob then + local parentdata=blob() + if parentdata then + report_resolving("loading configuration file %a",filename) + data=table.merged(parentdata,data) + end + end + end + data=data and data.content + if data then + if trace_locating then + report_resolving("loading configuration file %a",filename) + report_resolving() + end + local variables=data.variables or {} + local warning=false + for k,v in next,data do + local variant=type(v) + if variant=="table" then + initializesetter(filename,k,v) + elseif variables[k]==nil then + if trace_locating and not warning then + report_resolving("variables like %a in configuration file %a should move to the 'variables' subtable", + k,resolvers.resolve(filename)) + warning=true + end + variables[k]=v + end + end + setups[pathname]=variables + if resolvers.luacnfstate=="default" then + local cnfspec=variables["TEXMFCNF"] + if cnfspec then + if trace_locating then + report_resolving("reloading configuration due to TEXMF redefinition") + end + resolvers.setenv("TEXMFCNF",cnfspec) + instance.specification={} + identify_configuration_files() + load_configuration_files() + resolvers.luacnfstate="configuration" + break + end + end + else + if trace_locating then + report_resolving("skipping configuration file %a (no content)",filename) + end + setups[pathname]={} + instance.loaderror=true + end + elseif trace_locating then + report_resolving("skipping configuration file %a (no valid format)",filename) + end + instance.order[#instance.order+1]=instance.setups[pathname] + if instance.loaderror then + break + end + end + elseif trace_locating then + report_resolving("warning: no lua configuration files found") + end +end +local function load_file_databases() + instance.loaderror,instance.files=false,allocate() + if not instance.renewcache then + local hashes=instance.hashes + for k=1,#hashes do + local hash=hashes[k] + resolvers.hashers.byscheme(hash.type,hash.name) + if instance.loaderror then break end + end + end +end +local function locate_file_databases() + local texmfpaths=resolvers.expandedpathlist("TEXMF") + if #texmfpaths>0 then + for i=1,#texmfpaths do + local path=collapsepath(texmfpaths[i]) + path=gsub(path,"/+$","") + local stripped=lpegmatch(inhibitstripper,path) + if stripped~="" then + local runtime=stripped==path + path=resolvers.cleanpath(path) + local spec=resolvers.splitmethod(stripped) + if runtime and (spec.noscheme or spec.scheme=="file") then + stripped="tree:///"..stripped + elseif spec.scheme=="cache" or spec.scheme=="file" then + stripped=spec.path + end if trace_locating then - report_files("file locator '%s' found as '%s'",name,realname) + if runtime then + report_resolving("locating list of %a (runtime) (%s)",path,stripped) + else + report_resolving("locating list of %a (cached)",path) + end end - resolvers.appendhash('file',name,true) -- cache + methodhandler('locators',stripped) + end + end + if trace_locating then + report_resolving() + end + elseif trace_locating then + report_resolving("no texmf paths are defined (using TEXMF)") + end +end +local function generate_file_databases() + local hashes=instance.hashes + for k=1,#hashes do + local hash=hashes[k] + methodhandler('generators',hash.name) + end + if trace_locating then + report_resolving() + end +end +local function save_file_databases() + for i=1,#instance.hashes do + local hash=instance.hashes[i] + local cachename=hash.name + if hash.cache then + local content=instance.files[cachename] + caches.collapsecontent(content) + if trace_locating then + report_resolving("saving tree %a",cachename) + end + caches.savecontent(cachename,"files",content) elseif trace_locating then - report_files("file locator '%s' not found",name) + report_resolving("not saving runtime tree %a",cachename) + end + end +end +function resolvers.renew(hashname) + if hashname and hashname~="" then + local expanded=resolvers.expansion(hashname) or "" + if expanded~="" then + if trace_locating then + report_resolving("identifying tree %a from %a",expanded,hashname) + end + hashname=expanded + else + if trace_locating then + report_resolving("identifying tree %a",hashname) + end + end + local realpath=resolvers.resolve(hashname) + if lfs.isdir(realpath) then + if trace_locating then + report_resolving("using path %a",realpath) + end + methodhandler('generators',hashname) + local content=instance.files[hashname] + caches.collapsecontent(content) + if trace_locating then + report_resolving("saving tree %a",hashname) + end + caches.savecontent(hashname,"files",content) + else + report_resolving("invalid path %a",realpath) end + end end - -function hashers.file(specification) - local name = specification.filename - local content = caches.loadcontent(name,'files') - resolvers.registerfilehash(name,content,content==nil) +local function load_databases() + locate_file_databases() + if instance.diskcache and not instance.renewcache then + load_file_databases() + if instance.loaderror then + generate_file_databases() + save_file_databases() + end + else + generate_file_databases() + if instance.renewcache then + save_file_databases() + end + end end - -function generators.file(specification) - local path = specification.filename - local content = resolvers.scanfiles(path,false,true) -- scan once - resolvers.registerfilehash(path,content,true) +function resolvers.appendhash(type,name,cache) + if not instance.hashed[name] then + if trace_locating then + report_resolving("hash %a appended",name) + end + insert(instance.hashes,{ type=type,name=name,cache=cache } ) + instance.hashed[name]=cache + end end - -concatinators.file = file.join - -function finders.file(specification,filetype) - local filename = specification.filename - local foundname = resolvers.findfile(filename,filetype) - if foundname and foundname ~= "" then - if trace_locating then - report_files("file finder: '%s' found",filename) - end - return foundname +function resolvers.prependhash(type,name,cache) + if not instance.hashed[name] then + if trace_locating then + report_resolving("hash %a prepended",name) + end + insert(instance.hashes,1,{ type=type,name=name,cache=cache } ) + instance.hashed[name]=cache + end +end +function resolvers.extendtexmfvariable(specification) + local t=resolvers.splitpath(getenv("TEXMF")) + insert(t,1,specification) + local newspec=concat(t,",") + if instance.environment["TEXMF"] then + instance.environment["TEXMF"]=newspec + elseif instance.variables["TEXMF"] then + instance.variables["TEXMF"]=newspec + else + end + reset_hashes() +end +function resolvers.splitexpansions() + local ie=instance.expansions + for k,v in next,ie do + local t,tn,h,p={},0,{},splitconfigurationpath(v) + for kk=1,#p do + local vv=p[kk] + if vv~="" and not h[vv] then + tn=tn+1 + t[tn]=vv + h[vv]=true + end + end + if #t>1 then + ie[k]=t else - if trace_locating then - report_files("file finder: %s' not found",filename) - end - return finders.notfound() + ie[k]=t[1] end + end end - --- The default textopener will be overloaded later on. - -function openers.helpers.textopener(tag,filename,f) - return { - reader = function() return f:read () end, - close = function() logs.show_close(filename) return f:close() end, - } +function resolvers.datastate() + return caches.contentstate() end - -function openers.file(specification,filetype) - local filename = specification.filename - if filename and filename ~= "" then - local f = io.open(filename,"r") - if f then - if trace_locating then - report_files("file opener, '%s' opened",filename) - end - return openers.helpers.textopener("file",filename,f) +function resolvers.variable(name) + local name=name and lpegmatch(dollarstripper,name) + local result=name and instance.variables[name] + return result~=nil and result or "" +end +function resolvers.expansion(name) + local name=name and lpegmatch(dollarstripper,name) + local result=name and instance.expansions[name] + return result~=nil and result or "" +end +function resolvers.unexpandedpathlist(str) + local pth=resolvers.variable(str) + local lst=resolvers.splitpath(pth) + return expandedpathfromlist(lst) +end +function resolvers.unexpandedpath(str) + return joinpath(resolvers.unexpandedpathlist(str)) +end +local done={} +function resolvers.resetextrapath() + local ep=instance.extra_paths + if not ep then + ep,done={},{} + instance.extra_paths=ep + elseif #ep>0 then + instance.lists,done={},{} + end +end +function resolvers.registerextrapath(paths,subpaths) + paths=settings_to_array(paths) + subpaths=settings_to_array(subpaths) + local ep=instance.extra_paths or {} + local oldn=#ep + local newn=oldn + local nofpaths=#paths + local nofsubpaths=#subpaths + if nofpaths>0 then + if nofsubpaths>0 then + for i=1,nofpaths do + local p=paths[i] + for j=1,nofsubpaths do + local s=subpaths[j] + local ps=p.."/"..s + if not done[ps] then + newn=newn+1 + ep[newn]=resolvers.cleanpath(ps) + done[ps]=true + end + end + end + else + for i=1,nofpaths do + local p=paths[i] + if not done[p] then + newn=newn+1 + ep[newn]=resolvers.cleanpath(p) + done[p]=true + end + end + end + elseif nofsubpaths>0 then + for i=1,oldn do + for j=1,nofsubpaths do + local s=subpaths[j] + local ps=ep[i].."/"..s + if not done[ps] then + newn=newn+1 + ep[newn]=resolvers.cleanpath(ps) + done[ps]=true + end + end + end + end + if newn>0 then + instance.extra_paths=ep + end + if newn>oldn then + instance.lists={} + end +end +local function made_list(instance,list) + local ep=instance.extra_paths + if not ep or #ep==0 then + return list + else + local done,new,newn={},{},0 + for k=1,#list do + local v=list[k] + if not done[v] then + if find(v,"^[%.%/]$") then + done[v]=true + newn=newn+1 + new[newn]=v + else + break end + end end - if trace_locating then - report_files("file opener, '%s' not found",filename) + for k=1,#ep do + local v=ep[k] + if not done[v] then + done[v]=true + newn=newn+1 + new[newn]=v + end + end + for k=1,#list do + local v=list[k] + if not done[v] then + done[v]=true + newn=newn+1 + new[newn]=v + end end - return openers.notfound() + return new + end end - -function loaders.file(specification,filetype) - local filename = specification.filename - if filename and filename ~= "" then - local f = io.open(filename,"rb") - if f then - logs.show_load(filename) - if trace_locating then - report_files("file loader, '%s' loaded",filename) - end - local s = f:read("*a") - if checkgarbage then - checkgarbage(#s) - end - f:close() - if s then - return true, s, #s - end - end +function resolvers.cleanpathlist(str) + local t=resolvers.expandedpathlist(str) + if t then + for i=1,#t do + t[i]=collapsepath(resolvers.cleanpath(t[i])) end - if trace_locating then - report_files("file loader, '%s' not found",filename) + end + return t +end +function resolvers.expandpath(str) + return joinpath(resolvers.expandedpathlist(str)) +end +function resolvers.expandedpathlist(str) + if not str then + return {} + elseif instance.savelists then + str=lpegmatch(dollarstripper,str) + local lists=instance.lists + local lst=lists[str] + if not lst then + local l=made_list(instance,resolvers.splitpath(resolvers.expansion(str))) + lst=expandedpathfromlist(l) + lists[str]=lst end - return loaders.notfound() + return lst + else + local lst=resolvers.splitpath(resolvers.expansion(str)) + return made_list(instance,expandedpathfromlist(lst)) + end end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['data-con'] = { - version = 1.100, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local format, lower, gsub = string.format, string.lower, string.gsub - -local trace_cache = false trackers.register("resolvers.cache", function(v) trace_cache = v end) -local trace_containers = false trackers.register("resolvers.containers", function(v) trace_containers = v end) -local trace_storage = false trackers.register("resolvers.storage", function(v) trace_storage = v end) - ---[[ldx-- -

Once we found ourselves defining similar cache constructs -several times, containers were introduced. Containers are used -to collect tables in memory and reuse them when possible based -on (unique) hashes (to be provided by the calling function).

- -

Caching to disk is disabled by default. Version numbers are -stored in the saved table which makes it possible to change the -table structures without bothering about the disk cache.

- -

Examples of usage can be found in the font related code.

---ldx]]-- - -containers = containers or { } -local containers = containers -containers.usecache = true - -local report_containers = logs.reporter("resolvers","containers") - -local function report(container,tag,name) - if trace_cache or trace_containers then - report_containers("container: %s, tag: %s, name: %s",container.subcategory,tag,name or 'invalid') +function resolvers.expandedpathlistfromvariable(str) + str=lpegmatch(dollarstripper,str) + local tmp=resolvers.variableofformatorsuffix(str) + return resolvers.expandedpathlist(tmp~="" and tmp or str) +end +function resolvers.expandpathfromvariable(str) + return joinpath(resolvers.expandedpathlistfromvariable(str)) +end +function resolvers.expandbraces(str) + local ori=str + local pth=expandedpathfromlist(resolvers.splitpath(ori)) + return joinpath(pth) +end +function resolvers.registerfilehash(name,content,someerror) + if content then + instance.files[name]=content + else + instance.files[name]={} + if somerror==true then + instance.loaderror=someerror end + end end - -local allocated = { } - -local mt = { - __index = function(t,k) - if k == "writable" then - local writable = caches.getwritablepath(t.category,t.subcategory) or { "." } - t.writable = writable - return writable - elseif k == "readables" then - local readables = caches.getreadablepaths(t.category,t.subcategory) or { "." } - t.readables = readables - return readables - end - end, - __storage__ = true -} - -function containers.define(category, subcategory, version, enabled) - if category and subcategory then - local c = allocated[category] - if not c then - c = { } - allocated[category] = c - end - local s = c[subcategory] - if not s then - s = { - category = category, - subcategory = subcategory, - storage = { }, - enabled = enabled, - version = version or math.pi, -- after all, this is TeX - trace = false, - -- writable = caches.getwritablepath and caches.getwritablepath (category,subcategory) or { "." }, - -- readables = caches.getreadablepaths and caches.getreadablepaths(category,subcategory) or { "." }, - } - setmetatable(s,mt) - c[subcategory] = s +local function isreadable(name) + local readable=lfs.isfile(name) + if trace_detail then + if readable then + report_resolving("file %a is readable",name) + else + report_resolving("file %a is not readable",name) + end + end + return readable +end +local function collect_files(names) + local filelist,noffiles={},0 + for k=1,#names do + local fname=names[k] + if trace_detail then + report_resolving("checking name %a",fname) + end + local bname=filebasename(fname) + local dname=filedirname(fname) + if dname=="" or find(dname,"^%.") then + dname=false + else + dname=gsub(dname,"%*",".*") + dname="/"..dname.."$" + end + local hashes=instance.hashes + for h=1,#hashes do + local hash=hashes[h] + local blobpath=hash.name + local files=blobpath and instance.files[blobpath] + if files then + if trace_detail then + report_resolving("deep checking %a, base %a, pattern %a",blobpath,bname,dname) + end + local blobfile=files[bname] + if not blobfile then + local rname="remap:"..bname + blobfile=files[rname] + if blobfile then + bname=files[rname] + blobfile=files[bname] + end + end + if blobfile then + local blobroot=files.__path__ or blobpath + if type(blobfile)=='string' then + if not dname or find(blobfile,dname) then + local variant=hash.type + local search=filejoin(blobroot,blobfile,bname) + local result=methodhandler('concatinators',hash.type,blobroot,blobfile,bname) + if trace_detail then + report_resolving("match: variant %a, search %a, result %a",variant,search,result) + end + noffiles=noffiles+1 + filelist[noffiles]={ variant,search,result } + end + else + for kk=1,#blobfile do + local vv=blobfile[kk] + if not dname or find(vv,dname) then + local variant=hash.type + local search=filejoin(blobroot,vv,bname) + local result=methodhandler('concatinators',hash.type,blobroot,vv,bname) + if trace_detail then + report_resolving("match: variant %a, search %a, result %a",variant,search,result) + end + noffiles=noffiles+1 + filelist[noffiles]={ variant,search,result } + end + end + end end - return s + elseif trace_locating then + report_resolving("no match in %a (%s)",blobpath,bname) + end end + end + return noffiles>0 and filelist or nil end - -function containers.is_usable(container, name) - return container.enabled and caches and caches.is_writable(container.writable, name) +local fit={} +function resolvers.registerintrees(filename,format,filetype,usedmethod,foundname) + local foundintrees=instance.foundintrees + if usedmethod=="direct" and filename==foundname and fit[foundname] then + else + local t={ + filename=filename, + format=format~="" and format or nil, + filetype=filetype~="" and filetype or nil, + usedmethod=usedmethod, + foundname=foundname, + } + fit[foundname]=t + foundintrees[#foundintrees+1]=t + end end - -function containers.is_valid(container, name) - if name and name ~= "" then - local storage = container.storage[name] - return storage and storage.cache_version == container.version +local function can_be_dir(name) + local fakepaths=instance.fakepaths + if not fakepaths[name] then + if lfs.isdir(name) then + fakepaths[name]=1 else - return false + fakepaths[name]=2 end + end + return fakepaths[name]==1 end - -function containers.read(container,name) - local storage = container.storage - local stored = storage[name] - if not stored and container.enabled and caches and containers.usecache then - stored = caches.loaddata(container.readables,name) - if stored and stored.cache_version == container.version then - report(container,"loaded",name) - else - stored = nil +local preparetreepattern=Cs((P(".")/"%%."+P("-")/"%%-"+P(1))^0*Cc("$")) +local collect_instance_files +local function find_analyze(filename,askedformat,allresults) + local filetype,wantedfiles,ext='',{},suffixonly(filename) + wantedfiles[#wantedfiles+1]=filename + if askedformat=="" then + if ext=="" or not suffixmap[ext] then + local defaultsuffixes=resolvers.defaultsuffixes + for i=1,#defaultsuffixes do + local forcedname=filename..'.'..defaultsuffixes[i] + wantedfiles[#wantedfiles+1]=forcedname + filetype=resolvers.formatofsuffix(forcedname) + if trace_locating then + report_resolving("forcing filetype %a",filetype) end - storage[name] = stored - elseif stored then - report(container,"reusing",name) + end + else + filetype=resolvers.formatofsuffix(filename) + if trace_locating then + report_resolving("using suffix based filetype %a",filetype) + end end - return stored -end - -function containers.write(container, name, data) - if data then - data.cache_version = container.version - if container.enabled and caches then - local unique, shared = data.unique, data.shared - data.unique, data.shared = nil, nil - caches.savedata(container.writable, name, data) - report(container,"saved",name) - data.unique, data.shared = unique, shared + else + if ext=="" or not suffixmap[ext] then + local format_suffixes=suffixes[askedformat] + if format_suffixes then + for i=1,#format_suffixes do + wantedfiles[#wantedfiles+1]=filename.."."..format_suffixes[i] end - report(container,"stored",name) - container.storage[name] = data + end end - return data + filetype=askedformat + if trace_locating then + report_resolving("using given filetype %a",filetype) + end + end + return filetype,wantedfiles end - -function containers.content(container,name) - return container.storage[name] +local function find_direct(filename,allresults) + if not dangerous[askedformat] and isreadable(filename) then + if trace_detail then + report_resolving("file %a found directly",filename) + end + return "direct",{ filename } + end end - -function containers.cleanname(name) - return (gsub(lower(name),"[^%w%d]+","-")) +local function find_wildcard(filename,allresults) + if find(filename,'%*') then + if trace_locating then + report_resolving("checking wildcard %a",filename) + end + local method,result=resolvers.findwildcardfiles(filename) + if result then + return "wildcard",result + end + end +end +local function find_qualified(filename,allresults,askedformat,alsostripped) + if not file.is_qualified_path(filename) then + return + end + if trace_locating then + report_resolving("checking qualified name %a",filename) + end + if isreadable(filename) then + if trace_detail then + report_resolving("qualified file %a found",filename) + end + return "qualified",{ filename } + end + if trace_detail then + report_resolving("locating qualified file %a",filename) + end + local forcedname,suffix="",suffixonly(filename) + if suffix=="" then + local format_suffixes=askedformat=="" and resolvers.defaultsuffixes or suffixes[askedformat] + if format_suffixes then + for i=1,#format_suffixes do + local s=format_suffixes[i] + forcedname=filename.."."..s + if isreadable(forcedname) then + if trace_locating then + report_resolving("no suffix, forcing format filetype %a",s) + end + return "qualified",{ forcedname } + end + end + end + end + if alsostripped and suffix and suffix~="" then + local basename=filebasename(filename) + local pattern=lpegmatch(preparetreepattern,filename) + local savedformat=askedformat + local format=savedformat or "" + if format=="" then + askedformat=resolvers.formatofsuffix(suffix) + end + if not format then + askedformat="othertextfiles" + end + if basename~=filename then + local resolved=collect_instance_files(basename,askedformat,allresults) + if #resolved==0 then + local lowered=lower(basename) + if filename~=lowered then + resolved=collect_instance_files(lowered,askedformat,allresults) + end + end + resolvers.format=savedformat + if #resolved>0 then + local result={} + for r=1,#resolved do + local rr=resolved[r] + if find(rr,pattern) then + result[#result+1]=rr + end + end + if #result>0 then + return "qualified",result + end + end + end + end end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['data-use'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local format, lower, gsub, find = string.format, string.lower, string.gsub, string.find - -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) - -local report_mounts = logs.reporter("resolvers","mounts") - -local resolvers = resolvers - --- we will make a better format, maybe something xml or just text or lua - -resolvers.automounted = resolvers.automounted or { } - -function resolvers.automount(usecache) - local mountpaths = resolvers.cleanpathlist(resolvers.expansion('TEXMFMOUNT')) - if (not mountpaths or #mountpaths == 0) and usecache then - mountpaths = caches.getreadablepaths("mount") - end - if mountpaths and #mountpaths > 0 then - statistics.starttiming(resolvers.instance) - for k=1,#mountpaths do - local root = mountpaths[k] - local f = io.open(root.."/url.tmi") - if f then - for line in f:lines() do - if line then - if find(line,"^[%%#%-]") then -- or %W - -- skip - elseif find(line,"^zip://") then - if trace_locating then - report_mounts("mounting %s",line) - end - table.insert(resolvers.automounted,line) - resolvers.usezipfile(line) +local function check_subpath(fname) + if isreadable(fname) then + if trace_detail then + report_resolving("found %a by deep scanning",fname) + end + return fname + end +end +local function find_intree(filename,filetype,wantedfiles,allresults) + local typespec=resolvers.variableofformat(filetype) + local pathlist=resolvers.expandedpathlist(typespec) + local method="intree" + if pathlist and #pathlist>0 then + local filelist=collect_files(wantedfiles) + local dirlist={} + if filelist then + for i=1,#filelist do + dirlist[i]=filedirname(filelist[i][3]).."/" + end + end + if trace_detail then + report_resolving("checking filename %a",filename) + end + local resolve=resolvers.resolve + local result={} + for k=1,#pathlist do + local path=pathlist[k] + local pathname=lpegmatch(inhibitstripper,path) + local doscan=path==pathname + if not find (pathname,'//$') then + doscan=false + end + local done=false + if filelist then + local expression=makepathexpression(pathname) + if trace_detail then + report_resolving("using pattern %a for path %a",expression,pathname) + end + for k=1,#filelist do + local fl=filelist[k] + local f=fl[2] + local d=dirlist[k] + if find(d,expression) or find(resolve(d),expression) then + result[#result+1]=resolve(fl[3]) + done=true + if allresults then + if trace_detail then + report_resolving("match to %a in hash for file %a and path %a, continue scanning",expression,f,d) + end + else + if trace_detail then + report_resolving("match to %a in hash for file %a and path %a, quit scanning",expression,f,d) + end + break + end + elseif trace_detail then + report_resolving("no match to %a in hash for file %a and path %a",expression,f,d) + end + end + end + if done then + method="database" + else + method="filesystem" + pathname=gsub(pathname,"/+$","") + pathname=resolve(pathname) + local scheme=url.hasscheme(pathname) + if not scheme or scheme=="file" then + local pname=gsub(pathname,"%.%*$",'') + if not find(pname,"%*") then + if can_be_dir(pname) then + for k=1,#wantedfiles do + local w=wantedfiles[k] + local fname=check_subpath(filejoin(pname,w)) + if fname then + result[#result+1]=fname + done=true + if not allresults then + break + end + end + end + if not done and doscan then + local files=resolvers.simplescanfiles(pname,false,true) + for k=1,#wantedfiles do + local w=wantedfiles[k] + local subpath=files[w] + if not subpath or subpath=="" then + elseif type(subpath)=="string" then + local fname=check_subpath(filejoin(pname,subpath,w)) + if fname then + result[#result+1]=fname + done=true + if not allresults then + break + end + end + else + for i=1,#subpath do + local sp=subpath[i] + if sp=="" then + else + local fname=check_subpath(filejoin(pname,sp,w)) + if fname then + result[#result+1]=fname + done=true + if not allresults then + break + end end + end end + if done and not allresults then + break + end + end end - f:close() + end end + else + end end - statistics.stoptiming(resolvers.instance) + end + if done and not allresults then + break + end end -end - --- status info - -statistics.register("used config file", function() return caches.configfiles() end) -statistics.register("used cache path", function() return caches.usedpaths() end) - --- experiment (code will move) - -function statistics.savefmtstatus(texname,formatbanner,sourcefile) -- texname == formatname - local enginebanner = status.list().banner - if formatbanner and enginebanner and sourcefile then - local luvname = file.replacesuffix(texname,"luv") - local luvdata = { - enginebanner = enginebanner, - formatbanner = formatbanner, - sourcehash = md5.hex(io.loaddata(resolvers.findfile(sourcefile)) or "unknown"), - sourcefile = sourcefile, - } - io.savedata(luvname,table.serialize(luvdata,true)) + if #result>0 then + return method,result end + end end - -function statistics.checkfmtstatus(texname) - local enginebanner = status.list().banner - if enginebanner and texname then - local luvname = file.replacesuffix(texname,"luv") - if lfs.isfile(luvname) then - local luv = dofile(luvname) - if luv and luv.sourcefile then - local sourcehash = md5.hex(io.loaddata(resolvers.findfile(luv.sourcefile)) or "unknown") - local luvbanner = luv.enginebanner or "?" - if luvbanner ~= enginebanner then - return format("engine mismatch (luv: %s <> bin: %s)",luvbanner,enginebanner) - end - local luvhash = luv.sourcehash or "?" - if luvhash ~= sourcehash then - return format("source mismatch (luv: %s <> bin: %s)",luvhash,sourcehash) - end - else - return "invalid status file" - end - else - return "missing status file" +local function find_onpath(filename,filetype,wantedfiles,allresults) + if trace_detail then + report_resolving("checking filename %a, filetype %a, wanted files %a",filename,filetype,concat(wantedfiles," | ")) + end + local result={} + for k=1,#wantedfiles do + local fname=wantedfiles[k] + if fname and isreadable(fname) then + filename=fname + result[#result+1]=filejoin('.',fname) + if not allresults then + break + end + end + end + if #result>0 then + return "onpath",result + end +end +local function find_otherwise(filename,filetype,wantedfiles,allresults) + local filelist=collect_files(wantedfiles) + local fl=filelist and filelist[1] + if fl then + return "otherwise",{ resolvers.resolve(fl[3]) } + end +end +collect_instance_files=function(filename,askedformat,allresults) + askedformat=askedformat or "" + filename=collapsepath(filename) + if allresults then + local filetype,wantedfiles=find_analyze(filename,askedformat) + local results={ + { find_direct (filename,true) }, + { find_wildcard (filename,true) }, + { find_qualified(filename,true,askedformat) }, + { find_intree (filename,filetype,wantedfiles,true) }, + { find_onpath (filename,filetype,wantedfiles,true) }, + { find_otherwise(filename,filetype,wantedfiles,true) }, + } + local result,status,done={},{},{} + for k,r in next,results do + local method,list=r[1],r[2] + if method and list then + for i=1,#list do + local c=collapsepath(list[i]) + if not done[c] then + result[#result+1]=c + done[c]=true + end + status[#status+1]=formatters["%-10s: %s"](method,c) + end + end + end + if trace_detail then + report_resolving("lookup status: %s",table.serialize(status,filename)) + end + return result,status + else + local method,result,stamp,filetype,wantedfiles + if instance.remember then + stamp=formatters["%s--%s"](filename,askedformat) + result=stamp and instance.found[stamp] + if result then + if trace_locating then + report_resolving("remembered file %a",filename) end + return result + end end - return true -end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['data-zip'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- partly redone .. needs testing - -local format, find, match = string.format, string.find, string.match - -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) - -local report_zip = logs.reporter("resolvers","zip") - --- zip:///oeps.zip?name=bla/bla.tex --- zip:///oeps.zip?tree=tex/texmf-local --- zip:///texmf.zip?tree=/tex/texmf --- zip:///texmf.zip?tree=/tex/texmf-local --- zip:///texmf-mine.zip?tree=/tex/texmf-projects - -local resolvers = resolvers - -zip = zip or { } -local zip = zip - -zip.archives = zip.archives or { } -local archives = zip.archives - -zip.registeredfiles = zip.registeredfiles or { } -local registeredfiles = zip.registeredfiles - -local limited = false - -directives.register("system.inputmode", function(v) - if not limited then - local i_limiter = io.i_limiter(v) - if i_limiter then - zip.open = i_limiter.protect(zip.open) - limited = true + method,result=find_direct(filename) + if not result then + method,result=find_wildcard(filename) + if not result then + method,result=find_qualified(filename,false,askedformat) + if not result then + filetype,wantedfiles=find_analyze(filename,askedformat) + method,result=find_intree(filename,filetype,wantedfiles) + if not result then + method,result=find_onpath(filename,filetype,wantedfiles) + if not result then + method,result=find_otherwise(filename,filetype,wantedfiles) + end + end end + end end -end) - -local function validzip(str) -- todo: use url splitter - if not find(str,"^zip://") then - return "zip:///" .. str + if result and #result>0 then + local foundname=collapsepath(result[1]) + resolvers.registerintrees(filename,askedformat,filetype,method,foundname) + result={ foundname } else - return str + result={} end -end - -function zip.openarchive(name) - if not name or name == "" then - return nil - else - local arch = archives[name] - if not arch then - local full = resolvers.findfile(name) or "" - arch = (full ~= "" and zip.open(full)) or false - archives[name] = arch - end - return arch + if stamp then + if trace_locating then + report_resolving("remembering file %a",filename) + end + instance.found[stamp]=result end + return result + end end - -function zip.closearchive(name) - if not name or (name == "" and archives[name]) then - zip.close(archives[name]) - archives[name] = nil +local function findfiles(filename,filetype,allresults) + local result,status=collect_instance_files(filename,filetype or "",allresults) + if not result or #result==0 then + local lowered=lower(filename) + if filename~=lowered then + result,status=collect_instance_files(lowered,filetype or "",allresults) end + end + return result or {},status end - -function resolvers.locators.zip(specification) - local archive = specification.filename - local zipfile = archive and archive ~= "" and zip.openarchive(archive) -- tricky, could be in to be initialized tree - if trace_locating then - if zipfile then - report_zip("locator, archive '%s' found",archive) - else - report_zip("locator, archive '%s' not found",archive) - end - end +function resolvers.findfiles(filename,filetype) + return findfiles(filename,filetype,true) end - -function resolvers.hashers.zip(specification) - local archive = specification.filename - if trace_locating then - report_zip("loading file '%s'",archive) +function resolvers.findfile(filename,filetype) + return findfiles(filename,filetype,false)[1] or "" +end +function resolvers.findpath(filename,filetype) + return filedirname(findfiles(filename,filetype,false)[1] or "") +end +local function findgivenfiles(filename,allresults) + local bname,result=filebasename(filename),{} + local hashes=instance.hashes + local noffound=0 + for k=1,#hashes do + local hash=hashes[k] + local files=instance.files[hash.name] or {} + local blist=files[bname] + if not blist then + local rname="remap:"..bname + blist=files[rname] + if blist then + bname=files[rname] + blist=files[bname] + end + end + if blist then + if type(blist)=='string' then + local found=methodhandler('concatinators',hash.type,hash.name,blist,bname) or "" + if found~="" then + noffound=noffound+1 + result[noffound]=resolvers.resolve(found) + if not allresults then + break + end + end + else + for kk=1,#blist do + local vv=blist[kk] + local found=methodhandler('concatinators',hash.type,hash.name,vv,bname) or "" + if found~="" then + noffound=noffound+1 + result[noffound]=resolvers.resolve(found) + if not allresults then break end + end + end + end end - resolvers.usezipfile(specification.original) + end + return result end - -function resolvers.concatinators.zip(zipfile,path,name) -- ok ? - if not path or path == "" then - return format('%s?name=%s',zipfile,name) - else - return format('%s?name=%s/%s',zipfile,path,name) - end +function resolvers.findgivenfiles(filename) + return findgivenfiles(filename,true) end - -function resolvers.finders.zip(specification) - local original = specification.original - local archive = specification.filename - if archive then - local query = url.query(specification.query) - local queryname = query.name - if queryname then - local zfile = zip.openarchive(archive) - if zfile then - if trace_locating then - report_zip("finder, archive '%s' found",archive) - end - local dfile = zfile:open(queryname) - if dfile then - dfile = zfile:close() - if trace_locating then - report_zip("finder, file '%s' found",queryname) - end - return specification.original - elseif trace_locating then - report_zip("finder, file '%s' not found",queryname) - end - elseif trace_locating then - report_zip("finder, unknown archive '%s'",archive) - end +function resolvers.findgivenfile(filename) + return findgivenfiles(filename,false)[1] or "" +end +local function doit(path,blist,bname,tag,variant,result,allresults) + local done=false + if blist and variant then + local resolve=resolvers.resolve + if type(blist)=='string' then + if find(lower(blist),path) then + local full=methodhandler('concatinators',variant,tag,blist,bname) or "" + result[#result+1]=resolve(full) + done=true + end + else + for kk=1,#blist do + local vv=blist[kk] + if find(lower(vv),path) then + local full=methodhandler('concatinators',variant,tag,vv,bname) or "" + result[#result+1]=resolve(full) + done=true + if not allresults then break end end + end end - if trace_locating then - report_zip("finder, '%s' not found",original) - end - return resolvers.finders.notfound() + end + return done end - -function resolvers.openers.zip(specification) - local original = specification.original - local archive = specification.filename - if archive then - local query = url.query(specification.query) - local queryname = query.name - if queryname then - local zfile = zip.openarchive(archive) - if zfile then - if trace_locating then - report_zip("opener, archive '%s' opened",archive) - end - local dfile = zfile:open(queryname) - if dfile then - if trace_locating then - report_zip("opener, file '%s' found",queryname) - end - return resolvers.openers.helpers.textopener('zip',original,dfile) - elseif trace_locating then - report_zip("opener, file '%s' not found",queryname) - end - elseif trace_locating then - report_zip("opener, unknown archive '%s'",archive) - end +local makewildcard=Cs( + (P("^")^0*P("/")*P(-1)+P(-1))/".*"+(P("^")^0*P("/")/"")^0*(P("*")/".*"+P("-")/"%%-"+P(".")/"%%."+P("?")/"."+P("\\")/"/"+P(1))^0 +) +function resolvers.wildcardpattern(pattern) + return lpegmatch(makewildcard,pattern) or pattern +end +local function findwildcardfiles(filename,allresults,result) + result=result or {} + local base=filebasename(filename) + local dirn=filedirname(filename) + local path=lower(lpegmatch(makewildcard,dirn) or dirn) + local name=lower(lpegmatch(makewildcard,base) or base) + local files,done=instance.files,false + if find(name,"%*") then + local hashes=instance.hashes + for k=1,#hashes do + local hash=hashes[k] + local hashname,hashtype=hash.name,hash.type + for kk,hh in next,files[hashname] do + if not find(kk,"^remap:") then + if find(lower(kk),name) then + if doit(path,hh,kk,hashname,hashtype,result,allresults) then done=true end + if done and not allresults then break end + end end + end end - if trace_locating then - report_zip("opener, '%s' not found",original) + else + local hashes=instance.hashes + for k=1,#hashes do + local hash=hashes[k] + local hashname,hashtype=hash.name,hash.type + if doit(path,files[hashname][bname],bname,hashname,hashtype,result,allresults) then done=true end + if done and not allresults then break end end - return resolvers.openers.notfound() + end + return result end - -function resolvers.loaders.zip(specification) - local original = specification.original - local archive = specification.filename - if archive then - local query = url.query(specification.query) - local queryname = query.name - if queryname then - local zfile = zip.openarchive(archive) - if zfile then - if trace_locating then - report_zip("loader, archive '%s' opened",archive) - end - local dfile = zfile:open(queryname) - if dfile then - logs.show_load(original) - if trace_locating then - report_zip("loader, file '%s' loaded",original) - end - local s = dfile:read("*all") - dfile:close() - return true, s, #s - elseif trace_locating then - report_zip("loader, file '%s' not found",queryname) - end - elseif trace_locating then - report_zip("loader, unknown archive '%s'",archive) - end - end - end +function resolvers.findwildcardfiles(filename,result) + return findwildcardfiles(filename,true,result) +end +function resolvers.findwildcardfile(filename) + return findwildcardfiles(filename,false)[1] or "" +end +function resolvers.automount() +end +function resolvers.load(option) + statistics.starttiming(instance) + identify_configuration_files() + load_configuration_files() + if option~="nofiles" then + load_databases() + resolvers.automount() + end + statistics.stoptiming(instance) + local files=instance.files + return files and next(files) and true +end +function resolvers.loadtime() + return statistics.elapsedtime(instance) +end +local function report(str) + if trace_locating then + report_resolving(str) + else + print(str) + end +end +function resolvers.dowithfilesandreport(command,files,...) + if files and #files>0 then if trace_locating then - report_zip("loader, '%s' not found",original) + report('') end - return resolvers.openers.notfound() -end - --- zip:///somefile.zip --- zip:///somefile.zip?tree=texmf-local -> mount - -function resolvers.usezipfile(archive) - local specification = resolvers.splitmethod(archive) -- to be sure - local archive = specification.filename - if archive and not registeredfiles[archive] then - local z = zip.openarchive(archive) - if z then - local instance = resolvers.instance - local tree = url.query(specification.query).tree or "" - if trace_locating then - report_zip("registering, registering archive '%s'",archive) - end - statistics.starttiming(instance) - resolvers.prependhash('zip',archive) - resolvers.extendtexmfvariable(archive) -- resets hashes too - registeredfiles[archive] = z - instance.files[archive] = resolvers.registerzipfile(z,tree) - statistics.stoptiming(instance) - elseif trace_locating then - report_zip("registering, unknown archive '%s'",archive) + if type(files)=="string" then + files={ files } + end + for f=1,#files do + local file=files[f] + local result=command(file,...) + if type(result)=='string' then + report(result) + else + for i=1,#result do + report(result[i]) end - elseif trace_locating then - report_zip("registering, '%s' not found",archive) + end end + end end - -function resolvers.registerzipfile(z,tree) - local files, filter = { }, "" - if tree == "" then - filter = "^(.+)/(.-)$" +function resolvers.showpath(str) + return joinpath(resolvers.expandedpathlist(resolvers.formatofvariable(str))) +end +function resolvers.registerfile(files,name,path) + if files[name] then + if type(files[name])=='string' then + files[name]={ files[name],path } else - filter = format("^%s/(.+)/(.-)$",tree) + files[name]=path end - if trace_locating then - report_zip("registering, using filter '%s'",filter) - end - local register, n = resolvers.registerfile, 0 - for i in z:files() do - local path, name = match(i.filename,filter) - if path then - if name and name ~= '' then - register(files, name, path) - n = n + 1 + else + files[name]=path + end +end +function resolvers.dowithpath(name,func) + local pathlist=resolvers.expandedpathlist(name) + for i=1,#pathlist do + func("^"..resolvers.cleanpath(pathlist[i])) + end +end +function resolvers.dowithvariable(name,func) + func(expandedvariable(name)) +end +function resolvers.locateformat(name) + local engine=environment.ownmain or "luatex" + local barename=file.removesuffix(name) + local fullname=file.addsuffix(barename,"fmt") + local fmtname=caches.getfirstreadablefile(fullname,"formats",engine) or "" + if fmtname=="" then + fmtname=resolvers.findfile(fullname) + fmtname=resolvers.cleanpath(fmtname) + end + if fmtname~="" then + local barename=file.removesuffix(fmtname) + local luaname=file.addsuffix(barename,luasuffixes.lua) + local lucname=file.addsuffix(barename,luasuffixes.luc) + local luiname=file.addsuffix(barename,luasuffixes.lui) + if lfs.isfile(luiname) then + return barename,luiname + elseif lfs.isfile(lucname) then + return barename,lucname + elseif lfs.isfile(luaname) then + return barename,luaname + end + end + return nil,nil +end +function resolvers.booleanvariable(str,default) + local b=resolvers.expansion(str) + if b=="" then + return default + else + b=toboolean(b) + return (b==nil and default) or b + end +end +function resolvers.dowithfilesintree(pattern,handle,before,after) + local instance=resolvers.instance + local hashes=instance.hashes + for i=1,#hashes do + local hash=hashes[i] + local blobtype=hash.type + local blobpath=hash.name + if blobpath then + if before then + before(blobtype,blobpath,pattern) + end + local files=instance.files[blobpath] + local total,checked,done=0,0,0 + if files then + for k,v in table.sortedhash(files) do + total=total+1 + if find(k,"^remap:") then + elseif find(k,pattern) then + if type(v)=="string" then + checked=checked+1 + if handle(blobtype,blobpath,v,k) then + done=done+1 + end else - -- directory + checked=checked+#v + for i=1,#v do + if handle(blobtype,blobpath,v[i],k) then + done=done+1 + end + end end - else - register(files, i.filename, '') - n = n + 1 + end end + end + if after then + after(blobtype,blobpath,pattern,total,checked,done) + end end - report_zip("registering, %s files registered",n) - return files + end end +resolvers.obsolete=resolvers.obsolete or {} +local obsolete=resolvers.obsolete +resolvers.find_file=resolvers.findfile obsolete.find_file=resolvers.findfile +resolvers.find_files=resolvers.findfiles obsolete.find_files=resolvers.findfiles end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-tre'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- \input tree://oeps1/**/oeps.tex - -local find, gsub, format = string.find, string.gsub, string.format +package.loaded["data-pre"] = package.loaded["data-pre"] or true -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) +-- original size: 6430, stripped down to: 4219 -local report_trees = logs.reporter("resolvers","trees") - -local resolvers = resolvers - -local done, found, notfound = { }, { }, resolvers.finders.notfound - -function resolvers.finders.tree(specification) - local spec = specification.filename - local fnd = found[spec] - if fnd == nil then - if spec ~= "" then - local path, name = file.dirname(spec), file.basename(spec) - if path == "" then path = "." end - local hash = done[path] - if not hash then - local pattern = path .. "/*" -- we will use the proper splitter - hash = dir.glob(pattern) - done[path] = hash - end - local pattern = "/" .. gsub(name,"([%.%-%+])", "%%%1") .. "$" - for k=1,#hash do - local v = hash[k] - if find(v,pattern) then - found[spec] = v - return v - end - end - end - fnd = notfound() -- false - found[spec] = fnd +if not modules then modules={} end modules ['data-pre']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local resolvers=resolvers +local prefixes=utilities.storage.allocate() +resolvers.prefixes=prefixes +local cleanpath,findgivenfile,expansion=resolvers.cleanpath,resolvers.findgivenfile,resolvers.expansion +local getenv=resolvers.getenv +local P,S,R,C,Cs,lpegmatch=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.Cs,lpeg.match +local joinpath,basename,dirname=file.join,file.basename,file.dirname +local getmetatable,rawset,type=getmetatable,rawset,type +prefixes.environment=function(str) + return cleanpath(expansion(str)) +end +prefixes.relative=function(str,n) + if io.exists(str) then + elseif io.exists("./"..str) then + str="./"..str + else + local p="../" + for i=1,n or 2 do + if io.exists(p..str) then + str=p..str + break + else + p=p.."../" + end + end + end + return cleanpath(str) +end +prefixes.auto=function(str) + local fullname=prefixes.relative(str) + if not lfs.isfile(fullname) then + fullname=prefixes.locate(str) + end + return fullname +end +prefixes.locate=function(str) + local fullname=findgivenfile(str) or "" + return cleanpath((fullname~="" and fullname) or str) +end +prefixes.filename=function(str) + local fullname=findgivenfile(str) or "" + return cleanpath(basename((fullname~="" and fullname) or str)) +end +prefixes.pathname=function(str) + local fullname=findgivenfile(str) or "" + return cleanpath(dirname((fullname~="" and fullname) or str)) +end +prefixes.selfautoloc=function(str) + return cleanpath(joinpath(getenv('SELFAUTOLOC'),str)) +end +prefixes.selfautoparent=function(str) + return cleanpath(joinpath(getenv('SELFAUTOPARENT'),str)) +end +prefixes.selfautodir=function(str) + return cleanpath(joinpath(getenv('SELFAUTODIR'),str)) +end +prefixes.home=function(str) + return cleanpath(joinpath(getenv('HOME'),str)) +end +local function toppath() + local inputstack=resolvers.inputstack + if not inputstack then + return "." + end + local pathname=dirname(inputstack[#inputstack] or "") + if pathname=="" then + return "." + else + return pathname + end +end +resolvers.toppath=toppath +prefixes.toppath=function(str) + return cleanpath(joinpath(toppath(),str)) +end +prefixes.env=prefixes.environment +prefixes.rel=prefixes.relative +prefixes.loc=prefixes.locate +prefixes.kpse=prefixes.locate +prefixes.full=prefixes.locate +prefixes.file=prefixes.filename +prefixes.path=prefixes.pathname +function resolvers.allprefixes(separator) + local all=table.sortedkeys(prefixes) + if separator then + for i=1,#all do + all[i]=all[i]..":" end - return fnd + end + return all end - -function resolvers.locators.tree(specification) - local name = specification.filename - local realname = resolvers.resolve(name) -- no shortcut - if realname and realname ~= '' and lfs.isdir(realname) then - if trace_locating then - report_trees("locator '%s' found",realname) - end - resolvers.appendhash('tree',name,false) -- don't cache - elseif trace_locating then - report_trees("locator '%s' not found",name) +local function _resolve_(method,target) + local action=prefixes[method] + if action then + return action(target) + else + return method..":"..target + end +end +local resolved,abstract={},{} +function resolvers.resetresolve(str) + resolved,abstract={},{} +end +local pattern=Cs((C(R("az")^2)*P(":")*C((1-S(" \"\';,"))^1)/_resolve_+P(1))^0) +local function resolve(str) + if type(str)=="table" then + local t={} + for i=1,#str do + t[i]=resolve(str[i]) + end + return t + else + local res=resolved[str] + if not res then + res=lpegmatch(pattern,str) + resolved[str]=res + abstract[res]=str end + return res + end end - -function resolvers.hashers.tree(specification) - local name = specification.filename - if trace_locating then - report_trees("analysing '%s'",name) +local function unresolve(str) + return abstract[str] or str +end +resolvers.resolve=resolve +resolvers.unresolve=unresolve +if type(os.uname)=="function" then + for k,v in next,os.uname() do + if not prefixes[k] then + prefixes[k]=function() return v end end - resolvers.methodhandler("hashers",name) - - resolvers.generators.file(specification) + end +end +if os.type=="unix" then + local pattern + local function makepattern(t,k,v) + if t then + rawset(t,k,v) + end + local colon=P(":") + for k,v in table.sortedpairs(prefixes) do + if p then + p=P(k)+p + else + p=P(k) + end + end + pattern=Cs((p*colon+colon/";"+P(1))^0) + end + makepattern() + getmetatable(prefixes).__newindex=makepattern + function resolvers.repath(str) + return lpegmatch(pattern,str) + end +else + function resolvers.repath(str) + return str + end end - -resolvers.concatinators.tree = resolvers.concatinators.file -resolvers.generators.tree = resolvers.generators.file -resolvers.openers.tree = resolvers.openers.file -resolvers.loaders.tree = resolvers.loaders.file end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-crl'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} +package.loaded["data-inp"] = package.loaded["data-inp"] or true --- this one is replaced by data-sch.lua -- +-- original size: 910, stripped down to: 823 -local gsub = string.gsub +if not modules then modules={} end modules ['data-inp']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local allocate=utilities.storage.allocate +local resolvers=resolvers +local methodhandler=resolvers.methodhandler +local registermethod=resolvers.registermethod +local finders=allocate { helpers={},notfound=function() end } +local openers=allocate { helpers={},notfound=function() end } +local loaders=allocate { helpers={},notfound=function() return false,nil,0 end } +registermethod("finders",finders,"uri") +registermethod("openers",openers,"uri") +registermethod("loaders",loaders,"uri") +resolvers.finders=finders +resolvers.openers=openers +resolvers.loaders=loaders -local resolvers = resolvers -local finders, openers, loaders = resolvers.finders, resolvers.openers, resolvers.loaders +end -- of closure -resolvers.curl = resolvers.curl or { } -local curl = resolvers.curl +do -- create closure to overcome 200 locals limit -local cached = { } +package.loaded["data-out"] = package.loaded["data-out"] or true -local function runcurl(specification) - local original = specification.original - -- local scheme = specification.scheme - local cleanname = gsub(original,"[^%a%d%.]+","-") - local cachename = caches.setfirstwritablefile(cleanname,"curl") - if not cached[original] then - if not io.exists(cachename) then - cached[original] = cachename - local command = "curl --silent --create-dirs --output " .. cachename .. " " .. original - os.spawn(command) - end - if io.exists(cachename) then - cached[original] = cachename - else - cached[original] = "" - end - end - return cached[original] -end +-- original size: 530, stripped down to: 475 --- old code: we could be cleaner using specification (see schemes) +if not modules then modules={} end modules ['data-out']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local allocate=utilities.storage.allocate +local resolvers=resolvers +local registermethod=resolvers.registermethod +local savers=allocate { helpers={} } +resolvers.savers=savers +registermethod("savers",savers,"uri") -local function finder(specification,filetype) - return resolvers.methodhandler("finders",runcurl(specification),filetype) -end -local opener = openers.file -local loader = loaders.file +end -- of closure + +do -- create closure to overcome 200 locals limit -local function install(scheme) - finders[scheme] = finder - openers[scheme] = opener - loaders[scheme] = loader -end +package.loaded["data-fil"] = package.loaded["data-fil"] or true -resolvers.curl.install = install +-- original size: 3801, stripped down to: 3231 -install('http') -install('https') -install('ftp') +if not modules then modules={} end modules ['data-fil']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) +local report_files=logs.reporter("resolvers","files") +local resolvers=resolvers +local finders,openers,loaders,savers=resolvers.finders,resolvers.openers,resolvers.loaders,resolvers.savers +local locators,hashers,generators,concatinators=resolvers.locators,resolvers.hashers,resolvers.generators,resolvers.concatinators +local checkgarbage=utilities.garbagecollector and utilities.garbagecollector.check +function locators.file(specification) + local name=specification.filename + local realname=resolvers.resolve(name) + if realname and realname~='' and lfs.isdir(realname) then + if trace_locating then + report_files("file locator %a found as %a",name,realname) + end + resolvers.appendhash('file',name,true) + elseif trace_locating then + report_files("file locator %a not found",name) + end +end +function hashers.file(specification) + local name=specification.filename + local content=caches.loadcontent(name,'files') + resolvers.registerfilehash(name,content,content==nil) +end +function generators.file(specification) + local path=specification.filename + local content=resolvers.scanfiles(path,false,true) + resolvers.registerfilehash(path,content,true) +end +concatinators.file=file.join +function finders.file(specification,filetype) + local filename=specification.filename + local foundname=resolvers.findfile(filename,filetype) + if foundname and foundname~="" then + if trace_locating then + report_files("file finder: %a found",filename) + end + return foundname + else + if trace_locating then + report_files("file finder: %a not found",filename) + end + return finders.notfound() + end +end +function openers.helpers.textopener(tag,filename,f) + return { + reader=function() return f:read () end, + close=function() logs.show_close(filename) return f:close() end, + } +end +function openers.file(specification,filetype) + local filename=specification.filename + if filename and filename~="" then + local f=io.open(filename,"r") + if f then + if trace_locating then + report_files("file opener: %a opened",filename) + end + return openers.helpers.textopener("file",filename,f) + end + end + if trace_locating then + report_files("file opener: %a not found",filename) + end + return openers.notfound() +end +function loaders.file(specification,filetype) + local filename=specification.filename + if filename and filename~="" then + local f=io.open(filename,"rb") + if f then + logs.show_load(filename) + if trace_locating then + report_files("file loader: %a loaded",filename) + end + local s=f:read("*a") + if checkgarbage then + checkgarbage(#s) + end + f:close() + if s then + return true,s,#s + end + end + end + if trace_locating then + report_files("file loader: %a not found",filename) + end + return loaders.notfound() +end end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-lua'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} +package.loaded["data-con"] = package.loaded["data-con"] or true --- some loading stuff ... we might move this one to slot 2 depending --- on the developments (the loaders must not trigger kpse); we could --- of course use a more extensive lib path spec +-- original size: 4940, stripped down to: 3580 -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) +if not modules then modules={} end modules ['data-con']={ + version=1.100, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local format,lower,gsub=string.format,string.lower,string.gsub +local trace_cache=false trackers.register("resolvers.cache",function(v) trace_cache=v end) +local trace_containers=false trackers.register("resolvers.containers",function(v) trace_containers=v end) +local trace_storage=false trackers.register("resolvers.storage",function(v) trace_storage=v end) +containers=containers or {} +local containers=containers +containers.usecache=true +local report_containers=logs.reporter("resolvers","containers") +local allocated={} +local mt={ + __index=function(t,k) + if k=="writable" then + local writable=caches.getwritablepath(t.category,t.subcategory) or { "." } + t.writable=writable + return writable + elseif k=="readables" then + local readables=caches.getreadablepaths(t.category,t.subcategory) or { "." } + t.readables=readables + return readables + end + end, + __storage__=true +} +function containers.define(category,subcategory,version,enabled) + if category and subcategory then + local c=allocated[category] + if not c then + c={} + allocated[category]=c + end + local s=c[subcategory] + if not s then + s={ + category=category, + subcategory=subcategory, + storage={}, + enabled=enabled, + version=version or math.pi, + trace=false, + } + setmetatable(s,mt) + c[subcategory]=s + end + return s + end +end +function containers.is_usable(container,name) + return container.enabled and caches and caches.is_writable(container.writable,name) +end +function containers.is_valid(container,name) + if name and name~="" then + local storage=container.storage[name] + return storage and storage.cache_version==container.version + else + return false + end +end +function containers.read(container,name) + local storage=container.storage + local stored=storage[name] + if not stored and container.enabled and caches and containers.usecache then + stored=caches.loaddata(container.readables,name) + if stored and stored.cache_version==container.version then + if trace_cache or trace_containers then + report_containers("action %a, category %a, name %a","load",container.subcategory,name) + end + else + stored=nil + end + storage[name]=stored + elseif stored then + if trace_cache or trace_containers then + report_containers("action %a, category %a, name %a","reuse",container.subcategory,name) + end + end + return stored +end +function containers.write(container,name,data) + if data then + data.cache_version=container.version + if container.enabled and caches then + local unique,shared=data.unique,data.shared + data.unique,data.shared=nil,nil + caches.savedata(container.writable,name,data) + if trace_cache or trace_containers then + report_containers("action %a, category %a, name %a","save",container.subcategory,name) + end + data.unique,data.shared=unique,shared + end + if trace_cache or trace_containers then + report_containers("action %a, category %a, name %a","store",container.subcategory,name) + end + container.storage[name]=data + end + return data +end +function containers.content(container,name) + return container.storage[name] +end +function containers.cleanname(name) + return (gsub(lower(name),"[^%w%d]+","-")) +end -local report_libraries = logs.reporter("resolvers","libraries") -local gsub, insert = string.gsub, table.insert -local unpack = unpack or table.unpack +end -- of closure -local resolvers, package = resolvers, package +do -- create closure to overcome 200 locals limit -local libformats = { 'luatexlibs', 'tex', 'texmfscripts', 'othertextfiles' } -- 'luainputs' -local clibformats = { 'lib' } +package.loaded["data-use"] = package.loaded["data-use"] or true -local _path_, libpaths, _cpath_, clibpaths +-- original size: 3913, stripped down to: 2998 -function package.libpaths() - if not _path_ or package.path ~= _path_ then - _path_ = package.path - libpaths = file.splitpath(_path_,";") - end - return libpaths +if not modules then modules={} end modules ['data-use']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local format,lower,gsub,find=string.format,string.lower,string.gsub,string.find +local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) +local report_mounts=logs.reporter("resolvers","mounts") +local resolvers=resolvers +resolvers.automounted=resolvers.automounted or {} +function resolvers.automount(usecache) + local mountpaths=resolvers.cleanpathlist(resolvers.expansion('TEXMFMOUNT')) + if (not mountpaths or #mountpaths==0) and usecache then + mountpaths=caches.getreadablepaths("mount") + end + if mountpaths and #mountpaths>0 then + statistics.starttiming(resolvers.instance) + for k=1,#mountpaths do + local root=mountpaths[k] + local f=io.open(root.."/url.tmi") + if f then + for line in f:lines() do + if line then + if find(line,"^[%%#%-]") then + elseif find(line,"^zip://") then + if trace_locating then + report_mounts("mounting %a",line) + end + table.insert(resolvers.automounted,line) + resolvers.usezipfile(line) + end + end + end + f:close() + end + end + statistics.stoptiming(resolvers.instance) + end +end +statistics.register("used config file",function() return caches.configfiles() end) +statistics.register("used cache path",function() return caches.usedpaths() end) +function statistics.savefmtstatus(texname,formatbanner,sourcefile) + local enginebanner=status.list().banner + if formatbanner and enginebanner and sourcefile then + local luvname=file.replacesuffix(texname,"luv") + local luvdata={ + enginebanner=enginebanner, + formatbanner=formatbanner, + sourcehash=md5.hex(io.loaddata(resolvers.findfile(sourcefile)) or "unknown"), + sourcefile=sourcefile, + } + io.savedata(luvname,table.serialize(luvdata,true)) + end end - -function package.clibpaths() - if not _cpath_ or package.cpath ~= _cpath_ then - _cpath_ = package.cpath - clibpaths = file.splitpath(_cpath_,";") +function statistics.checkfmtstatus(texname) + local enginebanner=status.list().banner + if enginebanner and texname then + local luvname=file.replacesuffix(texname,"luv") + if lfs.isfile(luvname) then + local luv=dofile(luvname) + if luv and luv.sourcefile then + local sourcehash=md5.hex(io.loaddata(resolvers.findfile(luv.sourcefile)) or "unknown") + local luvbanner=luv.enginebanner or "?" + if luvbanner~=enginebanner then + return format("engine mismatch (luv: %s <> bin: %s)",luvbanner,enginebanner) + end + local luvhash=luv.sourcehash or "?" + if luvhash~=sourcehash then + return format("source mismatch (luv: %s <> bin: %s)",luvhash,sourcehash) + end + else + return "invalid status file" + end + else + return "missing status file" end - return clibpaths + end + return true end -local function thepath(...) - local t = { ... } t[#t+1] = "?.lua" - local path = file.join(unpack(t)) - if trace_locating then - report_libraries("! appending '%s' to 'package.path'",path) - end - return path -end -local p_libpaths, a_libpaths = { }, { } +end -- of closure -function package.appendtolibpath(...) - insert(a_libpath,thepath(...)) -end +do -- create closure to overcome 200 locals limit -function package.prependtolibpath(...) - insert(p_libpaths,1,thepath(...)) -end +package.loaded["data-zip"] = package.loaded["data-zip"] or true --- beware, we need to return a loadfile result ! +-- original size: 8489, stripped down to: 6757 -local function loaded(libpaths,name,simple) - for i=1,#libpaths do -- package.path, might become option - local libpath = libpaths[i] - local resolved = gsub(libpath,"%?",simple) - if trace_locating then -- more detail - report_libraries("! checking for '%s' on 'package.path': '%s' => '%s'",simple,libpath,resolved) - end - if file.is_readable(resolved) then - if trace_locating then - report_libraries("! lib '%s' located via 'package.path': '%s'",name,resolved) - end - return loadfile(resolved) - end +if not modules then modules={} end modules ['data-zip']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local format,find,match=string.format,string.find,string.match +local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) +local report_zip=logs.reporter("resolvers","zip") +local resolvers=resolvers +zip=zip or {} +local zip=zip +zip.archives=zip.archives or {} +local archives=zip.archives +zip.registeredfiles=zip.registeredfiles or {} +local registeredfiles=zip.registeredfiles +local limited=false +directives.register("system.inputmode",function(v) + if not limited then + local i_limiter=io.i_limiter(v) + if i_limiter then + zip.open=i_limiter.protect(zip.open) + limited=true + end + end +end) +local function validzip(str) + if not find(str,"^zip://") then + return "zip:///"..str + else + return str + end +end +function zip.openarchive(name) + if not name or name=="" then + return nil + else + local arch=archives[name] + if not arch then + local full=resolvers.findfile(name) or "" + arch=(full~="" and zip.open(full)) or false + archives[name]=arch end + return arch + end end - -package.loaders[2] = function(name) -- was [#package.loaders+1] - if file.suffix(name) == "" then - name = file.addsuffix(name,"lua") -- maybe a list - if trace_locating then -- mode detail - report_libraries("! locating '%s' with forced suffix",name) - end +function zip.closearchive(name) + if not name or (name=="" and archives[name]) then + zip.close(archives[name]) + archives[name]=nil + end +end +function resolvers.locators.zip(specification) + local archive=specification.filename + local zipfile=archive and archive~="" and zip.openarchive(archive) + if trace_locating then + if zipfile then + report_zip("locator: archive %a found",archive) else - if trace_locating then -- mode detail - report_libraries("! locating '%s'",name) - end + report_zip("locator: archive %a not found",archive) end - for i=1,#libformats do - local format = libformats[i] - local resolved = resolvers.findfile(name,format) or "" - if trace_locating then -- mode detail - report_libraries("! checking for '%s' using 'libformat path': '%s'",name,format) - end - if resolved ~= "" then - if trace_locating then - report_libraries("! lib '%s' located via environment: '%s'",name,resolved) - end - return loadfile(resolved) + end +end +function resolvers.hashers.zip(specification) + local archive=specification.filename + if trace_locating then + report_zip("loading file %a",archive) + end + resolvers.usezipfile(specification.original) +end +function resolvers.concatinators.zip(zipfile,path,name) + if not path or path=="" then + return format('%s?name=%s',zipfile,name) + else + return format('%s?name=%s/%s',zipfile,path,name) + end +end +function resolvers.finders.zip(specification) + local original=specification.original + local archive=specification.filename + if archive then + local query=url.query(specification.query) + local queryname=query.name + if queryname then + local zfile=zip.openarchive(archive) + if zfile then + if trace_locating then + report_zip("finder: archive %a found",archive) + end + local dfile=zfile:open(queryname) + if dfile then + dfile=zfile:close() + if trace_locating then + report_zip("finder: file %a found",queryname) + end + return specification.original + elseif trace_locating then + report_zip("finder: file %a not found",queryname) end + elseif trace_locating then + report_zip("finder: unknown archive %a",archive) + end end - -- libpaths - local libpaths, clibpaths = package.libpaths(), package.clibpaths() - local simple = gsub(name,"%.lua$","") - local simple = gsub(simple,"%.","/") - local resolved = loaded(p_libpaths,name,simple) or loaded(libpaths,name,simple) or loaded(a_libpaths,name,simple) - if resolved then - return resolved - end - -- - local libname = file.addsuffix(simple,os.libsuffix) - for i=1,#clibformats do - -- better have a dedicated loop - local format = clibformats[i] - local paths = resolvers.expandedpathlistfromvariable(format) - for p=1,#paths do - local path = paths[p] - local resolved = file.join(path,libname) - if trace_locating then -- mode detail - report_libraries("! checking for '%s' using 'clibformat path': '%s'",libname,path) - end - if file.is_readable(resolved) then - if trace_locating then - report_libraries("! lib '%s' located via 'clibformat': '%s'",libname,resolved) - end - return package.loadlib(resolved,name) - end + end + if trace_locating then + report_zip("finder: %a not found",original) + end + return resolvers.finders.notfound() +end +function resolvers.openers.zip(specification) + local original=specification.original + local archive=specification.filename + if archive then + local query=url.query(specification.query) + local queryname=query.name + if queryname then + local zfile=zip.openarchive(archive) + if zfile then + if trace_locating then + report_zip("opener; archive %a opened",archive) + end + local dfile=zfile:open(queryname) + if dfile then + if trace_locating then + report_zip("opener: file %a found",queryname) + end + return resolvers.openers.helpers.textopener('zip',original,dfile) + elseif trace_locating then + report_zip("opener: file %a not found",queryname) end + elseif trace_locating then + report_zip("opener: unknown archive %a",archive) + end end - for i=1,#clibpaths do -- package.path, might become option - local libpath = clibpaths[i] - local resolved = gsub(libpath,"?",simple) - if trace_locating then -- more detail - report_libraries("! checking for '%s' on 'package.cpath': '%s'",simple,libpath) - end - if file.is_readable(resolved) then - if trace_locating then - report_libraries("! lib '%s' located via 'package.cpath': '%s'",name,resolved) - end - return package.loadlib(resolved,name) + end + if trace_locating then + report_zip("opener: %a not found",original) + end + return resolvers.openers.notfound() +end +function resolvers.loaders.zip(specification) + local original=specification.original + local archive=specification.filename + if archive then + local query=url.query(specification.query) + local queryname=query.name + if queryname then + local zfile=zip.openarchive(archive) + if zfile then + if trace_locating then + report_zip("loader: archive %a opened",archive) + end + local dfile=zfile:open(queryname) + if dfile then + logs.show_load(original) + if trace_locating then + report_zip("loader; file %a loaded",original) + end + local s=dfile:read("*all") + dfile:close() + return true,s,#s + elseif trace_locating then + report_zip("loader: file %a not found",queryname) end + elseif trace_locating then + report_zip("loader; unknown archive %a",archive) + end end - -- just in case the distribution is messed up - if trace_loading then -- more detail - report_libraries("! checking for '%s' using 'luatexlibs': '%s'",name) + end + if trace_locating then + report_zip("loader: %a not found",original) + end + return resolvers.openers.notfound() +end +function resolvers.usezipfile(archive) + local specification=resolvers.splitmethod(archive) + local archive=specification.filename + if archive and not registeredfiles[archive] then + local z=zip.openarchive(archive) + if z then + local instance=resolvers.instance + local tree=url.query(specification.query).tree or "" + if trace_locating then + report_zip("registering: archive %a",archive) + end + statistics.starttiming(instance) + resolvers.prependhash('zip',archive) + resolvers.extendtexmfvariable(archive) + registeredfiles[archive]=z + instance.files[archive]=resolvers.registerzipfile(z,tree) + statistics.stoptiming(instance) + elseif trace_locating then + report_zip("registering: unknown archive %a",archive) end - local resolved = resolvers.findfile(file.basename(name),'luatexlibs') or "" - if resolved ~= "" then - if trace_locating then - report_libraries("! lib '%s' located by basename via environment: '%s'",name,resolved) - end - return loadfile(resolved) + elseif trace_locating then + report_zip("registering: archive %a not found",archive) + end +end +function resolvers.registerzipfile(z,tree) + local files,filter={},"" + if tree=="" then + filter="^(.+)/(.-)$" + else + filter=format("^%s/(.+)/(.-)$",tree) + end + if trace_locating then + report_zip("registering: using filter %a",filter) + end + local register,n=resolvers.registerfile,0 + for i in z:files() do + local path,name=match(i.filename,filter) + if path then + if name and name~='' then + register(files,name,path) + n=n+1 + else + end + else + register(files,i.filename,'') + n=n+1 end + end + report_zip("registering: %s files registered",n) + return files +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["data-tre"] = package.loaded["data-tre"] or true + +-- original size: 2508, stripped down to: 2074 + +if not modules then modules={} end modules ['data-tre']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local find,gsub,format=string.find,string.gsub,string.format +local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) +local report_trees=logs.reporter("resolvers","trees") +local resolvers=resolvers +local done,found,notfound={},{},resolvers.finders.notfound +function resolvers.finders.tree(specification) + local spec=specification.filename + local fnd=found[spec] + if fnd==nil then + if spec~="" then + local path,name=file.dirname(spec),file.basename(spec) + if path=="" then path="." end + local hash=done[path] + if not hash then + local pattern=path.."/*" + hash=dir.glob(pattern) + done[path]=hash + end + local pattern="/"..gsub(name,"([%.%-%+])","%%%1").."$" + for k=1,#hash do + local v=hash[k] + if find(v,pattern) then + found[spec]=v + return v + end + end + end + fnd=notfound() + found[spec]=fnd + end + return fnd +end +function resolvers.locators.tree(specification) + local name=specification.filename + local realname=resolvers.resolve(name) + if realname and realname~='' and lfs.isdir(realname) then if trace_locating then - report_libraries('? unable to locate lib: %s',name) + report_trees("locator %a found",realname) end --- return "unable to locate " .. name + resolvers.appendhash('tree',name,false) + elseif trace_locating then + report_trees("locator %a not found",name) + end end - -resolvers.loadlualib = require - --- -- -- -- - -package.obsolete = package.obsolete or { } - -package.append_libpath = appendtolibpath -- will become obsolete -package.prepend_libpath = prependtolibpath -- will become obsolete - -package.obsolete.append_libpath = appendtolibpath -- will become obsolete -package.obsolete.prepend_libpath = prependtolibpath -- will become obsolete +function resolvers.hashers.tree(specification) + local name=specification.filename + if trace_locating then + report_trees("analysing %a",name) + end + resolvers.methodhandler("hashers",name) + resolvers.generators.file(specification) +end +resolvers.concatinators.tree=resolvers.concatinators.file +resolvers.generators.tree=resolvers.generators.file +resolvers.openers.tree=resolvers.openers.file +resolvers.loaders.tree=resolvers.loaders.file end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-aux'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local find = string.find -local type, next = type, next - -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) - -local resolvers = resolvers +package.loaded["data-sch"] = package.loaded["data-sch"] or true -local report_scripts = logs.reporter("resolvers","scripts") +-- original size: 6202, stripped down to: 5149 -function resolvers.updatescript(oldname,newname) -- oldname -> own.name, not per se a suffix - local scriptpath = "scripts/context/lua" - newname = file.addsuffix(newname,"lua") - local oldscript = resolvers.cleanpath(oldname) - if trace_locating then - report_scripts("to be replaced old script %s", oldscript) - end - local newscripts = resolvers.findfiles(newname) or { } - if #newscripts == 0 then - if trace_locating then - report_scripts("unable to locate new script") - end +if not modules then modules={} end modules ['data-sch']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local load=load +local gsub,concat,format=string.gsub,table.concat,string.format +local finders,openers,loaders=resolvers.finders,resolvers.openers,resolvers.loaders +local trace_schemes=false trackers.register("resolvers.schemes",function(v) trace_schemes=v end) +local report_schemes=logs.reporter("resolvers","schemes") +local http=require("socket.http") +local ltn12=require("ltn12") +local resolvers=resolvers +local schemes=resolvers.schemes or {} +resolvers.schemes=schemes +local cleaners={} +schemes.cleaners=cleaners +local threshold=24*60*60 +directives.register("schemes.threshold",function(v) threshold=tonumber(v) or threshold end) +function cleaners.none(specification) + return specification.original +end +function cleaners.strip(specification) + return (gsub(specification.original,"[^%a%d%.]+","-")) +end +function cleaners.md5(specification) + return file.addsuffix(md5.hex(specification.original),file.suffix(specification.path)) +end +local cleaner=cleaners.strip +directives.register("schemes.cleanmethod",function(v) cleaner=cleaners[v] or cleaners.strip end) +function resolvers.schemes.cleanname(specification) + local hash=cleaner(specification) + if trace_schemes then + report_schemes("hashing %a to %a",specification.original,hash) + end + return hash +end +local cached,loaded,reused,thresholds,handlers={},{},{},{},{} +local function runcurl(name,cachename) + local command="curl --silent --create-dirs --output "..cachename.." "..name + os.spawn(command) +end +local function fetch(specification) + local original=specification.original + local scheme=specification.scheme + local cleanname=schemes.cleanname(specification) + local cachename=caches.setfirstwritablefile(cleanname,"schemes") + if not cached[original] then + statistics.starttiming(schemes) + if not io.exists(cachename) or (os.difftime(os.time(),lfs.attributes(cachename).modification)>(thresholds[protocol] or threshold)) then + cached[original]=cachename + local handler=handlers[scheme] + if handler then + if trace_schemes then + report_schemes("fetching %a, protocol %a, method %a",original,scheme,"built-in") + end + logs.flush() + handler(specification,cachename) + else + if trace_schemes then + report_schemes("fetching %a, protocol %a, method %a",original,scheme,"curl") + end + logs.flush() + runcurl(original,cachename) + end + end + if io.exists(cachename) then + cached[original]=cachename + if trace_schemes then + report_schemes("using cached %a, protocol %a, cachename %a",original,scheme,cachename) + end else - for i=1,#newscripts do - local newscript = resolvers.cleanpath(newscripts[i]) - if trace_locating then - report_scripts("checking new script %s", newscript) - end - if oldscript == newscript then - if trace_locating then - report_scripts("old and new script are the same") - end - elseif not find(newscript,scriptpath) then - if trace_locating then - report_scripts("new script should come from %s",scriptpath) - end - elseif not (find(oldscript,file.removesuffix(newname).."$") or find(oldscript,newname.."$")) then - if trace_locating then - report_scripts("invalid new script name") - end - else - local newdata = io.loaddata(newscript) - if newdata then - if trace_locating then - report_scripts("old script content replaced by new content") - end - io.savedata(oldscript,newdata) - break - elseif trace_locating then - report_scripts("unable to load new script") - end - end - end + cached[original]="" + if trace_schemes then + report_schemes("using missing %a, protocol %a",original,scheme) + end + end + loaded[scheme]=loaded[scheme]+1 + statistics.stoptiming(schemes) + else + if trace_schemes then + report_schemes("reusing %a, protocol %a",original,scheme) end + reused[scheme]=reused[scheme]+1 + end + return cached[original] +end +local function finder(specification,filetype) + return resolvers.methodhandler("finders",fetch(specification),filetype) +end +local opener=openers.file +local loader=loaders.file +local function install(scheme,handler,newthreshold) + handlers [scheme]=handler + loaded [scheme]=0 + reused [scheme]=0 + finders [scheme]=finder + openers [scheme]=opener + loaders [scheme]=loader + thresholds[scheme]=newthreshold or threshold +end +schemes.install=install +local function http_handler(specification,cachename) + local tempname=cachename..".tmp" + local f=io.open(tempname,"wb") + local status,message=http.request { + url=specification.original, + sink=ltn12.sink.file(f) + } + if not status then + os.remove(tempname) + else + os.remove(cachename) + os.rename(tempname,cachename) + end + return cachename +end +install('http',http_handler) +install('https') +install('ftp') +statistics.register("scheme handling time",function() + local l,r,nl,nr={},{},0,0 + for k,v in table.sortedhash(loaded) do + if v>0 then + nl=nl+1 + l[nl]=k..":"..v + end + end + for k,v in table.sortedhash(reused) do + if v>0 then + nr=nr+1 + r[nr]=k..":"..v + end + end + local n=nl+nr + if n>0 then + l=nl>0 and concat(l) or "none" + r=nr>0 and concat(r) or "none" + return format("%s seconds, %s processed, threshold %s seconds, loaded: %s, reused: %s", + statistics.elapsedtime(schemes),n,threshold,l,r) + else + return nil + end +end) +local httprequest=http.request +local toquery=url.toquery +local function fetchstring(url,data) + local q=data and toquery(data) + if q then + url=url.."?"..q + end + local reply=httprequest(url) + return reply +end +schemes.fetchstring=fetchstring +function schemes.fetchtable(url,data) + local reply=fetchstring(url,data) + if reply then + local s=load("return "..reply) + if s then + return s() + end + end end @@ -15312,78 +15079,198 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-tmf'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local resolvers = resolvers +package.loaded["data-lua"] = package.loaded["data-lua"] or true -local report_tds = logs.reporter("resolvers","tds") +-- original size: 4861, stripped down to: 3693 --- = << --- ? ?? --- < += --- > =+ +if not modules then modules={} end modules ['data-lua']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local resolvers,package=resolvers,package +local gsub=string.gsub +local concat=table.concat +local addsuffix=file.addsuffix +local P,S,Cs,lpegmatch=lpeg.P,lpeg.S,lpeg.Cs,lpeg.match +local libsuffixes={ 'tex','lua' } +local clibsuffixes={ 'lib' } +local libformats={ 'TEXINPUTS','LUAINPUTS' } +local clibformats={ 'CLUAINPUTS' } +local helpers=package.helpers +trackers.register("resolvers.libraries",function(v) helpers.trace=v end) +trackers.register("resolvers.locating",function(v) helpers.trace=v end) +helpers.report=logs.reporter("resolvers","libraries") +local pattern=Cs(P("!")^0/""*(P("/")*P(-1)/"/"+P("/")^1/"/"+1)^0) +local function cleanpath(path) + return resolvers.resolve(lpegmatch(pattern,path)) +end +helpers.cleanpath=cleanpath +local loadedaslib=helpers.loadedaslib +local loadedbylua=helpers.loadedbylua +local loadedbypath=helpers.loadedbypath +local notloaded=helpers.notloaded +local getlibpaths=package.libpaths +local getclibpaths=package.clibpaths +function helpers.libpaths(libhash) + local libpaths={} + for i=1,#libformats do + local paths=resolvers.expandedpathlistfromvariable(libformats[i]) + for i=1,#paths do + local path=cleanpath(paths[i]) + if not libhash[path] then + libpaths[#libpaths+1]=path + libhash[path]=true + end + end + end + return libpaths +end +function helpers.clibpaths(clibhash) + local clibpaths={} + for i=1,#clibformats do + local paths=resolvers.expandedpathlistfromvariable(clibformats[i]) + for i=1,#paths do + local path=cleanpath(paths[i]) + if not clibhash[path] then + clibpaths[#clibpaths+1]=path + clibhash[path]=true + end + end + end + return clibpaths +end +local function loadedbyformat(name,rawname,suffixes,islib) + local trace=helpers.trace + local report=helpers.report + if trace then + report("locating %a as %a using formats %a",rawname,name,suffixes) + end + for i=1,#suffixes do + local format=suffixes[i] + local resolved=resolvers.findfile(name,format) or "" + if trace then + report("checking %a using format %a",name,format) + end + if resolved~="" then + if trace then + report("lib %a located on %a",name,resolved) + end + if islib then + return true,loadedaslib(resolved,rawname) + else + return true,loadfile(resolved) + end + end + end +end +helpers.loadedbyformat=loadedbyformat +local pattern=Cs((((1-S("\\/"))^0*(S("\\/")^1/"/"))^0*(P(".")^1/"/"+P(1))^1)*-1) +local function lualibfile(name) + return lpegmatch(pattern,name) or name +end +helpers.lualibfile=lualibfile +function helpers.loaded(name) + local thename=lualibfile(name) + local luaname=addsuffix(thename,"lua") + local libname=addsuffix(thename,os.libsuffix) + local libpaths=getlibpaths() + local clibpaths=getclibpaths() + local done,result=loadedbyformat(luaname,name,libsuffixes,false) + if done then + return result + end + local done,result=loadedbyformat(libname,name,clibsuffixes,true) + if done then + return result + end + local done,result=loadedbypath(luaname,name,libpaths,false,"lua") + if done then + return result + end + local done,result=loadedbypath(luaname,name,clibpaths,false,"lua") + if done then + return result + end + local done,result=loadedbypath(libname,name,clibpaths,true,"lib") + if done then + return result + end + local done,result=loadedbylua(name) + if done then + return result + end + return notloaded(name) +end +resolvers.loadlualib=require -function resolvers.load_tree(tree,resolve) - if type(tree) == "string" and tree ~= "" then - local getenv, setenv = resolvers.getenv, resolvers.setenv +end -- of closure - -- later might listen to the raw osenv var as well - local texos = "texmf-" .. os.platform +do -- create closure to overcome 200 locals limit - local oldroot = environment.texroot - local newroot = file.collapsepath(tree) +package.loaded["data-aux"] = package.loaded["data-aux"] or true - local newtree = file.join(newroot,texos) - local newpath = file.join(newtree,"bin") +-- original size: 2394, stripped down to: 2005 - if not lfs.isdir(newtree) then - report_tds("no '%s' under tree %s",texos,tree) - os.exit() +if not modules then modules={} end modules ['data-aux']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local find=string.find +local type,next=type,next +local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) +local resolvers=resolvers +local report_scripts=logs.reporter("resolvers","scripts") +function resolvers.updatescript(oldname,newname) + local scriptpath="scripts/context/lua" + newname=file.addsuffix(newname,"lua") + local oldscript=resolvers.cleanpath(oldname) + if trace_locating then + report_scripts("to be replaced old script %a",oldscript) + end + local newscripts=resolvers.findfiles(newname) or {} + if #newscripts==0 then + if trace_locating then + report_scripts("unable to locate new script") + end + else + for i=1,#newscripts do + local newscript=resolvers.cleanpath(newscripts[i]) + if trace_locating then + report_scripts("checking new script %a",newscript) + end + if oldscript==newscript then + if trace_locating then + report_scripts("old and new script are the same") end - if not lfs.isdir(newpath) then - report_tds("no '%s/bin' under tree %s",texos,tree) - os.exit() + elseif not find(newscript,scriptpath) then + if trace_locating then + report_scripts("new script should come from %a",scriptpath) end - - local texmfos = newtree - - environment.texroot = newroot - environment.texos = texos - environment.texmfos = texmfos - - -- Beware, we need to obey the relocatable autoparent so we - -- set TEXMFCNF to its raw value. This is somewhat tricky when - -- we run a mkii job from within. Therefore, in mtxrun, there - -- is a resolve applied when we're in mkii/kpse mode or when - -- --resolve is passed to mtxrun. Maybe we should also set the - -- local AUTOPARENT etc. although these are alwasy set new. - - if resolve then - -- resolvers.luacnfspec = resolvers.joinpath(resolvers.resolve(resolvers.expandedpathfromlist(resolvers.splitpath(resolvers.luacnfspec)))) - resolvers.luacnfspec = resolvers.resolve(resolvers.luacnfspec) + elseif not (find(oldscript,file.removesuffix(newname).."$") or find(oldscript,newname.."$")) then + if trace_locating then + report_scripts("invalid new script name") + end + else + local newdata=io.loaddata(newscript) + if newdata then + if trace_locating then + report_scripts("old script content replaced by new content") + end + io.savedata(oldscript,newdata) + break + elseif trace_locating then + report_scripts("unable to load new script") end - - setenv('SELFAUTOPARENT', newroot) - setenv('SELFAUTODIR', newtree) - setenv('SELFAUTOLOC', newpath) - setenv('TEXROOT', newroot) - setenv('TEXOS', texos) - setenv('TEXMFOS', texmfos) - setenv('TEXMFCNF', resolvers.luacnfspec,true) -- already resolved - setenv('PATH', newpath .. io.pathseparator .. getenv('PATH')) - - report_tds("changing from root '%s' to '%s'",oldroot,newroot) - report_tds("prepending '%s' to PATH",newpath) - report_tds("setting TEXMFCNF to '%s'",resolvers.luacnfspec) - report_tds() + end end + end end @@ -15391,81 +15278,132 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-lst'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" +package.loaded["data-tmf"] = package.loaded["data-tmf"] or true + +-- original size: 2600, stripped down to: 1627 + +if not modules then modules={} end modules ['data-tmf']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" } +local resolvers=resolvers +local report_tds=logs.reporter("resolvers","tds") +function resolvers.load_tree(tree,resolve) + if type(tree)=="string" and tree~="" then + local getenv,setenv=resolvers.getenv,resolvers.setenv + local texos="texmf-"..os.platform + local oldroot=environment.texroot + local newroot=file.collapsepath(tree) + local newtree=file.join(newroot,texos) + local newpath=file.join(newtree,"bin") + if not lfs.isdir(newtree) then + report_tds("no %a under tree %a",texos,tree) + os.exit() + end + if not lfs.isdir(newpath) then + report_tds("no '%s/bin' under tree %a",texos,tree) + os.exit() + end + local texmfos=newtree + environment.texroot=newroot + environment.texos=texos + environment.texmfos=texmfos + if resolve then + resolvers.luacnfspec=resolvers.resolve(resolvers.luacnfspec) + end + setenv('SELFAUTOPARENT',newroot) + setenv('SELFAUTODIR',newtree) + setenv('SELFAUTOLOC',newpath) + setenv('TEXROOT',newroot) + setenv('TEXOS',texos) + setenv('TEXMFOS',texmfos) + setenv('TEXMFCNF',resolvers.luacnfspec,true) + setenv('PATH',newpath..io.pathseparator..getenv('PATH')) + report_tds("changing from root %a to %a",oldroot,newroot) + report_tds("prepending %a to PATH",newpath) + report_tds("setting TEXMFCNF to %a",resolvers.luacnfspec) + report_tds() + end +end --- used in mtxrun, can be loaded later .. todo -local find, concat, upper, format = string.find, table.concat, string.upper, string.format -local fastcopy, sortedpairs = table.fastcopy, table.sortedpairs +end -- of closure -resolvers.listers = resolvers.listers or { } +do -- create closure to overcome 200 locals limit -local resolvers = resolvers +package.loaded["data-lst"] = package.loaded["data-lst"] or true -local report_lists = logs.reporter("resolvers","lists") +-- original size: 2654, stripped down to: 2301 +if not modules then modules={} end modules ['data-lst']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local find,concat,upper,format=string.find,table.concat,string.upper,string.format +local fastcopy,sortedpairs=table.fastcopy,table.sortedpairs +resolvers.listers=resolvers.listers or {} +local resolvers=resolvers +local report_lists=logs.reporter("resolvers","lists") local function tabstr(str) - if type(str) == 'table' then - return concat(str," | ") - else - return str - end + if type(str)=='table' then + return concat(str," | ") + else + return str + end end - function resolvers.listers.variables(pattern) - local instance = resolvers.instance - local environment = instance.environment - local variables = instance.variables - local expansions = instance.expansions - local pattern = upper(pattern or "") - local configured = { } - local order = instance.order - for i=1,#order do - for k, v in next, order[i] do - if v ~= nil and configured[k] == nil then - configured[k] = v - end - end - end - local env = fastcopy(environment) - local var = fastcopy(variables) - local exp = fastcopy(expansions) - for key, value in sortedpairs(configured) do - if key ~= "" and (pattern == "" or find(upper(key),pattern)) then - report_lists(key) - report_lists(" env: %s",tabstr(rawget(environment,key)) or "unset") - report_lists(" var: %s",tabstr(configured[key]) or "unset") - report_lists(" exp: %s",tabstr(expansions[key]) or "unset") - report_lists(" res: %s",tabstr(resolvers.resolve(expansions[key])) or "unset") - end - end - instance.environment = fastcopy(env) - instance.variables = fastcopy(var) - instance.expansions = fastcopy(exp) -end - -function resolvers.listers.configurations(report) - local configurations = resolvers.instance.specification - local report = report or texio.write_nl - for i=1,#configurations do - report(format("file : %s",resolvers.resolve(configurations[i]))) - end - report("") - local list = resolvers.expandedpathfromlist(resolvers.splitpath(resolvers.luacnfspec)) - for i=1,#list do - local li = resolvers.resolve(list[i]) - if lfs.isdir(li) then - report(format("path - %s",li)) - else - report(format("path + %s",li)) - end + local instance=resolvers.instance + local environment=instance.environment + local variables=instance.variables + local expansions=instance.expansions + local pattern=upper(pattern or "") + local configured={} + local order=instance.order + for i=1,#order do + for k,v in next,order[i] do + if v~=nil and configured[k]==nil then + configured[k]=v + end + end + end + local env=fastcopy(environment) + local var=fastcopy(variables) + local exp=fastcopy(expansions) + for key,value in sortedpairs(configured) do + if key~="" and (pattern=="" or find(upper(key),pattern)) then + report_lists(key) + report_lists(" env: %s",tabstr(rawget(environment,key)) or "unset") + report_lists(" var: %s",tabstr(configured[key]) or "unset") + report_lists(" exp: %s",tabstr(expansions[key]) or "unset") + report_lists(" res: %s",tabstr(resolvers.resolve(expansions[key])) or "unset") + end + end + instance.environment=fastcopy(env) + instance.variables=fastcopy(var) + instance.expansions=fastcopy(exp) +end +local report_resolved=logs.reporter("system","resolved") +function resolvers.listers.configurations() + local configurations=resolvers.instance.specification + for i=1,#configurations do + report_resolved("file : %s",resolvers.resolve(configurations[i])) + end + report_resolved("") + local list=resolvers.expandedpathfromlist(resolvers.splitpath(resolvers.luacnfspec)) + for i=1,#list do + local li=resolvers.resolve(list[i]) + if lfs.isdir(li) then + report_resolved("path - %s",li) + else + report_resolved("path + %s",li) end + end end @@ -15473,279 +15411,444 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['luat-sta'] = { - version = 1.001, - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} +package.loaded["util-lib"] = package.loaded["util-lib"] or true --- this code is used in the updater +-- original size: 10762, stripped down to: 5269 -local gmatch, match = string.gmatch, string.match -local type = type +if not modules then modules={} end modules ['util-lib']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files", +} +local gsub,find=string.gsub,string.find +local pathpart,nameonly,joinfile=file.pathpart,file.nameonly,file.join +local findfile,findfiles=resolvers and resolvers.findfile,resolvers and resolvers.findfiles +local loaded=package.loaded +local report_swiglib=logs.reporter("swiglib") +local trace_swiglib=false trackers.register("resolvers.swiglib",function(v) trace_swiglib=v end) +local done=false +local function requireswiglib(required,version) + local library=loaded[required] + if library==nil then + local required_full=gsub(required,"%.","/") + local required_path=pathpart(required_full) + local required_base=nameonly(required_full) + local required_name=required_base.."."..os.libsuffix + local version=type(version)=="string" and version~="" and version or false + local engine=environment.ownmain or false + if trace_swiglib and not done then + local list=resolvers.expandedpathlistfromvariable("lib") + for i=1,#list do + report_swiglib("tds path %i: %s",i,list[i]) + end + end + local function found(locate,asked_library,how,...) + if trace_swiglib then + report_swiglib("checking %s: %a",how,asked_library) + end + return locate(asked_library,...) + end + local function check(locate,...) + local found=nil + if version then + local asked_library=joinfile(required_path,version,required_name) + if trace_swiglib then + report_swiglib("checking %s: %a","with version",asked_library) + end + found=locate(asked_library,...) + end + if not found or found=="" then + local asked_library=joinfile(required_path,required_name) + if trace_swiglib then + report_swiglib("checking %s: %a","with version",asked_library) + end + found=locate(asked_library,...) + end + return found and found~="" and found or false + end + local function attempt(checkpattern) + if trace_swiglib then + report_swiglib("checking tds lib paths strictly") + end + local found=findfile and check(findfile,"lib") + if found and (not checkpattern or find(found,checkpattern)) then + return found + end + if trace_swiglib then + report_swiglib("checking tds lib paths with wildcard") + end + local asked_library=joinfile(required_path,".*",required_name) + if trace_swiglib then + report_swiglib("checking %s: %a","latest version",asked_library) + end + local list=findfiles(asked_library,"lib",true) + if list and #list>0 then + table.sort(list) + local found=list[#list] + if found and (not checkpattern or find(found,checkpattern)) then + return found + end + end + if trace_swiglib then + report_swiglib("checking clib paths") + end + package.extraclibpath(environment.ownpath) + local paths=package.clibpaths() + for i=1,#paths do + local found=check(lfs.isfile) + if found and (not checkpattern or find(found,checkpattern)) then + return found + end + end + return false + end + local found_library=nil + if engine then + if trace_swiglib then + report_swiglib("attemp 1, engine %a",engine) + end + found_library=attempt("/"..engine.."/") + if not found_library then + if trace_swiglib then + report_swiglib("attemp 2, no engine",asked_library) + end + found_library=attempt() + end + else + found_library=attempt() + end + if not found_library then + if trace_swiglib then + report_swiglib("not found: %a",asked_library) + end + library=false + else + local path=pathpart(found_library) + local base=nameonly(found_library) + dir.push(path) + if trace_swiglib then + report_swiglib("found: %a",found_library) + end + library=package.loadlib(found_library,"luaopen_"..required_base) + if type(library)=="function" then + library=library() + else + library=false + end + dir.pop() + end + if not library then + report_swiglib("unknown: %a",required) + elseif trace_swiglib then + report_swiglib("stored: %a",required) + end + loaded[required]=library + else + report_swiglib("reused: %a",required) + end + return library +end +local savedrequire=require +function require(name,version) + if find(name,"^swiglib%.") then + return requireswiglib(name,version) + else + return savedrequire(name) + end +end +local swiglibs={} +function swiglib(name,version) + local library=swiglibs[name] + if not library then + statistics.starttiming(swiglibs) + report_swiglib("loading %a",name) + library=requireswiglib("swiglib."..name,version) + swiglibs[name]=library + statistics.stoptiming(swiglibs) + end + return library +end +statistics.register("used swiglibs",function() + if next(swiglibs) then + return string.format("%s, initial load time %s seconds",table.concat(table.sortedkeys(swiglibs)," "),statistics.elapsedtime(swiglibs)) + end +end) -states = states or { } -local states = states -states.data = states.data or { } -local data = states.data +end -- of closure + +do -- create closure to overcome 200 locals limit -states.hash = states.hash or { } -local hash = states.hash +package.loaded["luat-sta"] = package.loaded["luat-sta"] or true -states.tag = states.tag or "" -states.filename = states.filename or "" +-- original size: 5703, stripped down to: 2507 +if not modules then modules={} end modules ['luat-sta']={ + version=1.001, + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local gmatch,match=string.gmatch,string.match +local type=type +states=states or {} +local states=states +states.data=states.data or {} +local data=states.data +states.hash=states.hash or {} +local hash=states.hash +states.tag=states.tag or "" +states.filename=states.filename or "" function states.save(filename,tag) - tag = tag or states.tag - filename = file.addsuffix(filename or states.filename,'lus') - io.savedata(filename, - "-- generator : luat-sta.lua\n" .. - "-- state tag : " .. tag .. "\n\n" .. - table.serialize(data[tag or states.tag] or {},true) - ) + tag=tag or states.tag + filename=file.addsuffix(filename or states.filename,'lus') + io.savedata(filename, + "-- generator : luat-sta.lua\n".."-- state tag : "..tag.."\n\n"..table.serialize(data[tag or states.tag] or {},true) + ) end - function states.load(filename,tag) - states.filename = filename - states.tag = tag or "whatever" - states.filename = file.addsuffix(states.filename,'lus') - data[states.tag], hash[states.tag] = (io.exists(filename) and dofile(filename)) or { }, { } + states.filename=filename + states.tag=tag or "whatever" + states.filename=file.addsuffix(states.filename,'lus') + data[states.tag],hash[states.tag]=(io.exists(filename) and dofile(filename)) or {},{} end - local function set_by_tag(tag,key,value,default,persistent) - local d, h = data[tag], hash[tag] - if d then - if type(d) == "table" then - local dkey, hkey = key, key - local pre, post = match(key,"(.+)%.([^%.]+)$") - if pre and post then - for k in gmatch(pre,"[^%.]+") do - local dk = d[k] - if not dk then - dk = { } - d[k] = dk - elseif type(dk) == "string" then - -- invalid table, unable to upgrade structure - -- hope for the best or delete the state file - break - end - d = dk - end - dkey, hkey = post, key - end - if value == nil then - value = default - elseif value == false then - -- special case - elseif persistent then - value = value or d[dkey] or default - else - value = value or default - end - d[dkey], h[hkey] = value, value - elseif type(d) == "string" then - -- weird - data[tag], hash[tag] = value, value - end - end + local d,h=data[tag],hash[tag] + if d then + if type(d)=="table" then + local dkey,hkey=key,key + local pre,post=match(key,"(.+)%.([^%.]+)$") + if pre and post then + for k in gmatch(pre,"[^%.]+") do + local dk=d[k] + if not dk then + dk={} + d[k]=dk + elseif type(dk)=="string" then + break + end + d=dk + end + dkey,hkey=post,key + end + if value==nil then + value=default + elseif value==false then + elseif persistent then + value=value or d[dkey] or default + else + value=value or default + end + d[dkey],h[hkey]=value,value + elseif type(d)=="string" then + data[tag],hash[tag]=value,value + end + end end - local function get_by_tag(tag,key,default) - local h = hash[tag] - if h and h[key] then - return h[key] - else - local d = data[tag] - if d then - for k in gmatch(key,"[^%.]+") do - local dk = d[k] - if dk ~= nil then - d = dk - else - return default - end - end - if d == false then - return false - else - return d or default - end + local h=hash[tag] + if h and h[key] then + return h[key] + else + local d=data[tag] + if d then + for k in gmatch(key,"[^%.]+") do + local dk=d[k] + if dk~=nil then + d=dk + else + return default end + end + if d==false then + return false + else + return d or default + end end + end end - -states.set_by_tag = set_by_tag -states.get_by_tag = get_by_tag - +states.set_by_tag=set_by_tag +states.get_by_tag=get_by_tag function states.set(key,value,default,persistent) - set_by_tag(states.tag,key,value,default,persistent) + set_by_tag(states.tag,key,value,default,persistent) end - function states.get(key,default) - return get_by_tag(states.tag,key,default) + return get_by_tag(states.tag,key,default) end - - - end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['luat-fmt'] = { - version = 1.001, - comment = "companion to mtxrun", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - - -local format = string.format - -local report_format = logs.reporter("resolvers","formats") - --- helper for mtxrun +package.loaded["luat-fmt"] = package.loaded["luat-fmt"] or true -local quoted = string.quoted +-- original size: 5951, stripped down to: 4922 -local function primaryflags() -- not yet ok - local trackers = environment.argument("trackers") - local directives = environment.argument("directives") - local flags = "" - if trackers and trackers ~= "" then - flags = flags .. "--trackers=" .. quoted(trackers) - end - if directives and directives ~= "" then - flags = flags .. "--directives=" .. quoted(directives) - end - return flags +if not modules then modules={} end modules ['luat-fmt']={ + version=1.001, + comment="companion to mtxrun", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local format=string.format +local concat=table.concat +local quoted=string.quoted +local luasuffixes=utilities.lua.suffixes +local report_format=logs.reporter("resolvers","formats") +local function primaryflags() + local trackers=environment.argument("trackers") + local directives=environment.argument("directives") + local flags={} + if trackers and trackers~="" then + flags={ "--trackers="..quoted(trackers) } + end + if directives and directives~="" then + flags={ "--directives="..quoted(directives) } + end + if environment.argument("jit") then + flags={ "--jiton" } + end + return concat(flags," ") end - function environment.make_format(name) - -- change to format path (early as we need expanded paths) - local olddir = lfs.currentdir() - local path = caches.getwritablepath("formats") or "" -- maybe platform - if path ~= "" then - lfs.chdir(path) - end - report_format("format path: %s",lfs.currentdir()) - -- check source file - local texsourcename = file.addsuffix(name,"mkiv") - local fulltexsourcename = resolvers.findfile(texsourcename,"tex") or "" - if fulltexsourcename == "" then - texsourcename = file.addsuffix(name,"tex") - fulltexsourcename = resolvers.findfile(texsourcename,"tex") or "" - end - if fulltexsourcename == "" then - report_format("no tex source file with name: %s (mkiv or tex)",name) - lfs.chdir(olddir) - return - else - report_format("using tex source file: %s",fulltexsourcename) - end - local texsourcepath = dir.expandname(file.dirname(fulltexsourcename)) -- really needed - -- check specification - local specificationname = file.replacesuffix(fulltexsourcename,"lus") - local fullspecificationname = resolvers.findfile(specificationname,"tex") or "" - if fullspecificationname == "" then - specificationname = file.join(texsourcepath,"context.lus") - fullspecificationname = resolvers.findfile(specificationname,"tex") or "" - end - if fullspecificationname == "" then - report_format("unknown stub specification: %s",specificationname) - lfs.chdir(olddir) - return - end - local specificationpath = file.dirname(fullspecificationname) - -- load specification - local usedluastub = nil - local usedlualibs = dofile(fullspecificationname) - if type(usedlualibs) == "string" then - usedluastub = file.join(file.dirname(fullspecificationname),usedlualibs) - elseif type(usedlualibs) == "table" then - report_format("using stub specification: %s",fullspecificationname) - local texbasename = file.basename(name) - local luastubname = file.addsuffix(texbasename,"lua") - local lucstubname = file.addsuffix(texbasename,"luc") - -- pack libraries in stub - report_format("creating initialization file: %s",luastubname) - utilities.merger.selfcreate(usedlualibs,specificationpath,luastubname) - -- compile stub file (does not save that much as we don't use this stub at startup any more) - local strip = resolvers.booleanvariable("LUACSTRIP", true) - if utilities.lua.compile(luastubname,lucstubname) and lfs.isfile(lucstubname) then - report_format("using compiled initialization file: %s",lucstubname) - usedluastub = lucstubname - else - report_format("using uncompiled initialization file: %s",luastubname) - usedluastub = luastubname - end + local engine=environment.ownmain or "luatex" + local olddir=dir.current() + local path=caches.getwritablepath("formats",engine) or "" + if path~="" then + lfs.chdir(path) + end + report_format("using format path %a",dir.current()) + local texsourcename=file.addsuffix(name,"mkiv") + local fulltexsourcename=resolvers.findfile(texsourcename,"tex") or "" + if fulltexsourcename=="" then + texsourcename=file.addsuffix(name,"tex") + fulltexsourcename=resolvers.findfile(texsourcename,"tex") or "" + end + if fulltexsourcename=="" then + report_format("no tex source file with name %a (mkiv or tex)",name) + lfs.chdir(olddir) + return + else + report_format("using tex source file %a",fulltexsourcename) + end + local texsourcepath=dir.expandname(file.dirname(fulltexsourcename)) + local specificationname=file.replacesuffix(fulltexsourcename,"lus") + local fullspecificationname=resolvers.findfile(specificationname,"tex") or "" + if fullspecificationname=="" then + specificationname=file.join(texsourcepath,"context.lus") + fullspecificationname=resolvers.findfile(specificationname,"tex") or "" + end + if fullspecificationname=="" then + report_format("unknown stub specification %a",specificationname) + lfs.chdir(olddir) + return + end + local specificationpath=file.dirname(fullspecificationname) + local usedluastub=nil + local usedlualibs=dofile(fullspecificationname) + if type(usedlualibs)=="string" then + usedluastub=file.join(file.dirname(fullspecificationname),usedlualibs) + elseif type(usedlualibs)=="table" then + report_format("using stub specification %a",fullspecificationname) + local texbasename=file.basename(name) + local luastubname=file.addsuffix(texbasename,luasuffixes.lua) + local lucstubname=file.addsuffix(texbasename,luasuffixes.luc) + report_format("creating initialization file %a",luastubname) + utilities.merger.selfcreate(usedlualibs,specificationpath,luastubname) + if utilities.lua.compile(luastubname,lucstubname) and lfs.isfile(lucstubname) then + report_format("using compiled initialization file %a",lucstubname) + usedluastub=lucstubname else - report_format("invalid stub specification: %s",fullspecificationname) - lfs.chdir(olddir) - return - end - -- generate format - local command = format("luatex --ini %s --lua=%s %s %sdump",primaryflags(),quoted(usedluastub),quoted(fulltexsourcename),os.platform == "unix" and "\\\\" or "\\") - report_format("running command: %s\n",command) - os.spawn(command) - -- remove related mem files - local pattern = file.removesuffix(file.basename(usedluastub)).."-*.mem" - -- report_format("removing related mplib format with pattern '%s'", pattern) - local mp = dir.glob(pattern) - if mp then - for i=1,#mp do - local name = mp[i] - report_format("removing related mplib format %s", file.basename(name)) - os.remove(name) - end + report_format("using uncompiled initialization file %a",luastubname) + usedluastub=luastubname end + else + report_format("invalid stub specification %a",fullspecificationname) lfs.chdir(olddir) + return + end + local command=format("%s --ini %s --lua=%s %s %sdump",engine,primaryflags(),quoted(usedluastub),quoted(fulltexsourcename),os.platform=="unix" and "\\\\" or "\\") + report_format("running command: %s\n",command) + os.spawn(command) + local pattern=file.removesuffix(file.basename(usedluastub)).."-*.mem" + local mp=dir.glob(pattern) + if mp then + for i=1,#mp do + local name=mp[i] + report_format("removing related mplib format %a",file.basename(name)) + os.remove(name) + end + end + lfs.chdir(olddir) end - function environment.run_format(name,data,more) - -- hm, rather old code here; we can now use the file.whatever functions - if name and name ~= "" then - local barename = file.removesuffix(name) - local fmtname = caches.getfirstreadablefile(file.addsuffix(barename,"fmt"),"formats") - if fmtname == "" then - fmtname = resolvers.findfile(file.addsuffix(barename,"fmt")) or "" - end - fmtname = resolvers.cleanpath(fmtname) - if fmtname == "" then - report_format("no format with name: %s",name) - else - local barename = file.removesuffix(name) -- expanded name - local luaname = file.addsuffix(barename,"luc") - if not lfs.isfile(luaname) then - luaname = file.addsuffix(barename,"lua") - end - if not lfs.isfile(luaname) then - report_format("using format name: %s",fmtname) - report_format("no luc/lua with name: %s",barename) - else - local command = format("luatex %s --fmt=%s --lua=%s %s %s",primaryflags(),quoted(barename),quoted(luaname),quoted(data),more ~= "" and quoted(more) or "") - report_format("running command: %s",command) - os.spawn(command) - end - end + if name and name~="" then + local engine=environment.ownmain or "luatex" + local barename=file.removesuffix(name) + local fmtname=caches.getfirstreadablefile(file.addsuffix(barename,"fmt"),"formats",engine) + if fmtname=="" then + fmtname=resolvers.findfile(file.addsuffix(barename,"fmt")) or "" + end + fmtname=resolvers.cleanpath(fmtname) + if fmtname=="" then + report_format("no format with name %a",name) + else + local barename=file.removesuffix(name) + local luaname=file.addsuffix(barename,"luc") + if not lfs.isfile(luaname) then + luaname=file.addsuffix(barename,"lua") + end + if not lfs.isfile(luaname) then + report_format("using format name %a",fmtname) + report_format("no luc/lua file with name %a",barename) + else + local command=format("%s %s --fmt=%s --lua=%s %s %s",engine,primaryflags(),quoted(barename),quoted(luaname),quoted(data),more~="" and quoted(more) or "") + report_format("running command: %s",command) + os.spawn(command) + end end + end end end -- of closure + +-- used libraries : l-lua.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-md5.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-sto.lua util-prs.lua util-fmt.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-mrg.lua util-tpl.lua util-env.lua luat-env.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua +-- skipped libraries : - +-- original bytes : 660467 +-- stripped bytes : 242438 + -- end library merge -own = { } -- not local, might change +-- We need this hack till luatex is fixed. +-- +-- for k,v in pairs(arg) do print(k,v) end + +if arg and (arg[0] == 'luatex' or arg[0] == 'luatex.exe') and arg[1] == "--luaonly" then + arg[-1]=arg[0] arg[0]=arg[2] for k=3,#arg do arg[k-2]=arg[k] end arg[#arg]=nil arg[#arg]=nil +end + +-- End of hack. + +local format, gsub, gmatch, match, find = string.format, string.gsub, string.gmatch, string.match, string.find +local concat = table.concat + +local ownname = environment and environment.ownname or arg[0] or 'mtxrun.lua' +local ownpath = gsub(match(ownname,"^(.+)[\\/].-$") or ".","\\","/") +local owntree = environment and environment.ownpath or ownpath -own.libs = { -- order can be made better +local ownlibs = { -- order can be made better + 'l-lua.lua', + 'l-lpeg.lua', + 'l-function.lua', 'l-string.lua', 'l-table.lua', - 'l-lpeg.lua', 'l-io.lua', 'l-number.lua', 'l-set.lua', @@ -15758,19 +15861,23 @@ own.libs = { -- order can be made better 'l-unicode.lua', 'l-math.lua', + 'util-str.lua', -- code might move to l-string 'util-tab.lua', 'util-sto.lua', - 'util-mrg.lua', - 'util-lua.lua', 'util-prs.lua', 'util-fmt.lua', - 'util-deb.lua', - 'trac-inf.lua', 'trac-set.lua', 'trac-log.lua', - 'trac-pro.lua', + 'trac-inf.lua', -- was before trac-set + 'trac-pro.lua', -- not really needed + 'util-lua.lua', -- indeed here? + 'util-deb.lua', + + 'util-mrg.lua', + 'util-tpl.lua', + 'util-env.lua', 'luat-env.lua', -- can come before inf (as in mkiv) 'lxml-tab.lua', @@ -15780,6 +15887,8 @@ own.libs = { -- order can be made better 'lxml-aux.lua', 'lxml-xml.lua', + 'trac-xml.lua', + 'data-ini.lua', 'data-exp.lua', 'data-env.lua', @@ -15796,35 +15905,20 @@ own.libs = { -- order can be made better -- 'data-bin.lua', 'data-zip.lua', 'data-tre.lua', - 'data-crl.lua', + 'data-sch.lua', 'data-lua.lua', 'data-aux.lua', -- updater 'data-tmf.lua', 'data-lst.lua', + 'util-lib.lua', -- swiglib + 'luat-sta.lua', 'luat-fmt.lua', -} - --- We need this hack till luatex is fixed. --- --- for k,v in pairs(arg) do print(k,v) end - -if arg and (arg[0] == 'luatex' or arg[0] == 'luatex.exe') and arg[1] == "--luaonly" then - arg[-1]=arg[0] arg[0]=arg[2] for k=3,#arg do arg[k-2]=arg[k] end arg[#arg]=nil arg[#arg]=nil -end --- End of hack. - -local format, gsub, gmatch, match, find = string.format, string.gsub, string.gmatch, string.match, string.find -local concat = table.concat - -own.name = (environment and environment.ownname) or arg[0] or 'mtxrun.lua' -own.path = gsub(match(own.name,"^(.+)[\\/].-$") or ".","\\","/") - -local ownpath, owntree = own.path, environment and environment.ownpath or own.path +} -own.list = { +local ownlist = { '.', ownpath , ownpath .. "/../sources", -- HH's development path @@ -15838,13 +15932,21 @@ own.list = { owntree .. "/../../../texmf/tex/context/base", } -if own.path == "." then table.remove(own.list,1) end +if ownpath == "." then table.remove(ownlist,1) end + +own = { + name = ownname, + path = ownpath, + tree = owntree, + list = ownlist, + libs = ownlibs, +} local function locate_libs() - for l=1,#own.libs do - local lib = own.libs[l] - for p =1,#own.list do - local pth = own.list[p] + for l=1,#ownlibs do + local lib = ownlibs[l] + for p =1,#ownlist do + local pth = ownlist[p] local filename = pth .. "/" .. lib local found = lfs.isfile(filename) if found then @@ -15858,8 +15960,8 @@ end local function load_libs() local found = locate_libs() if found then - for l=1,#own.libs do - local filename = found .. "/" .. own.libs[l] + for l=1,#ownlibs do + local filename = found .. "/" .. ownlibs[l] local codeblob = loadfile(filename) if codeblob then codeblob() @@ -15933,55 +16035,85 @@ if not environment.experiments then environment.experiments = e_experiments end local instance = resolvers.reset() local helpinfo = [[ ---script run an mtx script (lua prefered method) (--noquotes), no script gives list ---execute run a script or program (texmfstart method) (--noquotes) ---resolve resolve prefixed arguments ---ctxlua run internally (using preloaded libs) ---internal run script using built in libraries (same as --ctxlua) ---locate locate given filename in database (default) or system (--first --all --detail) - ---autotree use texmf tree cf. env 'texmfstart_tree' or 'texmfstarttree' ---tree=pathtotree use given texmf tree (default file: 'setuptex.tmf') ---environment=name use given (tmf) environment file ---path=runpath go to given path before execution ---ifchanged=filename only execute when given file has changed (md checksum) ---iftouched=old,new only execute when given file has changed (time stamp) - ---makestubs create stubs for (context related) scripts ---removestubs remove stubs (context related) scripts ---stubpath=binpath paths where stubs wil be written ---windows create windows (mswin) stubs ---unix create unix (linux) stubs - ---verbose give a bit more info ---trackers=list enable given trackers ---progname=str format or backend - ---edit launch editor with found file ---launch (--all) launch files like manuals, assumes os support - ---timedrun run a script an time its run ---autogenerate regenerate databases if needed (handy when used to run context in an editor) - ---usekpse use kpse as fallback (when no mkiv and cache installed, often slower) ---forcekpse force using kpse (handy when no mkiv and cache installed but less functionality) - ---prefixes show supported prefixes - ---generate generate file database - ---variables show configuration variables ---configurations show configuration order - ---expand-braces expand complex variable ---expand-path expand variable (resolve paths) ---expand-var expand variable (resolve references) ---show-path show path expansion of ... ---var-value report value of variable ---find-file report file location ---find-path report path of file - ---pattern=str filter variables + + + + mtxrun + ConTeXt TDS Runner Tool + 1.31 + + + + + run an mtx script (lua prefered method) (), no script gives list + run a script or program (texmfstart method) () + resolve prefixed arguments + run internally (using preloaded libs) + run script using built in libraries (same as ) + locate given filename in database (default) or system ( ) + + + use texmf tree cf. env texmfstart_tree or texmfstarttree + use given texmf tree (default file: setuptex.tmf) + use given (tmf) environment file + go to given path before execution + only execute when given file has changed (md checksum) + only execute when given file has changed (time stamp) + + + create stubs for (context related) scripts + remove stubs (context related) scripts + paths where stubs wil be written + create windows (mswin) stubs + create unix (linux) stubs + + + give a bit more info + enable given trackers + format or backend + + + launch editor with found file + launch files like manuals, assumes os support () + + + run a script and time its run + regenerate databases if needed (handy when used to run context in an editor) + + + use kpse as fallback (when no mkiv and cache installed, often slower) + force using kpse (handy when no mkiv and cache installed but less functionality) + + + show supported prefixes + + + generate file database + + + show configuration variables + show configuration order + + + show (known) directives + show (known) trackers + show (known) experiments + + + expand complex variable + expand variable (resolve paths) + expand variable (resolve references) + show path expansion of ... + report value of variable + report file location + report path of file + + + filter variables + + + + ]] local application = logs.application { @@ -16093,7 +16225,8 @@ function runners.execute_script(fullname,internal,nosplit) elseif state == 'skip' then return true elseif state == "run" then - local path, name, suffix, result = file.dirname(fullname), file.basename(fullname), file.extname(fullname), "" + local path, name, suffix = file.splitname(fullname) + local result = "" if path ~= "" then result = fullname elseif name then @@ -16104,7 +16237,7 @@ function runners.execute_script(fullname,internal,nosplit) name = gsub(name,"^script:","") if suffix == "" and runners.registered[name] and runners.registered[name][1] then name = runners.registered[name][1] - suffix = file.extname(name) + suffix = file.suffix(name) end if suffix == "" then -- loop over known suffixes @@ -16131,7 +16264,7 @@ function runners.execute_script(fullname,internal,nosplit) environment.ownscript = result dofile(result) else - local binary = runners.applications[file.extname(result)] + local binary = runners.applications[file.suffix(result)] result = string.quoted(string.unquoted(result)) -- if string.match(result,' ') and not string.match(result,"^\".*\"$") then -- result = '"' .. result .. '"' @@ -16324,7 +16457,7 @@ function resolvers.launch(str) -- maybe we also need to test on mtxrun.launcher.suffix environment -- variable or on windows consult the assoc and ftype vars and such local launchers = runners.launchers[os.platform] if launchers then - local suffix = file.extname(str) if suffix then + local suffix = file.suffix(str) if suffix then local runner = launchers[suffix] if runner then str = runner .. " " .. str end @@ -16383,7 +16516,7 @@ function runners.find_mtx_script(filename) end filename = file.addsuffix(filename,"lua") local basename = file.removesuffix(file.basename(filename)) - local suffix = file.extname(filename) + local suffix = file.suffix(filename) -- qualified path, raw name local fullname = file.is_qualified_path(filename) and io.exists(filename) and filename if fullname and fullname ~= "" then @@ -16438,7 +16571,7 @@ function runners.execute_ctx_script(filename,...) runners.register_arguments(...) local arguments = environment.arguments_after local fullname = runners.find_mtx_script(filename) or "" - if file.extname(fullname) == "cld" then + if file.suffix(fullname) == "cld" then -- handy in editors where we force --autopdf report("running cld script: %s",filename) table.insert(arguments,1,fullname) @@ -16546,6 +16679,21 @@ function runners.timed(action) statistics.timed(action) end +function runners.associate(filename) + os.launch(filename) +end + +function runners.gethelp(filename) + local url = environment.argument("url") + if url and url ~= "" then + local command = string.gsub(environment.argument("command") or "unknown","^%s*\\*(.-)%s*$","%1") + url = utilities.templates.replace(url,{ command = command }) + os.launch(url) + else + report("no --url given") + end +end + -- this is a bit dirty ... first we store the first filename and next we -- split the arguments so that we only see the ones meant for this script -- ... later we will use the second half @@ -16647,8 +16795,18 @@ else end +if e_argument("script") or e_argument("scripts") then + + -- run a script by loading it (using libs), pass args + + runners.loadbase() + if is_mkii_stub then + ok = runners.execute_script(filename,false,true) + else + ok = runners.execute_ctx_script(filename) + end -if e_argument("selfmerge") then +elseif e_argument("selfmerge") then -- embed used libraries @@ -16678,17 +16836,6 @@ elseif e_argument("ctxlua") or e_argument("internal") then runners.loadbase() ok = runners.execute_script(filename,true) -elseif e_argument("script") or e_argument("scripts") then - - -- run a script by loading it (using libs), pass args - - runners.loadbase() - if is_mkii_stub then - ok = runners.execute_script(filename,false,true) - else - ok = runners.execute_ctx_script(filename) - end - elseif e_argument("execute") then -- execute script @@ -16715,6 +16862,14 @@ elseif e_argument("launch") then runners.loadbase() runners.launch_file(filename) +elseif e_argument("associate") then + + runners.associate(filename) + +elseif e_argument("gethelp") then + + runners.gethelp() + elseif e_argument("makestubs") then -- make stubs (depricated) @@ -16806,7 +16961,7 @@ elseif e_argument("find-path") then elseif e_argument("expand-braces") then - -- luatools: runners.execute_ctx_script("mtx-base","--expand-braces",filename + -- luatools: runners.execute_ctx_script("mtx-base","--expand-braces",filename) resolvers.load("nofiles") runners.register_arguments(filename) @@ -16908,6 +17063,23 @@ elseif e_argument("version") then application.version() +elseif e_argument("directives") then + + directives.show() + +elseif e_argument("trackers") then + + trackers.show() + +elseif e_argument("experiments") then + + experiments.show() + +elseif e_argument("exporthelp") then + + runners.loadbase() + application.export(e_argument("exporthelp"),filename) + elseif e_argument("help") or filename=='help' or filename == "" then application.help() @@ -16938,7 +17110,6 @@ elseif environment.files[1] == 'texmfcnf.lua' then -- so that we don't need to l resolvers.listers.configurations() else - runners.loadbase() runners.execute_ctx_script("mtx-base",filename) @@ -16955,4 +17126,4 @@ end if ok == false then ok = 1 elseif ok == true then ok = 0 end -os.exit(ok) +os.exit(ok,true) -- true forces a cleanup in 5.2+ diff --git a/Master/texmf-dist/context/data/scite/scite-context-readme.pdf b/Master/texmf-dist/context/data/scite/scite-context-readme.pdf index 5e68aecc783..05473e597a0 100644 Binary files a/Master/texmf-dist/context/data/scite/scite-context-readme.pdf and b/Master/texmf-dist/context/data/scite/scite-context-readme.pdf differ diff --git a/Master/texmf-dist/context/data/scite/scite-context-readme.tex b/Master/texmf-dist/context/data/scite/scite-context-readme.tex index 0789401b83c..3eb7d422ab2 100644 --- a/Master/texmf-dist/context/data/scite/scite-context-readme.tex +++ b/Master/texmf-dist/context/data/scite/scite-context-readme.tex @@ -113,56 +113,50 @@ {\em This manual is under (re)construction.} -For a long time at \PRAGMA\ we used \TEXEDIT, an editor we'd -written in \MODULA. It had some project management features and -recognized the project structure in \CONTEXT\ documents. Later we -rewrote this to a platform independent reimplementation called -\TEXWORK\ written in \PERLTK\ (not to be confused with the editor -with the plural name). - -In the beginning of the century I can into \SCITE, written by Neil -Hodgson. Although the mentioned editors provide some functionality -not present in \SCITE\ we decided to use that editor because it -frees us from maintaining our own. I ported our \TEX\ and -\METAPOST\ (line based) syntax highlighting to \SCITE\ and got a -lot of others for free. - -After a while I found out that there was an extension interface -written in \LUA. I played with it and wrote a few extensions too. -This pleasant experience later triggered the \LUATEX\ project. - -A decade into the century \SCITE\ got another new feature: you can -write dynamic external lexers in \LUA\ using \LPEG. As in the -meantime \CONTEXT\ has evolved in a \TEX/\LUA\ hybrid, it made -sense to look into this. The result is a couple of lexers that -suit \TEX, \METAPOST\ and \LUA\ usage in \CONTEXT\ \MKIV. As we -also use \XML\ as input and output format a lexer for \XML\ is also -provided. And because \PDF\ is one of the backend formats lexing of -\PDF\ is also implemented. \footnote {In the process some of the -general lexing framework was adapted to suit our demands for speed. -We ship these files as well.} - -In the \CONTEXT\ (standalone) distribution you will find the -relevant files under: +For a long time at \PRAGMA\ we used \TEXEDIT, an editor we'd written in \MODULA. +It had some project management features and recognized the project structure in +\CONTEXT\ documents. Later we rewrote this to a platform independent +reimplementation called \TEXWORK\ written in \PERLTK\ (not to be confused with +the editor with the plural name). + +In the beginning of the century I can into \SCITE, written by Neil Hodgson. +Although the mentioned editors provide some functionality not present in \SCITE\ +we decided to use that editor because it frees us from maintaining our own. I +ported our \TEX\ and \METAPOST\ (line based) syntax highlighting to \SCITE\ and +got a lot of others for free. + +After a while I found out that there was an extension interface written in \LUA. +I played with it and wrote a few extensions too. This pleasant experience later +triggered the \LUATEX\ project. + +A decade into the century \SCITE\ got another new feature: you can write dynamic +external lexers in \LUA\ using \LPEG. As in the meantime \CONTEXT\ has evolved in +a \TEX/\LUA\ hybrid, it made sense to look into this. The result is a couple of +lexers that suit \TEX, \METAPOST\ and \LUA\ usage in \CONTEXT\ \MKIV. As we also +use \XML\ as input and output format a lexer for \XML\ is also provided. And +because \PDF\ is one of the backend formats lexing of \PDF\ is also implemented. +\footnote {In the process some of the general lexing framework was adapted to +suit our demands for speed. We ship these files as well.} + +In the \CONTEXT\ (standalone) distribution you will find the relevant files +under: \starttyping /tex/texmf-context/context/data/scite \stoptyping -Normally a user will not have to dive into the implementation -details but in principle you can tweak the properties files to -suit your purpose. +Normally a user will not have to dive into the implementation details but in +principle you can tweak the properties files to suit your purpose. \subject{The look and feel} -The color scheme that we use is consistent over the lexers but we -use more colors that in the traditional lexing. For instance, -\TEX\ primitives, low level \TEX\ commands, \TEX\ constants, basic -file structure related commands, and user commands all get a -different treatment. When spell checking is turned on, we indicate -unknown words, but also words that are known but might need -checking, for instance because they have an uppercase character. -In \in {figure} [fig:colors] we some of that in practice. +The color scheme that we use is consistent over the lexers but we use more colors +that in the traditional lexing. For instance, \TEX\ primitives, low level \TEX\ +commands, \TEX\ constants, basic file structure related commands, and user +commands all get a different treatment. When spell checking is turned on, we +indicate unknown words, but also words that are known but might need checking, +for instance because they have an uppercase character. In \in {figure} +[fig:colors] we some of that in practice. \placefigure [page] @@ -177,31 +171,29 @@ In \in {figure} [fig:colors] we some of that in practice. \subject{Installing \SCITE} -Installing \SCITE\ is straightforward. We are most familiar with -\MSWINDOWS\ but for other operating systems installation is not -much different. First you need to fetch the archive from: +Installing \SCITE\ is straightforward. We are most familiar with \MSWINDOWS\ but +for other operating systems installation is not much different. First you need to +fetch the archive from: \starttyping www.scintilla.org \stoptyping -The \MSWINDOWS\ binaries are zipped in \type {wscite.zip}, and you -can unzip this in any directory you want as long as you make sure -that the binary ends up in your path or as shortcut on your -desktop. So, say that you install \SCITE\ in: +The \MSWINDOWS\ binaries are zipped in \type {wscite.zip}, and you can unzip this +in any directory you want as long as you make sure that the binary ends up in +your path or as shortcut on your desktop. So, say that you install \SCITE\ in: \starttyping c:\data\system\scite\wscite \stoptyping -You need to add this path to your local path definition. -Installing \SCITE\ to some known place has the advantage that you -can move it around. There are no special dependencies on the -operating system. +You need to add this path to your local path definition. Installing \SCITE\ to +some known place has the advantage that you can move it around. There are no +special dependencies on the operating system. -Next you need to install the lpeg lexers. \footnote {Versions -later than 2.11 will not run on \MSWINDOWS\ 2K. In that case you need -to comment the external lexer import.} These can be fetched from: +Next you need to install the lpeg lexers. \footnote {Versions later than 2.11 +will not run on \MSWINDOWS\ 2K. In that case you need to comment the external +lexer import.} These can be fetched from: \starttyping http://foicica.com/scintillua/ @@ -214,15 +206,17 @@ copy them in the same path as where the regular properties files live. \footnote \SCITE\ that influence the lexers in which case you have to wait till we have update them to suit those changes.} -For \UNIX, one can take a precompiled version as well. Here we -need to split the set of files into: +For \UNIX, one can take a precompiled version as well. Here we might need to split +the set of files into: \starttyping /usr/bin /usr/share/scite \stoptyping -The second path is hard coded in the binary. +The second path is hard coded in the binary and moving all files there probably works +okay. Beware: if you're on a 64 bit system, you need to rename the 64 bit \type {so} +library. If you want to use \CONTEXT, you need to copy the relevant files from @@ -230,21 +224,19 @@ If you want to use \CONTEXT, you need to copy the relevant files from /tex/texmf-context/context/data/scite \stoptyping -to the path were \SCITE\ keeps its property files (\type (*.properties). - -There is a file called \type {SciteGlobal.properties}. At the end -of that file (on \MSWINDOWS\ it is in the path where the Scite binary) -you then add a line to the end: +to the path were \SCITE\ keeps its property files (\type {*.properties}). There +is a file called \type {SciteGlobal.properties}. At the end of that file (on +\MSWINDOWS\ it is in the path where the Scite binary) you then add a line to the +end: \starttyping import scite-context-user \stoptyping -You need to restart \SCITE\ in order to see if things work out as -expected. +You need to restart \SCITE\ in order to see if things work out as expected. -Disabling the external lexer in a recent \SCITE\ is somewhat -tricky. In that case the end of that file looks like: +Disabling the external lexer in a recent \SCITE\ is somewhat tricky. In that case +the end of that file looks like: \starttyping imports.exclude=scite-context-external @@ -254,8 +246,8 @@ import scite-context-user In any case you need to make sure that the user file is loaded last. -After this, things should run as expected (given that \TEX\ runs -at the console as well). +After this, things should run as expected (given that \TEX\ runs at the console +as well). % In order to run the commands needed, we assume that the following programs % are installed: @@ -270,12 +262,11 @@ at the console as well). \subject{Fonts} -The configuration file defaults to the Dejavu fonts. These free -fonts are part of the \CONTEXT\ suite (also known as the -standalone distribution). Of course you can fetch them from \type -{http://dejavu-fonts.org} as well. You have to copy them to where -your operating system expects them. In the suite they are -available in +The configuration file defaults to the Dejavu fonts. These free fonts are part of +the \CONTEXT\ suite (also known as the standalone distribution). Of course you +can fetch them from \type {http://dejavu-fonts.org} as well. You have to copy +them to where your operating system expects them. In the suite they are available +in \starttyping /tex/texmf/fonts/truetype/public/dejavu @@ -283,9 +274,8 @@ available in \subject{An alternative approach} -If for some reason you prefer not to mess with property files in -the main \SCITE\ path, you can follow a different route and -selectively copy files to places. +If for some reason you prefer not to mess with property files in the main \SCITE\ +path, you can follow a different route and selectively copy files to places. The following files are needed for the lpeg based lexer: @@ -308,22 +298,21 @@ lexers/context/data/scite-context-data-metafun.lua lexers/themes/scite-context-theme.lua \stoptyping -The data files are needed because we cannot access property files -from within the lexer. If we could open a file we could use the -property files instead. +The data files are needed because we cannot access property files from within the +lexer. If we could open a file we could use the property files instead. -These files go to the \type {lexers} subpath in your \SCITE\ -installation. Normally this sits in the binary path. The -following files provide some extensions. On \MSWINDOWS\ you can copy -these files to the path where the \SCITE\ binary lives. +These files go to the \type {lexers} subpath in your \SCITE\ installation. +Normally this sits in the binary path. The following files provide some +extensions. On \MSWINDOWS\ you can copy these files to the path where the \SCITE\ +binary lives. \starttyping scite-ctx.lua \stoptyping -Because property files can only be loaded from the same path -where the (user) file loads them you need to copy the following -files to the same path where the loading is defined: +Because property files can only be loaded from the same path where the (user) +file loads them you need to copy the following files to the same path where the +loading is defined: \starttyping scite-context.properties @@ -367,41 +356,37 @@ to the file: SciTEUser.properties \stoptyping -Of course the pragma import is optional. You can comment either the -internal or external variant but there is no reason not to keep them both. +Of course the pragma import is optional. You can comment either the internal or +external variant but there is no reason not to keep them both. \subject{Extensions} -Just a quick not to some extensions. If you select a part of the -text (normally you do this with the shift key pressed) and you hit -\type {Shift-F11}, you get a menu with some options. More (robust) -ones will be provided at some point. +Just a quick not to some extensions. If you select a part of the text (normally +you do this with the shift key pressed) and you hit \type {Shift-F11}, you get a +menu with some options. More (robust) ones will be provided at some point. \subject{Spell checking} -If you want to have spell checking, you need have files with -correct words on each line. The first line of a file determines -the language: +If you want to have spell checking, you need have files with correct words on +each line. The first line of a file determines the language: \starttyping % language=uk \stoptyping -When you use the external lexers, you need to provide some files. -Given that you have a text file with valid words only, you can run -the following script: +When you use the external lexers, you need to provide some files. Given that you +have a text file with valid words only, you can run the following script: \starttyping mtxrun --script scite --words nl uk \stoptyping -This will convert files with names like \type {spell-nl.txt} into -\LUA\ files that you need to copy to the \type {lexers/data} path. -Spell checking happens realtime when you have the language -directive (just add a bogus character to disable it). Wrong words -are colored red, and words that might have a case problem are -colored orange. Recognized words are greyed and words with less -than three characters are ignored. +This will convert files with names like \type {spell-nl.txt} into \LUA\ files +that you need to copy to the \type {lexers/data} path. Spell checking happens +realtime when you have the language directive (just add a bogus character to +disable it). Wrong words are colored red, and words that might have a case +problem are colored orange. Recognized words are greyed and words with less than +three characters are ignored. In the case of internal lexers, the following file is needed: @@ -409,34 +394,32 @@ In the case of internal lexers, the following file is needed: spell-uk.txt \stoptyping -If you use the traditional lexer, this file is taken from the -path determined by the environment variable: +If you use the traditional lexer, this file is taken from the path determined by +the environment variable: \starttyping CTXSPELLPATH \stoptyping -As already mentioned, the lpeg lexer expects them in the data -path. This is because the \LUA\ instance that does the lexing is -rather minimalistic and lacks some libraries as well as cannot -access the main \SCITE\ state. +As already mentioned, the lpeg lexer expects them in the data path. This is +because the \LUA\ instance that does the lexing is rather minimalistic and lacks +some libraries as well as cannot access the main \SCITE\ state. -Spell checking in \type {txt} files is enabled by adding a first -line: +Spell checking in \type {txt} files is enabled by adding a first line: \starttyping [#!-%] language=uk \stoptyping -The first character on that line is one of the four mentioned -between square brackets. So, +The first character on that line is one of the four mentioned between square +brackets. So, \starttyping # language=uk \stoptyping -should work. For \XML\ files there are two methods. You can use the -following (at the start of the file): +should work. For \XML\ files there are two methods. You can use the following (at +the start of the file): \starttyping @@ -459,70 +442,63 @@ In a similar fashion you can drive the interface checking: \subject{Property files} -The internal lexers are controlled by the property files while the -external ones are steered with themes. Unfortunately there is -hardly any access to properties from the external lexer code nor -can we consult the file system and/or run programs like \type -{mtxrun}. This means that we cannot use configuration files in the -\CONTEXT\ distribution directly. Hopefully this changes with -future releases. +The internal lexers are controlled by the property files while the external ones +are steered with themes. Unfortunately there is hardly any access to properties +from the external lexer code nor can we consult the file system and/or run +programs like \type {mtxrun}. This means that we cannot use configuration files +in the \CONTEXT\ distribution directly. Hopefully this changes with future +releases. \subject{The external lexers} -These are the more advanced. They provide more detail and the \CONTEXT\ -lexer also supports nested \METAPOST\ and \LUA. Currently there is no -detailed configuration but this might change once they are stable. - -The external lexers operate on documents while the internal ones -operate on lines. This can make the external lexers slow on large -documents. We've optimized the code somewhat for speed and memory -consumption but there's only so much one can do. While lexing each -change in style needs a small table but allocating and garbage -collecting many small tables comes at a price. Of course in -practice this probably gets unnoticed. \footnote {I wrote the code -in 2011 on a more than 5 years old Dell M90 laptop, so I suppose -that speed is less an issue now.} - -In principle the external lexers can be used with \type -{textadept} which also uses \type {scintilla}. Actually, support -for lpeg lexing originates in \type {textadept}. Currently \type -{textadept} lacks a couple of features I like about \SCITE\ (for -instance it has no realtime logpane) and it's also still changing. -At some point the \CONTEXT\ distribution might ship with files -for \type {textadept} as well. - -The external lpeg lexers work okay with the \MSWINDOWS\ and -\LINUX\ versions of \SCITE, but unfortunately at the time of -writing this, the \LUA\ library that is needed is not available -for the \MACOSX\ version of \SCITE. Also, due to the fact that the -lexing framework is rather isolated, there are some issues that -cannot be addressed in the properly, at least not currently. - -In addition to \CONTEXT\ and \METAFUN\ lexing a \LUA\ lexer is -also provided so that we can handle \CONTEXT\ \LUA\ Document -(\CLD) files too. There is also an \XML\ lexer. This one also -provides spell checking. The \PDF\ lexer tries to do a good job on -\PDF\ files, but it has some limitations. There is also a simple -text file lexer that does spell checking. - -Don't worry if you see an orange rectangle in your \TEX\ or \XML\ -document. This indicates that there is a special space character -there, for instance \type {0xA0}, the nonbreakable space. Of course -we assume that you use \UTF8 as input encoding. +These are the more advanced. They provide more detail and the \CONTEXT\ lexer +also supports nested \METAPOST\ and \LUA. Currently there is no detailed +configuration but this might change once they are stable. + +The external lexers operate on documents while the internal ones operate on +lines. This can make the external lexers slow on large documents. We've optimized +the code somewhat for speed and memory consumption but there's only so much one +can do. While lexing each change in style needs a small table but allocating and +garbage collecting many small tables comes at a price. Of course in practice this +probably gets unnoticed. \footnote {I wrote the code in 2011 on a more than 5 +years old Dell M90 laptop, so I suppose that speed is less an issue now.} + +In principle the external lexers can be used with \type {textadept} which also +uses \type {scintilla}. Actually, support for lpeg lexing originates in \type +{textadept}. Currently \type {textadept} lacks a couple of features I like about +\SCITE\ (for instance it has no realtime logpane) and it's also still changing. +At some point the \CONTEXT\ distribution might ship with files for \type +{textadept} as well. + +The external lpeg lexers work okay with the \MSWINDOWS\ and \LINUX\ versions of +\SCITE, but unfortunately at the time of writing this, the \LUA\ library that is +needed is not available for the \MACOSX\ version of \SCITE. Also, due to the fact +that the lexing framework is rather isolated, there are some issues that cannot +be addressed in the properly, at least not currently. + +In addition to \CONTEXT\ and \METAFUN\ lexing a \LUA\ lexer is also provided so +that we can handle \CONTEXT\ \LUA\ Document (\CLD) files too. There is also an +\XML\ lexer. This one also provides spell checking. The \PDF\ lexer tries to do a +good job on \PDF\ files, but it has some limitations. There is also a simple text +file lexer that does spell checking. + +Don't worry if you see an orange rectangle in your \TEX\ or \XML\ document. This +indicates that there is a special space character there, for instance \type +{0xA0}, the nonbreakable space. Of course we assume that you use \UTF8 as input +encoding. \subject{The internal lexers} -\SCITE\ has quite some built in lexers. A lexer is responsible for -highlighting the syntax of your document. The way a \TEX\ file is -treated is configured in the file: +\SCITE\ has quite some built in lexers. A lexer is responsible for highlighting +the syntax of your document. The way a \TEX\ file is treated is configured in the +file: \starttyping tex.properties \stoptyping -You can edit this file to your needs using the menu entry under -\type {options} in the top bar. In this file, the following -settings apply to the \TEX\ lexer: +You can edit this file to your needs using the menu entry under \type {options} +in the top bar. In this file, the following settings apply to the \TEX\ lexer: \starttyping lexer.tex.interface.default=0 @@ -531,17 +507,16 @@ lexer.tex.comment.process=0 lexer.tex.auto.if=1 \stoptyping -The option \type {lexer.tex.interface.default} determines the way -keywords are highlighted. You can control the interface from your -document as well, which makes more sense that editing the -configuration file each time. +The option \type {lexer.tex.interface.default} determines the way keywords are +highlighted. You can control the interface from your document as well, which +makes more sense that editing the configuration file each time. \starttyping % interface=all|tex|nl|en|de|cz|it|ro|latex \stoptyping -The values in the properties file and the keywords in the preamble -line have the following meaning: +The values in the properties file and the keywords in the preamble line have the +following meaning: \starttabulate[|lT|lT|p|] \NC 0 \NC all \NC all commands (preceded by a backslash) \NC \NR @@ -555,19 +530,19 @@ line have the following meaning: \NC 8 \NC latex \NC \LATEX\ (apart from packages) \NC \NR \stoptabulate -The configuration file is set up in such a way that you can easily -add more keywords to the lists. The keywords for the second and -higher interfaces are defined in their own properties files. If -you're curious about the way this is configures, you can peek into -the property files that start with \type {scite-context}. When you -have \CONTEXT\ installed you can generate configuration files with +The configuration file is set up in such a way that you can easily add more +keywords to the lists. The keywords for the second and higher interfaces are +defined in their own properties files. If you're curious about the way this is +configures, you can peek into the property files that start with \type +{scite-context}. When you have \CONTEXT\ installed you can generate configuration +files with \starttyping mtxrun --script interface --scite \stoptyping -You need to make sure that you move the result to the right place so best -not mess around with this command and use the files from the distribution. +You need to make sure that you move the result to the right place so best not +mess around with this command and use the files from the distribution. Back to the properties in \type {tex.properties}. You can disable keyword coloring alltogether with: @@ -588,8 +563,8 @@ in a uniform color. But, when set to one, you will get as much colors as a \TEX\ source. It's a matter of taste what you choose. The lexer tries to cope with the \TEX\ syntax as good as possible and takes for -instance care of the funny \type {^^} notation. A special treatment is -applied to so called \type {\if}'s: +instance care of the funny \type {^^} notation. A special treatment is applied to +so called \type {\if}'s: \starttyping lexer.tex.auto.if=1 @@ -597,15 +572,15 @@ lexer.tex.auto.if=1 This is the default setting. When set to one, all \type {\ifwhatever}'s will be seen as a command. When set to zero, only the primitive \type {\if}'s will be -treated. In order not to confuse you, when this property is set to one, the -lexer will not color an \type {\ifwhatever} that follows an \type {\newif}. +treated. In order not to confuse you, when this property is set to one, the lexer +will not color an \type {\ifwhatever} that follows an \type {\newif}. \subject{The \METAPOST\ lexer} The \METAPOST\ lexer is set up slightly different from its \TEX\ counterpart, -first of all because \METAPOST\ is more a language that \TEX. As with the -\TEX\ lexer, we can control the interpretation of identifiers. The \METAPOST\ -specific configuration file is: +first of all because \METAPOST\ is more a language that \TEX. As with the \TEX\ +lexer, we can control the interpretation of identifiers. The \METAPOST\ specific +configuration file is: \starttyping metapost.properties @@ -674,13 +649,13 @@ find \type {mtxrun} in: or in a similar path that suits the operating system that you use. -When you hit \type{CTRL-12} your document will be processed. Take a look at -the \type {Tools} menu to see what more is provided. +When you hit \type{CTRL-12} your document will be processed. Take a look at the +\type {Tools} menu to see what more is provided. \subject{Extensions (using \LUA)} -When the \LUA\ extensions are loaded, you will see a message -in the log pane that looks like: +When the \LUA\ extensions are loaded, you will see a message in the log pane that +looks like: \starttyping - see scite-ctx.properties for configuring info @@ -710,9 +685,9 @@ This message tells you what extras are available. \subject{Templates} -There is an experimental template mechanism. One option is to define -templates in a properties file. The property file \type -{scite-ctx-context} contains definitions like: +There is an experimental template mechanism. One option is to define templates in +a properties file. The property file \type {scite-ctx-context} contains +definitions like: \starttyping command.25.$(file.patterns.context)=insert_template \ @@ -757,9 +732,9 @@ ctx.template.structure.itemize.example=\ \n \stoptyping -For larger projects it makes sense to keep templates with the -project. In one of our projects we have a directory in the -path where the project files are kept which holds template files: +For larger projects it makes sense to keep templates with the project. In one of +our projects we have a directory in the path where the project files are kept +which holds template files: \starttyping ..../ctx-templates/achtergronden.xml @@ -777,13 +752,13 @@ ctx.template.mathadore.achtergronden.file=smt-achtergronden.xml ctx.template.mathadore.bewijs.file=smt-bewijs.xml \stoptyping -However, when no such menu is defined, we will automatically scan -the directory and build the menu without user intervention. +However, when no such menu is defined, we will automatically scan the directory +and build the menu without user intervention. \subject{Using \SCITE} -The following keybindings are available in \SCITE. Most of this -list is taken from the on|-|line help pages. +The following keybindings are available in \SCITE. Most of this list is taken +from the on|-|line help pages. \startbuffer[keybindings] \starttabulate[|l|p|] diff --git a/Master/texmf-dist/doc/man/man1/context.man1.pdf b/Master/texmf-dist/doc/man/man1/context.man1.pdf index 8863439d028..8af8b7e3425 100644 Binary files a/Master/texmf-dist/doc/man/man1/context.man1.pdf and b/Master/texmf-dist/doc/man/man1/context.man1.pdf differ diff --git a/Master/texmf-dist/doc/man/man1/ctxtools.man1.pdf b/Master/texmf-dist/doc/man/man1/ctxtools.man1.pdf index b5e180f96c1..184fb1bd791 100644 Binary files a/Master/texmf-dist/doc/man/man1/ctxtools.man1.pdf and b/Master/texmf-dist/doc/man/man1/ctxtools.man1.pdf differ diff --git a/Master/texmf-dist/doc/man/man1/luatools.man1.pdf b/Master/texmf-dist/doc/man/man1/luatools.man1.pdf index 52b46b6bd83..07e8a332214 100644 Binary files a/Master/texmf-dist/doc/man/man1/luatools.man1.pdf and b/Master/texmf-dist/doc/man/man1/luatools.man1.pdf differ diff --git a/Master/texmf-dist/doc/man/man1/mtx-babel.man1.pdf b/Master/texmf-dist/doc/man/man1/mtx-babel.man1.pdf index aa026870301..e99515bd1a3 100644 Binary files a/Master/texmf-dist/doc/man/man1/mtx-babel.man1.pdf and b/Master/texmf-dist/doc/man/man1/mtx-babel.man1.pdf differ diff --git a/Master/texmf-dist/doc/man/man1/mtx-base.man1.pdf b/Master/texmf-dist/doc/man/man1/mtx-base.man1.pdf index 447b9341262..0128937e218 100644 Binary files a/Master/texmf-dist/doc/man/man1/mtx-base.man1.pdf and b/Master/texmf-dist/doc/man/man1/mtx-base.man1.pdf differ diff --git a/Master/texmf-dist/doc/man/man1/mtx-cache.man1.pdf b/Master/texmf-dist/doc/man/man1/mtx-cache.man1.pdf index 24ae98e4a16..d5ac026d450 100644 Binary files a/Master/texmf-dist/doc/man/man1/mtx-cache.man1.pdf and b/Master/texmf-dist/doc/man/man1/mtx-cache.man1.pdf differ diff --git a/Master/texmf-dist/doc/man/man1/mtx-chars.man1.pdf b/Master/texmf-dist/doc/man/man1/mtx-chars.man1.pdf index ff2eca72000..db7212c456c 100644 Binary files a/Master/texmf-dist/doc/man/man1/mtx-chars.man1.pdf and b/Master/texmf-dist/doc/man/man1/mtx-chars.man1.pdf differ diff --git a/Master/texmf-dist/doc/man/man1/mtx-check.man1.pdf b/Master/texmf-dist/doc/man/man1/mtx-check.man1.pdf index f8266ee15e3..ed13c231255 100644 Binary files a/Master/texmf-dist/doc/man/man1/mtx-check.man1.pdf and b/Master/texmf-dist/doc/man/man1/mtx-check.man1.pdf differ diff --git a/Master/texmf-dist/doc/man/man1/mtx-colors.man1.pdf b/Master/texmf-dist/doc/man/man1/mtx-colors.man1.pdf index 1c96774eee8..f50f39506a3 100644 Binary files a/Master/texmf-dist/doc/man/man1/mtx-colors.man1.pdf and b/Master/texmf-dist/doc/man/man1/mtx-colors.man1.pdf differ diff --git a/Master/texmf-dist/doc/man/man1/mtx-context.man1.pdf b/Master/texmf-dist/doc/man/man1/mtx-context.man1.pdf index 1fe6753ac13..26128939222 100644 Binary files a/Master/texmf-dist/doc/man/man1/mtx-context.man1.pdf and b/Master/texmf-dist/doc/man/man1/mtx-context.man1.pdf differ diff --git a/Master/texmf-dist/doc/man/man1/mtx-epub.man1.pdf b/Master/texmf-dist/doc/man/man1/mtx-epub.man1.pdf index f444aadf814..0061e4874e3 100644 Binary files a/Master/texmf-dist/doc/man/man1/mtx-epub.man1.pdf and b/Master/texmf-dist/doc/man/man1/mtx-epub.man1.pdf differ diff --git a/Master/texmf-dist/doc/man/man1/mtx-fcd.man1.pdf b/Master/texmf-dist/doc/man/man1/mtx-fcd.man1.pdf index 2809a922631..544d803da77 100644 Binary files a/Master/texmf-dist/doc/man/man1/mtx-fcd.man1.pdf and b/Master/texmf-dist/doc/man/man1/mtx-fcd.man1.pdf differ diff --git a/Master/texmf-dist/doc/man/man1/mtx-flac.man1.pdf b/Master/texmf-dist/doc/man/man1/mtx-flac.man1.pdf index 1d1cb8b58cb..f3942c3a4a7 100644 Binary files a/Master/texmf-dist/doc/man/man1/mtx-flac.man1.pdf and b/Master/texmf-dist/doc/man/man1/mtx-flac.man1.pdf differ diff --git a/Master/texmf-dist/doc/man/man1/mtx-fonts.man1.pdf b/Master/texmf-dist/doc/man/man1/mtx-fonts.man1.pdf index b5b1319c13d..5ec3b576e80 100644 Binary files a/Master/texmf-dist/doc/man/man1/mtx-fonts.man1.pdf and b/Master/texmf-dist/doc/man/man1/mtx-fonts.man1.pdf differ diff --git a/Master/texmf-dist/doc/man/man1/mtx-grep.man1.pdf b/Master/texmf-dist/doc/man/man1/mtx-grep.man1.pdf index db1dd5bb518..dd72717ee73 100644 Binary files a/Master/texmf-dist/doc/man/man1/mtx-grep.man1.pdf and b/Master/texmf-dist/doc/man/man1/mtx-grep.man1.pdf differ diff --git a/Master/texmf-dist/doc/man/man1/mtx-interface.man1.pdf b/Master/texmf-dist/doc/man/man1/mtx-interface.man1.pdf index f77c0d123c9..e5b55f70c7c 100644 Binary files a/Master/texmf-dist/doc/man/man1/mtx-interface.man1.pdf and b/Master/texmf-dist/doc/man/man1/mtx-interface.man1.pdf differ diff --git a/Master/texmf-dist/doc/man/man1/mtx-metapost.man1.pdf b/Master/texmf-dist/doc/man/man1/mtx-metapost.man1.pdf index e8a62b94201..6d230171cda 100644 --- a/Master/texmf-dist/doc/man/man1/mtx-metapost.man1.pdf +++ b/Master/texmf-dist/doc/man/man1/mtx-metapost.man1.pdf @@ -66,12 +66,12 @@ endobj - -2013-04-08T02:39:55+02:00 -2013-04-08T02:39:55+02:00 + +2013-04-09T19:03:22+02:00 +2013-04-09T19:03:22+02:00 groff version 1.22.2 - -Untitled + +Untitled @@ -81,8 +81,8 @@ endstream endobj 2 0 obj <>endobj xref 0 15 @@ -103,7 +103,7 @@ xref 0000001681 00000 n trailer << /Size 15 /Root 1 0 R /Info 2 0 R -/ID [<96EA08B77B0ED8E6BBE37608502BC60A><96EA08B77B0ED8E6BBE37608502BC60A>] +/ID [<6E1A53A428574766A673AD162625B182><6E1A53A428574766A673AD162625B182>] >> startxref 3258 diff --git a/Master/texmf-dist/doc/man/man1/mtx-metatex.man1.pdf b/Master/texmf-dist/doc/man/man1/mtx-metatex.man1.pdf index b62f414af46..da00bfe3b44 100644 Binary files a/Master/texmf-dist/doc/man/man1/mtx-metatex.man1.pdf and b/Master/texmf-dist/doc/man/man1/mtx-metatex.man1.pdf differ diff --git a/Master/texmf-dist/doc/man/man1/mtx-modules.man1.pdf b/Master/texmf-dist/doc/man/man1/mtx-modules.man1.pdf index 19ef7232d0f..8cbcff95c9a 100644 Binary files a/Master/texmf-dist/doc/man/man1/mtx-modules.man1.pdf and b/Master/texmf-dist/doc/man/man1/mtx-modules.man1.pdf differ diff --git a/Master/texmf-dist/doc/man/man1/mtx-package.man1.pdf b/Master/texmf-dist/doc/man/man1/mtx-package.man1.pdf index 722ca53bc61..802ade9c9d2 100644 Binary files a/Master/texmf-dist/doc/man/man1/mtx-package.man1.pdf and b/Master/texmf-dist/doc/man/man1/mtx-package.man1.pdf differ diff --git a/Master/texmf-dist/doc/man/man1/mtx-patterns.man1.pdf b/Master/texmf-dist/doc/man/man1/mtx-patterns.man1.pdf index 20964c47f9e..f72a612386e 100644 Binary files a/Master/texmf-dist/doc/man/man1/mtx-patterns.man1.pdf and b/Master/texmf-dist/doc/man/man1/mtx-patterns.man1.pdf differ diff --git a/Master/texmf-dist/doc/man/man1/mtx-pdf.man1.pdf b/Master/texmf-dist/doc/man/man1/mtx-pdf.man1.pdf index f5662e8ae4a..4757b9d0dcf 100644 Binary files a/Master/texmf-dist/doc/man/man1/mtx-pdf.man1.pdf and b/Master/texmf-dist/doc/man/man1/mtx-pdf.man1.pdf differ diff --git a/Master/texmf-dist/doc/man/man1/mtx-profile.man1.pdf b/Master/texmf-dist/doc/man/man1/mtx-profile.man1.pdf index 74f56d7b253..6446e52fd00 100644 --- a/Master/texmf-dist/doc/man/man1/mtx-profile.man1.pdf +++ b/Master/texmf-dist/doc/man/man1/mtx-profile.man1.pdf @@ -72,12 +72,12 @@ endobj - -2013-04-08T02:39:54+02:00 -2013-04-08T02:39:54+02:00 + +2013-04-09T19:03:20+02:00 +2013-04-09T19:03:20+02:00 groff version 1.22.2 - -Untitled + +Untitled @@ -87,8 +87,8 @@ endstream endobj 2 0 obj <>endobj xref 0 16 @@ -110,7 +110,7 @@ xref 0000001640 00000 n trailer << /Size 16 /Root 1 0 R /Info 2 0 R -/ID [<12D5B0B8D4EEBD69B189A1F1C48BFAD7><12D5B0B8D4EEBD69B189A1F1C48BFAD7>] +/ID [<5E717475DBAC500C93CD936280DBF96B><5E717475DBAC500C93CD936280DBF96B>] >> startxref 3217 diff --git a/Master/texmf-dist/doc/man/man1/mtx-rsync.man1.pdf b/Master/texmf-dist/doc/man/man1/mtx-rsync.man1.pdf index d9c41f3c033..6be6d17d1ee 100644 Binary files a/Master/texmf-dist/doc/man/man1/mtx-rsync.man1.pdf and b/Master/texmf-dist/doc/man/man1/mtx-rsync.man1.pdf differ diff --git a/Master/texmf-dist/doc/man/man1/mtx-scite.man1.pdf b/Master/texmf-dist/doc/man/man1/mtx-scite.man1.pdf index 28b5b948d02..cd33ad8ceab 100644 Binary files a/Master/texmf-dist/doc/man/man1/mtx-scite.man1.pdf and b/Master/texmf-dist/doc/man/man1/mtx-scite.man1.pdf differ diff --git a/Master/texmf-dist/doc/man/man1/mtx-server.man1.pdf b/Master/texmf-dist/doc/man/man1/mtx-server.man1.pdf index f6bf45b2fac..fc4894ac3bf 100644 --- a/Master/texmf-dist/doc/man/man1/mtx-server.man1.pdf +++ b/Master/texmf-dist/doc/man/man1/mtx-server.man1.pdf @@ -69,12 +69,12 @@ endobj - -2013-04-08T02:39:56+02:00 -2013-04-08T02:39:56+02:00 + +2013-04-09T19:03:22+02:00 +2013-04-09T19:03:22+02:00 groff version 1.22.2 - -Untitled + +Untitled @@ -84,8 +84,8 @@ endstream endobj 2 0 obj <>endobj xref 0 15 @@ -106,7 +106,7 @@ xref 0000001722 00000 n trailer << /Size 15 /Root 1 0 R /Info 2 0 R -/ID [<6167DA2FECF3699F8F1F3F428D8353B0><6167DA2FECF3699F8F1F3F428D8353B0>] +/ID [] >> startxref 3299 diff --git a/Master/texmf-dist/doc/man/man1/mtx-texworks.man1.pdf b/Master/texmf-dist/doc/man/man1/mtx-texworks.man1.pdf index 360fd07af93..e8fd492cbe5 100644 Binary files a/Master/texmf-dist/doc/man/man1/mtx-texworks.man1.pdf and b/Master/texmf-dist/doc/man/man1/mtx-texworks.man1.pdf differ diff --git a/Master/texmf-dist/doc/man/man1/mtx-timing.man1.pdf b/Master/texmf-dist/doc/man/man1/mtx-timing.man1.pdf index 679155e28be..bf9d09a89c3 100644 Binary files a/Master/texmf-dist/doc/man/man1/mtx-timing.man1.pdf and b/Master/texmf-dist/doc/man/man1/mtx-timing.man1.pdf differ diff --git a/Master/texmf-dist/doc/man/man1/mtx-tools.man1.pdf b/Master/texmf-dist/doc/man/man1/mtx-tools.man1.pdf index 715534fecd1..bf3904f830d 100644 Binary files a/Master/texmf-dist/doc/man/man1/mtx-tools.man1.pdf and b/Master/texmf-dist/doc/man/man1/mtx-tools.man1.pdf differ diff --git a/Master/texmf-dist/doc/man/man1/mtx-unzip.man1.pdf b/Master/texmf-dist/doc/man/man1/mtx-unzip.man1.pdf index 551f2e05952..1dbe6c75a6b 100644 --- a/Master/texmf-dist/doc/man/man1/mtx-unzip.man1.pdf +++ b/Master/texmf-dist/doc/man/man1/mtx-unzip.man1.pdf @@ -67,12 +67,12 @@ endobj - -2013-04-08T02:39:51+02:00 -2013-04-08T02:39:51+02:00 + +2013-04-09T19:03:16+02:00 +2013-04-09T19:03:16+02:00 groff version 1.22.2 - -Untitled + +Untitled @@ -82,8 +82,8 @@ endstream endobj 2 0 obj <>endobj xref 0 15 @@ -104,7 +104,7 @@ xref 0000001568 00000 n trailer << /Size 15 /Root 1 0 R /Info 2 0 R -/ID [] +/ID [] >> startxref 3145 diff --git a/Master/texmf-dist/doc/man/man1/mtx-update.man1.pdf b/Master/texmf-dist/doc/man/man1/mtx-update.man1.pdf index a21271c505f..dfae6747fab 100644 Binary files a/Master/texmf-dist/doc/man/man1/mtx-update.man1.pdf and b/Master/texmf-dist/doc/man/man1/mtx-update.man1.pdf differ diff --git a/Master/texmf-dist/doc/man/man1/mtx-watch.man1.pdf b/Master/texmf-dist/doc/man/man1/mtx-watch.man1.pdf index fe08191c9ff..3475ebeb3e6 100644 Binary files a/Master/texmf-dist/doc/man/man1/mtx-watch.man1.pdf and b/Master/texmf-dist/doc/man/man1/mtx-watch.man1.pdf differ diff --git a/Master/texmf-dist/doc/man/man1/mtxrun.man1.pdf b/Master/texmf-dist/doc/man/man1/mtxrun.man1.pdf index 5ed7141e9f4..8d02472ec5c 100644 Binary files a/Master/texmf-dist/doc/man/man1/mtxrun.man1.pdf and b/Master/texmf-dist/doc/man/man1/mtxrun.man1.pdf differ diff --git a/Master/texmf-dist/doc/man/man1/pstopdf.man1.pdf b/Master/texmf-dist/doc/man/man1/pstopdf.man1.pdf index 3c67c3ebcd4..6a6ab42d104 100644 Binary files a/Master/texmf-dist/doc/man/man1/pstopdf.man1.pdf and b/Master/texmf-dist/doc/man/man1/pstopdf.man1.pdf differ diff --git a/Master/texmf-dist/doc/man/man1/texexec.man1.pdf b/Master/texmf-dist/doc/man/man1/texexec.man1.pdf index 5823672f6af..4cae23799ea 100644 Binary files a/Master/texmf-dist/doc/man/man1/texexec.man1.pdf and b/Master/texmf-dist/doc/man/man1/texexec.man1.pdf differ diff --git a/Master/texmf-dist/doc/man/man1/texmfstart.man1.pdf b/Master/texmf-dist/doc/man/man1/texmfstart.man1.pdf index 9e34aaa82b2..3907022e379 100644 Binary files a/Master/texmf-dist/doc/man/man1/texmfstart.man1.pdf and b/Master/texmf-dist/doc/man/man1/texmfstart.man1.pdf differ diff --git a/Master/texmf-dist/scripts/context/lua/mtxrun.lua b/Master/texmf-dist/scripts/context/lua/mtxrun.lua index 00f63a5791d..d337ed75f74 100755 --- a/Master/texmf-dist/scripts/context/lua/mtxrun.lua +++ b/Master/texmf-dist/scripts/context/lua/mtxrun.lua @@ -56,7 +56,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-lua"] = package.loaded["l-lua"] or true --- original size: 10048, stripped down to: 5684 +-- original size: 10352, stripped down to: 5955 if not modules then modules={} end modules ['l-lua']={ version=1.001, @@ -247,7 +247,7 @@ local function loadedbypath(name,rawname,paths,islib,what) end local function notloaded(name) if helpers.trace then - helpers.report("? unable to locate library '%s'",name) + helpers.report("unable to locate library '%s'",name) end end helpers.loadedaslib=loadedaslib @@ -278,6 +278,16 @@ function helpers.loaded(name) end return notloaded(name) end +function helpers.unload(name) + if helpers.trace then + if package.loaded[name] then + helpers.report("unloading library '%s', %s",name,"done") + else + helpers.report("unloading library '%s', %s",name,"not loaded") + end + end + package.loaded[name]=nil +end end -- of closure @@ -12632,7 +12642,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-res"] = package.loaded["data-res"] or true --- original size: 60821, stripped down to: 42503 +-- original size: 60857, stripped down to: 42496 if not modules then modules={} end modules ['data-res']={ version=1.001, @@ -12807,7 +12817,7 @@ local slash=P("/") local pathexpressionpattern=Cs ( Cc("^")*( Cc("%")*S(".-")+slash^2*P(-1)/"/.*" -+slash^2/"/[^/]*/*"+(1-slash)*P(-1)*Cc("/")+P(1) ++slash^2/"/"+(1-slash)*P(-1)*Cc("/")+P(1) )^1*Cc("$") ) local cache={} @@ -15403,7 +15413,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-lib"] = package.loaded["util-lib"] or true --- original size: 8911, stripped down to: 4216 +-- original size: 10762, stripped down to: 5269 if not modules then modules={} end modules ['util-lib']={ version=1.001, @@ -15418,6 +15428,7 @@ local findfile,findfiles=resolvers and resolvers.findfile,resolvers and resolver local loaded=package.loaded local report_swiglib=logs.reporter("swiglib") local trace_swiglib=false trackers.register("resolvers.swiglib",function(v) trace_swiglib=v end) +local done=false local function requireswiglib(required,version) local library=loaded[required] if library==nil then @@ -15426,32 +15437,48 @@ local function requireswiglib(required,version) local required_base=nameonly(required_full) local required_name=required_base.."."..os.libsuffix local version=type(version)=="string" and version~="" and version or false + local engine=environment.ownmain or false + if trace_swiglib and not done then + local list=resolvers.expandedpathlistfromvariable("lib") + for i=1,#list do + report_swiglib("tds path %i: %s",i,list[i]) + end + end + local function found(locate,asked_library,how,...) + if trace_swiglib then + report_swiglib("checking %s: %a",how,asked_library) + end + return locate(asked_library,...) + end local function check(locate,...) - local found_library=nil + local found=nil if version then local asked_library=joinfile(required_path,version,required_name) if trace_swiglib then report_swiglib("checking %s: %a","with version",asked_library) end - found_library=locate(asked_library,...) - if not found_library or found_library==""then - asked_library=joinfile(required_path,required_name) - if trace_swiglib then - report_swiglib("checking %s: %a","without version",asked_library) - end - found_library=locate(asked_library,...) - end - else + found=locate(asked_library,...) + end + if not found or found=="" then local asked_library=joinfile(required_path,required_name) if trace_swiglib then - report_swiglib("checking %s: %a","without version",asked_library) + report_swiglib("checking %s: %a","with version",asked_library) end - found_library=locate(asked_library,...) + found=locate(asked_library,...) end - return found_library and found_library~="" and found_library or false + return found and found~="" and found or false end - local found_library=findfile and check(findfile,"lib") - if findfiles and not found_library then + local function attempt(checkpattern) + if trace_swiglib then + report_swiglib("checking tds lib paths strictly") + end + local found=findfile and check(findfile,"lib") + if found and (not checkpattern or find(found,checkpattern)) then + return found + end + if trace_swiglib then + report_swiglib("checking tds lib paths with wildcard") + end local asked_library=joinfile(required_path,".*",required_name) if trace_swiglib then report_swiglib("checking %s: %a","latest version",asked_library) @@ -15459,18 +15486,38 @@ local function requireswiglib(required,version) local list=findfiles(asked_library,"lib",true) if list and #list>0 then table.sort(list) - found_library=list[#list] + local found=list[#list] + if found and (not checkpattern or find(found,checkpattern)) then + return found + end + end + if trace_swiglib then + report_swiglib("checking clib paths") end - end - if not found_library then package.extraclibpath(environment.ownpath) local paths=package.clibpaths() for i=1,#paths do - local found_library=check(lfs.isfile) - if found_library then - break + local found=check(lfs.isfile) + if found and (not checkpattern or find(found,checkpattern)) then + return found end end + return false + end + local found_library=nil + if engine then + if trace_swiglib then + report_swiglib("attemp 1, engine %a",engine) + end + found_library=attempt("/"..engine.."/") + if not found_library then + if trace_swiglib then + report_swiglib("attemp 2, no engine",asked_library) + end + found_library=attempt() + end + else + found_library=attempt() end if not found_library then if trace_swiglib then @@ -15773,8 +15820,8 @@ end -- of closure -- used libraries : l-lua.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-md5.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-sto.lua util-prs.lua util-fmt.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-mrg.lua util-tpl.lua util-env.lua luat-env.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua -- skipped libraries : - --- original bytes : 658276 --- stripped bytes : 241564 +-- original bytes : 660467 +-- stripped bytes : 242438 -- end library merge diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/mtxrun.lua b/Master/texmf-dist/scripts/context/stubs/mswin/mtxrun.lua index 00f63a5791d..d337ed75f74 100644 --- a/Master/texmf-dist/scripts/context/stubs/mswin/mtxrun.lua +++ b/Master/texmf-dist/scripts/context/stubs/mswin/mtxrun.lua @@ -56,7 +56,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-lua"] = package.loaded["l-lua"] or true --- original size: 10048, stripped down to: 5684 +-- original size: 10352, stripped down to: 5955 if not modules then modules={} end modules ['l-lua']={ version=1.001, @@ -247,7 +247,7 @@ local function loadedbypath(name,rawname,paths,islib,what) end local function notloaded(name) if helpers.trace then - helpers.report("? unable to locate library '%s'",name) + helpers.report("unable to locate library '%s'",name) end end helpers.loadedaslib=loadedaslib @@ -278,6 +278,16 @@ function helpers.loaded(name) end return notloaded(name) end +function helpers.unload(name) + if helpers.trace then + if package.loaded[name] then + helpers.report("unloading library '%s', %s",name,"done") + else + helpers.report("unloading library '%s', %s",name,"not loaded") + end + end + package.loaded[name]=nil +end end -- of closure @@ -12632,7 +12642,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-res"] = package.loaded["data-res"] or true --- original size: 60821, stripped down to: 42503 +-- original size: 60857, stripped down to: 42496 if not modules then modules={} end modules ['data-res']={ version=1.001, @@ -12807,7 +12817,7 @@ local slash=P("/") local pathexpressionpattern=Cs ( Cc("^")*( Cc("%")*S(".-")+slash^2*P(-1)/"/.*" -+slash^2/"/[^/]*/*"+(1-slash)*P(-1)*Cc("/")+P(1) ++slash^2/"/"+(1-slash)*P(-1)*Cc("/")+P(1) )^1*Cc("$") ) local cache={} @@ -15403,7 +15413,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-lib"] = package.loaded["util-lib"] or true --- original size: 8911, stripped down to: 4216 +-- original size: 10762, stripped down to: 5269 if not modules then modules={} end modules ['util-lib']={ version=1.001, @@ -15418,6 +15428,7 @@ local findfile,findfiles=resolvers and resolvers.findfile,resolvers and resolver local loaded=package.loaded local report_swiglib=logs.reporter("swiglib") local trace_swiglib=false trackers.register("resolvers.swiglib",function(v) trace_swiglib=v end) +local done=false local function requireswiglib(required,version) local library=loaded[required] if library==nil then @@ -15426,32 +15437,48 @@ local function requireswiglib(required,version) local required_base=nameonly(required_full) local required_name=required_base.."."..os.libsuffix local version=type(version)=="string" and version~="" and version or false + local engine=environment.ownmain or false + if trace_swiglib and not done then + local list=resolvers.expandedpathlistfromvariable("lib") + for i=1,#list do + report_swiglib("tds path %i: %s",i,list[i]) + end + end + local function found(locate,asked_library,how,...) + if trace_swiglib then + report_swiglib("checking %s: %a",how,asked_library) + end + return locate(asked_library,...) + end local function check(locate,...) - local found_library=nil + local found=nil if version then local asked_library=joinfile(required_path,version,required_name) if trace_swiglib then report_swiglib("checking %s: %a","with version",asked_library) end - found_library=locate(asked_library,...) - if not found_library or found_library==""then - asked_library=joinfile(required_path,required_name) - if trace_swiglib then - report_swiglib("checking %s: %a","without version",asked_library) - end - found_library=locate(asked_library,...) - end - else + found=locate(asked_library,...) + end + if not found or found=="" then local asked_library=joinfile(required_path,required_name) if trace_swiglib then - report_swiglib("checking %s: %a","without version",asked_library) + report_swiglib("checking %s: %a","with version",asked_library) end - found_library=locate(asked_library,...) + found=locate(asked_library,...) end - return found_library and found_library~="" and found_library or false + return found and found~="" and found or false end - local found_library=findfile and check(findfile,"lib") - if findfiles and not found_library then + local function attempt(checkpattern) + if trace_swiglib then + report_swiglib("checking tds lib paths strictly") + end + local found=findfile and check(findfile,"lib") + if found and (not checkpattern or find(found,checkpattern)) then + return found + end + if trace_swiglib then + report_swiglib("checking tds lib paths with wildcard") + end local asked_library=joinfile(required_path,".*",required_name) if trace_swiglib then report_swiglib("checking %s: %a","latest version",asked_library) @@ -15459,18 +15486,38 @@ local function requireswiglib(required,version) local list=findfiles(asked_library,"lib",true) if list and #list>0 then table.sort(list) - found_library=list[#list] + local found=list[#list] + if found and (not checkpattern or find(found,checkpattern)) then + return found + end + end + if trace_swiglib then + report_swiglib("checking clib paths") end - end - if not found_library then package.extraclibpath(environment.ownpath) local paths=package.clibpaths() for i=1,#paths do - local found_library=check(lfs.isfile) - if found_library then - break + local found=check(lfs.isfile) + if found and (not checkpattern or find(found,checkpattern)) then + return found end end + return false + end + local found_library=nil + if engine then + if trace_swiglib then + report_swiglib("attemp 1, engine %a",engine) + end + found_library=attempt("/"..engine.."/") + if not found_library then + if trace_swiglib then + report_swiglib("attemp 2, no engine",asked_library) + end + found_library=attempt() + end + else + found_library=attempt() end if not found_library then if trace_swiglib then @@ -15773,8 +15820,8 @@ end -- of closure -- used libraries : l-lua.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-md5.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-sto.lua util-prs.lua util-fmt.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-mrg.lua util-tpl.lua util-env.lua luat-env.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua -- skipped libraries : - --- original bytes : 658276 --- stripped bytes : 241564 +-- original bytes : 660467 +-- stripped bytes : 242438 -- end library merge diff --git a/Master/texmf-dist/scripts/context/stubs/unix/mtxrun b/Master/texmf-dist/scripts/context/stubs/unix/mtxrun index 00f63a5791d..d337ed75f74 100755 --- a/Master/texmf-dist/scripts/context/stubs/unix/mtxrun +++ b/Master/texmf-dist/scripts/context/stubs/unix/mtxrun @@ -56,7 +56,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-lua"] = package.loaded["l-lua"] or true --- original size: 10048, stripped down to: 5684 +-- original size: 10352, stripped down to: 5955 if not modules then modules={} end modules ['l-lua']={ version=1.001, @@ -247,7 +247,7 @@ local function loadedbypath(name,rawname,paths,islib,what) end local function notloaded(name) if helpers.trace then - helpers.report("? unable to locate library '%s'",name) + helpers.report("unable to locate library '%s'",name) end end helpers.loadedaslib=loadedaslib @@ -278,6 +278,16 @@ function helpers.loaded(name) end return notloaded(name) end +function helpers.unload(name) + if helpers.trace then + if package.loaded[name] then + helpers.report("unloading library '%s', %s",name,"done") + else + helpers.report("unloading library '%s', %s",name,"not loaded") + end + end + package.loaded[name]=nil +end end -- of closure @@ -12632,7 +12642,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-res"] = package.loaded["data-res"] or true --- original size: 60821, stripped down to: 42503 +-- original size: 60857, stripped down to: 42496 if not modules then modules={} end modules ['data-res']={ version=1.001, @@ -12807,7 +12817,7 @@ local slash=P("/") local pathexpressionpattern=Cs ( Cc("^")*( Cc("%")*S(".-")+slash^2*P(-1)/"/.*" -+slash^2/"/[^/]*/*"+(1-slash)*P(-1)*Cc("/")+P(1) ++slash^2/"/"+(1-slash)*P(-1)*Cc("/")+P(1) )^1*Cc("$") ) local cache={} @@ -15403,7 +15413,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-lib"] = package.loaded["util-lib"] or true --- original size: 8911, stripped down to: 4216 +-- original size: 10762, stripped down to: 5269 if not modules then modules={} end modules ['util-lib']={ version=1.001, @@ -15418,6 +15428,7 @@ local findfile,findfiles=resolvers and resolvers.findfile,resolvers and resolver local loaded=package.loaded local report_swiglib=logs.reporter("swiglib") local trace_swiglib=false trackers.register("resolvers.swiglib",function(v) trace_swiglib=v end) +local done=false local function requireswiglib(required,version) local library=loaded[required] if library==nil then @@ -15426,32 +15437,48 @@ local function requireswiglib(required,version) local required_base=nameonly(required_full) local required_name=required_base.."."..os.libsuffix local version=type(version)=="string" and version~="" and version or false + local engine=environment.ownmain or false + if trace_swiglib and not done then + local list=resolvers.expandedpathlistfromvariable("lib") + for i=1,#list do + report_swiglib("tds path %i: %s",i,list[i]) + end + end + local function found(locate,asked_library,how,...) + if trace_swiglib then + report_swiglib("checking %s: %a",how,asked_library) + end + return locate(asked_library,...) + end local function check(locate,...) - local found_library=nil + local found=nil if version then local asked_library=joinfile(required_path,version,required_name) if trace_swiglib then report_swiglib("checking %s: %a","with version",asked_library) end - found_library=locate(asked_library,...) - if not found_library or found_library==""then - asked_library=joinfile(required_path,required_name) - if trace_swiglib then - report_swiglib("checking %s: %a","without version",asked_library) - end - found_library=locate(asked_library,...) - end - else + found=locate(asked_library,...) + end + if not found or found=="" then local asked_library=joinfile(required_path,required_name) if trace_swiglib then - report_swiglib("checking %s: %a","without version",asked_library) + report_swiglib("checking %s: %a","with version",asked_library) end - found_library=locate(asked_library,...) + found=locate(asked_library,...) end - return found_library and found_library~="" and found_library or false + return found and found~="" and found or false end - local found_library=findfile and check(findfile,"lib") - if findfiles and not found_library then + local function attempt(checkpattern) + if trace_swiglib then + report_swiglib("checking tds lib paths strictly") + end + local found=findfile and check(findfile,"lib") + if found and (not checkpattern or find(found,checkpattern)) then + return found + end + if trace_swiglib then + report_swiglib("checking tds lib paths with wildcard") + end local asked_library=joinfile(required_path,".*",required_name) if trace_swiglib then report_swiglib("checking %s: %a","latest version",asked_library) @@ -15459,18 +15486,38 @@ local function requireswiglib(required,version) local list=findfiles(asked_library,"lib",true) if list and #list>0 then table.sort(list) - found_library=list[#list] + local found=list[#list] + if found and (not checkpattern or find(found,checkpattern)) then + return found + end + end + if trace_swiglib then + report_swiglib("checking clib paths") end - end - if not found_library then package.extraclibpath(environment.ownpath) local paths=package.clibpaths() for i=1,#paths do - local found_library=check(lfs.isfile) - if found_library then - break + local found=check(lfs.isfile) + if found and (not checkpattern or find(found,checkpattern)) then + return found end end + return false + end + local found_library=nil + if engine then + if trace_swiglib then + report_swiglib("attemp 1, engine %a",engine) + end + found_library=attempt("/"..engine.."/") + if not found_library then + if trace_swiglib then + report_swiglib("attemp 2, no engine",asked_library) + end + found_library=attempt() + end + else + found_library=attempt() end if not found_library then if trace_swiglib then @@ -15773,8 +15820,8 @@ end -- of closure -- used libraries : l-lua.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-md5.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-sto.lua util-prs.lua util-fmt.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-mrg.lua util-tpl.lua util-env.lua luat-env.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua -- skipped libraries : - --- original bytes : 658276 --- stripped bytes : 241564 +-- original bytes : 660467 +-- stripped bytes : 242438 -- end library merge diff --git a/Master/texmf-dist/tex/context/base/chem-str.lua b/Master/texmf-dist/tex/context/base/chem-str.lua index 311b70b2d1b..dfcf0a3e143 100644 --- a/Master/texmf-dist/tex/context/base/chem-str.lua +++ b/Master/texmf-dist/tex/context/base/chem-str.lua @@ -45,6 +45,7 @@ local P, R, S, C, Cs, Ct, Cc, Cmt = lpeg.P, lpeg.R, lpeg.S, lpeg.C, lpeg.Cs, lpe local variables = interfaces and interfaces.variables local context = context local formatters = string.formatters +local texcount = tex.count local v_default = variables.default local v_small = variables.small @@ -625,6 +626,7 @@ local function checked(d,factor,unit,scale) end local function calculated(height,bottom,top,factor,unit,scale) + local scaled = 0 if height == v_none then -- this always wins height = "0pt" @@ -654,22 +656,23 @@ local function calculated(height,bottom,top,factor,unit,scale) bottom = bottom * ratio top = top * ratio end + scaled = height top = topoints(top) bottom = topoints(bottom) height = topoints(height) end - return height, bottom, top + return height, bottom, top, scaled end function chemistry.start(settings) - chemistry.structures = chemistry.structures + 1 + -- + local width = settings.width or v_fit + local height = settings.height or v_fit local unit = settings.unit or 655360 local factor = settings.factor or 3 local rulethickness = settings.rulethickness or 65536 local rulecolor = settings.rulecolor or "" local axiscolor = settings.framecolor or "" - local width = settings.width or v_fit - local height = settings.height or v_fit local scale = settings.scale or "normal" local rotation = settings.rotation or 0 local offset = settings.offset or 0 @@ -678,8 +681,6 @@ function chemistry.start(settings) local top = settings.top or v_fit local bottom = settings.bottom or v_fit -- - metacode = { } - -- align = settings.symalign or "auto" if trace_structure then report_chemistry("unit %p, factor %s, symalign %s",unit,factor,align) @@ -709,11 +710,26 @@ function chemistry.start(settings) -- unit = scale * unit -- - width, left, right = calculated(width, left, right,factor,unit,scale) - height, bottom, top = calculated(height,bottom,top, factor,unit,scale) + local sp_width = 0 + local sp_height = 0 + -- + width, left, right, sp_width = calculated(width, left, right,factor,unit,scale) + height, bottom, top, sp_height = calculated(height,bottom,top, factor,unit,scale) + -- + if width ~= "true" and height ~= "true" and texcount["@@trialtypesetting"] ~= 0 then + if trace_structure then + report_chemistry("skipping trial run") + end + context.hrule(sp_width,sp_height,0) -- maybe depth + return + end + -- + chemistry.structures = chemistry.structures + 1 -- rotation = tonumber(rotation) or 0 -- + metacode = { } + -- if trace_structure then report_chemistry("%s scale %a, rotation %a, width %s, height %s, left %s, right %s, top %s, bottom %s","used",scale,rotation,width,height,left,right,top,bottom) end @@ -729,32 +745,36 @@ function chemistry.start(settings) end function chemistry.stop() - metacode[#metacode+1] = f_stop_structure - local mpcode = concat(metacode,"\n") - if trace_metapost then - report_chemistry("metapost code:\n%s", mpcode) - end - if metapost.instance(chemistry.instance) then - f_initialize = nil + if metacode then + metacode[#metacode+1] = f_stop_structure + local mpcode = concat(metacode,"\n") + if trace_metapost then + report_chemistry("metapost code:\n%s", mpcode) + end + if metapost.instance(chemistry.instance) then + f_initialize = nil + end + metapost.graphic { + instance = chemistry.instance, + format = chemistry.format, + data = mpcode, + definitions = f_initialize, + } + t_initialize = "" + metacode = nil end - metapost.graphic { - instance = chemistry.instance, - format = chemistry.format, - data = mpcode, - definitions = f_initialize, - } - t_initialize = "" - metacode = nil end function chemistry.component(spec,text,settings) - rulethickness, rulecolor, offset = settings.rulethickness, settings.rulecolor - local spec = settings_to_array_with_repeat(spec,true) -- no lower? - local text = settings_to_array_with_repeat(text,true) --- inspect(spec) - metacode[#metacode+1] = f_start_component - process(1,spec,text,1,rulethickness,rulecolor) -- offset? - metacode[#metacode+1] = f_stop_component + if metacode then + rulethickness, rulecolor, offset = settings.rulethickness, settings.rulecolor + local spec = settings_to_array_with_repeat(spec,true) -- no lower? + local text = settings_to_array_with_repeat(text,true) + -- inspect(spec) + metacode[#metacode+1] = f_start_component + process(1,spec,text,1,rulethickness,rulecolor) -- offset? + metacode[#metacode+1] = f_stop_component + end end statistics.register("chemical formulas", function() diff --git a/Master/texmf-dist/tex/context/base/cont-new.mkiv b/Master/texmf-dist/tex/context/base/cont-new.mkiv index 530058fc92f..85a3cf0a5dd 100644 --- a/Master/texmf-dist/tex/context/base/cont-new.mkiv +++ b/Master/texmf-dist/tex/context/base/cont-new.mkiv @@ -11,14 +11,14 @@ %C therefore copyrighted by \PRAGMA. See mreadme.pdf for %C details. -\newcontextversion{2013.04.07 14:05} +\newcontextversion{2013.04.08 17:52} %D This file is loaded at runtime, thereby providing an excellent place for %D hacks, patches, extensions and new features. \unprotect -\writestatus\m!system{beware: some patches loaded from cont-new.mkiv} +% \writestatus\m!system{beware: some patches loaded from cont-new.mkiv} % \attribute152\zerocount : marks ... lots of sweeps so best early in list diff --git a/Master/texmf-dist/tex/context/base/context-version.pdf b/Master/texmf-dist/tex/context/base/context-version.pdf index 27388e1a084..916ff06beb2 100644 Binary files a/Master/texmf-dist/tex/context/base/context-version.pdf and b/Master/texmf-dist/tex/context/base/context-version.pdf differ diff --git a/Master/texmf-dist/tex/context/base/context-version.png b/Master/texmf-dist/tex/context/base/context-version.png index a8a38184bdc..b32e53f50b9 100644 Binary files a/Master/texmf-dist/tex/context/base/context-version.png and b/Master/texmf-dist/tex/context/base/context-version.png differ diff --git a/Master/texmf-dist/tex/context/base/context.mkiv b/Master/texmf-dist/tex/context/base/context.mkiv index 224944aec24..e11a57a4f94 100644 --- a/Master/texmf-dist/tex/context/base/context.mkiv +++ b/Master/texmf-dist/tex/context/base/context.mkiv @@ -25,7 +25,7 @@ %D up and the dependencies are more consistent. \edef\contextformat {\jobname} -\edef\contextversion{2013.04.07 14:05} +\edef\contextversion{2013.04.08 17:52} \edef\contextkind {beta} %D For those who want to use this: diff --git a/Master/texmf-dist/tex/context/base/data-res.lua b/Master/texmf-dist/tex/context/base/data-res.lua index f7e238a11a3..32e19cf130b 100644 --- a/Master/texmf-dist/tex/context/base/data-res.lua +++ b/Master/texmf-dist/tex/context/base/data-res.lua @@ -294,8 +294,9 @@ local pathexpressionpattern = Cs ( -- create lpeg instead (2013/2014) Cc("^") * ( Cc("%") * S(".-") + slash^2 * P(-1) / "/.*" --- + slash^2 / "/.-/" - + slash^2 / "/[^/]*/*" + -- + slash^2 / "/.-/" + -- + slash^2 / "/[^/]*/*" -- too general + + slash^2 / "/" + (1-slash) * P(-1) * Cc("/") + P(1) )^1 * Cc("$") -- yes or no $ diff --git a/Master/texmf-dist/tex/context/base/l-lua.lua b/Master/texmf-dist/tex/context/base/l-lua.lua index 538c65d7ea1..51816400ef4 100644 --- a/Master/texmf-dist/tex/context/base/l-lua.lua +++ b/Master/texmf-dist/tex/context/base/l-lua.lua @@ -356,7 +356,7 @@ end local function notloaded(name) if helpers.trace then - helpers.report("? unable to locate library '%s'",name) + helpers.report("unable to locate library '%s'",name) end end @@ -391,3 +391,14 @@ function helpers.loaded(name) end return notloaded(name) end + +function helpers.unload(name) + if helpers.trace then + if package.loaded[name] then + helpers.report("unloading library '%s', %s",name,"done") + else + helpers.report("unloading library '%s', %s",name,"not loaded") + end + end + package.loaded[name] = nil +end diff --git a/Master/texmf-dist/tex/context/base/math-ali.mkiv b/Master/texmf-dist/tex/context/base/math-ali.mkiv index a172dc8ae5f..0d9d38e200c 100644 --- a/Master/texmf-dist/tex/context/base/math-ali.mkiv +++ b/Master/texmf-dist/tex/context/base/math-ali.mkiv @@ -231,8 +231,16 @@ \let\align_text_normal_start\startalignment \let\align_text_normal_stop \stopalignment - \unexpanded\def\startalign{\ifmmode\expandafter\align_math_normal_start\else\expandafter\align_text_normal_start\fi} - \unexpanded\def\stopalign {\ifmmode\expandafter\align_math_normal_stop \else\expandafter\align_text_normal_stop \fi} + \unexpanded\def\startalign + {\ifmmode + \let\stopalign\align_math_normal_stop % cannot be an unexpanded def ... lookahead in align + \expandafter\align_math_normal_start + \else + \let\stopalign\align_text_normal_stop + \expandafter\align_text_normal_start + \fi} + + \let\stopalign\relax \let\startalignment\startalign \let\stopalignment \stopalign diff --git a/Master/texmf-dist/tex/context/base/status-files.pdf b/Master/texmf-dist/tex/context/base/status-files.pdf index 23ddb60ccf0..f43bcdc4e75 100644 Binary files a/Master/texmf-dist/tex/context/base/status-files.pdf and b/Master/texmf-dist/tex/context/base/status-files.pdf differ diff --git a/Master/texmf-dist/tex/context/base/status-lua.pdf b/Master/texmf-dist/tex/context/base/status-lua.pdf index 19f6e0668f0..f34d626b751 100644 Binary files a/Master/texmf-dist/tex/context/base/status-lua.pdf and b/Master/texmf-dist/tex/context/base/status-lua.pdf differ diff --git a/Master/texmf-dist/tex/context/base/tabl-xtb.lua b/Master/texmf-dist/tex/context/base/tabl-xtb.lua index cf9a4a0a603..59cfadbf210 100644 --- a/Master/texmf-dist/tex/context/base/tabl-xtb.lua +++ b/Master/texmf-dist/tex/context/base/tabl-xtb.lua @@ -59,6 +59,7 @@ local nodepool = nodes.pool local new_glue = nodepool.glue local new_kern = nodepool.kern local new_penalty = nodepool.penalty +local new_hlist = nodepool.hlist local v_stretch = variables.stretch local v_normal = variables.normal @@ -645,10 +646,13 @@ function xtables.construct() local list = drc.list if list then list.shift = list.height + list.depth - list = hpack_node_list(list) -- is somehow needed - list.width = 0 - list.height = 0 - list.depth = 0 +-- list = hpack_node_list(list) -- is somehow needed +-- list.width = 0 +-- list.height = 0 +-- list.depth = 0 +local h = new_hlist() +h.list = list +list = h if start then stop.next = list list.prev = stop diff --git a/Master/texmf-dist/tex/context/base/util-lib.lua b/Master/texmf-dist/tex/context/base/util-lib.lua index fde1f987e7f..601f2fb1d9f 100644 --- a/Master/texmf-dist/tex/context/base/util-lib.lua +++ b/Master/texmf-dist/tex/context/base/util-lib.lua @@ -65,6 +65,12 @@ or not using a wildcard lookup. This code is experimental and by providing a special abstract loader (called swiglib) we can start using the libraries. +A complication is that we might end up with a luajittex path matching before a +luatex path due to the path spec. One solution is to first check with the engine +prefixed. This could be prevented by a more strict lib pattern but that is not +always under our control. So, we first check for paths with engine in their name +and then without. + ]]-- -- seems to be clua in recent texlive @@ -81,6 +87,8 @@ local trace_swiglib = false trackers.register("resolvers.swiglib", function(v) -- We can check if there are more that one component, and if not, we can -- append 'core'. +local done = false + local function requireswiglib(required,version) local library = loaded[required] if library == nil then @@ -90,35 +98,55 @@ local function requireswiglib(required,version) local required_base = nameonly(required_full) local required_name = required_base .. "." .. os.libsuffix local version = type(version) == "string" and version ~= "" and version or false - -- helper + local engine = environment.ownmain or false + -- + if trace_swiglib and not done then + local list = resolvers.expandedpathlistfromvariable("lib") + for i=1,#list do + report_swiglib("tds path %i: %s",i,list[i]) + end + end + -- helpers + local function found(locate,asked_library,how,...) + if trace_swiglib then + report_swiglib("checking %s: %a",how,asked_library) + end + return locate(asked_library,...) + end local function check(locate,...) - local found_library = nil + local found = nil if version then local asked_library = joinfile(required_path,version,required_name) if trace_swiglib then report_swiglib("checking %s: %a","with version",asked_library) end - found_library = locate(asked_library,...) - if not found_library or found_library == ""then - asked_library = joinfile(required_path,required_name) - if trace_swiglib then - report_swiglib("checking %s: %a","without version",asked_library) - end - found_library = locate(asked_library,...) - end - else + found = locate(asked_library,...) + end + if not found or found == "" then local asked_library = joinfile(required_path,required_name) if trace_swiglib then - report_swiglib("checking %s: %a","without version",asked_library) + report_swiglib("checking %s: %a","with version",asked_library) end - found_library = locate(asked_library,...) + found = locate(asked_library,...) end - return found_library and found_library ~= "" and found_library or false + return found and found ~= "" and found or false end - -- check cnf spec using name and version - local found_library = findfile and check(findfile,"lib") - -- check cnf spec using wildcard - if findfiles and not found_library then + -- Alternatively we could first collect the locations and then do the two attempts + -- on this list but in practice this is not more efficient as we might have a fast + -- match anyway. + local function attempt(checkpattern) + -- check cnf spec using name and version + if trace_swiglib then + report_swiglib("checking tds lib paths strictly") + end + local found = findfile and check(findfile,"lib") + if found and (not checkpattern or find(found,checkpattern)) then + return found + end + -- check cnf spec using wildcard + if trace_swiglib then + report_swiglib("checking tds lib paths with wildcard") + end local asked_library = joinfile(required_path,".*",required_name) if trace_swiglib then report_swiglib("checking %s: %a","latest version",asked_library) @@ -126,19 +154,39 @@ local function requireswiglib(required,version) local list = findfiles(asked_library,"lib",true) if list and #list > 0 then table.sort(list) - found_library = list[#list] + local found = list[#list] + if found and (not checkpattern or find(found,checkpattern)) then + return found + end + end + -- Check clib paths using name and version. + if trace_swiglib then + report_swiglib("checking clib paths") end - end - -- check clib paths using name and version - if not found_library then package.extraclibpath(environment.ownpath) local paths = package.clibpaths() for i=1,#paths do - local found_library = check(lfs.isfile) - if found_library then - break + local found = check(lfs.isfile) + if found and (not checkpattern or find(found,checkpattern)) then + return found end end + return false + end + local found_library = nil + if engine then + if trace_swiglib then + report_swiglib("attemp 1, engine %a",engine) + end + found_library = attempt("/"..engine.."/") + if not found_library then + if trace_swiglib then + report_swiglib("attemp 2, no engine",asked_library) + end + found_library = attempt() + end + else + found_library = attempt() end -- load and initialize when found if not found_library then diff --git a/Master/texmf-dist/tex/generic/context/luatex/luatex-fonts-merged.lua b/Master/texmf-dist/tex/generic/context/luatex/luatex-fonts-merged.lua index 2cb036df01e..dff2d397537 100644 --- a/Master/texmf-dist/tex/generic/context/luatex/luatex-fonts-merged.lua +++ b/Master/texmf-dist/tex/generic/context/luatex/luatex-fonts-merged.lua @@ -1,6 +1,6 @@ -- merged file : luatex-fonts-merged.lua -- parent file : luatex-fonts.lua --- merge date : 04/07/13 14:05:24 +-- merge date : 04/08/13 17:52:17 do -- begin closure to overcome local limits and interference @@ -193,7 +193,7 @@ local function loadedbypath(name,rawname,paths,islib,what) end local function notloaded(name) if helpers.trace then - helpers.report("? unable to locate library '%s'",name) + helpers.report("unable to locate library '%s'",name) end end helpers.loadedaslib=loadedaslib @@ -224,6 +224,16 @@ function helpers.loaded(name) end return notloaded(name) end +function helpers.unload(name) + if helpers.trace then + if package.loaded[name] then + helpers.report("unloading library '%s', %s",name,"done") + else + helpers.report("unloading library '%s', %s",name,"not loaded") + end + end + package.loaded[name]=nil +end end -- closure -- cgit v1.2.3