diff options
Diffstat (limited to 'Master/texmf-dist/scripts')
92 files changed, 21090 insertions, 12724 deletions
diff --git a/Master/texmf-dist/scripts/context/lua/luatools.lua b/Master/texmf-dist/scripts/context/lua/luatools.lua index 433d1b8dc0a..1d87322c108 100755 --- a/Master/texmf-dist/scripts/context/lua/luatools.lua +++ b/Master/texmf-dist/scripts/context/lua/luatools.lua @@ -39,13 +39,16 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-string'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -local sub, gsub, find, match, gmatch, format, char, byte, rep = string.sub, string.gsub, string.find, string.match, string.gmatch, string.format, string.char, string.byte, string.rep +local sub, gsub, find, match, gmatch, format, char, byte, rep, lower = string.sub, string.gsub, string.find, string.match, string.gmatch, string.format, string.char, string.byte, string.rep, string.lower +local lpegmatch = lpeg.match + +-- some functions may disappear as they are not used anywhere if not string.split then @@ -85,8 +88,16 @@ function string:unquote() return (gsub(self,"^([\"\'])(.*)%1$","%2")) end +--~ function string:unquote() +--~ if find(self,"^[\'\"]") then +--~ return sub(self,2,-2) +--~ else +--~ return self +--~ end +--~ end + function string:quote() -- we could use format("%q") - return '"' .. self:unquote() .. '"' + return format("%q",self) end function string:count(pattern) -- variant 3 @@ -106,12 +117,23 @@ function string:limit(n,sentinel) end end -function string:strip() - return (gsub(self,"^%s*(.-)%s*$", "%1")) +--~ function string:strip() -- the .- is quite efficient +--~ -- return match(self,"^%s*(.-)%s*$") or "" +--~ -- return match(self,'^%s*(.*%S)') or '' -- posted on lua list +--~ return find(s,'^%s*$') and '' or match(s,'^%s*(.*%S)') +--~ end + +do -- roberto's variant: + local space = lpeg.S(" \t\v\n") + local nospace = 1 - space + local stripper = space^0 * lpeg.C((space^0 * nospace^1)^0) + function string.strip(str) + return lpegmatch(stripper,str) or "" + end end function string:is_empty() - return not find(find,"%S") + return not find(self,"%S") end function string:enhance(pattern,action) @@ -145,14 +167,14 @@ if not string.characters then local function nextchar(str, index) index = index + 1 - return (index <= #str) and index or nil, str:sub(index,index) + return (index <= #str) and index or nil, sub(str,index,index) end function string:characters() return nextchar, self, 0 end local function nextbyte(str, index) index = index + 1 - return (index <= #str) and index or nil, byte(str:sub(index,index)) + return (index <= #str) and index or nil, byte(sub(str,index,index)) end function string:bytes() return nextbyte, self, 0 @@ -165,7 +187,7 @@ end function string:rpadd(n,chr) local m = n-#self if m > 0 then - return self .. self.rep(chr or " ",m) + return self .. rep(chr or " ",m) else return self end @@ -174,7 +196,7 @@ end function string:lpadd(n,chr) local m = n-#self if m > 0 then - return self.rep(chr or " ",m) .. self + return rep(chr or " ",m) .. self else return self end @@ -222,6 +244,17 @@ function string:pattesc() return (gsub(self,".",patterns_escapes)) end +local simple_escapes = { + ["-"] = "%-", + ["."] = "%.", + ["?"] = ".", + ["*"] = ".*", +} + +function string:simpleesc() + return (gsub(self,".",simple_escapes)) +end + function string:tohash() local t = { } for s in gmatch(self,"([^, ]+)") do -- lpeg @@ -233,10 +266,10 @@ end local pattern = lpeg.Ct(lpeg.C(1)^0) function string:totable() - return pattern:match(self) + return lpegmatch(pattern,self) end ---~ for _, str in ipairs { +--~ local t = { --~ "1234567123456712345671234567", --~ "a\tb\tc", --~ "aa\tbb\tcc", @@ -244,7 +277,10 @@ end --~ "aaaa\tbbbb\tcccc", --~ "aaaaa\tbbbbb\tccccc", --~ "aaaaaa\tbbbbbb\tcccccc", ---~ } do print(string.tabtospace(str)) end +--~ } +--~ for k,v do +--~ print(string.tabtospace(t[k])) +--~ end function string.tabtospace(str,tab) -- we don't handle embedded newlines @@ -252,7 +288,7 @@ function string.tabtospace(str,tab) local s = find(str,"\t") if s then if not tab then tab = 7 end -- only when found - local d = tab-(s-1)%tab + local d = tab-(s-1) % tab if d > 0 then str = gsub(str,"\t",rep(" ",d),1) else @@ -271,6 +307,25 @@ function string:compactlong() -- strips newlines and leading spaces return self end +function string:striplong() -- strips newlines and leading spaces + self = gsub(self,"^%s*","") + self = gsub(self,"[\n\r]+ *","\n") + return self +end + +function string:topattern(lowercase,strict) + if lowercase then + self = lower(self) + end + self = gsub(self,".",simple_escapes) + if self == "" then + self = ".*" + elseif strict then + self = "^" .. self .. "$" + end + return self +end + end -- of closure @@ -278,58 +333,64 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-lpeg'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -local P, S, Ct, C, Cs, Cc = lpeg.P, lpeg.S, lpeg.Ct, lpeg.C, lpeg.Cs, lpeg.Cc - ---~ l-lpeg.lua : - ---~ lpeg.digit = lpeg.R('09')^1 ---~ lpeg.sign = lpeg.S('+-')^1 ---~ lpeg.cardinal = lpeg.P(lpeg.sign^0 * lpeg.digit^1) ---~ lpeg.integer = lpeg.P(lpeg.sign^0 * lpeg.digit^1) ---~ lpeg.float = lpeg.P(lpeg.sign^0 * lpeg.digit^0 * lpeg.P('.') * lpeg.digit^1) ---~ lpeg.number = lpeg.float + lpeg.integer ---~ lpeg.oct = lpeg.P("0") * lpeg.R('07')^1 ---~ lpeg.hex = lpeg.P("0x") * (lpeg.R('09') + lpeg.R('AF'))^1 ---~ lpeg.uppercase = lpeg.P("AZ") ---~ lpeg.lowercase = lpeg.P("az") - ---~ lpeg.eol = lpeg.S('\r\n\f')^1 -- includes formfeed ---~ lpeg.space = lpeg.S(' ')^1 ---~ lpeg.nonspace = lpeg.P(1-lpeg.space)^1 ---~ lpeg.whitespace = lpeg.S(' \r\n\f\t')^1 ---~ lpeg.nonwhitespace = lpeg.P(1-lpeg.whitespace)^1 - -local hash = { } +local lpeg = require("lpeg") + +lpeg.patterns = lpeg.patterns or { } -- so that we can share +local patterns = lpeg.patterns + +local P, R, S, Ct, C, Cs, Cc, V = lpeg.P, lpeg.R, lpeg.S, lpeg.Ct, lpeg.C, lpeg.Cs, lpeg.Cc, lpeg.V +local match = lpeg.match + +local digit, sign = R('09'), S('+-') +local cr, lf, crlf = P("\r"), P("\n"), P("\r\n") +local utf8byte = R("\128\191") + +patterns.utf8byte = utf8byte +patterns.utf8one = R("\000\127") +patterns.utf8two = R("\194\223") * utf8byte +patterns.utf8three = R("\224\239") * utf8byte * utf8byte +patterns.utf8four = R("\240\244") * utf8byte * utf8byte * utf8byte + +patterns.digit = digit +patterns.sign = sign +patterns.cardinal = sign^0 * digit^1 +patterns.integer = sign^0 * digit^1 +patterns.float = sign^0 * digit^0 * P('.') * digit^1 +patterns.number = patterns.float + patterns.integer +patterns.oct = P("0") * R("07")^1 +patterns.octal = patterns.oct +patterns.HEX = P("0x") * R("09","AF")^1 +patterns.hex = P("0x") * R("09","af")^1 +patterns.hexadecimal = P("0x") * R("09","AF","af")^1 +patterns.lowercase = R("az") +patterns.uppercase = R("AZ") +patterns.letter = patterns.lowercase + patterns.uppercase +patterns.space = S(" ") +patterns.eol = S("\n\r") +patterns.spacer = S(" \t\f\v") -- + string.char(0xc2, 0xa0) if we want utf (cf mail roberto) +patterns.newline = crlf + cr + lf +patterns.nonspace = 1 - patterns.space +patterns.nonspacer = 1 - patterns.spacer +patterns.whitespace = patterns.eol + patterns.spacer +patterns.nonwhitespace = 1 - patterns.whitespace +patterns.utf8 = patterns.utf8one + patterns.utf8two + patterns.utf8three + patterns.utf8four +patterns.utfbom = P('\000\000\254\255') + P('\255\254\000\000') + P('\255\254') + P('\254\255') + P('\239\187\191') function lpeg.anywhere(pattern) --slightly adapted from website - return P { P(pattern) + 1 * lpeg.V(1) } -end - -function lpeg.startswith(pattern) --slightly adapted - return P(pattern) + return P { P(pattern) + 1 * V(1) } -- why so complex? end function lpeg.splitter(pattern, action) return (((1-P(pattern))^1)/action+1)^0 end --- variant: - ---~ local parser = lpeg.Ct(lpeg.splitat(newline)) - -local crlf = P("\r\n") -local cr = P("\r") -local lf = P("\n") -local space = S(" \t\f\v") -- + string.char(0xc2, 0xa0) if we want utf (cf mail roberto) -local newline = crlf + cr + lf -local spacing = space^0 * newline - +local spacing = patterns.spacer^0 * patterns.newline -- sort of strip local empty = spacing * Cc("") local nonempty = Cs((1-spacing)^1) * spacing^-1 local content = (empty + nonempty)^1 @@ -337,15 +398,15 @@ local content = (empty + nonempty)^1 local capture = Ct(content^0) function string:splitlines() - return capture:match(self) + return match(capture,self) end -lpeg.linebyline = content -- better make a sublibrary +patterns.textline = content ---~ local p = lpeg.splitat("->",false) print(p:match("oeps->what->more")) -- oeps what more ---~ local p = lpeg.splitat("->",true) print(p:match("oeps->what->more")) -- oeps what->more ---~ local p = lpeg.splitat("->",false) print(p:match("oeps")) -- oeps ---~ local p = lpeg.splitat("->",true) print(p:match("oeps")) -- oeps +--~ local p = lpeg.splitat("->",false) print(match(p,"oeps->what->more")) -- oeps what more +--~ local p = lpeg.splitat("->",true) print(match(p,"oeps->what->more")) -- oeps what->more +--~ local p = lpeg.splitat("->",false) print(match(p,"oeps")) -- oeps +--~ local p = lpeg.splitat("->",true) print(match(p,"oeps")) -- oeps local splitters_s, splitters_m = { }, { } @@ -355,7 +416,7 @@ local function splitat(separator,single) separator = P(separator) if single then local other, any = C((1 - separator)^0), P(1) - splitter = other * (separator * C(any^0) + "") + splitter = other * (separator * C(any^0) + "") -- ? splitters_s[separator] = splitter else local other = C((1 - separator)^0) @@ -370,15 +431,72 @@ lpeg.splitat = splitat local cache = { } +function lpeg.split(separator,str) + local c = cache[separator] + if not c then + c = Ct(splitat(separator)) + cache[separator] = c + end + return match(c,str) +end + function string:split(separator) local c = cache[separator] if not c then c = Ct(splitat(separator)) cache[separator] = c end - return c:match(self) + return match(c,self) +end + +lpeg.splitters = cache + +local cache = { } + +function lpeg.checkedsplit(separator,str) + local c = cache[separator] + if not c then + separator = P(separator) + local other = C((1 - separator)^0) + c = Ct(separator^0 * other * (separator^1 * other)^0) + cache[separator] = c + end + return match(c,str) +end + +function string:checkedsplit(separator) + local c = cache[separator] + if not c then + separator = P(separator) + local other = C((1 - separator)^0) + c = Ct(separator^0 * other * (separator^1 * other)^0) + cache[separator] = c + end + return match(c,self) end +--~ function lpeg.append(list,pp) +--~ local p = pp +--~ for l=1,#list do +--~ if p then +--~ p = p + P(list[l]) +--~ else +--~ p = P(list[l]) +--~ end +--~ end +--~ return p +--~ end + +--~ from roberto's site: + +local f1 = string.byte + +local function f2(s) local c1, c2 = f1(s,1,2) return c1 * 64 + c2 - 12416 end +local function f3(s) local c1, c2, c3 = f1(s,1,3) return (c1 * 64 + c2) * 64 + c3 - 925824 end +local function f4(s) local c1, c2, c3, c4 = f1(s,1,4) return ((c1 * 64 + c2) * 64 + c3) * 64 + c4 - 63447168 end + +patterns.utf8byte = patterns.utf8one/f1 + patterns.utf8two/f2 + patterns.utf8three/f3 + patterns.utf8four/f4 + end -- of closure @@ -386,7 +504,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-table'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -395,9 +513,10 @@ if not modules then modules = { } end modules ['l-table'] = { table.join = table.concat local concat, sort, insert, remove = table.concat, table.sort, table.insert, table.remove -local format, find, gsub, lower, dump = string.format, string.find, string.gsub, string.lower, string.dump +local format, find, gsub, lower, dump, match = string.format, string.find, string.gsub, string.lower, string.dump, string.match local getmetatable, setmetatable = getmetatable, setmetatable -local type, next, tostring, ipairs = type, next, tostring, ipairs +local type, next, tostring, tonumber, ipairs = type, next, tostring, tonumber, ipairs +local unpack = unpack or table.unpack function table.strip(tab) local lst = { } @@ -412,6 +531,14 @@ function table.strip(tab) return lst end +function table.keys(t) + local k = { } + for key, _ in next, t do + k[#k+1] = key + end + return k +end + local function compare(a,b) return (tostring(a) < tostring(b)) end @@ -455,7 +582,7 @@ end table.sortedkeys = sortedkeys table.sortedhashkeys = sortedhashkeys -function table.sortedpairs(t) +function table.sortedhash(t) local s = sortedhashkeys(t) -- maybe just sortedkeys local n = 0 local function kv(s) @@ -466,6 +593,8 @@ function table.sortedpairs(t) return kv, s end +table.sortedpairs = table.sortedhash + function table.append(t, list) for _,v in next, list do insert(t,v) @@ -588,18 +717,18 @@ end -- slower than #t on indexed tables (#t only returns the size of the numerically indexed slice) -function table.is_empty(t) +function table.is_empty(t) -- obolete, use inline code instead return not t or not next(t) end -function table.one_entry(t) +function table.one_entry(t) -- obolete, use inline code instead local n = next(t) return n and not next(t,n) end -function table.starts_at(t) - return ipairs(t,1)(t,0) -end +--~ function table.starts_at(t) -- obsolete, not nice +--~ return ipairs(t,1)(t,0) +--~ end function table.tohash(t,value) local h = { } @@ -677,6 +806,8 @@ end -- -- local propername = lpeg.P(lpeg.R("AZ","az","__") * lpeg.R("09","AZ","az", "__")^0 * lpeg.P(-1) ) +-- problem: there no good number_to_string converter with the best resolution + local function do_serialize(root,name,depth,level,indexed) if level > 0 then depth = depth .. " " @@ -699,8 +830,9 @@ local function do_serialize(root,name,depth,level,indexed) handle(format("%s{",depth)) end end + -- we could check for k (index) being number (cardinal) if root and next(root) then - local first, last = nil, 0 -- #root cannot be trusted here + local first, last = nil, 0 -- #root cannot be trusted here (will be ok in 5.2 when ipairs is gone) if compact then -- NOT: for k=1,#root do (we need to quit at nil) for k,v in ipairs(root) do -- can we use next? @@ -721,10 +853,10 @@ local function do_serialize(root,name,depth,level,indexed) if hexify then handle(format("%s 0x%04X,",depth,v)) else - handle(format("%s %s,",depth,v)) + handle(format("%s %s,",depth,v)) -- %.99g end elseif t == "string" then - if reduce and (find(v,"^[%-%+]?[%d]-%.?[%d+]$") == 1) then + if reduce and tonumber(v) then handle(format("%s %s,",depth,v)) else handle(format("%s %q,",depth,v)) @@ -761,29 +893,29 @@ local function do_serialize(root,name,depth,level,indexed) --~ if hexify then --~ handle(format("%s %s=0x%04X,",depth,key(k),v)) --~ else - --~ handle(format("%s %s=%s,",depth,key(k),v)) + --~ handle(format("%s %s=%s,",depth,key(k),v)) -- %.99g --~ end if type(k) == "number" then -- or find(k,"^%d+$") then if hexify then handle(format("%s [0x%04X]=0x%04X,",depth,k,v)) else - handle(format("%s [%s]=%s,",depth,k,v)) + handle(format("%s [%s]=%s,",depth,k,v)) -- %.99g end elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then if hexify then handle(format("%s %s=0x%04X,",depth,k,v)) else - handle(format("%s %s=%s,",depth,k,v)) + handle(format("%s %s=%s,",depth,k,v)) -- %.99g end else if hexify then handle(format("%s [%q]=0x%04X,",depth,k,v)) else - handle(format("%s [%q]=%s,",depth,k,v)) + handle(format("%s [%q]=%s,",depth,k,v)) -- %.99g end end elseif t == "string" then - if reduce and (find(v,"^[%-%+]?[%d]-%.?[%d+]$") == 1) then + if reduce and tonumber(v) then --~ handle(format("%s %s=%s,",depth,key(k),v)) if type(k) == "number" then -- or find(k,"^%d+$") then if hexify then @@ -992,7 +1124,7 @@ function table.tofile(filename,root,name,reduce,noquotes,hexify) end end -local function flatten(t,f,complete) +local function flatten(t,f,complete) -- is this used? meybe a variant with next, ... for i=1,#t do local v = t[i] if type(v) == "table" then @@ -1021,6 +1153,24 @@ end table.flatten_one_level = table.unnest +-- a better one: + +local function flattened(t,f) + if not f then + f = { } + end + for k, v in next, t do + if type(v) == "table" then + flattened(v,f) + else + f[k] = v + end + end + return f +end + +table.flattened = flattened + -- the next three may disappear function table.remove_value(t,value) -- todo: n @@ -1156,7 +1306,7 @@ function table.clone(t,p) -- t is optional or nil or table elseif not t then t = { } end - setmetatable(t, { __index = function(_,key) return p[key] end }) + setmetatable(t, { __index = function(_,key) return p[key] end }) -- why not __index = p ? return t end @@ -1184,21 +1334,35 @@ function table.reverse(t) return tt end ---~ function table.keys(t) ---~ local k = { } ---~ for k,_ in next, t do ---~ k[#k+1] = k ---~ end ---~ return k ---~ end +function table.insert_before_value(t,value,extra) + for i=1,#t do + if t[i] == extra then + remove(t,i) + end + end + for i=1,#t do + if t[i] == value then + insert(t,i,extra) + return + end + end + insert(t,1,extra) +end ---~ function table.keys_as_string(t) ---~ local k = { } ---~ for k,_ in next, t do ---~ k[#k+1] = k ---~ end ---~ return concat(k,"") ---~ end +function table.insert_after_value(t,value,extra) + for i=1,#t do + if t[i] == extra then + remove(t,i) + end + end + for i=1,#t do + if t[i] == value then + insert(t,i+1,extra) + return + end + end + insert(t,#t+1,extra) +end end -- of closure @@ -1207,13 +1371,13 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-io'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -local byte = string.byte +local byte, find, gsub = string.byte, string.find, string.gsub if string.find(os.getenv("PATH"),";") then io.fileseparator, io.pathseparator = "\\", ";" @@ -1242,7 +1406,7 @@ function io.savedata(filename,data,joiner) elseif type(data) == "function" then data(f) else - f:write(data) + f:write(data or "") end f:close() return true @@ -1371,20 +1535,21 @@ function io.ask(question,default,options) end io.write(string.format(" ")) local answer = io.read() - answer = answer:gsub("^%s*(.*)%s*$","%1") + answer = gsub(answer,"^%s*(.*)%s*$","%1") if answer == "" and default then return default elseif not options then return answer else - for _,v in pairs(options) do - if v == answer then + for k=1,#options do + if options[k] == answer then return answer end end local pattern = "^" .. answer - for _,v in pairs(options) do - if v:find(pattern) then + for k=1,#options do + local v = options[k] + if find(v,pattern) then return v end end @@ -1399,20 +1564,22 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-number'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -local format = string.format +local tostring = tostring +local format, floor, insert, match = string.format, math.floor, table.insert, string.match +local lpegmatch = lpeg.match number = number or { } -- a,b,c,d,e,f = number.toset(100101) function number.toset(n) - return (tostring(n)):match("(.?)(.?)(.?)(.?)(.?)(.?)(.?)(.?)") + return match(tostring(n),"(.?)(.?)(.?)(.?)(.?)(.?)(.?)(.?)") end function number.toevenhex(n) @@ -1438,10 +1605,21 @@ end local one = lpeg.C(1-lpeg.S(''))^1 function number.toset(n) - return one:match(tostring(n)) + return lpegmatch(one,tostring(n)) end - +function number.bits(n,zero) + local t, i = { }, (zero and 0) or 1 + while n > 0 do + local m = n % 2 + if m > 0 then + insert(t,1,i) + end + n = floor(n/2) + i = i + 1 + end + return t +end end -- of closure @@ -1450,7 +1628,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-set'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -1540,46 +1718,63 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-os'] = { version = 1.001, - comment = "companion to luat-lub.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -local find = string.find +-- maybe build io.flush in os.execute + +local find, format, gsub = string.find, string.format, string.gsub +local random, ceil = math.random, math.ceil + +local execute, spawn, exec, ioflush = os.execute, os.spawn or os.execute, os.exec or os.execute, io.flush + +function os.execute(...) ioflush() return execute(...) end +function os.spawn (...) ioflush() return spawn (...) end +function os.exec (...) ioflush() return exec (...) end function os.resultof(command) - return io.popen(command,"r"):read("*all") + ioflush() -- else messed up logging + local handle = io.popen(command,"r") + if not handle then + -- print("unknown command '".. command .. "' in os.resultof") + return "" + else + return handle:read("*all") or "" + end end -if not os.exec then os.exec = os.execute end -if not os.spawn then os.spawn = os.execute end - ---~ os.type : windows | unix (new, we already guessed os.platform) ---~ os.name : windows | msdos | linux | macosx | solaris | .. | generic (new) +--~ os.type : windows | unix (new, we already guessed os.platform) +--~ os.name : windows | msdos | linux | macosx | solaris | .. | generic (new) +--~ os.platform : extended os.name with architecture if not io.fileseparator then if find(os.getenv("PATH"),";") then - io.fileseparator, io.pathseparator, os.platform = "\\", ";", os.type or "windows" + io.fileseparator, io.pathseparator, os.type = "\\", ";", os.type or "mswin" else - io.fileseparator, io.pathseparator, os.platform = "/" , ":", os.type or "unix" + io.fileseparator, io.pathseparator, os.type = "/" , ":", os.type or "unix" end end -os.platform = os.platform or os.type or (io.pathseparator == ";" and "windows") or "unix" +os.type = os.type or (io.pathseparator == ";" and "windows") or "unix" +os.name = os.name or (os.type == "windows" and "mswin" ) or "linux" + +if os.type == "windows" then + os.libsuffix, os.binsuffix = 'dll', 'exe' +else + os.libsuffix, os.binsuffix = 'so', '' +end function os.launch(str) - if os.platform == "windows" then + if os.type == "windows" then os.execute("start " .. str) -- os.spawn ? else os.execute(str .. " &") -- os.spawn ? end end -if not os.setenv then - function os.setenv() return false end -end - if not os.times then -- utime = user time -- stime = system time @@ -1609,64 +1804,218 @@ end --~ print(os.date("%H:%M:%S",os.gettimeofday())) --~ print(os.date("%H:%M:%S",os.time())) -os.arch = os.arch or function() - local a = os.resultof("uname -m") or "linux" - os.arch = function() - return a +-- no need for function anymore as we have more clever code and helpers now +-- this metatable trickery might as well disappear + +os.resolvers = os.resolvers or { } + +local resolvers = os.resolvers + +local osmt = getmetatable(os) or { __index = function(t,k) t[k] = "unset" return "unset" end } -- maybe nil +local osix = osmt.__index + +osmt.__index = function(t,k) + return (resolvers[k] or osix)(t,k) +end + +setmetatable(os,osmt) + +if not os.setenv then + + -- we still store them but they won't be seen in + -- child processes although we might pass them some day + -- using command concatination + + local env, getenv = { }, os.getenv + + function os.setenv(k,v) + env[k] = v end - return a + + function os.getenv(k) + return env[k] or getenv(k) + end + end -local platform +-- we can use HOSTTYPE on some platforms -function os.currentplatform(name,default) - if not platform then - local name = os.name or os.platform or name -- os.name is built in, os.platform is mine - if not name then - platform = default or "linux" - elseif name == "windows" or name == "mswin" or name == "win32" or name == "msdos" then - if os.getenv("PROCESSOR_ARCHITECTURE") == "AMD64" then - platform = "mswin-64" - else - platform = "mswin" - end +local name, platform = os.name or "linux", os.getenv("MTX_PLATFORM") or "" + +local function guess() + local architecture = os.resultof("uname -m") or "" + if architecture ~= "" then + return architecture + end + architecture = os.getenv("HOSTTYPE") or "" + if architecture ~= "" then + return architecture + end + return os.resultof("echo $HOSTTYPE") or "" +end + +if platform ~= "" then + + os.platform = platform + +elseif os.type == "windows" then + + -- we could set the variable directly, no function needed here + + function os.resolvers.platform(t,k) + local platform, architecture = "", os.getenv("PROCESSOR_ARCHITECTURE") or "" + if find(architecture,"AMD64") then + platform = "mswin-64" else - local architecture = os.arch() - if name == "linux" then - if find(architecture,"x86_64") then - platform = "linux-64" - elseif find(architecture,"ppc") then - platform = "linux-ppc" - else - platform = "linux" - end - elseif name == "macosx" then - if find(architecture,"i386") then - platform = "osx-intel" - else - platform = "osx-ppc" - end - elseif name == "sunos" then - if find(architecture,"sparc") then - platform = "solaris-sparc" - else -- if architecture == 'i86pc' - platform = "solaris-intel" - end - elseif name == "freebsd" then - if find(architecture,"amd64") then - platform = "freebsd-amd64" - else - platform = "freebsd" - end - else - platform = default or name - end + platform = "mswin" end - function os.currentplatform() - return platform + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +elseif name == "linux" then + + function os.resolvers.platform(t,k) + -- we sometims have HOSTTYPE set so let's check that first + local platform, architecture = "", os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" + if find(architecture,"x86_64") then + platform = "linux-64" + elseif find(architecture,"ppc") then + platform = "linux-ppc" + else + platform = "linux" + end + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +elseif name == "macosx" then + + --[[ + Identifying the architecture of OSX is quite a mess and this + is the best we can come up with. For some reason $HOSTTYPE is + a kind of pseudo environment variable, not known to the current + environment. And yes, uname cannot be trusted either, so there + is a change that you end up with a 32 bit run on a 64 bit system. + Also, some proper 64 bit intel macs are too cheap (low-end) and + therefore not permitted to run the 64 bit kernel. + ]]-- + + function os.resolvers.platform(t,k) + -- local platform, architecture = "", os.getenv("HOSTTYPE") or "" + -- if architecture == "" then + -- architecture = os.resultof("echo $HOSTTYPE") or "" + -- end + local platform, architecture = "", os.resultof("echo $HOSTTYPE") or "" + if architecture == "" then + -- print("\nI have no clue what kind of OSX you're running so let's assume an 32 bit intel.\n") + platform = "osx-intel" + elseif find(architecture,"i386") then + platform = "osx-intel" + elseif find(architecture,"x86_64") then + platform = "osx-64" + else + platform = "osx-ppc" + end + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +elseif name == "sunos" then + + function os.resolvers.platform(t,k) + local platform, architecture = "", os.resultof("uname -m") or "" + if find(architecture,"sparc") then + platform = "solaris-sparc" + else -- if architecture == 'i86pc' + platform = "solaris-intel" + end + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +elseif name == "freebsd" then + + function os.resolvers.platform(t,k) + local platform, architecture = "", os.resultof("uname -m") or "" + if find(architecture,"amd64") then + platform = "freebsd-amd64" + else + platform = "freebsd" + end + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +elseif name == "kfreebsd" then + + function os.resolvers.platform(t,k) + -- we sometims have HOSTTYPE set so let's check that first + local platform, architecture = "", os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" + if find(architecture,"x86_64") then + platform = "kfreebsd-64" + else + platform = "kfreebsd-i386" + end + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +else + + -- platform = "linux" + -- os.setenv("MTX_PLATFORM",platform) + -- os.platform = platform + + function os.resolvers.platform(t,k) + local platform = "linux" + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +end + +-- beware, we set the randomseed + +-- from wikipedia: Version 4 UUIDs use a scheme relying only on random numbers. This algorithm sets the +-- version number as well as two reserved bits. All other bits are set using a random or pseudorandom +-- data source. Version 4 UUIDs have the form xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx with hexadecimal +-- digits x and hexadecimal digits 8, 9, A, or B for y. e.g. f47ac10b-58cc-4372-a567-0e02b2c3d479. +-- +-- as we don't call this function too often there is not so much risk on repetition + +local t = { 8, 9, "a", "b" } + +function os.uuid() + return format("%04x%04x-4%03x-%s%03x-%04x-%04x%04x%04x", + random(0xFFFF),random(0xFFFF), + random(0x0FFF), + t[ceil(random(4))] or 8,random(0x0FFF), + random(0xFFFF), + random(0xFFFF),random(0xFFFF),random(0xFFFF) + ) +end + +local d + +function os.timezone(delta) + d = d or tonumber(tonumber(os.date("%H")-os.date("!%H"))) + if delta then + if d > 0 then + return format("+%02i:00",d) + else + return format("-%02i:00",-d) end + else + return 1 end - return platform end @@ -1676,7 +2025,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-file'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -1687,14 +2036,17 @@ if not modules then modules = { } end modules ['l-file'] = { file = file or { } local concat = table.concat -local find, gmatch, match, gsub = string.find, string.gmatch, string.match, string.gsub +local find, gmatch, match, gsub, sub, char = string.find, string.gmatch, string.match, string.gsub, string.sub, string.char +local lpegmatch = lpeg.match function file.removesuffix(filename) return (gsub(filename,"%.[%a%d]+$","")) end function file.addsuffix(filename, suffix) - if not find(filename,"%.[%a%d]+$") then + if not suffix or suffix == "" then + return filename + elseif not find(filename,"%.[%a%d]+$") then return filename .. "." .. suffix else return filename @@ -1717,20 +2069,39 @@ function file.nameonly(name) return (gsub(match(name,"^.+[/\\](.-)$") or name,"%..*$","")) end -function file.extname(name) - return match(name,"^.+%.([^/\\]-)$") or "" +function file.extname(name,default) + return match(name,"^.+%.([^/\\]-)$") or default or "" end file.suffix = file.extname ---~ print(file.join("x/","/y")) ---~ print(file.join("http://","/y")) ---~ print(file.join("http://a","/y")) ---~ print(file.join("http:///a","/y")) ---~ print(file.join("//nas-1","/y")) +--~ function file.join(...) +--~ local pth = concat({...},"/") +--~ pth = gsub(pth,"\\","/") +--~ local a, b = match(pth,"^(.*://)(.*)$") +--~ if a and b then +--~ return a .. gsub(b,"//+","/") +--~ end +--~ a, b = match(pth,"^(//)(.*)$") +--~ if a and b then +--~ return a .. gsub(b,"//+","/") +--~ end +--~ return (gsub(pth,"//+","/")) +--~ end + +local trick_1 = char(1) +local trick_2 = "^" .. trick_1 .. "/+" function file.join(...) - local pth = concat({...},"/") + local lst = { ... } + local a, b = lst[1], lst[2] + if a == "" then + lst[1] = trick_1 + elseif b and find(a,"^/+$") and find(b,"^/") then + lst[1] = "" + lst[2] = gsub(b,"^/+","") + end + local pth = concat(lst,"/") pth = gsub(pth,"\\","/") local a, b = match(pth,"^(.*://)(.*)$") if a and b then @@ -1740,17 +2111,28 @@ function file.join(...) if a and b then return a .. gsub(b,"//+","/") end + pth = gsub(pth,trick_2,"") return (gsub(pth,"//+","/")) end +--~ print(file.join("//","/y")) +--~ print(file.join("/","/y")) +--~ print(file.join("","/y")) +--~ print(file.join("/x/","/y")) +--~ print(file.join("x/","/y")) +--~ print(file.join("http://","/y")) +--~ print(file.join("http://a","/y")) +--~ print(file.join("http:///a","/y")) +--~ print(file.join("//nas-1","/y")) + function file.iswritable(name) local a = lfs.attributes(name) or lfs.attributes(file.dirname(name,".")) - return a and a.permissions:sub(2,2) == "w" + return a and sub(a.permissions,2,2) == "w" end function file.isreadable(name) local a = lfs.attributes(name) - return a and a.permissions:sub(1,1) == "r" + return a and sub(a.permissions,1,1) == "r" end file.is_readable = file.isreadable @@ -1758,36 +2140,50 @@ file.is_writable = file.iswritable -- todo: lpeg -function file.split_path(str) - local t = { } - str = gsub(str,"\\", "/") - str = gsub(str,"(%a):([;/])", "%1\001%2") - for name in gmatch(str,"([^;:]+)") do - if name ~= "" then - t[#t+1] = gsub(name,"\001",":") - end - end - return t +--~ function file.split_path(str) +--~ local t = { } +--~ str = gsub(str,"\\", "/") +--~ str = gsub(str,"(%a):([;/])", "%1\001%2") +--~ for name in gmatch(str,"([^;:]+)") do +--~ if name ~= "" then +--~ t[#t+1] = gsub(name,"\001",":") +--~ end +--~ end +--~ return t +--~ end + +local checkedsplit = string.checkedsplit + +function file.split_path(str,separator) + str = gsub(str,"\\","/") + return checkedsplit(str,separator or io.pathseparator) end function file.join_path(tab) return concat(tab,io.pathseparator) -- can have trailing // end +-- we can hash them weakly + function file.collapse_path(str) - str = gsub(str,"/%./","/") - local n, m = 1, 1 - while n > 0 or m > 0 do - str, n = gsub(str,"[^/%.]+/%.%.$","") - str, m = gsub(str,"[^/%.]+/%.%./","") - end - str = gsub(str,"([^/])/$","%1") - str = gsub(str,"^%./","") - str = gsub(str,"/%.$","") + str = gsub(str,"\\","/") + if find(str,"/") then + str = gsub(str,"^%./",(gsub(lfs.currentdir(),"\\","/")) .. "/") -- ./xx in qualified + str = gsub(str,"/%./","/") + local n, m = 1, 1 + while n > 0 or m > 0 do + str, n = gsub(str,"[^/%.]+/%.%.$","") + str, m = gsub(str,"[^/%.]+/%.%./","") + end + str = gsub(str,"([^/])/$","%1") + -- str = gsub(str,"^%./","") -- ./xx in qualified + str = gsub(str,"/%.$","") + end if str == "" then str = "." end return str end +--~ print(file.collapse_path("/a")) --~ print(file.collapse_path("a/./b/..")) --~ print(file.collapse_path("a/aa/../b/bb")) --~ print(file.collapse_path("a/../..")) @@ -1817,27 +2213,27 @@ end --~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * lpeg.C(noperiod^1) * -1 --~ function file.extname(name) ---~ return pattern:match(name) or "" +--~ return lpegmatch(pattern,name) or "" --~ end --~ local pattern = lpeg.Cs(((period * noperiod^1 * -1)/"" + 1)^1) --~ function file.removesuffix(name) ---~ return pattern:match(name) +--~ return lpegmatch(pattern,name) --~ end --~ local pattern = (noslashes^0 * slashes)^1 * lpeg.C(noslashes^1) * -1 --~ function file.basename(name) ---~ return pattern:match(name) or name +--~ return lpegmatch(pattern,name) or name --~ end --~ local pattern = (noslashes^0 * slashes)^1 * lpeg.Cp() * noslashes^1 * -1 --~ function file.dirname(name) ---~ local p = pattern:match(name) +--~ local p = lpegmatch(pattern,name) --~ if p then ---~ return name:sub(1,p-2) +--~ return sub(name,1,p-2) --~ else --~ return "" --~ end @@ -1846,7 +2242,7 @@ end --~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * lpeg.Cp() * noperiod^1 * -1 --~ function file.addsuffix(name, suffix) ---~ local p = pattern:match(name) +--~ local p = lpegmatch(pattern,name) --~ if p then --~ return name --~ else @@ -1857,9 +2253,9 @@ end --~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * lpeg.Cp() * noperiod^1 * -1 --~ function file.replacesuffix(name,suffix) ---~ local p = pattern:match(name) +--~ local p = lpegmatch(pattern,name) --~ if p then ---~ return name:sub(1,p-2) .. "." .. suffix +--~ return sub(name,1,p-2) .. "." .. suffix --~ else --~ return name .. "." .. suffix --~ end @@ -1868,11 +2264,11 @@ end --~ local pattern = (noslashes^0 * slashes)^0 * lpeg.Cp() * ((noperiod^1 * period)^1 * lpeg.Cp() + lpeg.P(true)) * noperiod^1 * -1 --~ function file.nameonly(name) ---~ local a, b = pattern:match(name) +--~ local a, b = lpegmatch(pattern,name) --~ if b then ---~ return name:sub(a,b-2) +--~ return sub(name,a,b-2) --~ elseif a then ---~ return name:sub(a) +--~ return sub(name,a) --~ else --~ return name --~ end @@ -1906,11 +2302,11 @@ local rootbased = lpeg.P("/") + letter*lpeg.P(":") -- ./name ../name /name c: :// name/name function file.is_qualified_path(filename) - return qualified:match(filename) + return lpegmatch(qualified,filename) ~= nil end function file.is_rootbased_path(filename) - return rootbased:match(filename) + return lpegmatch(rootbased,filename) ~= nil end local slash = lpeg.S("\\/") @@ -1923,16 +2319,25 @@ local base = lpeg.C((1-suffix)^0) local pattern = (drive + lpeg.Cc("")) * (path + lpeg.Cc("")) * (base + lpeg.Cc("")) * (suffix + lpeg.Cc("")) function file.splitname(str) -- returns drive, path, base, suffix - return pattern:match(str) + return lpegmatch(pattern,str) end --- function test(t) for k, v in pairs(t) do print(v, "=>", file.splitname(v)) end end +-- function test(t) for k, v in next, t do print(v, "=>", file.splitname(v)) end end -- -- test { "c:", "c:/aa", "c:/aa/bb", "c:/aa/bb/cc", "c:/aa/bb/cc.dd", "c:/aa/bb/cc.dd.ee" } -- test { "c:", "c:aa", "c:aa/bb", "c:aa/bb/cc", "c:aa/bb/cc.dd", "c:aa/bb/cc.dd.ee" } -- test { "/aa", "/aa/bb", "/aa/bb/cc", "/aa/bb/cc.dd", "/aa/bb/cc.dd.ee" } -- test { "aa", "aa/bb", "aa/bb/cc", "aa/bb/cc.dd", "aa/bb/cc.dd.ee" } +--~ -- todo: +--~ +--~ if os.type == "windows" then +--~ local currentdir = lfs.currentdir +--~ function lfs.currentdir() +--~ return (gsub(currentdir(),"\\","/")) +--~ end +--~ end + end -- of closure @@ -1997,7 +2402,7 @@ end function file.loadchecksum(name) if md5 then local data = io.loaddata(name .. ".md5") - return data and data:gsub("%s","") + return data and (gsub(data,"%s","")) end return nil end @@ -2018,14 +2423,15 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-url'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -local char, gmatch = string.char, string.gmatch +local char, gmatch, gsub = string.char, string.gmatch, string.gsub local tonumber, type = tonumber, type +local lpegmatch = lpeg.match -- from the spec (on the web): -- @@ -2049,7 +2455,9 @@ local hexdigit = lpeg.R("09","AF","af") local plus = lpeg.P("+") local escaped = (plus / " ") + (percent * lpeg.C(hexdigit * hexdigit) / tochar) -local scheme = lpeg.Cs((escaped+(1-colon-slash-qmark-hash))^0) * colon + lpeg.Cc("") +-- we assume schemes with more than 1 character (in order to avoid problems with windows disks) + +local scheme = lpeg.Cs((escaped+(1-colon-slash-qmark-hash))^2) * colon + lpeg.Cc("") local authority = slash * slash * lpeg.Cs((escaped+(1- slash-qmark-hash))^0) + lpeg.Cc("") local path = slash * lpeg.Cs((escaped+(1- qmark-hash))^0) + lpeg.Cc("") local query = qmark * lpeg.Cs((escaped+(1- hash))^0) + lpeg.Cc("") @@ -2057,25 +2465,51 @@ local fragment = hash * lpeg.Cs((escaped+(1- endofstring))^0 local parser = lpeg.Ct(scheme * authority * path * query * fragment) +-- todo: reconsider Ct as we can as well have five return values (saves a table) +-- so we can have two parsers, one with and one without + function url.split(str) - return (type(str) == "string" and parser:match(str)) or str + return (type(str) == "string" and lpegmatch(parser,str)) or str end +-- todo: cache them + function url.hashed(str) local s = url.split(str) + local somescheme = s[1] ~= "" return { - scheme = (s[1] ~= "" and s[1]) or "file", + scheme = (somescheme and s[1]) or "file", authority = s[2], - path = s[3], - query = s[4], - fragment = s[5], - original = str + path = s[3], + query = s[4], + fragment = s[5], + original = str, + noscheme = not somescheme, } end +function url.hasscheme(str) + return url.split(str)[1] ~= "" +end + +function url.addscheme(str,scheme) + return (url.hasscheme(str) and str) or ((scheme or "file:///") .. str) +end + +function url.construct(hash) + local fullurl = hash.sheme .. "://".. hash.authority .. hash.path + if hash.query then + fullurl = fullurl .. "?".. hash.query + end + if hash.fragment then + fullurl = fullurl .. "?".. hash.fragment + end + return fullurl +end + function url.filename(filename) local t = url.hashed(filename) - return (t.scheme == "file" and t.path:gsub("^/([a-zA-Z])([:|])/)","%1:")) or filename + return (t.scheme == "file" and (gsub(t.path,"^/([a-zA-Z])([:|])/)","%1:"))) or filename end function url.query(str) @@ -2129,17 +2563,26 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-dir'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } +-- dir.expand_name will be merged with cleanpath and collapsepath + local type = type -local find, gmatch = string.find, string.gmatch +local find, gmatch, match, gsub = string.find, string.gmatch, string.match, string.gsub +local lpegmatch = lpeg.match dir = dir or { } +-- handy + +function dir.current() + return (gsub(lfs.currentdir(),"\\","/")) +end + -- optimizing for no string.find (*) does not save time local attributes = lfs.attributes @@ -2170,6 +2613,35 @@ end dir.glob_pattern = glob_pattern +local function collect_pattern(path,patt,recurse,result) + local ok, scanner + result = result or { } + if path == "/" then + ok, scanner = xpcall(function() return walkdir(path..".") end, function() end) -- kepler safe + else + ok, scanner = xpcall(function() return walkdir(path) end, function() end) -- kepler safe + end + if ok and type(scanner) == "function" then + if not find(path,"/$") then path = path .. '/' end + for name in scanner do + local full = path .. name + local attr = attributes(full) + local mode = attr.mode + if mode == 'file' then + if find(full,patt) then + result[name] = attr + end + elseif recurse and (mode == "directory") and (name ~= '.') and (name ~= "..") then + attr.list = collect_pattern(full,patt,recurse) + result[name] = attr + end + end + end + return result +end + +dir.collect_pattern = collect_pattern + local P, S, R, C, Cc, Cs, Ct, Cv, V = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.Cc, lpeg.Cs, lpeg.Ct, lpeg.Cv, lpeg.V local pattern = Ct { @@ -2189,29 +2661,48 @@ local filter = Cs ( ( )^0 ) local function glob(str,t) - if type(str) == "table" then - local t = t or { } - for s=1,#str do - glob(str[s],t) + if type(t) == "function" then + if type(str) == "table" then + for s=1,#str do + glob(str[s],t) + end + elseif lfs.isfile(str) then + t(str) + else + local split = lpegmatch(pattern,str) + if split then + local root, path, base = split[1], split[2], split[3] + local recurse = find(base,"%*%*") + local start = root .. path + local result = lpegmatch(filter,start .. base) + glob_pattern(start,result,recurse,t) + end end - return t - elseif lfs.isfile(str) then - local t = t or { } - t[#t+1] = str - return t else - local split = pattern:match(str) - if split then + if type(str) == "table" then local t = t or { } - local action = action or function(name) t[#t+1] = name end - local root, path, base = split[1], split[2], split[3] - local recurse = find(base,"%*%*") - local start = root .. path - local result = filter:match(start .. base) - glob_pattern(start,result,recurse,action) + for s=1,#str do + glob(str[s],t) + end + return t + elseif lfs.isfile(str) then + local t = t or { } + t[#t+1] = str return t else - return { } + local split = lpegmatch(pattern,str) + if split then + local t = t or { } + local action = action or function(name) t[#t+1] = name end + local root, path, base = split[1], split[2], split[3] + local recurse = find(base,"%*%*") + local start = root .. path + local result = lpegmatch(filter,start .. base) + glob_pattern(start,result,recurse,action) + return t + else + return { } + end end end end @@ -2273,11 +2764,12 @@ end local make_indeed = true -- false -if string.find(os.getenv("PATH"),";") then +if string.find(os.getenv("PATH"),";") then -- os.type == "windows" function dir.mkdirs(...) - local str, pth = "", "" - for _, s in ipairs({...}) do + local str, pth, t = "", "", { ... } + for i=1,#t do + local s = t[i] if s ~= "" then if str ~= "" then str = str .. "/" .. s @@ -2288,13 +2780,13 @@ if string.find(os.getenv("PATH"),";") then end local first, middle, last local drive = false - first, middle, last = str:match("^(//)(//*)(.*)$") + first, middle, last = match(str,"^(//)(//*)(.*)$") if first then -- empty network path == local path else - first, last = str:match("^(//)/*(.-)$") + first, last = match(str,"^(//)/*(.-)$") if first then - middle, last = str:match("([^/]+)/+(.-)$") + middle, last = match(str,"([^/]+)/+(.-)$") if middle then pth = "//" .. middle else @@ -2302,11 +2794,11 @@ if string.find(os.getenv("PATH"),";") then last = "" end else - first, middle, last = str:match("^([a-zA-Z]:)(/*)(.-)$") + first, middle, last = match(str,"^([a-zA-Z]:)(/*)(.-)$") if first then pth, drive = first .. middle, true else - middle, last = str:match("^(/*)(.-)$") + middle, last = match(str,"^(/*)(.-)$") if not middle then last = str end @@ -2340,34 +2832,31 @@ if string.find(os.getenv("PATH"),";") then --~ print(dir.mkdirs("///a/b/c")) --~ print(dir.mkdirs("a/bbb//ccc/")) - function dir.expand_name(str) - local first, nothing, last = str:match("^(//)(//*)(.*)$") + function dir.expand_name(str) -- will be merged with cleanpath and collapsepath + local first, nothing, last = match(str,"^(//)(//*)(.*)$") if first then - first = lfs.currentdir() .. "/" - first = first:gsub("\\","/") + first = dir.current() .. "/" end if not first then - first, last = str:match("^(//)/*(.*)$") + first, last = match(str,"^(//)/*(.*)$") end if not first then - first, last = str:match("^([a-zA-Z]:)(.*)$") + first, last = match(str,"^([a-zA-Z]:)(.*)$") if first and not find(last,"^/") then local d = lfs.currentdir() if lfs.chdir(first) then - first = lfs.currentdir() - first = first:gsub("\\","/") + first = dir.current() end lfs.chdir(d) end end if not first then - first, last = lfs.currentdir(), str - first = first:gsub("\\","/") + first, last = dir.current(), str end - last = last:gsub("//","/") - last = last:gsub("/%./","/") - last = last:gsub("^/*","") - first = first:gsub("/*$","") + last = gsub(last,"//","/") + last = gsub(last,"/%./","/") + last = gsub(last,"^/*","") + first = gsub(first,"/*$","") if last == "" then return first else @@ -2378,8 +2867,9 @@ if string.find(os.getenv("PATH"),";") then else function dir.mkdirs(...) - local str, pth = "", "" - for _, s in ipairs({...}) do + local str, pth, t = "", "", { ... } + for i=1,#t do + local s = t[i] if s ~= "" then if str ~= "" then str = str .. "/" .. s @@ -2388,7 +2878,7 @@ else end end end - str = str:gsub("/+","/") + str = gsub(str,"/+","/") if find(str,"^/") then pth = "/" for s in gmatch(str,"[^/]+") do @@ -2422,12 +2912,12 @@ else --~ print(dir.mkdirs("///a/b/c")) --~ print(dir.mkdirs("a/bbb//ccc/")) - function dir.expand_name(str) + function dir.expand_name(str) -- will be merged with cleanpath and collapsepath if not find(str,"^/") then str = lfs.currentdir() .. "/" .. str end - str = str:gsub("//","/") - str = str:gsub("/%./","/") + str = gsub(str,"//","/") + str = gsub(str,"/%./","/") return str end @@ -2442,7 +2932,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-boolean'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -2503,19 +2993,40 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-unicode'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } +if not unicode then + + unicode = { utf8 = { } } + + local floor, char = math.floor, string.char + + function unicode.utf8.utfchar(n) + if n < 0x80 then + return char(n) + elseif n < 0x800 then + return char(0xC0 + floor(n/0x40)) .. char(0x80 + (n % 0x40)) + elseif n < 0x10000 then + return char(0xE0 + floor(n/0x1000)) .. char(0x80 + (floor(n/0x40) % 0x40)) .. char(0x80 + (n % 0x40)) + elseif n < 0x40000 then + return char(0xF0 + floor(n/0x40000)) .. char(0x80 + floor(n/0x1000)) .. char(0x80 + (floor(n/0x40) % 0x40)) .. char(0x80 + (n % 0x40)) + else -- wrong: + -- return char(0xF1 + floor(n/0x1000000)) .. char(0x80 + floor(n/0x40000)) .. char(0x80 + floor(n/0x1000)) .. char(0x80 + (floor(n/0x40) % 0x40)) .. char(0x80 + (n % 0x40)) + return "?" + end + end + +end + utf = utf or unicode.utf8 local concat, utfchar, utfgsub = table.concat, utf.char, utf.gsub local char, byte, find, bytepairs = string.char, string.byte, string.find, string.bytepairs -unicode = unicode or { } - -- 0 EF BB BF UTF-8 -- 1 FF FE UTF-16-little-endian -- 2 FE FF UTF-16-big-endian @@ -2530,14 +3041,20 @@ unicode.utfname = { [4] = 'utf-32-be' } -function unicode.utftype(f) -- \000 fails ! +-- \000 fails in <= 5.0 but is valid in >=5.1 where %z is depricated + +function unicode.utftype(f) local str = f:read(4) if not str then f:seek('set') return 0 - elseif find(str,"^%z%z\254\255") then + -- elseif find(str,"^%z%z\254\255") then -- depricated + -- elseif find(str,"^\000\000\254\255") then -- not permitted and bugged + elseif find(str,"\000\000\254\255",1,true) then -- seems to work okay (TH) return 4 - elseif find(str,"^\255\254%z%z") then + -- elseif find(str,"^\255\254%z%z") then -- depricated + -- elseif find(str,"^\255\254\000\000") then -- not permitted and bugged + elseif find(str,"\255\254\000\000",1,true) then -- seems to work okay (TH) return 3 elseif find(str,"^\254\255") then f:seek('set',2) @@ -2681,7 +3198,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-math'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -2728,7 +3245,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-utils'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -2736,6 +3253,10 @@ if not modules then modules = { } end modules ['l-utils'] = { -- hm, quite unreadable +local gsub = string.gsub +local concat = table.concat +local type, next = type, next + if not utils then utils = { } end if not utils.merger then utils.merger = { } end if not utils.lua then utils.lua = { } end @@ -2773,7 +3294,7 @@ function utils.merger._self_load_(name) end if data and utils.merger.strip_comment then -- saves some 20K - data = data:gsub("%-%-~[^\n\r]*[\r\n]", "") + data = gsub(data,"%-%-~[^\n\r]*[\r\n]", "") end return data or "" end @@ -2791,7 +3312,7 @@ end function utils.merger._self_swap_(data,code) if data ~= "" then - return (data:gsub(utils.merger.pattern, function(s) + return (gsub(data,utils.merger.pattern, function(s) return "\n\n" .. "-- "..utils.merger.m_begin .. "\n" .. code .. "\n" .. "-- "..utils.merger.m_end .. "\n\n" end, 1)) else @@ -2801,8 +3322,8 @@ end --~ stripper: --~ ---~ data = string.gsub(data,"%-%-~[^\n]*\n","") ---~ data = string.gsub(data,"\n\n+","\n") +--~ data = gsub(data,"%-%-~[^\n]*\n","") +--~ data = gsub(data,"\n\n+","\n") function utils.merger._self_libs_(libs,list) local result, f, frozen = { }, nil, false @@ -2810,9 +3331,10 @@ function utils.merger._self_libs_(libs,list) if type(libs) == 'string' then libs = { libs } end if type(list) == 'string' then list = { list } end local foundpath = nil - for _, lib in ipairs(libs) do - for _, pth in ipairs(list) do - pth = string.gsub(pth,"\\","/") -- file.clean_path + for i=1,#libs do + local lib = libs[i] + for j=1,#list do + local pth = gsub(list[j],"\\","/") -- file.clean_path utils.report("checking library path %s",pth) local name = pth .. "/" .. lib if lfs.isfile(name) then @@ -2824,7 +3346,8 @@ function utils.merger._self_libs_(libs,list) if foundpath then utils.report("using library path %s",foundpath) local right, wrong = { }, { } - for _, lib in ipairs(libs) do + for i=1,#libs do + local lib = libs[i] local fullname = foundpath .. "/" .. lib if lfs.isfile(fullname) then -- right[#right+1] = lib @@ -2838,15 +3361,15 @@ function utils.merger._self_libs_(libs,list) end end if #right > 0 then - utils.report("merged libraries: %s",table.concat(right," ")) + utils.report("merged libraries: %s",concat(right," ")) end if #wrong > 0 then - utils.report("skipped libraries: %s",table.concat(wrong," ")) + utils.report("skipped libraries: %s",concat(wrong," ")) end else utils.report("no valid library path found") end - return table.concat(result, "\n\n") + return concat(result, "\n\n") end function utils.merger.selfcreate(libs,list,target) @@ -2904,16 +3427,28 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-aux'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } +-- for inline, no store split : for s in string.gmatch(str,",* *([^,]+)") do .. end + aux = aux or { } local concat, format, gmatch = table.concat, string.format, string.gmatch local tostring, type = tostring, type +local lpegmatch = lpeg.match + +local P, R, V = lpeg.P, lpeg.R, lpeg.V + +local escape, left, right = P("\\"), P('{'), P('}') + +lpeg.patterns.balanced = P { + [1] = ((escape * (left+right)) + (1 - (left+right)) + V(2))^0, + [2] = left * V(1) * right +} local space = lpeg.P(' ') local equal = lpeg.P("=") @@ -2921,7 +3456,7 @@ local comma = lpeg.P(",") local lbrace = lpeg.P("{") local rbrace = lpeg.P("}") local nobrace = 1 - (lbrace+rbrace) -local nested = lpeg.P{ lbrace * (nobrace + lpeg.V(1))^0 * rbrace } +local nested = lpeg.P { lbrace * (nobrace + lpeg.V(1))^0 * rbrace } local spaces = space^0 local value = lpeg.P(lbrace * lpeg.C((nobrace + nested)^0) * rbrace) + lpeg.C((nested + (1-comma))^0) @@ -2959,13 +3494,13 @@ function aux.make_settings_to_hash_pattern(set,how) end end -function aux.settings_to_hash(str) +function aux.settings_to_hash(str,existing) if str and str ~= "" then - hash = { } + hash = existing or { } if moretolerant then - pattern_b_s:match(str) + lpegmatch(pattern_b_s,str) else - pattern_a_s:match(str) + lpegmatch(pattern_a_s,str) end return hash else @@ -2973,39 +3508,41 @@ function aux.settings_to_hash(str) end end -function aux.settings_to_hash_tolerant(str) +function aux.settings_to_hash_tolerant(str,existing) if str and str ~= "" then - hash = { } - pattern_b_s:match(str) + hash = existing or { } + lpegmatch(pattern_b_s,str) return hash else return { } end end -function aux.settings_to_hash_strict(str) +function aux.settings_to_hash_strict(str,existing) if str and str ~= "" then - hash = { } - pattern_c_s:match(str) + hash = existing or { } + lpegmatch(pattern_c_s,str) return next(hash) and hash else return nil end end -local seperator = comma * space^0 +local separator = comma * space^0 local value = lpeg.P(lbrace * lpeg.C((nobrace + nested)^0) * rbrace) + lpeg.C((nested + (1-comma))^0) -local pattern = lpeg.Ct(value*(seperator*value)^0) +local pattern = lpeg.Ct(value*(separator*value)^0) -- "aap, {noot}, mies" : outer {} removes, leading spaces ignored aux.settings_to_array_pattern = pattern +-- we could use a weak table as cache + function aux.settings_to_array(str) if not str or str == "" then return { } else - return pattern:match(str) + return lpegmatch(pattern,str) end end @@ -3014,10 +3551,10 @@ local function set(t,v) end local value = lpeg.P(lpeg.Carg(1)*value) / set -local pattern = value*(seperator*value)^0 * lpeg.Carg(1) +local pattern = value*(separator*value)^0 * lpeg.Carg(1) function aux.add_settings_to_array(t,str) - return pattern:match(str, nil, t) + return lpegmatch(pattern,str,nil,t) end function aux.hash_to_string(h,separator,yes,no,strict,omit) @@ -3065,6 +3602,13 @@ function aux.settings_to_set(str,t) return t end +local value = lbrace * lpeg.C((nobrace + nested)^0) * rbrace +local pattern = lpeg.Ct((space + value)^0) + +function aux.arguments_to_table(str) + return lpegmatch(pattern,str) +end + -- temporary here function aux.getparameters(self,class,parentclass,settings) @@ -3073,36 +3617,31 @@ function aux.getparameters(self,class,parentclass,settings) sc = table.clone(self[parent]) self[class] = sc end - aux.add_settings_to_array(sc, settings) + aux.settings_to_hash(settings,sc) end -- temporary here -local digit = lpeg.R("09") -local period = lpeg.P(".") -local zero = lpeg.P("0") - ---~ local finish = lpeg.P(-1) ---~ local nodigit = (1-digit) + finish ---~ local case_1 = (period * zero^1 * #nodigit)/"" -- .000 ---~ local case_2 = (period * (1-(zero^0/"") * #nodigit)^1 * (zero^0/"") * nodigit) -- .010 .10 .100100 - +local digit = lpeg.R("09") +local period = lpeg.P(".") +local zero = lpeg.P("0") local trailingzeros = zero^0 * -digit -- suggested by Roberto R -local case_1 = period * trailingzeros / "" -local case_2 = period * (digit - trailingzeros)^1 * (trailingzeros / "") - -local number = digit^1 * (case_1 + case_2) -local stripper = lpeg.Cs((number + 1)^0) +local case_1 = period * trailingzeros / "" +local case_2 = period * (digit - trailingzeros)^1 * (trailingzeros / "") +local number = digit^1 * (case_1 + case_2) +local stripper = lpeg.Cs((number + 1)^0) --~ local sample = "bla 11.00 bla 11 bla 0.1100 bla 1.00100 bla 0.00 bla 0.001 bla 1.1100 bla 0.100100100 bla 0.00100100100" --~ collectgarbage("collect") --~ str = string.rep(sample,10000) --~ local ts = os.clock() ---~ stripper:match(str) ---~ print(#str, os.clock()-ts, stripper:match(sample)) +--~ lpegmatch(stripper,str) +--~ print(#str, os.clock()-ts, lpegmatch(stripper,sample)) + +lpeg.patterns.strip_zeros = stripper function aux.strip_zeros(str) - return stripper:match(str) + return lpegmatch(stripper,str) end function aux.definetable(target) -- defines undefined tables @@ -3126,6 +3665,24 @@ function aux.accesstable(target) return t end +-- as we use this a lot ... + +--~ function aux.cachefunction(action,weak) +--~ local cache = { } +--~ if weak then +--~ setmetatable(cache, { __mode = "kv" } ) +--~ end +--~ local function reminder(str) +--~ local found = cache[str] +--~ if not found then +--~ found = action(str) +--~ cache[str] = found +--~ end +--~ return found +--~ end +--~ return reminder, cache +--~ end + end -- of closure @@ -3133,7 +3690,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['trac-tra'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to trac-tra.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -3143,12 +3700,17 @@ if not modules then modules = { } end modules ['trac-tra'] = { -- bound to a variable, like node.new, node.copy etc (contrary to for instance -- node.has_attribute which is bound to a has_attribute local variable in mkiv) +local debug = require "debug" + +local getinfo = debug.getinfo +local type, next = type, next +local concat = table.concat +local format, find, lower, gmatch, gsub = string.format, string.find, string.lower, string.gmatch, string.gsub + debugger = debugger or { } local counters = { } local names = { } -local getinfo = debug.getinfo -local format, find, lower, gmatch = string.format, string.find, string.lower, string.gmatch -- one @@ -3187,10 +3749,10 @@ function debugger.showstats(printer,threshold) local total, grandtotal, functions = 0, 0, 0 printer("\n") -- ugly but ok -- table.sort(counters) - for func, count in pairs(counters) do + for func, count in next, counters do if count > threshold then local name = getname(func) - if not name:find("for generator") then + if not find(name,"for generator") then printer(format("%8i %s", count, name)) total = total + count end @@ -3222,7 +3784,7 @@ end --~ local total, grandtotal, functions = 0, 0, 0 --~ printer("\n") -- ugly but ok --~ -- table.sort(counters) ---~ for func, count in pairs(counters) do +--~ for func, count in next, counters do --~ if count > threshold then --~ printer(format("%8i %s", count, func)) --~ total = total + count @@ -3276,38 +3838,77 @@ end --~ print("") --~ debugger.showstats(print,3) -trackers = trackers or { } +setters = setters or { } +setters.data = setters.data or { } -local data, done = { }, { } +--~ local function set(t,what,value) +--~ local data, done = t.data, t.done +--~ if type(what) == "string" then +--~ what = aux.settings_to_array(what) -- inefficient but ok +--~ end +--~ for i=1,#what do +--~ local w = what[i] +--~ for d, f in next, data do +--~ if done[d] then +--~ -- prevent recursion due to wildcards +--~ elseif find(d,w) then +--~ done[d] = true +--~ for i=1,#f do +--~ f[i](value) +--~ end +--~ end +--~ end +--~ end +--~ end -local function set(what,value) +local function set(t,what,value) + local data, done = t.data, t.done if type(what) == "string" then - what = aux.settings_to_array(what) + what = aux.settings_to_hash(what) -- inefficient but ok end - for i=1,#what do - local w = what[i] + for w, v in next, what do + if v == "" then + v = value + else + v = toboolean(v) + end for d, f in next, data do if done[d] then -- prevent recursion due to wildcards elseif find(d,w) then done[d] = true for i=1,#f do - f[i](value) + f[i](v) end end end end end -local function reset() - for d, f in next, data do +local function reset(t) + for d, f in next, t.data do for i=1,#f do f[i](false) end end end -function trackers.register(what,...) +local function enable(t,what) + set(t,what,true) +end + +local function disable(t,what) + local data = t.data + if not what or what == "" then + t.done = { } + reset(t) + else + set(t,what,false) + end +end + +function setters.register(t,what,...) + local data = t.data what = lower(what) local w = data[what] if not w then @@ -3319,32 +3920,32 @@ function trackers.register(what,...) if typ == "function" then w[#w+1] = fnc elseif typ == "string" then - w[#w+1] = function(value) set(fnc,value,nesting) end + w[#w+1] = function(value) set(t,fnc,value,nesting) end end end end -function trackers.enable(what) - done = { } - set(what,true) +function setters.enable(t,what) + local e = t.enable + t.enable, t.done = enable, { } + enable(t,string.simpleesc(tostring(what))) + t.enable, t.done = e, { } end -function trackers.disable(what) - done = { } - if not what or what == "" then - trackers.reset(what) - else - set(what,false) - end +function setters.disable(t,what) + local e = t.disable + t.disable, t.done = disable, { } + disable(t,string.simpleesc(tostring(what))) + t.disable, t.done = e, { } end -function trackers.reset(what) - done = { } - reset() +function setters.reset(t) + t.done = { } + reset(t) end -function trackers.list() -- pattern - local list = table.sortedkeys(data) +function setters.list(t) -- pattern + local list = table.sortedkeys(t.data) local user, system = { }, { } for l=1,#list do local what = list[l] @@ -3357,6 +3958,78 @@ function trackers.list() -- pattern return user, system end +function setters.show(t) + commands.writestatus("","") + local list = setters.list(t) + for k=1,#list do + commands.writestatus(t.name,list[k]) + end + commands.writestatus("","") +end + +-- we could have used a bit of oo and the trackers:enable syntax but +-- there is already a lot of code around using the singular tracker + +-- we could make this into a module + +function setters.new(name) + local t + t = { + data = { }, + name = name, + enable = function(...) setters.enable (t,...) end, + disable = function(...) setters.disable (t,...) end, + register = function(...) setters.register(t,...) end, + list = function(...) setters.list (t,...) end, + show = function(...) setters.show (t,...) end, + } + setters.data[name] = t + return t +end + +trackers = setters.new("trackers") +directives = setters.new("directives") +experiments = setters.new("experiments") + +-- nice trick: we overload two of the directives related functions with variants that +-- do tracing (itself using a tracker) .. proof of concept + +local trace_directives = false local trace_directives = false trackers.register("system.directives", function(v) trace_directives = v end) +local trace_experiments = false local trace_experiments = false trackers.register("system.experiments", function(v) trace_experiments = v end) + +local e = directives.enable +local d = directives.disable + +function directives.enable(...) + commands.writestatus("directives","enabling: %s",concat({...}," ")) + e(...) +end + +function directives.disable(...) + commands.writestatus("directives","disabling: %s",concat({...}," ")) + d(...) +end + +local e = experiments.enable +local d = experiments.disable + +function experiments.enable(...) + commands.writestatus("experiments","enabling: %s",concat({...}," ")) + e(...) +end + +function experiments.disable(...) + commands.writestatus("experiments","disabling: %s",concat({...}," ")) + d(...) +end + +-- a useful example + +directives.register("system.nostatistics", function(v) + statistics.enable = not v +end) + + end -- of closure @@ -3364,7 +4037,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['luat-env'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -3376,10 +4049,10 @@ if not modules then modules = { } end modules ['luat-env'] = { -- evolved before bytecode arrays were available and so a lot of -- code has disappeared already. -local trace_verbose = false trackers.register("resolvers.verbose", function(v) trace_verbose = v end) -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v trackers.enable("resolvers.verbose") end) +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) -local format = string.format +local format, sub, match, gsub, find = string.format, string.sub, string.match, string.gsub, string.find +local unquote, quote = string.unquote, string.quote -- precautions @@ -3413,13 +4086,14 @@ if not environment.jobname then environ function environment.initialize_arguments(arg) local arguments, files = { }, { } environment.arguments, environment.files, environment.sortedflags = arguments, files, nil - for index, argument in pairs(arg) do + for index=1,#arg do + local argument = arg[index] if index > 0 then - local flag, value = argument:match("^%-+(.+)=(.-)$") + local flag, value = match(argument,"^%-+(.-)=(.-)$") if flag then - arguments[flag] = string.unquote(value or "") + arguments[flag] = unquote(value or "") else - flag = argument:match("^%-+(.+)") + flag = match(argument,"^%-+(.+)") if flag then arguments[flag] = true else @@ -3446,25 +4120,30 @@ function environment.argument(name,partial) return arguments[name] elseif partial then if not sortedflags then - sortedflags = { } - for _,v in pairs(table.sortedkeys(arguments)) do - sortedflags[#sortedflags+1] = "^" .. v + sortedflags = table.sortedkeys(arguments) + for k=1,#sortedflags do + sortedflags[k] = "^" .. sortedflags[k] end environment.sortedflags = sortedflags end -- example of potential clash: ^mode ^modefile - for _,v in ipairs(sortedflags) do - if name:find(v) then - return arguments[v:sub(2,#v)] + for k=1,#sortedflags do + local v = sortedflags[k] + if find(name,v) then + return arguments[sub(v,2,#v)] end end end return nil end +environment.argument("x",true) + function environment.split_arguments(separator) -- rather special, cut-off before separator local done, before, after = false, { }, { } - for _,v in ipairs(environment.original_arguments) do + local original_arguments = environment.original_arguments + for k=1,#original_arguments do + local v = original_arguments[k] if not done and v == separator then done = true elseif done then @@ -3481,16 +4160,17 @@ function environment.reconstruct_commandline(arg,noquote) if noquote and #arg == 1 then local a = arg[1] a = resolvers.resolve(a) - a = a:unquote() + a = unquote(a) return a - elseif next(arg) then + elseif #arg > 0 then local result = { } - for _,a in ipairs(arg) do -- ipairs 1 .. #n + for i=1,#arg do + local a = arg[i] a = resolvers.resolve(a) - a = a:unquote() - a = a:gsub('"','\\"') -- tricky - if a:find(" ") then - result[#result+1] = a:quote() + a = unquote(a) + a = gsub(a,'"','\\"') -- tricky + if find(a," ") then + result[#result+1] = quote(a) else result[#result+1] = a end @@ -3503,17 +4183,18 @@ end if arg then - -- new, reconstruct quoted snippets (maybe better just remnove the " then and add them later) + -- new, reconstruct quoted snippets (maybe better just remove the " then and add them later) local newarg, instring = { }, false - for index, argument in ipairs(arg) do - if argument:find("^\"") then - newarg[#newarg+1] = argument:gsub("^\"","") - if not argument:find("\"$") then + for index=1,#arg do + local argument = arg[index] + if find(argument,"^\"") then + newarg[#newarg+1] = gsub(argument,"^\"","") + if not find(argument,"\"$") then instring = true end - elseif argument:find("\"$") then - newarg[#newarg] = newarg[#newarg] .. " " .. argument:gsub("\"$","") + elseif find(argument,"\"$") then + newarg[#newarg] = newarg[#newarg] .. " " .. gsub(argument,"\"$","") instring = false elseif instring then newarg[#newarg] = newarg[#newarg] .. " " .. argument @@ -3568,12 +4249,12 @@ function environment.luafilechunk(filename) -- used for loading lua bytecode in filename = file.replacesuffix(filename, "lua") local fullname = environment.luafile(filename) if fullname and fullname ~= "" then - if trace_verbose then + if trace_locating then logs.report("fileio","loading file %s", fullname) end return environment.loadedluacode(fullname) else - if trace_verbose then + if trace_locating then logs.report("fileio","unknown file %s", filename) end return nil @@ -3593,7 +4274,7 @@ function environment.loadluafile(filename, version) -- when not overloaded by explicit suffix we look for a luc file first local fullname = (lucname and environment.luafile(lucname)) or "" if fullname ~= "" then - if trace_verbose then + if trace_locating then logs.report("fileio","loading %s", fullname) end chunk = loadfile(fullname) -- this way we don't need a file exists check @@ -3611,7 +4292,7 @@ function environment.loadluafile(filename, version) if v == version then return true else - if trace_verbose then + if trace_locating then logs.report("fileio","version mismatch for %s: lua=%s, luc=%s", filename, v, version) end environment.loadluafile(filename) @@ -3622,12 +4303,12 @@ function environment.loadluafile(filename, version) end fullname = (luaname and environment.luafile(luaname)) or "" if fullname ~= "" then - if trace_verbose then + if trace_locating then logs.report("fileio","loading %s", fullname) end chunk = loadfile(fullname) -- this way we don't need a file exists check if not chunk then - if verbose then + if trace_locating then logs.report("fileio","unknown file %s", filename) end else @@ -3645,7 +4326,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['trac-inf'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to trac-inf.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -3670,6 +4351,14 @@ function statistics.hastimer(instance) return instance and instance.starttime end +function statistics.resettiming(instance) + if not instance then + notimer = { timing = 0, loadtime = 0 } + else + instance.timing, instance.loadtime = 0, 0 + end +end + function statistics.starttiming(instance) if not instance then notimer = { } @@ -3684,6 +4373,8 @@ function statistics.starttiming(instance) if not instance.loadtime then instance.loadtime = 0 end + else +--~ logs.report("system","nested timing (%s)",tostring(instance)) end instance.timing = it + 1 end @@ -3729,6 +4420,12 @@ function statistics.elapsedindeed(instance) return t > statistics.threshold end +function statistics.elapsedseconds(instance,rest) -- returns nil if 0 seconds + if statistics.elapsedindeed(instance) then + return format("%s seconds %s", statistics.elapsedtime(instance),rest or "") + end +end + -- general function function statistics.register(tag,fnc) @@ -3807,14 +4504,32 @@ function statistics.timed(action,report) report("total runtime: %s",statistics.elapsedtime(timer)) end +-- where, not really the best spot for this: + +commands = commands or { } + +local timer + +function commands.resettimer() + statistics.resettiming(timer) + statistics.starttiming(timer) +end + +function commands.elapsedtime() + statistics.stoptiming(timer) + tex.sprint(statistics.elapsedtime(timer)) +end + +commands.resettimer() + end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['luat-log'] = { +if not modules then modules = { } end modules ['trac-log'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to trac-log.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -3822,7 +4537,11 @@ if not modules then modules = { } end modules ['luat-log'] = { -- this is old code that needs an overhaul -local write_nl, write, format = texio.write_nl or print, texio.write or io.write, string.format +--~ io.stdout:setvbuf("no") +--~ io.stderr:setvbuf("no") + +local write_nl, write = texio.write_nl or print, texio.write or io.write +local format, gmatch = string.format, string.gmatch local texcount = tex and tex.count if texlua then @@ -3903,25 +4622,48 @@ function logs.tex.line(fmt,...) -- new end end +--~ function logs.tex.start_page_number() +--~ local real, user, sub = texcount.realpageno, texcount.userpageno, texcount.subpageno +--~ if real > 0 then +--~ if user > 0 then +--~ if sub > 0 then +--~ write(format("[%s.%s.%s",real,user,sub)) +--~ else +--~ write(format("[%s.%s",real,user)) +--~ end +--~ else +--~ write(format("[%s",real)) +--~ end +--~ else +--~ write("[-") +--~ end +--~ end + +--~ function logs.tex.stop_page_number() +--~ write("]") +--~ end + +local real, user, sub + function logs.tex.start_page_number() - local real, user, sub = texcount.realpageno, texcount.userpageno, texcount.subpageno + real, user, sub = texcount.realpageno, texcount.userpageno, texcount.subpageno +end + +function logs.tex.stop_page_number() if real > 0 then if user > 0 then if sub > 0 then - write(format("[%s.%s.%s",real,user,sub)) + logs.report("pages", "flushing realpage %s, userpage %s, subpage %s",real,user,sub) else - write(format("[%s.%s",real,user)) + logs.report("pages", "flushing realpage %s, userpage %s",real,user) end else - write(format("[%s",real)) + logs.report("pages", "flushing realpage %s",real) end else - write("[-") + logs.report("pages", "flushing page") end -end - -function logs.tex.stop_page_number() - write("]") + io.flush() end logs.tex.report_job_stat = statistics.show_job_stat @@ -4021,7 +4763,7 @@ end function logs.setprogram(_name_,_banner_,_verbose_) name, banner = _name_, _banner_ if _verbose_ then - trackers.enable("resolvers.verbose") + trackers.enable("resolvers.locating") end logs.set_method("tex") logs.report = report -- also used in libraries @@ -4034,9 +4776,9 @@ end function logs.setverbose(what) if what then - trackers.enable("resolvers.verbose") + trackers.enable("resolvers.locating") else - trackers.disable("resolvers.verbose") + trackers.disable("resolvers.locating") end logs.verbose = what or false end @@ -4053,7 +4795,7 @@ logs.report = logs.tex.report logs.simple = logs.tex.report function logs.reportlines(str) -- todo: <lines></lines> - for line in str:gmatch("(.-)[\n\r]") do + for line in gmatch(str,"(.-)[\n\r]") do logs.report(line) end end @@ -4064,8 +4806,12 @@ end logs.simpleline = logs.reportline -function logs.help(message,option) +function logs.reportbanner() -- for scripts too logs.report(banner) +end + +function logs.help(message,option) + logs.reportbanner() logs.reportline() logs.reportlines(message) local moreinfo = logs.moreinfo or "" @@ -4097,6 +4843,11 @@ end --~ logs.system(syslogname,"context","test","fonts","font %s recached due to newer version (%s)","blabla","123") --~ end +function logs.fatal(where,...) + logs.report(where,"fatal error: %s, aborting now",format(...)) + os.exit() +end + end -- of closure @@ -4104,10 +4855,10 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-inp'] = { version = 1.001, + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files", - comment = "companion to luat-lib.tex", } -- After a few years using the code the large luat-inp.lua file @@ -4119,7 +4870,7 @@ if not modules then modules = { } end modules ['data-inp'] = { -- * some public auxiliary functions were made private -- -- TODO: os.getenv -> os.env[] --- TODO: instances.[hashes,cnffiles,configurations,522] -> ipairs (alles check, sneller) +-- TODO: instances.[hashes,cnffiles,configurations,522] -- TODO: check escaping in find etc, too much, too slow -- This lib is multi-purpose and can be loaded again later on so that @@ -4140,12 +4891,13 @@ if not modules then modules = { } end modules ['data-inp'] = { local format, gsub, find, lower, upper, match, gmatch = string.format, string.gsub, string.find, string.lower, string.upper, string.match, string.gmatch local concat, insert, sortedkeys = table.concat, table.insert, table.sortedkeys local next, type = next, type +local lpegmatch = lpeg.match -local trace_locating, trace_detail, trace_verbose = false, false, false +local trace_locating, trace_detail, trace_expansions = false, false, false -trackers.register("resolvers.verbose", function(v) trace_verbose = v end) -trackers.register("resolvers.locating", function(v) trace_locating = v trackers.enable("resolvers.verbose") end) -trackers.register("resolvers.detail", function(v) trace_detail = v trackers.enable("resolvers.verbose,resolvers.detail") end) +trackers.register("resolvers.locating", function(v) trace_locating = v end) +trackers.register("resolvers.details", function(v) trace_detail = v end) +trackers.register("resolvers.expansions", function(v) trace_expansions = v end) -- todo if not resolvers then resolvers = { @@ -4169,7 +4921,7 @@ resolvers.generators.notfound = { nil } resolvers.cacheversion = '1.0.1' resolvers.cnfname = 'texmf.cnf' resolvers.luaname = 'texmfcnf.lua' -resolvers.homedir = os.env[os.platform == "windows" and 'USERPROFILE'] or os.env['HOME'] or '~' +resolvers.homedir = os.env[os.type == "windows" and 'USERPROFILE'] or os.env['HOME'] or '~' resolvers.cnfdefault = '{$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c}' local dummy_path_expr = "^!*unset/*$" @@ -4211,8 +4963,8 @@ suffixes['lua'] = { 'lua', 'luc', 'tma', 'tmc' } alternatives['map files'] = 'map' alternatives['enc files'] = 'enc' -alternatives['cid files'] = 'cid' -alternatives['fea files'] = 'fea' +alternatives['cid maps'] = 'cid' -- great, why no cid files +alternatives['font feature files'] = 'fea' -- and fea files here alternatives['opentype fonts'] = 'otf' alternatives['truetype fonts'] = 'ttf' alternatives['truetype collections'] = 'ttc' @@ -4228,6 +4980,11 @@ formats ['sfd'] = 'SFDFONTS' suffixes ['sfd'] = { 'sfd' } alternatives['subfont definition files'] = 'sfd' +-- lib paths + +formats ['lib'] = 'CLUAINPUTS' -- new (needs checking) +suffixes['lib'] = (os.libsuffix and { os.libsuffix }) or { 'dll', 'so' } + -- In practice we will work within one tds tree, but i want to keep -- the option open to build tools that look at multiple trees, which is -- why we keep the tree specific data in a table. We used to pass the @@ -4350,8 +5107,10 @@ local function check_configuration() -- not yet ok, no time for debugging now -- bad luck end fix("LUAINPUTS" , ".;$TEXINPUTS;$TEXMFSCRIPTS") -- no progname, hm - fix("FONTFEATURES", ".;$TEXMF/fonts/fea//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") - fix("FONTCIDMAPS" , ".;$TEXMF/fonts/cid//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") + -- this will go away some day + fix("FONTFEATURES", ".;$TEXMF/fonts/{data,fea}//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") + fix("FONTCIDMAPS" , ".;$TEXMF/fonts/{data,cid}//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") + -- fix("LUATEXLIBS" , ".;$TEXMF/luatex/lua//") end @@ -4366,7 +5125,7 @@ function resolvers.settrace(n) -- no longer number but: 'locating' or 'detail' end end -resolvers.settrace(os.getenv("MTX.resolvers.TRACE") or os.getenv("MTX_INPUT_TRACE")) +resolvers.settrace(os.getenv("MTX_INPUT_TRACE")) function resolvers.osenv(key) local ie = instance.environment @@ -4454,37 +5213,43 @@ end -- work that well; the parsing is ok, but dealing with the resulting -- table is a pain because we need to work inside-out recursively +local function do_first(a,b) + local t = { } + for s in gmatch(b,"[^,]+") do t[#t+1] = a .. s end + return "{" .. concat(t,",") .. "}" +end + +local function do_second(a,b) + local t = { } + for s in gmatch(a,"[^,]+") do t[#t+1] = s .. b end + return "{" .. concat(t,",") .. "}" +end + +local function do_both(a,b) + local t = { } + for sa in gmatch(a,"[^,]+") do + for sb in gmatch(b,"[^,]+") do + t[#t+1] = sa .. sb + end + end + return "{" .. concat(t,",") .. "}" +end + +local function do_three(a,b,c) + return a .. b.. c +end + local function splitpathexpr(str, t, validate) -- no need for further optimization as it is only called a - -- few times, we can use lpeg for the sub; we could move - -- the local functions outside the body + -- few times, we can use lpeg for the sub + if trace_expansions then + logs.report("fileio","expanding variable '%s'",str) + end t = t or { } str = gsub(str,",}",",@}") str = gsub(str,"{,","{@,") -- str = "@" .. str .. "@" local ok, done - local function do_first(a,b) - local t = { } - for s in gmatch(b,"[^,]+") do t[#t+1] = a .. s end - return "{" .. concat(t,",") .. "}" - end - local function do_second(a,b) - local t = { } - for s in gmatch(a,"[^,]+") do t[#t+1] = s .. b end - return "{" .. concat(t,",") .. "}" - end - local function do_both(a,b) - local t = { } - for sa in gmatch(a,"[^,]+") do - for sb in gmatch(b,"[^,]+") do - t[#t+1] = sa .. sb - end - end - return "{" .. concat(t,",") .. "}" - end - local function do_three(a,b,c) - return a .. b.. c - end while true do done = false while true do @@ -4515,6 +5280,11 @@ local function splitpathexpr(str, t, validate) t[#t+1] = s end end + if trace_expansions then + for k=1,#t do + logs.report("fileio","% 4i: %s",k,t[k]) + end + end return t end @@ -4554,18 +5324,27 @@ end -- also we now follow the stupid route: if not set then just assume *one* -- cnf file under texmf (i.e. distribution) -resolvers.ownpath = resolvers.ownpath or nil -resolvers.ownbin = resolvers.ownbin or arg[-2] or arg[-1] or arg[0] or "luatex" -resolvers.autoselfdir = true -- false may be handy for debugging +local args = environment and environment.original_arguments or arg -- this needs a cleanup + +resolvers.ownbin = resolvers.ownbin or args[-2] or arg[-2] or args[-1] or arg[-1] or arg[0] or "luatex" +resolvers.ownbin = gsub(resolvers.ownbin,"\\","/") function resolvers.getownpath() - if not resolvers.ownpath then - if resolvers.autoselfdir and os.selfdir then - resolvers.ownpath = os.selfdir - else - local binary = resolvers.ownbin - if os.platform == "windows" then - binary = file.replacesuffix(binary,"exe") + local ownpath = resolvers.ownpath or os.selfdir + if not ownpath or ownpath == "" or ownpath == "unset" then + ownpath = args[-1] or arg[-1] + ownpath = ownpath and file.dirname(gsub(ownpath,"\\","/")) + if not ownpath or ownpath == "" then + ownpath = args[-0] or arg[-0] + ownpath = ownpath and file.dirname(gsub(ownpath,"\\","/")) + end + local binary = resolvers.ownbin + if not ownpath or ownpath == "" then + ownpath = ownpath and file.dirname(binary) + end + if not ownpath or ownpath == "" then + if os.binsuffix ~= "" then + binary = file.replacesuffix(binary,os.binsuffix) end for p in gmatch(os.getenv("PATH"),"[^"..io.pathseparator.."]+") do local b = file.join(p,binary) @@ -4577,30 +5356,39 @@ function resolvers.getownpath() local olddir = lfs.currentdir() if lfs.chdir(p) then local pp = lfs.currentdir() - if trace_verbose and p ~= pp then - logs.report("fileio","following symlink %s to %s",p,pp) + if trace_locating and p ~= pp then + logs.report("fileio","following symlink '%s' to '%s'",p,pp) end - resolvers.ownpath = pp + ownpath = pp lfs.chdir(olddir) else - if trace_verbose then - logs.report("fileio","unable to check path %s",p) + if trace_locating then + logs.report("fileio","unable to check path '%s'",p) end - resolvers.ownpath = p + ownpath = p end break end end end - if not resolvers.ownpath then resolvers.ownpath = '.' end + if not ownpath or ownpath == "" then + ownpath = "." + logs.report("fileio","forcing fallback ownpath .") + elseif trace_locating then + logs.report("fileio","using ownpath '%s'",ownpath) + end end - return resolvers.ownpath + resolvers.ownpath = ownpath + function resolvers.getownpath() + return resolvers.ownpath + end + return ownpath end local own_places = { "SELFAUTOLOC", "SELFAUTODIR", "SELFAUTOPARENT", "TEXMFCNF" } local function identify_own() - local ownpath = resolvers.getownpath() or lfs.currentdir() + local ownpath = resolvers.getownpath() or dir.current() local ie = instance.environment if ownpath then if resolvers.env('SELFAUTOLOC') == "" then os.env['SELFAUTOLOC'] = file.collapse_path(ownpath) end @@ -4613,10 +5401,10 @@ local function identify_own() if resolvers.env('TEXMFCNF') == "" then os.env['TEXMFCNF'] = resolvers.cnfdefault end if resolvers.env('TEXOS') == "" then os.env['TEXOS'] = resolvers.env('SELFAUTODIR') end if resolvers.env('TEXROOT') == "" then os.env['TEXROOT'] = resolvers.env('SELFAUTOPARENT') end - if trace_verbose then + if trace_locating then for i=1,#own_places do local v = own_places[i] - logs.report("fileio","variable %s set to %s",v,resolvers.env(v) or "unknown") + logs.report("fileio","variable '%s' set to '%s'",v,resolvers.env(v) or "unknown") end end identify_own = function() end @@ -4648,10 +5436,8 @@ end local function load_cnf_file(fname) fname = resolvers.clean_path(fname) local lname = file.replacesuffix(fname,'lua') - local f = io.open(lname) - if f then -- this will go - f:close() - local dname = file.dirname(fname) + if lfs.isfile(lname) then + local dname = file.dirname(fname) -- fname ? if not instance.configuration[dname] then resolvers.load_data(dname,'configuration',lname and file.basename(lname)) instance.order[#instance.order+1] = instance.configuration[dname] @@ -4659,8 +5445,8 @@ local function load_cnf_file(fname) else f = io.open(fname) if f then - if trace_verbose then - logs.report("fileio","loading %s", fname) + if trace_locating then + logs.report("fileio","loading configuration file %s", fname) end local line, data, n, k, v local dname = file.dirname(fname) @@ -4694,14 +5480,16 @@ local function load_cnf_file(fname) end end f:close() - elseif trace_verbose then - logs.report("fileio","skipping %s", fname) + elseif trace_locating then + logs.report("fileio","skipping configuration file '%s'", fname) end end end local function collapse_cnf_data() -- potential optimization: pass start index (setup and configuration are shared) - for _,c in ipairs(instance.order) do + local order = instance.order + for i=1,#order do + local c = order[i] for k,v in next, c do if not instance.variables[k] then if instance.environment[k] then @@ -4717,19 +5505,24 @@ end function resolvers.load_cnf() local function loadoldconfigdata() - for _, fname in ipairs(instance.cnffiles) do - load_cnf_file(fname) + local cnffiles = instance.cnffiles + for i=1,#cnffiles do + load_cnf_file(cnffiles[i]) end end -- instance.cnffiles contain complete names now ! + -- we still use a funny mix of cnf and new but soon + -- we will switch to lua exclusively as we only use + -- the file to collect the tree roots if #instance.cnffiles == 0 then - if trace_verbose then + if trace_locating then logs.report("fileio","no cnf files found (TEXMFCNF may not be set/known)") end else - instance.rootpath = instance.cnffiles[1] - for k,fname in ipairs(instance.cnffiles) do - instance.cnffiles[k] = file.collapse_path(gsub(fname,"\\",'/')) + local cnffiles = instance.cnffiles + instance.rootpath = cnffiles[1] + for k=1,#cnffiles do + instance.cnffiles[k] = file.collapse_path(cnffiles[k]) end for i=1,3 do instance.rootpath = file.dirname(instance.rootpath) @@ -4757,8 +5550,9 @@ function resolvers.load_lua() -- yet harmless else instance.rootpath = instance.luafiles[1] - for k,fname in ipairs(instance.luafiles) do - instance.luafiles[k] = file.collapse_path(gsub(fname,"\\",'/')) + local luafiles = instance.luafiles + for k=1,#luafiles do + instance.luafiles[k] = file.collapse_path(luafiles[k]) end for i=1,3 do instance.rootpath = file.dirname(instance.rootpath) @@ -4790,14 +5584,14 @@ end function resolvers.append_hash(type,tag,name) if trace_locating then - logs.report("fileio","= hash append: %s",tag) + logs.report("fileio","hash '%s' appended",tag) end insert(instance.hashes, { ['type']=type, ['tag']=tag, ['name']=name } ) end function resolvers.prepend_hash(type,tag,name) if trace_locating then - logs.report("fileio","= hash prepend: %s",tag) + logs.report("fileio","hash '%s' prepended",tag) end insert(instance.hashes, 1, { ['type']=type, ['tag']=tag, ['name']=name } ) end @@ -4821,9 +5615,11 @@ end -- locators function resolvers.locatelists() - for _, path in ipairs(resolvers.clean_path_list('TEXMF')) do - if trace_verbose then - logs.report("fileio","locating list of %s",path) + local texmfpaths = resolvers.clean_path_list('TEXMF') + for i=1,#texmfpaths do + local path = texmfpaths[i] + if trace_locating then + logs.report("fileio","locating list of '%s'",path) end resolvers.locatedatabase(file.collapse_path(path)) end @@ -4836,11 +5632,11 @@ end function resolvers.locators.tex(specification) if specification and specification ~= '' and lfs.isdir(specification) then if trace_locating then - logs.report("fileio",'! tex locator found: %s',specification) + logs.report("fileio","tex locator '%s' found",specification) end resolvers.append_hash('file',specification,filename) elseif trace_locating then - logs.report("fileio",'? tex locator not found: %s',specification) + logs.report("fileio","tex locator '%s' not found",specification) end end @@ -4854,7 +5650,9 @@ function resolvers.loadfiles() instance.loaderror = false instance.files = { } if not instance.renewcache then - for _, hash in ipairs(instance.hashes) do + local hashes = instance.hashes + for k=1,#hashes do + local hash = hashes[k] resolvers.hashdatabase(hash.tag,hash.name) if instance.loaderror then break end end @@ -4868,8 +5666,9 @@ end -- generators: function resolvers.loadlists() - for _, hash in ipairs(instance.hashes) do - resolvers.generatedatabase(hash.tag) + local hashes = instance.hashes + for i=1,#hashes do + resolvers.generatedatabase(hashes[i].tag) end end @@ -4881,10 +5680,27 @@ end local weird = lpeg.P(".")^1 + lpeg.anywhere(lpeg.S("~`!#$%^&*()={}[]:;\"\'||<>,?\n\r\t")) +--~ local l_forbidden = lpeg.S("~`!#$%^&*()={}[]:;\"\'||\\/<>,?\n\r\t") +--~ local l_confusing = lpeg.P(" ") +--~ local l_character = lpeg.patterns.utf8 +--~ local l_dangerous = lpeg.P(".") + +--~ local l_normal = (l_character - l_forbidden - l_confusing - l_dangerous) * (l_character - l_forbidden - l_confusing^2)^0 * lpeg.P(-1) +--~ ----- l_normal = l_normal * lpeg.Cc(true) + lpeg.Cc(false) + +--~ local function test(str) +--~ print(str,lpeg.match(l_normal,str)) +--~ end +--~ test("ヒラギノ明朝 Pro W3") +--~ test("..ヒラギノ明朝 Pro W3") +--~ test(":ヒラギノ明朝 Pro W3;") +--~ test("ヒラギノ明朝 /Pro W3;") +--~ test("ヒラギノ明朝 Pro W3") + function resolvers.generators.tex(specification) local tag = specification - if trace_verbose then - logs.report("fileio","scanning path %s",specification) + if trace_locating then + logs.report("fileio","scanning path '%s'",specification) end instance.files[tag] = { } local files = instance.files[tag] @@ -4900,7 +5716,8 @@ function resolvers.generators.tex(specification) full = spec end for name in directory(full) do - if not weird:match(name) then + if not lpegmatch(weird,name) then + -- if lpegmatch(l_normal,name) then local mode = attributes(full..name,'mode') if mode == 'file' then if path then @@ -4933,7 +5750,7 @@ function resolvers.generators.tex(specification) end end action() - if trace_verbose then + if trace_locating then logs.report("fileio","%s files found on %s directories with %s uppercase remappings",n,m,r) end end @@ -4948,11 +5765,48 @@ end -- we join them and split them after the expansion has taken place. This -- is more convenient. +--~ local checkedsplit = string.checkedsplit + +local cache = { } + +local splitter = lpeg.Ct(lpeg.splitat(lpeg.S(os.type == "windows" and ";" or ":;"))) + +local function split_kpse_path(str) -- beware, this can be either a path or a {specification} + local found = cache[str] + if not found then + if str == "" then + found = { } + else + str = gsub(str,"\\","/") +--~ local split = (find(str,";") and checkedsplit(str,";")) or checkedsplit(str,io.pathseparator) +local split = lpegmatch(splitter,str) + found = { } + for i=1,#split do + local s = split[i] + if not find(s,"^{*unset}*") then + found[#found+1] = s + end + end + if trace_expansions then + logs.report("fileio","splitting path specification '%s'",str) + for k=1,#found do + logs.report("fileio","% 4i: %s",k,found[k]) + end + end + cache[str] = found + end + end + return found +end + +resolvers.split_kpse_path = split_kpse_path + function resolvers.splitconfig() - for i,c in ipairs(instance) do - for k,v in pairs(c) do + for i=1,#instance do + local c = instance[i] + for k,v in next, c do if type(v) == 'string' then - local t = file.split_path(v) + local t = split_kpse_path(v) if #t > 1 then c[k] = t end @@ -4962,21 +5816,25 @@ function resolvers.splitconfig() end function resolvers.joinconfig() - for i,c in ipairs(instance.order) do - for k,v in pairs(c) do -- ipairs? + local order = instance.order + for i=1,#order do + local c = order[i] + for k,v in next, c do -- indexed? if type(v) == 'table' then c[k] = file.join_path(v) end end end end + function resolvers.split_path(str) if type(str) == 'table' then return str else - return file.split_path(str) + return split_kpse_path(str) end end + function resolvers.join_path(str) if type(str) == 'table' then return file.join_path(str) @@ -4988,8 +5846,9 @@ end function resolvers.splitexpansions() local ie = instance.expansions for k,v in next, ie do - local t, h = { }, { } - for _,vv in ipairs(file.split_path(v)) do + local t, h, p = { }, { }, split_kpse_path(v) + for kk=1,#p do + local vv = p[kk] if vv ~= "" and not h[vv] then t[#t+1] = vv h[vv] = true @@ -5036,11 +5895,15 @@ function resolvers.serialize(files) end t[#t+1] = "return {" if instance.sortdata then - for _, k in pairs(sortedkeys(files)) do -- ipairs + local sortedfiles = sortedkeys(files) + for i=1,#sortedfiles do + local k = sortedfiles[i] local fk = files[k] if type(fk) == 'table' then t[#t+1] = "\t['" .. k .. "']={" - for _, kk in pairs(sortedkeys(fk)) do -- ipairs + local sortedfk = sortedkeys(fk) + for j=1,#sortedfk do + local kk = sortedfk[j] t[#t+1] = dump(kk,fk[kk],"\t\t") end t[#t+1] = "\t}," @@ -5065,12 +5928,18 @@ function resolvers.serialize(files) return concat(t,"\n") end +local data_state = { } + +function resolvers.data_state() + return data_state or { } +end + function resolvers.save_data(dataname, makename) -- untested without cache overload for cachename, files in next, instance[dataname] do local name = (makename or file.join)(cachename,dataname) local luaname, lucname = name .. ".lua", name .. ".luc" - if trace_verbose then - logs.report("fileio","preparing %s for %s",dataname,cachename) + if trace_locating then + logs.report("fileio","preparing '%s' for '%s'",dataname,cachename) end for k, v in next, files do if type(v) == "table" and #v == 1 then @@ -5084,24 +5953,25 @@ function resolvers.save_data(dataname, makename) -- untested without cache overl date = os.date("%Y-%m-%d"), time = os.date("%H:%M:%S"), content = files, + uuid = os.uuid(), } local ok = io.savedata(luaname,resolvers.serialize(data)) if ok then - if trace_verbose then - logs.report("fileio","%s saved in %s",dataname,luaname) + if trace_locating then + logs.report("fileio","'%s' saved in '%s'",dataname,luaname) end if utils.lua.compile(luaname,lucname,false,true) then -- no cleanup but strip - if trace_verbose then - logs.report("fileio","%s compiled to %s",dataname,lucname) + if trace_locating then + logs.report("fileio","'%s' compiled to '%s'",dataname,lucname) end else - if trace_verbose then - logs.report("fileio","compiling failed for %s, deleting file %s",dataname,lucname) + if trace_locating then + logs.report("fileio","compiling failed for '%s', deleting file '%s'",dataname,lucname) end os.remove(lucname) end - elseif trace_verbose then - logs.report("fileio","unable to save %s in %s (access error)",dataname,luaname) + elseif trace_locating then + logs.report("fileio","unable to save '%s' in '%s' (access error)",dataname,luaname) end end end @@ -5113,19 +5983,20 @@ function resolvers.load_data(pathname,dataname,filename,makename) -- untested wi if blob then local data = blob() if data and data.content and data.type == dataname and data.version == resolvers.cacheversion then - if trace_verbose then - logs.report("fileio","loading %s for %s from %s",dataname,pathname,filename) + data_state[#data_state+1] = data.uuid + if trace_locating then + logs.report("fileio","loading '%s' for '%s' from '%s'",dataname,pathname,filename) end instance[dataname][pathname] = data.content else - if trace_verbose then - logs.report("fileio","skipping %s for %s from %s",dataname,pathname,filename) + if trace_locating then + logs.report("fileio","skipping '%s' for '%s' from '%s'",dataname,pathname,filename) end instance[dataname][pathname] = { } instance.loaderror = true end - elseif trace_verbose then - logs.report("fileio","skipping %s for %s from %s",dataname,pathname,filename) + elseif trace_locating then + logs.report("fileio","skipping '%s' for '%s' from '%s'",dataname,pathname,filename) end end @@ -5144,15 +6015,17 @@ function resolvers.resetconfig() end function resolvers.loadnewconfig() - for _, cnf in ipairs(instance.luafiles) do + local luafiles = instance.luafiles + for i=1,#luafiles do + local cnf = luafiles[i] local pathname = file.dirname(cnf) local filename = file.join(pathname,resolvers.luaname) local blob = loadfile(filename) if blob then local data = blob() if data then - if trace_verbose then - logs.report("fileio","loading configuration file %s",filename) + if trace_locating then + logs.report("fileio","loading configuration file '%s'",filename) end if true then -- flatten to variable.progname @@ -5173,14 +6046,14 @@ function resolvers.loadnewconfig() instance['setup'][pathname] = data end else - if trace_verbose then - logs.report("fileio","skipping configuration file %s",filename) + if trace_locating then + logs.report("fileio","skipping configuration file '%s'",filename) end instance['setup'][pathname] = { } instance.loaderror = true end - elseif trace_verbose then - logs.report("fileio","skipping configuration file %s",filename) + elseif trace_locating then + logs.report("fileio","skipping configuration file '%s'",filename) end instance.order[#instance.order+1] = instance.setup[pathname] if instance.loaderror then break end @@ -5189,7 +6062,9 @@ end function resolvers.loadoldconfig() if not instance.renewcache then - for _, cnf in ipairs(instance.cnffiles) do + local cnffiles = instance.cnffiles + for i=1,#cnffiles do + local cnf = cnffiles[i] local dname = file.dirname(cnf) resolvers.load_data(dname,'configuration') instance.order[#instance.order+1] = instance.configuration[dname] @@ -5379,7 +6254,7 @@ end function resolvers.expanded_path_list(str) if not str then - return ep or { } + return ep or { } -- ep ? elseif instance.savelists then -- engine+progname hash str = gsub(str,"%$","") @@ -5397,9 +6272,9 @@ end function resolvers.expanded_path_list_from_var(str) -- brrr local tmp = resolvers.var_of_format_or_suffix(gsub(str,"%$","")) if tmp ~= "" then - return resolvers.expanded_path_list(str) - else return resolvers.expanded_path_list(tmp) + else + return resolvers.expanded_path_list(str) end end @@ -5446,9 +6321,9 @@ function resolvers.isreadable.file(name) local readable = lfs.isfile(name) -- brrr if trace_detail then if readable then - logs.report("fileio","+ readable: %s",name) + logs.report("fileio","file '%s' is readable",name) else - logs.report("fileio","- readable: %s", name) + logs.report("fileio","file '%s' is not readable", name) end end return readable @@ -5464,7 +6339,7 @@ local function collect_files(names) for k=1,#names do local fname = names[k] if trace_detail then - logs.report("fileio","? blobpath asked: %s",fname) + logs.report("fileio","checking name '%s'",fname) end local bname = file.basename(fname) local dname = file.dirname(fname) @@ -5480,7 +6355,7 @@ local function collect_files(names) local files = blobpath and instance.files[blobpath] if files then if trace_detail then - logs.report("fileio",'? blobpath do: %s (%s)',blobpath,bname) + logs.report("fileio","deep checking '%s' (%s)",blobpath,bname) end local blobfile = files[bname] if not blobfile then @@ -5514,7 +6389,7 @@ local function collect_files(names) end end elseif trace_locating then - logs.report("fileio",'! blobpath no: %s (%s)',blobpath,bname) + logs.report("fileio","no match in '%s' (%s)",blobpath,bname) end end end @@ -5564,14 +6439,13 @@ end local function collect_instance_files(filename,collected) -- todo : plugin (scanners, checkers etc) local result = collected or { } local stamp = nil - filename = file.collapse_path(filename) -- elsewhere - filename = file.collapse_path(gsub(filename,"\\","/")) -- elsewhere + filename = file.collapse_path(filename) -- speed up / beware: format problem if instance.remember then stamp = filename .. "--" .. instance.engine .. "--" .. instance.progname .. "--" .. instance.format if instance.found[stamp] then if trace_locating then - logs.report("fileio",'! remembered: %s',filename) + logs.report("fileio","remembering file '%s'",filename) end return instance.found[stamp] end @@ -5579,7 +6453,7 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan if not dangerous[instance.format or "?"] then if resolvers.isreadable.file(filename) then if trace_detail then - logs.report("fileio",'= found directly: %s',filename) + logs.report("fileio","file '%s' found directly",filename) end instance.found[stamp] = { filename } return { filename } @@ -5587,13 +6461,13 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan end if find(filename,'%*') then if trace_locating then - logs.report("fileio",'! wildcard: %s', filename) + logs.report("fileio","checking wildcard '%s'", filename) end result = resolvers.find_wildcard_files(filename) elseif file.is_qualified_path(filename) then if resolvers.isreadable.file(filename) then if trace_locating then - logs.report("fileio",'! qualified: %s', filename) + logs.report("fileio","qualified name '%s'", filename) end result = { filename } else @@ -5603,7 +6477,7 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan forcedname = filename .. ".tex" if resolvers.isreadable.file(forcedname) then if trace_locating then - logs.report("fileio",'! no suffix, forcing standard filetype: tex') + logs.report("fileio","no suffix, forcing standard filetype 'tex'") end result, ok = { forcedname }, true end @@ -5613,7 +6487,7 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan forcedname = filename .. "." .. s if resolvers.isreadable.file(forcedname) then if trace_locating then - logs.report("fileio",'! no suffix, forcing format filetype: %s', s) + logs.report("fileio","no suffix, forcing format filetype '%s'", s) end result, ok = { forcedname }, true break @@ -5625,7 +6499,7 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan -- try to find in tree (no suffix manipulation), here we search for the -- matching last part of the name local basename = file.basename(filename) - local pattern = (filename .. "$"):gsub("([%.%-])","%%%1") + local pattern = gsub(filename .. "$","([%.%-])","%%%1") local savedformat = instance.format local format = savedformat or "" if format == "" then @@ -5635,19 +6509,21 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan instance.format = "othertextfiles" -- kind of everything, maybe texinput is better end -- - local resolved = collect_instance_files(basename) - if #result == 0 then - local lowered = lower(basename) - if filename ~= lowered then - resolved = collect_instance_files(lowered) + if basename ~= filename then + local resolved = collect_instance_files(basename) + if #result == 0 then + local lowered = lower(basename) + if filename ~= lowered then + resolved = collect_instance_files(lowered) + end end - end - resolvers.format = savedformat - -- - for r=1,#resolved do - local rr = resolved[r] - if rr:find(pattern) then - result[#result+1], ok = rr, true + resolvers.format = savedformat + -- + for r=1,#resolved do + local rr = resolved[r] + if find(rr,pattern) then + result[#result+1], ok = rr, true + end end end -- a real wildcard: @@ -5656,14 +6532,14 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan -- local filelist = collect_files({basename}) -- for f=1,#filelist do -- local ff = filelist[f][3] or "" - -- if ff:find(pattern) then + -- if find(ff,pattern) then -- result[#result+1], ok = ff, true -- end -- end -- end end if not ok and trace_locating then - logs.report("fileio",'? qualified: %s', filename) + logs.report("fileio","qualified name '%s'", filename) end end else @@ -5682,12 +6558,12 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan wantedfiles[#wantedfiles+1] = forcedname filetype = resolvers.format_of_suffix(forcedname) if trace_locating then - logs.report("fileio",'! forcing filetype: %s',filetype) + logs.report("fileio","forcing filetype '%s'",filetype) end else filetype = resolvers.format_of_suffix(filename) if trace_locating then - logs.report("fileio",'! using suffix based filetype: %s',filetype) + logs.report("fileio","using suffix based filetype '%s'",filetype) end end else @@ -5699,7 +6575,7 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan end filetype = instance.format if trace_locating then - logs.report("fileio",'! using given filetype: %s',filetype) + logs.report("fileio","using given filetype '%s'",filetype) end end local typespec = resolvers.variable_of_format(filetype) @@ -5707,9 +6583,7 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan if not pathlist or #pathlist == 0 then -- no pathlist, access check only / todo == wildcard if trace_detail then - logs.report("fileio",'? filename: %s',filename) - logs.report("fileio",'? filetype: %s',filetype or '?') - logs.report("fileio",'? wanted files: %s',concat(wantedfiles," | ")) + logs.report("fileio","checking filename '%s', filetype '%s', wanted files '%s'",filename, filetype or '?',concat(wantedfiles," | ")) end for k=1,#wantedfiles do local fname = wantedfiles[k] @@ -5730,36 +6604,59 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan else -- list search local filelist = collect_files(wantedfiles) - local doscan, recurse + local dirlist = { } + if filelist then + for i=1,#filelist do + dirlist[i] = file.dirname(filelist[i][2]) .. "/" + end + end if trace_detail then - logs.report("fileio",'? filename: %s',filename) + logs.report("fileio","checking filename '%s'",filename) end -- a bit messy ... esp the doscan setting here + local doscan for k=1,#pathlist do local path = pathlist[k] if find(path,"^!!") then doscan = false else doscan = true end - if find(path,"//$") then recurse = true else recurse = false end local pathname = gsub(path,"^!+", '') done = false -- using file list - if filelist and not (done and not instance.allresults) and recurse then - -- compare list entries with permitted pattern - pathname = gsub(pathname,"([%-%.])","%%%1") -- this also influences - pathname = gsub(pathname,"/+$", '/.*') -- later usage of pathname - pathname = gsub(pathname,"//", '/.-/') -- not ok for /// but harmless - local expr = "^" .. pathname + if filelist then + local expression + -- compare list entries with permitted pattern -- /xx /xx// + if not find(pathname,"/$") then + expression = pathname .. "/" + else + expression = pathname + end + expression = gsub(expression,"([%-%.])","%%%1") -- this also influences + expression = gsub(expression,"//+$", '/.*') -- later usage of pathname + expression = gsub(expression,"//", '/.-/') -- not ok for /// but harmless + expression = "^" .. expression .. "$" + if trace_detail then + logs.report("fileio","using pattern '%s' for path '%s'",expression,pathname) + end for k=1,#filelist do local fl = filelist[k] local f = fl[2] - if find(f,expr) then - if trace_detail then - logs.report("fileio",'= found in hash: %s',f) - end + local d = dirlist[k] + if find(d,expression) then --- todo, test for readable result[#result+1] = fl[3] resolvers.register_in_trees(f) -- for tracing used files done = true - if not instance.allresults then break end + if instance.allresults then + if trace_detail then + logs.report("fileio","match in hash for file '%s' on path '%s', continue scanning",f,d) + end + else + if trace_detail then + logs.report("fileio","match in hash for file '%s' on path '%s', quit scanning",f,d) + end + break + end + elseif trace_detail then + logs.report("fileio","no match in hash for file '%s' on path '%s'",f,d) end end end @@ -5775,7 +6672,7 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan local fname = file.join(ppname,w) if resolvers.isreadable.file(fname) then if trace_detail then - logs.report("fileio",'= found by scanning: %s',fname) + logs.report("fileio","found '%s' by scanning",fname) end result[#result+1] = fname done = true @@ -5838,7 +6735,7 @@ function resolvers.find_given_files(filename) local hashes = instance.hashes for k=1,#hashes do local hash = hashes[k] - local files = instance.files[hash.tag] + local files = instance.files[hash.tag] or { } local blist = files[bname] if not blist then local rname = "remap:"..bname @@ -5948,9 +6845,9 @@ function resolvers.load(option) statistics.starttiming(instance) resolvers.resetconfig() resolvers.identify_cnf() - resolvers.load_lua() + resolvers.load_lua() -- will become the new method resolvers.expand_variables() - resolvers.load_cnf() + resolvers.load_cnf() -- will be skipped when we have a lua file resolvers.expand_variables() if option ~= "nofiles" then resolvers.load_hash() @@ -5962,22 +6859,23 @@ end function resolvers.for_files(command, files, filetype, mustexist) if files and #files > 0 then local function report(str) - if trace_verbose then + if trace_locating then logs.report("fileio",str) -- has already verbose else print(str) end end - if trace_verbose then - report('') + if trace_locating then + report('') -- ? end - for _, file in ipairs(files) do + for f=1,#files do + local file = files[f] local result = command(file,filetype,mustexist) if type(result) == 'string' then report(result) else - for _,v in ipairs(result) do - report(v) + for i=1,#result do + report(result[i]) -- could be unpack end end end @@ -6024,18 +6922,19 @@ end function table.sequenced(t,sep) -- temp here local s = { } - for k, v in pairs(t) do -- pairs? - s[#s+1] = k .. "=" .. v + for k, v in next, t do -- indexed? + s[#s+1] = k .. "=" .. tostring(v) end return concat(s, sep or " | ") end function resolvers.methodhandler(what, filename, filetype) -- ... + filename = file.collapse_path(filename) local specification = (type(filename) == "string" and resolvers.splitmethod(filename)) or filename -- no or { }, let it bomb local scheme = specification.scheme if resolvers[what][scheme] then if trace_locating then - logs.report("fileio",'= handler: %s -> %s -> %s',specification.original,what,table.sequenced(specification)) + logs.report("fileio","handler '%s' -> '%s' -> '%s'",specification.original,what,table.sequenced(specification)) end return resolvers[what][scheme](filename,filetype) -- todo: specification else @@ -6055,8 +6954,9 @@ function resolvers.clean_path(str) end function resolvers.do_with_path(name,func) - for _, v in pairs(resolvers.expanded_path_list(name)) do -- pairs? - func("^"..resolvers.clean_path(v)) + local pathlist = resolvers.expanded_path_list(name) + for i=1,#pathlist do + func("^"..resolvers.clean_path(pathlist[i])) end end @@ -6065,7 +6965,9 @@ function resolvers.do_with_var(name,func) end function resolvers.with_files(pattern,handle) - for _, hash in ipairs(instance.hashes) do + local hashes = instance.hashes + for i=1,#hashes do + local hash = hashes[i] local blobpath = hash.tag local blobtype = hash.type if blobpath then @@ -6080,7 +6982,7 @@ function resolvers.with_files(pattern,handle) if type(v) == "string" then handle(blobtype,blobpath,v,k) else - for _,vv in pairs(v) do -- ipairs? + for _,vv in next, v do -- indexed handle(blobtype,blobpath,vv,k) end end @@ -6092,7 +6994,7 @@ function resolvers.with_files(pattern,handle) end function resolvers.locate_format(name) - local barename, fmtname = name:gsub("%.%a+$",""), "" + local barename, fmtname = gsub(name,"%.%a+$",""), "" if resolvers.usecache then local path = file.join(caches.setpath("formats")) -- maybe platform fmtname = file.join(path,barename..".fmt") or "" @@ -6140,7 +7042,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-tmp'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -6164,7 +7066,7 @@ luatools with a recache feature.</p> local format, lower, gsub = string.format, string.lower, string.gsub -local trace_cache = false trackers.register("resolvers.cache", function(v) trace_cache = v end) +local trace_cache = false trackers.register("resolvers.cache", function(v) trace_cache = v end) -- not used yet caches = caches or { } @@ -6251,7 +7153,8 @@ function caches.setpath(...) caches.path = '.' end caches.path = resolvers.clean_path(caches.path) - if not table.is_empty({...}) then + local dirs = { ... } + if #dirs > 0 then local pth = dir.mkdirs(caches.path,...) return pth end @@ -6297,6 +7200,7 @@ function caches.savedata(filepath,filename,data,raw) if raw then reduce, simplify = false, false end + data.cache_uuid = os.uuid() if caches.direct then file.savedata(tmaname, table.serialize(data,'return',false,true,false)) -- no hex else @@ -6322,7 +7226,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-inp'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -6343,7 +7247,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-out'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -6359,7 +7263,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-con'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -6370,8 +7274,6 @@ local format, lower, gsub = string.format, string.lower, string.gsub local trace_cache = false trackers.register("resolvers.cache", function(v) trace_cache = v end) local trace_containers = false trackers.register("resolvers.containers", function(v) trace_containers = v end) local trace_storage = false trackers.register("resolvers.storage", function(v) trace_storage = v end) -local trace_verbose = false trackers.register("resolvers.verbose", function(v) trace_verbose = v end) -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v trackers.enable("resolvers.verbose") end) --[[ldx-- <p>Once we found ourselves defining similar cache constructs @@ -6435,7 +7337,7 @@ end function containers.is_valid(container, name) if name and name ~= "" then local storage = container.storage[name] - return storage and not table.is_empty(storage) and storage.cache_version == container.version + return storage and storage.cache_version == container.version else return false end @@ -6487,16 +7389,15 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-use'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -local format, lower, gsub = string.format, string.lower, string.gsub +local format, lower, gsub, find = string.format, string.lower, string.gsub, string.find -local trace_verbose = false trackers.register("resolvers.verbose", function(v) trace_verbose = v end) -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v trackers.enable("resolvers.verbose") end) +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) -- since we want to use the cache instead of the tree, we will now -- reimplement the saver. @@ -6540,19 +7441,20 @@ resolvers.automounted = resolvers.automounted or { } function resolvers.automount(usecache) local mountpaths = resolvers.clean_path_list(resolvers.expansion('TEXMFMOUNT')) - if table.is_empty(mountpaths) and usecache then + if (not mountpaths or #mountpaths == 0) and usecache then mountpaths = { caches.setpath("mount") } end - if not table.is_empty(mountpaths) then + if mountpaths and #mountpaths > 0 then statistics.starttiming(resolvers.instance) - for k, root in pairs(mountpaths) do + for k=1,#mountpaths do + local root = mountpaths[k] local f = io.open(root.."/url.tmi") if f then for line in f:lines() do if line then - if line:find("^[%%#%-]") then -- or %W + if find(line,"^[%%#%-]") then -- or %W -- skip - elseif line:find("^zip://") then + elseif find(line,"^zip://") then if trace_locating then logs.report("fileio","mounting %s",line) end @@ -6597,11 +7499,13 @@ function statistics.check_fmt_status(texname) local luv = dofile(luvname) if luv and luv.sourcefile then local sourcehash = md5.hex(io.loaddata(resolvers.find_file(luv.sourcefile)) or "unknown") - if luv.enginebanner and luv.enginebanner ~= enginebanner then - return "engine mismatch" + local luvbanner = luv.enginebanner or "?" + if luvbanner ~= enginebanner then + return string.format("engine mismatch (luv:%s <> bin:%s)",luvbanner,enginebanner) end - if luv.sourcehash and luv.sourcehash ~= sourcehash then - return "source mismatch" + local luvhash = luv.sourcehash or "?" + if luvhash ~= sourcehash then + return string.format("source mismatch (luv:%s <> bin:%s)",luvhash,sourcehash) end else return "invalid status file" @@ -6727,7 +7631,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-aux'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -6735,47 +7639,47 @@ if not modules then modules = { } end modules ['data-aux'] = { local find = string.find -local trace_verbose = false trackers.register("resolvers.verbose", function(v) trace_verbose = v end) +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) function resolvers.update_script(oldname,newname) -- oldname -> own.name, not per se a suffix local scriptpath = "scripts/context/lua" newname = file.addsuffix(newname,"lua") local oldscript = resolvers.clean_path(oldname) - if trace_verbose then + if trace_locating then logs.report("fileio","to be replaced old script %s", oldscript) end local newscripts = resolvers.find_files(newname) or { } if #newscripts == 0 then - if trace_verbose then + if trace_locating then logs.report("fileio","unable to locate new script") end else for i=1,#newscripts do local newscript = resolvers.clean_path(newscripts[i]) - if trace_verbose then + if trace_locating then logs.report("fileio","checking new script %s", newscript) end if oldscript == newscript then - if trace_verbose then + if trace_locating then logs.report("fileio","old and new script are the same") end elseif not find(newscript,scriptpath) then - if trace_verbose then + if trace_locating then logs.report("fileio","new script should come from %s",scriptpath) end elseif not (find(oldscript,file.removesuffix(newname).."$") or find(oldscript,newname.."$")) then - if trace_verbose then + if trace_locating then logs.report("fileio","invalid new script name") end else local newdata = io.loaddata(newscript) if newdata then - if trace_verbose then + if trace_locating then logs.report("fileio","old script content replaced by new content") end io.savedata(oldscript,newdata) break - elseif trace_verbose then + elseif trace_locating then logs.report("fileio","unable to load new script") end end @@ -6790,7 +7694,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-lst'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -6814,7 +7718,9 @@ local function list(list,report) local instance = resolvers.instance local pat = upper(pattern or "","") local report = report or texio.write_nl - for _,key in pairs(table.sortedkeys(list)) do + local sorted = table.sortedkeys(list) + for i=1,#sorted do + local key = sorted[i] if instance.pattern == "" or find(upper(key),pat) then if instance.kpseonly then if instance.kpsevars[key] then @@ -6833,11 +7739,14 @@ function resolvers.listers.expansions() list(resolvers.instance.expansions) end function resolvers.listers.configurations(report) local report = report or texio.write_nl local instance = resolvers.instance - for _,key in ipairs(table.sortedkeys(instance.kpsevars)) do + local sorted = table.sortedkeys(instance.kpsevars) + for i=1,#sorted do + local key = sorted[i] if not instance.pattern or (instance.pattern=="") or find(key,instance.pattern) then report(format("%s\n",key)) - for i,c in ipairs(instance.order) do - local str = c[key] + local order = instance.order + for i=1,#order do + local str = order[i][key] if str then report(format("\t%s\t%s",i,str)) end @@ -6943,7 +7852,7 @@ if not resolvers then os.exit() end -logs.setprogram('LuaTools',"TDS Management Tool 1.31",environment.arguments["verbose"] or false) +logs.setprogram('LuaTools',"TDS Management Tool 1.32",environment.arguments["verbose"] or false) local instance = resolvers.reset() @@ -7000,6 +7909,12 @@ end if environment.arguments["trace"] then resolvers.settrace(environment.arguments["trace"]) end +local trackspec = environment.argument("trackers") or environment.argument("track") + +if trackspec then + trackers.enable(trackspec) +end + runners = runners or { } messages = messages or { } @@ -7033,6 +7948,7 @@ messages.help = [[ --engine=str target engine --progname=str format or backend --pattern=str filter variables +--trackers=list enable given trackers ]] function runners.make_format(texname) @@ -7091,8 +8007,9 @@ function runners.make_format(texname) logs.simple("using uncompiled initialization file: %s",luaname) end else - for _, v in pairs({instance.luaname, instance.progname, barename}) do - v = string.gsub(v..".lua","%.lua%.lua$",".lua") + local what = { instance.luaname, instance.progname, barename } + for k=1,#what do + local v = string.gsub(what[k]..".lua","%.lua%.lua$",".lua") if v and (v ~= "") then luaname = resolvers.find_files(v)[1] or "" if luaname ~= "" then @@ -7116,7 +8033,8 @@ function runners.make_format(texname) logs.simple("using lua initialization file: %s",luaname) local mp = dir.glob(file.removesuffix(file.basename(luaname)).."-*.mem") if mp and #mp > 0 then - for _, name in ipairs(mp) do + for i=1,#mp do + local name = mp[i] logs.simple("removing related mplib format %s", file.basename(name)) os.remove(name) end diff --git a/Master/texmf-dist/scripts/context/lua/luatools.rme b/Master/texmf-dist/scripts/context/lua/luatools.rme index b320e1184b5..901e9a9a3fc 100644 --- a/Master/texmf-dist/scripts/context/lua/luatools.rme +++ b/Master/texmf-dist/scripts/context/lua/luatools.rme @@ -1,3 +1,3 @@ On MSWindows the luatools.lua script is called -with luatools.cmd. On Unix you can either rename +with luatools.exe. On Unix you can either rename luatools.lua to luatools, or use a symlink. diff --git a/Master/texmf-dist/scripts/context/lua/mtx-babel.lua b/Master/texmf-dist/scripts/context/lua/mtx-babel.lua index e241e933454..01e2ba4b223 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-babel.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-babel.lua @@ -415,7 +415,7 @@ do end -logs.extendbanner("Babel Conversion Tools 1.2",true) +logs.extendbanner("Babel Input To UTF Conversion 1.20",true) messages.help = [[ --language=string conversion language (e.g. greek) diff --git a/Master/texmf-dist/scripts/context/lua/mtx-cache.lua b/Master/texmf-dist/scripts/context/lua/mtx-cache.lua index a1fbed8255d..c2a0db00d89 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-cache.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-cache.lua @@ -22,9 +22,11 @@ function scripts.cache.collect_two(...) return path, rest end +local suffixes = { "afm", "tfm", "def", "enc", "otf", "mp", "data" } + function scripts.cache.process_one(action) - for k, v in ipairs({ "afm", "tfm", "def", "enc", "otf", "mp", "data" }) do - action("fonts", v) + for i=1,#suffixes do + action("fonts", suffixes[i]) end end @@ -35,13 +37,10 @@ end -- todo: recursive delete of paths function scripts.cache.remove(list,keep) - local keepsuffixes = { } - for _, v in ipairs(keep or {}) do - keepsuffixes[v] = true - end - local n = 0 - for _,filename in ipairs(list) do - if filename:find("luatex%-cache") then -- safeguard + local n, keepsuffixes = 0, table.tohash(keep or { }) + for i=1,#list do + local filename = list[i] + if string.find(filename,"luatex%-cache") then -- safeguard if not keepsuffixes[file.extname(filename) or ""] then os.remove(filename) n = n + 1 @@ -76,7 +75,7 @@ function scripts.cache.list(all) end) end -logs.extendbanner("Cache Tools 0.10") +logs.extendbanner("ConTeXt & MetaTeX Cache Management 0.10") messages.help = [[ --purge remove not used files diff --git a/Master/texmf-dist/scripts/context/lua/mtx-chars.lua b/Master/texmf-dist/scripts/context/lua/mtx-chars.lua index d4330ca304d..6acacfbd22e 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-chars.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-chars.lua @@ -11,17 +11,17 @@ local format, concat, utfchar, upper = string.format, table.concat, unicode.utf8 scripts = scripts or { } scripts.chars = scripts.chars or { } -local banner = [[ --- filename : char-mth.lua --- comment : companion to char-mth.tex (in ConTeXt) --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- license : see context related readme files --- comment : generated from data file downloaded from STIX website - -if not versions then versions = { } end versions['char-mth'] = 1.001 -if not characters then characters = { } end -]] - +--~ local banner = [[ +--~ -- filename : char-mth.lua +--~ -- comment : companion to char-mth.tex (in ConTeXt) +--~ -- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +--~ -- license : see context related readme files +--~ -- comment : generated from data file downloaded from STIX website +--~ +--~ if not versions then versions = { } end versions['char-mth'] = 1.001 +--~ if not characters then characters = { } end +--~ ]] +--~ --~ function scripts.chars.stixtomkiv(inname,outname) --~ if inname == "" then --~ logs.report("aquiring math data","invalid datafilename") @@ -248,7 +248,7 @@ function scripts.chars.makeencoutf() end local f = open("xetx-cls.tex",banner_utf_classes) if f then - for k, v in pairs(xtxclasses) do + for k, v in next, xtxclasses do f:write(format("\\defineXTXcharinjectionclass[lb:%s]\n",k)) end f:write("\n") @@ -301,7 +301,7 @@ function scripts.chars.makeencoutf() end end -logs.extendbanner("Character Tools 0.10") +logs.extendbanner("MkII Character Table Generators 0.10") messages.help = [[ --stix convert stix table to math table diff --git a/Master/texmf-dist/scripts/context/lua/mtx-check.lua b/Master/texmf-dist/scripts/context/lua/mtx-check.lua index 6be7f276597..4266ddf0d46 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-check.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-check.lua @@ -40,6 +40,8 @@ do local l_s, r_s = P("["), P("]") local l_g, r_g = P("{"), P("}") + local okay = lpeg.P("{[}") + lpeg.P("{]}") + local esc = P("\\") local cr = P("\r") local lf = P("\n") @@ -72,7 +74,7 @@ do ["tokens"] = (V("ignore") + V("whatever") + V("grouped") + V("setup") + V("display") + V("inline") + V("errors") + 1)^0, ["whatever"] = line + esc * 1 + C(P("%") * (1-line)^0), ["grouped"] = l_g * (V("whatever") + V("grouped") + V("setup") + V("display") + V("inline") + (1 - l_g - r_g))^0 * r_g, - ["setup"] = l_s * (V("whatever") + V("grouped") + V("setup") + V("display") + V("inline") + (1 - l_s - r_s))^0 * r_s, + ["setup"] = l_s * (okay + V("whatever") + V("grouped") + V("setup") + V("display") + V("inline") + (1 - l_s - r_s))^0 * r_s, ["display"] = d_m * (V("whatever") + V("grouped") + (1 - d_m))^0 * d_m, ["inline"] = i_m * (V("whatever") + V("grouped") + (1 - i_m))^0 * i_m, ["errors"] = (V("gerror")+ V("serror") + V("derror") + V("ierror")), @@ -103,8 +105,10 @@ function scripts.checker.check(filename) local str = io.loaddata(filename) if str then validator.check(str) - if #validator.errors > 0 then - for k, v in ipairs(validator.errors) do + local errors = validator.errors + if #errors > 0 then + for k=1,#errors do + local v = errors[k] local kind, position, line = v[1], v[2], v[3] local data = str:sub(position-30,position+30) data = data:gsub("(.)", { @@ -123,7 +127,7 @@ function scripts.checker.check(filename) end end -logs.extendbanner("Syntax Checking 0.10",true) +logs.extendbanner("Basic ConTeXt Syntax Checking 0.10",true) messages.help = [[ --convert check tex file for errors diff --git a/Master/texmf-dist/scripts/context/lua/mtx-context.lua b/Master/texmf-dist/scripts/context/lua/mtx-context.lua index 71d2eee5611..79e74e407ff 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-context.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-context.lua @@ -66,7 +66,7 @@ do function ctxrunner.reflag(flags) local t = { } - for _, flag in pairs(flags) do + for _, flag in next, flags do local key, value = flag:match("^(.-)=(.+)$") if key and value then t[key] = value @@ -122,22 +122,24 @@ do return end end - if table.is_empty(ctxdata.prepfiles) then - logs.simple("nothing prepared, no ctl file saved") - os.remove(ctlname) - else + local prepfiles = ctxdata.prepfiles + if prepfiles and next(prepfiles) then logs.simple("saving logdata in: %s",ctlname) f = io.open(ctlname,'w') if f then f:write("<?xml version='1.0' standalone='yes'?>\n\n") f:write(string.format("<ctx:preplist local='%s'>\n",yn(ctxdata.runlocal))) ---~ for name, value in pairs(ctxdata.prepfiles) do - for _, name in ipairs(table.sortedkeys(ctxdata.prepfiles)) do - f:write(string.format("\t<ctx:prepfile done='%s'>%s</ctx:prepfile>\n",yn(ctxdata.prepfiles[name]),name)) + local sorted = table.sortedkeys(prepfiles) + for i=1,#sorted do + local name = sorted[i] + f:write(string.format("\t<ctx:prepfile done='%s'>%s</ctx:prepfile>\n",yn(prepfiles[name]),name)) end f:write("</ctx:preplist>\n") f:close() end + else + logs.simple("nothing prepared, no ctl file saved") + os.remove(ctlname) end end @@ -153,7 +155,7 @@ do print(table.serialize(ctxdata.modules)) print(table.serialize(ctxdata.filters)) print(table.serialize(ctxdata.modes)) - print(xml.serialize(ctxdata.xmldata)) + print(xml.tostring(ctxdata.xmldata)) end function ctxrunner.manipulate(ctxdata,ctxname,defaultname) @@ -180,7 +182,7 @@ do local found = lfs.isfile(usedname) if not found then - for _, path in pairs(ctxdata.locations) do + for _, path in next, ctxdata.locations do local fullname = file.join(path,ctxdata.ctxname) if lfs.isfile(fullname) then usedname, found = fullname, true @@ -189,6 +191,9 @@ do end end + usedname = resolvers.find_file(ctxdata.ctxname,"tex") + found = usedname ~= "" + if not found and defaultname and defaultname ~= "" and lfs.isfile(defaultname) then usedname, found = defaultname, true end @@ -218,40 +223,40 @@ do ctxdata.flags = ctxrunner.reflag(ctxdata.flags) - for _, message in ipairs(ctxdata.messages) do - logs.simple("ctx comment: %s", xml.tostring(message)) + local messages = ctxdata.messages + for i=1,#messages do + logs.simple("ctx comment: %s", xml.tostring(messages[i])) end - xml.each(ctxdata.xmldata,"ctx:value[@name='job']", function(ek,e,k) - e[k] = ctxdata.variables['job'] or "" - end) + for r, d, k in xml.elements(ctxdata.xmldata,"ctx:value[@name='job']") do + d[k] = ctxdata.variables['job'] or "" + end local commands = { } - xml.each(ctxdata.xmldata,"/ctx:job/ctx:preprocess/ctx:processors/ctx:processor", function(r,d,k) - local ek = d[k] - commands[ek.at and ek.at['name'] or "unknown"] = ek - end) + for e in xml.collected(ctxdata.xmldata,"/ctx:job/ctx:preprocess/ctx:processors/ctx:processor") do + commands[e.at and e.at['name'] or "unknown"] = e + end - local suffix = xml.filter(ctxdata.xmldata,"/ctx:job/ctx:preprocess/attribute(suffix)") or ctxdata.suffix - local runlocal = xml.filter(ctxdata.xmldata,"/ctx:job/ctx:preprocess/ctx:processors/attribute(local)") + local suffix = xml.filter(ctxdata.xmldata,"/ctx:job/ctx:preprocess/attribute('suffix')") or ctxdata.suffix + local runlocal = xml.filter(ctxdata.xmldata,"/ctx:job/ctx:preprocess/ctx:processors/attribute('local')") runlocal = toboolean(runlocal) - for _, files in ipairs(xml.filters.elements(ctxdata.xmldata,"/ctx:job/ctx:preprocess/ctx:files")) do - for _, pattern in ipairs(xml.filters.elements(files,"ctx:file")) do + for files in xml.collected(ctxdata.xmldata,"/ctx:job/ctx:preprocess/ctx:files") do + for pattern in xml.collected(files,"ctx:file") do preprocessor = pattern.at['processor'] or "" if preprocessor ~= "" then ctxdata.variables['old'] = ctxdata.jobname - xml.each(ctxdata.xmldata,"ctx:value", function(r,d,k) + for r, d, k in xml.elements(ctxdata.xmldata,"ctx:value") do local ek = d[k] local ekat = ek.at['name'] if ekat == 'old' then d[k] = ctxrunner.substitute(ctxdata.variables[ekat] or "") end - end) + end pattern = ctxrunner.justtext(xml.tostring(pattern)) @@ -260,7 +265,9 @@ do local pluspath = false if #oldfiles == 0 then -- message: no files match pattern - for _, p in ipairs(ctxdata.paths) do + local paths = ctxdata.paths + for i=1,#paths do + local p = paths[i] local oldfiles = dir.glob(path.join(p,pattern)) if #oldfiles > 0 then pluspath = true @@ -271,15 +278,18 @@ do if #oldfiles == 0 then -- message: no old files else - for _, oldfile in ipairs(oldfiles) do - newfile = oldfile .. "." .. suffix -- addsuffix will add one only + for i=1,#oldfiles do + local oldfile = oldfiles[i] + local newfile = oldfile .. "." .. suffix -- addsuffix will add one only if ctxdata.runlocal then newfile = file.basename(newfile) end if oldfile ~= newfile and file.needsupdate(oldfile,newfile) then -- message: oldfile needs preprocessing -- os.remove(newfile) - for _, pp in ipairs(preprocessor:split(',')) do + local splitted = preprocessor:split(',') + for i=1,#splitted do + local pp = splitted[i] local command = commands[pp] if command then command = xml.copy(command) @@ -290,27 +300,27 @@ do if ctxdata.runlocal then newfile = file.basename(newfile) end - xml.each(command,"ctx:old", function(r,d,k) + for r, d, k in xml.elements(command,"ctx:old") do d[k] = ctxrunner.substitute(oldfile) - end) - xml.each(command,"ctx:new", function(r,d,k) + end + for r, d, k in xml.elements(command,"ctx:new") do d[k] = ctxrunner.substitute(newfile) - end) - -- message: preprocessing #{oldfile} into #{newfile} using #{pp} + end ctxdata.variables['old'] = oldfile ctxdata.variables['new'] = newfile - xml.each(command,"ctx:value", function(r,d,k) + for r, d, k in xml.elements(command,"ctx:value") do local ek = d[k] local ekat = ek.at and ek.at['name'] if ekat then d[k] = ctxrunner.substitute(ctxdata.variables[ekat] or "") end - end) + end -- potential optimization: when mtxrun run internal - command = xml.text(command) - command = ctxrunner.justtext(command) -- command is still xml element here + command = xml.content(command) + command = ctxrunner.justtext(command) logs.simple("command: %s",command) local result = os.spawn(command) or 0 + -- somehow we get the wrong return value if result > 0 then logs.simple("error, return code: %s",result) end @@ -340,6 +350,14 @@ do end + function ctxrunner.preppedfile(ctxdata,filename) + if ctxdata.prepfiles[file.basename(filename)] then + return filename .. ".prep" + else + return filename + end + end + end -- rest @@ -352,7 +370,9 @@ scripts.context.multipass = { function scripts.context.multipass.hashfiles(jobname) local hash = { } - for _, suffix in ipairs(scripts.context.multipass.suffixes) do + local suffixes = scripts.context.multipass.suffixes + for i=1,#suffixes do + local suffix = suffixes[i] local full = jobname .. suffix hash[full] = md5.hex(io.loaddata(full) or "unknown") end @@ -360,7 +380,7 @@ function scripts.context.multipass.hashfiles(jobname) end function scripts.context.multipass.changed(oldhash, newhash) - for k,v in pairs(oldhash) do + for k,v in next, oldhash do if v ~= newhash[k] then return true end @@ -398,7 +418,7 @@ function scripts.context.multipass.makeoptionfile(jobname,ctxdata,kindofrun,curr end local function setvalues(flag,format,plural) if type(flag) == "table" then - for k, v in pairs(flag) do + for k, v in next, flag do f:write(format:format(v),"\n") end else @@ -434,9 +454,17 @@ function scripts.context.multipass.makeoptionfile(jobname,ctxdata,kindofrun,curr if type(environment.argument("track")) == "string" then setvalue ("track" , "\\enabletrackers[%s]") end + if type(environment.argument("trackers")) == "string" then + setvalue ("trackers" , "\\enabletrackers[%s]") + end + if type(environment.argument("directives")) == "string" then + setvalue ("directives", "\\enabledirectives[%s]") + end setfixed ("timing" , "\\usemodule[timing]") setfixed ("batchmode" , "\\batchmode") + setfixed ("batch" , "\\batchmode") setfixed ("nonstopmode" , "\\nonstopmode") + setfixed ("nonstop" , "\\nonstopmode") setfixed ("tracefiles" , "\\tracefilestrue") setfixed ("nostats" , "\\nomkivstatistics") setfixed ("paranoid" , "\\def\\maxreadlevel{1}") @@ -451,10 +479,11 @@ function scripts.context.multipass.makeoptionfile(jobname,ctxdata,kindofrun,curr -- setalways("%% process info") -- - setalways( "\\setupsystem[\\c!n=%s,\\c!m=%s]", kindofrun or 0, currentrun or 0) - setalways( "\\setupsystem[\\c!type=%s]",os.platform) - setvalue ("inputfile" , "\\setupsystem[inputfile=%s]") + -- setvalue ("inputfile" , "\\setupsystem[inputfile=%s]") + setalways( "\\setupsystem[inputfile=%s]",environment.argument("input") or environment.files[1] or "\\jobname") setvalue ("result" , "\\setupsystem[file=%s]") + setalways( "\\setupsystem[\\c!n=%s,\\c!m=%s]", kindofrun or 0, currentrun or 0) + -- setalways( "\\setupsystem[\\c!type=%s]",os.type) -- windows or unix setvalues("path" , "\\usepath[%s]") setvalue ("setuppath" , "\\setupsystem[\\c!directory={%s}]") setvalue ("randomseed" , "\\setupsystem[\\c!random=%s]") @@ -557,18 +586,15 @@ scripts.context.interfaces = { it = "cont-it", ro = "cont-ro", pe = "cont-pe", - -- for taco and me - -- xp = "cont-xp", } scripts.context.defaultformats = { "cont-en", "cont-nl", --- "cont-xp", "mptopdf", -- "metatex", "metafun", - "plain" +-- "plain" } local function analyze(filename) @@ -585,6 +611,9 @@ local function analyze(filename) elseif line:find("^<?xml ") then t.type = "xml" end + if t.nofruns then + scripts.context.multipass.nofruns = t.nofruns + end if not t.engine then t.engine = 'luatex' end @@ -594,12 +623,12 @@ local function analyze(filename) return nil end -local function makestub(format,filename) +local function makestub(format,filename,prepname) local stubname = file.replacesuffix(file.basename(filename),'run') local f = io.open(stubname,'w') if f then f:write("\\starttext\n") - f:write(string.format(format,filename),"\n") + f:write(string.format(format,prepname or filename),"\n") f:write("\\stoptext\n") f:close() filename = stubname @@ -607,27 +636,32 @@ local function makestub(format,filename) return filename end -function scripts.context.openpdf(name) - os.spawn(string.format('pdfopen --file "%s" 2>&1', file.replacesuffix(name,"pdf"))) -end -function scripts.context.closepdf(name) - os.spawn(string.format('pdfclose --file "%s" 2>&1', file.replacesuffix(name,"pdf"))) -end - --~ function scripts.context.openpdf(name) ---~ -- somehow two instances start up, one with a funny filename ---~ os.spawn(string.format("\"c:/program files/kde/bin/okular.exe\" --unique %s",file.replacesuffix(name,"pdf"))) +--~ os.spawn(string.format('pdfopen --file "%s" 2>&1', file.replacesuffix(name,"pdf"))) --~ end --~ function scripts.context.closepdf(name) ---~ -- +--~ os.spawn(string.format('pdfclose --file "%s" 2>&1', file.replacesuffix(name,"pdf"))) --~ end +local pdfview -- delayed loading + +function scripts.context.openpdf(name) + pdfview = pdfview or dofile(resolvers.find_file("l-pdfview.lua","tex")) + logs.simple("pdfview methods: %s, current method: %s, MTX_PDFVIEW_METHOD=%s",pdfview.methods(),pdfview.method,os.getenv(pdfview.METHOD) or "<unset>") + pdfview.open(file.replacesuffix(name,"pdf")) +end + +function scripts.context.closepdf(name) + pdfview = pdfview or dofile(resolvers.find_file("l-pdfview.lua","tex")) + pdfview.close(file.replacesuffix(name,"pdf")) +end + function scripts.context.run(ctxdata,filename) -- filename overloads environment.files local files = (filename and { filename }) or environment.files if ctxdata then -- todo: interface - for k,v in pairs(ctxdata.flags) do + for k,v in next, ctxdata.flags do environment.setargument(k,v) end end @@ -648,7 +682,8 @@ function scripts.context.run(ctxdata,filename) end -- if formatfile and scriptfile then - for _, filename in ipairs(files) do + for i=1,#files do + local filename = files[i] local basename, pathname = file.basename(filename), file.dirname(filename) local jobname = file.removesuffix(basename) if pathname == "" then @@ -657,11 +692,18 @@ function scripts.context.run(ctxdata,filename) -- look at the first line local a = analyze(filename) if a and (a.engine == 'pdftex' or a.engine == 'xetex' or environment.argument("pdftex") or environment.argument("xetex")) then - local texexec = resolvers.find_file("texexec.rb") or "" - if texexec ~= "" then - os.setenv("RUBYOPT","") - local command = string.format("ruby %s %s",texexec,environment.reconstruct_commandline(environment.arguments_after)) - os.exec(command) + if false then + -- we need to write a top etc too and run mp etc so it's not worth the + -- trouble, so it will take a while before the next is finished + -- + -- require "mtx-texutil.lua" + else + local texexec = resolvers.find_file("texexec.rb") or "" + if texexec ~= "" then + os.setenv("RUBYOPT","") + local command = string.format("ruby %s %s",texexec,environment.reconstruct_commandline(environment.arguments_after)) + os.exec(command) + end end else if a and a.interface and a.interface ~= interface then @@ -677,6 +719,7 @@ function scripts.context.run(ctxdata,filename) end if formatfile and scriptfile then -- we default to mkiv xml ! + -- the --prep argument might become automatic (and noprep) local suffix = file.extname(filename) or "?" if scripts.context.xmlsuffixes[suffix] or environment.argument("forcexml") then if environment.argument("mkii") then @@ -688,6 +731,9 @@ function scripts.context.run(ctxdata,filename) filename = makestub("\\ctxlua{context.runfile('%s')}",filename) elseif scripts.context.luasuffixes[suffix] or environment.argument("forcelua") then filename = makestub("\\ctxlua{dofile('%s')}",filename) + elseif environment.argument("prep") then + -- we need to keep the original jobname + filename = makestub("\\readfile{%s}{}{}",filename,ctxrunner.preppedfile(ctxdata,filename)) end -- -- todo: also other stubs @@ -701,7 +747,7 @@ function scripts.context.run(ctxdata,filename) oldbase = file.removesuffix(jobname) newbase = file.removesuffix(resultname) if oldbase ~= newbase then - for _, suffix in pairs(scripts.context.beforesuffixes) do + for _, suffix in next, scripts.context.beforesuffixes do local oldname = file.addsuffix(oldbase,suffix) local newname = file.addsuffix(newbase,suffix) local tmpname = "keep-"..oldname @@ -732,9 +778,13 @@ function scripts.context.run(ctxdata,filename) end -- local flags = { } - if environment.argument("batchmode") then + if environment.argument("batchmode") or environment.argument("batch") then flags[#flags+1] = "--interaction=batchmode" end + if environment.argument("synctex") then + logs.simple("warning: syntex is enabled") -- can add upto 5% runtime + flags[#flags+1] = "--synctex=1" + end flags[#flags+1] = "--fmt=" .. string.quote(formatfile) flags[#flags+1] = "--lua=" .. string.quote(scriptfile) flags[#flags+1] = "--backend=pdf" @@ -754,16 +804,16 @@ function scripts.context.run(ctxdata,filename) --~ scripts.context.make(formatname) --~ returncode, errorstring = os.spawn(command) --~ if returncode == 3 then - --~ logs.simple("fatal error, return code 3, message: %s",errorstring or "?") + --~ logs.simple("ks: return code 3, message: %s",errorstring or "?") --~ os.exit(1) --~ end --~ end if not returncode then - logs.simple("fatal error, no return code, message: %s",errorstring or "?") + logs.simple("fatal error: no return code, message: %s",errorstring or "?") os.exit(1) break elseif returncode > 0 then - logs.simple("fatal error, return code: %s",returncode or "?") + logs.simple("fatal error: return code: %s",returncode or "?") os.exit(returncode) break else @@ -784,24 +834,24 @@ function scripts.context.run(ctxdata,filename) logs.simple("arrange run: %s",command) local returncode, errorstring = os.spawn(command) if not returncode then - logs.simple("fatal error, no return code, message: %s",errorstring or "?") + logs.simple("fatal error: no return code, message: %s",errorstring or "?") os.exit(1) elseif returncode > 0 then - logs.simple("fatal error, return code: %s",returncode or "?") + logs.simple("fatal error: return code: %s",returncode or "?") os.exit(returncode) end end -- if environment.argument("purge") then - scripts.context.purge_job(filename) + scripts.context.purge_job(jobname) elseif environment.argument("purgeall") then - scripts.context.purge_job(filename,true) + scripts.context.purge_job(jobname,true) end -- os.remove(jobname..".top") -- if resultname then - for _, suffix in pairs(scripts.context.aftersuffixes) do + for _, suffix in next, scripts.context.aftersuffixes do local oldname = file.addsuffix(oldbase,suffix) local newname = file.addsuffix(newbase,suffix) local tmpname = "keep-"..oldname @@ -910,9 +960,11 @@ function scripts.context.make(name) (environment.argument("xetex") and "mtxrun texexec.rb --make --xetex " ) or false, } local list = (name and { name }) or (environment.files[1] and environment.files) or scripts.context.defaultformats - for _, name in ipairs(list) do + for i=1,#list do + local name = list[i] name = scripts.context.interfaces[name] or name - for _, runner in ipairs(runners) do + for i=1,#runners do + local runner = runners[i] if runner then local command = runner .. name logs.simple("running command: %s",command) @@ -970,24 +1022,29 @@ local loaded = false function scripts.context.metapost() local filename = environment.files[1] or "" ---~ local tempname = "mtx-context-metapost.tex" ---~ local tempdata = string.format(template,"metafun",filename) ---~ io.savedata(tempname,tempdata) ---~ environment.files[1] = tempname ---~ environment.setargument("result",file.removesuffix(filename)) ---~ environment.setargument("once",true) ---~ scripts.context.run() if not loaded then dofile(resolvers.find_file("mlib-run.lua")) loaded = true commands = commands or { } commands.writestatus = logs.report end - local formatname = environment.arguments("format") or "metafun" + local formatname = environment.argument("format") or "metafun" if formatname == "" or type(format) == "boolean" then formatname = "metafun" end - if environment.arguments("svg") then + if environment.argument("pdf") then + local basename = file.removesuffix(filename) + local resultname = environment.argument("result") or basename + local jobname = "mtx-context-metapost" + local tempname = file.addsuffix(jobname,"tex") + io.savedata(tempname,string.format(template,"metafun",filename)) + environment.files[1] = tempname + environment.setargument("result",resultname) + environment.setargument("once",true) + scripts.context.run() + scripts.context.purge_job(jobname,true) + scripts.context.purge_job(resultname,true) + elseif environment.argument("svg") then metapost.directrun(formatname,filename,"svg") else metapost.directrun(formatname,filename,"mps") @@ -1041,7 +1098,7 @@ local function purge_file(dfile,cfile) if os.remove(dfile) then return file.basename(dfile) end - else + elseif dfile then if os.remove(dfile) then return file.basename(dfile) end @@ -1049,22 +1106,24 @@ local function purge_file(dfile,cfile) end function scripts.context.purge_job(jobname,all) - jobname = file.basename(jobname) - local filebase = file.removesuffix(jobname) - local deleted = { } - for _, suffix in ipairs(obsolete_results) do - deleted[#deleted+1] = purge_file(filebase.."."..suffix,filebase..".pdf") - end - for _, suffix in ipairs(temporary_runfiles) do - deleted[#deleted+1] = purge_file(filebase.."."..suffix) - end - if all then - for _, suffix in ipairs(persistent_runfiles) do - deleted[#deleted+1] = purge_file(filebase.."."..suffix) + if jobname and jobname ~= "" then + jobname = file.basename(jobname) + local filebase = file.removesuffix(jobname) + local deleted = { } + for i=1,#obsolete_results do + deleted[#deleted+1] = purge_file(filebase.."."..obsolete_results[i],filebase..".pdf") + end + for i=1,#temporary_runfiles do + deleted[#deleted+1] = purge_file(filebase.."."..temporary_runfiles[i]) + end + if all then + for i=1,#persistent_runfiles do + deleted[#deleted+1] = purge_file(filebase.."."..persistent_runfiles[i]) + end + end + if #deleted > 0 then + logs.simple("purged files: %s", table.join(deleted,", ")) end - end - if #deleted > 0 then - logs.simple("purged files: %s", table.join(deleted,", ")) end end @@ -1077,7 +1136,8 @@ function scripts.context.purge(all) local persistent = table.tohash(persistent_runfiles) local generic = table.tohash(generic_files) local deleted = { } - for _, name in ipairs(files) do + for i=1,#files do + local name = files[i] local suffix = file.extname(name) local basename = file.basename(name) if obsolete[suffix] or temporary[suffix] or persistent[suffix] or generic[basename] then @@ -1146,7 +1206,8 @@ function scripts.context.extras(pattern) else logs.extendbanner(extra) end - for k,v in ipairs(list) do + for i=1,#list do + local v = list[i] local data = io.loaddata(v) or "" data = string.match(data,"begin help(.-)end help") if data then @@ -1154,6 +1215,7 @@ function scripts.context.extras(pattern) for s in string.gmatch(data,"%% *(.-)[\n\r]") do h[#h+1] = s end + h[#h+1] = "" logs.help(table.concat(h,"\n"),"nomoreinfo") end end @@ -1191,8 +1253,15 @@ end -- todo: we need to do a dummy run -function scripts.context.track() - environment.files = { "m-track" } +function scripts.context.trackers() + environment.files = { resolvers.find_file("m-trackers.tex") } + scripts.context.multipass.nofruns = 1 + scripts.context.run() + -- maybe filter from log +end + +function scripts.context.directives() + environment.files = { resolvers.find_file("m-directives.tex") } scripts.context.multipass.nofruns = 1 scripts.context.run() -- maybe filter from log @@ -1338,35 +1407,65 @@ function scripts.context.update() end end -logs.extendbanner("ConTeXt Tools 0.51",true) +logs.extendbanner("ConTeXt Process Management 0.51",true) messages.help = [[ --run process (one or more) files (default action) --make create context formats ---generate generate file database etc. ---ctx=name use ctx file ---version report installed context version + +--ctx=name use ctx file (process management specification) +--interface use specified user interface (default: en) + +--autopdf close pdf file in viewer and start pdf viewer afterwards +--purge(all) purge files either or not after a run (--pattern=...) + +--usemodule=list load the given module or style, normally part o fthe distribution +--environment=list load the given environment file first (document styles) +--mode=list enable given the modes (conditional processing in styles) +--path=list also consult the given paths when files are looked for +--arguments=list set variables that can be consulted during a run (key/value pairs) +--randomseed=number set the randomseed +--result=name rename the resulting output to the given name +--trackers=list show/set tracker variables +--directives=list show/set directive variables + --forcexml force xml stub (optional flag: --mkii) --forcecld force cld (context lua document) stub ---autopdf close pdf file in viewer and start pdf viewer afterwards ---once only one run ---purge(all) purge files (--pattern=...) ---result=name rename result to given name ---arrange run extra arrange pass + +--arrange run extra imposition pass, given that the style sets up imposition +--noarrange ignore imposition specifications in the style + +--once only run once (no multipass data file is produced) +--batchmode run without stopping and don't show messages on the console +--nonstopmode run without stopping + +--generate generate file database etc. (as luatools does) +--paranoid don't descend to .. and ../.. +--version report installed context version --expert expert options ---interface use specified user interface ]] +-- filter=list is kind of obsolete +-- color is obsolete for mkiv, always on +-- separation is obsolete for mkiv, no longer available +-- output is currently obsolete for mkiv +-- setuppath=list must check +-- modefile=name must check +-- input=name load the given inputfile (must check) + messages.expert = [[ expert options: --touch update context version number (remake needed afterwards, also provide --expert) +--nostats omit runtime statistics at the end of the run --update update context from website (not to be confused with contextgarden) --profile profile job (use: mtxrun --script profile --analyse) ---track show/set tracker variables --timing generate timing and statistics overview +--tracefiles show some extra info when locating files (at the tex end) + --extra=name process extra (mtx-context-<name> in distribution) +--extras show extras ]] messages.private = [[ @@ -1387,6 +1486,8 @@ special options: if environment.argument("once") then scripts.context.multipass.nofruns = 1 +elseif environment.argument("runs") then + scripts.context.multipass.nofruns = tonumber(environment.argument("runs")) or nil end if environment.argument("profile") then @@ -1417,12 +1518,22 @@ elseif environment.argument("update") then scripts.context.update() elseif environment.argument("expert") then logs.help(table.join({ messages.expert, messages.private, messages.special },"\n")) +elseif environment.argument("extras") then + scripts.context.extras() elseif environment.argument("extra") then scripts.context.extra() elseif environment.argument("help") then - logs.help(messages.help) -elseif environment.argument("track") and type(environment.argument("track")) == "boolean" then - scripts.context.track() + if environment.files[1] == "extras" then + scripts.context.extras() + else + logs.help(messages.help) + end +elseif environment.argument("trackers") and type(environment.argument("trackers")) == "boolean" then + scripts.context.trackers() +elseif environment.argument("directives") and type(environment.argument("directives")) == "boolean" then + scripts.context.directives() +elseif environment.argument("track") and type(environment.argument("track")) == "boolean" then -- for old times sake, will go + scripts.context.trackers() elseif environment.files[1] then -- scripts.context.timed(scripts.context.run) scripts.context.timed(scripts.context.autoctx) diff --git a/Master/texmf-dist/scripts/context/lua/mtx-convert.lua b/Master/texmf-dist/scripts/context/lua/mtx-convert.lua index c0c383b176f..62198a62188 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-convert.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-convert.lua @@ -11,7 +11,7 @@ if not modules then modules = { } end modules ['mtx-convert'] = { graphics = graphics or { } graphics.converters = graphics.converters or { } -local gsprogram = (os.platform == "windows" and "gswin32c") or "gs" +local gsprogram = (os.type == "windows" and "gswin32c") or "gs" local gstemplate = "%s -q -sDEVICE=pdfwrite -dEPSCrop -dNOPAUSE -dNOCACHE -dBATCH -dAutoRotatePages=/None -dProcessColorModel=/DeviceCMYK -sOutputFile=%s %s -c quit" function graphics.converters.eps(oldname,newname) @@ -31,6 +31,7 @@ function graphics.converters.jpg(oldname,newname) return imtemplate[quality]:format(improgram,oldname,newname) end +graphics.converters.gif = graphics.converters.jpg graphics.converters.tif = graphics.converters.jpg graphics.converters.tiff = graphics.converters.jpg graphics.converters.png = graphics.converters.jpg @@ -111,13 +112,14 @@ function scripts.convert.convertall() end function scripts.convert.convertgiven() - for _, name in ipairs(environment.files) do - graphics.converters.convertfile(name) + local files = environment.files + for i=1,#files do + graphics.converters.convertfile(files[i]) end end -logs.extendbanner("Graphic Conversion Tools 0.10",true) +logs.extendbanner("ConTeXT Graphic Conversion Helpers 0.10",true) messages.help = [[ --convertall convert all graphics on path diff --git a/Master/texmf-dist/scripts/context/lua/mtx-fonts.lua b/Master/texmf-dist/scripts/context/lua/mtx-fonts.lua index dd319047590..74012ae38d3 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-fonts.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-fonts.lua @@ -15,30 +15,81 @@ dofile(resolvers.find_file("font-mis.lua","tex")) scripts = scripts or { } scripts.fonts = scripts.fonts or { } -function scripts.fonts.reload(verbose) - fonts.names.load(true,verbose) -end - -function scripts.fonts.names(name) - name = name or "luatex-fonts-names.lua" +function fonts.names.simple() + local simpleversion = 1.001 + local simplelist = { "ttf", "otf", "ttc", "dfont" } + local name = "luatex-fonts-names.lua" + fonts.names.filters.list = simplelist + fonts.names.version = simpleversion -- this number is the same as in font-dum.lua + logs.report("fontnames","generating font database for 'luatex-fonts' version %s",fonts.names.version) fonts.names.identify(true) local data = fonts.names.data if data then - data.fallback_mapping = nil + local simplemappings = { } + local simplified = { + mappings = simplemappings, + version = simpleversion, + } + local specifications = data.specifications + for i=1,#simplelist do + local format = simplelist[i] + for tag, index in next, data.mappings[format] do + local s = specifications[index] + simplemappings[tag] = { s.rawname, s.filename, s.subfont } + end + end logs.report("fontnames","saving names in '%s'",name) - io.savedata(name,table.serialize(data,true)) + io.savedata(name,table.serialize(simplified,true)) + local data = io.loaddata(resolvers.find_file("font-dum.lua","tex")) + local dummy = string.match(data,"fonts%.names%.version%s*=%s*([%d%.]+)") + if tonumber(dummy) ~= simpleversion then + logs.report("fontnames","warning: version number %s in 'font-dum' does not match database version number %s",dummy or "?",simpleversion) + end elseif lfs.isfile(name) then os.remove(name) end end -local function showfeatures(v,n,f,s,t) - logs.simple("fontname: %s",v) - logs.simple("fullname: %s",n) - logs.simple("filename: %s",f) - local features = fonts.get_features(f,t) +function scripts.fonts.reload() + if environment.argument("simple") then + fonts.names.simple() + else + fonts.names.load(true) + end +end + +local function subfont(sf) + if sf then + return string.format("index: % 2s", sf) + else + return "" + end +end + +local function fontweight(fw) + if fw then + return string.format("conflict: %s", fw) + else + return "" + end +end + +local function showfeatures(tag,specification) + logs.simple("mapping : %s",tag) + logs.simple("fontname: %s",specification.fontname) + logs.simple("fullname: %s",specification.fullname) + logs.simple("filename: %s",specification.filename) + logs.simple("family : %s",specification.familyname or "<nofamily>") + logs.simple("weight : %s",specification.weight or "<noweight>") + logs.simple("style : %s",specification.style or "<nostyle>") + logs.simple("width : %s",specification.width or "<nowidth>") + logs.simple("variant : %s",specification.variant or "<novariant>") + logs.simple("subfont : %s",subfont(specification.subfont)) + logs.simple("fweight : %s",fontweight(specification.fontweight)) + -- maybe more + local features = fonts.get_features(specification.filename,specification.format) if features then - for what, v in table.sortedpairs(features) do + for what, v in table.sortedhash(features) do local data = features[what] if data and next(data) then logs.simple() @@ -46,9 +97,9 @@ local function showfeatures(v,n,f,s,t) logs.simple() logs.simple("feature script languages") logs.simple() - for f,ff in table.sortedpairs(data) do + for f,ff in table.sortedhash(data) do local done = false - for s, ss in table.sortedpairs(ff) do + for s, ss in table.sortedhash(ff) do if s == "*" then s = "all" end if ss ["*"] then ss["*"] = nil ss.all = true end if done then @@ -61,56 +112,146 @@ local function showfeatures(v,n,f,s,t) end end end + else + logs.simple() + logs.simple("no features") + logs.simple() end logs.reportline() end -function scripts.fonts.list(pattern,reload,all,info) +local function reloadbase(reload) if reload then logs.simple("fontnames, reloading font database") - end - -- make a function for this - pattern = pattern:lower() - pattern = pattern:gsub("%-","%%-") - pattern = pattern:gsub("%.","%%.") - pattern = pattern:gsub("%*",".*") - pattern = pattern:gsub("%?",".?") - if pattern == "" then - pattern = ".*" - else - pattern = "^" .. pattern .. "$" - end - -- - local t = fonts.names.list(pattern,reload) - if reload then + names.load(true) logs.simple("fontnames, done\n\n") end +end + +local function list_specifications(t,info) if t then - local s, w = table.sortedkeys(t), { 0, 0, 0 } - local function action(f) - for k,v in ipairs(s) do - local type, name, file, sub = unpack(t[v]) - f(v,name,file,sub,type) + local s = table.sortedkeys(t) + if info then + for k=1,#s do + local v = s[k] + showfeatures(v,t[v]) + end + else + for k=1,#s do + local v = s[k] + local entry = t[v] + s[k] = { + entry.familyname or "<nofamily>", + entry.weight or "<noweight>", + entry.style or "<nostyle>", + entry.width or "<nowidth>", + entry.variant or "<novariant>", + entry.fontname, + entry.filename, + subfont(entry.subfont), + fontweight(entry.fontweight), + } + e[k] = entry + end + table.formatcolumns(s) + for k=1,#s do + local v = s[k] + texio.write_nl(v) end end - action(function(v,n,f,s,t) - if #v > w[1] then w[1] = #v end - if #n > w[2] then w[2] = #n end - if #f > w[3] then w[3] = #f end - end) - action(function(v,n,f,s,t) - if s then s = "(sub)" else s = "" end - if info then - showfeatures(v,n,f,s,t) - else - local str = string.format("%s %s %s %s",v:padd(w[1]," "),n:padd(w[2]," "),f:padd(w[3]," "), s) - print(str:strip()) + end +end + +local function list_matches(t,info) + if t then + local s, w = table.sortedkeys(t), { 0, 0, 0 } + if info then + for k=1,#s do + local v = s[k] + showfeatures(v,t[v]) end - end) + else + for k=1,#s do + local v = s[k] + local entry = t[v] + s[k] = { + v, + entry.fontname, + entry.filename, + subfont(entry.subfont) + } + end + table.formatcolumns(s) + for k=1,#s do + texio.write_nl(s[k]) + end + end end end -function scripts.fonts.save(name,sub) +function scripts.fonts.list() + + local all = environment.argument("all") + local info = environment.argument("info") + local reload = environment.argument("reload") + local pattern = environment.argument("pattern") + local filter = environment.argument("filter") + local given = environment.files[1] + + reloadbase(reload) + + if environment.argument("name") then + if pattern then + --~ mtxrun --script font --list --name --pattern=*somename* + list_matches(fonts.names.list(string.topattern(pattern,true),reload,all),info) + elseif filter then + logs.report("fontnames","not supported: --list --name --filter",name) + elseif given then + --~ mtxrun --script font --list --name somename + list_matches(fonts.names.list(given,reload,all),info) + else + logs.report("fontnames","not supported: --list --name <no specification>",name) + end + elseif environment.argument("spec") then + if pattern then + --~ mtxrun --script font --list --spec --pattern=*somename* + logs.report("fontnames","not supported: --list --spec --pattern",name) + elseif filter then + --~ mtxrun --script font --list --spec --filter="fontname=somename" + list_specifications(fonts.names.getlookups(filter),info) + elseif given then + --~ mtxrun --script font --list --spec somename + list_specifications(fonts.names.collectspec(given,reload,all),info) + else + logs.report("fontnames","not supported: --list --spec <no specification>",name) + end + elseif environment.argument("file") then + if pattern then + --~ mtxrun --script font --list --file --pattern=*somename* + list_specifications(fonts.names.collectfiles(string.topattern(pattern,true),reload,all),info) + elseif filter then + logs.report("fontnames","not supported: --list --spec",name) + elseif given then + --~ mtxrun --script font --list --file somename + list_specifications(fonts.names.collectfiles(given,reload,all),info) + else + logs.report("fontnames","not supported: --list --file <no specification>",name) + end + elseif pattern then + --~ mtxrun --script font --list --pattern=*somename* + list_matches(fonts.names.list(string.topattern(pattern,true),reload,all),info) + elseif given then + --~ mtxrun --script font --list somename + list_matches(fonts.names.list(given,reload,all),info) + else + logs.report("fontnames","not supported: --list <no specification>",name) + end + +end + +function scripts.fonts.save() + local name = environment.files[1] or "" + local sub = environment.files[2] or "" local function save(savename,fontblob) if fontblob then savename = savename:lower() .. ".lua" @@ -128,12 +269,15 @@ function scripts.fonts.save(name,sub) if fontinfo then logs.simple("font: %s located as %s",name,filename) if fontinfo[1] then - for _, v in ipairs(fontinfo) do + for k=1,#fontinfo do + local v = fontinfo[k] save(v.fontname,fontloader.open(filename,v.fullname)) end else save(fontinfo.fullname,fontloader.open(filename)) end + else + logs.simple("font: %s cannot be read",filename) end else logs.simple("font: %s not saved",filename) @@ -141,35 +285,61 @@ function scripts.fonts.save(name,sub) else logs.simple("font: %s not found",name) end + else + logs.simple("font: no name given") end end -logs.extendbanner("Font Tools 0.20",true) +logs.extendbanner("ConTeXt Font Database Management 0.21",true) messages.help = [[ ---reload generate new font database ---list [--info] list installed fonts (show info) --save save open type font in raw table ---names generate 'luatex-fonts-names.lua' (not for context!) ---pattern=str filter files ---all provide alternatives +--reload generate new font database +--reload --simple generate 'luatex-fonts-names.lua' (not for context!) + +--list --name list installed fonts, filter by name [--pattern] +--list --spec list installed fonts, filter by spec [--filter] +--list --file list installed fonts, filter by file [--pattern] + +--pattern=str filter files using pattern +--filter=list key-value pairs +--all show all found instances +--info give more details +--track=list enable trackers + +examples of searches: + +mtxrun --script font --list somename (== --pattern=*somename*) + +mtxrun --script font --list --name somename +mtxrun --script font --list --name --pattern=*somename* + +mtxrun --script font --list --spec somename +mtxrun --script font --list --spec somename-bold-italic +mtxrun --script font --list --spec --pattern=*somename* +mtxrun --script font --list --spec --filter="fontname=somename" +mtxrun --script font --list --spec --filter="familyname=somename,weight=bold,style=italic,width=condensed" + +mtxrun --script font --list --file somename +mtxrun --script font --list --file --pattern=*somename* ]] -if environment.argument("reload") then - scripts.fonts.reload(true) -elseif environment.argument("names") then - scripts.fonts.names() -elseif environment.argument("list") then - local pattern = environment.argument("pattern") or environment.files[1] or "" - local all = environment.argument("all") - local info = environment.argument("info") - local reload = environment.argument("reload") - scripts.fonts.list(pattern,reload,all,info) +local track = environment.argument("track") + +if track then trackers.enable(track) end + +if environment.argument("names") then + environment.setargument("reload",true) + environment.setargument("simple",true) +end + +if environment.argument("list") then + scripts.fonts.list() +elseif environment.argument("reload") then + scripts.fonts.reload() elseif environment.argument("save") then - local name = environment.files[1] or "" - local sub = environment.files[2] or "" - scripts.fonts.save(name,sub) + scripts.fonts.save() else logs.help(messages.help) end diff --git a/Master/texmf-dist/scripts/context/lua/mtx-grep.lua b/Master/texmf-dist/scripts/context/lua/mtx-grep.lua index a6617d711f7..9604bc9f85d 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-grep.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-grep.lua @@ -70,7 +70,9 @@ function scripts.grep.find(pattern, files, offset) end local capture = (content/check)^0 for i=offset or 1, #files do - for _, nam in ipairs(dir.glob(files[i])) do + local globbed = dir.glob(files[i]) + for i=1,#globbed do + local nam = globbed[i] name = nam local data = io.loaddata(name) if data then diff --git a/Master/texmf-dist/scripts/context/lua/mtx-interface.lua b/Master/texmf-dist/scripts/context/lua/mtx-interface.lua index 264a2dbe4f4..730a030d930 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-interface.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-interface.lua @@ -18,7 +18,8 @@ local messageinterfaces = { 'en','cs','de','it','nl','ro','fr','pe','no' } function flushers.scite(interface,collection) local result, i = {}, 0 result[#result+1] = format("keywordclass.macros.context.%s=",interface) - for _, command in ipairs(collection) do + for i=1,#collection do + local command = collection[i] if i==0 then result[#result+1] = "\\\n" i = 5 @@ -38,7 +39,8 @@ function flushers.jedit(interface,collection) result[#result+1] = "<MODE>" result[#result+1] = "\t<RULES>" result[#result+1] = "\t\t<KEYWORDS>" - for _, command in ipairs(collection) do + for i=1,#collection do + local command = collection[i] result[#result+1] = format("\t\t\t<KEYWORD2>%s</KEYWORD2>",command) end result[#result+1] = "\t\t</KEYWORDS>" @@ -52,7 +54,8 @@ function flushers.bbedit(interface,collection) result[#result+1] = "<?xml version='1.0'?>" result[#result+1] = "<key>BBLMKeywordList</key>" result[#result+1] = "<array>" - for _, command in ipairs(collection) do + for i=1,#collection do + local command = collection[i] result[#result+1] = format("\t<string>\\%s</string>",command) end result[#result+1] = "</array>" @@ -60,7 +63,8 @@ function flushers.bbedit(interface,collection) end function flushers.raw(interface,collection) - for _, command in ipairs(collection) do + for i=1,#collection do + local command = collection[i] logs.simple(command) end end @@ -74,7 +78,8 @@ function scripts.interface.editor(editor) if xmlfile == "" then logs.simple("unable to locate cont-en.xml") end - for _, interface in ipairs(interfaces) do + for i=1,#interfaces do + local interface = interfaces[i] local keyfile = resolvers.find_file(format("keys-%s.xml",interface)) or "" if keyfile == "" then logs.simple("unable to locate keys-*.xml") @@ -138,7 +143,7 @@ function scripts.interface.check() end function scripts.interface.context() - local filename = resolvers.find_file("mult-def.lua") or "" + local filename = resolvers.find_file(environment.files[1] or "mult-def.lua") or "" if filename ~= "" then local interface = dofile(filename) if interface and next(interface) then @@ -150,7 +155,9 @@ function scripts.interface.context() texresult[#texresult+1] = format("%% definitions for interface %s for language %s\n%%",what,language) xmlresult[#xmlresult+1] = format("\t<!-- definitions for interface %s for language %s -->\n",what,language) xmlresult[#xmlresult+1] = format("\t<cd:%s>",what) - for _, key in ipairs(table.sortedkeys(t)) do + local sorted = table.sortedkeys(t) + for i=1,#sorted do + local key = sorted[i] local v = t[key] local value = v[language] or v["en"] if not value then @@ -178,7 +185,7 @@ function scripts.interface.context() return a .. b .. c .. b end) end - for language, _ in pairs(commands.setuplayout) do + for language, _ in next, commands.setuplayout do local texresult, xmlresult = { }, { } texresult[#texresult+1] = format("%% this file is auto-generated, don't edit this file\n%%") xmlresult[#xmlresult+1] = format("<?xml version='1.0'?>\n",tag) @@ -213,10 +220,11 @@ function scripts.interface.context() end function scripts.interface.messages() - local filename = resolvers.find_file("mult-mes.lua") or "" + local filename = resolvers.find_file(environment.files[1] or "mult-mes.lua") or "" if filename ~= "" then local messages = dofile(filename) - for _, interface in ipairs(messageinterfaces) do + for i=1,#messageinterfaces do + local interface = messageinterfaces[i] local texresult = { } for category, data in next, messages do for tag, message in next, data do @@ -234,7 +242,7 @@ function scripts.interface.messages() end end -logs.extendbanner("Interface Tools 0.11",true) +logs.extendbanner("ConTeXt Interface Related Goodies 0.11",true) messages.help = [[ --scite generate scite interface diff --git a/Master/texmf-dist/scripts/context/lua/mtx-metatex.lua b/Master/texmf-dist/scripts/context/lua/mtx-metatex.lua index f8c871a7b1c..4453e2ccb47 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-metatex.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-metatex.lua @@ -49,7 +49,7 @@ function scripts.metatex.timed(action) statistics.timed(action) end -logs.extendbanner("MetaTeX Tools 0.10",true) +logs.extendbanner("MetaTeX Process Management 0.10",true) messages.help = [[ --run process (one or more) files (default action) diff --git a/Master/texmf-dist/scripts/context/lua/mtx-modules.lua b/Master/texmf-dist/scripts/context/lua/mtx-modules.lua new file mode 100644 index 00000000000..3a348593f8e --- /dev/null +++ b/Master/texmf-dist/scripts/context/lua/mtx-modules.lua @@ -0,0 +1,167 @@ +if not modules then modules = { } end modules ['mtx-modules'] = { + version = 1.002, + comment = "companion to mtxrun.lua", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +scripts = scripts or { } +scripts.modules = scripts.modules or { } + +-- Documentation can be woven into a source file. This script can generates +-- a file with the documentation and source fragments properly tagged. The +-- documentation is included as comment: +-- +-- %D ...... some kind of documentation +-- %M ...... macros needed for documenation +-- %S B begin skipping +-- %S E end skipping +-- +-- The generated file is structured as: +-- +-- \starttypen +-- \startmodule[type=suffix] +-- \startdocumentation +-- \stopdocumentation +-- \startdefinition +-- \stopdefinition +-- \stopmodule +-- \stoptypen +-- +-- Macro definitions specific to the documentation are not surrounded by +-- start-stop commands. The suffix specificaction can be overruled at runtime, +-- but defaults to the file extension. This specification can be used for language +-- depended verbatim typesetting. + +local find, format, sub, is_empty, strip = string.find, string.format, string.sub, string.is_empty, string.strip + +local function source_to_ted(inpname,outname,filetype) + local inp = io.open(inpname) + if not inp then + logs.simple("unable to open '%s'",inpname) + return + end + local out = io.open(outname,"w") + if not out then + logs.simple("unable to open '%s'",outname) + return + end + logs.simple("converting '%s' to '%s'",inpname,outname) + local skiplevel, indocument, indefinition = 0, false, false + out:write(format("\\startmodule[type=%s]\n",filetype or file.suffix(inpname))) + for line in inp:lines() do +--~ line = strip(line) + if find(line,"^%%D ") or find(line,"^%%D$") then + if skiplevel == 0 then + local someline = (#line < 3 and "") or sub(line,4,#line) + if indocument then + out:write(format("%s\n",someline)) + else + if indefinition then + out:write("\\stopdefinition\n") + indefinition = false + end + if not indocument then + out:write("\n\\startdocumentation\n") + end + out:write(format("%s\n",someline)) + indocument = true + end + end + elseif find(line,"^%%M ") or find(line,"^%%M$") then + if skiplevel == 0 then + local someline = (#line < 3 and "") or sub(line,4,#line) + out:write(format("%s\n",someline)) + end + elseif find(line,"^%%S B") then + skiplevel = skiplevel + 1 + elseif find(line,"^%%S E") then + skiplevel = skiplevel - 1 + elseif find(line,"^%%") then + -- nothing + elseif skiplevel == 0 then + inlocaldocument = indocument + inlocaldocument = false + local someline = line + if indocument then + out:write("\\stopdocumentation\n") + indocument = false + end + if indefinition then + if is_empty(someline) then + out:write("\\stopdefinition\n") + indefinition = false + else + out:write(format("%s\n",someline)) + end + elseif not is_empty(someline) then + out:write("\n\\startdefinition\n") + indefinition = true + if inlocaldocument then + -- nothing + else + out:write(format("%s\n",someline)) + end + end + end + end + if indocument then + out:write("\\stopdocumentation\n") + end + if indefinition then + out:write("\\stopdefinition\n") + end + out:write("\\stopmodule\n") + out:close() + inp:close() + return true +end + +local suffixes = table.tohash { 'tex','mkii','mkiv','mp' } + +function scripts.modules.process(runtex) + local processed = { } + local prep = environment.argument("prep") + local files = environment.files + for i=1,#files do + local shortname = files[i] + local suffix = file.suffix(shortname) + if suffixes[suffix] then + local longname + if prep then + longname = shortname .. ".prep" + else + longname = file.removesuffix(shortname) .. "-" .. suffix .. ".ted" + end + local done = source_to_ted(shortname,longname) + if done and runtex then + os.execute(format("mtxrun --script context --usemodule=mod-01 %s",longname)) + processed[#processed+1] = longname + end + end + end + for i=1,#processed do + local name = processed[i] + logs.simple("modules","processed: %s",name) + end +end + +-- context --ctx=m-modules.ctx xxx.mkiv + + +logs.extendbanner("ConTeXt Module Documentation Generators 1.00",true) + +messages.help = [[ +--convert convert source files (tex, mkii, mkiv, mp) to 'ted' files +--process process source files (tex, mkii, mkiv, mp) to 'pdf' files +--prep use original name with suffix 'prep' appended +]] + +if environment.argument("process") then + scripts.modules.process(true) +elseif environment.argument("convert") then + scripts.modules.process(false) +else + logs.help(messages.help) +end diff --git a/Master/texmf-dist/scripts/context/lua/mtx-mptopdf.lua b/Master/texmf-dist/scripts/context/lua/mtx-mptopdf.lua index 4243625adb9..342ff1c284a 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-mptopdf.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-mptopdf.lua @@ -1,6 +1,6 @@ if not modules then modules = { } end modules ['mtx-mptopdf'] = { version = 1.303, - comment = "companion to mtxrun.lua", + comment = "companion to mtxrun.lua, patched by HH so errors are his", author = "Taco Hoekwater, Elvenkind BV, Dordrecht NL", copyright = "Elvenkind BV / ConTeXt Development Team", license = "see context related readme files" @@ -10,40 +10,30 @@ scripts = scripts or { } scripts.mptopdf = scripts.mptopdf or { } scripts.mptopdf.aux = scripts.mptopdf.aux or { } -do - -- setup functions and variables here +local dosish = os.type == 'windows' +local miktex = dosish and environment.TEXSYSTEM and environment.TEXSYSTEM:find("miktex") +local escapeshell = environment.SHELL and environment.SHELL:find("sh") and true - local dosish, miktex, escapeshell = false, false, false +function scripts.mptopdf.aux.find_latex(fname) + local d = io.loaddata(fname) or "" + return d:find("\\documentstyle") or d:find("\\documentclass") or d:find("\\begin{document}") +end - if os.platform == 'windows' then - dosish = true - if environment.TEXSYSTEM and environment.TEXSYSTEM:find("miktex") then - miktex = true +function scripts.mptopdf.aux.do_convert (fname) + local command, done, pdfdest = "", 0, "" + if fname:find(".%d+$") or fname:find("%.mps$") then + if miktex then + command = "pdftex -undump=mptopdf" + else + command = "pdftex -fmt=mptopdf -progname=context" end - end - if environment.SHELL and environment.SHELL:find("sh") then - escapeshell = true - end - - function scripts.mptopdf.aux.find_latex(fname) - local d = io.loaddata(fname) or "" - return d:find("\\documentstyle") or d:find("\\documentclass") or d:find("\\begin{document}") - end - - function scripts.mptopdf.aux.do_convert (fname) - local command, done, pdfdest = "", 0, "" - if fname:find(".%d+$") or fname:find("%.mps$") then - if miktex then - command = "pdftex -undump=mptopdf" - else - command = "pdftex -fmt=mptopdf -progname=context" - end - if dosish then - command = string.format('%s \\relax "%s"',command,fname) - else - command = string.format('%s \\\\relax "%s"',command,fname) - end - os.execute(command) + if dosish then + command = string.format('%s \\relax "%s"',command,fname) + else + command = string.format('%s \\\\relax "%s"',command,fname) + end + local result = os.execute(command) + if result == 0 then local name, suffix = file.nameonly(fname), file.extname(fname) local pdfsrc = name .. ".pdf" if lfs.isfile(pdfsrc) then @@ -55,29 +45,28 @@ do done = 1 end end - return done, pdfdest end + return done, pdfdest +end - function scripts.mptopdf.aux.make_mps(fn,latex,rawmp,metafun) - local rest, mpbin = latex and " --tex=latex " or " ", "" - if rawmp then - if metafun then - mpbin = "mpost --progname=mpost --mem=metafun" - else - mpbin = "mpost --mem=mpost" - end +function scripts.mptopdf.aux.make_mps(fn,latex,rawmp,metafun) + local rest, mpbin = latex and " --tex=latex " or " ", "" + if rawmp then + if metafun then + mpbin = "mpost --progname=mpost --mem=metafun" else - if latex then - mpbin = "mpost --mem=mpost" - else - mpbin = "texexec --mptex" - end + mpbin = "mpost --mem=mpost" end - local runner = mpbin .. rest .. fn - logs.simple("running: %s\n", runner) - return (os.execute(runner)) - end - + else + if latex then + mpbin = "mpost --mem=mpost" + else + mpbin = "texexec --mptex" + end + end + local runner = mpbin .. rest .. fn + logs.simple("running: %s\n", runner) + return (os.execute(runner)) end function scripts.mptopdf.convertall() @@ -97,7 +86,8 @@ function scripts.mptopdf.convertall() exit(1) end local report = { } - for _,fn in ipairs(files) do + for i=1,#files do + local fn = files[i] local success, name = scripts.mptopdf.aux.do_convert(fn) if success > 0 then report[#report+1] = { fn, name } @@ -106,11 +96,12 @@ function scripts.mptopdf.convertall() if #report > 0 then logs.simple("number of converted files: %i", #report) logs.simple("") - for _, r in ipairs(report) do + for i=1,#report do + local r = report[i] logs.simple("%s => %s", r[1], r[2]) end else - logs.simple("no input files match %s", table.concat(files,' ')) + logs.simple("no files are converted") end else logs.simple("no files match %s", table.concat(environment.files,' ')) diff --git a/Master/texmf-dist/scripts/context/lua/mtx-mtxworks.lua b/Master/texmf-dist/scripts/context/lua/mtx-mtxworks.lua new file mode 100644 index 00000000000..1239ae4c5b5 --- /dev/null +++ b/Master/texmf-dist/scripts/context/lua/mtx-mtxworks.lua @@ -0,0 +1,14 @@ +if not modules then modules = { } end modules ['mtx-mtxworks'] = { + version = 1.002, + comment = "companion to mtxrun.lua", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- this is a shortcut to "mtxrun --script texworks --start" + +environment.setargument("start",true) + +require "mtx-texworks" + diff --git a/Master/texmf-dist/scripts/context/lua/mtx-package.lua b/Master/texmf-dist/scripts/context/lua/mtx-package.lua index 06c89907adf..b36fc0ed89c 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-package.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-package.lua @@ -55,7 +55,7 @@ function scripts.package.merge_luatex_files(name,strip) end end -logs.extendbanner("Package Tools 0.1",true) +logs.extendbanner("Distribution Related Goodies 0.10",true) messages.help = [[ --merge merge 'loadmodule' into merge file diff --git a/Master/texmf-dist/scripts/context/lua/mtx-patterns.lua b/Master/texmf-dist/scripts/context/lua/mtx-patterns.lua index 7f130465bdf..293016991ff 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-patterns.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-patterns.lua @@ -14,25 +14,25 @@ scripts.patterns = scripts.patterns or { } scripts.patterns.list = { { "??", "hyph-ar.tex", "arabic" }, { "bg", "hyph-bg.tex", "bulgarian" }, --- { "ca", "hyph-ca.tex", "" }, + { "ca", "hyph-ca.tex", "catalan" }, { "??", "hyph-cop.tex", "coptic" }, { "cs", "hyph-cs.tex", "czech" }, - { "??", "hyph-cy.tex", "welsh" }, + { "cy", "hyph-cy.tex", "welsh" }, { "da", "hyph-da.tex", "danish" }, { "deo", "hyph-de-1901.tex", "german, old spelling" }, { "de", "hyph-de-1996.tex", "german, new spelling" }, --~ { "??", "hyph-el-monoton.tex", "" }, --~ { "??", "hyph-el-polyton.tex", "" }, ---~ { "agr", "hyph-grc", "ancient greek" }, + { "agr", "hyph-grc", "ancient greek" }, --~ { "???", "hyph-x-ibycus", "ancient greek in ibycus encoding" }, --~ { "gr", "", "" }, - { "??", "hyph-eo.tex", "esperanto" }, + { "eo", "hyph-eo.tex", "esperanto" }, { "gb", "hyph-en-gb.tex", "british english" }, { "us", "hyph-en-us.tex", "american english" }, { "es", "hyph-es.tex", "spanish" }, { "et", "hyph-et.tex", "estonian" }, { "eu", "hyph-eu.tex", "basque" }, -- ba is Bashkir! - { "??", "hyph-fa.tex", "farsi" }, + { "fa", "hyph-fa.tex", "farsi" }, { "fi", "hyph-fi.tex", "finnish" }, { "fr", "hyph-fr.tex", "french" }, -- { "??", "hyph-ga.tex", "" }, @@ -43,11 +43,11 @@ scripts.patterns.list = { { "hu", "hyph-hu.tex", "hungarian" }, { "??", "hyph-ia.tex", "interlingua" }, { "??", "hyph-id.tex", "indonesian" }, - { "??", "hyph-is.tex", "icelandic" }, + { "is", "hyph-is.tex", "icelandic" }, { "it", "hyph-it.tex", "italian" }, { "la", "hyph-la.tex", "latin" }, - { "??", "hyph-mn-cyrl.tex", "mongolian, cyrillic script" }, - { "??", "hyph-mn-cyrl-x-new.tex", "mongolian, cyrillic script (new patterns)" }, + { "lt", "hyph-lt.tex", "lithuanian" }, + { "mn", "hyph-mn-cyrl.tex", "mongolian, cyrillic script" }, { "nb", "hyph-nb.tex", "norwegian bokmål" }, { "nl", "hyph-nl.tex", "dutch" }, { "nn", "hyph-nn.tex", "norwegian nynorsk" }, @@ -55,13 +55,14 @@ scripts.patterns.list = { { "pt", "hyph-pt.tex", "portuguese" }, { "ro", "hyph-ro.tex", "romanian" }, { "ru", "hyph-ru.tex", "russian" }, - { "sk", "hyph-sk.tex", "" }, + { "sk", "hyph-sk.tex", "slovak" }, { "sl", "hyph-sl.tex", "slovenian" }, - { "??", "hyph-sr-cyrl.tex", "serbian" }, + { "sr", "hyph-sr-cyrl.tex", "serbian" }, { "sv", "hyph-sv.tex", "swedish" }, { "tr", "hyph-tr.tex", "turkish" }, + { "tk", "hyph-tk.tex", "turkman" }, { "uk", "hyph-uk.tex", "ukrainian" }, - { "??", "hyph-zh-latn.tex", "zh-latn, chinese Pinyin" }, + { "zh", "hyph-zh-latn.tex", "zh-latn, chinese Pinyin" }, } @@ -152,7 +153,7 @@ function scripts.patterns.load(path,name,mnemonic,fullcheck) end h.patterns = nil h.hyphenation = nil - for k, v in pairs(h) do + for k, v in next, h do if not permitted_commands[k] then okay = false end if mnemonic then logs.simple("command \\%s found in language %s, file %s, n=%s",k,mnemonic,name,v) @@ -161,7 +162,7 @@ function scripts.patterns.load(path,name,mnemonic,fullcheck) end end if not environment.argument("fast") then - for k, v in pairs(c) do + for k, v in next, c do if mnemonic then logs.simple("command \\%s found in comment of language %s, file %s, n=%s",k,mnemonic,name,v) else @@ -221,7 +222,7 @@ function scripts.patterns.load(path,name,mnemonic,fullcheck) end end local stripped = { } - for k, v in pairs(p) do + for k, v in next, p do if mnemonic then logs.simple("invalid character %s (0x%04X) in patterns of language %s, file %s, n=%s",char(k),k,mnemonic,name,v) else @@ -233,7 +234,7 @@ function scripts.patterns.load(path,name,mnemonic,fullcheck) stripped[k] = true end end - for k, v in pairs(h) do + for k, v in next, h do if mnemonic then logs.simple("invalid character %s (0x%04X) in exceptions of language %s, file %s, n=%s",char(k),k,mnemonic,name,v) else @@ -246,7 +247,7 @@ function scripts.patterns.load(path,name,mnemonic,fullcheck) end end local stripset = "" - for k, v in pairs(stripped) do + for k, v in next, stripped do logs.simple("entries that contain character %s will be omitted",char(k)) stripset = stripset .. "%" .. char(k) end @@ -292,8 +293,10 @@ end function scripts.patterns.check() local path = environment.argument("path") or "." local found = false - if #environment.files > 0 then - for _, name in ipairs(environment.files) do + local files = environment.files + if #files > 0 then + for i=1,#files do + local name = files[i] logs.simple("checking language file %s", name) local okay = scripts.patterns.load(path,name,nil,not environment.argument("fast")) if #environment.files > 1 then @@ -301,7 +304,7 @@ function scripts.patterns.check() end end else - for k, v in pairs(scripts.patterns.list) do + for k, v in next, scripts.patterns.list do local mnemonic, name = v[1], v[2] logs.simple("checking language %s, file %s", mnemonic, name) local okay = scripts.patterns.load(path,name,mnemonic,not environment.argument("fast")) @@ -322,7 +325,7 @@ function scripts.patterns.convert() if path == destination then logs.simple("source path and destination path should differ (use --path and/or --destination)") else - for k, v in pairs(scripts.patterns.list) do + for k, v in next, scripts.patterns.list do local mnemonic, name = v[1], v[2] logs.simple("converting language %s, file %s", mnemonic, name) local okay, patterns, hyphenations, comment, stripped, pused, hused = scripts.patterns.load(path,name,false) @@ -337,7 +340,7 @@ function scripts.patterns.convert() end end -logs.extendbanner("Pattern Tools 0.20",true) +logs.extendbanner("ConTeXt Pattern File Management 0.20",true) messages.help = [[ --convert generate context language files (mnemonic driven, if not given then all) @@ -360,3 +363,4 @@ end -- mtxrun --script pattern --check --path=c:/data/develop/svn-hyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns -- mtxrun --script pattern --check --fast --path=c:/data/develop/svn-hyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns -- mtxrun --script pattern --convert --path=c:/data/develop/svn-hyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns --destination=e:/tmp/patterns +-- mtxrun --script pattern --convert --path=c:/data/develop/svn-hyphen/branches/luatex/hyph-utf8/tex/generic/hyph-utf8/patterns/tex --destination=e:/tmp/patterns diff --git a/Master/texmf-dist/scripts/context/lua/mtx-profile.lua b/Master/texmf-dist/scripts/context/lua/mtx-profile.lua index d99f7e926bc..11d48d0399b 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-profile.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-profile.lua @@ -54,9 +54,13 @@ function scripts.profiler.analyse(filename) f:close() print("") local loaded = { } - for _, filename in ipairs(table.sortedkeys(times)) do + local sortedtable.sortedkeys(times) + for i=1,#sorted do + local filename = sorted[i] local functions = times[filename] - for _, functionname in ipairs(table.sortedkeys(functions)) do + local sorted = table.sortedkeys(functions) + for i=1,#sorted do + local functionname = sorted[i] local totaltime = functions[functionname] local count = counts[functionname] totalcount = totalcount + count @@ -81,7 +85,9 @@ function scripts.profiler.analyse(filename) end end print("") - for _, call in ipairs(table.sortedkeys(calls)) do + local sorted = table.sortedkeys(calls) + for i=1,#sorted do + local call = sorted[i] local n = calls[call] totalcalls = totalcalls + n if n > callthreshold then @@ -95,7 +101,7 @@ function scripts.profiler.analyse(filename) end end -function scripts.profiler.analyse(filename) +function scripts.profiler.x_analyse(filename) local f = io.open(filename) local calls = { } local lines = 0 @@ -148,7 +154,7 @@ end --~ scripts.profiler.analyse("t:/manuals/mk/mk-fonts-profile.lua") --~ scripts.profiler.analyse("t:/manuals/mk/mk-introduction-profile.lua") -logs.extendbanner("LuaTeX Profiler 1.00",true) +logs.extendbanner("ConTeXt MkIV LuaTeX Profiler 1.00",true) messages.help = [[ --analyse analyse lua calls diff --git a/Master/texmf-dist/scripts/context/lua/mtx-scite.lua b/Master/texmf-dist/scripts/context/lua/mtx-scite.lua new file mode 100644 index 00000000000..d5f0a5344e5 --- /dev/null +++ b/Master/texmf-dist/scripts/context/lua/mtx-scite.lua @@ -0,0 +1,166 @@ +if not modules then modules = { } end modules ['mtx-scite'] = { + version = 1.001, + comment = "companion to mtxrun.lua", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- todo: append to global properties else order of loading problem +-- linux problem ... files are under root protection so we need --install + +scripts = scripts or { } +scripts.scite = scripts.scite or { } + +local scitesignals = { "scite-context.rme", "context.properties" } +local screenfont = "lmtypewriter10-regular.ttf" + +function scripts.scite.start(indeed) + local usedsignal, datapath, fullname, workname, userpath, fontpath + if os.type == "windows" then + workname = "scite.exe" + userpath = os.getenv("USERPROFILE") or "" + fontpath = os.getenv("SYSTEMROOT") + fontpath = (fontpath and file.join(fontpath,"fonts")) or "" + else + workname = "scite" + userpath = os.getenv("HOME") or "" + fontpath = "" + end + local binpaths = file.split_path(os.getenv("PATH")) or file.split_path(os.getenv("path")) + for i=1,#scitesignals do + local scitesignal = scitesignals[i] + local scitepath = resolvers.find_file(scitesignal,"other text files") or "" + if scitepath ~= "" then + scitepath = file.dirname(scitepath) -- data + if scitepath == "" then + scitepath = resolvers.clean_path(lfs.currentdir()) + else + usedsignal, datapath = scitesignal, scitepath + break + end + end + end + if not datapath or datapath == "" then + logs.simple("invalid datapath, maybe you need to regenerate the file database") + return false + end + if not binpaths or #binpaths == 0 then + logs.simple("invalid binpath") + return false + end + for i=1,#binpaths do + local p = file.join(binpaths[i],workname) + if lfs.isfile(p) and lfs.attributes(p,"size") > 10000 then -- avoind stub + fullname = p + break + end + end + if not fullname then + logs.simple("unable to locate %s",workname) + return false + end + local properties = dir.glob(file.join(datapath,"*.properties")) + local luafiles = dir.glob(file.join(datapath,"*.lua")) + local extrafont = resolvers.find_file(screenfont,"truetype font") or "" + local pragmafound = dir.glob(file.join(datapath,"pragma.properties")) + if userpath == "" then + logs.simple("unable to figure out userpath") + return false + end + local verbose = environment.argument("verbose") + local tobecopied, logdata = { }, { } + local function check_state(fullname,newpath) + local basename = file.basename(fullname) + local destination = file.join(newpath,basename) + local pa, da = lfs.attributes(fullname), lfs.attributes(destination) + if not da then + logdata[#logdata+1] = { "new : %s", basename } + tobecopied[#tobecopied+1] = { fullname, destination } + elseif pa.modification > da.modification then + logdata[#logdata+1] = { "outdated : %s", basename } + tobecopied[#tobecopied+1] = { fullname, destination } + else + logdata[#logdata+1] = { "up to date : %s", basename } + end + end + for i=1,#properties do + check_state(properties[i],userpath) + end + for i=1,#luafiles do + check_state(luafiles[i],userpath) + end + if fontpath ~= "" then + check_state(extrafont,fontpath) + end + local userpropfile = "SciTEUser.properties" + if os.name ~= "windows" then + userpropfile = "." .. userpropfile + end + local fullpropfile = file.join(userpath,userpropfile) + local userpropdata = io.loaddata(fullpropfile) or "" + local propfiledone = false + if pragmafound then + if userpropdata == "" then + logdata[#logdata+1] = { "error : no user properties found on '%s'", fullpropfile } + elseif string.find(userpropdata,"import *pragma") then + logdata[#logdata+1] = { "up to date : 'import pragma' in '%s'", userpropfile } + else + logdata[#logdata+1] = { "yet unset : 'import pragma' in '%s'", userpropfile } + userproperties = userpropdata .. "\n\nimport pragma\n\n" + propfiledone = true + end + else + if string.find(userpropdata,"import *context") then + logdata[#logdata+1] = { "up to date : 'import context' in '%s'", userpropfile } + else + logdata[#logdata+1] = { "yet unset : 'import context' in '%s'", userpropfile } + userproperties = userpropdata .. "\n\nimport context\n\n" + propfiledone = true + end + end + if not indeed or verbose then + logs.simple("used signal: %s", usedsignal) + logs.simple("data path : %s", datapath) + logs.simple("full name : %s", fullname) + logs.simple("user path : %s", userpath) + logs.simple("extra font : %s", extrafont) + end + if #logdata > 0 then + logs.simple("") + for k=1,#logdata do + local v = logdata[k] + logs.simple(v[1],v[2]) + end + end + if indeed then + if #tobecopied > 0 then + logs.simple("warning : copying updated files") + for i=1,#tobecopied do + local what = tobecopied[i] + logs.simple("copying : '%s' => '%s'",what[1],what[2]) + file.copy(what[1],what[2]) + end + end + if propfiledone then + logs.simple("saving : '%s'",userpropfile) + io.savedata(fullpropfile,userpropdata) + end + os.launch(fullname) + end +end + +logs.extendbanner("Scite Startup Script 1.00",true) + +messages.help = [[ +--start [--verbose] start scite +--test report what will happen +]] + +if environment.argument("start") then + scripts.scite.start(true) +elseif environment.argument("test") then + scripts.scite.start() +else + logs.help(messages.help) +end diff --git a/Master/texmf-dist/scripts/context/lua/mtx-server-ctx-fonttest.lua b/Master/texmf-dist/scripts/context/lua/mtx-server-ctx-fonttest.lua index f593c2e351d..b2a993bf836 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-server-ctx-fonttest.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-server-ctx-fonttest.lua @@ -6,7 +6,7 @@ if not modules then modules = { } end modules ['mtx-server-ctx-fonttest'] = { license = "see context related readme files" } -dofile(resolvers.find_file("l-aux.lua","tex")) +--~ dofile(resolvers.find_file("l-aux.lua","tex")) dofile(resolvers.find_file("trac-lmx.lua","tex")) dofile(resolvers.find_file("font-ott.lua","tex")) dofile(resolvers.find_file("font-syn.lua","tex")) @@ -21,8 +21,11 @@ local temppath = caches.setpath("temp","mtx-server-ctx-fonttest") local basename = "mtx-server-ctx-fonttest-data.lua" local basepath = temppath -for _, suffix in ipairs { "tex", "pdf", "log" } do - os.remove(file.join(temppath,file.addsuffix(tempname,suffix))) +local remove_suffixes = { "tex", "pdf", "log" } +local what_options = { "trace", "basemode" } + +for i=1,#remove_suffixes do + os.remove(file.join(temppath,file.addsuffix(tempname,remove_suffixes[i]))) end local process_templates = { } @@ -30,7 +33,7 @@ local process_templates = { } process_templates.default = [[ \starttext \setcharactermirroring[1] - \definefontfeature[sample][%s] + \definefontfeature[sample][analyse=yes,%s] \definedfont[name:%s*sample] \startTEXpage[offset=3pt] \detokenize{%s} @@ -147,9 +150,10 @@ local cache = { } local function showfeatures(f) if f then + logs.simple("processing font '%s'",f) local features = cache[f] if features == nil then - features = fonts.get_features(f) + features = fonts.get_features(resolvers.find_file(f)) if not features then logs.simple("building cache for '%s'",f) io.savedata(file.join(temppath,file.addsuffix(tempname,"tex")),format(process_templates.cache,f,f)) @@ -166,18 +170,18 @@ local function showfeatures(f) local function show(what) local data = features[what] if data and next(data) then - for f,ff in pairs(data) do + for f,ff in next, data do if find(f,"<") then -- ignore aat for the moment else fea[f] = true - for s, ss in pairs(ff) do + for s, ss in next, ff do if find(s,"%*") then -- ignore * else scr[s] = true local rs = rev[s] if not rs then rs = {} rev[s] = rs end - for k, l in pairs(ss) do + for k, l in next, ss do if find(k,"%*") then -- ignore * else @@ -192,16 +196,16 @@ local function showfeatures(f) end end end - for what, v in table.sortedpairs(features) do + for what, v in table.sortedhash(features) do show(what) end local stupid = { } stupid[#stupid+1] = "var feature_hash = new Array ;" - for s, sr in pairs(rev) do + for s, sr in next, rev do stupid[#stupid+1] = format("feature_hash['%s'] = new Array ;",s) - for l, lr in pairs(sr) do + for l, lr in next, sr do stupid[#stupid+1] = format("feature_hash['%s']['%s'] = new Array ;",s,l) - for f, fr in pairs(lr) do + for f, fr in next, lr do stupid[#stupid+1] = format("feature_hash['%s']['%s']['%s'] = true ;",s,l,f) end end @@ -217,25 +221,50 @@ local function showfeatures(f) end end +local template_h = [[ +<tr> + <th>safe name </th> + <th>family name </th> + <th>style-variant-weight-width </th> + <th>font name </th> + <th>weight </th> + <th>filename</th> +</tr>]] + +local template_d = [[ +<tr> + <td><a href='mtx-server-ctx-fonttest.lua?selection=%s'>%s</a> </td> + <td>%s </td> + <td>%s-%s-%s-%s </td> + <td>%s </td> + <td>%s </td> + <td>%s</td> +</tr>]] + local function select_font() - local t = fonts.names.list(".*") + local t = fonts.names.list(".*",false,true) if t then local listoffonts = { } - local t = fonts.names.list(".*") - if t then - listoffonts[#listoffonts+1] = "<table>" - listoffonts[#listoffonts+1] = "<tr><th>safe name</th><th>font name</th><th>filename</th><th>sub font </th><th>type</th></tr>" - for k, id in ipairs(table.sortedkeys(t)) do - local ti = t[id] - local type, name, file, sub = ti[1], ti[2], ti[3], ti[4] - if type == "otf" or type == "ttf" or type == "ttc" then - if sub then sub = "(sub)" else sub = "" end - listoffonts[#listoffonts+1] = format("<tr><td><a href='mtx-server-ctx-fonttest.lua?selection=%s'>%s</a></td><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>",id,id,name,file,sub,type) - end + listoffonts[#listoffonts+1] = "<table>" + listoffonts[#listoffonts+1] = template_h + for k, v in table.sortedhash(t) do + local kind = v.format + if kind == "otf" or kind == "ttf" or kind == "ttc" then + local fontname = v.fontname + listoffonts[#listoffonts+1] = format(template_d, fontname, fontname, + v.familyname or "", + t.variant or "normal", + t.weight or "normal", + t.width or "normal", + t.style or "normal", + v.rawname or fontname, + v.fontweight or "", + v.filename or "" + ) end - listoffonts[#listoffonts+1] = "</table>" - return concat(listoffonts,"\n") end + listoffonts[#listoffonts+1] = "</table>" + return concat(listoffonts,"\n") end return "<b>no fonts</b>" end @@ -260,65 +289,80 @@ local result_template = [[ scripts.webserver.registerpath(temppath) +local function get_specification(name) + return fonts.names.resolvedspecification(name or "") +end + local function edit_font(currentfont,detail,tempname) - local fontname, fontfile, issub = fonts.names.specification(currentfont or "") - local htmldata = showfeatures(fontfile) - if htmldata then - local features, languages, scripts, options = { }, { }, { }, { } - for k,v in ipairs(table.sortedkeys(htmldata.scripts)) do - local s = fonts.otf.tables.scripts[v] or v - if detail and v == detail.script then - scripts[#scripts+1] = format("<input title='%s' id='s-%s' type='radio' name='script' value='%s' onclick='check_script()' checked='checked'/> <span id='t-s-%s'>%s</span>",s,v,v,v,v) - else - scripts[#scripts+1] = format("<input title='%s' id='s-%s' type='radio' name='script' value='%s' onclick='check_script()' /> <span id='t-s-%s'>%s</span>",s,v,v,v,v) + logs.simple("entering edit mode for '%s'",currentfont) + local specification = get_specification(currentfont) + if specification then + local htmldata = showfeatures(specification.filename) + if htmldata then + local features, languages, scripts, options = { }, { }, { }, { } + local sorted = table.sortedkeys(htmldata.scripts) + for k=1,#sorted do + local v = sorted[k] + local s = fonts.otf.tables.scripts[v] or v + if detail and v == detail.script then + scripts[#scripts+1] = format("<input title='%s' id='s-%s' type='radio' name='script' value='%s' onclick='check_script()' checked='checked'/> <span id='t-s-%s'>%s</span>",s,v,v,v,v) + else + scripts[#scripts+1] = format("<input title='%s' id='s-%s' type='radio' name='script' value='%s' onclick='check_script()' /> <span id='t-s-%s'>%s</span>",s,v,v,v,v) + end end - end - for k,v in ipairs(table.sortedkeys(htmldata.languages)) do - local l = fonts.otf.tables.languages[v] or v - if detail and v == detail.language then - languages[#languages+1] = format("<input title='%s' id='l-%s' type='radio' name='language' value='%s' onclick='check_language()' checked='checked'/> <span id='t-l-%s'>%s</span>",l,v,v,v,v) - else - languages[#languages+1] = format("<input title='%s' id='l-%s' type='radio' name='language' value='%s' onclick='check_language()' /> <span id='t-l-%s'>%s</span>",l,v,v,v,v) + local sorted = table.sortedkeys(htmldata.languages) + for k=1,#sorted do + local v = sorted[k] + local l = fonts.otf.tables.languages[v] or v + if detail and v == detail.language then + languages[#languages+1] = format("<input title='%s' id='l-%s' type='radio' name='language' value='%s' onclick='check_language()' checked='checked'/> <span id='t-l-%s'>%s</span>",l,v,v,v,v) + else + languages[#languages+1] = format("<input title='%s' id='l-%s' type='radio' name='language' value='%s' onclick='check_language()' /> <span id='t-l-%s'>%s</span>",l,v,v,v,v) + end end - end - for k,v in ipairs(table.sortedkeys(htmldata.features)) do - local f = fonts.otf.tables.features[v] or v - if detail and detail["f-"..v] then - features[#features+1] = format("<input title='%s' id='f-%s' type='checkbox' name='f-%s' onclick='check_feature()' checked='checked'/> <span id='t-f-%s'>%s</span>",f,v,v,v,v) - else - features[#features+1] = format("<input title='%s' id='f-%s' type='checkbox' name='f-%s' onclick='check_feature()' /> <span id='t-f-%s'>%s</span>",f,v,v,v,v) + local sorted = table.sortedkeys(htmldata.features) + for k=1,#sorted do + local v = sorted[k] + local f = fonts.otf.tables.features[v] or v + if detail and detail["f-"..v] then + features[#features+1] = format("<input title='%s' id='f-%s' type='checkbox' name='f-%s' onclick='check_feature()' checked='checked'/> <span id='t-f-%s'>%s</span>",f,v,v,v,v) + else + features[#features+1] = format("<input title='%s' id='f-%s' type='checkbox' name='f-%s' onclick='check_feature()' /> <span id='t-f-%s'>%s</span>",f,v,v,v,v) + end end - end - for k, v in ipairs { "trace", "basemode" } do - if detail and detail["o-"..v] then - options[#options+1] = format("<input id='o-%s' type='checkbox' name='o-%s' checked='checked'/> %s",v,v,v) + for k=1,#what_options do + local v = what_options[k] + if detail and detail["o-"..v] then + options[#options+1] = format("<input id='o-%s' type='checkbox' name='o-%s' checked='checked'/> %s",v,v,v) + else + options[#options+1] = format("<input id='o-%s' type='checkbox' name='o-%s'/> %s",v,v,v) + end + end + local e = format(edit_template, + (detail and detail.sampletext) or sample_line,(detail and detail.name) or "no name",(detail and detail.title) or "", + concat(scripts," "),concat(languages," "),concat(features," "),concat(options," ")) + if tempname then + local pdffile, texfile = file.addsuffix(tempname,"pdf"), file.addsuffix(tempname,"tex") + local r = format(result_template,pdffile,texfile,pdffile) + return e .. r, htmldata.javascript or "" else - options[#options+1] = format("<input id='o-%s' type='checkbox' name='o-%s'/> %s",v,v,v) + return e, htmldata.javascript or "" end - end - local e = format(edit_template, - (detail and detail.sampletext) or sample_line,(detail and detail.name) or "no name",(detail and detail.title) or "", - concat(scripts," "),concat(languages," "),concat(features," "),concat(options," ")) - if tempname then - local pdffile, texfile = file.addsuffix(tempname,"pdf"), file.addsuffix(tempname,"tex") - local r = format(result_template,pdffile,texfile,pdffile) - return e .. r, htmldata.javascript or "" else - return e, htmldata.javascript or "" + return "error, nothing set up yet" end else - return "error, nothing set up yet" + return "error, no info about font" end end local function process_font(currentfont,detail) -- maybe just fontname - local fontname, fontfile, issub = fonts.names.specification(currentfont or "") local features = { "mode=node", format("language=%s",detail.language or "dflt"), format("script=%s",detail.script or "dflt"), } - for k,v in pairs(detail) do + for k,v in next, detail do local f = match(k,"^f%-(.*)$") if f then features[#features+1] = format("%s=yes",f) @@ -361,25 +405,38 @@ local function show_log(currentfont,detail) end local function show_font(currentfont,detail) - local fontname, fontfile, issub = fonts.names.specification(currentfont or "") - local features = fonts.get_features(fontfile) + local specification = get_specification(currentfont) + local features = fonts.get_features(specification.filename) local result = { } result[#result+1] = format("<h1>names</h1>",what) result[#result+1] = "<table>" - result[#result+1] = format("<tr><td class='tc'>fontname:</td><td>%s</td></tr>",currentfont) - result[#result+1] = format("<tr><td class='tc'>fullname:</td><td>%s</td></tr>",fontname) - result[#result+1] = format("<tr><td class='tc'>filename:</td><td>%s</td></tr>",fontfile) + result[#result+1] = format("<tr><td class='tc'>fontname: </td><td>%s</td></tr>",currentfont) + result[#result+1] = format("<tr><td class='tc'>fullname: </td><td>%s</td></tr>",specification.fontname or "-") + result[#result+1] = format("<tr><td class='tc'>filename: </td><td>%s</td></tr>",specification.fontfile or "-") + result[#result+1] = format("<tr><td class='tc'>familyname: </td><td>%s</td></tr>",specification.familyname or "-") + result[#result+1] = format("<tr><td class='tc'>fontweight: </td><td>%s</td></tr>",specification.fontweight or "-") + result[#result+1] = format("<tr><td class='tc'>format: </td><td>%s</td></tr>",specification.format or "-") + result[#result+1] = format("<tr><td class='tc'>fullname: </td><td>%s</td></tr>",specification.fullname or "-") + result[#result+1] = format("<tr><td class='tc'>subfamily: </td><td>%s</td></tr>",specification.subfamily or "-") + result[#result+1] = format("<tr><td class='tc'>rawname: </td><td>%s</td></tr>",specification.rawname or "-") + result[#result+1] = format("<tr><td class='tc'>designsize: </td><td>%s</td></tr>",specification.designsize or "-") + result[#result+1] = format("<tr><td class='tc'>minimumsize:</td><td>%s</td></tr>",specification.minsize or "-") + result[#result+1] = format("<tr><td class='tc'>maximumsize:</td><td>%s</td></tr>",specification.maxsize or "-") + result[#result+1] = format("<tr><td class='tc'>style: </td><td>%s</td></tr>",specification.style ~= "" and specification.style or "normal") + result[#result+1] = format("<tr><td class='tc'>variant: </td><td>%s</td></tr>",specification.variant ~= "" and specification.variant or "normal") + result[#result+1] = format("<tr><td class='tc'>weight: </td><td>%s</td></tr>",specification.weight ~= "" and specification.weight or "normal") + result[#result+1] = format("<tr><td class='tc'>width: </td><td>%s</td></tr>",specification.width ~= "" and specification.width or "normal") result[#result+1] = "</table>" if features then - for what, v in table.sortedpairs(features) do + for what, v in table.sortedhash(features) do local data = features[what] if data and next(data) then result[#result+1] = format("<h1>%s features</h1>",what) result[#result+1] = "<table>" result[#result+1] = "<tr><th>feature</th><th>tag </th><th>script </th><th>languages </th></tr>" - for f,ff in table.sortedpairs(data) do + for f,ff in table.sortedhash(data) do local done = false - for s, ss in table.sortedpairs(ff) do + for s, ss in table.sortedhash(ff) do if s == "*" then s = "all" end if ss ["*"] then ss["*"] = nil ss.all = true end if done then @@ -457,10 +514,10 @@ local function loadstored(detail,currentfont,name) detail.title = storage.title or detail.title detail.sampletext = storage.text or detail.sampletext detail.name = name or "no name" - for k,v in pairs(storage.features) do + for k,v in next, storage.features do detail["f-"..k] = v end - for k,v in pairs(storage.options) do + for k,v in next, storage.options do detail["o-"..k] = v end end @@ -486,19 +543,20 @@ local function deletestored(detail,currentfont,name) end local function save_font(currentfont,detail) - local fontname, fontfile, issub = fonts.names.specification(currentfont or "") + local specification = get_specification(currentfont) local name, title, script, language, features, options, text = currentfont, "", "dflt", "dflt", { }, { }, "" if detail then - local htmldata = showfeatures(fontfile) + local htmldata = showfeatures(specification.filename) script = detail.script or script language = detail.language or language text = string.strip(detail.sampletext or text) name = string.strip(detail.name or name) title = string.strip(detail.title or title) - for k,v in pairs(htmldata.features) do + for k,v in next, htmldata.features do if detail["f-"..k] then features[k] = true end end - for k,v in ipairs { "trace", "basemode" } do + for k=1,#what_options do + local v = what_options[k] if detail["o-"..v] then options[k] = true end end end @@ -518,8 +576,8 @@ local function load_font(currentfont) local storage = loadbase(datafile) local result = {} result[#result+1] = format("<tr><th>del </th><th>name </th><th>font </th><th>fontname </th><th>script </th><th>language </th><th>features </th><th>title </th><th>sampletext </th></tr>") - for k,v in table.sortedpairs(storage) do - local fontname, fontfile, issub = fonts.names.specification(v.font or "") + for k,v in table.sortedhash(storage) do + local fontname, fontfile = get_specification(v.font) result[#result+1] = format("<tr><td><a href='mtx-server-ctx-fonttest.lua?deletename=%s'>x</a> </td><td><a href='mtx-server-ctx-fonttest.lua?loadname=%s'>%s</a> </td><td>%s </td<td>%s </td><td>%s </td><td>%s </td><td>%s </td><td>%s </td><td>%s </td></tr>", k,k,k,v.font,fontname,v.script,v.language,concat(table.sortedkeys(v.features)," "),v.title or "no title",v.text or "") end @@ -562,6 +620,13 @@ local status_template = [[ <input type="hidden" name="currentfont" value="%s" /> ]] +local variables = { + ['color-background-one'] = lmx.get('color-background-green'), + ['color-background-two'] = lmx.get('color-background-blue'), + ['title'] = 'ConTeXt Font Tester', + ['formaction'] = "mtx-server-ctx-fonttest.lua", +} + function doit(configuration,filename,hashed) local start = os.clock() @@ -589,65 +654,52 @@ function doit(configuration,filename,hashed) action = "extras" end - lmx.restore() - - local fontname, fontfile, issub = fonts.names.specification(currentfont or "") + local fontname, fontfile = get_specification(currentfont) if fontfile then - lmx.variables['title-default'] = format('ConTeXt Font Tester: %s (%s)',fontname,fontfile) + variables.title = format('ConTeXt Font Tester: %s (%s)',fontname,fontfile) else - lmx.variables['title-default'] = 'ConTeXt Font Tester' + variables.title = 'ConTeXt Font Tester' end - lmx.variables['color-background-green'] = '#4F6F6F' - lmx.variables['color-background-blue'] = '#6F6F8F' - lmx.variables['color-background-yellow'] = '#8F8F6F' - lmx.variables['color-background-purple'] = '#8F6F8F' - - lmx.variables['color-background-body'] = '#808080' - lmx.variables['color-background-main'] = '#3F3F3F' - lmx.variables['color-background-one'] = lmx.variables['color-background-green'] - lmx.variables['color-background-two'] = lmx.variables['color-background-blue'] - - lmx.variables['title'] = lmx.variables['title-default'] - - lmx.set('title', lmx.get('title')) - lmx.set('color-background-one', lmx.get('color-background-green')) - lmx.set('color-background-two', lmx.get('color-background-blue')) - -- lua table and adapt - lmx.set('formaction', "mtx-server-ctx-fonttest.lua") + local buttons = { 'process', 'select', 'save', 'load', 'edit', 'reset', 'features', 'source', 'log', 'info', 'extras'} + local menu = { } - local menu = { } - for k, v in ipairs { 'process', 'select', 'save', 'load', 'edit', 'reset', 'features', 'source', 'log', 'info', 'extras'} do - menu[#menu+1] = format("<button name='action' value='%s' type='submit'>%s</button>",v,v) + for i=1,#buttons do + local button = buttons[i] + menu[#menu+1] = format("<button name='action' value='%s' type='submit'>%s</button>",button,button) end - lmx.set('menu', concat(menu," ")) - logs.simple("action: %s",action or "no action") + variables.menu = concat(menu," ") + variables.status = format(status_template,currentfont or "") + variables.maintext = "" + variables.javascriptdata = "" + variables.javascripts = "" + variables.javascriptinit = "" - lmx.set("status",format(status_template,currentfont or "")) + logs.simple("action: %s",action or "no action") local result if action == "select" then - lmx.set('maintext',select_font()) + variables.maintext = select_font() elseif action == "info" then - lmx.set('maintext',info_about()) + variables.maintext = info_about() elseif action == "extras" then - lmx.set('maintext',do_extras()) + variables.maintext = do_extras() elseif currentfont and currentfont ~= "" then if action == "save" then - lmx.set('maintext',save_font(currentfont,detail)) + variables.maintext = save_font(currentfont,detail) elseif action == "load" then - lmx.set('maintext',load_font(currentfont,detail)) + variables.maintext = load_font(currentfont,detail) elseif action == "source" then - lmx.set('maintext',show_source(currentfont,detail)) + variables.maintext = show_source(currentfont,detail) elseif action == "log" then - lmx.set('maintext',show_log(currentfont,detail)) + variables.maintext = show_log(currentfont,detail) elseif action == "features" then - lmx.set('maintext',show_font(currentfont,detail)) + variables.maintext = show_font(currentfont,detail) else local e, s if action == "process" then @@ -659,16 +711,16 @@ function doit(configuration,filename,hashed) else e, s = process_font(currentfont,detail) end - lmx.set('maintext',e) - lmx.set('javascriptdata',s) - lmx.set('javascripts',javascripts) - lmx.set('javascriptinit', "check_form()") + variables.maintext = e + variables.javascriptdata = s + variables.javascripts = javascripts + variables.javascriptinit = "check_form()" end else - lmx.set('maintext',select_font()) + variables.maintext = select_font() end - result = { content = lmx.convert('context-fonttest.lmx') } + result = { content = lmx.convert('context-fonttest.lmx',false,variables) } logs.simple("time spent on page: %0.03f seconds",os.clock()-start) diff --git a/Master/texmf-dist/scripts/context/lua/mtx-server-ctx-help.lua b/Master/texmf-dist/scripts/context/lua/mtx-server-ctx-help.lua index c53d9f6e0f2..2f072f97747 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-server-ctx-help.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-server-ctx-help.lua @@ -6,8 +6,10 @@ if not modules then modules = { } end modules ['mtx-server-ctx-help'] = { license = "see context related readme files" } ---~ dofile(resolvers.find_file("l-xml.lua","tex")) -dofile(resolvers.find_file("l-aux.lua","tex")) +-- todo in lua interface: noargument, oneargument, twoarguments, threearguments + +--~ dofile(resolvers.find_file("l-aux.lua","tex")) +--~ dofile(resolvers.find_file("l-url.lua","tex")) dofile(resolvers.find_file("trac-lmx.lua","tex")) -- problem ... serialize parent stack @@ -277,18 +279,23 @@ document.setups.translations = document.setups.translations or { } document.setups.formats = { - interface = [[<a href='mtx-server-ctx-help.lua?interface=%s'>%s</a>]], - href = [[<a href='mtx-server-ctx-help.lua?command=%s'>%s</a>]], - source = [[<a href='mtx-server-ctx-help.lua?source=%s'>%s</a>]], - optional_single = "[optional string %s]", - optional_list = "[optional list %s]", - mandate_single = "[mandate string %s]", - mandate_list = "[mandate list %s]", - parameter = [[<tr><td width='15%%'>%s</td><td width='15%%'>%s</td><td width='70%%'>%s</td></tr>]], - parameters = [[<table width='100%%'>%s</table>]], - listing = [[<pre><t>%s</t></listing>]], - special = "<i>%s</i>", - default = "<u>%s</u>", + open_command = { [[\%s]], [[context.%s (]] }, + close_command = { [[]], [[ )]] }, + connector = { [[]], [[, ]] }, + href_in_list = { [[<a href='mtx-server-ctx-help.lua?command=%s&mode=%s'>%s</a>]], [[<a href='mtx-server-ctx-help.lua?command=%s&mode=%s'>%s</a>]] }, + href_as_command = { [[<a href='mtx-server-ctx-help.lua?command=%s&mode=%s'>\%s</a>]], [[<a href='mtx-server-ctx-help.lua?command=%s&mode=%s'>context.%s</a>]] }, + interface = [[<a href='mtx-server-ctx-help.lua?interface=%s&mode=%s'>%s</a>]], + source = [[<a href='mtx-server-ctx-help.lua?source=%s&mode=%s'>%s</a>]], + modes = { [[<a href='mtx-server-ctx-help.lua?mode=2'>lua mode</a>]], [[<a href='mtx-server-ctx-help.lua?mode=1'>tex mode</a>]] }, + optional_single = { "[optional string %s]", "{optional string %s}" }, + optional_list = { "[optional list %s]", "{optional table %s}" } , + mandate_single = { "[mandate string %s]", "{mandate string %s}" }, + mandate_list = { "[mandate list %s]", "{mandate list %s}" }, + parameter = [[<tr><td width='15%%'>%s</td><td width='15%%'>%s</td><td width='70%%'>%s</td></tr>]], + parameters = [[<table width='100%%'>%s</table>]], + listing = [[<pre><t>%s</t></listing>]], + special = [[<i>%s</i>]], + default = [[<u>%s</u>]], } local function translate(tag,int,noformat) @@ -298,7 +305,7 @@ local function translate(tag,int,noformat) if noformat then return ti[tag] or te[tag] or tag else - return document.setups.formats.special:format(ti[tag] or te[tag] or tag) + return format(document.setups.formats.special,ti[tag] or te[tag] or tag) end end @@ -307,7 +314,7 @@ local function translated(e,int) local s = attributes.type or "?" local tag = s:match("^cd:(.*)$") if attributes.default == "yes" then - return document.setups.formats.default:format(tag) + return format(document.setups.formats.default,tag or "?") elseif tag then return translate(tag,int) else @@ -318,7 +325,8 @@ end document.setups.loaded = document.setups.loaded or { } document.setups.current = { } -document.setups.showsources = false +document.setups.showsources = true +document.setups.mode = 1 function document.setups.load(filename) filename = resolvers.find_file(filename) or "" @@ -358,16 +366,15 @@ end function document.setups.csname(ek,int) local cs = "" - local at = ek.at + local at = ek.at or { } if at.type == 'environment' then cs = translate("start",int,true) .. cs end - for r, d, k in xml.elements(ek,'cd:sequence/(cd:string|variable)') do - local dk = d[k] - if dk.tg == "string" then - cs = cs .. dk.at.value + for e in xml.collected(ek,'cd:sequence/(cd:string|variable)') do + if e.tg == "string" then + cs = cs .. e.at.value else - cs = cs .. dk.at.value -- to be translated + cs = cs .. e.at.value -- to be translated end end return cs @@ -380,9 +387,8 @@ function document.setups.names() names = { } local name = document.setups.name local csname = document.setups.csname - for r, d, k in xml.elements(current.root,'cd:command') do - local dk = d[k] - names[#names+1] = { dk.at.name, csname(dk,int) } + for e in xml.collected(current.root,'cd:command') do + names[#names+1] = { e.at.name, csname(e,int) } end table.sort(names, function(a,b) return a[2]:lower() < b[2]:lower() end) current.names = names @@ -403,8 +409,9 @@ end function document.setups.showused() local current = document.setups.current if current.root and next(current.used) then - for k,v in ipairs(table.sortedkeys(current.used)) do - xml.sprint(current.used[v]) + local sorted = table.sortedkeys(current.used) + for i=1,#sorted do + xml.sprint(current.used[sorted[i]]) end end end @@ -412,12 +419,12 @@ function document.setups.showall() local current = document.setups.current if current.root then local list = { } - xml.each_element(current.root,"cd:command", function(r,d,t) - local ek = d[t] - list[document.setups.name(ek)] = ek - end ) - for k,v in ipairs(table.sortedkeys(list)) do - xml.sprint(list[v]) + for e in xml.collected(current.root,"cd:command") do + list[document.setups.name(e)] = e + end + local sorted = table.sortedkeys(list) + for i=1,#sorted do + xml.sprint(list[sorted[i]]) end end end @@ -431,46 +438,56 @@ function document.setups.resolve(name) end end -function document.setups.collect(name,int) +function document.setups.collect(name,int,lastmode) local current = document.setups.current local formats = document.setups.formats - local command = xml.filter(current.root,format("cd:command[@name='%s']",name)) + local command = xml.filter(current.root,format("cd:command[@name='%s']/first()",name)) if command then - local attributes = command.at + local attributes = command.at or { } local data = { command = command, category = attributes.category or "", } if document.setups.showsources then - data.source = (attributes.file and formats.source:format(attributes.file,attributes.file)) or "" + data.source = (attributes.file and formats.source:format(attributes.file,lastmode,attributes.file)) or "" else data.source = attributes.file or "" end - local sequence, n = { "\\" .. document.setups.csname(command,int) }, 0 - local arguments = { } + local n, sequence, tags = 0, { }, { } + sequence[#sequence+1] = formats.open_command[lastmode]:format(document.setups.csname(command,int)) + local arguments, tag = { }, "" for r, d, k in xml.elements(command,"(cd:keywords|cd:assignments)") do n = n + 1 local attributes = d[k].at + if #sequence > 1 then + local c = formats.connector[lastmode] + if c ~= "" then + sequence[#sequence+1] = c + end + end if attributes.optional == 'yes' then if attributes.list == 'yes' then - sequence[#sequence+1] = formats.optional_list:format(n) + tag = formats.optional_list[lastmode]:format(n) else - sequence[#sequence+1] = formats.optional_single:format(n) + tag = formats.optional_single[lastmode]:format(n) end else if attributes.list == 'yes' then - sequence[#sequence+1] = formats.mandate_list:format(n) + tag = formats.mandate_list[lastmode]:format(n) else - sequence[#sequence+1] = formats.mandate_single:format(n) + tag = formats.mandate_single[lastmode]:format(n) end end + sequence[#sequence+1] = tag + tags[#tags+1] = tag end + sequence[#sequence+1] = formats.close_command[lastmode] data.sequence = concat(sequence, " ") local parameters, n = { }, 0 for r, d, k in xml.elements(command,"(cd:keywords|cd:assignments)") do n = n + 1 if d[k].tg == "keywords" then - local left = sequence[n+1] + local left = tags[n] local right = { } for r, d, k in xml.elements(d[k],"(cd:constant|cd:resolve)") do local tag = d[k].tg @@ -488,13 +505,13 @@ function document.setups.collect(name,int) end parameters[#parameters+1] = formats.parameter:format(left,"",concat(right, ", ")) else - local what = sequence[n+1] + local what = tags[n] for r, d, k in xml.elements(d[k],"(cd:parameter|cd:inherit)") do local tag = d[k].tg local left, right = d[k].at.name or "?", { } if tag == "inherit" then local name = d[k].at.name or "?" - local goto = document.setups.formats.href:format(name,"\\"..name) + local goto = document.setups.formats.href_as_command[lastmode]:format(name,lastmode,name) if #parameters > 0 and not parameters[#parameters]:find("<br/>") then parameters[#parameters+1] = formats.parameter:format("<br/>","","") end @@ -521,7 +538,8 @@ function document.setups.collect(name,int) end parameters[#parameters+1] = formats.parameter:format("<br/>","","") end - data.parameters = parameters + data.parameters = parameters or { } + data.mode = formats.modes[lastmode or 1] return data else return nil @@ -532,25 +550,6 @@ end tex = tex or { } -lmx.variables['color-background-green'] = '#4F6F6F' -lmx.variables['color-background-blue'] = '#6F6F8F' -lmx.variables['color-background-yellow'] = '#8F8F6F' -lmx.variables['color-background-purple'] = '#8F6F8F' - -lmx.variables['color-background-body'] = '#808080' -lmx.variables['color-background-main'] = '#3F3F3F' -lmx.variables['color-background-main-left'] = '#3F3F3F' -lmx.variables['color-background-main-right'] = '#5F5F5F' -lmx.variables['color-background-one'] = lmx.variables['color-background-green'] -lmx.variables['color-background-two'] = lmx.variables['color-background-blue'] - -lmx.variables['title-default'] = 'ConTeXt Help Information' -lmx.variables['title'] = lmx.variables['title-default'] - -function lmx.loadedfile(filename) - return io.loaddata(resolvers.find_file(filename)) -- return resolvers.texdatablob(filename) -end - -- -- -- local interfaces = { @@ -564,7 +563,19 @@ local interfaces = { romanian = 'ro', } -local lastinterface, lastcommand, lastsource = "en", "", "" +local lastinterface, lastcommand, lastsource, lastmode = "en", "", "", 1 + +local variables = { + ['color-background-main-left'] = '#3F3F3F', + ['color-background-main-right'] = '#5F5F5F', + ['color-background-one'] = lmx.get('color-background-green'), + ['color-background-two'] = lmx.get('color-background-blue'), + ['title'] = 'ConTeXt Help Information', +} + +--~ function lmx.loadedfile(filename) +--~ return io.loaddata(resolvers.find_file(filename)) -- return resolvers.texdatablob(filename) +--~ end local function doit(configuration,filename,hashed) @@ -572,9 +583,12 @@ local function doit(configuration,filename,hashed) local start = os.clock() - local detail = aux.settings_to_hash(hashed.query or "") + local detail = url.query(hashed.query or "") - lastinterface, lastcommand, lastsource = detail.interface or lastinterface, detail.command or lastcommand, detail.source or lastsource + lastinterface = detail.interface or lastinterface + lastcommand = detail.command or lastcommand + lastsource = detail.source or lastsource + lastmode = tonumber(detail.mode or lastmode) or 1 if lastinterface then logs.simple("checking interface: %s",lastinterface) @@ -587,58 +601,61 @@ local function doit(configuration,filename,hashed) local result = { content = "error" } local names, refs, ints = document.setups.names(lastinterface), { }, { } - for k,v in ipairs(names) do - refs[k] = document.setups.formats.href:format(v[1],v[2]) + for k=1,#names do + local v = names[k] + refs[k] = formats.href_in_list[lastmode]:format(v[1],lastmode,v[2]) end - for k,v in ipairs(table.sortedkeys(interfaces)) do - ints[k] = document.setups.formats.interface:format(interfaces[v],v) + if lastmode ~= 2 then + local sorted = table.sortedkeys(interfaces) + for k=1,#sorted do + local v = sorted[k] + ints[k] = formats.interface:format(interfaces[v],lastmode,v) + end end - lmx.restore() - lmx.set('title', 'ConTeXt Help Information') - lmx.set('color-background-one', lmx.get('color-background-green')) - lmx.set('color-background-two', lmx.get('color-background-blue')) - local n = concat(refs,"<br/>") local i = concat(ints,"<br/><br/>") if div then - lmx.set('names',div:format(n)) - lmx.set('interfaces',div:format(i)) + variables.names = div:format(n) + variables.interfaces = div:format(i) else - lmx.set('names', n) - lmx.set('interfaces', i) + variables.names = n + variables.interfaces = i end -- first we need to add information about mkii/mkiv + variables.maintitle = "no definition" + variables.maintext = "" + variables.extra = "" + if document.setups.showsources and lastsource and lastsource ~= "" then -- todo: mkii, mkiv, tex (can be different) local data = io.loaddata(resolvers.find_file(lastsource)) - lmx.set('maintitle', lastsource) - lmx.set('maintext', formats.listing:format(data)) + variables.maintitle = lastsource + variables.maintext = formats.listing:format(data) lastsource = "" elseif lastcommand and lastcommand ~= "" then - local data = document.setups.collect(lastcommand,lastinterface) + local data = document.setups.collect(lastcommand,lastinterface,lastmode) if data then - lmx.set('maintitle', data.sequence) - local extra = { } - for k, v in ipairs { "environment", "category", "source" } do + local what, extra = { "environment", "category", "source", "mode" }, { } + for k=1,#what do + local v = what[k] if data[v] and data[v] ~= "" then lmx.set(v, data[v]) extra[#extra+1] = v .. ": " .. data[v] end end - lmx.set('extra', concat(extra,", ")) - lmx.set('maintext', formats.parameters:format(concat(data.parameters))) + variables.maintitle = data.sequence + variables.maintext = formats.parameters:format(concat(data.parameters)) + variables.extra = concat(extra," ") else - lmx.set('maintext', "select command") + variables.maintext = "select command" end - else - lmx.set('maintext', "no definition") end - local content = lmx.convert('context-help.lmx') + local content = lmx.convert('context-help.lmx',false,variables) logs.simple("time spent on page: %0.03f seconds",os.clock()-start) diff --git a/Master/texmf-dist/scripts/context/lua/mtx-server-ctx-startup.lua b/Master/texmf-dist/scripts/context/lua/mtx-server-ctx-startup.lua index fcb757b3e60..59536c36cdb 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-server-ctx-startup.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-server-ctx-startup.lua @@ -10,25 +10,6 @@ dofile(resolvers.find_file("trac-lmx.lua","tex")) function doit(configuration,filename,hashed) - lmx.restore() - - lmx.variables['color-background-green'] = '#4F6F6F' - lmx.variables['color-background-blue'] = '#6F6F8F' - lmx.variables['color-background-yellow'] = '#8F8F6F' - lmx.variables['color-background-purple'] = '#8F6F8F' - - lmx.variables['color-background-body'] = '#808080' - lmx.variables['color-background-main'] = '#3F3F3F' - lmx.variables['color-background-one'] = lmx.variables['color-background-green'] - lmx.variables['color-background-two'] = lmx.variables['color-background-blue'] - - lmx.variables['title'] = "Overview Of Goodies" - - lmx.set('title', lmx.get('title')) - lmx.set('color-background-one', lmx.get('color-background-green')) - lmx.set('color-background-two', lmx.get('color-background-blue')) - - local list = { } local root = file.dirname(resolvers.find_file("mtx-server.lua") or ".") if root == "" then root = "." end @@ -42,11 +23,16 @@ function doit(configuration,filename,hashed) end end - lmx.set('maintext',table.concat(list,"\n")) - - result = { content = lmx.convert('context-base.lmx') } + local variables = { + ['color-background-one'] = lmx.get('color-background-green'), + ['color-background-two'] = lmx.get('color-background-blue'), + ['title'] = "Overview Of Goodies", + ['color-background-one'] = lmx.get('color-background-green'), + ['color-background-two'] = lmx.get('color-background-blue'), + ['maintext'] = table.concat(list,"\n"), + } - return result + return { content = lmx.convert('context-base.lmx',false,variables) } end diff --git a/Master/texmf-dist/scripts/context/lua/mtx-server.lua b/Master/texmf-dist/scripts/context/lua/mtx-server.lua index 615506ac0ce..dc0befcaa2f 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-server.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-server.lua @@ -252,7 +252,8 @@ function scripts.webserver.run(configuration) end -- locate root and index file in tex tree if not lfs.isdir(configuration.root) then - for _, name in ipairs(indices) do + for i=1,#indices do + local name = indices[i] local root = resolvers.resolve("path:" .. name) or "" if root ~= "" then configuration.root = root @@ -263,7 +264,8 @@ function scripts.webserver.run(configuration) end configuration.root = dir.expand_name(configuration.root) if not configuration.index then - for _, name in ipairs(indices) do + for i=1,#indices do + local name = indices[i] if lfs.isfile(file.join(configuration.root,name)) then configuration.index = name -- we will prepend the rootpath later break @@ -281,8 +283,11 @@ function scripts.webserver.run(configuration) logs.simple("scripts subpath: %s",configuration.scripts) logs.simple("context services: http://localhost:%s/mtx-server-ctx-startup.lua",configuration.port) local server = assert(socket.bind("*", configuration.port)) +--~ local reading = { server } while true do -- no multiple clients local start = os.clock() +--~ local input = socket.select(reading) +--~ local client = input:accept() local client = server:accept() client:settimeout(configuration.timeout or 60) local request, e = client:receive() @@ -323,7 +328,7 @@ function scripts.webserver.run(configuration) end end -logs.extendbanner("Simple Webserver 0.10") +logs.extendbanner("Simple Webserver For Helpers 0.10") messages.help = [[ --start start server diff --git a/Master/texmf-dist/scripts/context/lua/mtx-texworks.lua b/Master/texmf-dist/scripts/context/lua/mtx-texworks.lua index f525d5336e2..73ab846cd7f 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-texworks.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-texworks.lua @@ -20,29 +20,31 @@ local texworkspaths = { } local texworkssignal = "texworks-context.rme" -local texworkininame = "TeXworks.ini" +local texworkininame = "texworks.ini" function scripts.texworks.start(indeed) - local is_mswin = os.platform == "windows" - local workname = (is_mswin and "texworks.exe") or "TeXworks" + local workname = (os.type == "windows" and "texworks.exe") or "texworks" local fullname = nil local binpaths = file.split_path(os.getenv("PATH")) or file.split_path(os.getenv("path")) - local datapath = resolvers.find_file(texworkssignal,"other text files") or "" + local usedsignal = texworkssignal + local datapath = resolvers.find_file(usedsignal,"other text files") or "" if datapath ~= "" then datapath = file.dirname(datapath) -- data if datapath == "" then - datapath = resolvers.ownpath + datapath = resolvers.clean_path(lfs.currentdir()) end else - datapath = resolvers.find_file(texworkininame,"other text files") or "" + usedsignal = texworkininame + datapath = resolvers.find_file(usedsignal,"other text files") or "" if datapath == "" then - datapath = resolvers.find_file(string.lower(texworkininame),"other text files") or "" + usedsignal = string.lower(usedsignal) + datapath = resolvers.find_file(usedsignal,"other text files") or "" end if datapath ~= "" and lfs.isfile(datapath) then datapath = file.dirname(datapath) -- TUG datapath = file.dirname(datapath) -- data if datapath == "" then - datapath = resolvers.ownpath + datapath = resolvers.clean_path(lfs.currentdir()) end end end @@ -56,7 +58,7 @@ function scripts.texworks.start(indeed) end for i=1,#binpaths do local p = file.join(binpaths[i],workname) - if lfs.isfile(p) then + if lfs.isfile(p) and lfs.attributes(p,"size") > 10000 then -- avoind stub fullname = p break end @@ -65,22 +67,23 @@ function scripts.texworks.start(indeed) logs.simple("unable to locate %s",workname) return false end - for _, subpath in ipairs(texworkspaths) do - dir.makedirs(file.join(datapath,subpath)) + for i=1,#texworkspaths do + dir.makedirs(file.join(datapath,texworkspaths[i])) end os.setenv("TW_INIPATH",datapath) os.setenv("TW_LIBPATH",datapath) if not indeed or environment.argument("verbose") then - logs.simple("data path: %s", datapath) - logs.simple("full name: %s", fullname) + logs.simple("used signal: %s", usedsignal) + logs.simple("data path : %s", datapath) + logs.simple("full name : %s", fullname) + logs.simple("set paths : TW_INIPATH TW_LIBPATH") end if indeed then os.launch(fullname) end end - -logs.extendbanner("TeXworks startup script 1.0",true) +logs.extendbanner("TeXworks Startup Script 1.00",true) messages.help = [[ --start [--verbose] start texworks diff --git a/Master/texmf-dist/scripts/context/lua/mtx-timing.lua b/Master/texmf-dist/scripts/context/lua/mtx-timing.lua index 1dcb9aa0e0d..40e33cdaef0 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-timing.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-timing.lua @@ -55,15 +55,17 @@ local html_menu = [[ local directrun = true -function goodies.progress.make_svg(filename,other) +local what = { "parameters", "nodes" } + +function plugins.progress.make_svg(filename,other) local metadata, menudata, c = { }, { }, 0 metadata[#metadata+1] = 'outputformat := "svg" ;' - for _, kind in pairs { "parameters", "nodes" } do - local mdk = { } + for i=1,#what do + local kind, mdk = what[i], { } menudata[kind] = mdk - for n, name in pairs(goodies.progress[kind](filename)) do - local first = goodies.progress.path(filename,name) - local second = goodies.progress.path(filename,other) + for n, name in next, plugins.progress[kind](filename) do + local first = plugins.progress.path(filename,name) + local second = plugins.progress.path(filename,other) c = c + 1 metadata[#metadata+1] = format(meta,c,first,second) mdk[#mdk+1] = { name, c } @@ -88,18 +90,19 @@ function goodies.progress.make_svg(filename,other) end end -function goodies.progress.makehtml(filename,other,menudata,metadata) +function plugins.progress.makehtml(filename,other,menudata,metadata) local graphics = { } local result = { graphics = graphics } - for _, kind in pairs { "parameters", "nodes" } do + for i=1,#what do + local kind, menu = what[i], { } local md = menudata[kind] - local menu = { } result[kind] = menu - for k, v in ipairs(md) do + for k=1,#md do + local v = md[k] local name, number = v[1], v[2] - local min = goodies.progress.bot(filename,name) - local max = goodies.progress.top(filename,name) - local pages = goodies.progress.pages(filename) + local min = plugins.progress.bot(filename,name) + local max = plugins.progress.top(filename,name) + local pages = plugins.progress.pages(filename) local average = math.round(max/pages) if directrun then local data = metadata[number] @@ -119,55 +122,44 @@ function goodies.progress.makehtml(filename,other,menudata,metadata) return result end -function goodies.progress.valid_file(name) +function plugins.progress.valid_file(name) return name and name ~= "" and lfs.isfile(name .. "-luatex-progress.lut") end -function goodies.progress.make_lmx_page(name,launch,remove) +function plugins.progress.make_lmx_page(name,launch,remove) + local filename = name .. "-luatex-progress" local other = "elapsed_time" local template = 'context-timing.lmx' - lmx.variables['color-background-green'] = '#4F6F6F' - lmx.variables['color-background-blue'] = '#6F6F8F' - lmx.variables['color-background-yellow'] = '#8F8F6F' - lmx.variables['color-background-purple'] = '#8F6F8F' - - lmx.variables['color-background-body'] = '#808080' - lmx.variables['color-background-main'] = '#3F3F3F' - lmx.variables['color-background-one'] = lmx.variables['color-background-green'] - lmx.variables['color-background-two'] = lmx.variables['color-background-blue'] + plugins.progress.convert(filename) - lmx.variables['title-default'] = 'ConTeXt Timing Information' - lmx.variables['title'] = lmx.variables['title-default'] + local menudata, metadata = plugins.progress.make_svg(filename,other) + local htmldata = plugins.progress.makehtml(filename,other,menudata,metadata) lmx.htmfile = function(name) return name .. "-timing.xhtml" end lmx.lmxfile = function(name) return resolvers.find_file(name,'tex') end - lmx.set('title', format('ConTeXt Timing Information: %s',file.basename(name))) - lmx.set('color-background-one', lmx.get('color-background-green')) - lmx.set('color-background-two', lmx.get('color-background-blue')) - - goodies.progress.convert(filename) - - local menudata, metadata = goodies.progress.make_svg(filename,other) - local htmldata = goodies.progress.makehtml(filename,other,menudata,metadata) - - lmx.set('parametersmenu', concat(htmldata.parameters, " ")) - lmx.set('nodesmenu', concat(htmldata.nodes, " ")) - lmx.set('graphics', concat(htmldata.graphics, "\n\n")) + local variables = { + ['title-default'] = 'ConTeXt Timing Information', + ['title'] = format('ConTeXt Timing Information: %s',file.basename(name)), + ['parametersmenu'] = concat(htmldata.parameters, " "), + ['nodesmenu'] = concat(htmldata.nodes, " "), + ['graphics'] = concat(htmldata.graphics, "\n\n"), + ['color-background-one'] = lmx.get('color-background-green'), + ['color-background-two'] = lmx.get('color-background-blue'), + } if launch then - local htmfile = lmx.show(template) + local htmfile = lmx.show(template,variables) if remove then os.sleep(1) -- give time to launch os.remove(htmfile) end else - lmx.make(template) + lmx.make(template,variables) end - lmx.restore() end scripts = scripts or { } @@ -176,17 +168,17 @@ scripts.timings = scripts.timings or { } function scripts.timings.xhtml(filename) if filename == "" then logs.simple("provide filename") - elseif not goodies.progress.valid_file(filename) then + elseif not plugins.progress.valid_file(filename) then logs.simple("first run context again with the --timing option") else local basename = file.removesuffix(filename) local launch = environment.argument("launch") local remove = environment.argument("remove") - goodies.progress.make_lmx_page(basename,launch,remove) + plugins.progress.make_lmx_page(basename,launch,remove) end end -logs.extendbanner("ConTeXt Timing Tools 0.1",true) +logs.extendbanner("ConTeXt Timing Tools 0.10",true) messages.help = [[ --xhtml make xhtml file diff --git a/Master/texmf-dist/scripts/context/lua/mtx-tools.lua b/Master/texmf-dist/scripts/context/lua/mtx-tools.lua index 87fd51dc60d..bf4add16840 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-tools.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-tools.lua @@ -6,9 +6,7 @@ if not modules then modules = { } end modules ['mtx-tools'] = { license = "see context related readme files" } --- data tables by Thomas A. Schmitz - -local find, gsub = string.find, string.gsub +local find, format, sub, rep, gsub, lower = string.find, string.format, string.sub, string.rep, string.gsub, string.lower scripts = scripts or { } scripts.tools = scripts.tools or { } @@ -17,7 +15,9 @@ local bomb_1, bomb_2 = "^\254\255", "^\239\187\191" function scripts.tools.disarmutfbomb() local force, done = environment.argument("force"), false - for _, name in ipairs(environment.files) do + local files = environment.files + for i=1,#files do + local name = files[i] if lfs.isfile(name) then local data = io.loaddata(name) if not data then @@ -44,14 +44,133 @@ function scripts.tools.disarmutfbomb() end end -logs.extendbanner("All Kind Of Tools 1.0",true) +function scripts.tools.downcase() + local pattern = environment.argument('pattern') or "*" + local recurse = environment.argument('recurse') + local force = environment.argument('force') + local n = 0 + if recurse and not find(pattern,"^%*%*%/") then + pattern = "**/*" .. pattern + end + dir.glob(pattern,function(name) + local basename = file.basename(name) + if lower(basename) ~= basename then + n = n + 1 + if force then + os.rename(name,lower(name)) + end + end + end) + if n > 0 then + if force then + logs.simple("%s files renamed",n) + else + logs.simple("use --force to do a real rename (%s files involved)",n) + end + else + logs.simple("nothing to do") + end +end + + +function scripts.tools.dirtoxml() + + local join, removesuffix, extname, date = file.join, file.removesuffix, file.extname, os.date + + local xmlns = "http://www.pragma-ade.com/rlg/xmldir.rng" + local timestamp = "%Y-%m-%d %H:%M" + + local pattern = environment.argument('pattern') or ".*" + local url = environment.argument('url') or "no-url" + local root = environment.argument('root') or "." + local outputfile = environment.argument('output') + + local recurse = environment.argument('recurse') + local stripname = environment.argument('stripname') + local longname = environment.argument('longname') + + local function flush(list,result,n,path) + n, result = n or 1, result or { } + local d = rep(" ",n) + for name, attr in table.sortedhash(list) do + local mode = attr.mode + if mode == "file" then + result[#result+1] = format("%s<file name='%s'>",d,(longname and path and join(path,name)) or name) + result[#result+1] = format("%s <base>%s</base>",d,removesuffix(name)) + result[#result+1] = format("%s <type>%s</type>",d,extname(name)) + result[#result+1] = format("%s <size>%s</size>",d,attr.size) + result[#result+1] = format("%s <permissions>%s</permissions>",d,sub(attr.permissions,7,9)) + result[#result+1] = format("%s <date>%s</date>",d,date(timestamp,attr.modification)) + result[#result+1] = format("%s</file>",d) + elseif mode == "directory" then + result[#result+1] = format("%s<directory name='%s'>",d,name) + flush(attr.list,result,n+1,(path and join(path,name)) or name) + result[#result+1] = format("%s</directory>",d) + end + end + end + + if not pattern or pattern == "" then + logs.report('provide --pattern=') + return + end + + if stripname then + pattern = file.dirname(pattern) + end + + local luapattern = string.topattern(pattern,true) + + lfs.chdir(root) + + local list = dir.collect_pattern(root,luapattern,recurse) + + if list[outputfile] then + list[outputfile] = nil + end + + local result = { "<?xml version='1.0'?>" } + result[#result+1] = format("<files url=%q root=%q pattern=%q luapattern=%q xmlns='%s' timestamp='%s'>",url,root,pattern,luapattern,xmlns,date(timestamp)) + flush(list,result) + result[#result+1] = "</files>" + + result = table.concat(result,"\n") + + if not outputfile or outputfile == "" then + texio.write_nl(result) + else + io.savedata(outputfile,result) + end + +end + +logs.extendbanner("Some File Related Goodies 1.01",true) messages.help = [[ --disarmutfbomb remove utf bomb if present + --force remove indeed + +--dirtoxml glob directory into xml + --pattern glob pattern (default: *) + --url url attribute (no processing) + --root the root of the globbed path (default: .) + --output output filename (console by default) + --recurse recurse into subdirecories + --stripname take pathpart of given pattern + --longname set name attributes to full path name + +--downcase + --pattern glob pattern (default: *) + --recurse recurse into subdirecories + --force downcase indeed ]] if environment.argument("disarmutfbomb") then scripts.tools.disarmutfbomb() +elseif environment.argument("dirtoxml") then + scripts.tools.dirtoxml() +elseif environment.argument("downcase") then + scripts.tools.downcase() else logs.help(messages.help) end diff --git a/Master/texmf-dist/scripts/context/lua/mtx-update.lua b/Master/texmf-dist/scripts/context/lua/mtx-update.lua index bc6ca402639..b56083d3809 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-update.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-update.lua @@ -86,10 +86,12 @@ scripts.update.engines = { ["xetex"] = { { "base/xetex/", "texmf" }, { "fonts/new/", "texmf" }, + { "bin/luatex/<platform>/", "texmf-<platform>" }, -- tools { "bin/xetex/<platform>/", "texmf-<platform>" }, }, ["pdftex"] = { { "fonts/old/", "texmf" }, + { "bin/luatex/<platform>/", "texmf-<platform>" }, -- tools { "bin/pdftex/<platform>/", "texmf-<platform>" }, }, ["all"] = { @@ -112,30 +114,34 @@ scripts.update.goodies = { } scripts.update.platforms = { - ["mswin"] = "mswin", - ["windows"] = "mswin", - ["win32"] = "mswin", - ["win"] = "mswin", - ["linux"] = "linux", - ["freebsd"] = "freebsd", - ["freebsd-amd64"] = "freebsd-amd64", - ["linux-32"] = "linux", - ["linux-64"] = "linux-64", - ["linux32"] = "linux", - ["linux64"] = "linux-64", - ["linux-ppc"] = "linux-ppc", - ["ppc"] = "linux-ppc", - ["osx"] = "osx-intel", - ["macosx"] = "osx-intel", - ["osx-intel"] = "osx-intel", - ["osx-ppc"] = "osx-ppc", - ["osx-powerpc"] = "osx-ppc", - ["osxintel"] = "osx-intel", - ["osxppc"] = "osx-ppc", - ["osxpowerpc"] = "osx-ppc", - ["solaris-intel"] = "solaris-intel", - ["solaris-sparc"] = "solaris-sparc", - ["solaris"] = "solaris-sparc", + ["mswin"] = "mswin", + ["windows"] = "mswin", + ["win32"] = "mswin", + ["win"] = "mswin", + ["linux"] = "linux", + ["freebsd"] = "freebsd", + ["freebsd-amd64"] = "freebsd-amd64", + ["kfreebsd"] = "kfreebsd-i386", + ["kfreebsd-i386"] = "kfreebsd-i386", + ["kfreebsd-amd64"] = "kfreebsd-amd64", + ["linux-32"] = "linux", + ["linux-64"] = "linux-64", + ["linux32"] = "linux", + ["linux64"] = "linux-64", + ["linux-ppc"] = "linux-ppc", + ["ppc"] = "linux-ppc", + ["osx"] = "osx-intel", + ["macosx"] = "osx-intel", + ["osx-intel"] = "osx-intel", + ["osx-ppc"] = "osx-ppc", + ["osx-powerpc"] = "osx-ppc", + ["osx-64"] = "osx-64", + ["osxintel"] = "osx-intel", + ["osxppc"] = "osx-ppc", + ["osxpowerpc"] = "osx-ppc", + ["solaris-intel"] = "solaris-intel", + ["solaris-sparc"] = "solaris-sparc", + ["solaris"] = "solaris-sparc", } -- the list is filled up later (when we know what modules to download) @@ -144,12 +150,14 @@ scripts.update.modules = { } function scripts.update.run(str) - logs.report("run", str) + -- important, otherwise formats fly to a weird place + -- (texlua sets luatex as the engine, we need to reset that or to fix texexec :) + os.setenv("engine",nil) if environment.argument("force") then - -- important, otherwise formats fly to a weird place - -- (texlua sets luatex as the engine, we need to reset that or to fix texexec :) - os.setenv("engine",nil) + logs.report("run", str) os.execute(str) + else + logs.report("dry run", str) end end @@ -165,16 +173,16 @@ function scripts.update.synchronize() logs.report("update","start") - local texroot = scripts.update.fullpath(states.get("paths.root")) - local engines = states.get('engines') or { } - local platforms = states.get('platforms') or { } - local repositories = states.get('repositories') -- minimals - local bin = states.get("rsync.program") -- rsync - local url = states.get("rsync.server") -- contextgarden.net - local version = states.get("context.version") -- current (or beta) - local extras = states.get("extras") -- extras (like modules) - local goodies = states.get("goodies") -- goodies (like editors) - local force = environment.argument("force") + local texroot = scripts.update.fullpath(states.get("paths.root")) + local engines = states.get('engines') or { } + local platforms = states.get('platforms') or { } + local repositories = states.get('repositories') -- minimals + local bin = states.get("rsync.program") -- rsync + local url = states.get("rsync.server") -- contextgarden.net + local version = states.get("context.version") -- current (or beta) + local extras = states.get("extras") -- extras (like modules) + local goodies = states.get("goodies") -- goodies (like editors) + local force = environment.argument("force") bin = string.gsub(bin,"\\","/") @@ -188,18 +196,19 @@ function scripts.update.synchronize() if force then dir.mkdirs(format("%s/%s", texroot, "texmf-cache")) dir.mkdirs(format("%s/%s", texroot, "texmf-local")) + dir.mkdirs(format("%s/%s", texroot, "texmf-project")) end if ok or not force then - local fetched, individual, osplatform = { }, { }, os.currentplatform() + local fetched, individual, osplatform = { }, { }, os.platform -- takes a collection as argument and returns a list of folders local function collection_to_list_of_folders(collection, platform) local archives = {} - for _, c in ipairs(collection) do - local archive = c[1] + for i=1,#collection do + local archive = collection[i][1] archive = archive:gsub("<platform>", platform) archive = archive:gsub("<version>", version) archives[#archives+1] = archive @@ -220,7 +229,7 @@ function scripts.update.synchronize() return prefix .. concat(list_of_folders, format(" %s", prefix)) end - -- example of usage: print(list_of_folders_to_rsync_string(collection_to_list_of_folders(scripts.update.base, os.currentplatform))) + -- example of usage: print(list_of_folders_to_rsync_string(collection_to_list_of_folders(scripts.update.base, os.platform))) -- rename function and add some more functionality: -- * recursive/non-recursive (default: non-recursive) @@ -255,7 +264,8 @@ function scripts.update.synchronize() local available_modules = get_list_of_files_from_rsync({"modules/"}) -- hash of requested modules -- local h = table.tohash(extras:split(",")) - for _, s in ipairs(available_modules) do + for i=1,#available_modules do + local s = available_modules[i] -- if extras == "all" or h[s] then if extras.all or extras[s] then scripts.update.modules[#scripts.update.modules+1] = { format("modules/%s/",s), "texmf-context" } @@ -269,7 +279,8 @@ function scripts.update.synchronize() if collection and platform then platform = scripts.update.platforms[platform] if platform then - for _, c in ipairs(collection) do + for i=1,#collection do + local c = collection[i] local archive = c[1]:gsub("<platform>", platform) local destination = format("%s/%s", texroot, c[2]:gsub("<platform>", platform)) destination = destination:gsub("\\","/") @@ -283,30 +294,32 @@ function scripts.update.synchronize() end end - for platform, _ in pairs(platforms) do + for platform, _ in next, platforms do add_collection(scripts.update.base,platform) end - for platform, _ in pairs(platforms) do + for platform, _ in next, platforms do add_collection(scripts.update.modules,platform) end - for engine, _ in pairs(engines) do - for platform, _ in pairs(platforms) do + for engine, _ in next, engines do + for platform, _ in next, platforms do add_collection(scripts.update.engines[engine],platform) end end if goodies and type(goodies) == "table" then - for goodie, _ in pairs(goodies) do - for platform, _ in pairs(platforms) do + for goodie, _ in next, goodies do + for platform, _ in next, platforms do add_collection(scripts.update.goodies[goodie],platform) end end end local combined = { } - for _, repository in ipairs(scripts.update.repositories) do + local update_repositories = scripts.update.repositories + for i=1,#update_repositories do + local repository = update_repositories[i] if repositories[repository] then - for _, v in pairs(individual) do + for _, v in next, individual do local archive, destination = v[1], v[2] local cd = combined[destination] if not cd then @@ -318,14 +331,14 @@ function scripts.update.synchronize() end end if logs.verbose then - for k, v in pairs(combined) do + for k, v in next, combined do logs.report("update", k) - for k,v in ipairs(v) do - logs.report("update", " <= " .. v) + for i=1,#v do + logs.report("update", " <= " .. v[i]) end end end - for destination, archive in pairs(combined) do + for destination, archive in next, combined do local archives, command = concat(archive," "), "" -- local normalflags, deleteflags = states.get("rsync.flags.normal"), states.get("rsync.flags.delete") -- if environment.argument("keep") or destination:find("%.$") then @@ -334,13 +347,17 @@ function scripts.update.synchronize() -- command = format("%s %s %s %s'%s' '%s'", bin, normalflags, deleteflags, url, archives, destination) -- end local normalflags, deleteflags = states.get("rsync.flags.normal"), "" + local dryrunflags = "" + if not environment.argument("force") then + dryrunflags = "--dry-run" + end if (destination:find("texmf$") or destination:find("texmf%-context$")) and (not environment.argument("keep")) then deleteflags = states.get("rsync.flags.delete") end - command = format("%s %s %s %s'%s' '%s'", bin, normalflags, deleteflags, url, archives, destination) - logs.report("mtx update", format("running command: %s",command)) + command = format("%s %s %s %s %s'%s' '%s'", bin, normalflags, deleteflags, dryrunflags, url, archives, destination) + --logs.report("mtx update", format("running command: %s",command)) if not fetched[command] then - scripts.update.run(command) + scripts.update.run(command,true) fetched[command] = command end end @@ -363,7 +380,7 @@ function scripts.update.synchronize() end end - for platform, _ in pairs(platforms) do + for platform, _ in next, platforms do update_script('luatools',platform) update_script('mtxrun',platform) end @@ -374,12 +391,20 @@ function scripts.update.synchronize() if not force then logs.report("update", "use --force to really update files") end + + resolvers.load_tree(texroot) -- else we operate in the wrong tree + + -- update filename database for pdftex/xetex + scripts.update.run("mktexlsr") + -- update filename database for luatex + scripts.update.run("luatools --generate") + logs.report("update","done") end function table.fromhash(t) local h = { } - for k, v in pairs(t) do -- no ipairs here + for k, v in next, t do -- not indexed if v then h[#h+1] = k end end return h @@ -390,34 +415,34 @@ function scripts.update.make() logs.report("make","start") - local force = environment.argument("force") - local texroot = scripts.update.fullpath(states.get("paths.root")) - local engines = states.get('engines') - local goodies = states.get('goodies') + local force = environment.argument("force") + local texroot = scripts.update.fullpath(states.get("paths.root")) + local engines = states.get('engines') + local goodies = states.get('goodies') local platforms = states.get('platforms') - local formats = states.get('formats') + local formats = states.get('formats') resolvers.load_tree(texroot) - -- update filename database for pdftex/xetex + scripts.update.run("mktexlsr") - -- update filename database for luatex scripts.update.run("luatools --generate") + local askedformats = formats local texformats = table.tohash(scripts.update.texformats) local mpformats = table.tohash(scripts.update.mpformats) - for k,v in pairs(texformats) do + for k,v in next, texformats do if not askedformats[k] then texformats[k] = nil end end - for k,v in pairs(mpformats) do + for k,v in next, mpformats do if not askedformats[k] then mpformats[k] = nil end end local formatlist = concat(table.fromhash(texformats), " ") if formatlist ~= "" then - for engine in pairs(engines) do + for engine in next, engines do if engine == "luatex" then scripts.update.run(format("context --make")) -- maybe also formatlist else @@ -438,7 +463,7 @@ function scripts.update.make() logs.report("make","done") end -logs.extendbanner("Download Tools 0.20",true) +logs.extendbanner("ConTeXt Minimals Updater 0.21",true) messages.help = [[ --platform=string platform (windows, linux, linux-64, osx-intel, osx-ppc, linux-ppc) @@ -476,7 +501,7 @@ if scripts.savestate then states.set("rsync.program", environment.argument("rsync"), "rsync", true) -- ok states.set("rsync.server", environment.argument("server"), "contextgarden.net::", true) -- ok states.set("rsync.module", environment.argument("module"), "minimals", true) -- ok - states.set("rsync.flags.normal", environment.argument("flags"), "-rpztlv --stats", true) -- ok + states.set("rsync.flags.normal", environment.argument("flags"), "-rpztlv", true) -- ok states.set("rsync.flags.delete", nil, "--delete", true) -- ok states.set("paths.root", environment.argument("texroot"), "tex", true) -- ok @@ -490,7 +515,7 @@ if scripts.savestate then local valid = scripts.update.engines for r in gmatch(environment.argument("engine") or "all","([^, ]+)") do if r == "all" then - for k, v in pairs(valid) do + for k, v in next, valid do if k ~= "all" then states.set("engines." .. k, true) end @@ -500,7 +525,7 @@ if scripts.savestate then end end local valid = scripts.update.platforms - for r in gmatch(environment.argument("platform") or os.currentplatform(),"([^, ]+)") do + for r in gmatch(environment.argument("platform") or os.platform,"([^, ]+)") do if valid[r] then states.set("platforms." .. r, true) end end diff --git a/Master/texmf-dist/scripts/context/lua/mtx-watch.lua b/Master/texmf-dist/scripts/context/lua/mtx-watch.lua index 2e4dcf6efec..10f01cf8698 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-watch.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-watch.lua @@ -9,160 +9,247 @@ if not modules then modules = { } end modules ['mtx-watch'] = { scripts = scripts or { } scripts.watch = scripts.watch or { } -do +local format, concat, difftime, time = string.format, table.concat, os.difftime, os.time +local next, type = next, type - function scripts.watch.save_exa_modes(joblog,ctmname) +-- the machine/instance matches the server app we use + +local machine = socket.dns.gethostname() or "unknown-machine" +local instance = string.match(machine,"(%d+)$") or "0" + +function scripts.watch.save_exa_modes(joblog,ctmname) + local values = joblog and joblog.values + if values then local t= { } - if joblog then - t[#t+1] = "<?xml version='1.0' standalone='yes'?>\n" - t[#t+1] = "<exa:variables xmlns:exa='htpp://www.pragma-ade.com/schemas/exa-variables.rng'>" - if joblog.values then - for k, v in pairs(joblog.values) do - t[#t+1] = string.format("\t<exa:variable label='%s'>%s</exa:variable>", k, tostring(v)) + t[#t+1] = "<?xml version='1.0' standalone='yes'?>\n" + t[#t+1] = "<exa:variables xmlns:exa='htpp://www.pragma-ade.com/schemas/exa-variables.rng'>" + for k, v in next, joblog.values do + t[#t+1] = format("\t<exa:variable label='%s'>%s</exa:variable>", k, tostring(v)) + end + t[#t+1] = "</exa:variables>" + io.savedata(ctmname,concat(t,"\n")) + else + os.remove(ctmname) + end +end + +local function toset(t) + if type(t) == "table" then + return concat(t,",") + else + return t + end +end + +local function noset(t) + if type(t) == "table" then + return t[1] + else + return t + end +end + +local lfsdir, lfsattributes = lfs.dir, lfs.attributes + +local function glob(files,path) + for name in lfsdir(path) do + if name:find("^%.") then + -- skip . and .. + else + name = path .. "/" .. name + local a = lfsattributes(name) + if not a then + -- weird + elseif a.mode == "directory" then + if name:find("graphics$") or name:find("figures$") or name:find("resources$") then + -- skip these too + else + glob(files,name) end - else - t[#t+1] = "<!-- no modes -->" + elseif name:find(".%luj$") then + files[name] = a.change or a.ctime or a.modification or a.mtime end - t[#t+1] = "</exa:variables>" end - os.remove(ctmname) - io.savedata(ctmname,table.concat(t,"\n")) end +end - function scripts.watch.watch() - local delay = environment.argument("delay") or 5 - local logpath = environment.argument("logpath") or "" - local pipe = environment.argument("pipe") or false - if #environment.files > 0 then - for _, path in ipairs(environment.files) do - logs.report("watch", "watching path ".. path) +local clock = os.gettimeofday or os.time -- we cannot trust os.clock on linux + +function scripts.watch.watch() + local delay = tonumber(environment.argument("delay") or 5) or 5 + if delay == 0 then + delay = .25 + end + local logpath = environment.argument("logpath") or "" + local pipe = environment.argument("pipe") or false + local watcher = "mtxwatch.run" + local paths = environment.files + if #paths > 0 then + if environment.argument("automachine") then + logpath = string.gsub(logpath,"/machine/","/"..machine.."/") + for i=1,#paths do + paths[i] = string.gsub(paths[i],"/machine/","/"..machine.."/") end - local function glob(files,path) - for name in lfs.dir(path) do - if name:find("^%.") then - -- skip . and .. - else - name = path .. "/" .. name - local a = lfs.attributes(name) - if not a then - -- weird - elseif a.mode == "directory" then - if name:find("graphics$") or name:find("figures$") or name:find("resources$") then - -- skip these too + end + for i=1,#paths do + logs.report("watch", "watching path ".. paths[i]) + end + local function process() + local done = false + for i=1,#paths do + local path = paths[i] + lfs.chdir(path) + local files = { } + glob(files,path) + table.sort(files) -- what gets sorted here, todo: by time + for name, time in next, files do + --~ local ok, joblog = xpcall(function() return dofile(name) end, function() end ) + local ok, joblog = pcall(dofile,name) + if ok and joblog then + if joblog.status == "processing" then + logs.report("watch",format("aborted job, %s added to queue",name)) + joblog.status = "queued" + io.savedata(name, table.serialize(joblog,true)) + elseif joblog.status == "queued" then + local command = joblog.command + if command then + local replacements = { + inputpath = toset((joblog.paths and joblog.paths.input ) or "."), + outputpath = noset((joblog.paths and joblog.paths.output) or "."), + filename = joblog.filename or "", + } + -- todo: revision path etc + command = command:gsub("%%(.-)%%", replacements) + if command ~= "" then + joblog.status = "processing" + joblog.runtime = clock() + io.savedata(name, table.serialize(joblog,true)) + logs.report("watch",format("running: %s", command)) + local newpath = file.dirname(name) + io.flush() + local result = "" + local ctmname = file.basename(replacements.filename) + if ctmname == "" then ctmname = name end -- use self as fallback + ctmname = file.replacesuffix(ctmname,"ctm") + if newpath ~= "" and newpath ~= "." then + local oldpath = lfs.currentdir() + lfs.chdir(newpath) + scripts.watch.save_exa_modes(joblog,ctmname) + if pipe then result = os.resultof(command) else result = os.spawn(command) end + lfs.chdir(oldpath) + else + scripts.watch.save_exa_modes(joblog,ctmname) + if pipe then result = os.resultof(command) else result = os.spawn(command) end + end + logs.report("watch",format("return value: %s", result)) + done = true + local path, base = replacements.outputpath, file.basename(replacements.filename) + joblog.runtime = clock() - joblog.runtime + if base ~= "" then + joblog.result = file.replacesuffix(file.join(path,base),"pdf") + joblog.size = lfs.attributes(joblog.result,"size") + end + joblog.status = "finished" + else + joblog.status = "invalid command" + end else - glob(files,name) + joblog.status = "no command" + end + -- pcall, when error sleep + again + -- todo: just one log file and append + io.savedata(name, table.serialize(joblog,true)) + if logpath and logpath ~= "" then + local name = file.join(logpath,os.uuid() .. ".lua") + io.savedata(name, table.serialize(joblog,true)) + logs.report("watch", "saving joblog in " .. name) end - elseif name:find(".%luj$") then - files[name] = a.change or a.ctime or a.modification or a.mtime end end end end - local function toset(t) - if type(t) == "table" then - return table.concat(t,",") - else - return t + end + local n, start = 0, time() +--~ local function wait() +--~ io.flush() +--~ if not done then +--~ n = n + 1 +--~ if n >= 10 then +--~ logs.report("watch", format("run time: %i seconds, memory usage: %0.3g MB", difftime(time(),start), (status.luastate_bytes/1024)/1000)) +--~ n = 0 +--~ end +--~ os.sleep(delay) +--~ end +--~ end + local wtime = 0 + local function wait() + io.flush() + if not done then + n = n + 1 + if n >= 10 then + logs.report("watch", format("run time: %i seconds, memory usage: %0.3g MB", difftime(time(),start), (status.luastate_bytes/1024)/1000)) + n = 0 end - end - local function noset(t) - if type(t) == "table" then - return t[1] - else - return t + local ttime = 0 + while ttime <= delay do + local wt = lfs.attributes(watcher,"mtime") + if wt and wt ~= wtime then + -- fast signal that there is a request + wtime = wt + break + end + ttime = ttime + 0.2 + os.sleep(0.2) end end - local function process() - local done = false - for _, path in ipairs(environment.files) do - lfs.chdir(path) - local files = { } - glob(files,path) - table.sort(files) -- what gets sorted here - for name, time in pairs(files) do - --~ local ok, joblog = xpcall(function() return dofile(name) end, function() end ) - local ok, joblog = pcall(dofile,name) - if ok and joblog then - if joblog.status == "processing" then - logs.report("watch",string.format("aborted job, %s added to queue",name)) - joblog.status = "queued" - io.savedata(name, table.serialize(joblog,true)) - elseif joblog.status == "queued" then - local command = joblog.command - if command then - local replacements = { - inputpath = toset((joblog.paths and joblog.paths.input ) or "."), - outputpath = noset((joblog.paths and joblog.paths.output) or "."), - filename = joblog.filename or "", - } - command = command:gsub("%%(.-)%%", replacements) - if command ~= "" then - joblog.status = "processing" - joblog.runtime = os.time() -- os.clock() - io.savedata(name, table.serialize(joblog,true)) - logs.report("watch",string.format("running: %s", command)) - local newpath = file.dirname(name) - io.flush() - local result = "" - local ctmname = file.basename(replacements.filename) - if ctmname == "" then ctmname = name end -- use self as fallback - ctmname = file.replacesuffix(ctmname,"ctm") - if newpath ~= "" and newpath ~= "." then - local oldpath = lfs.currentdir() - lfs.chdir(newpath) - scripts.watch.save_exa_modes(joblog,ctmname) - if pipe then result = os.resultof(command) else result = os.spawn(command) end - lfs.chdir(oldpath) - else - scripts.watch.save_exa_modes(joblog,ctmname) - if pipe then result = os.resultof(command) else result = os.spawn(command) end - end - logs.report("watch",string.format("return value: %s", result)) - done = true - local path, base = replacements.outputpath, file.basename(replacements.filename) - joblog.runtime = os.time() - joblog.runtime -- os.clock() - joblog.runtime - joblog.result = file.replacesuffix(file.join(path,base),"pdf") - joblog.size = lfs.attributes(joblog.result,"size") - joblog.status = "finished" - else - joblog.status = "invalid command" - end - else - joblog.status = "no command" - end - -- pcall, when error sleep + again - io.savedata(name, table.serialize(joblog,true)) - if logpath ~= "" then - local name = string.format("%s/%s%04i%09i.lua", logpath, os.time(), math.floor((os.clock()*100)%1000), math.random(99999999)) - io.savedata(name, table.serialize(joblog,true)) - logs.report("watch", "saving joblog ".. name) + end + local cleanupdelay, cleanup = environment.argument("cleanup"), false + if cleanupdelay then + local lasttime = time() + cleanup = function() + local currenttime = time() + local delta = difftime(currenttime,lasttime) + if delta > cleanupdelay then + lasttime = currenttime + for i=1,#paths do + local path = paths[i] + if string.find(path,"%.") then + -- safeguard, we want a fully qualified path + else + local files = dir.glob(file.join(path,"*")) + for i=1,#files do + local name = files[i] + local filetime = lfs.attributes(name,"modification") + local delta = difftime(currenttime,filetime) + if delta > cleanupdelay then + -- logs.report("watch",format("cleaning up '%s'",name)) + os.remove(name) end end end end end end - local n, start = 0, os.clock() - local function wait() - io.flush() - if not done then - n = n + 1 - if n >= 10 then - logs.report("watch", string.format("run time: %i seconds, memory usage: %0.3g MB", os.clock() - start, (status.luastate_bytes/1024)/1000)) - n = 0 - end - os.sleep(delay) - end + else + cleanup = function() + -- nothing end - while true do + end + while true do + if false then +--~ if true then + process() + cleanup() + wait() + else pcall(process) + pcall(cleanup) pcall(wait) end - else - logs.report("watch", "no paths to watch") end + else + logs.report("watch", "no paths to watch") end - end function scripts.watch.collect_logs(path) -- clean 'm up too @@ -171,10 +258,11 @@ function scripts.watch.collect_logs(path) -- clean 'm up too local files = dir.globfiles(path,false,"^%d+%.lua$") local collection = { } local valid = table.tohash({"filename","result","runtime","size","status"}) - for _, name in ipairs(files) do + for i=1,#files do + local name = files[i] local t = dofile(name) if t and type(t) == "table" and t.status then - for k, v in pairs(t) do + for k, v in next, t do if not valid[k] then t[k] = nil end @@ -186,21 +274,21 @@ function scripts.watch.collect_logs(path) -- clean 'm up too end function scripts.watch.save_logs(collection,path) -- play safe - if collection and not table.is_empty(collection) then + if collection and next(collection) then path = path or environment.argument("logpath") or "" path = (path == "" and ".") or path - local filename = string.format("%s/collected-%s.lua",path,tostring(os.time())) + local filename = format("%s/collected-%s.lua",path,tostring(time())) io.savedata(filename,table.serialize(collection,true)) local check = dofile(filename) - for k,v in pairs(check) do + for k,v in next, check do if not collection[k] then logs.error("watch", "error in saving file") os.remove(filename) return false end end - for k,v in pairs(check) do - os.remove(string.format("%s.lua",k)) + for k,v in next, check do + os.remove(format("%s.lua",k)) end return true else @@ -213,10 +301,11 @@ function scripts.watch.collect_collections(path) -- removes duplicates path = (path == "" and ".") or path local files = dir.globfiles(path,false,"^collected%-%d+%.lua$") local collection = { } - for _, name in ipairs(files) do + for i=1,#files do + local name = files[i] local t = dofile(name) if t and type(t) == "table" then - for k, v in pairs(t) do + for k, v in next, t do collection[k] = v end end @@ -227,26 +316,56 @@ end function scripts.watch.show_logs(path) -- removes duplicates local collection = scripts.watch.collect_collections(path) or { } local max = 0 - for k,v in pairs(collection) do + for k,v in next, collection do v = v.filename or "?" if #v > max then max = #v end end - print(max) - for k,v in ipairs(table.sortedkeys(collection)) do + -- print(max) + local sorted = table.sortedkeys(collection) + for k=1,#sorted do + local v = sorted[k] local c = collection[v] local f, s, r, n = c.filename or "?", c.status or "?", c.runtime or 0, c.size or 0 - logs.report("watch", string.format("%s %s %3i %8i %s",string.padd(f,max," "),string.padd(s,10," "),r,n,v)) + logs.report("watch", format("%s %s %3i %8i %s",string.padd(f,max," "),string.padd(s,10," "),r,n,v)) + end +end + +function scripts.watch.cleanup_stale_files() -- removes duplicates + local path = environment.files[1] + local delay = tonumber(environment.argument("cleanup")) + local force = environment.argument("force") + if not path or path == "." then + logs.report("watch","provide qualified path") + elseif not delay then + logs.report("watch","missing --cleanup=delay") + else + logs.report("watch","dryrun, use --force for real cleanup") + local files = dir.glob(file.join(path,"*")) + local rtime = time() + for i=1,#files do + local name = files[i] + local mtime = lfs.attributes(name,"modification") + local delta = difftime(rtime,mtime) + if delta > delay then + logs.report("watch",format("cleaning up '%s'",name)) + if force then + os.remove(name) + end + end + end end end -logs.extendbanner("Watchdog 1.00",true) +logs.extendbanner("ConTeXt Request Watchdog 1.00",true) messages.help = [[ --logpath optional path for log files ---watch watch given path +--watch watch given path [--delay] --pipe use pipe instead of execute --delay delay between sweeps +--automachine replace /machine/ in path /<servername>/ --collect condense log files +--cleanup=delay remove files in given path [--force] --showlog show log data ]] @@ -254,6 +373,8 @@ if environment.argument("watch") then scripts.watch.watch() elseif environment.argument("collect") then scripts.watch.save_logs(scripts.watch.collect_logs()) +elseif environment.argument("cleanup") then + scripts.watch.save_logs(scripts.watch.cleanup_stale_files()) elseif environment.argument("showlog") then scripts.watch.show_logs() else diff --git a/Master/texmf-dist/scripts/context/lua/mtxrun.lua b/Master/texmf-dist/scripts/context/lua/mtxrun.lua index 82d1edecbc5..b99327692d7 100755 --- a/Master/texmf-dist/scripts/context/lua/mtxrun.lua +++ b/Master/texmf-dist/scripts/context/lua/mtxrun.lua @@ -48,13 +48,16 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-string'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -local sub, gsub, find, match, gmatch, format, char, byte, rep = string.sub, string.gsub, string.find, string.match, string.gmatch, string.format, string.char, string.byte, string.rep +local sub, gsub, find, match, gmatch, format, char, byte, rep, lower = string.sub, string.gsub, string.find, string.match, string.gmatch, string.format, string.char, string.byte, string.rep, string.lower +local lpegmatch = lpeg.match + +-- some functions may disappear as they are not used anywhere if not string.split then @@ -94,8 +97,16 @@ function string:unquote() return (gsub(self,"^([\"\'])(.*)%1$","%2")) end +--~ function string:unquote() +--~ if find(self,"^[\'\"]") then +--~ return sub(self,2,-2) +--~ else +--~ return self +--~ end +--~ end + function string:quote() -- we could use format("%q") - return '"' .. self:unquote() .. '"' + return format("%q",self) end function string:count(pattern) -- variant 3 @@ -115,12 +126,23 @@ function string:limit(n,sentinel) end end -function string:strip() - return (gsub(self,"^%s*(.-)%s*$", "%1")) +--~ function string:strip() -- the .- is quite efficient +--~ -- return match(self,"^%s*(.-)%s*$") or "" +--~ -- return match(self,'^%s*(.*%S)') or '' -- posted on lua list +--~ return find(s,'^%s*$') and '' or match(s,'^%s*(.*%S)') +--~ end + +do -- roberto's variant: + local space = lpeg.S(" \t\v\n") + local nospace = 1 - space + local stripper = space^0 * lpeg.C((space^0 * nospace^1)^0) + function string.strip(str) + return lpegmatch(stripper,str) or "" + end end function string:is_empty() - return not find(find,"%S") + return not find(self,"%S") end function string:enhance(pattern,action) @@ -154,14 +176,14 @@ if not string.characters then local function nextchar(str, index) index = index + 1 - return (index <= #str) and index or nil, str:sub(index,index) + return (index <= #str) and index or nil, sub(str,index,index) end function string:characters() return nextchar, self, 0 end local function nextbyte(str, index) index = index + 1 - return (index <= #str) and index or nil, byte(str:sub(index,index)) + return (index <= #str) and index or nil, byte(sub(str,index,index)) end function string:bytes() return nextbyte, self, 0 @@ -174,7 +196,7 @@ end function string:rpadd(n,chr) local m = n-#self if m > 0 then - return self .. self.rep(chr or " ",m) + return self .. rep(chr or " ",m) else return self end @@ -183,7 +205,7 @@ end function string:lpadd(n,chr) local m = n-#self if m > 0 then - return self.rep(chr or " ",m) .. self + return rep(chr or " ",m) .. self else return self end @@ -231,6 +253,17 @@ function string:pattesc() return (gsub(self,".",patterns_escapes)) end +local simple_escapes = { + ["-"] = "%-", + ["."] = "%.", + ["?"] = ".", + ["*"] = ".*", +} + +function string:simpleesc() + return (gsub(self,".",simple_escapes)) +end + function string:tohash() local t = { } for s in gmatch(self,"([^, ]+)") do -- lpeg @@ -242,10 +275,10 @@ end local pattern = lpeg.Ct(lpeg.C(1)^0) function string:totable() - return pattern:match(self) + return lpegmatch(pattern,self) end ---~ for _, str in ipairs { +--~ local t = { --~ "1234567123456712345671234567", --~ "a\tb\tc", --~ "aa\tbb\tcc", @@ -253,7 +286,10 @@ end --~ "aaaa\tbbbb\tcccc", --~ "aaaaa\tbbbbb\tccccc", --~ "aaaaaa\tbbbbbb\tcccccc", ---~ } do print(string.tabtospace(str)) end +--~ } +--~ for k,v do +--~ print(string.tabtospace(t[k])) +--~ end function string.tabtospace(str,tab) -- we don't handle embedded newlines @@ -261,7 +297,7 @@ function string.tabtospace(str,tab) local s = find(str,"\t") if s then if not tab then tab = 7 end -- only when found - local d = tab-(s-1)%tab + local d = tab-(s-1) % tab if d > 0 then str = gsub(str,"\t",rep(" ",d),1) else @@ -280,6 +316,25 @@ function string:compactlong() -- strips newlines and leading spaces return self end +function string:striplong() -- strips newlines and leading spaces + self = gsub(self,"^%s*","") + self = gsub(self,"[\n\r]+ *","\n") + return self +end + +function string:topattern(lowercase,strict) + if lowercase then + self = lower(self) + end + self = gsub(self,".",simple_escapes) + if self == "" then + self = ".*" + elseif strict then + self = "^" .. self .. "$" + end + return self +end + end -- of closure @@ -287,58 +342,64 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-lpeg'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -local P, S, Ct, C, Cs, Cc = lpeg.P, lpeg.S, lpeg.Ct, lpeg.C, lpeg.Cs, lpeg.Cc - ---~ l-lpeg.lua : - ---~ lpeg.digit = lpeg.R('09')^1 ---~ lpeg.sign = lpeg.S('+-')^1 ---~ lpeg.cardinal = lpeg.P(lpeg.sign^0 * lpeg.digit^1) ---~ lpeg.integer = lpeg.P(lpeg.sign^0 * lpeg.digit^1) ---~ lpeg.float = lpeg.P(lpeg.sign^0 * lpeg.digit^0 * lpeg.P('.') * lpeg.digit^1) ---~ lpeg.number = lpeg.float + lpeg.integer ---~ lpeg.oct = lpeg.P("0") * lpeg.R('07')^1 ---~ lpeg.hex = lpeg.P("0x") * (lpeg.R('09') + lpeg.R('AF'))^1 ---~ lpeg.uppercase = lpeg.P("AZ") ---~ lpeg.lowercase = lpeg.P("az") - ---~ lpeg.eol = lpeg.S('\r\n\f')^1 -- includes formfeed ---~ lpeg.space = lpeg.S(' ')^1 ---~ lpeg.nonspace = lpeg.P(1-lpeg.space)^1 ---~ lpeg.whitespace = lpeg.S(' \r\n\f\t')^1 ---~ lpeg.nonwhitespace = lpeg.P(1-lpeg.whitespace)^1 - -local hash = { } +local lpeg = require("lpeg") + +lpeg.patterns = lpeg.patterns or { } -- so that we can share +local patterns = lpeg.patterns + +local P, R, S, Ct, C, Cs, Cc, V = lpeg.P, lpeg.R, lpeg.S, lpeg.Ct, lpeg.C, lpeg.Cs, lpeg.Cc, lpeg.V +local match = lpeg.match + +local digit, sign = R('09'), S('+-') +local cr, lf, crlf = P("\r"), P("\n"), P("\r\n") +local utf8byte = R("\128\191") + +patterns.utf8byte = utf8byte +patterns.utf8one = R("\000\127") +patterns.utf8two = R("\194\223") * utf8byte +patterns.utf8three = R("\224\239") * utf8byte * utf8byte +patterns.utf8four = R("\240\244") * utf8byte * utf8byte * utf8byte + +patterns.digit = digit +patterns.sign = sign +patterns.cardinal = sign^0 * digit^1 +patterns.integer = sign^0 * digit^1 +patterns.float = sign^0 * digit^0 * P('.') * digit^1 +patterns.number = patterns.float + patterns.integer +patterns.oct = P("0") * R("07")^1 +patterns.octal = patterns.oct +patterns.HEX = P("0x") * R("09","AF")^1 +patterns.hex = P("0x") * R("09","af")^1 +patterns.hexadecimal = P("0x") * R("09","AF","af")^1 +patterns.lowercase = R("az") +patterns.uppercase = R("AZ") +patterns.letter = patterns.lowercase + patterns.uppercase +patterns.space = S(" ") +patterns.eol = S("\n\r") +patterns.spacer = S(" \t\f\v") -- + string.char(0xc2, 0xa0) if we want utf (cf mail roberto) +patterns.newline = crlf + cr + lf +patterns.nonspace = 1 - patterns.space +patterns.nonspacer = 1 - patterns.spacer +patterns.whitespace = patterns.eol + patterns.spacer +patterns.nonwhitespace = 1 - patterns.whitespace +patterns.utf8 = patterns.utf8one + patterns.utf8two + patterns.utf8three + patterns.utf8four +patterns.utfbom = P('\000\000\254\255') + P('\255\254\000\000') + P('\255\254') + P('\254\255') + P('\239\187\191') function lpeg.anywhere(pattern) --slightly adapted from website - return P { P(pattern) + 1 * lpeg.V(1) } -end - -function lpeg.startswith(pattern) --slightly adapted - return P(pattern) + return P { P(pattern) + 1 * V(1) } -- why so complex? end function lpeg.splitter(pattern, action) return (((1-P(pattern))^1)/action+1)^0 end --- variant: - ---~ local parser = lpeg.Ct(lpeg.splitat(newline)) - -local crlf = P("\r\n") -local cr = P("\r") -local lf = P("\n") -local space = S(" \t\f\v") -- + string.char(0xc2, 0xa0) if we want utf (cf mail roberto) -local newline = crlf + cr + lf -local spacing = space^0 * newline - +local spacing = patterns.spacer^0 * patterns.newline -- sort of strip local empty = spacing * Cc("") local nonempty = Cs((1-spacing)^1) * spacing^-1 local content = (empty + nonempty)^1 @@ -346,15 +407,15 @@ local content = (empty + nonempty)^1 local capture = Ct(content^0) function string:splitlines() - return capture:match(self) + return match(capture,self) end -lpeg.linebyline = content -- better make a sublibrary +patterns.textline = content ---~ local p = lpeg.splitat("->",false) print(p:match("oeps->what->more")) -- oeps what more ---~ local p = lpeg.splitat("->",true) print(p:match("oeps->what->more")) -- oeps what->more ---~ local p = lpeg.splitat("->",false) print(p:match("oeps")) -- oeps ---~ local p = lpeg.splitat("->",true) print(p:match("oeps")) -- oeps +--~ local p = lpeg.splitat("->",false) print(match(p,"oeps->what->more")) -- oeps what more +--~ local p = lpeg.splitat("->",true) print(match(p,"oeps->what->more")) -- oeps what->more +--~ local p = lpeg.splitat("->",false) print(match(p,"oeps")) -- oeps +--~ local p = lpeg.splitat("->",true) print(match(p,"oeps")) -- oeps local splitters_s, splitters_m = { }, { } @@ -364,7 +425,7 @@ local function splitat(separator,single) separator = P(separator) if single then local other, any = C((1 - separator)^0), P(1) - splitter = other * (separator * C(any^0) + "") + splitter = other * (separator * C(any^0) + "") -- ? splitters_s[separator] = splitter else local other = C((1 - separator)^0) @@ -379,15 +440,72 @@ lpeg.splitat = splitat local cache = { } +function lpeg.split(separator,str) + local c = cache[separator] + if not c then + c = Ct(splitat(separator)) + cache[separator] = c + end + return match(c,str) +end + function string:split(separator) local c = cache[separator] if not c then c = Ct(splitat(separator)) cache[separator] = c end - return c:match(self) + return match(c,self) +end + +lpeg.splitters = cache + +local cache = { } + +function lpeg.checkedsplit(separator,str) + local c = cache[separator] + if not c then + separator = P(separator) + local other = C((1 - separator)^0) + c = Ct(separator^0 * other * (separator^1 * other)^0) + cache[separator] = c + end + return match(c,str) +end + +function string:checkedsplit(separator) + local c = cache[separator] + if not c then + separator = P(separator) + local other = C((1 - separator)^0) + c = Ct(separator^0 * other * (separator^1 * other)^0) + cache[separator] = c + end + return match(c,self) end +--~ function lpeg.append(list,pp) +--~ local p = pp +--~ for l=1,#list do +--~ if p then +--~ p = p + P(list[l]) +--~ else +--~ p = P(list[l]) +--~ end +--~ end +--~ return p +--~ end + +--~ from roberto's site: + +local f1 = string.byte + +local function f2(s) local c1, c2 = f1(s,1,2) return c1 * 64 + c2 - 12416 end +local function f3(s) local c1, c2, c3 = f1(s,1,3) return (c1 * 64 + c2) * 64 + c3 - 925824 end +local function f4(s) local c1, c2, c3, c4 = f1(s,1,4) return ((c1 * 64 + c2) * 64 + c3) * 64 + c4 - 63447168 end + +patterns.utf8byte = patterns.utf8one/f1 + patterns.utf8two/f2 + patterns.utf8three/f3 + patterns.utf8four/f4 + end -- of closure @@ -395,7 +513,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-table'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -404,9 +522,58 @@ if not modules then modules = { } end modules ['l-table'] = { table.join = table.concat local concat, sort, insert, remove = table.concat, table.sort, table.insert, table.remove -local format, find, gsub, lower, dump = string.format, string.find, string.gsub, string.lower, string.dump +local format, find, gsub, lower, dump, match = string.format, string.find, string.gsub, string.lower, string.dump, string.match local getmetatable, setmetatable = getmetatable, setmetatable -local type, next, tostring, ipairs = type, next, tostring, ipairs +local type, next, tostring, tonumber, ipairs = type, next, tostring, tonumber, ipairs + +-- Starting with version 5.2 Lua no longer provide ipairs, which makes +-- sense. As we already used the for loop and # in most places the +-- impact on ConTeXt was not that large; the remaining ipairs already +-- have been replaced. In a similar fashio we also hardly used pairs. +-- +-- Just in case, we provide the fallbacks as discussed in Programming +-- in Lua (http://www.lua.org/pil/7.3.html): + +if not ipairs then + + -- for k, v in ipairs(t) do ... end + -- for k=1,#t do local v = t[k] ... end + + local function iterate(a,i) + i = i + 1 + local v = a[i] + if v ~= nil then + return i, v --, nil + end + end + + function ipairs(a) + return iterate, a, 0 + end + +end + +if not pairs then + + -- for k, v in pairs(t) do ... end + -- for k, v in next, t do ... end + + function pairs(t) + return next, t -- , nil + end + +end + +-- Also, unpack has been moved to the table table, and for compatiility +-- reasons we provide both now. + +if not table.unpack then + table.unpack = _G.unpack +elseif not unpack then + _G.unpack = table.unpack +end + +-- extra functions, some might go (when not used) function table.strip(tab) local lst = { } @@ -421,6 +588,14 @@ function table.strip(tab) return lst end +function table.keys(t) + local k = { } + for key, _ in next, t do + k[#k+1] = key + end + return k +end + local function compare(a,b) return (tostring(a) < tostring(b)) end @@ -464,7 +639,7 @@ end table.sortedkeys = sortedkeys table.sortedhashkeys = sortedhashkeys -function table.sortedpairs(t) +function table.sortedhash(t) local s = sortedhashkeys(t) -- maybe just sortedkeys local n = 0 local function kv(s) @@ -475,6 +650,8 @@ function table.sortedpairs(t) return kv, s end +table.sortedpairs = table.sortedhash + function table.append(t, list) for _,v in next, list do insert(t,v) @@ -583,7 +760,7 @@ end table.fastcopy = fastcopy table.copy = copy --- rougly: copy-loop : unpack : sub == 0.9 : 0.4 : 0.45 (so in critical apps, use unpack) +-- roughly: copy-loop : unpack : sub == 0.9 : 0.4 : 0.45 (so in critical apps, use unpack) function table.sub(t,i,j) return { unpack(t,i,j) } @@ -597,18 +774,18 @@ end -- slower than #t on indexed tables (#t only returns the size of the numerically indexed slice) -function table.is_empty(t) +function table.is_empty(t) -- obolete, use inline code instead return not t or not next(t) end -function table.one_entry(t) +function table.one_entry(t) -- obolete, use inline code instead local n = next(t) return n and not next(t,n) end -function table.starts_at(t) - return ipairs(t,1)(t,0) -end +--~ function table.starts_at(t) -- obsolete, not nice anyway +--~ return ipairs(t,1)(t,0) +--~ end function table.tohash(t,value) local h = { } @@ -686,6 +863,8 @@ end -- -- local propername = lpeg.P(lpeg.R("AZ","az","__") * lpeg.R("09","AZ","az", "__")^0 * lpeg.P(-1) ) +-- problem: there no good number_to_string converter with the best resolution + local function do_serialize(root,name,depth,level,indexed) if level > 0 then depth = depth .. " " @@ -708,8 +887,9 @@ local function do_serialize(root,name,depth,level,indexed) handle(format("%s{",depth)) end end + -- we could check for k (index) being number (cardinal) if root and next(root) then - local first, last = nil, 0 -- #root cannot be trusted here + local first, last = nil, 0 -- #root cannot be trusted here (will be ok in 5.2 when ipairs is gone) if compact then -- NOT: for k=1,#root do (we need to quit at nil) for k,v in ipairs(root) do -- can we use next? @@ -730,10 +910,10 @@ local function do_serialize(root,name,depth,level,indexed) if hexify then handle(format("%s 0x%04X,",depth,v)) else - handle(format("%s %s,",depth,v)) + handle(format("%s %s,",depth,v)) -- %.99g end elseif t == "string" then - if reduce and (find(v,"^[%-%+]?[%d]-%.?[%d+]$") == 1) then + if reduce and tonumber(v) then handle(format("%s %s,",depth,v)) else handle(format("%s %q,",depth,v)) @@ -770,29 +950,29 @@ local function do_serialize(root,name,depth,level,indexed) --~ if hexify then --~ handle(format("%s %s=0x%04X,",depth,key(k),v)) --~ else - --~ handle(format("%s %s=%s,",depth,key(k),v)) + --~ handle(format("%s %s=%s,",depth,key(k),v)) -- %.99g --~ end if type(k) == "number" then -- or find(k,"^%d+$") then if hexify then handle(format("%s [0x%04X]=0x%04X,",depth,k,v)) else - handle(format("%s [%s]=%s,",depth,k,v)) + handle(format("%s [%s]=%s,",depth,k,v)) -- %.99g end elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then if hexify then handle(format("%s %s=0x%04X,",depth,k,v)) else - handle(format("%s %s=%s,",depth,k,v)) + handle(format("%s %s=%s,",depth,k,v)) -- %.99g end else if hexify then handle(format("%s [%q]=0x%04X,",depth,k,v)) else - handle(format("%s [%q]=%s,",depth,k,v)) + handle(format("%s [%q]=%s,",depth,k,v)) -- %.99g end end elseif t == "string" then - if reduce and (find(v,"^[%-%+]?[%d]-%.?[%d+]$") == 1) then + if reduce and tonumber(v) then --~ handle(format("%s %s=%s,",depth,key(k),v)) if type(k) == "number" then -- or find(k,"^%d+$") then if hexify then @@ -1001,7 +1181,7 @@ function table.tofile(filename,root,name,reduce,noquotes,hexify) end end -local function flatten(t,f,complete) +local function flatten(t,f,complete) -- is this used? meybe a variant with next, ... for i=1,#t do local v = t[i] if type(v) == "table" then @@ -1030,6 +1210,24 @@ end table.flatten_one_level = table.unnest +-- a better one: + +local function flattened(t,f) + if not f then + f = { } + end + for k, v in next, t do + if type(v) == "table" then + flattened(v,f) + else + f[k] = v + end + end + return f +end + +table.flattened = flattened + -- the next three may disappear function table.remove_value(t,value) -- todo: n @@ -1165,7 +1363,7 @@ function table.clone(t,p) -- t is optional or nil or table elseif not t then t = { } end - setmetatable(t, { __index = function(_,key) return p[key] end }) + setmetatable(t, { __index = function(_,key) return p[key] end }) -- why not __index = p ? return t end @@ -1193,21 +1391,36 @@ function table.reverse(t) return tt end ---~ function table.keys(t) ---~ local k = { } ---~ for k,_ in next, t do ---~ k[#k+1] = k ---~ end ---~ return k ---~ end +function table.insert_before_value(t,value,extra) + for i=1,#t do + if t[i] == extra then + remove(t,i) + end + end + for i=1,#t do + if t[i] == value then + insert(t,i,extra) + return + end + end + insert(t,1,extra) +end + +function table.insert_after_value(t,value,extra) + for i=1,#t do + if t[i] == extra then + remove(t,i) + end + end + for i=1,#t do + if t[i] == value then + insert(t,i+1,extra) + return + end + end + insert(t,#t+1,extra) +end ---~ function table.keys_as_string(t) ---~ local k = { } ---~ for k,_ in next, t do ---~ k[#k+1] = k ---~ end ---~ return concat(k,"") ---~ end end -- of closure @@ -1216,13 +1429,13 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-io'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -local byte = string.byte +local byte, find, gsub = string.byte, string.find, string.gsub if string.find(os.getenv("PATH"),";") then io.fileseparator, io.pathseparator = "\\", ";" @@ -1251,7 +1464,7 @@ function io.savedata(filename,data,joiner) elseif type(data) == "function" then data(f) else - f:write(data) + f:write(data or "") end f:close() return true @@ -1380,20 +1593,21 @@ function io.ask(question,default,options) end io.write(string.format(" ")) local answer = io.read() - answer = answer:gsub("^%s*(.*)%s*$","%1") + answer = gsub(answer,"^%s*(.*)%s*$","%1") if answer == "" and default then return default elseif not options then return answer else - for _,v in pairs(options) do - if v == answer then + for k=1,#options do + if options[k] == answer then return answer end end local pattern = "^" .. answer - for _,v in pairs(options) do - if v:find(pattern) then + for k=1,#options do + local v = options[k] + if find(v,pattern) then return v end end @@ -1408,20 +1622,22 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-number'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -local format = string.format +local tostring = tostring +local format, floor, insert, match = string.format, math.floor, table.insert, string.match +local lpegmatch = lpeg.match number = number or { } -- a,b,c,d,e,f = number.toset(100101) function number.toset(n) - return (tostring(n)):match("(.?)(.?)(.?)(.?)(.?)(.?)(.?)(.?)") + return match(tostring(n),"(.?)(.?)(.?)(.?)(.?)(.?)(.?)(.?)") end function number.toevenhex(n) @@ -1447,10 +1663,21 @@ end local one = lpeg.C(1-lpeg.S(''))^1 function number.toset(n) - return one:match(tostring(n)) + return lpegmatch(one,tostring(n)) end - +function number.bits(n,zero) + local t, i = { }, (zero and 0) or 1 + while n > 0 do + local m = n % 2 + if m > 0 then + insert(t,1,i) + end + n = floor(n/2) + i = i + 1 + end + return t +end end -- of closure @@ -1459,7 +1686,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-set'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -1549,46 +1776,63 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-os'] = { version = 1.001, - comment = "companion to luat-lub.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -local find = string.find +-- maybe build io.flush in os.execute + +local find, format, gsub = string.find, string.format, string.gsub +local random, ceil = math.random, math.ceil + +local execute, spawn, exec, ioflush = os.execute, os.spawn or os.execute, os.exec or os.execute, io.flush + +function os.execute(...) ioflush() return execute(...) end +function os.spawn (...) ioflush() return spawn (...) end +function os.exec (...) ioflush() return exec (...) end function os.resultof(command) - return io.popen(command,"r"):read("*all") + ioflush() -- else messed up logging + local handle = io.popen(command,"r") + if not handle then + -- print("unknown command '".. command .. "' in os.resultof") + return "" + else + return handle:read("*all") or "" + end end -if not os.exec then os.exec = os.execute end -if not os.spawn then os.spawn = os.execute end - ---~ os.type : windows | unix (new, we already guessed os.platform) ---~ os.name : windows | msdos | linux | macosx | solaris | .. | generic (new) +--~ os.type : windows | unix (new, we already guessed os.platform) +--~ os.name : windows | msdos | linux | macosx | solaris | .. | generic (new) +--~ os.platform : extended os.name with architecture if not io.fileseparator then if find(os.getenv("PATH"),";") then - io.fileseparator, io.pathseparator, os.platform = "\\", ";", os.type or "windows" + io.fileseparator, io.pathseparator, os.type = "\\", ";", os.type or "mswin" else - io.fileseparator, io.pathseparator, os.platform = "/" , ":", os.type or "unix" + io.fileseparator, io.pathseparator, os.type = "/" , ":", os.type or "unix" end end -os.platform = os.platform or os.type or (io.pathseparator == ";" and "windows") or "unix" +os.type = os.type or (io.pathseparator == ";" and "windows") or "unix" +os.name = os.name or (os.type == "windows" and "mswin" ) or "linux" + +if os.type == "windows" then + os.libsuffix, os.binsuffix = 'dll', 'exe' +else + os.libsuffix, os.binsuffix = 'so', '' +end function os.launch(str) - if os.platform == "windows" then + if os.type == "windows" then os.execute("start " .. str) -- os.spawn ? else os.execute(str .. " &") -- os.spawn ? end end -if not os.setenv then - function os.setenv() return false end -end - if not os.times then -- utime = user time -- stime = system time @@ -1618,64 +1862,218 @@ end --~ print(os.date("%H:%M:%S",os.gettimeofday())) --~ print(os.date("%H:%M:%S",os.time())) -os.arch = os.arch or function() - local a = os.resultof("uname -m") or "linux" - os.arch = function() - return a +-- no need for function anymore as we have more clever code and helpers now +-- this metatable trickery might as well disappear + +os.resolvers = os.resolvers or { } + +local resolvers = os.resolvers + +local osmt = getmetatable(os) or { __index = function(t,k) t[k] = "unset" return "unset" end } -- maybe nil +local osix = osmt.__index + +osmt.__index = function(t,k) + return (resolvers[k] or osix)(t,k) +end + +setmetatable(os,osmt) + +if not os.setenv then + + -- we still store them but they won't be seen in + -- child processes although we might pass them some day + -- using command concatination + + local env, getenv = { }, os.getenv + + function os.setenv(k,v) + env[k] = v + end + + function os.getenv(k) + return env[k] or getenv(k) end - return a + end -local platform +-- we can use HOSTTYPE on some platforms -function os.currentplatform(name,default) - if not platform then - local name = os.name or os.platform or name -- os.name is built in, os.platform is mine - if not name then - platform = default or "linux" - elseif name == "windows" or name == "mswin" or name == "win32" or name == "msdos" then - if os.getenv("PROCESSOR_ARCHITECTURE") == "AMD64" then - platform = "mswin-64" - else - platform = "mswin" - end +local name, platform = os.name or "linux", os.getenv("MTX_PLATFORM") or "" + +local function guess() + local architecture = os.resultof("uname -m") or "" + if architecture ~= "" then + return architecture + end + architecture = os.getenv("HOSTTYPE") or "" + if architecture ~= "" then + return architecture + end + return os.resultof("echo $HOSTTYPE") or "" +end + +if platform ~= "" then + + os.platform = platform + +elseif os.type == "windows" then + + -- we could set the variable directly, no function needed here + + function os.resolvers.platform(t,k) + local platform, architecture = "", os.getenv("PROCESSOR_ARCHITECTURE") or "" + if find(architecture,"AMD64") then + platform = "mswin-64" else - local architecture = os.arch() - if name == "linux" then - if find(architecture,"x86_64") then - platform = "linux-64" - elseif find(architecture,"ppc") then - platform = "linux-ppc" - else - platform = "linux" - end - elseif name == "macosx" then - if find(architecture,"i386") then - platform = "osx-intel" - else - platform = "osx-ppc" - end - elseif name == "sunos" then - if find(architecture,"sparc") then - platform = "solaris-sparc" - else -- if architecture == 'i86pc' - platform = "solaris-intel" - end - elseif name == "freebsd" then - if find(architecture,"amd64") then - platform = "freebsd-amd64" - else - platform = "freebsd" - end - else - platform = default or name - end + platform = "mswin" end - function os.currentplatform() - return platform + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +elseif name == "linux" then + + function os.resolvers.platform(t,k) + -- we sometims have HOSTTYPE set so let's check that first + local platform, architecture = "", os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" + if find(architecture,"x86_64") then + platform = "linux-64" + elseif find(architecture,"ppc") then + platform = "linux-ppc" + else + platform = "linux" + end + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +elseif name == "macosx" then + + --[[ + Identifying the architecture of OSX is quite a mess and this + is the best we can come up with. For some reason $HOSTTYPE is + a kind of pseudo environment variable, not known to the current + environment. And yes, uname cannot be trusted either, so there + is a change that you end up with a 32 bit run on a 64 bit system. + Also, some proper 64 bit intel macs are too cheap (low-end) and + therefore not permitted to run the 64 bit kernel. + ]]-- + + function os.resolvers.platform(t,k) + -- local platform, architecture = "", os.getenv("HOSTTYPE") or "" + -- if architecture == "" then + -- architecture = os.resultof("echo $HOSTTYPE") or "" + -- end + local platform, architecture = "", os.resultof("echo $HOSTTYPE") or "" + if architecture == "" then + -- print("\nI have no clue what kind of OSX you're running so let's assume an 32 bit intel.\n") + platform = "osx-intel" + elseif find(architecture,"i386") then + platform = "osx-intel" + elseif find(architecture,"x86_64") then + platform = "osx-64" + else + platform = "osx-ppc" end + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +elseif name == "sunos" then + + function os.resolvers.platform(t,k) + local platform, architecture = "", os.resultof("uname -m") or "" + if find(architecture,"sparc") then + platform = "solaris-sparc" + else -- if architecture == 'i86pc' + platform = "solaris-intel" + end + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +elseif name == "freebsd" then + + function os.resolvers.platform(t,k) + local platform, architecture = "", os.resultof("uname -m") or "" + if find(architecture,"amd64") then + platform = "freebsd-amd64" + else + platform = "freebsd" + end + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +elseif name == "kfreebsd" then + + function os.resolvers.platform(t,k) + -- we sometims have HOSTTYPE set so let's check that first + local platform, architecture = "", os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" + if find(architecture,"x86_64") then + platform = "kfreebsd-64" + else + platform = "kfreebsd-i386" + end + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +else + + -- platform = "linux" + -- os.setenv("MTX_PLATFORM",platform) + -- os.platform = platform + + function os.resolvers.platform(t,k) + local platform = "linux" + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +end + +-- beware, we set the randomseed + +-- from wikipedia: Version 4 UUIDs use a scheme relying only on random numbers. This algorithm sets the +-- version number as well as two reserved bits. All other bits are set using a random or pseudorandom +-- data source. Version 4 UUIDs have the form xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx with hexadecimal +-- digits x and hexadecimal digits 8, 9, A, or B for y. e.g. f47ac10b-58cc-4372-a567-0e02b2c3d479. +-- +-- as we don't call this function too often there is not so much risk on repetition + +local t = { 8, 9, "a", "b" } + +function os.uuid() + return format("%04x%04x-4%03x-%s%03x-%04x-%04x%04x%04x", + random(0xFFFF),random(0xFFFF), + random(0x0FFF), + t[ceil(random(4))] or 8,random(0x0FFF), + random(0xFFFF), + random(0xFFFF),random(0xFFFF),random(0xFFFF) + ) +end + +local d + +function os.timezone(delta) + d = d or tonumber(tonumber(os.date("%H")-os.date("!%H"))) + if delta then + if d > 0 then + return format("+%02i:00",d) + else + return format("-%02i:00",-d) + end + else + return 1 end - return platform end @@ -1685,7 +2083,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-file'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -1696,14 +2094,17 @@ if not modules then modules = { } end modules ['l-file'] = { file = file or { } local concat = table.concat -local find, gmatch, match, gsub = string.find, string.gmatch, string.match, string.gsub +local find, gmatch, match, gsub, sub, char = string.find, string.gmatch, string.match, string.gsub, string.sub, string.char +local lpegmatch = lpeg.match function file.removesuffix(filename) return (gsub(filename,"%.[%a%d]+$","")) end function file.addsuffix(filename, suffix) - if not find(filename,"%.[%a%d]+$") then + if not suffix or suffix == "" then + return filename + elseif not find(filename,"%.[%a%d]+$") then return filename .. "." .. suffix else return filename @@ -1726,20 +2127,39 @@ function file.nameonly(name) return (gsub(match(name,"^.+[/\\](.-)$") or name,"%..*$","")) end -function file.extname(name) - return match(name,"^.+%.([^/\\]-)$") or "" +function file.extname(name,default) + return match(name,"^.+%.([^/\\]-)$") or default or "" end file.suffix = file.extname ---~ print(file.join("x/","/y")) ---~ print(file.join("http://","/y")) ---~ print(file.join("http://a","/y")) ---~ print(file.join("http:///a","/y")) ---~ print(file.join("//nas-1","/y")) +--~ function file.join(...) +--~ local pth = concat({...},"/") +--~ pth = gsub(pth,"\\","/") +--~ local a, b = match(pth,"^(.*://)(.*)$") +--~ if a and b then +--~ return a .. gsub(b,"//+","/") +--~ end +--~ a, b = match(pth,"^(//)(.*)$") +--~ if a and b then +--~ return a .. gsub(b,"//+","/") +--~ end +--~ return (gsub(pth,"//+","/")) +--~ end + +local trick_1 = char(1) +local trick_2 = "^" .. trick_1 .. "/+" function file.join(...) - local pth = concat({...},"/") + local lst = { ... } + local a, b = lst[1], lst[2] + if a == "" then + lst[1] = trick_1 + elseif b and find(a,"^/+$") and find(b,"^/") then + lst[1] = "" + lst[2] = gsub(b,"^/+","") + end + local pth = concat(lst,"/") pth = gsub(pth,"\\","/") local a, b = match(pth,"^(.*://)(.*)$") if a and b then @@ -1749,17 +2169,28 @@ function file.join(...) if a and b then return a .. gsub(b,"//+","/") end + pth = gsub(pth,trick_2,"") return (gsub(pth,"//+","/")) end +--~ print(file.join("//","/y")) +--~ print(file.join("/","/y")) +--~ print(file.join("","/y")) +--~ print(file.join("/x/","/y")) +--~ print(file.join("x/","/y")) +--~ print(file.join("http://","/y")) +--~ print(file.join("http://a","/y")) +--~ print(file.join("http:///a","/y")) +--~ print(file.join("//nas-1","/y")) + function file.iswritable(name) local a = lfs.attributes(name) or lfs.attributes(file.dirname(name,".")) - return a and a.permissions:sub(2,2) == "w" + return a and sub(a.permissions,2,2) == "w" end function file.isreadable(name) local a = lfs.attributes(name) - return a and a.permissions:sub(1,1) == "r" + return a and sub(a.permissions,1,1) == "r" end file.is_readable = file.isreadable @@ -1767,36 +2198,50 @@ file.is_writable = file.iswritable -- todo: lpeg -function file.split_path(str) - local t = { } - str = gsub(str,"\\", "/") - str = gsub(str,"(%a):([;/])", "%1\001%2") - for name in gmatch(str,"([^;:]+)") do - if name ~= "" then - t[#t+1] = gsub(name,"\001",":") - end - end - return t +--~ function file.split_path(str) +--~ local t = { } +--~ str = gsub(str,"\\", "/") +--~ str = gsub(str,"(%a):([;/])", "%1\001%2") +--~ for name in gmatch(str,"([^;:]+)") do +--~ if name ~= "" then +--~ t[#t+1] = gsub(name,"\001",":") +--~ end +--~ end +--~ return t +--~ end + +local checkedsplit = string.checkedsplit + +function file.split_path(str,separator) + str = gsub(str,"\\","/") + return checkedsplit(str,separator or io.pathseparator) end function file.join_path(tab) return concat(tab,io.pathseparator) -- can have trailing // end +-- we can hash them weakly + function file.collapse_path(str) - str = gsub(str,"/%./","/") - local n, m = 1, 1 - while n > 0 or m > 0 do - str, n = gsub(str,"[^/%.]+/%.%.$","") - str, m = gsub(str,"[^/%.]+/%.%./","") - end - str = gsub(str,"([^/])/$","%1") - str = gsub(str,"^%./","") - str = gsub(str,"/%.$","") + str = gsub(str,"\\","/") + if find(str,"/") then + str = gsub(str,"^%./",(gsub(lfs.currentdir(),"\\","/")) .. "/") -- ./xx in qualified + str = gsub(str,"/%./","/") + local n, m = 1, 1 + while n > 0 or m > 0 do + str, n = gsub(str,"[^/%.]+/%.%.$","") + str, m = gsub(str,"[^/%.]+/%.%./","") + end + str = gsub(str,"([^/])/$","%1") + -- str = gsub(str,"^%./","") -- ./xx in qualified + str = gsub(str,"/%.$","") + end if str == "" then str = "." end return str end +--~ print(file.collapse_path("/a")) --~ print(file.collapse_path("a/./b/..")) --~ print(file.collapse_path("a/aa/../b/bb")) --~ print(file.collapse_path("a/../..")) @@ -1826,27 +2271,27 @@ end --~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * lpeg.C(noperiod^1) * -1 --~ function file.extname(name) ---~ return pattern:match(name) or "" +--~ return lpegmatch(pattern,name) or "" --~ end --~ local pattern = lpeg.Cs(((period * noperiod^1 * -1)/"" + 1)^1) --~ function file.removesuffix(name) ---~ return pattern:match(name) +--~ return lpegmatch(pattern,name) --~ end --~ local pattern = (noslashes^0 * slashes)^1 * lpeg.C(noslashes^1) * -1 --~ function file.basename(name) ---~ return pattern:match(name) or name +--~ return lpegmatch(pattern,name) or name --~ end --~ local pattern = (noslashes^0 * slashes)^1 * lpeg.Cp() * noslashes^1 * -1 --~ function file.dirname(name) ---~ local p = pattern:match(name) +--~ local p = lpegmatch(pattern,name) --~ if p then ---~ return name:sub(1,p-2) +--~ return sub(name,1,p-2) --~ else --~ return "" --~ end @@ -1855,7 +2300,7 @@ end --~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * lpeg.Cp() * noperiod^1 * -1 --~ function file.addsuffix(name, suffix) ---~ local p = pattern:match(name) +--~ local p = lpegmatch(pattern,name) --~ if p then --~ return name --~ else @@ -1866,9 +2311,9 @@ end --~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * lpeg.Cp() * noperiod^1 * -1 --~ function file.replacesuffix(name,suffix) ---~ local p = pattern:match(name) +--~ local p = lpegmatch(pattern,name) --~ if p then ---~ return name:sub(1,p-2) .. "." .. suffix +--~ return sub(name,1,p-2) .. "." .. suffix --~ else --~ return name .. "." .. suffix --~ end @@ -1877,11 +2322,11 @@ end --~ local pattern = (noslashes^0 * slashes)^0 * lpeg.Cp() * ((noperiod^1 * period)^1 * lpeg.Cp() + lpeg.P(true)) * noperiod^1 * -1 --~ function file.nameonly(name) ---~ local a, b = pattern:match(name) +--~ local a, b = lpegmatch(pattern,name) --~ if b then ---~ return name:sub(a,b-2) +--~ return sub(name,a,b-2) --~ elseif a then ---~ return name:sub(a) +--~ return sub(name,a) --~ else --~ return name --~ end @@ -1915,11 +2360,11 @@ local rootbased = lpeg.P("/") + letter*lpeg.P(":") -- ./name ../name /name c: :// name/name function file.is_qualified_path(filename) - return qualified:match(filename) + return lpegmatch(qualified,filename) ~= nil end function file.is_rootbased_path(filename) - return rootbased:match(filename) + return lpegmatch(rootbased,filename) ~= nil end local slash = lpeg.S("\\/") @@ -1932,16 +2377,25 @@ local base = lpeg.C((1-suffix)^0) local pattern = (drive + lpeg.Cc("")) * (path + lpeg.Cc("")) * (base + lpeg.Cc("")) * (suffix + lpeg.Cc("")) function file.splitname(str) -- returns drive, path, base, suffix - return pattern:match(str) + return lpegmatch(pattern,str) end --- function test(t) for k, v in pairs(t) do print(v, "=>", file.splitname(v)) end end +-- function test(t) for k, v in next, t do print(v, "=>", file.splitname(v)) end end -- -- test { "c:", "c:/aa", "c:/aa/bb", "c:/aa/bb/cc", "c:/aa/bb/cc.dd", "c:/aa/bb/cc.dd.ee" } -- test { "c:", "c:aa", "c:aa/bb", "c:aa/bb/cc", "c:aa/bb/cc.dd", "c:aa/bb/cc.dd.ee" } -- test { "/aa", "/aa/bb", "/aa/bb/cc", "/aa/bb/cc.dd", "/aa/bb/cc.dd.ee" } -- test { "aa", "aa/bb", "aa/bb/cc", "aa/bb/cc.dd", "aa/bb/cc.dd.ee" } +--~ -- todo: +--~ +--~ if os.type == "windows" then +--~ local currentdir = lfs.currentdir +--~ function lfs.currentdir() +--~ return (gsub(currentdir(),"\\","/")) +--~ end +--~ end + end -- of closure @@ -2006,7 +2460,7 @@ end function file.loadchecksum(name) if md5 then local data = io.loaddata(name .. ".md5") - return data and data:gsub("%s","") + return data and (gsub(data,"%s","")) end return nil end @@ -2025,19 +2479,168 @@ end -- of closure do -- create closure to overcome 200 locals limit +if not modules then modules = { } end modules ['l-url'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local char, gmatch, gsub = string.char, string.gmatch, string.gsub +local tonumber, type = tonumber, type +local lpegmatch = lpeg.match + +-- from the spec (on the web): +-- +-- foo://example.com:8042/over/there?name=ferret#nose +-- \_/ \______________/\_________/ \_________/ \__/ +-- | | | | | +-- scheme authority path query fragment +-- | _____________________|__ +-- / \ / \ +-- urn:example:animal:ferret:nose + +url = url or { } + +local function tochar(s) + return char(tonumber(s,16)) +end + +local colon, qmark, hash, slash, percent, endofstring = lpeg.P(":"), lpeg.P("?"), lpeg.P("#"), lpeg.P("/"), lpeg.P("%"), lpeg.P(-1) + +local hexdigit = lpeg.R("09","AF","af") +local plus = lpeg.P("+") +local escaped = (plus / " ") + (percent * lpeg.C(hexdigit * hexdigit) / tochar) + +-- we assume schemes with more than 1 character (in order to avoid problems with windows disks) + +local scheme = lpeg.Cs((escaped+(1-colon-slash-qmark-hash))^2) * colon + lpeg.Cc("") +local authority = slash * slash * lpeg.Cs((escaped+(1- slash-qmark-hash))^0) + lpeg.Cc("") +local path = slash * lpeg.Cs((escaped+(1- qmark-hash))^0) + lpeg.Cc("") +local query = qmark * lpeg.Cs((escaped+(1- hash))^0) + lpeg.Cc("") +local fragment = hash * lpeg.Cs((escaped+(1- endofstring))^0) + lpeg.Cc("") + +local parser = lpeg.Ct(scheme * authority * path * query * fragment) + +-- todo: reconsider Ct as we can as well have five return values (saves a table) +-- so we can have two parsers, one with and one without + +function url.split(str) + return (type(str) == "string" and lpegmatch(parser,str)) or str +end + +-- todo: cache them + +function url.hashed(str) + local s = url.split(str) + local somescheme = s[1] ~= "" + return { + scheme = (somescheme and s[1]) or "file", + authority = s[2], + path = s[3], + query = s[4], + fragment = s[5], + original = str, + noscheme = not somescheme, + } +end + +function url.hasscheme(str) + return url.split(str)[1] ~= "" +end + +function url.addscheme(str,scheme) + return (url.hasscheme(str) and str) or ((scheme or "file:///") .. str) +end + +function url.construct(hash) + local fullurl = hash.sheme .. "://".. hash.authority .. hash.path + if hash.query then + fullurl = fullurl .. "?".. hash.query + end + if hash.fragment then + fullurl = fullurl .. "?".. hash.fragment + end + return fullurl +end + +function url.filename(filename) + local t = url.hashed(filename) + return (t.scheme == "file" and (gsub(t.path,"^/([a-zA-Z])([:|])/)","%1:"))) or filename +end + +function url.query(str) + if type(str) == "string" then + local t = { } + for k, v in gmatch(str,"([^&=]*)=([^&=]*)") do + t[k] = v + end + return t + else + return str + end +end + +--~ print(url.filename("file:///c:/oeps.txt")) +--~ print(url.filename("c:/oeps.txt")) +--~ print(url.filename("file:///oeps.txt")) +--~ print(url.filename("file:///etc/test.txt")) +--~ print(url.filename("/oeps.txt")) + +--~ from the spec on the web (sort of): +--~ +--~ function test(str) +--~ print(table.serialize(url.hashed(str))) +--~ end +--~ +--~ test("%56pass%20words") +--~ test("file:///c:/oeps.txt") +--~ test("file:///c|/oeps.txt") +--~ test("file:///etc/oeps.txt") +--~ test("file://./etc/oeps.txt") +--~ test("file:////etc/oeps.txt") +--~ test("ftp://ftp.is.co.za/rfc/rfc1808.txt") +--~ test("http://www.ietf.org/rfc/rfc2396.txt") +--~ test("ldap://[2001:db8::7]/c=GB?objectClass?one#what") +--~ test("mailto:John.Doe@example.com") +--~ test("news:comp.infosystems.www.servers.unix") +--~ test("tel:+1-816-555-1212") +--~ test("telnet://192.0.2.16:80/") +--~ test("urn:oasis:names:specification:docbook:dtd:xml:4.1.2") +--~ test("/etc/passwords") +--~ test("http://www.pragma-ade.com/spaced%20name") + +--~ test("zip:///oeps/oeps.zip#bla/bla.tex") +--~ test("zip:///oeps/oeps.zip?bla/bla.tex") + + +end -- of closure + +do -- create closure to overcome 200 locals limit + if not modules then modules = { } end modules ['l-dir'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } +-- dir.expand_name will be merged with cleanpath and collapsepath + local type = type -local find, gmatch = string.find, string.gmatch +local find, gmatch, match, gsub = string.find, string.gmatch, string.match, string.gsub +local lpegmatch = lpeg.match dir = dir or { } +-- handy + +function dir.current() + return (gsub(lfs.currentdir(),"\\","/")) +end + -- optimizing for no string.find (*) does not save time local attributes = lfs.attributes @@ -2068,6 +2671,35 @@ end dir.glob_pattern = glob_pattern +local function collect_pattern(path,patt,recurse,result) + local ok, scanner + result = result or { } + if path == "/" then + ok, scanner = xpcall(function() return walkdir(path..".") end, function() end) -- kepler safe + else + ok, scanner = xpcall(function() return walkdir(path) end, function() end) -- kepler safe + end + if ok and type(scanner) == "function" then + if not find(path,"/$") then path = path .. '/' end + for name in scanner do + local full = path .. name + local attr = attributes(full) + local mode = attr.mode + if mode == 'file' then + if find(full,patt) then + result[name] = attr + end + elseif recurse and (mode == "directory") and (name ~= '.') and (name ~= "..") then + attr.list = collect_pattern(full,patt,recurse) + result[name] = attr + end + end + end + return result +end + +dir.collect_pattern = collect_pattern + local P, S, R, C, Cc, Cs, Ct, Cv, V = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.Cc, lpeg.Cs, lpeg.Ct, lpeg.Cv, lpeg.V local pattern = Ct { @@ -2087,29 +2719,48 @@ local filter = Cs ( ( )^0 ) local function glob(str,t) - if type(str) == "table" then - local t = t or { } - for s=1,#str do - glob(str[s],t) + if type(t) == "function" then + if type(str) == "table" then + for s=1,#str do + glob(str[s],t) + end + elseif lfs.isfile(str) then + t(str) + else + local split = lpegmatch(pattern,str) + if split then + local root, path, base = split[1], split[2], split[3] + local recurse = find(base,"%*%*") + local start = root .. path + local result = lpegmatch(filter,start .. base) + glob_pattern(start,result,recurse,t) + end end - return t - elseif lfs.isfile(str) then - local t = t or { } - t[#t+1] = str - return t else - local split = pattern:match(str) - if split then + if type(str) == "table" then local t = t or { } - local action = action or function(name) t[#t+1] = name end - local root, path, base = split[1], split[2], split[3] - local recurse = find(base,"%*%*") - local start = root .. path - local result = filter:match(start .. base) - glob_pattern(start,result,recurse,action) + for s=1,#str do + glob(str[s],t) + end + return t + elseif lfs.isfile(str) then + local t = t or { } + t[#t+1] = str return t else - return { } + local split = lpegmatch(pattern,str) + if split then + local t = t or { } + local action = action or function(name) t[#t+1] = name end + local root, path, base = split[1], split[2], split[3] + local recurse = find(base,"%*%*") + local start = root .. path + local result = lpegmatch(filter,start .. base) + glob_pattern(start,result,recurse,action) + return t + else + return { } + end end end end @@ -2171,11 +2822,12 @@ end local make_indeed = true -- false -if string.find(os.getenv("PATH"),";") then +if string.find(os.getenv("PATH"),";") then -- os.type == "windows" function dir.mkdirs(...) - local str, pth = "", "" - for _, s in ipairs({...}) do + local str, pth, t = "", "", { ... } + for i=1,#t do + local s = t[i] if s ~= "" then if str ~= "" then str = str .. "/" .. s @@ -2186,13 +2838,13 @@ if string.find(os.getenv("PATH"),";") then end local first, middle, last local drive = false - first, middle, last = str:match("^(//)(//*)(.*)$") + first, middle, last = match(str,"^(//)(//*)(.*)$") if first then -- empty network path == local path else - first, last = str:match("^(//)/*(.-)$") + first, last = match(str,"^(//)/*(.-)$") if first then - middle, last = str:match("([^/]+)/+(.-)$") + middle, last = match(str,"([^/]+)/+(.-)$") if middle then pth = "//" .. middle else @@ -2200,11 +2852,11 @@ if string.find(os.getenv("PATH"),";") then last = "" end else - first, middle, last = str:match("^([a-zA-Z]:)(/*)(.-)$") + first, middle, last = match(str,"^([a-zA-Z]:)(/*)(.-)$") if first then pth, drive = first .. middle, true else - middle, last = str:match("^(/*)(.-)$") + middle, last = match(str,"^(/*)(.-)$") if not middle then last = str end @@ -2238,34 +2890,31 @@ if string.find(os.getenv("PATH"),";") then --~ print(dir.mkdirs("///a/b/c")) --~ print(dir.mkdirs("a/bbb//ccc/")) - function dir.expand_name(str) - local first, nothing, last = str:match("^(//)(//*)(.*)$") + function dir.expand_name(str) -- will be merged with cleanpath and collapsepath + local first, nothing, last = match(str,"^(//)(//*)(.*)$") if first then - first = lfs.currentdir() .. "/" - first = first:gsub("\\","/") + first = dir.current() .. "/" end if not first then - first, last = str:match("^(//)/*(.*)$") + first, last = match(str,"^(//)/*(.*)$") end if not first then - first, last = str:match("^([a-zA-Z]:)(.*)$") + first, last = match(str,"^([a-zA-Z]:)(.*)$") if first and not find(last,"^/") then local d = lfs.currentdir() if lfs.chdir(first) then - first = lfs.currentdir() - first = first:gsub("\\","/") + first = dir.current() end lfs.chdir(d) end end if not first then - first, last = lfs.currentdir(), str - first = first:gsub("\\","/") + first, last = dir.current(), str end - last = last:gsub("//","/") - last = last:gsub("/%./","/") - last = last:gsub("^/*","") - first = first:gsub("/*$","") + last = gsub(last,"//","/") + last = gsub(last,"/%./","/") + last = gsub(last,"^/*","") + first = gsub(first,"/*$","") if last == "" then return first else @@ -2276,8 +2925,9 @@ if string.find(os.getenv("PATH"),";") then else function dir.mkdirs(...) - local str, pth = "", "" - for _, s in ipairs({...}) do + local str, pth, t = "", "", { ... } + for i=1,#t do + local s = t[i] if s ~= "" then if str ~= "" then str = str .. "/" .. s @@ -2286,7 +2936,7 @@ else end end end - str = str:gsub("/+","/") + str = gsub(str,"/+","/") if find(str,"^/") then pth = "/" for s in gmatch(str,"[^/]+") do @@ -2320,12 +2970,12 @@ else --~ print(dir.mkdirs("///a/b/c")) --~ print(dir.mkdirs("a/bbb//ccc/")) - function dir.expand_name(str) + function dir.expand_name(str) -- will be merged with cleanpath and collapsepath if not find(str,"^/") then str = lfs.currentdir() .. "/" .. str end - str = str:gsub("//","/") - str = str:gsub("/%./","/") + str = gsub(str,"//","/") + str = gsub(str,"/%./","/") return str end @@ -2340,7 +2990,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-boolean'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -2401,7 +3051,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-math'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -2448,7 +3098,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-utils'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -2456,6 +3106,10 @@ if not modules then modules = { } end modules ['l-utils'] = { -- hm, quite unreadable +local gsub = string.gsub +local concat = table.concat +local type, next = type, next + if not utils then utils = { } end if not utils.merger then utils.merger = { } end if not utils.lua then utils.lua = { } end @@ -2493,7 +3147,7 @@ function utils.merger._self_load_(name) end if data and utils.merger.strip_comment then -- saves some 20K - data = data:gsub("%-%-~[^\n\r]*[\r\n]", "") + data = gsub(data,"%-%-~[^\n\r]*[\r\n]", "") end return data or "" end @@ -2511,7 +3165,7 @@ end function utils.merger._self_swap_(data,code) if data ~= "" then - return (data:gsub(utils.merger.pattern, function(s) + return (gsub(data,utils.merger.pattern, function(s) return "\n\n" .. "-- "..utils.merger.m_begin .. "\n" .. code .. "\n" .. "-- "..utils.merger.m_end .. "\n\n" end, 1)) else @@ -2521,8 +3175,8 @@ end --~ stripper: --~ ---~ data = string.gsub(data,"%-%-~[^\n]*\n","") ---~ data = string.gsub(data,"\n\n+","\n") +--~ data = gsub(data,"%-%-~[^\n]*\n","") +--~ data = gsub(data,"\n\n+","\n") function utils.merger._self_libs_(libs,list) local result, f, frozen = { }, nil, false @@ -2530,9 +3184,10 @@ function utils.merger._self_libs_(libs,list) if type(libs) == 'string' then libs = { libs } end if type(list) == 'string' then list = { list } end local foundpath = nil - for _, lib in ipairs(libs) do - for _, pth in ipairs(list) do - pth = string.gsub(pth,"\\","/") -- file.clean_path + for i=1,#libs do + local lib = libs[i] + for j=1,#list do + local pth = gsub(list[j],"\\","/") -- file.clean_path utils.report("checking library path %s",pth) local name = pth .. "/" .. lib if lfs.isfile(name) then @@ -2544,7 +3199,8 @@ function utils.merger._self_libs_(libs,list) if foundpath then utils.report("using library path %s",foundpath) local right, wrong = { }, { } - for _, lib in ipairs(libs) do + for i=1,#libs do + local lib = libs[i] local fullname = foundpath .. "/" .. lib if lfs.isfile(fullname) then -- right[#right+1] = lib @@ -2558,15 +3214,15 @@ function utils.merger._self_libs_(libs,list) end end if #right > 0 then - utils.report("merged libraries: %s",table.concat(right," ")) + utils.report("merged libraries: %s",concat(right," ")) end if #wrong > 0 then - utils.report("skipped libraries: %s",table.concat(wrong," ")) + utils.report("skipped libraries: %s",concat(wrong," ")) end else utils.report("no valid library path found") end - return table.concat(result, "\n\n") + return concat(result, "\n\n") end function utils.merger.selfcreate(libs,list,target) @@ -2624,16 +3280,28 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-aux'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } +-- for inline, no store split : for s in string.gmatch(str,",* *([^,]+)") do .. end + aux = aux or { } local concat, format, gmatch = table.concat, string.format, string.gmatch local tostring, type = tostring, type +local lpegmatch = lpeg.match + +local P, R, V = lpeg.P, lpeg.R, lpeg.V + +local escape, left, right = P("\\"), P('{'), P('}') + +lpeg.patterns.balanced = P { + [1] = ((escape * (left+right)) + (1 - (left+right)) + V(2))^0, + [2] = left * V(1) * right +} local space = lpeg.P(' ') local equal = lpeg.P("=") @@ -2641,7 +3309,7 @@ local comma = lpeg.P(",") local lbrace = lpeg.P("{") local rbrace = lpeg.P("}") local nobrace = 1 - (lbrace+rbrace) -local nested = lpeg.P{ lbrace * (nobrace + lpeg.V(1))^0 * rbrace } +local nested = lpeg.P { lbrace * (nobrace + lpeg.V(1))^0 * rbrace } local spaces = space^0 local value = lpeg.P(lbrace * lpeg.C((nobrace + nested)^0) * rbrace) + lpeg.C((nested + (1-comma))^0) @@ -2679,13 +3347,13 @@ function aux.make_settings_to_hash_pattern(set,how) end end -function aux.settings_to_hash(str) +function aux.settings_to_hash(str,existing) if str and str ~= "" then - hash = { } + hash = existing or { } if moretolerant then - pattern_b_s:match(str) + lpegmatch(pattern_b_s,str) else - pattern_a_s:match(str) + lpegmatch(pattern_a_s,str) end return hash else @@ -2693,39 +3361,41 @@ function aux.settings_to_hash(str) end end -function aux.settings_to_hash_tolerant(str) +function aux.settings_to_hash_tolerant(str,existing) if str and str ~= "" then - hash = { } - pattern_b_s:match(str) + hash = existing or { } + lpegmatch(pattern_b_s,str) return hash else return { } end end -function aux.settings_to_hash_strict(str) +function aux.settings_to_hash_strict(str,existing) if str and str ~= "" then - hash = { } - pattern_c_s:match(str) + hash = existing or { } + lpegmatch(pattern_c_s,str) return next(hash) and hash else return nil end end -local seperator = comma * space^0 +local separator = comma * space^0 local value = lpeg.P(lbrace * lpeg.C((nobrace + nested)^0) * rbrace) + lpeg.C((nested + (1-comma))^0) -local pattern = lpeg.Ct(value*(seperator*value)^0) +local pattern = lpeg.Ct(value*(separator*value)^0) -- "aap, {noot}, mies" : outer {} removes, leading spaces ignored aux.settings_to_array_pattern = pattern +-- we could use a weak table as cache + function aux.settings_to_array(str) if not str or str == "" then return { } else - return pattern:match(str) + return lpegmatch(pattern,str) end end @@ -2734,10 +3404,10 @@ local function set(t,v) end local value = lpeg.P(lpeg.Carg(1)*value) / set -local pattern = value*(seperator*value)^0 * lpeg.Carg(1) +local pattern = value*(separator*value)^0 * lpeg.Carg(1) function aux.add_settings_to_array(t,str) - return pattern:match(str, nil, t) + return lpegmatch(pattern,str,nil,t) end function aux.hash_to_string(h,separator,yes,no,strict,omit) @@ -2785,6 +3455,13 @@ function aux.settings_to_set(str,t) return t end +local value = lbrace * lpeg.C((nobrace + nested)^0) * rbrace +local pattern = lpeg.Ct((space + value)^0) + +function aux.arguments_to_table(str) + return lpegmatch(pattern,str) +end + -- temporary here function aux.getparameters(self,class,parentclass,settings) @@ -2793,36 +3470,31 @@ function aux.getparameters(self,class,parentclass,settings) sc = table.clone(self[parent]) self[class] = sc end - aux.add_settings_to_array(sc, settings) + aux.settings_to_hash(settings,sc) end -- temporary here -local digit = lpeg.R("09") -local period = lpeg.P(".") -local zero = lpeg.P("0") - ---~ local finish = lpeg.P(-1) ---~ local nodigit = (1-digit) + finish ---~ local case_1 = (period * zero^1 * #nodigit)/"" -- .000 ---~ local case_2 = (period * (1-(zero^0/"") * #nodigit)^1 * (zero^0/"") * nodigit) -- .010 .10 .100100 - +local digit = lpeg.R("09") +local period = lpeg.P(".") +local zero = lpeg.P("0") local trailingzeros = zero^0 * -digit -- suggested by Roberto R -local case_1 = period * trailingzeros / "" -local case_2 = period * (digit - trailingzeros)^1 * (trailingzeros / "") - -local number = digit^1 * (case_1 + case_2) -local stripper = lpeg.Cs((number + 1)^0) +local case_1 = period * trailingzeros / "" +local case_2 = period * (digit - trailingzeros)^1 * (trailingzeros / "") +local number = digit^1 * (case_1 + case_2) +local stripper = lpeg.Cs((number + 1)^0) --~ local sample = "bla 11.00 bla 11 bla 0.1100 bla 1.00100 bla 0.00 bla 0.001 bla 1.1100 bla 0.100100100 bla 0.00100100100" --~ collectgarbage("collect") --~ str = string.rep(sample,10000) --~ local ts = os.clock() ---~ stripper:match(str) ---~ print(#str, os.clock()-ts, stripper:match(sample)) +--~ lpegmatch(stripper,str) +--~ print(#str, os.clock()-ts, lpegmatch(stripper,sample)) + +lpeg.patterns.strip_zeros = stripper function aux.strip_zeros(str) - return stripper:match(str) + return lpegmatch(stripper,str) end function aux.definetable(target) -- defines undefined tables @@ -2846,6 +3518,375 @@ function aux.accesstable(target) return t end +--~ function string.commaseparated(str) +--~ return gmatch(str,"([^,%s]+)") +--~ end + +-- as we use this a lot ... + +--~ function aux.cachefunction(action,weak) +--~ local cache = { } +--~ if weak then +--~ setmetatable(cache, { __mode = "kv" } ) +--~ end +--~ local function reminder(str) +--~ local found = cache[str] +--~ if not found then +--~ found = action(str) +--~ cache[str] = found +--~ end +--~ return found +--~ end +--~ return reminder, cache +--~ end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['trac-tra'] = { + version = 1.001, + comment = "companion to trac-tra.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- the <anonymous> tag is kind of generic and used for functions that are not +-- bound to a variable, like node.new, node.copy etc (contrary to for instance +-- node.has_attribute which is bound to a has_attribute local variable in mkiv) + +local debug = require "debug" + +local getinfo = debug.getinfo +local type, next = type, next +local concat = table.concat +local format, find, lower, gmatch, gsub = string.format, string.find, string.lower, string.gmatch, string.gsub + +debugger = debugger or { } + +local counters = { } +local names = { } + +-- one + +local function hook() + local f = getinfo(2,"f").func + local n = getinfo(2,"Sn") +-- if n.what == "C" and n.name then print (n.namewhat .. ': ' .. n.name) end + if f then + local cf = counters[f] + if cf == nil then + counters[f] = 1 + names[f] = n + else + counters[f] = cf + 1 + end + end +end +local function getname(func) + local n = names[func] + if n then + if n.what == "C" then + return n.name or '<anonymous>' + else + -- source short_src linedefined what name namewhat nups func + local name = n.name or n.namewhat or n.what + if not name or name == "" then name = "?" end + return format("%s : %s : %s", n.short_src or "unknown source", n.linedefined or "--", name) + end + else + return "unknown" + end +end +function debugger.showstats(printer,threshold) + printer = printer or texio.write or print + threshold = threshold or 0 + local total, grandtotal, functions = 0, 0, 0 + printer("\n") -- ugly but ok + -- table.sort(counters) + for func, count in next, counters do + if count > threshold then + local name = getname(func) + if not find(name,"for generator") then + printer(format("%8i %s", count, name)) + total = total + count + end + end + grandtotal = grandtotal + count + functions = functions + 1 + end + printer(format("functions: %s, total: %s, grand total: %s, threshold: %s\n", functions, total, grandtotal, threshold)) +end + +-- two + +--~ local function hook() +--~ local n = getinfo(2) +--~ if n.what=="C" and not n.name then +--~ local f = tostring(debug.traceback()) +--~ local cf = counters[f] +--~ if cf == nil then +--~ counters[f] = 1 +--~ names[f] = n +--~ else +--~ counters[f] = cf + 1 +--~ end +--~ end +--~ end +--~ function debugger.showstats(printer,threshold) +--~ printer = printer or texio.write or print +--~ threshold = threshold or 0 +--~ local total, grandtotal, functions = 0, 0, 0 +--~ printer("\n") -- ugly but ok +--~ -- table.sort(counters) +--~ for func, count in next, counters do +--~ if count > threshold then +--~ printer(format("%8i %s", count, func)) +--~ total = total + count +--~ end +--~ grandtotal = grandtotal + count +--~ functions = functions + 1 +--~ end +--~ printer(format("functions: %s, total: %s, grand total: %s, threshold: %s\n", functions, total, grandtotal, threshold)) +--~ end + +-- rest + +function debugger.savestats(filename,threshold) + local f = io.open(filename,'w') + if f then + debugger.showstats(function(str) f:write(str) end,threshold) + f:close() + end +end + +function debugger.enable() + debug.sethook(hook,"c") +end + +function debugger.disable() + debug.sethook() +--~ counters[debug.getinfo(2,"f").func] = nil +end + +function debugger.tracing() + local n = tonumber(os.env['MTX.TRACE.CALLS']) or tonumber(os.env['MTX_TRACE_CALLS']) or 0 + if n > 0 then + function debugger.tracing() return true end ; return true + else + function debugger.tracing() return false end ; return false + end +end + +--~ debugger.enable() + +--~ print(math.sin(1*.5)) +--~ print(math.sin(1*.5)) +--~ print(math.sin(1*.5)) +--~ print(math.sin(1*.5)) +--~ print(math.sin(1*.5)) + +--~ debugger.disable() + +--~ print("") +--~ debugger.showstats() +--~ print("") +--~ debugger.showstats(print,3) + +setters = setters or { } +setters.data = setters.data or { } + +--~ local function set(t,what,value) +--~ local data, done = t.data, t.done +--~ if type(what) == "string" then +--~ what = aux.settings_to_array(what) -- inefficient but ok +--~ end +--~ for i=1,#what do +--~ local w = what[i] +--~ for d, f in next, data do +--~ if done[d] then +--~ -- prevent recursion due to wildcards +--~ elseif find(d,w) then +--~ done[d] = true +--~ for i=1,#f do +--~ f[i](value) +--~ end +--~ end +--~ end +--~ end +--~ end + +local function set(t,what,value) + local data, done = t.data, t.done + if type(what) == "string" then + what = aux.settings_to_hash(what) -- inefficient but ok + end + for w, v in next, what do + if v == "" then + v = value + else + v = toboolean(v) + end + for d, f in next, data do + if done[d] then + -- prevent recursion due to wildcards + elseif find(d,w) then + done[d] = true + for i=1,#f do + f[i](v) + end + end + end + end +end + +local function reset(t) + for d, f in next, t.data do + for i=1,#f do + f[i](false) + end + end +end + +local function enable(t,what) + set(t,what,true) +end + +local function disable(t,what) + local data = t.data + if not what or what == "" then + t.done = { } + reset(t) + else + set(t,what,false) + end +end + +function setters.register(t,what,...) + local data = t.data + what = lower(what) + local w = data[what] + if not w then + w = { } + data[what] = w + end + for _, fnc in next, { ... } do + local typ = type(fnc) + if typ == "function" then + w[#w+1] = fnc + elseif typ == "string" then + w[#w+1] = function(value) set(t,fnc,value,nesting) end + end + end +end + +function setters.enable(t,what) + local e = t.enable + t.enable, t.done = enable, { } + enable(t,string.simpleesc(tostring(what))) + t.enable, t.done = e, { } +end + +function setters.disable(t,what) + local e = t.disable + t.disable, t.done = disable, { } + disable(t,string.simpleesc(tostring(what))) + t.disable, t.done = e, { } +end + +function setters.reset(t) + t.done = { } + reset(t) +end + +function setters.list(t) -- pattern + local list = table.sortedkeys(t.data) + local user, system = { }, { } + for l=1,#list do + local what = list[l] + if find(what,"^%*") then + system[#system+1] = what + else + user[#user+1] = what + end + end + return user, system +end + +function setters.show(t) + commands.writestatus("","") + local list = setters.list(t) + for k=1,#list do + commands.writestatus(t.name,list[k]) + end + commands.writestatus("","") +end + +-- we could have used a bit of oo and the trackers:enable syntax but +-- there is already a lot of code around using the singular tracker + +-- we could make this into a module + +function setters.new(name) + local t + t = { + data = { }, + name = name, + enable = function(...) setters.enable (t,...) end, + disable = function(...) setters.disable (t,...) end, + register = function(...) setters.register(t,...) end, + list = function(...) setters.list (t,...) end, + show = function(...) setters.show (t,...) end, + } + setters.data[name] = t + return t +end + +trackers = setters.new("trackers") +directives = setters.new("directives") +experiments = setters.new("experiments") + +-- nice trick: we overload two of the directives related functions with variants that +-- do tracing (itself using a tracker) .. proof of concept + +local trace_directives = false local trace_directives = false trackers.register("system.directives", function(v) trace_directives = v end) +local trace_experiments = false local trace_experiments = false trackers.register("system.experiments", function(v) trace_experiments = v end) + +local e = directives.enable +local d = directives.disable + +function directives.enable(...) + commands.writestatus("directives","enabling: %s",concat({...}," ")) + e(...) +end + +function directives.disable(...) + commands.writestatus("directives","disabling: %s",concat({...}," ")) + d(...) +end + +local e = experiments.enable +local d = experiments.disable + +function experiments.enable(...) + commands.writestatus("experiments","enabling: %s",concat({...}," ")) + e(...) +end + +function experiments.disable(...) + commands.writestatus("experiments","disabling: %s",concat({...}," ")) + d(...) +end + +-- a useful example + +directives.register("system.nostatistics", function(v) + statistics.enable = not v +end) + + end -- of closure @@ -2859,6 +3900,12 @@ if not modules then modules = { } end modules ['lxml-tab'] = { license = "see context related readme files" } +-- this module needs a cleanup: check latest lpeg, passing args, (sub)grammar, etc etc +-- stripping spaces from e.g. cont-en.xml saves .2 sec runtime so it's not worth the +-- trouble + +local trace_entities = false trackers.register("xml.entities", function(v) trace_entities = v end) + --[[ldx-- <p>The parser used here is inspired by the variant discussed in the lua book, but handles comment and processing instructions, has a different structure, provides @@ -2866,18 +3913,6 @@ parent access; a first version used different trickery but was less optimized to went this route. First we had a find based parser, now we have an <l n='lpeg'/> based one. The find based parser can be found in l-xml-edu.lua along with other older code.</p> -<p>Expecially the lpath code is experimental, we will support some of xpath, but -only things that make sense for us; as compensation it is possible to hook in your -own functions. Apart from preprocessing content for <l n='context'/> we also need -this module for process management, like handling <l n='ctx'/> and <l n='rlx'/> -files.</p> - -<typing> -a/b/c /*/c -a/b/c/first() a/b/c/last() a/b/c/index(n) a/b/c/index(-n) -a/b/c/text() a/b/c/text(1) a/b/c/text(-1) a/b/c/text(n) -</typing> - <p>Beware, the interface may change. For instance at, ns, tg, dt may get more verbose names. Once the code is stable we will also remove some tracing and optimize the code.</p> @@ -2888,26 +3923,11 @@ xml = xml or { } --~ local xml = xml local concat, remove, insert = table.concat, table.remove, table.insert -local type, next, setmetatable = type, next, setmetatable -local format, lower, find = string.format, string.lower, string.find - ---[[ldx-- -<p>This module can be used stand alone but also inside <l n='mkiv'/> in -which case it hooks into the tracker code. Therefore we provide a few -functions that set the tracers.</p> ---ldx]]-- - -local trace_remap = false - -if trackers then - trackers.register("xml.remap", function(v) trace_remap = v end) -end - -function xml.settrace(str,value) - if str == "remap" then - trace_remap = value or false - end -end +local type, next, setmetatable, getmetatable, tonumber = type, next, setmetatable, getmetatable, tonumber +local format, lower, find, match, gsub = string.format, string.lower, string.find, string.match, string.gsub +local utfchar = unicode.utf8.char +local lpegmatch = lpeg.match +local P, S, R, C, V, C, Cs = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.C, lpeg.Cs --[[ldx-- <p>First a hack to enable namespace resolving. A namespace is characterized by @@ -2919,7 +3939,7 @@ much cleaner.</p> xml.xmlns = xml.xmlns or { } -local check = lpeg.P(false) +local check = P(false) local parse = check --[[ldx-- @@ -2932,8 +3952,8 @@ xml.registerns("mml","mathml") --ldx]]-- function xml.registerns(namespace, pattern) -- pattern can be an lpeg - check = check + lpeg.C(lpeg.P(lower(pattern))) / namespace - parse = lpeg.P { lpeg.P(check) + 1 * lpeg.V(1) } + check = check + C(P(lower(pattern))) / namespace + parse = P { P(check) + 1 * V(1) } end --[[ldx-- @@ -2947,7 +3967,7 @@ xml.checkns("m","http://www.w3.org/mathml") --ldx]]-- function xml.checkns(namespace,url) - local ns = parse:match(lower(url)) + local ns = lpegmatch(parse,lower(url)) if ns and namespace ~= ns then xml.xmlns[namespace] = ns end @@ -2965,7 +3985,7 @@ This returns <t>mml</t>. --ldx]]-- function xml.resolvens(url) - return parse:match(lower(url)) or "" + return lpegmatch(parse,lower(url)) or "" end --[[ldx-- @@ -3004,27 +4024,36 @@ local x = xml.convert(somestring) <p>An optional second boolean argument tells this function not to create a root element.</p> ---ldx]]-- -xml.strip_cm_and_dt = false -- an extra global flag, in case we have many includes +<p>Valid entities are:</p> + +<typing> +<!ENTITY xxxx SYSTEM "yyyy" NDATA zzzz> +<!ENTITY xxxx PUBLIC "yyyy" > +<!ENTITY xxxx "yyyy" > +</typing> +--ldx]]-- -- not just one big nested table capture (lpeg overflow) local nsremap, resolvens = xml.xmlns, xml.resolvens -local stack, top, dt, at, xmlns, errorstr, entities = {}, {}, {}, {}, {}, nil, {} +local stack, top, dt, at, xmlns, errorstr, entities = { }, { }, { }, { }, { }, nil, { } +local strip, cleanup, utfize, resolve, resolve_predefined, unify_predefined = false, false, false, false, false, false +local dcache, hcache, acache = { }, { }, { } -local mt = { __tostring = xml.text } +local mt = { } -function xml.check_error(top,toclose) - return "" +function initialize_mt(root) + mt = { __index = root } -- will be redefined later end -local strip = false -local cleanup = false +function xml.setproperty(root,k,v) + getmetatable(root).__index[k] = v +end -function xml.set_text_cleanup(fnc) - cleanup = fnc +function xml.check_error(top,toclose) + return "" end local function add_attribute(namespace,tag,value) @@ -3034,12 +4063,31 @@ local function add_attribute(namespace,tag,value) if tag == "xmlns" then xmlns[#xmlns+1] = resolvens(value) at[tag] = value + elseif namespace == "" then + at[tag] = value elseif namespace == "xmlns" then xml.checkns(tag,value) at["xmlns:" .. tag] = value else - at[tag] = value + -- for the moment this way: + at[namespace .. ":" .. tag] = value + end +end + +local function add_empty(spacing, namespace, tag) + if #spacing > 0 then + dt[#dt+1] = spacing + end + local resolved = (namespace == "" and xmlns[#xmlns]) or nsremap[namespace] or namespace + top = stack[#stack] + dt = top.dt + local t = { ns=namespace or "", rn=resolved, tg=tag, at=at, dt={}, __p__ = top } + dt[#dt+1] = t + setmetatable(t, mt) + if at.xmlns then + remove(xmlns) end + at = { } end local function add_begin(spacing, namespace, tag) @@ -3067,28 +4115,12 @@ local function add_end(spacing, namespace, tag) end dt = top.dt dt[#dt+1] = toclose - dt[0] = top + -- dt[0] = top -- nasty circular reference when serializing table if toclose.at.xmlns then remove(xmlns) end end -local function add_empty(spacing, namespace, tag) - if #spacing > 0 then - dt[#dt+1] = spacing - end - local resolved = (namespace == "" and xmlns[#xmlns]) or nsremap[namespace] or namespace - top = stack[#stack] - dt = top.dt - local t = { ns=namespace or "", rn=resolved, tg=tag, at=at, dt={}, __p__ = top } - dt[#dt+1] = t - setmetatable(t, mt) - if at.xmlns then - remove(xmlns) - end - at = { } -end - local function add_text(text) if cleanup and #text > 0 then dt[#dt+1] = cleanup(text) @@ -3104,7 +4136,7 @@ local function add_special(what, spacing, text) if strip and (what == "@cm@" or what == "@dt@") then -- forget it else - dt[#dt+1] = { special=true, ns="", tg=what, dt={text} } + dt[#dt+1] = { special=true, ns="", tg=what, dt={ text } } end end @@ -3112,7 +4144,199 @@ local function set_message(txt) errorstr = "garbage at the end of the file: " .. gsub(txt,"([ \n\r\t]*)","") end -local P, S, R, C, V = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V +local reported_attribute_errors = { } + +local function attribute_value_error(str) + if not reported_attribute_errors[str] then + logs.report("xml","invalid attribute value: %q",str) + reported_attribute_errors[str] = true + at._error_ = str + end + return str +end +local function attribute_specification_error(str) + if not reported_attribute_errors[str] then + logs.report("xml","invalid attribute specification: %q",str) + reported_attribute_errors[str] = true + at._error_ = str + end + return str +end + +function xml.unknown_dec_entity_format(str) return (str == "" and "&error;") or format("&%s;",str) end +function xml.unknown_hex_entity_format(str) return format("&#x%s;",str) end +function xml.unknown_any_entity_format(str) return format("&#x%s;",str) end + +local function fromhex(s) + local n = tonumber(s,16) + if n then + return utfchar(n) + else + return format("h:%s",s), true + end +end + +local function fromdec(s) + local n = tonumber(s) + if n then + return utfchar(n) + else + return format("d:%s",s), true + end +end + +-- one level expansion (simple case), no checking done + +local rest = (1-P(";"))^0 +local many = P(1)^0 + +local parsedentity = + P("&") * (P("#x")*(rest/fromhex) + P("#")*(rest/fromdec)) * P(";") * P(-1) + + (P("#x")*(many/fromhex) + P("#")*(many/fromdec)) + +-- parsing in the xml file + +local predefined_unified = { + [38] = "&", + [42] = """, + [47] = "'", + [74] = "<", + [76] = "&gr;", +} + +local predefined_simplified = { + [38] = "&", amp = "&", + [42] = '"', quot = '"', + [47] = "'", apos = "'", + [74] = "<", lt = "<", + [76] = ">", gt = ">", +} + +local function handle_hex_entity(str) + local h = hcache[str] + if not h then + local n = tonumber(str,16) + h = unify_predefined and predefined_unified[n] + if h then + if trace_entities then + logs.report("xml","utfize, converting hex entity &#x%s; into %s",str,h) + end + elseif utfize then + h = (n and utfchar(n)) or xml.unknown_hex_entity_format(str) or "" + if not n then + logs.report("xml","utfize, ignoring hex entity &#x%s;",str) + elseif trace_entities then + logs.report("xml","utfize, converting hex entity &#x%s; into %s",str,h) + end + else + if trace_entities then + logs.report("xml","found entity &#x%s;",str) + end + h = "&#x" .. str .. ";" + end + hcache[str] = h + end + return h +end + +local function handle_dec_entity(str) + local d = dcache[str] + if not d then + local n = tonumber(str) + d = unify_predefined and predefined_unified[n] + if d then + if trace_entities then + logs.report("xml","utfize, converting dec entity &#%s; into %s",str,d) + end + elseif utfize then + d = (n and utfchar(n)) or xml.unknown_dec_entity_format(str) or "" + if not n then + logs.report("xml","utfize, ignoring dec entity &#%s;",str) + elseif trace_entities then + logs.report("xml","utfize, converting dec entity &#%s; into %s",str,h) + end + else + if trace_entities then + logs.report("xml","found entity &#%s;",str) + end + d = "&#" .. str .. ";" + end + dcache[str] = d + end + return d +end + +xml.parsedentitylpeg = parsedentity + +local function handle_any_entity(str) + if resolve then + local a = acache[str] -- per instance ! todo + if not a then + a = resolve_predefined and predefined_simplified[str] + if a then + -- one of the predefined + elseif type(resolve) == "function" then + a = resolve(str) or entities[str] + else + a = entities[str] + end + if a then + if trace_entities then + logs.report("xml","resolved entity &%s; -> %s (internal)",str,a) + end + a = lpegmatch(parsedentity,a) or a + else + if xml.unknown_any_entity_format then + a = xml.unknown_any_entity_format(str) or "" + end + if a then + if trace_entities then + logs.report("xml","resolved entity &%s; -> %s (external)",str,a) + end + else + if trace_entities then + logs.report("xml","keeping entity &%s;",str) + end + if str == "" then + a = "&error;" + else + a = "&" .. str .. ";" + end + end + end + acache[str] = a + elseif trace_entities then + if not acache[str] then + logs.report("xml","converting entity &%s; into %s",str,a) + acache[str] = a + end + end + return a + else + local a = acache[str] + if not a then + if trace_entities then + logs.report("xml","found entity &%s;",str) + end + a = resolve_predefined and predefined_simplified[str] + if a then + -- one of the predefined + acache[str] = a + elseif str == "" then + a = "&error;" + acache[str] = a + else + a = "&" .. str .. ";" + acache[str] = a + end + end + return a + end +end + +local function handle_end_entity(chr) + logs.report("xml","error in entity, %q found instead of ';'",chr) +end local space = S(' \r\n\t') local open = P('<') @@ -3122,24 +4346,50 @@ local dquote = S('"') local equal = P('=') local slash = P('/') local colon = P(':') +local semicolon = P(';') +local ampersand = P('&') local valid = R('az', 'AZ', '09') + S('_-.') local name_yes = C(valid^1) * colon * C(valid^1) local name_nop = C(P(true)) * C(valid^1) local name = name_yes + name_nop +local utfbom = lpeg.patterns.utfbom -- no capture +local spacing = C(space^0) -local utfbom = P('\000\000\254\255') + P('\255\254\000\000') + - P('\255\254') + P('\254\255') + P('\239\187\191') -- no capture +----- entitycontent = (1-open-semicolon)^0 +local anyentitycontent = (1-open-semicolon-space-close)^0 +local hexentitycontent = R("AF","af","09")^0 +local decentitycontent = R("09")^0 +local parsedentity = P("#")/"" * ( + P("x")/"" * (hexentitycontent/handle_hex_entity) + + (decentitycontent/handle_dec_entity) + ) + (anyentitycontent/handle_any_entity) +local entity = ampersand/"" * parsedentity * ( (semicolon/"") + #(P(1)/handle_end_entity)) + +local text_unparsed = C((1-open)^1) +local text_parsed = Cs(((1-open-ampersand)^1 + entity)^1) -local spacing = C(space^0) -local justtext = C((1-open)^1) local somespace = space^1 local optionalspace = space^0 -local value = (squote * C((1 - squote)^0) * squote) + (dquote * C((1 - dquote)^0) * dquote) -local attribute = (somespace * name * optionalspace * equal * optionalspace * value) / add_attribute -local attributes = attribute^0 +----- value = (squote * C((1 - squote)^0) * squote) + (dquote * C((1 - dquote)^0) * dquote) -- ampersand and < also invalid in value +local value = (squote * Cs((entity + (1 - squote))^0) * squote) + (dquote * Cs((entity + (1 - dquote))^0) * dquote) -- ampersand and < also invalid in value + +local endofattributes = slash * close + close -- recovery of flacky html +local whatever = space * name * optionalspace * equal +local wrongvalue = C(P(1-whatever-close)^1 + P(1-close)^1) / attribute_value_error +----- wrongvalue = C(P(1-whatever-endofattributes)^1 + P(1-endofattributes)^1) / attribute_value_error +----- wrongvalue = C(P(1-space-endofattributes)^1) / attribute_value_error +local wrongvalue = Cs(P(entity + (1-space-endofattributes))^1) / attribute_value_error -local text = justtext / add_text +local attributevalue = value + wrongvalue + +local attribute = (somespace * name * optionalspace * equal * optionalspace * attributevalue) / add_attribute +----- attributes = (attribute)^0 + +local attributes = (attribute + somespace^-1 * (((1-endofattributes)^1)/attribute_specification_error))^0 + +local parsedtext = text_parsed / add_text +local unparsedtext = text_unparsed / add_text local balanced = P { "[" * ((1 - S"[]") + V(1))^0 * "]" } -- taken from lpeg manual, () example local emptyelement = (spacing * open * name * attributes * optionalspace * slash * close) / add_empty @@ -3157,19 +4407,27 @@ local someinstruction = C((1 - endinstruction)^0) local somecomment = C((1 - endcomment )^0) local somecdata = C((1 - endcdata )^0) -local function entity(k,v) entities[k] = v end +local function normalentity(k,v ) entities[k] = v end +local function systementity(k,v,n) entities[k] = v end +local function publicentity(k,v,n) entities[k] = v end local begindoctype = open * P("!DOCTYPE") local enddoctype = close local beginset = P("[") local endset = P("]") -local doctypename = C((1-somespace)^0) +local doctypename = C((1-somespace-close)^0) local elementdoctype = optionalspace * P("<!ELEMENT") * (1-close)^0 * close -local entitydoctype = optionalspace * P("<!ENTITY") * somespace * (doctypename * somespace * value)/entity * optionalspace * close -local publicdoctype = doctypename * somespace * P("PUBLIC") * somespace * value * somespace * value * somespace -local systemdoctype = doctypename * somespace * P("SYSTEM") * somespace * value * somespace -local definitiondoctype= doctypename * somespace * beginset * P(elementdoctype + entitydoctype)^0 * optionalspace * endset -local simpledoctype = (1-close)^1 -- * balanced^0 + +local normalentitytype = (doctypename * somespace * value)/normalentity +local publicentitytype = (doctypename * somespace * P("PUBLIC") * somespace * value)/publicentity +local systementitytype = (doctypename * somespace * P("SYSTEM") * somespace * value * somespace * P("NDATA") * somespace * doctypename)/systementity +local entitydoctype = optionalspace * P("<!ENTITY") * somespace * (systementitytype + publicentitytype + normalentitytype) * optionalspace * close + +local doctypeset = beginset * optionalspace * P(elementdoctype + entitydoctype + space)^0 * optionalspace * endset +local definitiondoctype= doctypename * somespace * doctypeset +local publicdoctype = doctypename * somespace * P("PUBLIC") * somespace * value * somespace * value * somespace * doctypeset +local systemdoctype = doctypename * somespace * P("SYSTEM") * somespace * value * somespace * doctypeset +local simpledoctype = (1-close)^1 -- * balanced^0 local somedoctype = C((somespace * (publicdoctype + systemdoctype + definitiondoctype + simpledoctype) * optionalspace)^0) local instruction = (spacing * begininstruction * someinstruction * endinstruction) / function(...) add_special("@pi@",...) end @@ -3179,60 +4437,116 @@ local doctype = (spacing * begindoctype * somedoctype * enddoct -- nicer but slower: -- --- local instruction = (lpeg.Cc("@pi@") * spacing * begininstruction * someinstruction * endinstruction) / add_special --- local comment = (lpeg.Cc("@cm@") * spacing * begincomment * somecomment * endcomment ) / add_special --- local cdata = (lpeg.Cc("@cd@") * spacing * begincdata * somecdata * endcdata ) / add_special --- local doctype = (lpeg.Cc("@dt@") * spacing * begindoctype * somedoctype * enddoctype ) / add_special +-- local instruction = (Cc("@pi@") * spacing * begininstruction * someinstruction * endinstruction) / add_special +-- local comment = (Cc("@cm@") * spacing * begincomment * somecomment * endcomment ) / add_special +-- local cdata = (Cc("@cd@") * spacing * begincdata * somecdata * endcdata ) / add_special +-- local doctype = (Cc("@dt@") * spacing * begindoctype * somedoctype * enddoctype ) / add_special -local trailer = space^0 * (justtext/set_message)^0 +local trailer = space^0 * (text_unparsed/set_message)^0 -- comment + emptyelement + text + cdata + instruction + V("parent"), -- 6.5 seconds on 40 MB database file -- text + comment + emptyelement + cdata + instruction + V("parent"), -- 5.8 -- text + V("parent") + emptyelement + comment + cdata + instruction, -- 5.5 -local grammar = P { "preamble", +local grammar_parsed_text = P { "preamble", preamble = utfbom^0 * instruction^0 * (doctype + comment + instruction)^0 * V("parent") * trailer, parent = beginelement * V("children")^0 * endelement, - children = text + V("parent") + emptyelement + comment + cdata + instruction, + children = parsedtext + V("parent") + emptyelement + comment + cdata + instruction, } --- todo: xml.new + properties like entities and strip and such (store in root) +local grammar_unparsed_text = P { "preamble", + preamble = utfbom^0 * instruction^0 * (doctype + comment + instruction)^0 * V("parent") * trailer, + parent = beginelement * V("children")^0 * endelement, + children = unparsedtext + V("parent") + emptyelement + comment + cdata + instruction, +} -function xml.convert(data, no_root, strip_cm_and_dt, given_entities) -- maybe use table met k/v (given_entities may disapear) - strip = strip_cm_and_dt or xml.strip_cm_and_dt - stack, top, at, xmlns, errorstr, result, entities = {}, {}, {}, {}, nil, nil, given_entities or {} +-- maybe we will add settinsg to result as well + +local function xmlconvert(data, settings) + settings = settings or { } -- no_root strip_cm_and_dt given_entities parent_root error_handler + strip = settings.strip_cm_and_dt + utfize = settings.utfize_entities + resolve = settings.resolve_entities + resolve_predefined = settings.resolve_predefined_entities -- in case we have escaped entities + unify_predefined = settings.unify_predefined_entities -- & -> & + cleanup = settings.text_cleanup + stack, top, at, xmlns, errorstr, result, entities = { }, { }, { }, { }, nil, nil, settings.entities or { } + acache, hcache, dcache = { }, { }, { } -- not stored + reported_attribute_errors = { } + if settings.parent_root then + mt = getmetatable(settings.parent_root) + else + initialize_mt(top) + end stack[#stack+1] = top top.dt = { } dt = top.dt if not data or data == "" then errorstr = "empty xml file" - elseif not grammar:match(data) then - errorstr = "invalid xml file" + elseif utfize or resolve then + if lpegmatch(grammar_parsed_text,data) then + errorstr = "" + else + errorstr = "invalid xml file - parsed text" + end + elseif type(data) == "string" then + if lpegmatch(grammar_unparsed_text,data) then + errorstr = "" + else + errorstr = "invalid xml file - unparsed text" + end else - errorstr = "" + errorstr = "invalid xml file - no text at all" end if errorstr and errorstr ~= "" then - result = { dt = { { ns = "", tg = "error", dt = { errorstr }, at={}, er = true } }, error = true } + result = { dt = { { ns = "", tg = "error", dt = { errorstr }, at={ }, er = true } } } setmetatable(stack, mt) - if xml.error_handler then xml.error_handler("load",errorstr) end + local error_handler = settings.error_handler + if error_handler == false then + -- no error message + else + error_handler = error_handler or xml.error_handler + if error_handler then + xml.error_handler("load",errorstr) + end + end else result = stack[1] end - if not no_root then - result = { special = true, ns = "", tg = '@rt@', dt = result.dt, at={}, entities = entities } + if not settings.no_root then + result = { special = true, ns = "", tg = '@rt@', dt = result.dt, at={ }, entities = entities, settings = settings } setmetatable(result, mt) local rdt = result.dt for k=1,#rdt do local v = rdt[k] if type(v) == "table" and not v.special then -- always table -) result.ri = k -- rootindex +v.__p__ = result -- new, experiment, else we cannot go back to settings, we need to test this ! break end end end + if errorstr and errorstr ~= "" then + result.error = true + end return result end +xml.convert = xmlconvert + +function xml.inheritedconvert(data,xmldata) + local settings = xmldata.settings + settings.parent_root = xmldata -- to be tested + -- settings.no_root = true + local xc = xmlconvert(data,settings) + -- xc.settings = nil + -- xc.entities = nil + -- xc.special = nil + -- xc.ri = nil + -- print(xc.tg) + return xc +end + --[[ldx-- <p>Packaging data in an xml like table is done with the following function. Maybe it will go away (when not used).</p> @@ -3243,7 +4557,7 @@ function xml.is_valid(root) end function xml.package(tag,attributes,data) - local ns, tg = tag:match("^(.-):?([^:]+)$") + local ns, tg = match(tag,"^(.-):?([^:]+)$") local t = { ns = ns, tg = tg, dt = data or "", at = attributes or {} } setmetatable(t, mt) return t @@ -3261,21 +4575,19 @@ the whole file first. The function accepts a string representing a filename or a file handle.</p> --ldx]]-- -function xml.load(filename) +function xml.load(filename,settings) + local data = "" if type(filename) == "string" then + -- local data = io.loaddata(filename) - -todo: check type in io.loaddata local f = io.open(filename,'r') if f then - local root = xml.convert(f:read("*all")) + data = f:read("*all") f:close() - return root - else - return xml.convert("") end elseif filename then -- filehandle - return xml.convert(filename:read("*all")) - else - return xml.convert("") + data = filename:read("*all") end + return xmlconvert(data,settings) end --[[ldx-- @@ -3283,9 +4595,11 @@ end valid trees, which is what the next function does.</p> --ldx]]-- +local no_root = { no_root = true } + function xml.toxml(data) if type(data) == "string" then - local root = { xml.convert(data,true) } + local root = { xmlconvert(data,no_root) } return (#root > 1 and root) or root[1] else return data @@ -3305,7 +4619,7 @@ local function copy(old,tables) if not tables[old] then tables[old] = new end - for k,v in pairs(old) do + for k,v in next, old do new[k] = (type(v) == "table" and (tables[v] or copy(v, tables))) or v end local mt = getmetatable(old) @@ -3330,217 +4644,304 @@ alternative.</p> -- todo: add <?xml version='1.0' standalone='yes'?> when not present -local fallbackhandle = (tex and tex.sprint) or io.write - -local function serialize(e, handle, textconverter, attributeconverter, specialconverter, nocommands) - if not e then - return - elseif not nocommands then - local ec = e.command - if ec ~= nil then -- we can have all kind of types - if e.special then - local etg, edt = e.tg, e.dt - local spc = specialconverter and specialconverter[etg] - if spc then - local result = spc(edt[1]) - if result then - handle(result) - return - else - -- no need to handle any further - end - end - end - local xc = xml.command - if xc then - xc(e,ec) - return +function xml.checkbom(root) -- can be made faster + if root.ri then + local dt, found = root.dt, false + for k=1,#dt do + local v = dt[k] + if type(v) == "table" and v.special and v.tg == "@pi@" and find(v.dt[1],"xml.*version=") then + found = true + break end end + if not found then + insert(dt, 1, { special=true, ns="", tg="@pi@", dt = { "xml version='1.0' standalone='yes'"} } ) + insert(dt, 2, "\n" ) + end end - handle = handle or fallbackhandle - local etg = e.tg - if etg then - if e.special then - local edt = e.dt - local spc = specialconverter and specialconverter[etg] - if spc then - local result = spc(edt[1]) - if result then - handle(result) +end + +--[[ldx-- +<p>At the cost of some 25% runtime overhead you can first convert the tree to a string +and then handle the lot.</p> +--ldx]]-- + +-- new experimental reorganized serialize + +local function verbose_element(e,handlers) + local handle = handlers.handle + local serialize = handlers.serialize + local ens, etg, eat, edt, ern = e.ns, e.tg, e.at, e.dt, e.rn + local ats = eat and next(eat) and { } + if ats then + for k,v in next, eat do + ats[#ats+1] = format('%s=%q',k,v) + end + end + if ern and trace_remap and ern ~= ens then + ens = ern + end + if ens ~= "" then + if edt and #edt > 0 then + if ats then + handle("<",ens,":",etg," ",concat(ats," "),">") + else + handle("<",ens,":",etg,">") + end + for i=1,#edt do + local e = edt[i] + if type(e) == "string" then + handle(e) else - -- no need to handle any further - end - elseif etg == "@pi@" then - -- handle(format("<?%s?>",edt[1])) - handle("<?" .. edt[1] .. "?>") - elseif etg == "@cm@" then - -- handle(format("<!--%s-->",edt[1])) - handle("<!--" .. edt[1] .. "-->") - elseif etg == "@cd@" then - -- handle(format("<![CDATA[%s]]>",edt[1])) - handle("<![CDATA[" .. edt[1] .. "]]>") - elseif etg == "@dt@" then - -- handle(format("<!DOCTYPE %s>",edt[1])) - handle("<!DOCTYPE " .. edt[1] .. ">") - elseif etg == "@rt@" then - serialize(edt,handle,textconverter,attributeconverter,specialconverter,nocommands) + serialize(e,handlers) + end end + handle("</",ens,":",etg,">") else - local ens, eat, edt, ern = e.ns, e.at, e.dt, e.rn - local ats = eat and next(eat) and { } -- type test maybe faster if ats then - if attributeconverter then - for k,v in next, eat do - ats[#ats+1] = format('%s=%q',k,attributeconverter(v)) - end - else - for k,v in next, eat do - ats[#ats+1] = format('%s=%q',k,v) - end - end + handle("<",ens,":",etg," ",concat(ats," "),"/>") + else + handle("<",ens,":",etg,"/>") end - if ern and trace_remap and ern ~= ens then - ens = ern + end + else + if edt and #edt > 0 then + if ats then + handle("<",etg," ",concat(ats," "),">") + else + handle("<",etg,">") end - if ens ~= "" then - if edt and #edt > 0 then - if ats then - -- handle(format("<%s:%s %s>",ens,etg,concat(ats," "))) - handle("<" .. ens .. ":" .. etg .. " " .. concat(ats," ") .. ">") - else - -- handle(format("<%s:%s>",ens,etg)) - handle("<" .. ens .. ":" .. etg .. ">") - end - for i=1,#edt do - local e = edt[i] - if type(e) == "string" then - if textconverter then - handle(textconverter(e)) - else - handle(e) - end - else - serialize(e,handle,textconverter,attributeconverter,specialconverter,nocommands) - end - end - -- handle(format("</%s:%s>",ens,etg)) - handle("</" .. ens .. ":" .. etg .. ">") + for i=1,#edt do + local ei = edt[i] + if type(ei) == "string" then + handle(ei) else - if ats then - -- handle(format("<%s:%s %s/>",ens,etg,concat(ats," "))) - handle("<" .. ens .. ":" .. etg .. " " .. concat(ats," ") .. "/>") - else - -- handle(format("<%s:%s/>",ens,etg)) - handle("<" .. ens .. ":" .. etg .. "/>") - end + serialize(ei,handlers) end + end + handle("</",etg,">") + else + if ats then + handle("<",etg," ",concat(ats," "),"/>") else - if edt and #edt > 0 then - if ats then - -- handle(format("<%s %s>",etg,concat(ats," "))) - handle("<" .. etg .. " " .. concat(ats," ") .. ">") - else - -- handle(format("<%s>",etg)) - handle("<" .. etg .. ">") - end - for i=1,#edt do - local ei = edt[i] - if type(ei) == "string" then - if textconverter then - handle(textconverter(ei)) - else - handle(ei) - end - else - serialize(ei,handle,textconverter,attributeconverter,specialconverter,nocommands) - end - end - -- handle(format("</%s>",etg)) - handle("</" .. etg .. ">") - else - if ats then - -- handle(format("<%s %s/>",etg,concat(ats," "))) - handle("<" .. etg .. " " .. concat(ats," ") .. "/>") - else - -- handle(format("<%s/>",etg)) - handle("<" .. etg .. "/>") - end - end + handle("<",etg,"/>") end end - elseif type(e) == "string" then - if textconverter then - handle(textconverter(e)) + end +end + +local function verbose_pi(e,handlers) + handlers.handle("<?",e.dt[1],"?>") +end + +local function verbose_comment(e,handlers) + handlers.handle("<!--",e.dt[1],"-->") +end + +local function verbose_cdata(e,handlers) + handlers.handle("<![CDATA[", e.dt[1],"]]>") +end + +local function verbose_doctype(e,handlers) + handlers.handle("<!DOCTYPE ",e.dt[1],">") +end + +local function verbose_root(e,handlers) + handlers.serialize(e.dt,handlers) +end + +local function verbose_text(e,handlers) + handlers.handle(e) +end + +local function verbose_document(e,handlers) + local serialize = handlers.serialize + local functions = handlers.functions + for i=1,#e do + local ei = e[i] + if type(ei) == "string" then + functions["@tx@"](ei,handlers) else - handle(e) + serialize(ei,handlers) end - else - for i=1,#e do - local ei = e[i] - if type(ei) == "string" then - if textconverter then - handle(textconverter(ei)) - else - handle(ei) - end - else - serialize(ei,handle,textconverter,attributeconverter,specialconverter,nocommands) - end + end +end + +local function serialize(e,handlers,...) + local initialize = handlers.initialize + local finalize = handlers.finalize + local functions = handlers.functions + if initialize then + local state = initialize(...) + if not state == true then + return state end end + local etg = e.tg + if etg then + (functions[etg] or functions["@el@"])(e,handlers) + -- elseif type(e) == "string" then + -- functions["@tx@"](e,handlers) + else + functions["@dc@"](e,handlers) + end + if finalize then + return finalize() + end end -xml.serialize = serialize +local function xserialize(e,handlers) + local functions = handlers.functions + local etg = e.tg + if etg then + (functions[etg] or functions["@el@"])(e,handlers) + -- elseif type(e) == "string" then + -- functions["@tx@"](e,handlers) + else + functions["@dc@"](e,handlers) + end +end -function xml.checkbom(root) -- can be made faster - if root.ri then - local dt, found = root.dt, false - for k=1,#dt do - local v = dt[k] - if type(v) == "table" and v.special and v.tg == "@pi" and find(v.dt,"xml.*version=") then - found = true - break +local handlers = { } + +local function newhandlers(settings) + local t = table.copy(handlers.verbose or { }) -- merge + if settings then + for k,v in next, settings do + if type(v) == "table" then + tk = t[k] if not tk then tk = { } t[k] = tk end + for kk,vv in next, v do + tk[kk] = vv + end + else + t[k] = v end end - if not found then - insert(dt, 1, { special=true, ns="", tg="@pi@", dt = { "xml version='1.0' standalone='yes'"} } ) - insert(dt, 2, "\n" ) + if settings.name then + handlers[settings.name] = t end end + return t end +local nofunction = function() end + +function xml.sethandlersfunction(handler,name,fnc) + handler.functions[name] = fnc or nofunction +end + +function xml.gethandlersfunction(handler,name) + return handler.functions[name] +end + +function xml.gethandlers(name) + return handlers[name] +end + +newhandlers { + name = "verbose", + initialize = false, -- faster than nil and mt lookup + finalize = false, -- faster than nil and mt lookup + serialize = xserialize, + handle = print, + functions = { + ["@dc@"] = verbose_document, + ["@dt@"] = verbose_doctype, + ["@rt@"] = verbose_root, + ["@el@"] = verbose_element, + ["@pi@"] = verbose_pi, + ["@cm@"] = verbose_comment, + ["@cd@"] = verbose_cdata, + ["@tx@"] = verbose_text, + } +} + --[[ldx-- -<p>At the cost of some 25% runtime overhead you can first convert the tree to a string -and then handle the lot.</p> +<p>How you deal with saving data depends on your preferences. For a 40 MB database +file the timing on a 2.3 Core Duo are as follows (time in seconds):</p> + +<lines> +1.3 : load data from file to string +6.1 : convert string into tree +5.3 : saving in file using xmlsave +6.8 : converting to string using xml.tostring +3.6 : saving converted string in file +</lines> + +<p>Beware, these were timing with the old routine but measurements will not be that +much different I guess.</p> --ldx]]-- -function xml.tostring(root) -- 25% overhead due to collecting +-- maybe this will move to lxml-xml + +local result + +local xmlfilehandler = newhandlers { + name = "file", + initialize = function(name) result = io.open(name,"wb") return result end, + finalize = function() result:close() return true end, + handle = function(...) result:write(...) end, +} + +-- no checking on writeability here but not faster either +-- +-- local xmlfilehandler = newhandlers { +-- initialize = function(name) io.output(name,"wb") return true end, +-- finalize = function() io.close() return true end, +-- handle = io.write, +-- } + + +function xml.save(root,name) + serialize(root,xmlfilehandler,name) +end + +local result + +local xmlstringhandler = newhandlers { + name = "string", + initialize = function() result = { } return result end, + finalize = function() return concat(result) end, + handle = function(...) result[#result+1] = concat { ... } end +} + +local function xmltostring(root) -- 25% overhead due to collecting if root then if type(root) == 'string' then return root - elseif next(root) then -- next is faster than type (and >0 test) - local result = { } - serialize(root,function(s) result[#result+1] = s end) -- brrr, slow (direct printing is faster) - return concat(result,"") + else -- if next(root) then -- next is faster than type (and >0 test) + return serialize(root,xmlstringhandler) or "" end end return "" end +local function xmltext(root) -- inline + return (root and xmltostring(root)) or "" +end + +function initialize_mt(root) + mt = { __tostring = xmltext, __index = root } +end + +xml.defaulthandlers = handlers +xml.newhandlers = newhandlers +xml.serialize = serialize +xml.tostring = xmltostring + --[[ldx-- <p>The next function operated on the content only and needs a handle function that accepts a string.</p> --ldx]]-- -function xml.string(e,handle) +local function xmlstring(e,handle) if not handle or (e.special and e.tg ~= "@rt@") then -- nothing elseif e.tg then local edt = e.dt if edt then for i=1,#edt do - xml.string(edt[i],handle) + xmlstring(edt[i],handle) end end else @@ -3548,54 +4949,52 @@ function xml.string(e,handle) end end ---[[ldx-- -<p>How you deal with saving data depends on your preferences. For a 40 MB database -file the timing on a 2.3 Core Duo are as follows (time in seconds):</p> +xml.string = xmlstring -<lines> -1.3 : load data from file to string -6.1 : convert string into tree -5.3 : saving in file using xmlsave -6.8 : converting to string using xml.tostring -3.6 : saving converted string in file -</lines> - -<p>The save function is given below.</p> +--[[ldx-- +<p>A few helpers:</p> --ldx]]-- -function xml.save(root,name) - local f = io.open(name,"w") - if f then - xml.serialize(root,function(s) f:write(s) end) - f:close() +--~ xmlsetproperty(root,"settings",settings) + +function xml.settings(e) + while e do + local s = e.settings + if s then + return s + else + e = e.__p__ + end end + return nil end ---[[ldx-- -<p>A few helpers:</p> ---ldx]]-- - -function xml.body(root) - return (root.ri and root.dt[root.ri]) or root +function xml.root(e) + local r = e + while e do + e = e.__p__ + if e then + r = e + end + end + return r end -function xml.text(root) - return (root and xml.tostring(root)) or "" +function xml.parent(root) + return root.__p__ end -function xml.content(root) -- bugged - return (root and root.dt and xml.tostring(root.dt)) or "" +function xml.body(root) + return (root.ri and root.dt[root.ri]) or root -- not ok yet end -function xml.isempty(root, pattern) - if pattern == "" or pattern == "*" then - pattern = nil - end - if pattern then - -- todo - return false +function xml.name(root) + if not root then + return "" + elseif root.ns == "" then + return root.tg else - return not root or not root.dt or #root.dt == 0 or root.dt == "" + return root.ns .. ":" .. root.tg end end @@ -3603,18 +5002,15 @@ end <p>The next helper erases an element but keeps the table as it is, and since empty strings are not serialized (effectively) it does not harm. Copying the table would take more time. Usage:</p> - -<typing> -dt[k] = xml.empty() or xml.empty(dt,k) -</typing> --ldx]]-- -function xml.empty(dt,k) - if dt and k then - dt[k] = "" - return dt[k] - else - return "" +function xml.erase(dt,k) + if dt then + if k then + dt[k] = "" + else for k=1,#dt do + dt[1] = { "" } + end end end end @@ -3635,6 +5031,42 @@ function xml.assign(dt,k,root) end end +-- the following helpers may move + +--[[ldx-- +<p>The next helper assigns a tree (or string). Usage:</p> +<typing> +xml.tocdata(e) +xml.tocdata(e,"error") +</typing> +--ldx]]-- + +function xml.tocdata(e,wrapper) + local whatever = xmltostring(e.dt) + if wrapper then + whatever = format("<%s>%s</%s>",wrapper,whatever,wrapper) + end + local t = { special = true, ns = "", tg = "@cd@", at = {}, rn = "", dt = { whatever }, __p__ = e } + setmetatable(t,getmetatable(e)) + e.dt = { t } +end + +function xml.makestandalone(root) + if root.ri then + local dt = root.dt + for k=1,#dt do + local v = dt[k] + if type(v) == "table" and v.special and v.tg == "@pi@" then + local txt = v.dt[1] + if find(txt,"xml.*version=") then + v.dt[1] = txt .. " standalone='yes'" + break + end + end + end + end +end + end -- of closure @@ -3648,1420 +5080,1285 @@ if not modules then modules = { } end modules ['lxml-pth'] = { license = "see context related readme files" } +-- e.ni is only valid after a filter run + local concat, remove, insert = table.concat, table.remove, table.insert local type, next, tonumber, tostring, setmetatable, loadstring = type, next, tonumber, tostring, setmetatable, loadstring -local format, lower, gmatch, gsub, find = string.format, string.lower, string.gmatch, string.gsub, string.find +local format, upper, lower, gmatch, gsub, find, rep = string.format, string.upper, string.lower, string.gmatch, string.gsub, string.find, string.rep +local lpegmatch = lpeg.match + +-- beware, this is not xpath ... e.g. position is different (currently) and +-- we have reverse-sibling as reversed preceding sibling --[[ldx-- <p>This module can be used stand alone but also inside <l n='mkiv'/> in which case it hooks into the tracker code. Therefore we provide a few functions that set the tracers. Here we overload a previously defined function.</p> +<p>If I can get in the mood I will make a variant that is XSLT compliant +but I wonder if it makes sense.</P> --ldx]]-- -local trace_lpath = false - -if trackers then - trackers.register("xml.lpath", function(v) trace_lpath = v end) -end +--[[ldx-- +<p>Expecially the lpath code is experimental, we will support some of xpath, but +only things that make sense for us; as compensation it is possible to hook in your +own functions. Apart from preprocessing content for <l n='context'/> we also need +this module for process management, like handling <l n='ctx'/> and <l n='rlx'/> +files.</p> -local settrace = xml.settrace -- lxml-tab +<typing> +a/b/c /*/c +a/b/c/first() a/b/c/last() a/b/c/index(n) a/b/c/index(-n) +a/b/c/text() a/b/c/text(1) a/b/c/text(-1) a/b/c/text(n) +</typing> +--ldx]]-- -function xml.settrace(str,value) - if str == "lpath" then - trace_lpath = value or false - else - settrace(str,value) -- lxml-tab - end -end +local trace_lpath = false if trackers then trackers.register("xml.path", function(v) trace_lpath = v end) end +local trace_lparse = false if trackers then trackers.register("xml.parse", function(v) trace_lparse = v end) end +local trace_lprofile = false if trackers then trackers.register("xml.profile", function(v) trace_lpath = v trace_lparse = v trace_lprofile = v end) end --[[ldx-- -<p>We've now arrived at an intersting part: accessing the tree using a subset +<p>We've now arrived at an interesting part: accessing the tree using a subset of <l n='xpath'/> and since we're not compatible we call it <l n='lpath'/>. We will explain more about its usage in other documents.</p> --ldx]]-- -local lpathcalls = 0 -- statistics -local lpathcached = 0 -- statistics +local lpathcalls = 0 function xml.lpathcalls () return lpathcalls end +local lpathcached = 0 function xml.lpathcached() return lpathcached end -xml.functions = xml.functions or { } -xml.expressions = xml.expressions or { } +xml.functions = xml.functions or { } -- internal +xml.expressions = xml.expressions or { } -- in expressions +xml.finalizers = xml.finalizers or { } -- fast do-with ... (with return value other than collection) +xml.specialhandler = xml.specialhandler or { } local functions = xml.functions local expressions = xml.expressions +local finalizers = xml.finalizers -local actions = { - [10] = "stay", - [11] = "parent", - [12] = "subtree root", - [13] = "document root", - [14] = "any", - [15] = "many", - [16] = "initial", - [20] = "match", - [21] = "match one of", - [22] = "match and attribute eq", - [23] = "match and attribute ne", - [24] = "match one of and attribute eq", - [25] = "match one of and attribute ne", - [27] = "has attribute", - [28] = "has value", - [29] = "fast match", - [30] = "select", - [31] = "expression", - [40] = "processing instruction", -} - --- a rather dumb lpeg +finalizers.xml = finalizers.xml or { } +finalizers.tex = finalizers.tex or { } -local P, S, R, C, V, Cc = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc +local function fallback (t, name) + local fn = finalizers[name] + if fn then + t[name] = fn + else + logs.report("xml","unknown sub finalizer '%s'",tostring(name)) + fn = function() end + end + return fn +end --- instead of using functions we just parse a few names which saves a call --- later on +setmetatable(finalizers.xml, { __index = fallback }) +setmetatable(finalizers.tex, { __index = fallback }) -local lp_position = P("position()") / "ps" -local lp_index = P("index()") / "id" -local lp_text = P("text()") / "tx" -local lp_name = P("name()") / "(ns~='' and ns..':'..tg)" -- "((rt.ns~='' and rt.ns..':'..rt.tg) or '')" -local lp_tag = P("tag()") / "tg" -- (rt.tg or '') -local lp_ns = P("ns()") / "ns" -- (rt.ns or '') -local lp_noequal = P("!=") / "~=" + P("<=") + P(">=") + P("==") -local lp_doequal = P("=") / "==" -local lp_attribute = P("@") / "" * Cc("(at['") * R("az","AZ","--","__")^1 * Cc("'] or '')") +xml.defaultprotocol = "xml" -local lp_lua_function = C(R("az","AZ","--","__")^1 * (P(".") * R("az","AZ","--","__")^1)^1) * P("(") / function(t) -- todo: better . handling - return t .. "(" -end +-- as xsl does not follow xpath completely here we will also +-- be more liberal especially with regards to the use of | and +-- the rootpath: +-- +-- test : all 'test' under current +-- /test : 'test' relative to current +-- a|b|c : set of names +-- (a|b|c) : idem +-- ! : not +-- +-- after all, we're not doing transformations but filtering. in +-- addition we provide filter functions (last bit) +-- +-- todo: optimizer +-- +-- .. : parent +-- * : all kids +-- / : anchor here +-- // : /**/ +-- ** : all in between +-- +-- so far we had (more practical as we don't transform) +-- +-- {/test} : kids 'test' under current node +-- {test} : any kid with tag 'test' +-- {//test} : same as above -local lp_function = C(R("az","AZ","--","__")^1) * P("(") / function(t) -- todo: better . handling - if expressions[t] then - return "expressions." .. t .. "(" - else - return "expressions.error(" - end -end +-- evaluator (needs to be redone, for the moment copied) -local lparent = lpeg.P("(") -local rparent = lpeg.P(")") -local noparent = 1 - (lparent+rparent) -local nested = lpeg.P{lparent * (noparent + lpeg.V(1))^0 * rparent} -local value = lpeg.P(lparent * lpeg.C((noparent + nested)^0) * rparent) -- lpeg.P{"("*C(((1-S("()"))+V(1))^0)*")"} +-- todo: apply_axis(list,notable) and collection vs single --- if we use a dedicated namespace then we don't need to pass rt and k +local apply_axis = { } -local lp_special = (C(P("name")+P("text")+P("tag"))) * value / function(t,s) - if expressions[t] then - if s then - return "expressions." .. t .. "(r,k," .. s ..")" - else - return "expressions." .. t .. "(r,k)" +apply_axis['root'] = function(list) + local collected = { } + for l=1,#list do + local ll = list[l] + local rt = ll + while ll do + ll = ll.__p__ + if ll then + rt = ll + end end - else - return "expressions.error(" .. t .. ")" + collected[#collected+1] = rt end + return collected end -local converter = lpeg.Cs ( ( - lp_position + - lp_index + - lp_text + lp_name + -- fast one - lp_special + - lp_noequal + lp_doequal + - lp_attribute + - lp_lua_function + - lp_function + -1 )^1 ) - --- expressions,root,rootdt,k,e,edt,ns,tg,idx,hsh[tg] or 1 +apply_axis['self'] = function(list) +--~ local collected = { } +--~ for l=1,#list do +--~ collected[#collected+1] = list[l] +--~ end +--~ return collected + return list +end -local template = [[ - return function(expressions,r,d,k,e,dt,ns,tg,id,ps) - local at, tx = e.at or { }, dt[1] or "" - return %s +apply_axis['child'] = function(list) + local collected = { } + for l=1,#list do + local ll = list[l] + local dt = ll.dt + local en = 0 + for k=1,#dt do + local dk = dt[k] + if dk.tg then + collected[#collected+1] = dk + dk.ni = k -- refresh + en = en + 1 + dk.ei = en + end + end + ll.en = en end -]] - -local function make_expression(str) - str = converter:match(str) - return str, loadstring(format(template,str))() -end - -local map = { } - -local space = S(' \r\n\t') -local squote = S("'") -local dquote = S('"') -local lparent = P('(') -local rparent = P(')') -local atsign = P('@') -local lbracket = P('[') -local rbracket = P(']') -local exclam = P('!') -local period = P('.') -local eq = P('==') + P('=') -local ne = P('<>') + P('!=') -local star = P('*') -local slash = P('/') -local colon = P(':') -local bar = P('|') -local hat = P('^') -local valid = R('az', 'AZ', '09') + S('_-') -local name_yes = C(valid^1 + star) * colon * C(valid^1 + star) -- permits ns:* *:tg *:* -local name_nop = Cc("*") * C(valid^1) -local name = name_yes + name_nop -local number = C((S('+-')^0 * R('09')^1)) / tonumber -local names = (bar^0 * name)^1 -local morenames = name * (bar^0 * name)^1 -local instructiontag = P('pi::') -local spacing = C(space^0) -local somespace = space^1 -local optionalspace = space^0 -local text = C(valid^0) -local value = (squote * C((1 - squote)^0) * squote) + (dquote * C((1 - dquote)^0) * dquote) -local empty = 1-slash - -local is_eq = lbracket * atsign * name * eq * value * rbracket -local is_ne = lbracket * atsign * name * ne * value * rbracket -local is_attribute = lbracket * atsign * name * rbracket -local is_value = lbracket * value * rbracket -local is_number = lbracket * number * rbracket - -local nobracket = 1-(lbracket+rbracket) -- must be improved -local is_expression = lbracket * C(((C(nobracket^1))/make_expression)) * rbracket - -local is_expression = lbracket * (C(nobracket^1))/make_expression * rbracket - -local is_one = name -local is_none = exclam * name -local is_one_of = ((lparent * names * rparent) + morenames) -local is_none_of = exclam * ((lparent * names * rparent) + morenames) - -local stay = (period ) -local parent = (period * period ) / function( ) map[#map+1] = { 11 } end -local subtreeroot = (slash + hat ) / function( ) map[#map+1] = { 12 } end -local documentroot = (hat * hat ) / function( ) map[#map+1] = { 13 } end -local any = (star ) / function( ) map[#map+1] = { 14 } end -local many = (star * star ) / function( ) map[#map+1] = { 15 } end -local initial = (hat * hat * hat ) / function( ) map[#map+1] = { 16 } end - -local match = (is_one ) / function(...) map[#map+1] = { 20, true , ... } end -local match_one_of = (is_one_of ) / function(...) map[#map+1] = { 21, true , ... } end -local dont_match = (is_none ) / function(...) map[#map+1] = { 20, false, ... } end -local dont_match_one_of = (is_none_of ) / function(...) map[#map+1] = { 21, false, ... } end - -local match_and_eq = (is_one * is_eq ) / function(...) map[#map+1] = { 22, true , ... } end -local match_and_ne = (is_one * is_ne ) / function(...) map[#map+1] = { 23, true , ... } end -local dont_match_and_eq = (is_none * is_eq ) / function(...) map[#map+1] = { 22, false, ... } end -local dont_match_and_ne = (is_none * is_ne ) / function(...) map[#map+1] = { 23, false, ... } end - -local match_one_of_and_eq = (is_one_of * is_eq ) / function(...) map[#map+1] = { 24, true , ... } end -local match_one_of_and_ne = (is_one_of * is_ne ) / function(...) map[#map+1] = { 25, true , ... } end -local dont_match_one_of_and_eq = (is_none_of * is_eq ) / function(...) map[#map+1] = { 24, false, ... } end -local dont_match_one_of_and_ne = (is_none_of * is_ne ) / function(...) map[#map+1] = { 25, false, ... } end - -local has_attribute = (is_one * is_attribute) / function(...) map[#map+1] = { 27, true , ... } end -local has_value = (is_one * is_value ) / function(...) map[#map+1] = { 28, true , ... } end -local dont_has_attribute = (is_none * is_attribute) / function(...) map[#map+1] = { 27, false, ... } end -local dont_has_value = (is_none * is_value ) / function(...) map[#map+1] = { 28, false, ... } end -local position = (is_one * is_number ) / function(...) map[#map+1] = { 30, true, ... } end -local dont_position = (is_none * is_number ) / function(...) map[#map+1] = { 30, false, ... } end - -local expression = (is_one * is_expression)/ function(...) map[#map+1] = { 31, true, ... } end -local dont_expression = (is_none * is_expression)/ function(...) map[#map+1] = { 31, false, ... } end - -local self_expression = ( is_expression) / function(...) if #map == 0 then map[#map+1] = { 11 } end - map[#map+1] = { 31, true, "*", "*", ... } end -local dont_self_expression = (exclam * is_expression) / function(...) if #map == 0 then map[#map+1] = { 11 } end - map[#map+1] = { 31, false, "*", "*", ... } end - -local instruction = (instructiontag * text ) / function(...) map[#map+1] = { 40, ... } end -local nothing = (empty ) / function( ) map[#map+1] = { 15 } end -- 15 ? -local crap = (1-slash)^1 - --- a few ugly goodies: - -local docroottag = P('^^') / function( ) map[#map+1] = { 12 } end -local subroottag = P('^') / function( ) map[#map+1] = { 13 } end -local roottag = P('root::') / function( ) map[#map+1] = { 12 } end -local parenttag = P('parent::') / function( ) map[#map+1] = { 11 } end -local childtag = P('child::') -local selftag = P('self::') - --- there will be more and order will be optimized - -local selector = ( - instruction + --- many + any + -- brrr, not here ! - parent + stay + - dont_position + position + - dont_match_one_of_and_eq + dont_match_one_of_and_ne + - match_one_of_and_eq + match_one_of_and_ne + - dont_match_and_eq + dont_match_and_ne + - match_and_eq + match_and_ne + - dont_expression + expression + - dont_self_expression + self_expression + - has_attribute + has_value + - dont_match_one_of + match_one_of + - dont_match + match + - many + any + - crap + empty -) - -local grammar = P { "startup", - startup = (initial + documentroot + subtreeroot + roottag + docroottag + subroottag)^0 * V("followup"), - followup = ((slash + parenttag + childtag + selftag)^0 * selector)^1, -} + return collected +end -local function compose(str) - if not str or str == "" then - -- wildcard - return true - elseif str == '/' then - -- root - return false - else - map = { } - grammar:match(str) - if #map == 0 then - return true - else - local m = map[1][1] - if #map == 1 then - if m == 14 or m == 15 then - -- wildcard - return true - elseif m == 12 then - -- root - return false - end - elseif #map == 2 and m == 12 and map[2][1] == 20 then - -- return { { 29, map[2][2], map[2][3], map[2][4], map[2][5] } } - map[2][1] = 29 - return { map[2] } - end - if m ~= 11 and m ~= 12 and m ~= 13 and m ~= 14 and m ~= 15 and m ~= 16 then - insert(map, 1, { 16 }) +local function collect(list,collected) + local dt = list.dt + if dt then + local en = 0 + for k=1,#dt do + local dk = dt[k] + if dk.tg then + collected[#collected+1] = dk + dk.ni = k -- refresh + en = en + 1 + dk.ei = en + collect(dk,collected) end - -- print(gsub(table.serialize(map),"[ \n]+"," ")) - return map end + list.en = en end end +apply_axis['descendant'] = function(list) + local collected = { } + for l=1,#list do + collect(list[l],collected) + end + return collected +end -local cache = { } - -function xml.lpath(pattern,trace) - lpathcalls = lpathcalls + 1 - if type(pattern) == "string" then - local result = cache[pattern] - if result == nil then -- can be false which is valid -) - result = compose(pattern) - cache[pattern] = result - lpathcached = lpathcached + 1 +local function collect(list,collected) + local dt = list.dt + if dt then + local en = 0 + for k=1,#dt do + local dk = dt[k] + if dk.tg then + collected[#collected+1] = dk + dk.ni = k -- refresh + en = en + 1 + dk.ei = en + collect(dk,collected) + end end - if trace or trace_lpath then - xml.lshow(result) + list.en = en + end +end +apply_axis['descendant-or-self'] = function(list) + local collected = { } + for l=1,#list do + local ll = list[l] + if ll.special ~= true then -- catch double root + collected[#collected+1] = ll end - return result - else - return pattern + collect(ll,collected) end + return collected end -function xml.cached_patterns() - return cache +apply_axis['ancestor'] = function(list) + local collected = { } + for l=1,#list do + local ll = list[l] + while ll do + ll = ll.__p__ + if ll then + collected[#collected+1] = ll + end + end + end + return collected end --- we run out of locals (limited to 200) --- --- local fallbackreport = (texio and texio.write) or io.write - -function xml.lshow(pattern,report) --- report = report or fallbackreport - report = report or (texio and texio.write) or io.write - local lp = xml.lpath(pattern) - if lp == false then - report(" -: root\n") - elseif lp == true then - report(" -: wildcard\n") - else - if type(pattern) == "string" then - report(format("pattern: %s\n",pattern)) - end - for k=1,#lp do - local v = lp[k] - if #v > 1 then - local t = { } - for i=2,#v do - local vv = v[i] - if type(vv) == "string" then - t[#t+1] = (vv ~= "" and vv) or "#" - elseif type(vv) == "boolean" then - t[#t+1] = (vv and "==") or "<>" - end - end - report(format("%2i: %s %s -> %s\n", k,v[1],actions[v[1]],concat(t," "))) - else - report(format("%2i: %s %s\n", k,v[1],actions[v[1]])) +apply_axis['ancestor-or-self'] = function(list) + local collected = { } + for l=1,#list do + local ll = list[l] + collected[#collected+1] = ll + while ll do + ll = ll.__p__ + if ll then + collected[#collected+1] = ll end end end + return collected end -function xml.xshow(e,...) -- also handy when report is given, use () to isolate first e - local t = { ... } --- local report = (type(t[#t]) == "function" and t[#t]) or fallbackreport - local report = (type(t[#t]) == "function" and t[#t]) or (texio and texio.write) or io.write - if e == nil then - report("<!-- no element -->\n") - elseif type(e) ~= "table" then - report(tostring(e)) - elseif e.tg then - report(tostring(e) .. "\n") - else - for i=1,#e do - report(tostring(e[i]) .. "\n") +apply_axis['parent'] = function(list) + local collected = { } + for l=1,#list do + local pl = list[l].__p__ + if pl then + collected[#collected+1] = pl end end + return collected end ---[[ldx-- -<p>An <l n='lpath'/> is converted to a table with instructions for traversing the -tree. Hoever, simple cases are signaled by booleans. Because we don't know in -advance what we want to do with the found element the handle gets three arguments:</p> - -<lines> -<t>r</t> : the root element of the data table -<t>d</t> : the data table of the result -<t>t</t> : the index in the data table of the result -</lines> - -<p> Access to the root and data table makes it possible to construct insert and delete -functions.</p> ---ldx]]-- - -local functions = xml.functions -local expressions = xml.expressions - -expressions.contains = string.find -expressions.find = string.find -expressions.upper = string.upper -expressions.lower = string.lower -expressions.number = tonumber -expressions.boolean = toboolean - -expressions.oneof = function(s,...) -- slow - local t = {...} for i=1,#t do if s == t[i] then return true end end return false +apply_axis['attribute'] = function(list) + return { } end -expressions.error = function(str) - xml.error_handler("unknown function in lpath expression",str or "?") - return false +apply_axis['namespace'] = function(list) + return { } end -functions.text = function(root,k,n) -- unchecked, maybe one deeper - local t = type(t) - if t == "string" then - return t - else -- todo n - local rdt = root.dt - return (rdt and rdt[k]) or root[k] or "" - end +apply_axis['following'] = function(list) -- incomplete +--~ local collected = { } +--~ for l=1,#list do +--~ local ll = list[l] +--~ local p = ll.__p__ +--~ local d = p.dt +--~ for i=ll.ni+1,#d do +--~ local di = d[i] +--~ if type(di) == "table" then +--~ collected[#collected+1] = di +--~ break +--~ end +--~ end +--~ end +--~ return collected + return { } +end + +apply_axis['preceding'] = function(list) -- incomplete +--~ local collected = { } +--~ for l=1,#list do +--~ local ll = list[l] +--~ local p = ll.__p__ +--~ local d = p.dt +--~ for i=ll.ni-1,1,-1 do +--~ local di = d[i] +--~ if type(di) == "table" then +--~ collected[#collected+1] = di +--~ break +--~ end +--~ end +--~ end +--~ return collected + return { } end -functions.name = function(d,k,n) -- ns + tg - local found = false - n = n or 0 - if not k then - -- not found - elseif n == 0 then - local dk = d[k] - found = dk and (type(dk) == "table") and dk - elseif n < 0 then - for i=k-1,1,-1 do - local di = d[i] - if type(di) == "table" then - if n == -1 then - found = di - break - else - n = n + 1 - end - end - end - else - for i=k+1,#d,1 do +apply_axis['following-sibling'] = function(list) + local collected = { } + for l=1,#list do + local ll = list[l] + local p = ll.__p__ + local d = p.dt + for i=ll.ni+1,#d do local di = d[i] if type(di) == "table" then - if n == 1 then - found = di - break - else - n = n - 1 - end + collected[#collected+1] = di end end end - if found then - local ns, tg = found.rn or found.ns or "", found.tg - if ns ~= "" then - return ns .. ":" .. tg - else - return tg - end - else - return "" - end + return collected end -functions.tag = function(d,k,n) -- only tg - local found = false - n = n or 0 - if not k then - -- not found - elseif n == 0 then - local dk = d[k] - found = dk and (type(dk) == "table") and dk - elseif n < 0 then - for i=k-1,1,-1 do +apply_axis['preceding-sibling'] = function(list) + local collected = { } + for l=1,#list do + local ll = list[l] + local p = ll.__p__ + local d = p.dt + for i=1,ll.ni-1 do local di = d[i] if type(di) == "table" then - if n == -1 then - found = di - break - else - n = n + 1 - end + collected[#collected+1] = di end end - else - for i=k+1,#d,1 do + end + return collected +end + +apply_axis['reverse-sibling'] = function(list) -- reverse preceding + local collected = { } + for l=1,#list do + local ll = list[l] + local p = ll.__p__ + local d = p.dt + for i=ll.ni-1,1,-1 do local di = d[i] if type(di) == "table" then - if n == 1 then - found = di - break - else - n = n - 1 - end + collected[#collected+1] = di end end end - return (found and found.tg) or "" + return collected end -expressions.text = functions.text -expressions.name = functions.name -expressions.tag = functions.tag +apply_axis['auto-descendant-or-self'] = apply_axis['descendant-or-self'] +apply_axis['auto-descendant'] = apply_axis['descendant'] +apply_axis['auto-child'] = apply_axis['child'] +apply_axis['auto-self'] = apply_axis['self'] +apply_axis['initial-child'] = apply_axis['child'] -local function traverse(root,pattern,handle,reverse,index,parent,wildcard) -- multiple only for tags, not for namespaces - if not root then -- error - return false - elseif pattern == false then -- root - handle(root,root.dt,root.ri) - return false - elseif pattern == true then -- wildcard - local rootdt = root.dt - if rootdt then - local start, stop, step = 1, #rootdt, 1 - if reverse then - start, stop, step = stop, start, -1 - end - for k=start,stop,step do - if handle(root,rootdt,root.ri or k) then return false end - if not traverse(rootdt[k],true,handle,reverse) then return false end - end - end - return false - elseif root.dt then - index = index or 1 - local action = pattern[index] - local command = action[1] - if command == 29 then -- fast case /oeps - local rootdt = root.dt - for k=1,#rootdt do - local e = rootdt[k] - local tg = e.tg - if e.tg then - local ns = e.rn or e.ns - local ns_a, tg_a = action[3], action[4] - local matched = (ns_a == "*" or ns == ns_a) and (tg_a == "*" or tg == tg_a) - if not action[2] then matched = not matched end - if matched then - if handle(root,rootdt,k) then return false end - end - end - end - elseif command == 11 then -- parent - local ep = root.__p__ or parent - if index < #pattern then - if not traverse(ep,pattern,handle,reverse,index+1,root) then return false end - elseif handle(root,rootdt,k) then - return false +local function apply_nodes(list,directive,nodes) + -- todo: nodes[1] etc ... negated node name in set ... when needed + -- ... currently ignored + local maxn = #nodes + if maxn == 3 then --optimized loop + local nns, ntg = nodes[2], nodes[3] + if not nns and not ntg then -- wildcard + if directive then + return list + else + return { } end else - if (command == 16 or command == 12) and index == 1 then -- initial - -- wildcard = true - wildcard = command == 16 -- ok? - index = index + 1 - action = pattern[index] - command = action and action[1] or 0 -- something is wrong - end - if command == 11 then -- parent - local ep = root.__p__ or parent - if index < #pattern then - if not traverse(ep,pattern,handle,reverse,index+1,root) then return false end - elseif handle(root,rootdt,k) then - return false - end - else - local rootdt = root.dt - local start, stop, step, n, dn = 1, #rootdt, 1, 0, 1 - if command == 30 then - if action[5] < 0 then - start, stop, step = stop, start, -1 - dn = -1 - end - elseif reverse and index == #pattern then - start, stop, step = stop, start, -1 - end - local idx = 0 - local hsh = { } -- this will slooow down the lot - for k=start,stop,step do -- we used to have functions for all but a case is faster - local e = rootdt[k] - local ns, tg = e.rn or e.ns, e.tg - if tg then - -- we can optimize this for simple searches, but it probably does not pay off - hsh[tg] = (hsh[tg] or 0) + 1 - idx = idx + 1 - if command == 30 then - local ns_a, tg_a = action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - if not action[2] then matched = not matched end - if matched then - n = n + dn - if n == action[5] then - if index == #pattern then - if handle(root,rootdt,root.ri or k) then return false end - else - if not traverse(e,pattern,handle,reverse,index+1,root) then return false end - end - break - end - elseif wildcard then - if not traverse(e,pattern,handle,reverse,index,root,true) then return false end + local collected, m, p = { }, 0, nil + if not nns then -- only check tag + for l=1,#list do + local ll = list[l] + local ltg = ll.tg + if ltg then + if directive then + if ntg == ltg then + local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end + collected[#collected+1], ll.mi = ll, m end - else - local matched, multiple = false, false - if command == 20 then -- match - local ns_a, tg_a = action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - if not action[2] then matched = not matched end - elseif command == 21 then -- match one of - multiple = true - for i=3,#action,2 do - local ns_a, tg_a = action[i], action[i+1] - if (ns_a == "*" or ns == ns_a) and (tg == "*" or tg == tg_a) then - matched = true - break - end - end - if not action[2] then matched = not matched end - elseif command == 22 then -- eq - local ns_a, tg_a = action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - matched = matched and e.at[action[6]] == action[7] - elseif command == 23 then -- ne - local ns_a, tg_a = action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - if not action[2] then matched = not matched end - matched = mached and e.at[action[6]] ~= action[7] - elseif command == 24 then -- one of eq - multiple = true - for i=3,#action-2,2 do - local ns_a, tg_a = action[i], action[i+1] - if (ns_a == "*" or ns == ns_a) and (tg == "*" or tg == tg_a) then - matched = true - break - end - end - if not action[2] then matched = not matched end - matched = matched and e.at[action[#action-1]] == action[#action] - elseif command == 25 then -- one of ne - multiple = true - for i=3,#action-2,2 do - local ns_a, tg_a = action[i], action[i+1] - if (ns_a == "*" or ns == ns_a) and (tg == "*" or tg == tg_a) then - matched = true - break - end - end - if not action[2] then matched = not matched end - matched = matched and e.at[action[#action-1]] ~= action[#action] - elseif command == 27 then -- has attribute - local ns_a, tg_a = action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - if not action[2] then matched = not matched end - matched = matched and e.at[action[5]] - elseif command == 28 then -- has value - local edt, ns_a, tg_a = e.dt, action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - if not action[2] then matched = not matched end - matched = matched and edt and edt[1] == action[5] - elseif command == 31 then - local edt, ns_a, tg_a = e.dt, action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - if not action[2] then matched = not matched end - if matched then - matched = action[6](expressions,root,rootdt,k,e,edt,ns,tg,idx,hsh[tg] or 1) - end - end - if matched then -- combine tg test and at test - if index == #pattern then - if handle(root,rootdt,root.ri or k) then return false end - if wildcard then - if multiple then - if not traverse(e,pattern,handle,reverse,index,root,true) then return false end - else - -- maybe or multiple; anyhow, check on (section|title) vs just section and title in example in lxml - if not traverse(e,pattern,handle,reverse,index,root) then return false end - end - end - else - if not traverse(e,pattern,handle,reverse,index+1,root) then return false end - end - elseif command == 14 then -- any - if index == #pattern then - if handle(root,rootdt,root.ri or k) then return false end - else - if not traverse(e,pattern,handle,reverse,index+1,root) then return false end - end - elseif command == 15 then -- many - if index == #pattern then - if handle(root,rootdt,root.ri or k) then return false end - else - if not traverse(e,pattern,handle,reverse,index+1,root,true) then return false end - end - -- not here : 11 - elseif command == 11 then -- parent - local ep = e.__p__ or parent - if index < #pattern then - if not traverse(ep,pattern,handle,reverse,root,index+1) then return false end - elseif handle(root,rootdt,k) then - return false - end - elseif command == 40 and e.special and tg == "@pi@" then -- pi - local pi = action[2] - if pi ~= "" then - local pt = e.dt[1] - if pt and pt:find(pi) then - if handle(root,rootdt,k) then - return false - end - end - elseif handle(root,rootdt,k) then - return false - end - elseif wildcard then - if not traverse(e,pattern,handle,reverse,index,root,true) then return false end + elseif ntg ~= ltg then + local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end + collected[#collected+1], ll.mi = ll, m + end + end + end + elseif not ntg then -- only check namespace + for l=1,#list do + local ll = list[l] + local lns = ll.rn or ll.ns + if lns then + if directive then + if lns == nns then + local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end + collected[#collected+1], ll.mi = ll, m end + elseif lns ~= nns then + local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end + collected[#collected+1], ll.mi = ll, m end - else - -- not here : 11 - if command == 11 then -- parent - local ep = e.__p__ or parent - if index < #pattern then - if not traverse(ep,pattern,handle,reverse,index+1,root) then return false end - elseif handle(root,rootdt,k) then - return false + end + end + else -- check both + for l=1,#list do + local ll = list[l] + local ltg = ll.tg + if ltg then + local lns = ll.rn or ll.ns + local ok = ltg == ntg and lns == nns + if directive then + if ok then + local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end + collected[#collected+1], ll.mi = ll, m end - break -- else loop + elseif not ok then + local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end + collected[#collected+1], ll.mi = ll, m end end end end + return collected end + else + local collected, m, p = { }, 0, nil + for l=1,#list do + local ll = list[l] + local ltg = ll.tg + if ltg then + local lns = ll.rn or ll.ns + local ok = false + for n=1,maxn,3 do + local nns, ntg = nodes[n+1], nodes[n+2] + ok = (not ntg or ltg == ntg) and (not nns or lns == nns) + if ok then + break + end + end + if directive then + if ok then + local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end + collected[#collected+1], ll.mi = ll, m + end + elseif not ok then + local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end + collected[#collected+1], ll.mi = ll, m + end + end + end + return collected end - return true end -xml.traverse = traverse +local quit_expression = false ---[[ldx-- -<p>Next come all kind of locators and manipulators. The most generic function here -is <t>xml.filter(root,pattern)</t>. All registers functions in the filters namespace -can be path of a search path, as in:</p> +local function apply_expression(list,expression,order) + local collected = { } + quit_expression = false + for l=1,#list do + local ll = list[l] + if expression(list,ll,l,order) then -- nasty, order alleen valid als n=1 + collected[#collected+1] = ll + end + if quit_expression then + break + end + end + return collected +end + +local P, V, C, Cs, Cc, Ct, R, S, Cg, Cb = lpeg.P, lpeg.V, lpeg.C, lpeg.Cs, lpeg.Cc, lpeg.Ct, lpeg.R, lpeg.S, lpeg.Cg, lpeg.Cb + +local spaces = S(" \n\r\t\f")^0 +local lp_space = S(" \n\r\t\f") +local lp_any = P(1) +local lp_noequal = P("!=") / "~=" + P("<=") + P(">=") + P("==") +local lp_doequal = P("=") / "==" +local lp_or = P("|") / " or " +local lp_and = P("&") / " and " + +local lp_builtin = P ( + P("firstindex") / "1" + + P("lastindex") / "(#ll.__p__.dt or 1)" + + P("firstelement") / "1" + + P("lastelement") / "(ll.__p__.en or 1)" + + P("first") / "1" + + P("last") / "#list" + + P("rootposition") / "order" + + P("position") / "l" + -- is element in finalizer + P("order") / "order" + + P("element") / "(ll.ei or 1)" + + P("index") / "(ll.ni or 1)" + + P("match") / "(ll.mi or 1)" + + P("text") / "(ll.dt[1] or '')" + + -- P("name") / "(ll.ns~='' and ll.ns..':'..ll.tg)" + + P("name") / "((ll.ns~='' and ll.ns..':'..ll.tg) or ll.tg)" + + P("tag") / "ll.tg" + + P("ns") / "ll.ns" + ) * ((spaces * P("(") * spaces * P(")"))/"") + +local lp_attribute = (P("@") + P("attribute::")) / "" * Cc("(ll.at and ll.at['") * R("az","AZ","--","__")^1 * Cc("'])") +local lp_fastpos_p = ((P("+")^0 * R("09")^1 * P(-1)) / function(s) return "l==" .. s end) +local lp_fastpos_n = ((P("-") * R("09")^1 * P(-1)) / function(s) return "(" .. s .. "<0 and (#list+".. s .. "==l))" end) +local lp_fastpos = lp_fastpos_n + lp_fastpos_p +local lp_reserved = C("and") + C("or") + C("not") + C("div") + C("mod") + C("true") + C("false") + +local lp_lua_function = C(R("az","AZ","__")^1 * (P(".") * R("az","AZ","__")^1)^1) * ("(") / function(t) -- todo: better . handling + return t .. "(" +end -<typing> -local r, d, k = xml.filter(root,"/a/b/c/position(4)" -</typing> ---ldx]]-- +local lp_function = C(R("az","AZ","__")^1) * P("(") / function(t) -- todo: better . handling + if expressions[t] then + return "expr." .. t .. "(" + else + return "expr.error(" + end +end -local traverse, lpath, convert = xml.traverse, xml.lpath, xml.convert +local lparent = lpeg.P("(") +local rparent = lpeg.P(")") +local noparent = 1 - (lparent+rparent) +local nested = lpeg.P{lparent * (noparent + lpeg.V(1))^0 * rparent} +local value = lpeg.P(lparent * lpeg.C((noparent + nested)^0) * rparent) -- lpeg.P{"("*C(((1-S("()"))+V(1))^0)*")"} -xml.filters = { } +local lp_child = Cc("expr.child(ll,'") * R("az","AZ","--","__")^1 * Cc("')") +local lp_number = S("+-") * R("09")^1 +local lp_string = Cc("'") * R("az","AZ","--","__")^1 * Cc("'") +local lp_content = (P("'") * (1-P("'"))^0 * P("'") + P('"') * (1-P('"'))^0 * P('"')) -function xml.filters.default(root,pattern) - local rt, dt, dk - traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end) - return dt and dt[dk], rt, dt, dk -end +local cleaner -function xml.filters.attributes(root,pattern,arguments) - local rt, dt, dk - traverse(root, lpath(pattern), function(r,d,k) rt, dt, dk = r, d, k return true end) - local ekat = (dt and dt[dk] and dt[dk].at) or (rt and rt.at) - if ekat then - if arguments then - return ekat[arguments] or "", rt, dt, dk +local lp_special = (C(P("name")+P("text")+P("tag")+P("count")+P("child"))) * value / function(t,s) + if expressions[t] then + s = s and s ~= "" and lpegmatch(cleaner,s) + if s and s ~= "" then + return "expr." .. t .. "(ll," .. s ..")" else - return ekat, rt, dt, dk + return "expr." .. t .. "(ll)" end else - return { }, rt, dt, dk + return "expr.error(" .. t .. ")" end end -function xml.filters.reverse(root,pattern) - local rt, dt, dk - traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end, 'reverse') - return dt and dt[dk], rt, dt, dk -end +local content = + lp_builtin + + lp_attribute + + lp_special + + lp_noequal + lp_doequal + + lp_or + lp_and + + lp_reserved + + lp_lua_function + lp_function + + lp_content + -- too fragile + lp_child + + lp_any + +local converter = Cs ( + lp_fastpos + (P { lparent * (V(1))^0 * rparent + content } )^0 +) -function xml.filters.count(root,pattern,everything) - local n = 0 - traverse(root, lpath(pattern), function(r,d,t) - if everything or type(d[t]) == "table" then - n = n + 1 - end - end) - return n -end +cleaner = Cs ( ( +--~ lp_fastpos + + lp_reserved + + lp_number + + lp_string + +1 )^1 ) -function xml.filters.elements(root, pattern) -- == all - local t = { } - traverse(root, lpath(pattern), function(r,d,k) - local e = d[k] - if e then - t[#t+1] = e - end - end) - return t -end -function xml.filters.texts(root, pattern) - local t = { } - traverse(root, lpath(pattern), function(r,d,k) - local e = d[k] - if e and e.dt then - t[#t+1] = e.dt - end - end) - return t -end +--~ expr -function xml.filters.first(root,pattern) - local rt, dt, dk - traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end) - return dt and dt[dk], rt, dt, dk -end +local template_e = [[ + local expr = xml.expressions + return function(list,ll,l,order) + return %s + end +]] -function xml.filters.last(root,pattern) - local rt, dt, dk - traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end, 'reverse') - return dt and dt[dk], rt, dt, dk -end +local template_f_y = [[ + local finalizer = xml.finalizers['%s']['%s'] + return function(collection) + return finalizer(collection,%s) + end +]] -function xml.filters.index(root,pattern,arguments) - local rt, dt, dk, reverse, i = nil, nil, nil, false, tonumber(arguments or '1') or 1 - if i and i ~= 0 then - if i < 0 then - reverse, i = true, -i - end - traverse(root, lpath(pattern), function(r,d,k) rt, dt, dk, i = r, d, k, i-1 return i == 0 end, reverse) - if i == 0 then - return dt and dt[dk], rt, dt, dk - end +local template_f_n = [[ + return xml.finalizers['%s']['%s'] +]] + +-- + +local register_self = { kind = "axis", axis = "self" } -- , apply = apply_axis["self"] } +local register_parent = { kind = "axis", axis = "parent" } -- , apply = apply_axis["parent"] } +local register_descendant = { kind = "axis", axis = "descendant" } -- , apply = apply_axis["descendant"] } +local register_child = { kind = "axis", axis = "child" } -- , apply = apply_axis["child"] } +local register_descendant_or_self = { kind = "axis", axis = "descendant-or-self" } -- , apply = apply_axis["descendant-or-self"] } +local register_root = { kind = "axis", axis = "root" } -- , apply = apply_axis["root"] } +local register_ancestor = { kind = "axis", axis = "ancestor" } -- , apply = apply_axis["ancestor"] } +local register_ancestor_or_self = { kind = "axis", axis = "ancestor-or-self" } -- , apply = apply_axis["ancestor-or-self"] } +local register_attribute = { kind = "axis", axis = "attribute" } -- , apply = apply_axis["attribute"] } +local register_namespace = { kind = "axis", axis = "namespace" } -- , apply = apply_axis["namespace"] } +local register_following = { kind = "axis", axis = "following" } -- , apply = apply_axis["following"] } +local register_following_sibling = { kind = "axis", axis = "following-sibling" } -- , apply = apply_axis["following-sibling"] } +local register_preceding = { kind = "axis", axis = "preceding" } -- , apply = apply_axis["preceding"] } +local register_preceding_sibling = { kind = "axis", axis = "preceding-sibling" } -- , apply = apply_axis["preceding-sibling"] } +local register_reverse_sibling = { kind = "axis", axis = "reverse-sibling" } -- , apply = apply_axis["reverse-sibling"] } + +local register_auto_descendant_or_self = { kind = "axis", axis = "auto-descendant-or-self" } -- , apply = apply_axis["auto-descendant-or-self"] } +local register_auto_descendant = { kind = "axis", axis = "auto-descendant" } -- , apply = apply_axis["auto-descendant"] } +local register_auto_self = { kind = "axis", axis = "auto-self" } -- , apply = apply_axis["auto-self"] } +local register_auto_child = { kind = "axis", axis = "auto-child" } -- , apply = apply_axis["auto-child"] } + +local register_initial_child = { kind = "axis", axis = "initial-child" } -- , apply = apply_axis["initial-child"] } + +local register_all_nodes = { kind = "nodes", nodetest = true, nodes = { true, false, false } } + +local skip = { } + +local function errorrunner_e(str,cnv) + if not skip[str] then + logs.report("lpath","error in expression: %s => %s",str,cnv) + skip[str] = cnv or str end - return nil, nil, nil, nil + return false +end +local function errorrunner_f(str,arg) + logs.report("lpath","error in finalizer: %s(%s)",str,arg or "") + return false end -function xml.filters.attribute(root,pattern,arguments) - local rt, dt, dk - traverse(root, lpath(pattern), function(r,d,k) rt, dt, dk = r, d, k return true end) - local ekat = (dt and dt[dk] and dt[dk].at) or (rt and rt.at) - return (ekat and (ekat[arguments] or ekat[gsub(arguments,"^([\"\'])(.*)%1$","%2")])) or "" +local function register_nodes(nodetest,nodes) + return { kind = "nodes", nodetest = nodetest, nodes = nodes } end -function xml.filters.text(root,pattern,arguments) -- ?? why index, tostring slow - local dtk, rt, dt, dk = xml.filters.index(root,pattern,arguments) - if dtk then -- n - local dtkdt = dtk.dt - if not dtkdt then - return "", rt, dt, dk - elseif #dtkdt == 1 and type(dtkdt[1]) == "string" then - return dtkdt[1], rt, dt, dk - else - return xml.tostring(dtkdt), rt, dt, dk - end +local function register_expression(expression) + local converted = lpegmatch(converter,expression) + local runner = loadstring(format(template_e,converted)) + runner = (runner and runner()) or function() errorrunner_e(expression,converted) end + return { kind = "expression", expression = expression, converted = converted, evaluator = runner } +end + +local function register_finalizer(protocol,name,arguments) + local runner + if arguments and arguments ~= "" then + runner = loadstring(format(template_f_y,protocol or xml.defaultprotocol,name,arguments)) else - return "", rt, dt, dk + runner = loadstring(format(template_f_n,protocol or xml.defaultprotocol,name)) end + runner = (runner and runner()) or function() errorrunner_f(name,arguments) end + return { kind = "finalizer", name = name, arguments = arguments, finalizer = runner } end -function xml.filters.tag(root,pattern,n) - local tag = "" - traverse(root, lpath(pattern), function(r,d,k) - tag = xml.functions.tag(d,k,n and tonumber(n)) - return true - end) - return tag -end +local expression = P { "ex", + ex = "[" * C((V("sq") + V("dq") + (1 - S("[]")) + V("ex"))^0) * "]", + sq = "'" * (1 - S("'"))^0 * "'", + dq = '"' * (1 - S('"'))^0 * '"', +} -function xml.filters.name(root,pattern,n) - local tag = "" - traverse(root, lpath(pattern), function(r,d,k) - tag = xml.functions.name(d,k,n and tonumber(n)) - return true - end) - return tag +local arguments = P { "ar", + ar = "(" * Cs((V("sq") + V("dq") + V("nq") + P(1-P(")")))^0) * ")", + nq = ((1 - S("),'\""))^1) / function(s) return format("%q",s) end, + sq = P("'") * (1 - P("'"))^0 * P("'"), + dq = P('"') * (1 - P('"'))^0 * P('"'), +} + +-- todo: better arg parser + +local function register_error(str) + return { kind = "error", error = format("unparsed: %s",str) } end ---[[ldx-- -<p>For splitting the filter function from the path specification, we can -use string matching or lpeg matching. Here the difference in speed is -neglectable but the lpeg variant is more robust.</p> ---ldx]]-- +-- there is a difference in * and /*/ and so we need to catch a few special cases + +local special_1 = P("*") * Cc(register_auto_descendant) * Cc(register_all_nodes) -- last one not needed +local special_2 = P("/") * Cc(register_auto_self) +local special_3 = P("") * Cc(register_auto_self) + +local parser = Ct { "patterns", -- can be made a bit faster by moving pattern outside + + patterns = spaces * V("protocol") * spaces * ( + ( V("special") * spaces * P(-1) ) + + ( V("initial") * spaces * V("step") * spaces * (P("/") * spaces * V("step") * spaces)^0 ) + ), + + protocol = Cg(V("letters"),"protocol") * P("://") + Cg(Cc(nil),"protocol"), + + -- the / is needed for // as descendant or self is somewhat special + -- step = (V("shortcuts") + V("axis") * spaces * V("nodes")^0 + V("error")) * spaces * V("expressions")^0 * spaces * V("finalizer")^0, + step = ((V("shortcuts") + P("/") + V("axis")) * spaces * V("nodes")^0 + V("error")) * spaces * V("expressions")^0 * spaces * V("finalizer")^0, + + axis = V("descendant") + V("child") + V("parent") + V("self") + V("root") + V("ancestor") + + V("descendant_or_self") + V("following_sibling") + V("following") + + V("reverse_sibling") + V("preceding_sibling") + V("preceding") + V("ancestor_or_self") + + #(1-P(-1)) * Cc(register_auto_child), + + special = special_1 + special_2 + special_3, --- not faster but hipper ... although ... i can't get rid of the trailing / in the path + initial = (P("/") * spaces * Cc(register_initial_child))^-1, -local P, S, R, C, V, Cc = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc + error = (P(1)^1) / register_error, -local slash = P('/') -local name = (R("az","AZ","--","__"))^1 -local path = C(((1-slash)^0 * slash)^1) -local argument = P { "(" * C(((1 - S("()")) + V(1))^0) * ")" } -local action = Cc(1) * path * C(name) * argument -local attribute = Cc(2) * path * P('@') * C(name) -local direct = Cc(3) * Cc("../*") * slash^0 * C(name) * argument + shortcuts_a = V("s_descendant_or_self") + V("s_descendant") + V("s_child") + V("s_parent") + V("s_self") + V("s_root") + V("s_ancestor"), -local parser = direct + action + attribute + shortcuts = V("shortcuts_a") * (spaces * "/" * spaces * V("shortcuts_a"))^0, -local filters = xml.filters -local attribute_filter = xml.filters.attributes -local default_filter = xml.filters.default + s_descendant_or_self = (P("***/") + P("/")) * Cc(register_descendant_or_self), --- *** is a bonus + -- s_descendant_or_self = P("/") * Cc(register_descendant_or_self), + s_descendant = P("**") * Cc(register_descendant), + s_child = P("*") * #(1-P(":")) * Cc(register_child ), +-- s_child = P("*") * #(P("/")+P(-1)) * Cc(register_child ), + s_parent = P("..") * Cc(register_parent ), + s_self = P("." ) * Cc(register_self ), + s_root = P("^^") * Cc(register_root ), + s_ancestor = P("^") * Cc(register_ancestor ), --- todo: also hash, could be gc'd + descendant = P("descendant::") * Cc(register_descendant ), + child = P("child::") * Cc(register_child ), + parent = P("parent::") * Cc(register_parent ), + self = P("self::") * Cc(register_self ), + root = P('root::') * Cc(register_root ), + ancestor = P('ancestor::') * Cc(register_ancestor ), + descendant_or_self = P('descendant-or-self::') * Cc(register_descendant_or_self ), + ancestor_or_self = P('ancestor-or-self::') * Cc(register_ancestor_or_self ), + -- attribute = P('attribute::') * Cc(register_attribute ), + -- namespace = P('namespace::') * Cc(register_namespace ), + following = P('following::') * Cc(register_following ), + following_sibling = P('following-sibling::') * Cc(register_following_sibling ), + preceding = P('preceding::') * Cc(register_preceding ), + preceding_sibling = P('preceding-sibling::') * Cc(register_preceding_sibling ), + reverse_sibling = P('reverse-sibling::') * Cc(register_reverse_sibling ), + + nodes = (V("nodefunction") * spaces * P("(") * V("nodeset") * P(")") + V("nodetest") * V("nodeset")) / register_nodes, + + expressions = expression / register_expression, + + letters = R("az")^1, + name = (1-lpeg.S("/[]()|:*!"))^1, + negate = P("!") * Cc(false), + + nodefunction = V("negate") + P("not") * Cc(false) + Cc(true), + nodetest = V("negate") + Cc(true), + nodename = (V("negate") + Cc(true)) * spaces * ((V("wildnodename") * P(":") * V("wildnodename")) + (Cc(false) * V("wildnodename"))), + wildnodename = (C(V("name")) + P("*") * Cc(false)) * #(1-P("(")), + nodeset = spaces * Ct(V("nodename") * (spaces * P("|") * spaces * V("nodename"))^0) * spaces, + + finalizer = (Cb("protocol") * P("/")^-1 * C(V("name")) * arguments * P(-1)) / register_finalizer, + +} + +local cache = { } -function xml.filter(root,pattern) - local kind, a, b, c = parser:match(pattern) - if kind == 1 or kind == 3 then - return (filters[b] or default_filter)(root,a,c) - elseif kind == 2 then - return attribute_filter(root,a,b) +local function nodesettostring(set,nodetest) + local t = { } + for i=1,#set,3 do + local directive, ns, tg = set[i], set[i+1], set[i+2] + if not ns or ns == "" then ns = "*" end + if not tg or tg == "" then tg = "*" end + tg = (tg == "@rt@" and "[root]") or format("%s:%s",ns,tg) + t[#t+1] = (directive and tg) or format("not(%s)",tg) + end + if nodetest == false then + return format("not(%s)",concat(t,"|")) else - return default_filter(root,pattern) + return concat(t,"|") end end ---~ slightly faster, but first we need a proper test file ---~ ---~ local hash = { } ---~ ---~ function xml.filter(root,pattern) ---~ local h = hash[pattern] ---~ if not h then ---~ local kind, a, b, c = parser:match(pattern) ---~ if kind == 1 then ---~ h = { kind, filters[b] or default_filter, a, b, c } ---~ elseif kind == 2 then ---~ h = { kind, attribute_filter, a, b, c } ---~ else ---~ h = { kind, default_filter, a, b, c } ---~ end ---~ hash[pattern] = h ---~ end ---~ local kind = h[1] ---~ if kind == 1 then ---~ return h[2](root,h[2],h[4]) ---~ elseif kind == 2 then ---~ return h[2](root,h[2],h[3]) ---~ else ---~ return h[2](root,pattern) ---~ end ---~ end +local function tagstostring(list) + if #list == 0 then + return "no elements" + else + local t = { } + for i=1, #list do + local li = list[i] + local ns, tg = li.ns, li.tg + if not ns or ns == "" then ns = "*" end + if not tg or tg == "" then tg = "*" end + t[#t+1] = (tg == "@rt@" and "[root]") or format("%s:%s",ns,tg) + end + return concat(t," ") + end +end ---[[ldx-- -<p>The following functions collect elements and texts.</p> ---ldx]]-- +xml.nodesettostring = nodesettostring --- still somewhat bugged +local parse_pattern -- we have a harmless kind of circular reference -function xml.collect_elements(root, pattern, ignorespaces) - local rr, dd = { }, { } - traverse(root, lpath(pattern), function(r,d,k) - local dk = d and d[k] - if dk then - if ignorespaces and type(dk) == "string" and dk:find("[^%S]") then - -- ignore - else - local n = #rr+1 - rr[n], dd[n] = r, dk - end - end - end) - return dd, rr +local function lshow(parsed) + if type(parsed) == "string" then + parsed = parse_pattern(parsed) + end + local s = table.serialize_functions -- ugly + table.serialize_functions = false -- ugly + logs.report("lpath","%s://%s => %s",parsed.protocol or xml.defaultprotocol,parsed.pattern,table.serialize(parsed,false)) + table.serialize_functions = s -- ugly end -function xml.collect_texts(root, pattern, flatten) - local t = { } -- no r collector - traverse(root, lpath(pattern), function(r,d,k) - if d then - local ek = d[k] - local tx = ek and ek.dt - if flatten then - if tx then - t[#t+1] = xml.tostring(tx) or "" +xml.lshow = lshow + +local function add_comment(p,str) + local pc = p.comment + if not pc then + p.comment = { str } + else + pc[#pc+1] = str + end +end + +parse_pattern = function (pattern) -- the gain of caching is rather minimal + lpathcalls = lpathcalls + 1 + if type(pattern) == "table" then + return pattern + else + local parsed = cache[pattern] + if parsed then + lpathcached = lpathcached + 1 + else + parsed = lpegmatch(parser,pattern) + if parsed then + parsed.pattern = pattern + local np = #parsed + if np == 0 then + parsed = { pattern = pattern, register_self, state = "parsing error" } + logs.report("lpath","parsing error in '%s'",pattern) + lshow(parsed) else - t[#t+1] = "" + -- we could have done this with a more complex parser but this + -- is cleaner + local pi = parsed[1] + if pi.axis == "auto-child" then + if false then + add_comment(parsed, "auto-child replaced by auto-descendant-or-self") + parsed[1] = register_auto_descendant_or_self + else + add_comment(parsed, "auto-child replaced by auto-descendant") + parsed[1] = register_auto_descendant + end + elseif pi.axis == "initial-child" and np > 1 and parsed[2].axis then + add_comment(parsed, "initial-child removed") -- we could also make it a auto-self + remove(parsed,1) + end + local np = #parsed -- can have changed + if np > 1 then + local pnp = parsed[np] + if pnp.kind == "nodes" and pnp.nodetest == true then + local nodes = pnp.nodes + if nodes[1] == true and nodes[2] == false and nodes[3] == false then + add_comment(parsed, "redundant final wildcard filter removed") + remove(parsed,np) + end + end + end end else - t[#t+1] = tx or "" + parsed = { pattern = pattern } + end + cache[pattern] = parsed + if trace_lparse and not trace_lprofile then + lshow(parsed) end - else - t[#t+1] = "" end - end) - return t + return parsed + end end -function xml.collect_tags(root, pattern, nonamespace) - local t = { } - xml.traverse(root, xml.lpath(pattern), function(r,d,k) - local dk = d and d[k] - if dk and type(dk) == "table" then - local ns, tg = e.ns, e.tg - if nonamespace then - t[#t+1] = tg -- if needed we can return an extra table - elseif ns == "" then - t[#t+1] = tg - else - t[#t+1] = ns .. ":" .. tg +-- we can move all calls inline and then merge the trace back +-- technically we can combine axis and the next nodes which is +-- what we did before but this a bit cleaner (but slower too) +-- but interesting is that it's not that much faster when we +-- go inline +-- +-- beware: we need to return a collection even when we filter +-- else the (simple) cache gets messed up + +-- caching found lookups saves not that much (max .1 sec on a 8 sec run) +-- and it also messes up finalizers + +-- watch out: when there is a finalizer, it's always called as there +-- can be cases that a finalizer returns (or does) something in case +-- there is no match; an example of this is count() + +local profiled = { } xml.profiled = profiled + +local function profiled_apply(list,parsed,nofparsed,order) + local p = profiled[parsed.pattern] + if p then + p.tested = p.tested + 1 + else + p = { tested = 1, matched = 0, finalized = 0 } + profiled[parsed.pattern] = p + end + local collected = list + for i=1,nofparsed do + local pi = parsed[i] + local kind = pi.kind + if kind == "axis" then + collected = apply_axis[pi.axis](collected) + elseif kind == "nodes" then + collected = apply_nodes(collected,pi.nodetest,pi.nodes) + elseif kind == "expression" then + collected = apply_expression(collected,pi.evaluator,order) + elseif kind == "finalizer" then + collected = pi.finalizer(collected) + p.matched = p.matched + 1 + p.finalized = p.finalized + 1 + return collected + end + if not collected or #collected == 0 then + local pn = i < nofparsed and parsed[nofparsed] + if pn and pn.kind == "finalizer" then + collected = pn.finalizer(collected) + p.finalized = p.finalized + 1 + return collected end + return nil end - end) - return #t > 0 and {} + end + if collected then + p.matched = p.matched + 1 + end + return collected +end + +local function traced_apply(list,parsed,nofparsed,order) + if trace_lparse then + lshow(parsed) + end + logs.report("lpath", "collecting : %s",parsed.pattern) + logs.report("lpath", " root tags : %s",tagstostring(list)) + logs.report("lpath", " order : %s",order or "unset") + local collected = list + for i=1,nofparsed do + local pi = parsed[i] + local kind = pi.kind + if kind == "axis" then + collected = apply_axis[pi.axis](collected) + logs.report("lpath", "% 10i : ax : %s",(collected and #collected) or 0,pi.axis) + elseif kind == "nodes" then + collected = apply_nodes(collected,pi.nodetest,pi.nodes) + logs.report("lpath", "% 10i : ns : %s",(collected and #collected) or 0,nodesettostring(pi.nodes,pi.nodetest)) + elseif kind == "expression" then + collected = apply_expression(collected,pi.evaluator,order) + logs.report("lpath", "% 10i : ex : %s -> %s",(collected and #collected) or 0,pi.expression,pi.converted) + elseif kind == "finalizer" then + collected = pi.finalizer(collected) + logs.report("lpath", "% 10i : fi : %s : %s(%s)",(type(collected) == "table" and #collected) or 0,parsed.protocol or xml.defaultprotocol,pi.name,pi.arguments or "") + return collected + end + if not collected or #collected == 0 then + local pn = i < nofparsed and parsed[nofparsed] + if pn and pn.kind == "finalizer" then + collected = pn.finalizer(collected) + logs.report("lpath", "% 10i : fi : %s : %s(%s)",(type(collected) == "table" and #collected) or 0,parsed.protocol or xml.defaultprotocol,pn.name,pn.arguments or "") + return collected + end + return nil + end + end + return collected end ---[[ldx-- -<p>Often using an iterators looks nicer in the code than passing handler -functions. The <l n='lua'/> book describes how to use coroutines for that -purpose (<url href='http://www.lua.org/pil/9.3.html'/>). This permits -code like:</p> - -<typing> -for r, d, k in xml.elements(xml.load('text.xml'),"title") do - print(d[k]) +local function normal_apply(list,parsed,nofparsed,order) + local collected = list + for i=1,nofparsed do + local pi = parsed[i] + local kind = pi.kind + if kind == "axis" then + local axis = pi.axis + if axis ~= "self" then + collected = apply_axis[axis](collected) + end + elseif kind == "nodes" then + collected = apply_nodes(collected,pi.nodetest,pi.nodes) + elseif kind == "expression" then + collected = apply_expression(collected,pi.evaluator,order) + elseif kind == "finalizer" then + return pi.finalizer(collected) + end + if not collected or #collected == 0 then + local pf = i < nofparsed and parsed[nofparsed].finalizer + if pf then + return pf(collected) -- can be anything + end + return nil + end + end + return collected end -</typing> - -<p>Which will print all the titles in the document. The iterator variant takes -1.5 times the runtime of the function variant which is due to the overhead in -creating the wrapper. So, instead of:</p> -<typing> -function xml.filters.first(root,pattern) - for rt,dt,dk in xml.elements(root,pattern) - return dt and dt[dk], rt, dt, dk +local function parse_apply(list,pattern) + -- we avoid an extra call + local parsed = cache[pattern] + if parsed then + lpathcalls = lpathcalls + 1 + lpathcached = lpathcached + 1 + elseif type(pattern) == "table" then + lpathcalls = lpathcalls + 1 + parsed = pattern + else + parsed = parse_pattern(pattern) or pattern + end + if not parsed then + return + end + local nofparsed = #parsed + if nofparsed == 0 then + return -- something is wrong + end + local one = list[1] + if not one then + return -- something is wrong + elseif not trace_lpath then + return normal_apply(list,parsed,nofparsed,one.mi) + elseif trace_lprofile then + return profiled_apply(list,parsed,nofparsed,one.mi) + else + return traced_apply(list,parsed,nofparsed,one.mi) end - return nil, nil, nil, nil end -</typing> -<p>We use the function variants in the filters.</p> ---ldx]]-- - -local wrap, yield = coroutine.wrap, coroutine.yield +-- internal (parsed) -function xml.elements(root,pattern,reverse) - return wrap(function() traverse(root, lpath(pattern), yield, reverse) end) +expressions.child = function(e,pattern) + return parse_apply({ e },pattern) -- todo: cache end - -function xml.elements_only(root,pattern,reverse) - return wrap(function() traverse(root, lpath(pattern), function(r,d,k) yield(d[k]) end, reverse) end) +expressions.count = function(e,pattern) + local collected = parse_apply({ e },pattern) -- todo: cache + return (collected and #collected) or 0 end -function xml.each_element(root, pattern, handle, reverse) - local ok - traverse(root, lpath(pattern), function(r,d,k) ok = true handle(r,d,k) end, reverse) - return ok +-- external + +expressions.oneof = function(s,...) -- slow + local t = {...} for i=1,#t do if s == t[i] then return true end end return false +end +expressions.error = function(str) + xml.error_handler("unknown function in lpath expression",tostring(str or "?")) + return false +end +expressions.undefined = function(s) + return s == nil end -function xml.process_elements(root, pattern, handle) - traverse(root, lpath(pattern), function(r,d,k) - local dkdt = d[k].dt - if dkdt then - for i=1,#dkdt do - local v = dkdt[i] - if v.tg then handle(v) end - end - end - end) +expressions.quit = function(s) + if s or s == nil then + quit_expression = true + end + return true end -function xml.process_attributes(root, pattern, handle) - traverse(root, lpath(pattern), function(r,d,k) - local ek = d[k] - local a = ek.at or { } - handle(a) - if next(a) then -- next is faster than type (and >0 test) - ek.at = a - else - ek.at = nil - end - end) +expressions.print = function(...) + print(...) + return true end ---[[ldx-- -<p>We've now arrives at the functions that manipulate the tree.</p> ---ldx]]-- +expressions.contains = find +expressions.find = find +expressions.upper = upper +expressions.lower = lower +expressions.number = tonumber +expressions.boolean = toboolean -function xml.inject_element(root, pattern, element, prepend) - if root and element then - local matches, collect = { }, nil - if type(element) == "string" then - element = convert(element,true) - end - if element then - collect = function(r,d,k) matches[#matches+1] = { r, d, k, element } end - traverse(root, lpath(pattern), collect) - for i=1,#matches do - local m = matches[i] - local r, d, k, element, edt = m[1], m[2], m[3], m[4], nil - if element.ri then - element = element.dt[element.ri].dt - else - element = element.dt - end - if r.ri then - edt = r.dt[r.ri].dt - else - edt = d and d[k] and d[k].dt - end - if edt then - local be, af - if prepend then - be, af = xml.copy(element), edt - else - be, af = edt, xml.copy(element) - end - for i=1,#af do - be[#be+1] = af[i] - end - if r.ri then - r.dt[r.ri].dt = be - else - d[k].dt = be - end - else - -- r.dt = element.dt -- todo - end - end +-- user interface + +local function traverse(root,pattern,handle) + logs.report("xml","use 'xml.selection' instead for '%s'",pattern) + local collected = parse_apply({ root },pattern) + if collected then + for c=1,#collected do + local e = collected[c] + local r = e.__p__ + handle(r,r.dt,e.ni) end end end --- todo: copy ! - -function xml.insert_element(root, pattern, element, before) -- todo: element als functie - if root and element then - if pattern == "/" then - xml.inject_element(root, pattern, element, before) - else - local matches, collect = { }, nil - if type(element) == "string" then - element = convert(element,true) - end - if element and element.ri then - element = element.dt[element.ri] - end - if element then - collect = function(r,d,k) matches[#matches+1] = { r, d, k, element } end - traverse(root, lpath(pattern), collect) - for i=#matches,1,-1 do - local m = matches[i] - local r, d, k, element = m[1], m[2], m[3], m[4] - if not before then k = k + 1 end - if element.tg then - insert(d,k,element) -- untested ---~ elseif element.dt then ---~ for _,v in ipairs(element.dt) do -- i added ---~ insert(d,k,v) ---~ k = k + 1 ---~ end ---~ end - else - local edt = element.dt - if edt then - for i=1,#edt do - insert(d,k,edt[i]) - k = k + 1 - end - end - end - end +local function selection(root,pattern,handle) + local collected = parse_apply({ root },pattern) + if collected then + if handle then + for c=1,#collected do + handle(collected[c]) end + else + return collected end end end -xml.insert_element_after = xml.insert_element -xml.insert_element_before = function(r,p,e) xml.insert_element(r,p,e,true) end -xml.inject_element_after = xml.inject_element -xml.inject_element_before = function(r,p,e) xml.inject_element(r,p,e,true) end +xml.parse_parser = parser +xml.parse_pattern = parse_pattern +xml.parse_apply = parse_apply +xml.traverse = traverse -- old method, r, d, k +xml.selection = selection -- new method, simple handle -function xml.delete_element(root, pattern) - local matches, deleted = { }, { } - local collect = function(r,d,k) matches[#matches+1] = { r, d, k } end - traverse(root, lpath(pattern), collect) - for i=#matches,1,-1 do - local m = matches[i] - deleted[#deleted+1] = remove(m[2],m[3]) - end - return deleted +local lpath = parse_pattern + +xml.lpath = lpath + +function xml.cached_patterns() + return cache end -function xml.replace_element(root, pattern, element) - if type(element) == "string" then - element = convert(element,true) - end - if element and element.ri then - element = element.dt[element.ri] - end - if element then - traverse(root, lpath(pattern), function(rm, d, k) - d[k] = element.dt -- maybe not clever enough - end) +-- generic function finalizer (independant namespace) + +local function dofunction(collected,fnc) + if collected then + local f = functions[fnc] + if f then + for c=1,#collected do + f(collected[c]) + end + else + logs.report("xml","unknown function '%s'",fnc) + end end end -local function load_data(name) -- == io.loaddata - local f, data = io.open(name), "" - if f then - data = f:read("*all",'b') -- 'b' ? - f:close() - end - return data +xml.finalizers.xml["function"] = dofunction +xml.finalizers.tex["function"] = dofunction + +-- functions + +expressions.text = function(e,n) + local rdt = e.__p__.dt + return (rdt and rdt[n]) or "" end -function xml.include(xmldata,pattern,attribute,recursive,loaddata) - -- parse="text" (default: xml), encoding="" (todo) - -- attribute = attribute or 'href' - pattern = pattern or 'include' - loaddata = loaddata or load_data - local function include(r,d,k) - local ek, name = d[k], nil - if not attribute or attribute == "" then - local ekdt = ek.dt - name = (type(ekdt) == "table" and ekdt[1]) or ekdt - end - if not name then - if ek.at then - for a in gmatch(attribute or "href","([^|]+)") do - name = ek.at[a] - if name then break end +expressions.name = function(e,n) -- ns + tg + local found = false + n = tonumber(n) or 0 + if n == 0 then + found = type(e) == "table" and e + elseif n < 0 then + local d, k = e.__p__.dt, e.ni + for i=k-1,1,-1 do + local di = d[i] + if type(di) == "table" then + if n == -1 then + found = di + break + else + n = n + 1 end end end - local data = (name and name ~= "" and loaddata(name)) or "" - if data == "" then - xml.empty(d,k) - elseif ek.at["parse"] == "text" then -- for the moment hard coded - d[k] = xml.escaped(data) - else - local xi = xml.convert(data) - if not xi then - xml.empty(d,k) - else - if recursive then - xml.include(xi,pattern,attribute,recursive,loaddata) + else + local d, k = e.__p__.dt, e.ni + for i=k+1,#d,1 do + local di = d[i] + if type(di) == "table" then + if n == 1 then + found = di + break + else + n = n - 1 end - xml.assign(d,k,xi) end end end - xml.each_element(xmldata, pattern, include) + if found then + local ns, tg = found.rn or found.ns or "", found.tg + if ns ~= "" then + return ns .. ":" .. tg + else + return tg + end + else + return "" + end end -function xml.strip_whitespace(root, pattern, nolines) -- strips all leading and trailing space ! - traverse(root, lpath(pattern), function(r,d,k) - local dkdt = d[k].dt - if dkdt then -- can be optimized - local t = { } - for i=1,#dkdt do - local str = dkdt[i] - if type(str) == "string" then - - if str == "" then - -- stripped +expressions.tag = function(e,n) -- only tg + if not e then + return "" + else + local found = false + n = tonumber(n) or 0 + if n == 0 then + found = (type(e) == "table") and e -- seems to fail + elseif n < 0 then + local d, k = e.__p__.dt, e.ni + for i=k-1,1,-1 do + local di = d[i] + if type(di) == "table" then + if n == -1 then + found = di + break else - if nolines then - str = gsub(str,"[ \n\r\t]+"," ") - end - if str == "" then - -- stripped - else - t[#t+1] = str - end + n = n + 1 end - else - t[#t+1] = str end end - d[k].dt = t - end - end) -end - -local function rename_space(root, oldspace, newspace) -- fast variant - local ndt = #root.dt - for i=1,ndt or 0 do - local e = root[i] - if type(e) == "table" then - if e.ns == oldspace then - e.ns = newspace - if e.rn then - e.rn = newspace + else + local d, k = e.__p__.dt, e.ni + for i=k+1,#d,1 do + local di = d[i] + if type(di) == "table" then + if n == 1 then + found = di + break + else + n = n - 1 + end end end - local edt = e.dt - if edt then - rename_space(edt, oldspace, newspace) - end end + return (found and found.tg) or "" end end -xml.rename_space = rename_space +--[[ldx-- +<p>This is the main filter function. It returns whatever is asked for.</p> +--ldx]]-- -function xml.remap_tag(root, pattern, newtg) - traverse(root, lpath(pattern), function(r,d,k) - d[k].tg = newtg - end) +function xml.filter(root,pattern) -- no longer funny attribute handling here + return parse_apply({ root },pattern) end -function xml.remap_namespace(root, pattern, newns) - traverse(root, lpath(pattern), function(r,d,k) - d[k].ns = newns - end) + +--[[ldx-- +<p>Often using an iterators looks nicer in the code than passing handler +functions. The <l n='lua'/> book describes how to use coroutines for that +purpose (<url href='http://www.lua.org/pil/9.3.html'/>). This permits +code like:</p> + +<typing> +for r, d, k in xml.elements(xml.load('text.xml'),"title") do + print(d[k]) -- old method end -function xml.check_namespace(root, pattern, newns) - traverse(root, lpath(pattern), function(r,d,k) - local dk = d[k] - if (not dk.rn or dk.rn == "") and dk.ns == "" then - dk.rn = newns - end - end) +for e in xml.collected(xml.load('text.xml'),"title") do + print(e) -- new one end -function xml.remap_name(root, pattern, newtg, newns, newrn) - traverse(root, lpath(pattern), function(r,d,k) - local dk = d[k] - dk.tg = newtg - dk.ns = newns - dk.rn = newrn - end) +</typing> +--ldx]]-- + +local wrap, yield = coroutine.wrap, coroutine.yield + +function xml.elements(root,pattern,reverse) -- r, d, k + local collected = parse_apply({ root },pattern) + if collected then + if reverse then + return wrap(function() for c=#collected,1,-1 do + local e = collected[c] local r = e.__p__ yield(r,r.dt,e.ni) + end end) + else + return wrap(function() for c=1,#collected do + local e = collected[c] local r = e.__p__ yield(r,r.dt,e.ni) + end end) + end + end + return wrap(function() end) end -function xml.filters.found(root,pattern,check_content) - local found = false - traverse(root, lpath(pattern), function(r,d,k) - if check_content then - local dk = d and d[k] - found = dk and dk.dt and next(dk.dt) and true +function xml.collected(root,pattern,reverse) -- e + local collected = parse_apply({ root },pattern) + if collected then + if reverse then + return wrap(function() for c=#collected,1,-1 do yield(collected[c]) end end) else - found = true + return wrap(function() for c=1,#collected do yield(collected[c]) end end) end - return true - end) - return found + end + return wrap(function() end) end ---[[ldx-- -<p>Here are a few synonyms.</p> ---ldx]]-- -xml.filters.position = xml.filters.index +end -- of closure -xml.count = xml.filters.count -xml.index = xml.filters.index -xml.position = xml.filters.index -xml.first = xml.filters.first -xml.last = xml.filters.last -xml.found = xml.filters.found +do -- create closure to overcome 200 locals limit -xml.each = xml.each_element -xml.process = xml.process_element -xml.strip = xml.strip_whitespace -xml.collect = xml.collect_elements -xml.all = xml.collect_elements +if not modules then modules = { } end modules ['lxml-mis'] = { + version = 1.001, + comment = "this module is the basis for the lxml-* ones", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} -xml.insert = xml.insert_element_after -xml.inject = xml.inject_element_after -xml.after = xml.insert_element_after -xml.before = xml.insert_element_before -xml.delete = xml.delete_element -xml.replace = xml.replace_element +local concat = table.concat +local type, next, tonumber, tostring, setmetatable, loadstring = type, next, tonumber, tostring, setmetatable, loadstring +local format, gsub, match = string.format, string.gsub, string.match +local lpegmatch = lpeg.match --[[ldx-- -<p>The following helper functions best belong to the <t>lmxl-ini</t> +<p>The following helper functions best belong to the <t>lxml-ini</t> module. Some are here because we need then in the <t>mk</t> document and other manuals, others came up when playing with this module. Since this module is also used in <l n='mtxrun'/> we've put them here instead of loading mode modules there then needed.</p> --ldx]]-- -function xml.gsub(t,old,new) +local function xmlgsub(t,old,new) -- will be replaced local dt = t.dt if dt then for k=1,#dt do @@ -5069,28 +6366,26 @@ function xml.gsub(t,old,new) if type(v) == "string" then dt[k] = gsub(v,old,new) else - xml.gsub(v,old,new) + xmlgsub(v,old,new) end end end end +--~ xml.gsub = xmlgsub + function xml.strip_leading_spaces(dk,d,k) -- cosmetic, for manual - if d and k and d[k-1] and type(d[k-1]) == "string" then - local s = d[k-1]:match("\n(%s+)") - xml.gsub(dk,"\n"..string.rep(" ",#s),"\n") + if d and k then + local dkm = d[k-1] + if dkm and type(dkm) == "string" then + local s = match(dkm,"\n(%s+)") + xmlgsub(dk,"\n"..rep(" ",#s),"\n") + end end end -function xml.serialize_path(root,lpath,handle) - local dk, r, d, k = xml.first(root,lpath) - dk = xml.copy(dk) - xml.strip_leading_spaces(dk,d,k) - xml.serialize(dk,handle) -end - --~ xml.escapes = { ['&'] = '&', ['<'] = '<', ['>'] = '>', ['"'] = '"' } ---~ xml.unescapes = { } for k,v in pairs(xml.escapes) do xml.unescapes[v] = k end +--~ xml.unescapes = { } for k,v in next, xml.escapes do xml.unescapes[v] = k end --~ function xml.escaped (str) return (gsub(str,"(.)" , xml.escapes )) end --~ function xml.unescaped(str) return (gsub(str,"(&.-;)", xml.unescapes)) end @@ -5114,8 +6409,6 @@ local escaped = Cs(normal * (special * normal)^0) -- 100 * 1000 * "oeps< oeps> oeps&" : gsub:lpeg == 0153:0280:0151:0080 (last one by roberto) --- unescaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0) --- unescaped = Cs((((P("&")/"") * (P("lt")/"<" + P("gt")/">" + P("amp")/"&") * (P(";")/"")) + 1)^0) local normal = (1 - S"&")^0 local special = P("<")/"<" + P(">")/">" + P("&")/"&" local unescaped = Cs(normal * (special * normal)^0) @@ -5124,84 +6417,32 @@ local unescaped = Cs(normal * (special * normal)^0) local cleansed = Cs(((P("<") * (1-P(">"))^0 * P(">"))/"" + 1)^0) -function xml.escaped (str) return escaped :match(str) end -function xml.unescaped(str) return unescaped:match(str) end -function xml.cleansed (str) return cleansed :match(str) end +xml.escaped_pattern = escaped +xml.unescaped_pattern = unescaped +xml.cleansed_pattern = cleansed -function xml.join(t,separator,lastseparator) - if #t > 0 then - local result = { } - for k,v in pairs(t) do - result[k] = xml.tostring(v) - end - if lastseparator then - return concat(result,separator or "",1,#result-1) .. (lastseparator or "") .. result[#result] - else - return concat(result,separator) +function xml.escaped (str) return lpegmatch(escaped,str) end +function xml.unescaped(str) return lpegmatch(unescaped,str) end +function xml.cleansed (str) return lpegmatch(cleansed,str) end + +-- this might move + +function xml.fillin(root,pattern,str,check) + local e = xml.first(root,pattern) + if e then + local n = #e.dt + if not check or n == 0 or (n == 1 and e.dt[1] == "") then + e.dt = { str } end - else - return "" end end -function xml.statistics() - return { - lpathcalls = lpathcalls, - lpathcached = lpathcached, - } -end - --- xml.set_text_cleanup(xml.show_text_entities) --- xml.set_text_cleanup(xml.resolve_text_entities) - ---~ xml.lshow("/../../../a/(b|c)[@d='e']/f") ---~ xml.lshow("/../../../a/!(b|c)[@d='e']/f") ---~ xml.lshow("/../../../a/!b[@d!='e']/f") - ---~ x = xml.convert([[ ---~ <a> ---~ <b n='01'>01</b> ---~ <b n='02'>02</b> ---~ <b n='03'>03</b> ---~ <b n='04'>OK</b> ---~ <b n='05'>05</b> ---~ <b n='06'>06</b> ---~ <b n='07'>ALSO OK</b> ---~ </a> ---~ ]]) - ---~ xml.settrace("lpath",true) - ---~ xml.xshow(xml.first(x,"b[position() > 2 and position() < 5 and text() == 'ok']")) ---~ xml.xshow(xml.first(x,"b[position() > 2 and position() < 5 and text() == upper('ok')]")) ---~ xml.xshow(xml.first(x,"b[@n=='03' or @n=='08']")) ---~ xml.xshow(xml.all (x,"b[number(@n)>2 and number(@n)<6]")) ---~ xml.xshow(xml.first(x,"b[find(text(),'ALSO')]")) - ---~ str = [[ ---~ <?xml version="1.0" encoding="utf-8"?> ---~ <story line='mojca'> ---~ <windows>my secret</mouse> ---~ </story> ---~ ]] - ---~ x = xml.convert([[ ---~ <a><b n='01'>01</b><b n='02'>02</b><x>xx</x><b n='03'>03</b><b n='04'>OK</b></a> ---~ ]]) ---~ xml.xshow(xml.first(x,"b[tag(2) == 'x']")) ---~ xml.xshow(xml.first(x,"b[tag(1) == 'x']")) ---~ xml.xshow(xml.first(x,"b[tag(-1) == 'x']")) ---~ xml.xshow(xml.first(x,"b[tag(-2) == 'x']")) - ---~ print(xml.filter(x,"b/tag(2)")) ---~ print(xml.filter(x,"b/tag(1)")) - end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['lxml-ent'] = { +if not modules then modules = { } end modules ['lxml-aux'] = { version = 1.001, comment = "this module is the basis for the lxml-* ones", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", @@ -5209,457 +6450,836 @@ if not modules then modules = { } end modules ['lxml-ent'] = { license = "see context related readme files" } -local type, next, tonumber, tostring, setmetatable, loadstring = type, next, tonumber, tostring, setmetatable, loadstring -local format, gsub, find = string.format, string.gsub, string.find -local utfchar = unicode.utf8.char +-- not all functions here make sense anymore vbut we keep them for +-- compatibility reasons ---[[ldx-- -<p>We provide (at least here) two entity handlers. The more extensive -resolver consults a hash first, tries to convert to <l n='utf'/> next, -and finaly calls a handler when defines. When this all fails, the -original entity is returned.</p> ---ldx]]-- +local trace_manipulations = false trackers.register("lxml.manipulations", function(v) trace_manipulations = v end) -xml.entities = xml.entities or { } -- xml.entity_handler == function +local xmlparseapply, xmlconvert, xmlcopy, xmlname = xml.parse_apply, xml.convert, xml.copy, xml.name +local xmlinheritedconvert = xml.inheritedconvert -function xml.entity_handler(e) - return format("[%s]",e) -end +local type = type +local insert, remove = table.insert, table.remove +local gmatch, gsub = string.gmatch, string.gsub -local function toutf(s) - return utfchar(tonumber(s,16)) +local function report(what,pattern,c,e) + logs.report("xml","%s element '%s' (root: '%s', position: %s, index: %s, pattern: %s)",what,xmlname(e),xmlname(e.__p__),c,e.ni,pattern) end -local function utfize(root) - local d = root.dt - for k=1,#d do - local dk = d[k] - if type(dk) == "string" then - -- test prevents copying if no match - if find(dk,"&#x.-;") then - d[k] = gsub(dk,"&#x(.-);",toutf) +local function withelements(e,handle,depth) + if e and handle then + local edt = e.dt + if edt then + depth = depth or 0 + for i=1,#edt do + local e = edt[i] + if type(e) == "table" then + handle(e,depth) + withelements(e,handle,depth+1) + end end - else - utfize(dk) end end end -xml.utfize = utfize +xml.withelements = withelements -local function resolve(e) -- hex encoded always first, just to avoid mkii fallbacks - if find(e,"^#x") then - return utfchar(tonumber(e:sub(3),16)) - elseif find(e,"^#") then - return utfchar(tonumber(e:sub(2))) - else - local ee = xml.entities[e] -- we cannot shortcut this one (is reloaded) - if ee then - return ee - else - local h = xml.entity_handler - return (h and h(e)) or "&" .. e .. ";" +function xml.withelement(e,n,handle) -- slow + if e and n ~= 0 and handle then + local edt = e.dt + if edt then + if n > 0 then + for i=1,#edt do + local ei = edt[i] + if type(ei) == "table" then + if n == 1 then + handle(ei) + return + else + n = n - 1 + end + end + end + elseif n < 0 then + for i=#edt,1,-1 do + local ei = edt[i] + if type(ei) == "table" then + if n == -1 then + handle(ei) + return + else + n = n + 1 + end + end + end + end end end end -local function resolve_entities(root) - if not root.special or root.tg == "@rt@" then - local d = root.dt - for k=1,#d do - local dk = d[k] - if type(dk) == "string" then - if find(dk,"&.-;") then - d[k] = gsub(dk,"&(.-);",resolve) - end - else - resolve_entities(dk) +xml.elements_only = xml.collected + +function xml.each_element(root,pattern,handle,reverse) + local collected = xmlparseapply({ root },pattern) + if collected then + if reverse then + for c=#collected,1,-1 do + handle(collected[c]) + end + else + for c=1,#collected do + handle(collected[c]) end end + return collected end end -xml.resolve_entities = resolve_entities +xml.process_elements = xml.each_element -function xml.utfize_text(str) - if find(str,"&#") then - return (gsub(str,"&#x(.-);",toutf)) - else - return str +function xml.process_attributes(root,pattern,handle) + local collected = xmlparseapply({ root },pattern) + if collected and handle then + for c=1,#collected do + handle(collected[c].at) + end end + return collected end -function xml.resolve_text_entities(str) -- maybe an lpeg. maybe resolve inline - if find(str,"&") then - return (gsub(str,"&(.-);",resolve)) - else - return str - end +--[[ldx-- +<p>The following functions collect elements and texts.</p> +--ldx]]-- + +-- are these still needed -> lxml-cmp.lua + +function xml.collect_elements(root, pattern) + return xmlparseapply({ root },pattern) end -function xml.show_text_entities(str) - if find(str,"&") then - return (gsub(str,"&(.-);","[%1]")) - else - return str +function xml.collect_texts(root, pattern, flatten) -- todo: variant with handle + local collected = xmlparseapply({ root },pattern) + if collected and flatten then + local xmltostring = xml.tostring + for c=1,#collected do + collected[c] = xmltostring(collected[c].dt) + end end + return collected or { } end --- experimental, this will be done differently - -function xml.merge_entities(root) - local documententities = root.entities - local allentities = xml.entities - if documententities then - for k, v in next, documententities do - allentities[k] = v +function xml.collect_tags(root, pattern, nonamespace) + local collected = xmlparseapply({ root },pattern) + if collected then + local t = { } + for c=1,#collected do + local e = collected[c] + local ns, tg = e.ns, e.tg + if nonamespace then + t[#t+1] = tg + elseif ns == "" then + t[#t+1] = tg + else + t[#t+1] = ns .. ":" .. tg + end end + return t end end +--[[ldx-- +<p>We've now arrived at the functions that manipulate the tree.</p> +--ldx]]-- -end -- of closure +local no_root = { no_root = true } -do -- create closure to overcome 200 locals limit +function xml.redo_ni(d) + for k=1,#d do + local dk = d[k] + if type(dk) == "table" then + dk.ni = k + end + end +end -if not modules then modules = { } end modules ['lxml-mis'] = { - version = 1.001, - comment = "this module is the basis for the lxml-* ones", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} +local function xmltoelement(whatever,root) + if not whatever then + return nil + end + local element + if type(whatever) == "string" then + element = xmlinheritedconvert(whatever,root) + else + element = whatever -- we assume a table + end + if element.error then + return whatever -- string + end + if element then + --~ if element.ri then + --~ element = element.dt[element.ri].dt + --~ else + --~ element = element.dt + --~ end + end + return element +end -local concat = table.concat -local type, next, tonumber, tostring, setmetatable, loadstring = type, next, tonumber, tostring, setmetatable, loadstring -local format, gsub = string.format, string.gsub +xml.toelement = xmltoelement ---[[ldx-- -<p>The following helper functions best belong to the <t>lmxl-ini</t> -module. Some are here because we need then in the <t>mk</t> -document and other manuals, others came up when playing with -this module. Since this module is also used in <l n='mtxrun'/> we've -put them here instead of loading mode modules there then needed.</p> ---ldx]]-- +local function copiedelement(element,newparent) + if type(element) == "string" then + return element + else + element = xmlcopy(element).dt + if newparent and type(element) == "table" then + element.__p__ = newparent + end + return element + end +end -function xml.gsub(t,old,new) - local dt = t.dt - if dt then - for k=1,#dt do - local v = dt[k] - if type(v) == "string" then - dt[k] = gsub(v,old,new) - else - xml.gsub(v,old,new) +function xml.delete_element(root,pattern) + local collected = xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + local e = collected[c] + local p = e.__p__ + if p then + if trace_manipulations then + report('deleting',pattern,c,e) + end + local d = p.dt + remove(d,e.ni) + xml.redo_ni(d) -- can be made faster and inlined end end end end -function xml.strip_leading_spaces(dk,d,k) -- cosmetic, for manual - if d and k and d[k-1] and type(d[k-1]) == "string" then - local s = d[k-1]:match("\n(%s+)") - xml.gsub(dk,"\n"..string.rep(" ",#s),"\n") +function xml.replace_element(root,pattern,whatever) + local element = root and xmltoelement(whatever,root) + local collected = element and xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + local e = collected[c] + local p = e.__p__ + if p then + if trace_manipulations then + report('replacing',pattern,c,e) + end + local d = p.dt + d[e.ni] = copiedelement(element,p) + xml.redo_ni(d) -- probably not needed + end + end end end -function xml.serialize_path(root,lpath,handle) - local dk, r, d, k = xml.first(root,lpath) - dk = xml.copy(dk) - xml.strip_leading_spaces(dk,d,k) - xml.serialize(dk,handle) +local function inject_element(root,pattern,whatever,prepend) + local element = root and xmltoelement(whatever,root) + local collected = element and xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + local e = collected[c] + local r = e.__p__ + local d, k, rri = r.dt, e.ni, r.ri + local edt = (rri and d[rri].dt) or (d and d[k] and d[k].dt) + if edt then + local be, af + local cp = copiedelement(element,e) + if prepend then + be, af = cp, edt + else + be, af = edt, cp + end + for i=1,#af do + be[#be+1] = af[i] + end + if rri then + r.dt[rri].dt = be + else + d[k].dt = be + end + xml.redo_ni(d) + end + end + end end ---~ xml.escapes = { ['&'] = '&', ['<'] = '<', ['>'] = '>', ['"'] = '"' } ---~ xml.unescapes = { } for k,v in pairs(xml.escapes) do xml.unescapes[v] = k end +local function insert_element(root,pattern,whatever,before) -- todo: element als functie + local element = root and xmltoelement(whatever,root) + local collected = element and xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + local e = collected[c] + local r = e.__p__ + local d, k = r.dt, e.ni + if not before then + k = k + 1 + end + insert(d,k,copiedelement(element,r)) + xml.redo_ni(d) + end + end +end ---~ function xml.escaped (str) return (gsub(str,"(.)" , xml.escapes )) end ---~ function xml.unescaped(str) return (gsub(str,"(&.-;)", xml.unescapes)) end ---~ function xml.cleansed (str) return (gsub(str,"<.->" , '' )) end -- "%b<>" +xml.insert_element = insert_element +xml.insert_element_after = insert_element +xml.insert_element_before = function(r,p,e) insert_element(r,p,e,true) end +xml.inject_element = inject_element +xml.inject_element_after = inject_element +xml.inject_element_before = function(r,p,e) inject_element(r,p,e,true) end -local P, S, R, C, V, Cc, Cs = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc, lpeg.Cs +local function include(xmldata,pattern,attribute,recursive,loaddata) + -- parse="text" (default: xml), encoding="" (todo) + -- attribute = attribute or 'href' + pattern = pattern or 'include' + loaddata = loaddata or io.loaddata + local collected = xmlparseapply({ xmldata },pattern) + if collected then + for c=1,#collected do + local ek = collected[c] + local name = nil + local ekdt = ek.dt + local ekat = ek.at + local epdt = ek.__p__.dt + if not attribute or attribute == "" then + name = (type(ekdt) == "table" and ekdt[1]) or ekdt -- ckeck, probably always tab or str + end + if not name then + for a in gmatch(attribute or "href","([^|]+)") do + name = ekat[a] + if name then break end + end + end + local data = (name and name ~= "" and loaddata(name)) or "" + if data == "" then + epdt[ek.ni] = "" -- xml.empty(d,k) + elseif ekat["parse"] == "text" then + -- for the moment hard coded + epdt[ek.ni] = xml.escaped(data) -- d[k] = xml.escaped(data) + else +--~ local settings = xmldata.settings +--~ settings.parent_root = xmldata -- to be tested +--~ local xi = xmlconvert(data,settings) + local xi = xmlinheritedconvert(data,xmldata) + if not xi then + epdt[ek.ni] = "" -- xml.empty(d,k) + else + if recursive then + include(xi,pattern,attribute,recursive,loaddata) + end + epdt[ek.ni] = xml.body(xi) -- xml.assign(d,k,xi) + end + end + end + end +end --- 100 * 2500 * "oeps< oeps> oeps&" : gsub:lpeg|lpeg|lpeg --- --- 1021:0335:0287:0247 +xml.include = include --- 10 * 1000 * "oeps< oeps> oeps& asfjhalskfjh alskfjh alskfjh alskfjh ;al J;LSFDJ" --- --- 1559:0257:0288:0190 (last one suggested by roberto) +--~ local function manipulate(xmldata,pattern,manipulator) -- untested and might go away +--~ local collected = xmlparseapply({ xmldata },pattern) +--~ if collected then +--~ local xmltostring = xml.tostring +--~ for c=1,#collected do +--~ local e = collected[c] +--~ local data = manipulator(xmltostring(e)) +--~ if data == "" then +--~ epdt[e.ni] = "" +--~ else +--~ local xi = xmlinheritedconvert(data,xmldata) +--~ if not xi then +--~ epdt[e.ni] = "" +--~ else +--~ epdt[e.ni] = xml.body(xi) -- xml.assign(d,k,xi) +--~ end +--~ end +--~ end +--~ end +--~ end --- escaped = Cs((S("<&>") / xml.escapes + 1)^0) --- escaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0) -local normal = (1 - S("<&>"))^0 -local special = P("<")/"<" + P(">")/">" + P("&")/"&" -local escaped = Cs(normal * (special * normal)^0) +--~ xml.manipulate = manipulate --- 100 * 1000 * "oeps< oeps> oeps&" : gsub:lpeg == 0153:0280:0151:0080 (last one by roberto) +function xml.strip_whitespace(root, pattern, nolines) -- strips all leading and trailing space ! + local collected = xmlparseapply({ root },pattern) + if collected then + for i=1,#collected do + local e = collected[i] + local edt = e.dt + if edt then + local t = { } + for i=1,#edt do + local str = edt[i] + if type(str) == "string" then + if str == "" then + -- stripped + else + if nolines then + str = gsub(str,"[ \n\r\t]+"," ") + end + if str == "" then + -- stripped + else + t[#t+1] = str + end + end + else + --~ str.ni = i + t[#t+1] = str + end + end + e.dt = t + end + end + end +end --- unescaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0) --- unescaped = Cs((((P("&")/"") * (P("lt")/"<" + P("gt")/">" + P("amp")/"&") * (P(";")/"")) + 1)^0) -local normal = (1 - S"&")^0 -local special = P("<")/"<" + P(">")/">" + P("&")/"&" -local unescaped = Cs(normal * (special * normal)^0) +function xml.strip_whitespace(root, pattern, nolines, anywhere) -- strips all leading and trailing spacing + local collected = xmlparseapply({ root },pattern) -- beware, indices no longer are valid now + if collected then + for i=1,#collected do + local e = collected[i] + local edt = e.dt + if edt then + if anywhere then + local t = { } + for e=1,#edt do + local str = edt[e] + if type(str) ~= "string" then + t[#t+1] = str + elseif str ~= "" then + -- todo: lpeg for each case + if nolines then + str = gsub(str,"%s+"," ") + end + str = gsub(str,"^%s*(.-)%s*$","%1") + if str ~= "" then + t[#t+1] = str + end + end + end + e.dt = t + else + -- we can assume a regular sparse xml table with no successive strings + -- otherwise we should use a while loop + if #edt > 0 then + -- strip front + local str = edt[1] + if type(str) ~= "string" then + -- nothing + elseif str == "" then + remove(edt,1) + else + if nolines then + str = gsub(str,"%s+"," ") + end + str = gsub(str,"^%s+","") + if str == "" then + remove(edt,1) + else + edt[1] = str + end + end + end + if #edt > 1 then + -- strip end + local str = edt[#edt] + if type(str) ~= "string" then + -- nothing + elseif str == "" then + remove(edt) + else + if nolines then + str = gsub(str,"%s+"," ") + end + str = gsub(str,"%s+$","") + if str == "" then + remove(edt) + else + edt[#edt] = str + end + end + end + end + end + end + end +end --- 100 * 5000 * "oeps <oeps bla='oeps' foo='bar'> oeps </oeps> oeps " : gsub:lpeg == 623:501 msec (short tags, less difference) +local function rename_space(root, oldspace, newspace) -- fast variant + local ndt = #root.dt + for i=1,ndt or 0 do + local e = root[i] + if type(e) == "table" then + if e.ns == oldspace then + e.ns = newspace + if e.rn then + e.rn = newspace + end + end + local edt = e.dt + if edt then + rename_space(edt, oldspace, newspace) + end + end + end +end -local cleansed = Cs(((P("<") * (1-P(">"))^0 * P(">"))/"" + 1)^0) +xml.rename_space = rename_space -xml.escaped_pattern = escaped -xml.unescaped_pattern = unescaped -xml.cleansed_pattern = cleansed +function xml.remap_tag(root, pattern, newtg) + local collected = xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + collected[c].tg = newtg + end + end +end -function xml.escaped (str) return escaped :match(str) end -function xml.unescaped(str) return unescaped:match(str) end -function xml.cleansed (str) return cleansed :match(str) end +function xml.remap_namespace(root, pattern, newns) + local collected = xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + collected[c].ns = newns + end + end +end -function xml.join(t,separator,lastseparator) - if #t > 0 then - local result = { } - for k,v in pairs(t) do - result[k] = xml.tostring(v) +function xml.check_namespace(root, pattern, newns) + local collected = xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + local e = collected[c] + if (not e.rn or e.rn == "") and e.ns == "" then + e.rn = newns + end end - if lastseparator then - return concat(result,separator or "",1,#result-1) .. (lastseparator or "") .. result[#result] - else - return concat(result,separator) + end +end + +function xml.remap_name(root, pattern, newtg, newns, newrn) + local collected = xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + local e = collected[c] + e.tg, e.ns, e.rn = newtg, newns, newrn end - else - return "" end end +--[[ldx-- +<p>Here are a few synonyms.</p> +--ldx]]-- + +xml.each = xml.each_element +xml.process = xml.process_element +xml.strip = xml.strip_whitespace +xml.collect = xml.collect_elements +xml.all = xml.collect_elements + +xml.insert = xml.insert_element_after +xml.inject = xml.inject_element_after +xml.after = xml.insert_element_after +xml.before = xml.insert_element_before +xml.delete = xml.delete_element +xml.replace = xml.replace_element + end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['trac-tra'] = { +if not modules then modules = { } end modules ['lxml-xml'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "this module is the basis for the lxml-* ones", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } --- the <anonymous> tag is kind of generic and used for functions that are not --- bound to a variable, like node.new, node.copy etc (contrary to for instance --- node.has_attribute which is bound to a has_attribute local variable in mkiv) +local finalizers = xml.finalizers.xml +local xmlfilter = xml.filter -- we could inline this one for speed +local xmltostring = xml.tostring +local xmlserialize = xml.serialize -debugger = debugger or { } +local function first(collected) -- wrong ? + return collected and collected[1] +end -local counters = { } -local names = { } -local getinfo = debug.getinfo -local format, find, lower, gmatch = string.format, string.find, string.lower, string.gmatch +local function last(collected) + return collected and collected[#collected] +end --- one +local function all(collected) + return collected +end -local function hook() - local f = getinfo(2,"f").func - local n = getinfo(2,"Sn") --- if n.what == "C" and n.name then print (n.namewhat .. ': ' .. n.name) end - if f then - local cf = counters[f] - if cf == nil then - counters[f] = 1 - names[f] = n - else - counters[f] = cf + 1 +local function reverse(collected) + if collected then + local reversed = { } + for c=#collected,1,-1 do + reversed[#reversed+1] = collected[c] end + return reversed end end -local function getname(func) - local n = names[func] - if n then - if n.what == "C" then - return n.name or '<anonymous>' - else - -- source short_src linedefined what name namewhat nups func - local name = n.name or n.namewhat or n.what - if not name or name == "" then name = "?" end - return format("%s : %s : %s", n.short_src or "unknown source", n.linedefined or "--", name) - end - else - return "unknown" + +local function attribute(collected,name) + if collected and #collected > 0 then + local at = collected[1].at + return at and at[name] end end -function debugger.showstats(printer,threshold) - printer = printer or texio.write or print - threshold = threshold or 0 - local total, grandtotal, functions = 0, 0, 0 - printer("\n") -- ugly but ok - -- table.sort(counters) - for func, count in pairs(counters) do - if count > threshold then - local name = getname(func) - if not name:find("for generator") then - printer(format("%8i %s", count, name)) - total = total + count - end - end - grandtotal = grandtotal + count - functions = functions + 1 - end - printer(format("functions: %s, total: %s, grand total: %s, threshold: %s\n", functions, total, grandtotal, threshold)) + +local function att(id,name) + local at = id.at + return at and at[name] end --- two +local function count(collected) + return (collected and #collected) or 0 +end ---~ local function hook() ---~ local n = getinfo(2) ---~ if n.what=="C" and not n.name then ---~ local f = tostring(debug.traceback()) ---~ local cf = counters[f] ---~ if cf == nil then ---~ counters[f] = 1 ---~ names[f] = n ---~ else ---~ counters[f] = cf + 1 ---~ end ---~ end ---~ end ---~ function debugger.showstats(printer,threshold) ---~ printer = printer or texio.write or print ---~ threshold = threshold or 0 ---~ local total, grandtotal, functions = 0, 0, 0 ---~ printer("\n") -- ugly but ok ---~ -- table.sort(counters) ---~ for func, count in pairs(counters) do ---~ if count > threshold then ---~ printer(format("%8i %s", count, func)) ---~ total = total + count ---~ end ---~ grandtotal = grandtotal + count ---~ functions = functions + 1 ---~ end ---~ printer(format("functions: %s, total: %s, grand total: %s, threshold: %s\n", functions, total, grandtotal, threshold)) ---~ end +local function position(collected,n) + if collected then + n = tonumber(n) or 0 + if n < 0 then + return collected[#collected + n + 1] + elseif n > 0 then + return collected[n] + else + return collected[1].mi or 0 + end + end +end --- rest +local function match(collected) + return (collected and collected[1].mi) or 0 -- match +end -function debugger.savestats(filename,threshold) - local f = io.open(filename,'w') - if f then - debugger.showstats(function(str) f:write(str) end,threshold) - f:close() +local function index(collected) + if collected then + return collected[1].ni end end -function debugger.enable() - debug.sethook(hook,"c") +local function attributes(collected,arguments) + if collected then + local at = collected[1].at + if arguments then + return at[arguments] + elseif next(at) then + return at -- all of them + end + end end -function debugger.disable() - debug.sethook() ---~ counters[debug.getinfo(2,"f").func] = nil +local function chainattribute(collected,arguments) -- todo: optional levels + if collected then + local e = collected[1] + while e do + local at = e.at + if at then + local a = at[arguments] + if a then + return a + end + else + break -- error + end + e = e.__p__ + end + end + return "" end -function debugger.tracing() - local n = tonumber(os.env['MTX.TRACE.CALLS']) or tonumber(os.env['MTX_TRACE_CALLS']) or 0 - if n > 0 then - function debugger.tracing() return true end ; return true +local function raw(collected) -- hybrid + if collected then + local e = collected[1] or collected + return (e and xmlserialize(e)) or "" -- only first as we cannot concat function else - function debugger.tracing() return false end ; return false + return "" end end ---~ debugger.enable() - ---~ print(math.sin(1*.5)) ---~ print(math.sin(1*.5)) ---~ print(math.sin(1*.5)) ---~ print(math.sin(1*.5)) ---~ print(math.sin(1*.5)) - ---~ debugger.disable() - ---~ print("") ---~ debugger.showstats() ---~ print("") ---~ debugger.showstats(print,3) - -trackers = trackers or { } +local function text(collected) -- hybrid + if collected then + local e = collected[1] or collected + return (e and xmltostring(e.dt)) or "" + else + return "" + end +end -local data, done = { }, { } +local function texts(collected) + if collected then + local t = { } + for c=1,#collected do + local e = collection[c] + if e and e.dt then + t[#t+1] = e.dt + end + end + return t + end +end -local function set(what,value) - if type(what) == "string" then - what = aux.settings_to_array(what) +local function tag(collected,n) + if collected then + local c + if n == 0 or not n then + c = collected[1] + elseif n > 1 then + c = collected[n] + else + c = collected[#collected-n+1] + end + return c and c.tg end - for i=1,#what do - local w = what[i] - for d, f in next, data do - if done[d] then - -- prevent recursion due to wildcards - elseif find(d,w) then - done[d] = true - for i=1,#f do - f[i](value) - end +end + +local function name(collected,n) + if collected then + local c + if n == 0 or not n then + c = collected[1] + elseif n > 1 then + c = collected[n] + else + c = collected[#collected-n+1] + end + if c then + if c.ns == "" then + return c.tg + else + return c.ns .. ":" .. c.tg end end end end -local function reset() - for d, f in next, data do - for i=1,#f do - f[i](false) +local function tags(collected,nonamespace) + if collected then + local t = { } + for c=1,#collected do + local e = collected[c] + local ns, tg = e.ns, e.tg + if nonamespace or ns == "" then + t[#t+1] = tg + else + t[#t+1] = ns .. ":" .. tg + end end + return t end end -function trackers.register(what,...) - what = lower(what) - local w = data[what] - if not w then - w = { } - data[what] = w - end - for _, fnc in next, { ... } do - local typ = type(fnc) - if typ == "function" then - w[#w+1] = fnc - elseif typ == "string" then - w[#w+1] = function(value) set(fnc,value,nesting) end +local function empty(collected) + if collected then + for c=1,#collected do + local e = collected[c] + if e then + local edt = e.dt + if edt then + local n = #edt + if n == 1 then + local edk = edt[1] + local typ = type(edk) + if typ == "table" then + return false + elseif edk ~= "" then -- maybe an extra tester for spacing only + return false + end + elseif n > 1 then + return false + end + end + end end end + return true end -function trackers.enable(what) - done = { } - set(what,true) +finalizers.first = first +finalizers.last = last +finalizers.all = all +finalizers.reverse = reverse +finalizers.elements = all +finalizers.default = all +finalizers.attribute = attribute +finalizers.att = att +finalizers.count = count +finalizers.position = position +finalizers.match = match +finalizers.index = index +finalizers.attributes = attributes +finalizers.chainattribute = chainattribute +finalizers.text = text +finalizers.texts = texts +finalizers.tag = tag +finalizers.name = name +finalizers.tags = tags +finalizers.empty = empty + +-- shortcuts -- we could support xmlfilter(id,pattern,first) + +function xml.first(id,pattern) + return first(xmlfilter(id,pattern)) end -function trackers.disable(what) - done = { } - if not what or what == "" then - trackers.reset(what) +function xml.last(id,pattern) + return last(xmlfilter(id,pattern)) +end + +function xml.count(id,pattern) + return count(xmlfilter(id,pattern)) +end + +function xml.attribute(id,pattern,a,default) + return attribute(xmlfilter(id,pattern),a,default) +end + +function xml.raw(id,pattern) + if pattern then + return raw(xmlfilter(id,pattern)) else - set(what,false) + return raw(id) end end -function trackers.reset(what) - done = { } - reset() +function xml.text(id,pattern) + if pattern then + -- return text(xmlfilter(id,pattern)) + local collected = xmlfilter(id,pattern) + return (collected and xmltostring(collected[1].dt)) or "" + elseif id then + -- return text(id) + return xmltostring(id.dt) or "" + else + return "" + end end -function trackers.list() -- pattern - local list = table.sortedkeys(data) - local user, system = { }, { } - for l=1,#list do - local what = list[l] - if find(what,"^%*") then - system[#system+1] = what - else - user[#user+1] = what - end - end - return user, system +xml.content = text + +function xml.position(id,pattern,n) -- element + return position(xmlfilter(id,pattern),n) +end + +function xml.match(id,pattern) -- number + return match(xmlfilter(id,pattern)) +end + +function xml.empty(id,pattern) + return empty(xmlfilter(id,pattern)) end +xml.all = xml.filter +xml.index = xml.position +xml.found = xml.filter + end -- of closure @@ -5667,7 +7287,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['luat-env'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -5679,10 +7299,10 @@ if not modules then modules = { } end modules ['luat-env'] = { -- evolved before bytecode arrays were available and so a lot of -- code has disappeared already. -local trace_verbose = false trackers.register("resolvers.verbose", function(v) trace_verbose = v end) -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v trackers.enable("resolvers.verbose") end) +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) -local format = string.format +local format, sub, match, gsub, find = string.format, string.sub, string.match, string.gsub, string.find +local unquote, quote = string.unquote, string.quote -- precautions @@ -5716,13 +7336,14 @@ if not environment.jobname then environ function environment.initialize_arguments(arg) local arguments, files = { }, { } environment.arguments, environment.files, environment.sortedflags = arguments, files, nil - for index, argument in pairs(arg) do + for index=1,#arg do + local argument = arg[index] if index > 0 then - local flag, value = argument:match("^%-+(.+)=(.-)$") + local flag, value = match(argument,"^%-+(.-)=(.-)$") if flag then - arguments[flag] = string.unquote(value or "") + arguments[flag] = unquote(value or "") else - flag = argument:match("^%-+(.+)") + flag = match(argument,"^%-+(.+)") if flag then arguments[flag] = true else @@ -5749,25 +7370,30 @@ function environment.argument(name,partial) return arguments[name] elseif partial then if not sortedflags then - sortedflags = { } - for _,v in pairs(table.sortedkeys(arguments)) do - sortedflags[#sortedflags+1] = "^" .. v + sortedflags = table.sortedkeys(arguments) + for k=1,#sortedflags do + sortedflags[k] = "^" .. sortedflags[k] end environment.sortedflags = sortedflags end -- example of potential clash: ^mode ^modefile - for _,v in ipairs(sortedflags) do - if name:find(v) then - return arguments[v:sub(2,#v)] + for k=1,#sortedflags do + local v = sortedflags[k] + if find(name,v) then + return arguments[sub(v,2,#v)] end end end return nil end +environment.argument("x",true) + function environment.split_arguments(separator) -- rather special, cut-off before separator local done, before, after = false, { }, { } - for _,v in ipairs(environment.original_arguments) do + local original_arguments = environment.original_arguments + for k=1,#original_arguments do + local v = original_arguments[k] if not done and v == separator then done = true elseif done then @@ -5784,16 +7410,17 @@ function environment.reconstruct_commandline(arg,noquote) if noquote and #arg == 1 then local a = arg[1] a = resolvers.resolve(a) - a = a:unquote() + a = unquote(a) return a - elseif next(arg) then + elseif #arg > 0 then local result = { } - for _,a in ipairs(arg) do -- ipairs 1 .. #n + for i=1,#arg do + local a = arg[i] a = resolvers.resolve(a) - a = a:unquote() - a = a:gsub('"','\\"') -- tricky - if a:find(" ") then - result[#result+1] = a:quote() + a = unquote(a) + a = gsub(a,'"','\\"') -- tricky + if find(a," ") then + result[#result+1] = quote(a) else result[#result+1] = a end @@ -5806,17 +7433,18 @@ end if arg then - -- new, reconstruct quoted snippets (maybe better just remnove the " then and add them later) + -- new, reconstruct quoted snippets (maybe better just remove the " then and add them later) local newarg, instring = { }, false - for index, argument in ipairs(arg) do - if argument:find("^\"") then - newarg[#newarg+1] = argument:gsub("^\"","") - if not argument:find("\"$") then + for index=1,#arg do + local argument = arg[index] + if find(argument,"^\"") then + newarg[#newarg+1] = gsub(argument,"^\"","") + if not find(argument,"\"$") then instring = true end - elseif argument:find("\"$") then - newarg[#newarg] = newarg[#newarg] .. " " .. argument:gsub("\"$","") + elseif find(argument,"\"$") then + newarg[#newarg] = newarg[#newarg] .. " " .. gsub(argument,"\"$","") instring = false elseif instring then newarg[#newarg] = newarg[#newarg] .. " " .. argument @@ -5871,12 +7499,12 @@ function environment.luafilechunk(filename) -- used for loading lua bytecode in filename = file.replacesuffix(filename, "lua") local fullname = environment.luafile(filename) if fullname and fullname ~= "" then - if trace_verbose then + if trace_locating then logs.report("fileio","loading file %s", fullname) end return environment.loadedluacode(fullname) else - if trace_verbose then + if trace_locating then logs.report("fileio","unknown file %s", filename) end return nil @@ -5896,7 +7524,7 @@ function environment.loadluafile(filename, version) -- when not overloaded by explicit suffix we look for a luc file first local fullname = (lucname and environment.luafile(lucname)) or "" if fullname ~= "" then - if trace_verbose then + if trace_locating then logs.report("fileio","loading %s", fullname) end chunk = loadfile(fullname) -- this way we don't need a file exists check @@ -5914,7 +7542,7 @@ function environment.loadluafile(filename, version) if v == version then return true else - if trace_verbose then + if trace_locating then logs.report("fileio","version mismatch for %s: lua=%s, luc=%s", filename, v, version) end environment.loadluafile(filename) @@ -5925,12 +7553,12 @@ function environment.loadluafile(filename, version) end fullname = (luaname and environment.luafile(luaname)) or "" if fullname ~= "" then - if trace_verbose then + if trace_locating then logs.report("fileio","loading %s", fullname) end chunk = loadfile(fullname) -- this way we don't need a file exists check if not chunk then - if verbose then + if trace_locating then logs.report("fileio","unknown file %s", filename) end else @@ -5948,7 +7576,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['trac-inf'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to trac-inf.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -5973,6 +7601,14 @@ function statistics.hastimer(instance) return instance and instance.starttime end +function statistics.resettiming(instance) + if not instance then + notimer = { timing = 0, loadtime = 0 } + else + instance.timing, instance.loadtime = 0, 0 + end +end + function statistics.starttiming(instance) if not instance then notimer = { } @@ -5987,6 +7623,8 @@ function statistics.starttiming(instance) if not instance.loadtime then instance.loadtime = 0 end + else +--~ logs.report("system","nested timing (%s)",tostring(instance)) end instance.timing = it + 1 end @@ -6032,6 +7670,12 @@ function statistics.elapsedindeed(instance) return t > statistics.threshold end +function statistics.elapsedseconds(instance,rest) -- returns nil if 0 seconds + if statistics.elapsedindeed(instance) then + return format("%s seconds %s", statistics.elapsedtime(instance),rest or "") + end +end + -- general function function statistics.register(tag,fnc) @@ -6110,14 +7754,32 @@ function statistics.timed(action,report) report("total runtime: %s",statistics.elapsedtime(timer)) end +-- where, not really the best spot for this: + +commands = commands or { } + +local timer + +function commands.resettimer() + statistics.resettiming(timer) + statistics.starttiming(timer) +end + +function commands.elapsedtime() + statistics.stoptiming(timer) + tex.sprint(statistics.elapsedtime(timer)) +end + +commands.resettimer() + end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['luat-log'] = { +if not modules then modules = { } end modules ['trac-log'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to trac-log.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -6125,7 +7787,11 @@ if not modules then modules = { } end modules ['luat-log'] = { -- this is old code that needs an overhaul -local write_nl, write, format = texio.write_nl or print, texio.write or io.write, string.format +--~ io.stdout:setvbuf("no") +--~ io.stderr:setvbuf("no") + +local write_nl, write = texio.write_nl or print, texio.write or io.write +local format, gmatch = string.format, string.gmatch local texcount = tex and tex.count if texlua then @@ -6206,25 +7872,48 @@ function logs.tex.line(fmt,...) -- new end end +--~ function logs.tex.start_page_number() +--~ local real, user, sub = texcount.realpageno, texcount.userpageno, texcount.subpageno +--~ if real > 0 then +--~ if user > 0 then +--~ if sub > 0 then +--~ write(format("[%s.%s.%s",real,user,sub)) +--~ else +--~ write(format("[%s.%s",real,user)) +--~ end +--~ else +--~ write(format("[%s",real)) +--~ end +--~ else +--~ write("[-") +--~ end +--~ end + +--~ function logs.tex.stop_page_number() +--~ write("]") +--~ end + +local real, user, sub + function logs.tex.start_page_number() - local real, user, sub = texcount.realpageno, texcount.userpageno, texcount.subpageno + real, user, sub = texcount.realpageno, texcount.userpageno, texcount.subpageno +end + +function logs.tex.stop_page_number() if real > 0 then if user > 0 then if sub > 0 then - write(format("[%s.%s.%s",real,user,sub)) + logs.report("pages", "flushing realpage %s, userpage %s, subpage %s",real,user,sub) else - write(format("[%s.%s",real,user)) + logs.report("pages", "flushing realpage %s, userpage %s",real,user) end else - write(format("[%s",real)) + logs.report("pages", "flushing realpage %s",real) end else - write("[-") + logs.report("pages", "flushing page") end -end - -function logs.tex.stop_page_number() - write("]") + io.flush() end logs.tex.report_job_stat = statistics.show_job_stat @@ -6324,7 +8013,7 @@ end function logs.setprogram(_name_,_banner_,_verbose_) name, banner = _name_, _banner_ if _verbose_ then - trackers.enable("resolvers.verbose") + trackers.enable("resolvers.locating") end logs.set_method("tex") logs.report = report -- also used in libraries @@ -6337,9 +8026,9 @@ end function logs.setverbose(what) if what then - trackers.enable("resolvers.verbose") + trackers.enable("resolvers.locating") else - trackers.disable("resolvers.verbose") + trackers.disable("resolvers.locating") end logs.verbose = what or false end @@ -6356,7 +8045,7 @@ logs.report = logs.tex.report logs.simple = logs.tex.report function logs.reportlines(str) -- todo: <lines></lines> - for line in str:gmatch("(.-)[\n\r]") do + for line in gmatch(str,"(.-)[\n\r]") do logs.report(line) end end @@ -6367,8 +8056,12 @@ end logs.simpleline = logs.reportline -function logs.help(message,option) +function logs.reportbanner() -- for scripts too logs.report(banner) +end + +function logs.help(message,option) + logs.reportbanner() logs.reportline() logs.reportlines(message) local moreinfo = logs.moreinfo or "" @@ -6400,6 +8093,11 @@ end --~ logs.system(syslogname,"context","test","fonts","font %s recached due to newer version (%s)","blabla","123") --~ end +function logs.fatal(where,...) + logs.report(where,"fatal error: %s, aborting now",format(...)) + os.exit() +end + end -- of closure @@ -6407,10 +8105,10 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-inp'] = { version = 1.001, + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files", - comment = "companion to luat-lib.tex", } -- After a few years using the code the large luat-inp.lua file @@ -6422,7 +8120,7 @@ if not modules then modules = { } end modules ['data-inp'] = { -- * some public auxiliary functions were made private -- -- TODO: os.getenv -> os.env[] --- TODO: instances.[hashes,cnffiles,configurations,522] -> ipairs (alles check, sneller) +-- TODO: instances.[hashes,cnffiles,configurations,522] -- TODO: check escaping in find etc, too much, too slow -- This lib is multi-purpose and can be loaded again later on so that @@ -6443,12 +8141,13 @@ if not modules then modules = { } end modules ['data-inp'] = { local format, gsub, find, lower, upper, match, gmatch = string.format, string.gsub, string.find, string.lower, string.upper, string.match, string.gmatch local concat, insert, sortedkeys = table.concat, table.insert, table.sortedkeys local next, type = next, type +local lpegmatch = lpeg.match -local trace_locating, trace_detail, trace_verbose = false, false, false +local trace_locating, trace_detail, trace_expansions = false, false, false -trackers.register("resolvers.verbose", function(v) trace_verbose = v end) -trackers.register("resolvers.locating", function(v) trace_locating = v trackers.enable("resolvers.verbose") end) -trackers.register("resolvers.detail", function(v) trace_detail = v trackers.enable("resolvers.verbose,resolvers.detail") end) +trackers.register("resolvers.locating", function(v) trace_locating = v end) +trackers.register("resolvers.details", function(v) trace_detail = v end) +trackers.register("resolvers.expansions", function(v) trace_expansions = v end) -- todo if not resolvers then resolvers = { @@ -6472,7 +8171,7 @@ resolvers.generators.notfound = { nil } resolvers.cacheversion = '1.0.1' resolvers.cnfname = 'texmf.cnf' resolvers.luaname = 'texmfcnf.lua' -resolvers.homedir = os.env[os.platform == "windows" and 'USERPROFILE'] or os.env['HOME'] or '~' +resolvers.homedir = os.env[os.type == "windows" and 'USERPROFILE'] or os.env['HOME'] or '~' resolvers.cnfdefault = '{$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c}' local dummy_path_expr = "^!*unset/*$" @@ -6514,8 +8213,8 @@ suffixes['lua'] = { 'lua', 'luc', 'tma', 'tmc' } alternatives['map files'] = 'map' alternatives['enc files'] = 'enc' -alternatives['cid files'] = 'cid' -alternatives['fea files'] = 'fea' +alternatives['cid maps'] = 'cid' -- great, why no cid files +alternatives['font feature files'] = 'fea' -- and fea files here alternatives['opentype fonts'] = 'otf' alternatives['truetype fonts'] = 'ttf' alternatives['truetype collections'] = 'ttc' @@ -6531,6 +8230,11 @@ formats ['sfd'] = 'SFDFONTS' suffixes ['sfd'] = { 'sfd' } alternatives['subfont definition files'] = 'sfd' +-- lib paths + +formats ['lib'] = 'CLUAINPUTS' -- new (needs checking) +suffixes['lib'] = (os.libsuffix and { os.libsuffix }) or { 'dll', 'so' } + -- In practice we will work within one tds tree, but i want to keep -- the option open to build tools that look at multiple trees, which is -- why we keep the tree specific data in a table. We used to pass the @@ -6653,8 +8357,10 @@ local function check_configuration() -- not yet ok, no time for debugging now -- bad luck end fix("LUAINPUTS" , ".;$TEXINPUTS;$TEXMFSCRIPTS") -- no progname, hm - fix("FONTFEATURES", ".;$TEXMF/fonts/fea//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") - fix("FONTCIDMAPS" , ".;$TEXMF/fonts/cid//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") + -- this will go away some day + fix("FONTFEATURES", ".;$TEXMF/fonts/{data,fea}//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") + fix("FONTCIDMAPS" , ".;$TEXMF/fonts/{data,cid}//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") + -- fix("LUATEXLIBS" , ".;$TEXMF/luatex/lua//") end @@ -6669,7 +8375,7 @@ function resolvers.settrace(n) -- no longer number but: 'locating' or 'detail' end end -resolvers.settrace(os.getenv("MTX.resolvers.TRACE") or os.getenv("MTX_INPUT_TRACE")) +resolvers.settrace(os.getenv("MTX_INPUT_TRACE")) function resolvers.osenv(key) local ie = instance.environment @@ -6757,37 +8463,43 @@ end -- work that well; the parsing is ok, but dealing with the resulting -- table is a pain because we need to work inside-out recursively +local function do_first(a,b) + local t = { } + for s in gmatch(b,"[^,]+") do t[#t+1] = a .. s end + return "{" .. concat(t,",") .. "}" +end + +local function do_second(a,b) + local t = { } + for s in gmatch(a,"[^,]+") do t[#t+1] = s .. b end + return "{" .. concat(t,",") .. "}" +end + +local function do_both(a,b) + local t = { } + for sa in gmatch(a,"[^,]+") do + for sb in gmatch(b,"[^,]+") do + t[#t+1] = sa .. sb + end + end + return "{" .. concat(t,",") .. "}" +end + +local function do_three(a,b,c) + return a .. b.. c +end + local function splitpathexpr(str, t, validate) -- no need for further optimization as it is only called a - -- few times, we can use lpeg for the sub; we could move - -- the local functions outside the body + -- few times, we can use lpeg for the sub + if trace_expansions then + logs.report("fileio","expanding variable '%s'",str) + end t = t or { } str = gsub(str,",}",",@}") str = gsub(str,"{,","{@,") -- str = "@" .. str .. "@" local ok, done - local function do_first(a,b) - local t = { } - for s in gmatch(b,"[^,]+") do t[#t+1] = a .. s end - return "{" .. concat(t,",") .. "}" - end - local function do_second(a,b) - local t = { } - for s in gmatch(a,"[^,]+") do t[#t+1] = s .. b end - return "{" .. concat(t,",") .. "}" - end - local function do_both(a,b) - local t = { } - for sa in gmatch(a,"[^,]+") do - for sb in gmatch(b,"[^,]+") do - t[#t+1] = sa .. sb - end - end - return "{" .. concat(t,",") .. "}" - end - local function do_three(a,b,c) - return a .. b.. c - end while true do done = false while true do @@ -6818,6 +8530,11 @@ local function splitpathexpr(str, t, validate) t[#t+1] = s end end + if trace_expansions then + for k=1,#t do + logs.report("fileio","% 4i: %s",k,t[k]) + end + end return t end @@ -6857,18 +8574,27 @@ end -- also we now follow the stupid route: if not set then just assume *one* -- cnf file under texmf (i.e. distribution) -resolvers.ownpath = resolvers.ownpath or nil -resolvers.ownbin = resolvers.ownbin or arg[-2] or arg[-1] or arg[0] or "luatex" -resolvers.autoselfdir = true -- false may be handy for debugging +local args = environment and environment.original_arguments or arg -- this needs a cleanup + +resolvers.ownbin = resolvers.ownbin or args[-2] or arg[-2] or args[-1] or arg[-1] or arg[0] or "luatex" +resolvers.ownbin = gsub(resolvers.ownbin,"\\","/") function resolvers.getownpath() - if not resolvers.ownpath then - if resolvers.autoselfdir and os.selfdir then - resolvers.ownpath = os.selfdir - else - local binary = resolvers.ownbin - if os.platform == "windows" then - binary = file.replacesuffix(binary,"exe") + local ownpath = resolvers.ownpath or os.selfdir + if not ownpath or ownpath == "" or ownpath == "unset" then + ownpath = args[-1] or arg[-1] + ownpath = ownpath and file.dirname(gsub(ownpath,"\\","/")) + if not ownpath or ownpath == "" then + ownpath = args[-0] or arg[-0] + ownpath = ownpath and file.dirname(gsub(ownpath,"\\","/")) + end + local binary = resolvers.ownbin + if not ownpath or ownpath == "" then + ownpath = ownpath and file.dirname(binary) + end + if not ownpath or ownpath == "" then + if os.binsuffix ~= "" then + binary = file.replacesuffix(binary,os.binsuffix) end for p in gmatch(os.getenv("PATH"),"[^"..io.pathseparator.."]+") do local b = file.join(p,binary) @@ -6880,30 +8606,39 @@ function resolvers.getownpath() local olddir = lfs.currentdir() if lfs.chdir(p) then local pp = lfs.currentdir() - if trace_verbose and p ~= pp then - logs.report("fileio","following symlink %s to %s",p,pp) + if trace_locating and p ~= pp then + logs.report("fileio","following symlink '%s' to '%s'",p,pp) end - resolvers.ownpath = pp + ownpath = pp lfs.chdir(olddir) else - if trace_verbose then - logs.report("fileio","unable to check path %s",p) + if trace_locating then + logs.report("fileio","unable to check path '%s'",p) end - resolvers.ownpath = p + ownpath = p end break end end end - if not resolvers.ownpath then resolvers.ownpath = '.' end + if not ownpath or ownpath == "" then + ownpath = "." + logs.report("fileio","forcing fallback ownpath .") + elseif trace_locating then + logs.report("fileio","using ownpath '%s'",ownpath) + end + end + resolvers.ownpath = ownpath + function resolvers.getownpath() + return resolvers.ownpath end - return resolvers.ownpath + return ownpath end local own_places = { "SELFAUTOLOC", "SELFAUTODIR", "SELFAUTOPARENT", "TEXMFCNF" } local function identify_own() - local ownpath = resolvers.getownpath() or lfs.currentdir() + local ownpath = resolvers.getownpath() or dir.current() local ie = instance.environment if ownpath then if resolvers.env('SELFAUTOLOC') == "" then os.env['SELFAUTOLOC'] = file.collapse_path(ownpath) end @@ -6916,10 +8651,10 @@ local function identify_own() if resolvers.env('TEXMFCNF') == "" then os.env['TEXMFCNF'] = resolvers.cnfdefault end if resolvers.env('TEXOS') == "" then os.env['TEXOS'] = resolvers.env('SELFAUTODIR') end if resolvers.env('TEXROOT') == "" then os.env['TEXROOT'] = resolvers.env('SELFAUTOPARENT') end - if trace_verbose then + if trace_locating then for i=1,#own_places do local v = own_places[i] - logs.report("fileio","variable %s set to %s",v,resolvers.env(v) or "unknown") + logs.report("fileio","variable '%s' set to '%s'",v,resolvers.env(v) or "unknown") end end identify_own = function() end @@ -6951,10 +8686,8 @@ end local function load_cnf_file(fname) fname = resolvers.clean_path(fname) local lname = file.replacesuffix(fname,'lua') - local f = io.open(lname) - if f then -- this will go - f:close() - local dname = file.dirname(fname) + if lfs.isfile(lname) then + local dname = file.dirname(fname) -- fname ? if not instance.configuration[dname] then resolvers.load_data(dname,'configuration',lname and file.basename(lname)) instance.order[#instance.order+1] = instance.configuration[dname] @@ -6962,8 +8695,8 @@ local function load_cnf_file(fname) else f = io.open(fname) if f then - if trace_verbose then - logs.report("fileio","loading %s", fname) + if trace_locating then + logs.report("fileio","loading configuration file %s", fname) end local line, data, n, k, v local dname = file.dirname(fname) @@ -6997,14 +8730,16 @@ local function load_cnf_file(fname) end end f:close() - elseif trace_verbose then - logs.report("fileio","skipping %s", fname) + elseif trace_locating then + logs.report("fileio","skipping configuration file '%s'", fname) end end end local function collapse_cnf_data() -- potential optimization: pass start index (setup and configuration are shared) - for _,c in ipairs(instance.order) do + local order = instance.order + for i=1,#order do + local c = order[i] for k,v in next, c do if not instance.variables[k] then if instance.environment[k] then @@ -7020,19 +8755,24 @@ end function resolvers.load_cnf() local function loadoldconfigdata() - for _, fname in ipairs(instance.cnffiles) do - load_cnf_file(fname) + local cnffiles = instance.cnffiles + for i=1,#cnffiles do + load_cnf_file(cnffiles[i]) end end -- instance.cnffiles contain complete names now ! + -- we still use a funny mix of cnf and new but soon + -- we will switch to lua exclusively as we only use + -- the file to collect the tree roots if #instance.cnffiles == 0 then - if trace_verbose then + if trace_locating then logs.report("fileio","no cnf files found (TEXMFCNF may not be set/known)") end else - instance.rootpath = instance.cnffiles[1] - for k,fname in ipairs(instance.cnffiles) do - instance.cnffiles[k] = file.collapse_path(gsub(fname,"\\",'/')) + local cnffiles = instance.cnffiles + instance.rootpath = cnffiles[1] + for k=1,#cnffiles do + instance.cnffiles[k] = file.collapse_path(cnffiles[k]) end for i=1,3 do instance.rootpath = file.dirname(instance.rootpath) @@ -7060,8 +8800,9 @@ function resolvers.load_lua() -- yet harmless else instance.rootpath = instance.luafiles[1] - for k,fname in ipairs(instance.luafiles) do - instance.luafiles[k] = file.collapse_path(gsub(fname,"\\",'/')) + local luafiles = instance.luafiles + for k=1,#luafiles do + instance.luafiles[k] = file.collapse_path(luafiles[k]) end for i=1,3 do instance.rootpath = file.dirname(instance.rootpath) @@ -7093,14 +8834,14 @@ end function resolvers.append_hash(type,tag,name) if trace_locating then - logs.report("fileio","= hash append: %s",tag) + logs.report("fileio","hash '%s' appended",tag) end insert(instance.hashes, { ['type']=type, ['tag']=tag, ['name']=name } ) end function resolvers.prepend_hash(type,tag,name) if trace_locating then - logs.report("fileio","= hash prepend: %s",tag) + logs.report("fileio","hash '%s' prepended",tag) end insert(instance.hashes, 1, { ['type']=type, ['tag']=tag, ['name']=name } ) end @@ -7124,9 +8865,11 @@ end -- locators function resolvers.locatelists() - for _, path in ipairs(resolvers.clean_path_list('TEXMF')) do - if trace_verbose then - logs.report("fileio","locating list of %s",path) + local texmfpaths = resolvers.clean_path_list('TEXMF') + for i=1,#texmfpaths do + local path = texmfpaths[i] + if trace_locating then + logs.report("fileio","locating list of '%s'",path) end resolvers.locatedatabase(file.collapse_path(path)) end @@ -7139,11 +8882,11 @@ end function resolvers.locators.tex(specification) if specification and specification ~= '' and lfs.isdir(specification) then if trace_locating then - logs.report("fileio",'! tex locator found: %s',specification) + logs.report("fileio","tex locator '%s' found",specification) end resolvers.append_hash('file',specification,filename) elseif trace_locating then - logs.report("fileio",'? tex locator not found: %s',specification) + logs.report("fileio","tex locator '%s' not found",specification) end end @@ -7157,7 +8900,9 @@ function resolvers.loadfiles() instance.loaderror = false instance.files = { } if not instance.renewcache then - for _, hash in ipairs(instance.hashes) do + local hashes = instance.hashes + for k=1,#hashes do + local hash = hashes[k] resolvers.hashdatabase(hash.tag,hash.name) if instance.loaderror then break end end @@ -7171,8 +8916,9 @@ end -- generators: function resolvers.loadlists() - for _, hash in ipairs(instance.hashes) do - resolvers.generatedatabase(hash.tag) + local hashes = instance.hashes + for i=1,#hashes do + resolvers.generatedatabase(hashes[i].tag) end end @@ -7184,10 +8930,27 @@ end local weird = lpeg.P(".")^1 + lpeg.anywhere(lpeg.S("~`!#$%^&*()={}[]:;\"\'||<>,?\n\r\t")) +--~ local l_forbidden = lpeg.S("~`!#$%^&*()={}[]:;\"\'||\\/<>,?\n\r\t") +--~ local l_confusing = lpeg.P(" ") +--~ local l_character = lpeg.patterns.utf8 +--~ local l_dangerous = lpeg.P(".") + +--~ local l_normal = (l_character - l_forbidden - l_confusing - l_dangerous) * (l_character - l_forbidden - l_confusing^2)^0 * lpeg.P(-1) +--~ ----- l_normal = l_normal * lpeg.Cc(true) + lpeg.Cc(false) + +--~ local function test(str) +--~ print(str,lpeg.match(l_normal,str)) +--~ end +--~ test("ヒラギノ明朝 Pro W3") +--~ test("..ヒラギノ明朝 Pro W3") +--~ test(":ヒラギノ明朝 Pro W3;") +--~ test("ヒラギノ明朝 /Pro W3;") +--~ test("ヒラギノ明朝 Pro W3") + function resolvers.generators.tex(specification) local tag = specification - if trace_verbose then - logs.report("fileio","scanning path %s",specification) + if trace_locating then + logs.report("fileio","scanning path '%s'",specification) end instance.files[tag] = { } local files = instance.files[tag] @@ -7203,7 +8966,8 @@ function resolvers.generators.tex(specification) full = spec end for name in directory(full) do - if not weird:match(name) then + if not lpegmatch(weird,name) then + -- if lpegmatch(l_normal,name) then local mode = attributes(full..name,'mode') if mode == 'file' then if path then @@ -7236,7 +9000,7 @@ function resolvers.generators.tex(specification) end end action() - if trace_verbose then + if trace_locating then logs.report("fileio","%s files found on %s directories with %s uppercase remappings",n,m,r) end end @@ -7251,11 +9015,48 @@ end -- we join them and split them after the expansion has taken place. This -- is more convenient. +--~ local checkedsplit = string.checkedsplit + +local cache = { } + +local splitter = lpeg.Ct(lpeg.splitat(lpeg.S(os.type == "windows" and ";" or ":;"))) + +local function split_kpse_path(str) -- beware, this can be either a path or a {specification} + local found = cache[str] + if not found then + if str == "" then + found = { } + else + str = gsub(str,"\\","/") +--~ local split = (find(str,";") and checkedsplit(str,";")) or checkedsplit(str,io.pathseparator) +local split = lpegmatch(splitter,str) + found = { } + for i=1,#split do + local s = split[i] + if not find(s,"^{*unset}*") then + found[#found+1] = s + end + end + if trace_expansions then + logs.report("fileio","splitting path specification '%s'",str) + for k=1,#found do + logs.report("fileio","% 4i: %s",k,found[k]) + end + end + cache[str] = found + end + end + return found +end + +resolvers.split_kpse_path = split_kpse_path + function resolvers.splitconfig() - for i,c in ipairs(instance) do - for k,v in pairs(c) do + for i=1,#instance do + local c = instance[i] + for k,v in next, c do if type(v) == 'string' then - local t = file.split_path(v) + local t = split_kpse_path(v) if #t > 1 then c[k] = t end @@ -7265,21 +9066,25 @@ function resolvers.splitconfig() end function resolvers.joinconfig() - for i,c in ipairs(instance.order) do - for k,v in pairs(c) do -- ipairs? + local order = instance.order + for i=1,#order do + local c = order[i] + for k,v in next, c do -- indexed? if type(v) == 'table' then c[k] = file.join_path(v) end end end end + function resolvers.split_path(str) if type(str) == 'table' then return str else - return file.split_path(str) + return split_kpse_path(str) end end + function resolvers.join_path(str) if type(str) == 'table' then return file.join_path(str) @@ -7291,8 +9096,9 @@ end function resolvers.splitexpansions() local ie = instance.expansions for k,v in next, ie do - local t, h = { }, { } - for _,vv in ipairs(file.split_path(v)) do + local t, h, p = { }, { }, split_kpse_path(v) + for kk=1,#p do + local vv = p[kk] if vv ~= "" and not h[vv] then t[#t+1] = vv h[vv] = true @@ -7339,11 +9145,15 @@ function resolvers.serialize(files) end t[#t+1] = "return {" if instance.sortdata then - for _, k in pairs(sortedkeys(files)) do -- ipairs + local sortedfiles = sortedkeys(files) + for i=1,#sortedfiles do + local k = sortedfiles[i] local fk = files[k] if type(fk) == 'table' then t[#t+1] = "\t['" .. k .. "']={" - for _, kk in pairs(sortedkeys(fk)) do -- ipairs + local sortedfk = sortedkeys(fk) + for j=1,#sortedfk do + local kk = sortedfk[j] t[#t+1] = dump(kk,fk[kk],"\t\t") end t[#t+1] = "\t}," @@ -7368,12 +9178,18 @@ function resolvers.serialize(files) return concat(t,"\n") end +local data_state = { } + +function resolvers.data_state() + return data_state or { } +end + function resolvers.save_data(dataname, makename) -- untested without cache overload for cachename, files in next, instance[dataname] do local name = (makename or file.join)(cachename,dataname) local luaname, lucname = name .. ".lua", name .. ".luc" - if trace_verbose then - logs.report("fileio","preparing %s for %s",dataname,cachename) + if trace_locating then + logs.report("fileio","preparing '%s' for '%s'",dataname,cachename) end for k, v in next, files do if type(v) == "table" and #v == 1 then @@ -7387,24 +9203,25 @@ function resolvers.save_data(dataname, makename) -- untested without cache overl date = os.date("%Y-%m-%d"), time = os.date("%H:%M:%S"), content = files, + uuid = os.uuid(), } local ok = io.savedata(luaname,resolvers.serialize(data)) if ok then - if trace_verbose then - logs.report("fileio","%s saved in %s",dataname,luaname) + if trace_locating then + logs.report("fileio","'%s' saved in '%s'",dataname,luaname) end if utils.lua.compile(luaname,lucname,false,true) then -- no cleanup but strip - if trace_verbose then - logs.report("fileio","%s compiled to %s",dataname,lucname) + if trace_locating then + logs.report("fileio","'%s' compiled to '%s'",dataname,lucname) end else - if trace_verbose then - logs.report("fileio","compiling failed for %s, deleting file %s",dataname,lucname) + if trace_locating then + logs.report("fileio","compiling failed for '%s', deleting file '%s'",dataname,lucname) end os.remove(lucname) end - elseif trace_verbose then - logs.report("fileio","unable to save %s in %s (access error)",dataname,luaname) + elseif trace_locating then + logs.report("fileio","unable to save '%s' in '%s' (access error)",dataname,luaname) end end end @@ -7416,19 +9233,20 @@ function resolvers.load_data(pathname,dataname,filename,makename) -- untested wi if blob then local data = blob() if data and data.content and data.type == dataname and data.version == resolvers.cacheversion then - if trace_verbose then - logs.report("fileio","loading %s for %s from %s",dataname,pathname,filename) + data_state[#data_state+1] = data.uuid + if trace_locating then + logs.report("fileio","loading '%s' for '%s' from '%s'",dataname,pathname,filename) end instance[dataname][pathname] = data.content else - if trace_verbose then - logs.report("fileio","skipping %s for %s from %s",dataname,pathname,filename) + if trace_locating then + logs.report("fileio","skipping '%s' for '%s' from '%s'",dataname,pathname,filename) end instance[dataname][pathname] = { } instance.loaderror = true end - elseif trace_verbose then - logs.report("fileio","skipping %s for %s from %s",dataname,pathname,filename) + elseif trace_locating then + logs.report("fileio","skipping '%s' for '%s' from '%s'",dataname,pathname,filename) end end @@ -7447,15 +9265,17 @@ function resolvers.resetconfig() end function resolvers.loadnewconfig() - for _, cnf in ipairs(instance.luafiles) do + local luafiles = instance.luafiles + for i=1,#luafiles do + local cnf = luafiles[i] local pathname = file.dirname(cnf) local filename = file.join(pathname,resolvers.luaname) local blob = loadfile(filename) if blob then local data = blob() if data then - if trace_verbose then - logs.report("fileio","loading configuration file %s",filename) + if trace_locating then + logs.report("fileio","loading configuration file '%s'",filename) end if true then -- flatten to variable.progname @@ -7476,14 +9296,14 @@ function resolvers.loadnewconfig() instance['setup'][pathname] = data end else - if trace_verbose then - logs.report("fileio","skipping configuration file %s",filename) + if trace_locating then + logs.report("fileio","skipping configuration file '%s'",filename) end instance['setup'][pathname] = { } instance.loaderror = true end - elseif trace_verbose then - logs.report("fileio","skipping configuration file %s",filename) + elseif trace_locating then + logs.report("fileio","skipping configuration file '%s'",filename) end instance.order[#instance.order+1] = instance.setup[pathname] if instance.loaderror then break end @@ -7492,7 +9312,9 @@ end function resolvers.loadoldconfig() if not instance.renewcache then - for _, cnf in ipairs(instance.cnffiles) do + local cnffiles = instance.cnffiles + for i=1,#cnffiles do + local cnf = cnffiles[i] local dname = file.dirname(cnf) resolvers.load_data(dname,'configuration') instance.order[#instance.order+1] = instance.configuration[dname] @@ -7682,7 +9504,7 @@ end function resolvers.expanded_path_list(str) if not str then - return ep or { } + return ep or { } -- ep ? elseif instance.savelists then -- engine+progname hash str = gsub(str,"%$","") @@ -7700,9 +9522,9 @@ end function resolvers.expanded_path_list_from_var(str) -- brrr local tmp = resolvers.var_of_format_or_suffix(gsub(str,"%$","")) if tmp ~= "" then - return resolvers.expanded_path_list(str) - else return resolvers.expanded_path_list(tmp) + else + return resolvers.expanded_path_list(str) end end @@ -7749,9 +9571,9 @@ function resolvers.isreadable.file(name) local readable = lfs.isfile(name) -- brrr if trace_detail then if readable then - logs.report("fileio","+ readable: %s",name) + logs.report("fileio","file '%s' is readable",name) else - logs.report("fileio","- readable: %s", name) + logs.report("fileio","file '%s' is not readable", name) end end return readable @@ -7767,7 +9589,7 @@ local function collect_files(names) for k=1,#names do local fname = names[k] if trace_detail then - logs.report("fileio","? blobpath asked: %s",fname) + logs.report("fileio","checking name '%s'",fname) end local bname = file.basename(fname) local dname = file.dirname(fname) @@ -7783,7 +9605,7 @@ local function collect_files(names) local files = blobpath and instance.files[blobpath] if files then if trace_detail then - logs.report("fileio",'? blobpath do: %s (%s)',blobpath,bname) + logs.report("fileio","deep checking '%s' (%s)",blobpath,bname) end local blobfile = files[bname] if not blobfile then @@ -7817,7 +9639,7 @@ local function collect_files(names) end end elseif trace_locating then - logs.report("fileio",'! blobpath no: %s (%s)',blobpath,bname) + logs.report("fileio","no match in '%s' (%s)",blobpath,bname) end end end @@ -7867,14 +9689,13 @@ end local function collect_instance_files(filename,collected) -- todo : plugin (scanners, checkers etc) local result = collected or { } local stamp = nil - filename = file.collapse_path(filename) -- elsewhere - filename = file.collapse_path(gsub(filename,"\\","/")) -- elsewhere + filename = file.collapse_path(filename) -- speed up / beware: format problem if instance.remember then stamp = filename .. "--" .. instance.engine .. "--" .. instance.progname .. "--" .. instance.format if instance.found[stamp] then if trace_locating then - logs.report("fileio",'! remembered: %s',filename) + logs.report("fileio","remembering file '%s'",filename) end return instance.found[stamp] end @@ -7882,7 +9703,7 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan if not dangerous[instance.format or "?"] then if resolvers.isreadable.file(filename) then if trace_detail then - logs.report("fileio",'= found directly: %s',filename) + logs.report("fileio","file '%s' found directly",filename) end instance.found[stamp] = { filename } return { filename } @@ -7890,13 +9711,13 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan end if find(filename,'%*') then if trace_locating then - logs.report("fileio",'! wildcard: %s', filename) + logs.report("fileio","checking wildcard '%s'", filename) end result = resolvers.find_wildcard_files(filename) elseif file.is_qualified_path(filename) then if resolvers.isreadable.file(filename) then if trace_locating then - logs.report("fileio",'! qualified: %s', filename) + logs.report("fileio","qualified name '%s'", filename) end result = { filename } else @@ -7906,7 +9727,7 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan forcedname = filename .. ".tex" if resolvers.isreadable.file(forcedname) then if trace_locating then - logs.report("fileio",'! no suffix, forcing standard filetype: tex') + logs.report("fileio","no suffix, forcing standard filetype 'tex'") end result, ok = { forcedname }, true end @@ -7916,7 +9737,7 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan forcedname = filename .. "." .. s if resolvers.isreadable.file(forcedname) then if trace_locating then - logs.report("fileio",'! no suffix, forcing format filetype: %s', s) + logs.report("fileio","no suffix, forcing format filetype '%s'", s) end result, ok = { forcedname }, true break @@ -7928,7 +9749,7 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan -- try to find in tree (no suffix manipulation), here we search for the -- matching last part of the name local basename = file.basename(filename) - local pattern = (filename .. "$"):gsub("([%.%-])","%%%1") + local pattern = gsub(filename .. "$","([%.%-])","%%%1") local savedformat = instance.format local format = savedformat or "" if format == "" then @@ -7938,19 +9759,21 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan instance.format = "othertextfiles" -- kind of everything, maybe texinput is better end -- - local resolved = collect_instance_files(basename) - if #result == 0 then - local lowered = lower(basename) - if filename ~= lowered then - resolved = collect_instance_files(lowered) + if basename ~= filename then + local resolved = collect_instance_files(basename) + if #result == 0 then + local lowered = lower(basename) + if filename ~= lowered then + resolved = collect_instance_files(lowered) + end end - end - resolvers.format = savedformat - -- - for r=1,#resolved do - local rr = resolved[r] - if rr:find(pattern) then - result[#result+1], ok = rr, true + resolvers.format = savedformat + -- + for r=1,#resolved do + local rr = resolved[r] + if find(rr,pattern) then + result[#result+1], ok = rr, true + end end end -- a real wildcard: @@ -7959,14 +9782,14 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan -- local filelist = collect_files({basename}) -- for f=1,#filelist do -- local ff = filelist[f][3] or "" - -- if ff:find(pattern) then + -- if find(ff,pattern) then -- result[#result+1], ok = ff, true -- end -- end -- end end if not ok and trace_locating then - logs.report("fileio",'? qualified: %s', filename) + logs.report("fileio","qualified name '%s'", filename) end end else @@ -7985,12 +9808,12 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan wantedfiles[#wantedfiles+1] = forcedname filetype = resolvers.format_of_suffix(forcedname) if trace_locating then - logs.report("fileio",'! forcing filetype: %s',filetype) + logs.report("fileio","forcing filetype '%s'",filetype) end else filetype = resolvers.format_of_suffix(filename) if trace_locating then - logs.report("fileio",'! using suffix based filetype: %s',filetype) + logs.report("fileio","using suffix based filetype '%s'",filetype) end end else @@ -8002,7 +9825,7 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan end filetype = instance.format if trace_locating then - logs.report("fileio",'! using given filetype: %s',filetype) + logs.report("fileio","using given filetype '%s'",filetype) end end local typespec = resolvers.variable_of_format(filetype) @@ -8010,9 +9833,7 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan if not pathlist or #pathlist == 0 then -- no pathlist, access check only / todo == wildcard if trace_detail then - logs.report("fileio",'? filename: %s',filename) - logs.report("fileio",'? filetype: %s',filetype or '?') - logs.report("fileio",'? wanted files: %s',concat(wantedfiles," | ")) + logs.report("fileio","checking filename '%s', filetype '%s', wanted files '%s'",filename, filetype or '?',concat(wantedfiles," | ")) end for k=1,#wantedfiles do local fname = wantedfiles[k] @@ -8033,36 +9854,59 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan else -- list search local filelist = collect_files(wantedfiles) - local doscan, recurse + local dirlist = { } + if filelist then + for i=1,#filelist do + dirlist[i] = file.dirname(filelist[i][2]) .. "/" + end + end if trace_detail then - logs.report("fileio",'? filename: %s',filename) + logs.report("fileio","checking filename '%s'",filename) end -- a bit messy ... esp the doscan setting here + local doscan for k=1,#pathlist do local path = pathlist[k] if find(path,"^!!") then doscan = false else doscan = true end - if find(path,"//$") then recurse = true else recurse = false end local pathname = gsub(path,"^!+", '') done = false -- using file list - if filelist and not (done and not instance.allresults) and recurse then - -- compare list entries with permitted pattern - pathname = gsub(pathname,"([%-%.])","%%%1") -- this also influences - pathname = gsub(pathname,"/+$", '/.*') -- later usage of pathname - pathname = gsub(pathname,"//", '/.-/') -- not ok for /// but harmless - local expr = "^" .. pathname + if filelist then + local expression + -- compare list entries with permitted pattern -- /xx /xx// + if not find(pathname,"/$") then + expression = pathname .. "/" + else + expression = pathname + end + expression = gsub(expression,"([%-%.])","%%%1") -- this also influences + expression = gsub(expression,"//+$", '/.*') -- later usage of pathname + expression = gsub(expression,"//", '/.-/') -- not ok for /// but harmless + expression = "^" .. expression .. "$" + if trace_detail then + logs.report("fileio","using pattern '%s' for path '%s'",expression,pathname) + end for k=1,#filelist do local fl = filelist[k] local f = fl[2] - if find(f,expr) then - if trace_detail then - logs.report("fileio",'= found in hash: %s',f) - end + local d = dirlist[k] + if find(d,expression) then --- todo, test for readable result[#result+1] = fl[3] resolvers.register_in_trees(f) -- for tracing used files done = true - if not instance.allresults then break end + if instance.allresults then + if trace_detail then + logs.report("fileio","match in hash for file '%s' on path '%s', continue scanning",f,d) + end + else + if trace_detail then + logs.report("fileio","match in hash for file '%s' on path '%s', quit scanning",f,d) + end + break + end + elseif trace_detail then + logs.report("fileio","no match in hash for file '%s' on path '%s'",f,d) end end end @@ -8078,7 +9922,7 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan local fname = file.join(ppname,w) if resolvers.isreadable.file(fname) then if trace_detail then - logs.report("fileio",'= found by scanning: %s',fname) + logs.report("fileio","found '%s' by scanning",fname) end result[#result+1] = fname done = true @@ -8141,7 +9985,7 @@ function resolvers.find_given_files(filename) local hashes = instance.hashes for k=1,#hashes do local hash = hashes[k] - local files = instance.files[hash.tag] + local files = instance.files[hash.tag] or { } local blist = files[bname] if not blist then local rname = "remap:"..bname @@ -8251,9 +10095,9 @@ function resolvers.load(option) statistics.starttiming(instance) resolvers.resetconfig() resolvers.identify_cnf() - resolvers.load_lua() + resolvers.load_lua() -- will become the new method resolvers.expand_variables() - resolvers.load_cnf() + resolvers.load_cnf() -- will be skipped when we have a lua file resolvers.expand_variables() if option ~= "nofiles" then resolvers.load_hash() @@ -8265,22 +10109,23 @@ end function resolvers.for_files(command, files, filetype, mustexist) if files and #files > 0 then local function report(str) - if trace_verbose then + if trace_locating then logs.report("fileio",str) -- has already verbose else print(str) end end - if trace_verbose then - report('') + if trace_locating then + report('') -- ? end - for _, file in ipairs(files) do + for f=1,#files do + local file = files[f] local result = command(file,filetype,mustexist) if type(result) == 'string' then report(result) else - for _,v in ipairs(result) do - report(v) + for i=1,#result do + report(result[i]) -- could be unpack end end end @@ -8327,18 +10172,19 @@ end function table.sequenced(t,sep) -- temp here local s = { } - for k, v in pairs(t) do -- pairs? - s[#s+1] = k .. "=" .. v + for k, v in next, t do -- indexed? + s[#s+1] = k .. "=" .. tostring(v) end return concat(s, sep or " | ") end function resolvers.methodhandler(what, filename, filetype) -- ... + filename = file.collapse_path(filename) local specification = (type(filename) == "string" and resolvers.splitmethod(filename)) or filename -- no or { }, let it bomb local scheme = specification.scheme if resolvers[what][scheme] then if trace_locating then - logs.report("fileio",'= handler: %s -> %s -> %s',specification.original,what,table.sequenced(specification)) + logs.report("fileio","handler '%s' -> '%s' -> '%s'",specification.original,what,table.sequenced(specification)) end return resolvers[what][scheme](filename,filetype) -- todo: specification else @@ -8358,8 +10204,9 @@ function resolvers.clean_path(str) end function resolvers.do_with_path(name,func) - for _, v in pairs(resolvers.expanded_path_list(name)) do -- pairs? - func("^"..resolvers.clean_path(v)) + local pathlist = resolvers.expanded_path_list(name) + for i=1,#pathlist do + func("^"..resolvers.clean_path(pathlist[i])) end end @@ -8368,7 +10215,9 @@ function resolvers.do_with_var(name,func) end function resolvers.with_files(pattern,handle) - for _, hash in ipairs(instance.hashes) do + local hashes = instance.hashes + for i=1,#hashes do + local hash = hashes[i] local blobpath = hash.tag local blobtype = hash.type if blobpath then @@ -8383,7 +10232,7 @@ function resolvers.with_files(pattern,handle) if type(v) == "string" then handle(blobtype,blobpath,v,k) else - for _,vv in pairs(v) do -- ipairs? + for _,vv in next, v do -- indexed handle(blobtype,blobpath,vv,k) end end @@ -8395,7 +10244,7 @@ function resolvers.with_files(pattern,handle) end function resolvers.locate_format(name) - local barename, fmtname = name:gsub("%.%a+$",""), "" + local barename, fmtname = gsub(name,"%.%a+$",""), "" if resolvers.usecache then local path = file.join(caches.setpath("formats")) -- maybe platform fmtname = file.join(path,barename..".fmt") or "" @@ -8443,7 +10292,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-tmp'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -8467,7 +10316,7 @@ luatools with a recache feature.</p> local format, lower, gsub = string.format, string.lower, string.gsub -local trace_cache = false trackers.register("resolvers.cache", function(v) trace_cache = v end) +local trace_cache = false trackers.register("resolvers.cache", function(v) trace_cache = v end) -- not used yet caches = caches or { } @@ -8554,7 +10403,8 @@ function caches.setpath(...) caches.path = '.' end caches.path = resolvers.clean_path(caches.path) - if not table.is_empty({...}) then + local dirs = { ... } + if #dirs > 0 then local pth = dir.mkdirs(caches.path,...) return pth end @@ -8600,6 +10450,7 @@ function caches.savedata(filepath,filename,data,raw) if raw then reduce, simplify = false, false end + data.cache_uuid = os.uuid() if caches.direct then file.savedata(tmaname, table.serialize(data,'return',false,true,false)) -- no hex else @@ -8625,7 +10476,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-res'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -8660,6 +10511,14 @@ prefixes.relative = function(str,n) return resolvers.clean_path(str) end +prefixes.auto = function(str) + local fullname = prefixes.relative(str) + if not lfs.isfile(fullname) then + fullname = prefixes.locate(str) + end + return fullname +end + prefixes.locate = function(str) local fullname = resolvers.find_given_file(str) or "" return resolvers.clean_path((fullname ~= "" and fullname) or str) @@ -8683,6 +10542,16 @@ prefixes.full = prefixes.locate prefixes.file = prefixes.filename prefixes.path = prefixes.pathname +function resolvers.allprefixes(separator) + local all = table.sortedkeys(prefixes) + if separator then + for i=1,#all do + all[i] = all[i] .. ":" + end + end + return all +end + local function _resolve_(method,target) if prefixes[method] then return prefixes[method](target) @@ -8693,7 +10562,8 @@ end local function resolve(str) if type(str) == "table" then - for k, v in pairs(str) do -- ipairs + for k=1,#str do + local v = str[k] str[k] = resolve(v) or v end elseif str and str ~= "" then @@ -8706,7 +10576,7 @@ resolvers.resolve = resolve if os.uname then - for k, v in pairs(os.uname()) do + for k, v in next, os.uname() do if not prefixes[k] then prefixes[k] = function() return v end end @@ -8721,7 +10591,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-inp'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -8742,7 +10612,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-out'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -8758,7 +10628,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-con'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -8769,8 +10639,6 @@ local format, lower, gsub = string.format, string.lower, string.gsub local trace_cache = false trackers.register("resolvers.cache", function(v) trace_cache = v end) local trace_containers = false trackers.register("resolvers.containers", function(v) trace_containers = v end) local trace_storage = false trackers.register("resolvers.storage", function(v) trace_storage = v end) -local trace_verbose = false trackers.register("resolvers.verbose", function(v) trace_verbose = v end) -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v trackers.enable("resolvers.verbose") end) --[[ldx-- <p>Once we found ourselves defining similar cache constructs @@ -8834,7 +10702,7 @@ end function containers.is_valid(container, name) if name and name ~= "" then local storage = container.storage[name] - return storage and not table.is_empty(storage) and storage.cache_version == container.version + return storage and storage.cache_version == container.version else return false end @@ -8886,16 +10754,15 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-use'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -local format, lower, gsub = string.format, string.lower, string.gsub +local format, lower, gsub, find = string.format, string.lower, string.gsub, string.find -local trace_verbose = false trackers.register("resolvers.verbose", function(v) trace_verbose = v end) -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v trackers.enable("resolvers.verbose") end) +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) -- since we want to use the cache instead of the tree, we will now -- reimplement the saver. @@ -8939,19 +10806,20 @@ resolvers.automounted = resolvers.automounted or { } function resolvers.automount(usecache) local mountpaths = resolvers.clean_path_list(resolvers.expansion('TEXMFMOUNT')) - if table.is_empty(mountpaths) and usecache then + if (not mountpaths or #mountpaths == 0) and usecache then mountpaths = { caches.setpath("mount") } end - if not table.is_empty(mountpaths) then + if mountpaths and #mountpaths > 0 then statistics.starttiming(resolvers.instance) - for k, root in pairs(mountpaths) do + for k=1,#mountpaths do + local root = mountpaths[k] local f = io.open(root.."/url.tmi") if f then for line in f:lines() do if line then - if line:find("^[%%#%-]") then -- or %W + if find(line,"^[%%#%-]") then -- or %W -- skip - elseif line:find("^zip://") then + elseif find(line,"^zip://") then if trace_locating then logs.report("fileio","mounting %s",line) end @@ -8996,11 +10864,13 @@ function statistics.check_fmt_status(texname) local luv = dofile(luvname) if luv and luv.sourcefile then local sourcehash = md5.hex(io.loaddata(resolvers.find_file(luv.sourcefile)) or "unknown") - if luv.enginebanner and luv.enginebanner ~= enginebanner then - return "engine mismatch" + local luvbanner = luv.enginebanner or "?" + if luvbanner ~= enginebanner then + return string.format("engine mismatch (luv:%s <> bin:%s)",luvbanner,enginebanner) end - if luv.sourcehash and luv.sourcehash ~= sourcehash then - return "source mismatch" + local luvhash = luv.sourcehash or "?" + if luvhash ~= sourcehash then + return string.format("source mismatch (luv:%s <> bin:%s)",luvhash,sourcehash) end else return "invalid status file" @@ -9019,18 +10889,22 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-zip'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -local format, find = string.format, string.find +local format, find, match = string.format, string.find, string.match +local unpack = unpack or table.unpack -local trace_locating, trace_verbose = false, false +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) -trackers.register("resolvers.verbose", function(v) trace_verbose = v end) -trackers.register("resolvers.locating", function(v) trace_locating = v trace_verbose = v end) +-- zip:///oeps.zip?name=bla/bla.tex +-- zip:///oeps.zip?tree=tex/texmf-local +-- zip:///texmf.zip?tree=/tex/texmf +-- zip:///texmf.zip?tree=/tex/texmf-local +-- zip:///texmf-mine.zip?tree=/tex/texmf-projects zip = zip or { } zip.archives = zip.archives or { } @@ -9041,9 +10915,6 @@ local locators, hashers, concatinators = resolvers.locators, resolvers.hashers, local archives = zip.archives --- zip:///oeps.zip?name=bla/bla.tex --- zip:///oeps.zip?tree=tex/texmf-local - local function validzip(str) -- todo: use url splitter if not find(str,"^zip://") then return "zip:///" .. str @@ -9073,26 +10944,22 @@ function zip.closearchive(name) end end --- zip:///texmf.zip?tree=/tex/texmf --- zip:///texmf.zip?tree=/tex/texmf-local --- zip:///texmf-mine.zip?tree=/tex/texmf-projects - function locators.zip(specification) -- where is this used? startup zips (untested) specification = resolvers.splitmethod(specification) local zipfile = specification.path local zfile = zip.openarchive(name) -- tricky, could be in to be initialized tree if trace_locating then if zfile then - logs.report("fileio",'! zip locator, found: %s',specification.original) + logs.report("fileio","zip locator, archive '%s' found",specification.original) else - logs.report("fileio",'? zip locator, not found: %s',specification.original) + logs.report("fileio","zip locator, archive '%s' not found",specification.original) end end end function hashers.zip(tag,name) - if trace_verbose then - logs.report("fileio","loading zip file %s as %s",name,tag) + if trace_locating then + logs.report("fileio","loading zip file '%s' as '%s'",name,tag) end resolvers.usezipfile(format("%s?tree=%s",tag,name)) end @@ -9117,23 +10984,25 @@ function finders.zip(specification,filetype) local zfile = zip.openarchive(specification.path) if zfile then if trace_locating then - logs.report("fileio",'! zip finder, path: %s',specification.path) + logs.report("fileio","zip finder, archive '%s' found",specification.path) end local dfile = zfile:open(q.name) if dfile then dfile = zfile:close() if trace_locating then - logs.report("fileio",'+ zip finder, name: %s',q.name) + logs.report("fileio","zip finder, file '%s' found",q.name) end return specification.original + elseif trace_locating then + logs.report("fileio","zip finder, file '%s' not found",q.name) end elseif trace_locating then - logs.report("fileio",'? zip finder, path %s',specification.path) + logs.report("fileio","zip finder, unknown archive '%s'",specification.path) end end end if trace_locating then - logs.report("fileio",'- zip finder, name: %s',filename) + logs.report("fileio","zip finder, '%s' not found",filename) end return unpack(finders.notfound) end @@ -9146,20 +11015,25 @@ function openers.zip(specification) local zfile = zip.openarchive(zipspecification.path) if zfile then if trace_locating then - logs.report("fileio",'+ zip starter, path: %s',zipspecification.path) + logs.report("fileio","zip opener, archive '%s' opened",zipspecification.path) end local dfile = zfile:open(q.name) if dfile then logs.show_open(specification) + if trace_locating then + logs.report("fileio","zip opener, file '%s' found",q.name) + end return openers.text_opener(specification,dfile,'zip') + elseif trace_locating then + logs.report("fileio","zip opener, file '%s' not found",q.name) end elseif trace_locating then - logs.report("fileio",'- zip starter, path %s',zipspecification.path) + logs.report("fileio","zip opener, unknown archive '%s'",zipspecification.path) end end end if trace_locating then - logs.report("fileio",'- zip opener, name: %s',filename) + logs.report("fileio","zip opener, '%s' not found",filename) end return unpack(openers.notfound) end @@ -9172,25 +11046,27 @@ function loaders.zip(specification) local zfile = zip.openarchive(specification.path) if zfile then if trace_locating then - logs.report("fileio",'+ zip starter, path: %s',specification.path) + logs.report("fileio","zip loader, archive '%s' opened",specification.path) end local dfile = zfile:open(q.name) if dfile then logs.show_load(filename) if trace_locating then - logs.report("fileio",'+ zip loader, name: %s',filename) + logs.report("fileio","zip loader, file '%s' loaded",filename) end local s = dfile:read("*all") dfile:close() return true, s, #s + elseif trace_locating then + logs.report("fileio","zip loader, file '%s' not found",q.name) end elseif trace_locating then - logs.report("fileio",'- zip starter, path: %s',specification.path) + logs.report("fileio","zip loader, unknown archive '%s'",specification.path) end end end if trace_locating then - logs.report("fileio",'- zip loader, name: %s',filename) + logs.report("fileio","zip loader, '%s' not found",filename) end return unpack(openers.notfound) end @@ -9200,21 +11076,15 @@ end function resolvers.usezipfile(zipname) zipname = validzip(zipname) - if trace_locating then - logs.report("fileio",'! zip use, file: %s',zipname) - end local specification = resolvers.splitmethod(zipname) local zipfile = specification.path if zipfile and not zip.registeredfiles[zipname] then local tree = url.query(specification.query).tree or "" - if trace_locating then - logs.report("fileio",'! zip register, file: %s',zipname) - end local z = zip.openarchive(zipfile) if z then local instance = resolvers.instance if trace_locating then - logs.report("fileio","= zipfile, registering: %s",zipname) + logs.report("fileio","zip registering, registering archive '%s'",zipname) end statistics.starttiming(instance) resolvers.prepend_hash('zip',zipname,zipfile) @@ -9223,10 +11093,10 @@ function resolvers.usezipfile(zipname) instance.files[zipname] = resolvers.register_zip_file(z,tree or "") statistics.stoptiming(instance) elseif trace_locating then - logs.report("fileio","? zipfile, unknown: %s",zipname) + logs.report("fileio","zip registering, unknown archive '%s'",zipname) end elseif trace_locating then - logs.report("fileio",'! zip register, no file: %s',zipname) + logs.report("fileio","zip registering, '%s' not found",zipname) end end @@ -9238,11 +11108,11 @@ function resolvers.register_zip_file(z,tree) filter = format("^%s/(.+)/(.-)$",tree) end if trace_locating then - logs.report("fileio",'= zip filter: %s',filter) + logs.report("fileio","zip registering, using filter '%s'",filter) end local register, n = resolvers.register_file, 0 for i in z:files() do - local path, name = i.filename:match(filter) + local path, name = match(i.filename,filter) if path then if name and name ~= '' then register(files, name, path) @@ -9255,7 +11125,7 @@ function resolvers.register_zip_file(z,tree) n = n + 1 end end - logs.report("fileio",'= zip entries: %s',n) + logs.report("fileio","zip registering, %s files registered",n) return files end @@ -9266,12 +11136,14 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-crl'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } +local gsub = string.gsub + curl = curl or { } curl.cached = { } @@ -9280,9 +11152,9 @@ curl.cachepath = caches.definepath("curl") local finders, openers, loaders = resolvers.finders, resolvers.openers, resolvers.loaders function curl.fetch(protocol, name) - local cachename = curl.cachepath() .. "/" .. name:gsub("[^%a%d%.]+","-") --- cachename = cachename:gsub("[\\/]", io.fileseparator) - cachename = cachename:gsub("[\\]", "/") -- cleanup + local cachename = curl.cachepath() .. "/" .. gsub(name,"[^%a%d%.]+","-") +-- cachename = gsub(cachename,"[\\/]", io.fileseparator) + cachename = gsub(cachename,"[\\]", "/") -- cleanup if not curl.cached[name] then if not io.exists(cachename) then curl.cached[name] = cachename @@ -9328,6 +11200,164 @@ end -- of closure do -- create closure to overcome 200 locals limit +if not modules then modules = { } end modules ['data-lua'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- some loading stuff ... we might move this one to slot 2 depending +-- on the developments (the loaders must not trigger kpse); we could +-- of course use a more extensive lib path spec + +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) + +local gsub, insert = string.gsub, table.insert +local unpack = unpack or table.unpack + +local libformats = { 'luatexlibs', 'tex', 'texmfscripts', 'othertextfiles' } -- 'luainputs' +local clibformats = { 'lib' } + +local _path_, libpaths, _cpath_, clibpaths + +function package.libpaths() + if not _path_ or package.path ~= _path_ then + _path_ = package.path + libpaths = file.split_path(_path_,";") + end + return libpaths +end + +function package.clibpaths() + if not _cpath_ or package.cpath ~= _cpath_ then + _cpath_ = package.cpath + clibpaths = file.split_path(_cpath_,";") + end + return clibpaths +end + +local function thepath(...) + local t = { ... } t[#t+1] = "?.lua" + local path = file.join(unpack(t)) + if trace_locating then + logs.report("fileio","! appending '%s' to 'package.path'",path) + end + return path +end + +local p_libpaths, a_libpaths = { }, { } + +function package.append_libpath(...) + insert(a_libpath,thepath(...)) +end + +function package.prepend_libpath(...) + insert(p_libpaths,1,thepath(...)) +end + +-- beware, we need to return a loadfile result ! + +local function loaded(libpaths,name,simple) + for i=1,#libpaths do -- package.path, might become option + local libpath = libpaths[i] + local resolved = gsub(libpath,"%?",simple) + if trace_locating then -- more detail + logs.report("fileio","! checking for '%s' on 'package.path': '%s' => '%s'",simple,libpath,resolved) + end + if resolvers.isreadable.file(resolved) then + if trace_locating then + logs.report("fileio","! lib '%s' located via 'package.path': '%s'",name,resolved) + end + return loadfile(resolved) + end + end +end + + +package.loaders[2] = function(name) -- was [#package.loaders+1] + if trace_locating then -- mode detail + logs.report("fileio","! locating '%s'",name) + end + for i=1,#libformats do + local format = libformats[i] + local resolved = resolvers.find_file(name,format) or "" + if trace_locating then -- mode detail + logs.report("fileio","! checking for '%s' using 'libformat path': '%s'",name,format) + end + if resolved ~= "" then + if trace_locating then + logs.report("fileio","! lib '%s' located via environment: '%s'",name,resolved) + end + return loadfile(resolved) + end + end + -- libpaths + local libpaths, clibpaths = package.libpaths(), package.clibpaths() + local simple = gsub(name,"%.lua$","") + local simple = gsub(simple,"%.","/") + local resolved = loaded(p_libpaths,name,simple) or loaded(libpaths,name,simple) or loaded(a_libpaths,name,simple) + if resolved then + return resolved + end + -- + local libname = file.addsuffix(simple,os.libsuffix) + for i=1,#clibformats do + -- better have a dedicated loop + local format = clibformats[i] + local paths = resolvers.expanded_path_list_from_var(format) + for p=1,#paths do + local path = paths[p] + local resolved = file.join(path,libname) + if trace_locating then -- mode detail + logs.report("fileio","! checking for '%s' using 'clibformat path': '%s'",libname,path) + end + if resolvers.isreadable.file(resolved) then + if trace_locating then + logs.report("fileio","! lib '%s' located via 'clibformat': '%s'",libname,resolved) + end + return package.loadlib(resolved,name) + end + end + end + for i=1,#clibpaths do -- package.path, might become option + local libpath = clibpaths[i] + local resolved = gsub(libpath,"?",simple) + if trace_locating then -- more detail + logs.report("fileio","! checking for '%s' on 'package.cpath': '%s'",simple,libpath) + end + if resolvers.isreadable.file(resolved) then + if trace_locating then + logs.report("fileio","! lib '%s' located via 'package.cpath': '%s'",name,resolved) + end + return package.loadlib(resolved,name) + end + end + -- just in case the distribution is messed up + if trace_loading then -- more detail + logs.report("fileio","! checking for '%s' using 'luatexlibs': '%s'",name) + end + local resolved = resolvers.find_file(file.basename(name),'luatexlibs') or "" + if resolved ~= "" then + if trace_locating then + logs.report("fileio","! lib '%s' located by basename via environment: '%s'",name,resolved) + end + return loadfile(resolved) + end + if trace_locating then + logs.report("fileio",'? unable to locate lib: %s',name) + end +-- return "unable to locate " .. name +end + +resolvers.loadlualib = require + + +end -- of closure + +do -- create closure to overcome 200 locals limit + if not modules then modules = { } end modules ['luat-kps'] = { version = 1.001, comment = "companion to luatools.lua", @@ -9437,7 +11467,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-aux'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -9445,47 +11475,47 @@ if not modules then modules = { } end modules ['data-aux'] = { local find = string.find -local trace_verbose = false trackers.register("resolvers.verbose", function(v) trace_verbose = v end) +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) function resolvers.update_script(oldname,newname) -- oldname -> own.name, not per se a suffix local scriptpath = "scripts/context/lua" newname = file.addsuffix(newname,"lua") local oldscript = resolvers.clean_path(oldname) - if trace_verbose then + if trace_locating then logs.report("fileio","to be replaced old script %s", oldscript) end local newscripts = resolvers.find_files(newname) or { } if #newscripts == 0 then - if trace_verbose then + if trace_locating then logs.report("fileio","unable to locate new script") end else for i=1,#newscripts do local newscript = resolvers.clean_path(newscripts[i]) - if trace_verbose then + if trace_locating then logs.report("fileio","checking new script %s", newscript) end if oldscript == newscript then - if trace_verbose then + if trace_locating then logs.report("fileio","old and new script are the same") end elseif not find(newscript,scriptpath) then - if trace_verbose then + if trace_locating then logs.report("fileio","new script should come from %s",scriptpath) end elseif not (find(oldscript,file.removesuffix(newname).."$") or find(oldscript,newname.."$")) then - if trace_verbose then + if trace_locating then logs.report("fileio","invalid new script name") end else local newdata = io.loaddata(newscript) if newdata then - if trace_verbose then + if trace_locating then logs.report("fileio","old script content replaced by new content") end io.savedata(oldscript,newdata) break - elseif trace_verbose then + elseif trace_locating then logs.report("fileio","unable to load new script") end end @@ -9500,25 +11530,28 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-tmf'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } +local find, gsub, match = string.find, string.gsub, string.match +local getenv, setenv = os.getenv, os.setenv + -- loads *.tmf files in minimal tree roots (to be optimized and documented) function resolvers.check_environment(tree) logs.simpleline() - os.setenv('TMP', os.getenv('TMP') or os.getenv('TEMP') or os.getenv('TMPDIR') or os.getenv('HOME')) - os.setenv('TEXOS', os.getenv('TEXOS') or ("texmf-" .. os.currentplatform())) - os.setenv('TEXPATH', (tree or "tex"):gsub("\/+$",'')) - os.setenv('TEXMFOS', os.getenv('TEXPATH') .. "/" .. os.getenv('TEXOS')) + setenv('TMP', getenv('TMP') or getenv('TEMP') or getenv('TMPDIR') or getenv('HOME')) + setenv('TEXOS', getenv('TEXOS') or ("texmf-" .. os.platform)) + setenv('TEXPATH', gsub(tree or "tex","\/+$",'')) + setenv('TEXMFOS', getenv('TEXPATH') .. "/" .. getenv('TEXOS')) logs.simpleline() - logs.simple("preset : TEXPATH => %s", os.getenv('TEXPATH')) - logs.simple("preset : TEXOS => %s", os.getenv('TEXOS')) - logs.simple("preset : TEXMFOS => %s", os.getenv('TEXMFOS')) - logs.simple("preset : TMP => %s", os.getenv('TMP')) + logs.simple("preset : TEXPATH => %s", getenv('TEXPATH')) + logs.simple("preset : TEXOS => %s", getenv('TEXOS')) + logs.simple("preset : TEXMFOS => %s", getenv('TEXMFOS')) + logs.simple("preset : TMP => %s", getenv('TMP')) logs.simple('') end @@ -9526,27 +11559,27 @@ function resolvers.load_environment(name) -- todo: key=value as well as lua local f = io.open(name) if f then for line in f:lines() do - if line:find("^[%%%#]") then + if find(line,"^[%%%#]") then -- skip comment else - local key, how, value = line:match("^(.-)%s*([<=>%?]+)%s*(.*)%s*$") + local key, how, value = match(line,"^(.-)%s*([<=>%?]+)%s*(.*)%s*$") if how then - value = value:gsub("%%(.-)%%", function(v) return os.getenv(v) or "" end) + value = gsub(value,"%%(.-)%%", function(v) return getenv(v) or "" end) if how == "=" or how == "<<" then - os.setenv(key,value) + setenv(key,value) elseif how == "?" or how == "??" then - os.setenv(key,os.getenv(key) or value) + setenv(key,getenv(key) or value) elseif how == "<" or how == "+=" then - if os.getenv(key) then - os.setenv(key,os.getenv(key) .. io.fileseparator .. value) + if getenv(key) then + setenv(key,getenv(key) .. io.fileseparator .. value) else - os.setenv(key,value) + setenv(key,value) end elseif how == ">" or how == "=+" then - if os.getenv(key) then - os.setenv(key,value .. io.pathseparator .. os.getenv(key)) + if getenv(key) then + setenv(key,value .. io.pathseparator .. getenv(key)) else - os.setenv(key,value) + setenv(key,value) end end end @@ -9585,6 +11618,9 @@ if not modules then modules = { } end modules ['luat-sta'] = { -- this code is used in the updater +local gmatch, match = string.gmatch, string.match +local type = type + states = states or { } states.data = states.data or { } states.hash = states.hash or { } @@ -9613,13 +11649,17 @@ function states.set_by_tag(tag,key,value,default,persistent) if d then if type(d) == "table" then local dkey, hkey = key, key - local pre, post = key:match("(.+)%.([^%.]+)$") + local pre, post = match(key,"(.+)%.([^%.]+)$") if pre and post then - for k in pre:gmatch("[^%.]+") do + for k in gmatch(pre,"[^%.]+") do local dk = d[k] if not dk then dk = { } d[k] = dk + elseif type(dk) == "string" then + -- invalid table, unable to upgrade structure + -- hope for the best or delete the state file + break end d = dk end @@ -9647,7 +11687,7 @@ function states.get_by_tag(tag,key,default) else local d = states.data[tag] if d then - for k in key:gmatch("[^%.]+") do + for k in gmatch(key,"[^%.]+") do local dk = d[k] if dk then d = dk @@ -9782,6 +11822,7 @@ own.libs = { -- todo: check which ones are really needed 'l-os.lua', 'l-file.lua', 'l-md5.lua', + 'l-url.lua', 'l-dir.lua', 'l-boolean.lua', 'l-math.lua', @@ -9790,11 +11831,13 @@ own.libs = { -- todo: check which ones are really needed 'l-utils.lua', 'l-aux.lua', -- 'l-xml.lua', + 'trac-tra.lua', 'lxml-tab.lua', - 'lxml-pth.lua', - 'lxml-ent.lua', + 'lxml-lpt.lua', +-- 'lxml-ent.lua', 'lxml-mis.lua', - 'trac-tra.lua', + 'lxml-aux.lua', + 'lxml-xml.lua', 'luat-env.lua', 'trac-inf.lua', 'trac-log.lua', @@ -9809,7 +11852,7 @@ own.libs = { -- todo: check which ones are really needed -- 'data-bin.lua', 'data-zip.lua', 'data-crl.lua', --- 'data-lua.lua', + 'data-lua.lua', 'data-kps.lua', -- so that we can replace kpsewhich 'data-aux.lua', -- updater 'data-tmf.lua', -- tree files @@ -9827,7 +11870,8 @@ end -- End of hack. -own.name = (environment and environment.ownname) or arg[0] or 'luatools.lua' +own.name = (environment and environment.ownname) or arg[0] or 'luatools.lua' + own.path = string.match(own.name,"^(.+)[\\/].-$") or "." own.list = { '.' } @@ -9865,18 +11909,25 @@ if not resolvers then os.exit() end -logs.setprogram('MTXrun',"TDS Runner Tool 1.22",environment.arguments["verbose"] or false) +logs.setprogram('MTXrun',"TDS Runner Tool 1.24",environment.arguments["verbose"] or false) local instance = resolvers.reset() +local trackspec = environment.argument("trackers") or environment.argument("track") + +if trackspec then + trackers.enable(trackspec) +end + runners = runners or { } -- global messages = messages or { } messages.help = [[ ---script run an mtx script (--noquotes) ---execute run a script or program (--noquotes) +--script run an mtx script (lua prefered method) (--noquotes), no script gives list +--execute run a script or program (texmfstart method) (--noquotes) --resolve resolve prefixed arguments --ctxlua run internally (using preloaded libs) +--internal run script using built in libraries (same as --ctxlua) --locate locate given filename --autotree use texmf tree cf. env 'texmfstart_tree' or 'texmfstarttree' @@ -9893,16 +11944,20 @@ messages.help = [[ --unix create unix (linux) stubs --verbose give a bit more info +--trackers=list enable given trackers --engine=str target engine --progname=str format or backend --edit launch editor with found file --launch (--all) launch files like manuals, assumes os support ---intern run script using built in libraries +--timedrun run a script an time its run +--autogenerate regenerate databases if needed (handy when used to run context in an editor) + +--usekpse use kpse as fallback (when no mkiv and cache installed, often slower) +--forcekpse force using kpse (handy when no mkiv and cache installed but less functionality) ---usekpse use kpse as fallback (when no mkiv and cache installed, often slower) ---forcekpse force using kpse (handy when no mkiv and cache installed but less functionality) +--prefixes show supported prefixes ]] runners.applications = { @@ -9918,20 +11973,17 @@ runners.suffixes = { } runners.registered = { - texexec = { 'texexec.rb', true }, -- context mkii runner (only tool not to be luafied) + texexec = { 'texexec.rb', false }, -- context mkii runner (only tool not to be luafied) texutil = { 'texutil.rb', true }, -- old perl based index sorter for mkii (old versions need it) texfont = { 'texfont.pl', true }, -- perl script that makes mkii font metric files texfind = { 'texfind.pl', false }, -- perltk based tex searching tool, mostly used at pragma texshow = { 'texshow.pl', false }, -- perltk based context help system, will be luafied - -- texwork = { \texwork.pl', false }, -- perltk based editing environment, only used at pragma - + -- texwork = { 'texwork.pl', false }, -- perltk based editing environment, only used at pragma makempy = { 'makempy.pl', true }, mptopdf = { 'mptopdf.pl', true }, pstopdf = { 'pstopdf.rb', true }, -- converts ps (and some more) images, does some cleaning (replaced) - -- examplex = { 'examplex.rb', false }, concheck = { 'concheck.rb', false }, - runtools = { 'runtools.rb', true }, textools = { 'textools.rb', true }, tmftools = { 'tmftools.rb', true }, @@ -9943,7 +11995,6 @@ runners.registered = { xmltools = { 'xmltools.rb', true }, -- luatools = { 'luatools.lua', true }, mtxtools = { 'mtxtools.rb', true }, - pdftrimwhite = { 'pdftrimwhite.pl', false } } @@ -9952,6 +12003,13 @@ runners.launchers = { unix = { } } +-- like runners.libpath("framework"): looks on script's subpath + +function runners.libpath(...) + package.prepend_libpath(file.dirname(environment.ownscript),...) + package.prepend_libpath(file.dirname(environment.ownname) ,...) +end + function runners.prepare() local checkname = environment.argument("ifchanged") if checkname and checkname ~= "" then @@ -9996,7 +12054,7 @@ function runners.prepare() return "run" end -function runners.execute_script(fullname,internal) +function runners.execute_script(fullname,internal,nosplit) local noquote = environment.argument("noquotes") if fullname and fullname ~= "" then local state = runners.prepare() @@ -10036,17 +12094,20 @@ function runners.execute_script(fullname,internal) end end if result and result ~= "" then - local before, after = environment.split_arguments(fullname) -- already done - environment.arguments_before, environment.arguments_after = before, after + if not no_split then + local before, after = environment.split_arguments(fullname) -- already done + environment.arguments_before, environment.arguments_after = before, after + end if internal then - arg = { } for _,v in pairs(after) do arg[#arg+1] = v end + arg = { } for _,v in pairs(environment.arguments_after) do arg[#arg+1] = v end + environment.ownscript = result dofile(result) else local binary = runners.applications[file.extname(result)] if binary and binary ~= "" then result = binary .. " " .. result end - local command = result .. " " .. environment.reconstruct_commandline(after,noquote) + local command = result .. " " .. environment.reconstruct_commandline(environment.arguments_after,noquote) if logs.verbose then logs.simpleline() logs.simple("executing: %s",command) @@ -10054,8 +12115,24 @@ function runners.execute_script(fullname,internal) logs.simpleline() io.flush() end - local code = os.exec(command) -- maybe spawn - return code == 0 + -- no os.exec because otherwise we get the wrong return value + local code = os.execute(command) -- maybe spawn + if code == 0 then + return true + else + if binary then + binary = file.addsuffix(binary,os.binsuffix) + for p in string.gmatch(os.getenv("PATH"),"[^"..io.pathseparator.."]+") do + if lfs.isfile(file.join(p,binary)) then + return false + end + end + logs.simpleline() + logs.simple("This script needs '%s' which seems not to be installed.",binary) + logs.simpleline() + end + return false + end end end end @@ -10088,7 +12165,7 @@ function runners.execute_program(fullname) return false end --- the --usekpse flag will fallback on kpse +-- the --usekpse flag will fallback on kpse (hm, we can better update mtx-stubs) local windows_stub = '@echo off\013\010setlocal\013\010set ownpath=%%~dp0%%\013\010texlua "%%ownpath%%mtxrun.lua" --usekpse --execute %s %%*\013\010endlocal\013\010' local unix_stub = '#!/bin/sh\010mtxrun --usekpse --execute %s \"$@\"\010' @@ -10143,7 +12220,7 @@ function runners.locate_file(filename) end function runners.locate_platform() - runners.report_location(os.currentplatform()) + runners.report_location(os.platform) end function runners.report_location(result) @@ -10176,7 +12253,8 @@ end function runners.save_script_session(filename, list) local t = { } - for _, key in ipairs(list) do + for i=1,#list do + local key = list[i] t[key] = environment.arguments[key] end io.savedata(filename,table.serialize(t,true)) @@ -10265,20 +12343,22 @@ function runners.find_mtx_script(filename) if fullname and fullname ~= "" then return fullname end + -- mtx- prefix checking + local mtxprefix = (filename:find("^mtx%-") and "") or "mtx-" -- context namespace, mtx-<filename> - fullname = "mtx-" .. filename + fullname = mtxprefix .. filename fullname = found(fullname) or resolvers.find_file(fullname) if fullname and fullname ~= "" then return fullname end -- context namespace, mtx-<filename>s - fullname = "mtx-" .. basename .. "s" .. "." .. suffix + fullname = mtxprefix .. basename .. "s" .. "." .. suffix fullname = found(fullname) or resolvers.find_file(fullname) if fullname and fullname ~= "" then return fullname end -- context namespace, mtx-<filename minus trailing s> - fullname = "mtx-" .. basename:gsub("s$","") .. "." .. suffix + fullname = mtxprefix .. basename:gsub("s$","") .. "." .. suffix fullname = found(fullname) or resolvers.find_file(fullname) if fullname and fullname ~= "" then return fullname @@ -10288,9 +12368,17 @@ function runners.find_mtx_script(filename) return fullname end -function runners.execute_ctx_script(filename,arguments) +function runners.execute_ctx_script(filename) + local arguments = environment.arguments_after local fullname = runners.find_mtx_script(filename) or "" - -- retyr after generate but only if --autogenerate + if file.extname(fullname) == "cld" then + -- handy in editors where we force --autopdf + logs.simple("running cld script: %s",filename) + table.insert(arguments,1,fullname) + table.insert(arguments,"--autopdf") + fullname = runners.find_mtx_script("context") or "" + end + -- retry after generate but only if --autogenerate if fullname == "" and environment.argument("autogenerate") then -- might become the default instance.renewcache = true logs.setverbose(true) @@ -10319,32 +12407,51 @@ function runners.execute_ctx_script(filename,arguments) if logs.verbose then logs.simple("using script: %s\n",fullname) end + environment.ownscript = fullname dofile(fullname) local savename = environment.arguments['save'] - if savename and runners.save_list and not table.is_empty(runners.save_list or { }) then - if type(savename) ~= "string" then savename = file.basename(fullname) end - savename = file.replacesuffix(savename,"cfg") - runners.save_script_session(savename, runners.save_list) + if savename then + local save_list = runners.save_list + if save_list and next(save_list) then + if type(savename) ~= "string" then savename = file.basename(fullname) end + savename = file.replacesuffix(savename,"cfg") + runners.save_script_session(savename,save_list) + end end return true end else - logs.setverbose(true) - if filename == "" then - logs.simple("unknown script, no name given") + -- logs.setverbose(true) + if filename == "" or filename == "help" then local context = resolvers.find_file("mtx-context.lua") + logs.setverbose(true) if context ~= "" then local result = dir.glob((string.gsub(context,"mtx%-context","mtx-*"))) -- () needed local valid = { } - for _, scriptname in ipairs(result) do - scriptname = string.match(scriptname,".*mtx%-([^%-]-)%.lua") - if scriptname then - valid[#valid+1] = scriptname + table.sort(result) + for i=1,#result do + local scriptname = result[i] + local scriptbase = string.match(scriptname,".*mtx%-([^%-]-)%.lua") + if scriptbase then + local data = io.loaddata(scriptname) + local banner, version = string.match(data,"[\n\r]logs%.extendbanner%s*%(%s*[\"\']([^\n\r]+)%s*(%d+%.%d+)") + if banner then + valid[#valid+1] = { scriptbase, version, banner } + end end end if #valid > 0 then - logs.simple("known scripts: %s",table.concat(valid,", ")) + logs.reportbanner() + logs.reportline() + logs.simple("no script name given, known scripts:") + logs.simple() + for k=1,#valid do + local v = valid[k] + logs.simple("%-12s %4s %s",v[1],v[2],v[3]) + end end + else + logs.simple("no script name given") end else filename = file.addsuffix(filename,"lua") @@ -10358,6 +12465,12 @@ function runners.execute_ctx_script(filename,arguments) end end +function runners.prefixes() + logs.reportbanner() + logs.reportline() + logs.simple(table.concat(resolvers.allprefixes(true)," ")) +end + function runners.timedrun(filename) -- just for me if filename and filename ~= "" then runners.timed(function() os.execute(filename) end) @@ -10385,7 +12498,9 @@ instance.lsrmode = environment.argument("lsr") or false -- maybe the unset has to go to this level -if environment.argument("usekpse") or environment.argument("forcekpse") then +local is_mkii_stub = runners.registered[file.removesuffix(file.basename(filename))] + +if environment.argument("usekpse") or environment.argument("forcekpse") or is_mkii_stub then os.setenv("engine","") os.setenv("progname","") @@ -10420,7 +12535,7 @@ if environment.argument("usekpse") or environment.argument("forcekpse") then return (kpse_initialized():show_path(name)) or "" end - elseif environment.argument("usekpse") then + elseif environment.argument("usekpse") or is_mkii_stub then resolvers.load() @@ -10449,7 +12564,6 @@ else end - if environment.argument("selfmerge") then -- embed used libraries utils.merger.selfmerge(own.name,own.libs,own.list) @@ -10462,9 +12576,14 @@ elseif environment.argument("selfupdate") then elseif environment.argument("ctxlua") or environment.argument("internal") then -- run a script by loading it (using libs) ok = runners.execute_script(filename,true) -elseif environment.argument("script") or environment.argument("s") then +elseif environment.argument("script") or environment.argument("scripts") then -- run a script by loading it (using libs), pass args - ok = runners.execute_ctx_script(filename,after) + if is_mkii_stub then + -- execute mkii script + ok = runners.execute_script(filename,false,true) + else + ok = runners.execute_ctx_script(filename) + end elseif environment.argument("execute") then -- execute script ok = runners.execute_script(filename) @@ -10491,6 +12610,8 @@ elseif environment.argument("locate") then elseif environment.argument("platform")then -- locate platform runners.locate_platform() +elseif environment.argument("prefixes") then + runners.prefixes() elseif environment.argument("timedrun") then -- locate platform runners.timedrun(filename) @@ -10499,8 +12620,14 @@ elseif environment.argument("help") or filename=='help' or filename == "" then -- execute script elseif filename:find("^bin:") then ok = runners.execute_program(filename) +elseif is_mkii_stub then + -- execute mkii script + ok = runners.execute_script(filename,false,true) else - ok = runners.execute_script(filename) + ok = runners.execute_ctx_script(filename) + if not ok then + ok = runners.execute_script(filename) + end end if os.platform == "unix" then diff --git a/Master/texmf-dist/scripts/context/lua/mtxrun.rme b/Master/texmf-dist/scripts/context/lua/mtxrun.rme index 9cb56486aba..9850e389d74 100644 --- a/Master/texmf-dist/scripts/context/lua/mtxrun.rme +++ b/Master/texmf-dist/scripts/context/lua/mtxrun.rme @@ -1,3 +1,18 @@ -On MSWindows the mtxrun.lua script is called -with mtxrun.cmd. On Unix you can either rename -mtxrun.lua to mtxrun, or use a symlink. +On MSWindows the mtxrun.lua script is called with +mtxrun.exe. On Unix you can either rename mtxrun.lua +to mtxrun, or use a symlink. + +You can create additional stubs, like + +copy mtxrun.exe luatools.exe +copy mtxrun.exe texexec.exe +copy mtxrun.exe context.exe +copy mtxrun.exe mtx-server.exe + +The mtxrun.exe program is rather dump and only +intercepts mtxrun, luatools and texmfstart (for +old times sake) and passes the buck to mtxrun.lua +which happens to know enough of mkii to deal +with kpse based lookups and therefore acts like +texmfstart but when used with mkiv it behaves +more clever and looks for more. diff --git a/Master/texmf-dist/scripts/context/lua/scite-ctx.lua b/Master/texmf-dist/scripts/context/lua/scite-ctx.lua deleted file mode 100644 index 1b832928909..00000000000 --- a/Master/texmf-dist/scripts/context/lua/scite-ctx.lua +++ /dev/null @@ -1,843 +0,0 @@ --- version : 1.0.0 - 07/2005 (2008: lua 5.1) --- author : Hans Hagen - PRAGMA ADE - www.pragma-ade.com --- copyright : public domain or whatever suits --- remark : part of the context distribution, my first lua code - --- todo: name space for local functions - --- loading: scite-ctx.properties - --- # environment variable --- # --- # CTXSPELLPATH=t:/spell --- # --- # auto language detection --- # --- # % version =1.0 language=uk --- # <?xml version='1.0' language='uk' ?> - --- ext.lua.startup.script=$(SciteDefaultHome)/scite-ctx.lua --- --- # extension.$(file.patterns.context)=scite-ctx.lua --- # extension.$(file.patterns.example)=scite-ctx.lua --- --- # ext.lua.reset=1 --- # ext.lua.auto.reload=1 --- # ext.lua.startup.script=t:/lua/scite-ctx.lua --- --- ctx.menulist.default=\ --- wrap=wrap_text|\ --- unwrap=unwrap_text|\ --- sort=sort_text|\ --- document=document_text|\ --- quote=quote_text|\ --- compound=compound_text|\ --- check=check_text --- --- ctx.spellcheck.language=auto --- ctx.spellcheck.wordsize=4 --- ctx.spellcheck.wordpath=ENV(CTXSPELLPATH) --- --- ctx.spellcheck.wordfile.all=spell-uk.txt,spell-nl.txt --- --- ctx.spellcheck.wordfile.uk=spell-uk.txt --- ctx.spellcheck.wordfile.nl=spell-nl.txt --- ctx.spellcheck.wordsize.uk=4 --- ctx.spellcheck.wordsize.nl=4 --- --- command.name.21.*=CTX Action List --- command.subsystem.21.*=3 --- command.21.*=show_menu $(ctx.menulist.default) --- command.groupundo.21.*=yes --- command.shortcut.21.*=Shift+F11 --- --- command.name.22.*=CTX Check Text --- command.subsystem.22.*=3 --- command.22.*=check_text --- command.groupundo.22.*=yes --- command.shortcut.22.*=Ctrl+L --- --- command.name.23.*=CTX Wrap Text --- command.subsystem.23.*=3 --- command.23.*=wrap_text --- command.groupundo.23.*=yes --- command.shortcut.23.*=Ctrl+M --- --- # command.21.*=check_text --- # command.21.*=dofile e:\context\lua\scite-ctx.lua - --- generic functions - -local crlf = "\n" - -function traceln(str) - trace(str .. crlf) - io.flush() -end - -function table.found(tab, str) - local l, r, p - if #str == 0 then - return false - else - l, r = 1, #tab - while l <= r do - p = math.floor((l+r)/2) - if str < tab[p] then - r = p - 1 - elseif str > tab[p] then - l = p + 1 - else - return true - end - end - return false - end -end - -function string:grab(delimiter) - local list = {} - for snippet in self:gmatch(delimiter) do - list[#list+1] = snippet - end - return list -end - -function string:is_empty() - return not self:find("%S") -end - -function string:expand() - return (self:gsub("ENV%((%w+)%)", os.envvar)) -end - -function string:strip() - return (self:gsub("^%s*(.-)%s*$", "%1")) -end - -do - - local lower, gsub, sub = string.lower, string.gsub, string.sub - - function table.alphasort(list,i) - if i and i > 0 then - local function alphacmp(a,b) - return lower(gsub(sub(a,i),'0',' ')) < lower(gsub(sub(b,i),'0',' ')) - end - table.sort(list,alphacmp) - else - local function alphacmp(a,b) - return a:lower() < b:lower() - end - table.sort(list,alphacmp) - end - end - -end - -function io.exists(filename) - local ok, result, message = pcall(io.open,filename) - if result then - io.close(result) - return true - else - return false - end -end - -function os.envvar(str) - local s = os.getenv(str) - if s ~= '' then - return s - end - s = os.getenv(str:upper()) - if s ~= '' then - return s - end - s = os.getenv(str:lower()) - if s ~= '' then - return s - end -end - --- support functions, maybe editor namespace - --- function column_of_position(position) --- local line = editor:LineFromPosition(position) --- local oldposition = editor.CurrentPos --- local column = 0 --- editor:GotoPos(position) --- while editor.CurrentPos ~= 0 and line == editor:LineFromPosition(editor.CurrentPos) do --- editor:CharLeft() --- column = column + 1 --- end --- editor:GotoPos(oldposition) --- if line > 0 then --- return column -1 --- else --- return column --- end --- end - --- function line_of_position(position) --- return editor:LineFromPosition(position) --- end - -function extend_to_start() - local selectionstart = editor.SelectionStart - local selectionend = editor.SelectionEnd - local line = editor:LineFromPosition(selectionstart) - if line > 0 then - while line == editor:LineFromPosition(selectionstart-1) do - selectionstart = selectionstart - 1 - editor:SetSel(selectionstart,selectionend) - end - else - selectionstart = 0 - end - editor:SetSel(selectionstart,selectionend) - return selectionstart -end - -function extend_to_end() -- editor:LineEndExtend() does not work - local selectionstart = editor.SelectionStart - local selectionend = editor.SelectionEnd - local line = editor:LineFromPosition(selectionend) - while line == editor:LineFromPosition(selectionend+1) do - selectionend = selectionend + 1 - editor:SetSel(selectionstart,selectionend) - end - editor:SetSel(selectionstart,selectionend) - return selectionend -end - -function getfiletype() - local firstline = editor:GetLine(0) - if editor.Lexer == SCLEX_TEX then - return 'tex' - elseif editor.Lexer == SCLEX_XML then - return 'xml' - elseif firstline:find("^%%") then - return 'tex' - elseif firstline:find("^<%?xml") then - return 'xml' - else - return 'unknown' - end -end - --- inspired by LuaExt's scite_Files - -function get_dir_list(mask) - local f - if props['PLAT_GTK'] and props['PLAT_GTK'] ~= "" then - f = io.popen('ls -1 ' .. mask) - else - mask = mask:gsub('/','\\') - local tmpfile = 'scite-ctx.tmp' - local cmd = 'dir /b "' .. mask .. '" > ' .. tmpfile - os.execute(cmd) - f = io.open(tmpfile) - end - local files = {} - if not f then -- path check added - return files - end - for line in f:lines() do - files[#files+1] = line - end - f:close() - return files -end - --- banner - -do - - print("loading scite-ctx.lua definition file\n") - print("- see scite-ctx.properties for configuring info\n") - print("- ctx.spellcheck.wordpath set to " .. props['ctx.spellcheck.wordpath']) - if (props['ctx.spellcheck.wordpath']:lower()):find("ctxspellpath") then - if os.getenv('ctxspellpath') then - print("- ctxspellpath set to " .. os.getenv('CTXSPELLPATH')) - else - print("- 'ctxspellpath is not set") - end - print("- ctx.spellcheck.wordpath expands to " .. string.expand(props['ctx.spellcheck.wordpath'])) - end - print("\n- ctx.wraptext.length is set to " .. props['ctx.wraptext.length']) - if props['ctx.helpinfo'] ~= '' then - print("\n- key bindings:\n") - print(((string.strip(props['ctx.helpinfo'])):gsub("%s*\|%s*","\n"))) - end - print("\n- recognized first lines:\n") - print("xml <?xml version='1.0' language='nl'") - print("tex % language=nl") - -end - --- text functions - --- written while listening to Talk Talk - -local magicstring = string.rep("<ctx-crlf/>", 2) - -function wrap_text() - - -- We always go to the end of a line, so in fact some of - -- the variables set next are not needed. - - local length = props["ctx.wraptext.length"] - - if length == '' then length = 80 else length = tonumber(length) end - - local startposition = editor.SelectionStart - local endposition = editor.SelectionEnd - - if startposition == endposition then return end - - editor:LineEndExtend() - - startposition = editor.SelectionStart - endposition = editor.SelectionEnd - - -- local startline = line_of_position(startposition) - -- local endline = line_of_position(endposition) - -- local startcolumn = column_of_position(startposition) - -- local endcolumn = column_of_position(endposition) - -- - -- editor:SetSel(startposition,endposition) - - local startline = props['SelectionStartLine'] - local endline = props['SelectionEndLine'] - local startcolumn = props['SelectionStartColumn'] - 1 - local endcolumn = props['SelectionEndColumn'] - 1 - - local replacement = { } - local templine = '' - local indentation = string.rep(' ',startcolumn) - local selection = editor:GetSelText() - - selection = selection:gsub("[\n\r][\n\r]","\n") - selection = selection:gsub("\n\n+",' ' .. magicstring .. ' ') - selection = selection:gsub("^%s",'') - - for snippet in selection:gmatch("%S+") do - if snippet == magicstring then - replacement[#replacement+1] = templine - replacement[#replacement+1] = "" - templine = '' - elseif #templine + #snippet > length then - replacement[#replacement+1] = templine - templine = indentation .. snippet - elseif #templine == 0 then - templine = indentation .. snippet - else - templine = templine .. ' ' .. snippet - end - end - - replacement[#replacement+1] = templine - replacement[1] = replacement[1]:gsub("^%s+",'') - - if endcolumn == 0 then - replacement[#replacement+1] = "" - end - - editor:ReplaceSel(table.concat(replacement,"\n")) - -end - -function unwrap_text() - - local startposition = editor.SelectionStart - local endposition = editor.SelectionEnd - - if startposition == endposition then return end - - editor:HomeExtend() - editor:LineEndExtend() - - startposition = editor.SelectionStart - endposition = editor.SelectionEnd - - local magicstring = string.rep("<multiplelines/>", 2) - local selection = string.gsub(editor:GetSelText(),"[\n\r][\n\r]+", ' ' .. magicstring .. ' ') - local replacement = '' - - for snippet in selection:gmatch("%S+") do - if snippet == magicstring then - replacement = replacement .. "\n" - else - replacement = replacement .. snippet .. "\n" - end - end - - if endcolumn == 0 then replacement = replacement .. "\n" end - - editor:ReplaceSel(replacement) - -end - -function sort_text() - - local startposition = editor.SelectionStart - local endposition = editor.SelectionEnd - - if startposition == endposition then return end - - -- local startcolumn = column_of_position(startposition) - -- local endcolumn = column_of_position(endposition) - -- - -- editor:SetSel(startposition,endposition) - - local startline = props['SelectionStartLine'] - local endline = props['SelectionEndLine'] - local startcolumn = props['SelectionStartColumn'] - 1 - local endcolumn = props['SelectionEndColumn'] - 1 - - startposition = extend_to_start() - endposition = extend_to_end() - - local selection = string.gsub(editor:GetSelText(), "%s*$", '') - - list = string.grab(selection,"[^\n\r]+") - table.alphasort(list, startcolumn) - local replacement = table.concat(list, "\n") - - editor:GotoPos(startposition) - editor:SetSel(startposition,endposition) - - if endcolumn == 0 then replacement = replacement .. "\n" end - - editor:ReplaceSel(replacement) - -end - -function document_text() - - local startposition = editor.SelectionStart - local endposition = editor.SelectionEnd - - if startposition == endposition then return end - - startposition = extend_to_start() - endposition = extend_to_end() - - editor:SetSel(startposition,endposition) - - local filetype = getfiletype() - - local replacement = '' - for i = editor:LineFromPosition(startposition), editor:LineFromPosition(endposition) do - local str = editor:GetLine(i) - if filetype == 'xml' then - if str:find("^<%!%-%- .* %-%->%s*$") then - replacement = replacement .. str:gsub("^<%!%-%- (.*) %-%->(%s*)$","%1\n") - elseif not str:is_empty() then - replacement = replacement .. '<!-- ' .. str:gsub("(%s*)$",'') .. " -->\n" - else - replacement = replacement .. str - end - else - if str:find("^%%D%s+$") then - replacement = replacement .. "\n" - elseif str:find("^%%D ") then - replacement = replacement .. str:gsub("^%%D ",'') - else - replacement = replacement .. '%D ' .. str - end - end - end - - editor:ReplaceSel(replacement:gsub("[\n\r]$",'')) - -end - -function quote_text() - - local filetype, leftquotation, rightquotation = getfiletype(), '', '' - - if filetype == 'xml' then - leftquotation, rightquotation = "<quotation>", "</quotation>" - leftquote, rightquote = "<quotation>", "</quote>" - else - leftquotation, rightquotation = "\\quotation {", "}" - leftquote, rightquote = "\\quote {", "}" - end - - local replacement = editor:GetSelText() - replacement = replacement.gsub("\`\`(.-)\'\'", leftquotation .. "%1" .. rightquotation) - replacement = replacement.gsub("\"(.-)\"", leftquotation .. "%1" .. rightquotation) - replacement = replacement.gsub("\`(.-)\'", leftquote .. "%1" .. rightquote ) - replacement = replacement.gsub("\'(.-)\'", leftquote .. "%1" .. rightquote ) - editor:ReplaceSel(replacement) - -end - -function compound_text() - - local filetype = getfiletype() - - if filetype == 'xml' then - editor:ReplaceSel(string.gsub(editor:GetSelText(),"(>[^<%-][^<%-]+)([-\/])(%w%w+)","%1<compound token='%2'/>%3")) - else - editor:ReplaceSel(string.gsub(editor:GetSelText(),"([^\|])([-\/]+)([^\|])","%1|%2|%3")) - end - -end - --- written while listening to Alanis Morissette's acoustic --- Jagged Little Pill and Tori Amos' Beekeeper after --- reinstalling on my good old ATH-7 - -local language = props["ctx.spellcheck.language"] -local wordsize = props["ctx.spellcheck.wordsize"] -local wordpath = props["ctx.spellcheck.wordpath"] - -if language == '' then language = 'uk' end -if wordsize == '' then wordsize = 4 else wordsize = tonumber(wordsize) end - -local wordfile = "" -local wordlist = {} -local worddone = 0 - --- we use wordlist as a hash so that we can add entries without the --- need to sort and also use a fast (built in) search - --- function kpsewhich_file(filename,filetype,progname) --- local progflag, typeflag = '', '' --- local tempname = os.tmpname() --- if progname then --- progflag = " --progname=" .. progname .. " " --- end --- if filetype then --- typeflag = " --format=" .. filetype .. " " --- end --- local command = "kpsewhich" .. progflag .. typeflag .. " " .. filename .. " > " .. tempname --- os.execute(command) --- for line in io.lines(tempname) do --- return string.gsub(line, "\s*$", '') --- end --- end - -function check_text() - - local dlanguage = props["ctx.spellcheck.language"] - local dwordsize = props["ctx.spellcheck.wordsize"] - local dwordpath = props["ctx.spellcheck.wordpath"] - - if dlanguage ~= '' then dlanguage = tostring(language) end - if dwordsize ~= '' then dwordsize = tonumber(wordsize) end - - local firstline, skipfirst = editor:GetLine(0), false - local filetype, wordskip, wordgood = getfiletype(), '', '' - - if filetype == 'tex' then - wordskip = "\\" - elseif filetype == 'xml' then - wordskip = "<" - wordgood = ">" - end - - if props["ctx.spellcheck.language"] == 'auto' then - if filetype == 'tex' then - -- % version =1.0 language=uk - firstline = firstline:gsub("^%%%s*",'') - firstline = firstline:gsub("%s*$",'') - for key, val in firstline:gmatch("(%w+)=(%w+)") do - if key == "language" then - language = val - traceln("auto document language " .. "'" .. language .. "' (tex)") - end - end - skipfirst = true - elseif filetype == 'xml' then - -- <?xml version='1.0' language='uk' ?> - firstline = firstline:gsub("^%<%?xml%s*", '') - firstline = firstline:gsub("%s*%?%>%s*$", '') - for key, val in firstline:gmatch("(%w+)=[\"\'](.-)[\"\']") do - if key == "language" then - language = val - traceln("auto document language " .. "'" .. language .. "' (xml)") - end - end - skipfirst = true - end - end - - local fname = props["ctx.spellcheck.wordfile." .. language] - local fsize = props["ctx.spellcheck.wordsize." .. language] - - if fsize ~= '' then wordsize = tonumber(fsize) end - - if fname ~= '' and fname ~= wordfile then - wordfile, worddone, wordlist = fname, 0, {} - for filename in wordfile:gmatch("[^%,]+") do - if wordpath ~= '' then - filename = string.expand(wordpath) .. '/' .. filename - end - if io.exists(filename) then - traceln("loading " .. filename) - for line in io.lines(filename) do - if not line:find("^[\%\#\-]") then - str = line:gsub("%s*$", '') - rawset(wordlist,str,true) - worddone = worddone + 1 - end - end - else - traceln("unknown file '" .. filename .."'") - end - end - traceln(worddone .. " words loaded") - end - - reset_text() - - if worddone == 0 then - traceln("no (valid) language or wordfile specified") - else - traceln("start checking") - if wordskip ~= '' then - traceln("ignoring " .. wordskip .. "..." .. wordgood) - end - local i, j, lastpos, startpos, endpos, snippet, len, first = 0, 0, -1, 0, 0, '', 0, 0 - local ok, skip, ch = false, false, '' - if skipfirst then first = #firstline end - for k = first, editor.TextLength do - ch = editor:textrange(k,k+1) - if wordgood ~= '' and ch == wordgood then - skip = false - elseif ch == wordskip then - skip = true - end - if ch:find("%w") and not ch:find("%d") then - if not skip then - if ok then - endpos = k - else - startpos = k - endpos = k - ok = true - end - end - elseif ok and not skip then - len = endpos - startpos + 1 - if len >= wordsize then - snippet = editor:textrange(startpos,endpos+1) - i = i + 1 - if wordlist[snippet] or wordlist[snippet:lower()] then -- table.found(wordlist,snippet) - j = j + 1 - else - editor:StartStyling(startpos,INDICS_MASK) - editor:SetStyling(len,INDIC2_MASK) -- INDIC0_MASK+2 - end - end - ok = false - elseif wordgood == '' then - skip = (ch == wordskip) - end - end - traceln(i .. " words checked, " .. (i-j) .. " errors") - end - -end - -function reset_text() - editor:StartStyling(0,INDICS_MASK) - editor:SetStyling(editor.TextLength,INDIC_PLAIN) -end - --- menu - -local menuactions = {} -local menufunctions = {} - -function UserListShow(menutrigger, menulist) - local menuentries = {} - local list = string.grab(menulist,"[^%|]+") - menuactions = {} - for i=1, #list do - if list[i] ~= '' then - for key, val in list[i]:gmatch("%s*(.+)=(.+)%s*") do - menuentries[#menuentries+1] = key - menuactions[key] = val - end - end - end - local menustring = table.concat(menuentries,'|') - if menustring == "" then - traceln("There are no templates defined for this file type.") - else - editor.AutoCSeparator = string.byte('|') - editor:UserListShow(menutrigger,menustring) - editor.AutoCSeparator = string.byte(' ') - end -end - -function OnUserListSelection(trigger,choice) - if menufunctions[trigger] and menuactions[choice] then - return menufunctions[trigger](menuactions[choice]) - else - return false - end -end - --- main menu - -local menutrigger = 12 - -function show_menu(menulist) - UserListShow(menutrigger, menulist) -end - -function process_menu(action) - if not action:find("%(%)$") then - assert(loadstring(action .. "()"))() - else - assert(loadstring(action))() - end -end - -menufunctions[12] = process_menu - --- templates - -local templatetrigger = 13 - -local ctx_template_paths = { "./ctx-templates", "../ctx-templates", "../../ctx-templates" } -local ctx_auto_templates = false -local ctx_template_list = "" - -local ctx_path_list = {} -local ctx_path_done = {} -local ctx_path_name = {} - -function ctx_list_loaded(path) - return ctx_path_list[path] and #ctx_path_list[path] > 0 -end - -function insert_template(templatelist) - if props["ctx.template.scan"] == "yes" then - local path = props["FileDir"] - local rescan = props["ctx.template.rescan"] == "yes" - local suffix = props["ctx.template.suffix." .. props["FileExt"]] -- alas, no suffix expansion here - local current = path .. "+" .. props["FileExt"] - if rescan then - print("re-scanning enabled") - end - ctx_template_list = "" - if not ctx_path_done[path] or rescan then - local pattern = "*.*" - for i, pathname in ipairs(ctx_template_paths) do - print("scanning " .. path:gsub("\\","/") .. "/" .. pathname) - ctx_path_name[path] = pathname - ctx_path_list[path] = get_dir_list(pathname .. "/" .. pattern) - if ctx_list_loaded(path) then - print("finished locating template files") - break - end - end - if ctx_list_loaded(path) then - print(#ctx_path_list[path] .. " template files found") - else - print("no template files found") - end - end - if ctx_list_loaded(path) then - ctx_template_list = "" - local pattern = "%." .. suffix .. "$" - local n = 0 - for j, filename in ipairs(ctx_path_list[path]) do - if filename:find(pattern) then - n = n + 1 - local menuname = filename:gsub("%..-$","") - if ctx_template_list ~= "" then - ctx_template_list = ctx_template_list .. "|" - end - ctx_template_list = ctx_template_list .. menuname .. "=" .. ctx_path_name[path] .. "/" .. filename - end - end - if not ctx_path_done[path] then - print(n .. " suitable template files found") - end - end - ctx_path_done[path] = true - if ctx_template_list == "" then - ctx_auto_templates = false - else - ctx_auto_templates = true - templatelist = ctx_template_list - end - else - ctx_auto_templates = false - end - if templatelist ~= "" then - UserListShow(templatetrigger, templatelist) - end -end - - --- ctx.template.[whatever].[filetype] --- ctx.template.[whatever].data.[filetype] --- ctx.template.[whatever].file.[filetype] --- ctx.template.[whatever].list.[filetype] - -function process_template_one(action) - local text = nil - if ctx_auto_templates then - local f = io.open(action,"r") - if f then - text = string.gsub(f:read("*all"),"\n$","") - f:close() - else - print("unable to auto load template file " .. text) - text = nil - end - end - if not text or text == "" then - text = props["ctx.template." .. action .. ".file"] - if not text or text == "" then - text = props["ctx.template." .. action .. ".data"] - if not text or text == "" then - text = props["ctx.template." .. action] - end - else - local f = io.open(text,"r") - if f then - text = string.gsub(f:read("*all"),"\n$","") - f:close() - else - print("unable to load template file " .. text) - text = nil - end - end - end - if text then - text = text:gsub("\\n","\n") - local pos = text:find("%?") - text = text:gsub("%?","") - editor:insert(editor.CurrentPos,text) - if pos then - editor.CurrentPos = editor.CurrentPos + pos - 1 - editor.SelectionStart = editor.CurrentPos - editor.SelectionEnd = editor.CurrentPos - editor:GotoPos(editor.CurrentPos) - end - end -end - -menufunctions[13] = process_template_one -menufunctions[14] = process_template_two - --- command.name.26.*=Open Logfile --- command.subsystem.26.*=3 --- command.26.*=open_log --- command.save.before.26.*=2 --- command.groupundo.26.*=yes --- command.shortcut.26.*=Ctrl+E - -function open_log() - scite.Open(props['FileName'] .. ".log") -end diff --git a/Master/texmf-dist/scripts/context/lua/x-ldx.lua b/Master/texmf-dist/scripts/context/lua/x-ldx.lua index af5c9c0c87e..e0f21d68cd9 100644 --- a/Master/texmf-dist/scripts/context/lua/x-ldx.lua +++ b/Master/texmf-dist/scripts/context/lua/x-ldx.lua @@ -6,7 +6,8 @@ itself serves as an example of using <logo label='lua'/> in combination with <logo label='tex'/>. I will rewrite this using lpeg once I have the time to study that nice new -subsystem. +subsystem. On the other hand, we cannot expect proper <logo label='tex'/> +ad for educational purposed the syntax migh be wrong. --ldx]]-- banner = "version 1.0.1 - 2007+ - PRAGMA ADE / CONTEXT" @@ -126,79 +127,8 @@ ldx.make_index = true function ldx.enhance(data) -- i need to use lpeg and then we can properly autoindent -) local e = ldx.escape - for _,v in pairs(data) do - if v.code then - local dqs, sqs, com, cmt, cod = { }, { }, { }, { }, e(v.code) - cod = cod:gsub('\\"', "##d##") - cod = cod:gsub("\\'", "##s##") - cod = cod:gsub("%-%-%[%[.-%]%]%-%-", function(s) - cmt[#cmt+1] = s - return "[[[[".. #cmt .."]]]]" - end) - cod = cod:gsub("%-%-([^\n]*)", function(s) - com[#com+1] = s - return "[[".. #com .."]]" - end) - cod = cod:gsub("(%b\"\")", function(s) - dqs[#dqs+1] = s:sub(2,-2) or "" - return "<<<<".. #dqs ..">>>>" - end) - cod = cod:gsub("(%b\'\')", function(s) - sqs[#sqs+1] = s:sub(2,-2) or "" - return "<<".. #sqs ..">>" - end) - cod = cod:gsub("(%a+)",function(key) - local class = ldx.keywords.reserved[key] - if class then - return "<key class='" .. class .. "'>" .. key .. "</key>" - else - return key - end - end) - cod = cod:gsub("<<<<(%d+)>>>>", function(s) - return "<dqs>" .. dqs[tonumber(s)] .. "</dqs>" - end) - cod = cod:gsub("<<(%d+)>>", function(s) - return "<sqs>" .. sqs[tonumber(s)] .. "</sqs>" - end) - cod = cod:gsub("%[%[%[%[(%d+)%]%]%]%]", function(s) - return cmt[tonumber(s)] - end) - cod = cod:gsub("%[%[(%d+)%]%]", function(s) - return "<com>" .. com[tonumber(s)] .. "</com>" - end) - cod = cod:gsub("##d##", "\\\"") - cod = cod:gsub("##s##", "\\\'") - if ldx.make_index then - local lines = cod:split("\n") - local f = "(<key class='1'>function</key>)%s+([%w%.]+)%s*%(" - for k,v in pairs(lines) do - -- functies - v = v:gsub(f,function(key, str) - return "<function>" .. str .. "</function>(" - end) - -- variables - v = v:gsub("^([%w][%w%,%s]-)(=[^=])",function(str, rest) - local t = string.split(str, ",%s*") - for k,v in pairs(t) do - t[k] = "<variable>" .. v .. "</variable>" - end - return table.join(t,", ") .. rest - end) - -- so far - lines[k] = v - end - v.code = table.concat(lines,"\n") - else - v.code = cod - end - end - end -end - -function ldx.enhance(data) -- i need to use lpeg and then we can properly autoindent -) - local e = ldx.escape - for _,v in pairs(data) do + for k=1,#data do + local v = data[k] if v.code then local dqs, sqs, com, cmt, cod = { }, { }, { }, { }, e(v.code) cod = cod:gsub('\\"', "##d##") @@ -244,7 +174,8 @@ function ldx.enhance(data) -- i need to use lpeg and then we can properly autoin if ldx.make_index then local lines = cod:split("\n") local f = "(<key class='1'>function</key>)%s+([%w%.]+)%s*%(" - for k,v in pairs(lines) do + for k=1,#lines do + local v = lines[k] -- functies v = v:gsub(f,function(key, str) return "<function>" .. str .. "</function>(" @@ -252,8 +183,8 @@ function ldx.enhance(data) -- i need to use lpeg and then we can properly autoin -- variables v = v:gsub("^([%w][%w%,%s]-)(=[^=])",function(str, rest) local t = string.split(str, ",%s*") - for k,v in pairs(t) do - t[k] = "<variable>" .. v .. "</variable>" + for k=1,#t do + t[k] = "<variable>" .. t[k] .. "</variable>" end return table.join(t,", ") .. rest end) @@ -276,14 +207,17 @@ and by calculating the indentation we also avoid space troubles. It also makes it possible to change the indentation afterwards. --ldx]]-- -function ldx.as_xml(data) +function ldx.as_xml(data) -- ldx: not needed local t, cmode = { }, false t[#t+1] = "<?xml version='1.0' standalone='yes'?>\n" t[#t+1] = "\n<document xmlns:ldx='http://www.pragma-ade.com/schemas/ldx.rng' xmlns='http://www.pragma-ade.com/schemas/ldx.rng'>\n" - for _,v in pairs(data) do -- ldx: not needed + for k=1,#data do + local v = data[k] if v.code and not v.code:is_empty() then t[#t+1] = "\n<code>\n" - for k,v in pairs(v.code:split("\n")) do -- make this faster + local split = v.code:split("\n") + for k=1,#split do -- make this faster + local v = split[k] local a, b = v:find("^(%s+)") if v then v = v:gsub("[\n\r ]+$","") end if a and b then @@ -384,3 +318,5 @@ The main conversion call is: if arg and arg[1] then ldx.convert(arg[1],arg[2]) end + +--~ exit(1) diff --git a/Master/texmf-dist/scripts/context/perl/mptopdf.pl b/Master/texmf-dist/scripts/context/perl/mptopdf.pl index a6b946baa4d..41d1ae1f7ef 100644 --- a/Master/texmf-dist/scripts/context/perl/mptopdf.pl +++ b/Master/texmf-dist/scripts/context/perl/mptopdf.pl @@ -104,7 +104,7 @@ if (($pattern eq '')||($Help)) { my $error = system ($runner) ; if ($error) { print "\n$program : error while processing mp file\n" ; - exit ; + exit 1 ; } else { $pattern =~ s/\.mp$//io ; @files = glob "$pattern.*" ; @@ -131,7 +131,11 @@ foreach my $file (@files) { } else { $command = "$command \\\\relax $file" ; } - system($command) ; + my $error = system($command) ; + if ($error) { + print "\n$program : error while processing tex file\n" ; + exit 1 ; + } my $pdfsrc = basename($_).".pdf"; rename ($pdfsrc, "$_-$1.pdf") ; if (-e $pdfsrc) { diff --git a/Master/texmf-dist/scripts/context/perl/texshow.pl b/Master/texmf-dist/scripts/context/perl/texshow.pl deleted file mode 100644 index 629c28f99ec..00000000000 --- a/Master/texmf-dist/scripts/context/perl/texshow.pl +++ /dev/null @@ -1,956 +0,0 @@ -eval '(exit $?0)' && eval 'exec perl -w -S $0 ${1+"$@"}' && eval 'exec perl -w -S $0 $argv:q' - if 0; - -#D \module -#D [ file=texshow.pl, -#D version=2006.08.04, -#D title=TeXShow, -#D subtitle=showing \CONTEXT\ commands, -#D author=Taco Hoekwater, -#D date=\currentdate, -#D copyright={Taco Hoekwater}] - -#D Early 1999 \TEXSHOW\ showed up in the \CONTEXT\ distribution. At that time -#D the user interface was described in files named \type {setup*.tex}. The -#D program used a stripped down version of these definition files, generated -#D by \CONTEXT\ itself. \TEXSHOW\ shows you the commands, their (optional) -#D arguments, as well as the parameters and their values. For some five years -#D there was no need to change \TEXSHOW. However, when a few years ago we -#D started providing an \XML\ variant of the user interface definitions, Taco -#D came up with \TEXSHOW||\XML. Because Patricks \CONTEXT\ garden and tools -#D like \CTXTOOLS\ also use the \XML\ definitions, it's time to drop the old -#D \TEX\ based definitions and move forward. From now on Taco's version is the -#D one to be used. -#D -#D Hans Hagen - Januari 2005 -#D -#D ChangeLog: -#D \startitemize -#D \item Add keyboard bindings for quitting the app: Ctrl-q,Ctrl-x,Alt-F4 (2006/07/19) -#D \item Support for define --resolve (2006/08/04) -#D \stopitemize - -use strict; -use Getopt::Long ; -use XML::Parser; -use Data::Dumper; -use Tk; -use Tk::ROText ; -use Config; -use Time::HiRes; - -$Getopt::Long::passthrough = 1 ; # no error message -$Getopt::Long::autoabbrev = 1 ; # partial switch accepted - -my $ShowHelp = 0; -my $Debug = 0; -my $Editmode = 0; -my $Interface = 'cont-en'; -my $current_command; -my $current_interface; -my $current_part; -my @setup_files; - -my %setups; -my %commes; -my %descrs; -my %examps; -my %trees; -my %positions; -my %locations; -my %crosslinks; - - -&GetOptions - ( "help" => \$ShowHelp , - "interface=s" => \$Interface , - "debug" => \$Debug, - "edit" => \$Editmode) ; - -print "\n"; - -show('TeXShow-XML 0.3 beta','Taco Hoekwater 2006',"/"); - -print "\n"; - -if ($ShowHelp) { - show('--help','print this help'); - show('--interface=lg','primary interface'); - show('--debug','print debugging info'); - show('string','show info about command \'string\''); - show('string lg','show info about \'string\' in language \'lg\''); - print "\n"; - exit 0; -} - -my $command = $ARGV[0] || ''; -my $interface = $ARGV[1] || ''; -if ($interface =~ /^[a-z][a-z]$/i) { - $Interface = 'cont-' . lc($interface); -} elsif ($interface && $command =~ /^[a-z][a-z]$/i) { - show('debug',"switching '$interface' and '$command'"); - $Interface = 'cont-' . lc($command); - $command = $interface; -} - -if ($command =~ s/^\\//) { - show('debug','removed initial command backslash'); -} - -show('interface', $Interface); -if ($command){ - show ('command', "\\$command") ; -} - -print "\n"; - -show('status','searching for setup files'); - -my $setup_path; -my ($mainwindow,$interfaceframe,$partframe,$leftframe,$rightframe,$buttonframe); -my ($request,$listbox,$textwindow,%interfacebuttons,%partbuttons); - -my ($textfont,$userfont,$buttonfont); - -my $Part; - -if (setups_found($Interface)) { - $current_interface = ''; - $current_command = ''; - $current_part = 'Command'; - show('status','loading setups') ; - load_setups($Interface) ; - show ('status','initializing display') ; - initialize_display(); - change_setup(); - show_command ($command); - $mainwindow->deiconify(); - show ('status','entering main loop') ; - MainLoop () ; - show ('status','closing down') ; -} else { - show ('error','no setup files found') ; -} -print "\n"; - -sub initialize_display { - my $dosish = ($Config{'osname'} =~ /dos|win/i) ; - my $default_size = $dosish ? 9 : 12 ; - my $s_vertical = 30 ; - my $s_horizontal = 72 ; - my $c_horizontal = 24 ; - if (!$dosish) { - $textfont = "-adobe-courier-bold-r-normal--$default_size-120-75-75-m-70-iso8859-1" ; - $userfont = "-adobe-courier-bold-o-normal--$default_size-120-75-75-m-70-iso8859-1" ; - $buttonfont = "-adobe-helvetica-bold-r-normal--$default_size-120-75-75-p-69-iso8859-1"; - } else { - $textfont = "Courier $default_size " ; - $userfont = "Courier $default_size italic" ; - $buttonfont = "Helvetica $default_size bold " ; - } - $mainwindow = MainWindow -> new ( -title => 'ConTeXt commands' ) ; - $buttonframe = $mainwindow -> Frame () ; # buttons - $leftframe = $mainwindow -> Frame () ; # leftside - $rightframe = $mainwindow -> Frame(); - $request = $rightframe -> Entry (-font => $textfont, - -background => 'ivory1', - -width => $c_horizontal); - $listbox = $rightframe -> Scrolled ('Listbox', - -scrollbars => 'e', - -font => $textfont, - -width => $c_horizontal, - -selectbackground => 'gray', - -background => 'ivory1', - -selectmode => 'browse') ; - $textwindow = $leftframe -> Scrolled ('ROText', - -scrollbars => 'se', - -height => $s_vertical, - -width => $s_horizontal, - -wrap => 'none', - -background => 'ivory1', - -font => $textfont); - $interfaceframe = $leftframe -> Frame(); - $mainwindow -> withdraw() ; - $mainwindow -> resizable ('y', 'y') ; - foreach (@setup_files) { - $interfacebuttons{$_} = $buttonframe -> Radiobutton (-text => $_, - -value => $_, - -font => $buttonfont, - -selectcolor => 'ivory1', - -indicatoron => 0, - -command => \&change_setup, - -variable => \$Interface ); - - $interfacebuttons{$_} -> pack (-padx => '2p',-pady => '2p','-side' => 'left' ); - } - foreach (qw(Command Description Comments Examples)) { - $partbuttons{$_} = $interfaceframe -> Radiobutton (-text => $_, - -value => $_, - -font => $buttonfont, - -selectcolor => 'ivory1', - -indicatoron => 0, - -command => \&change_part, - -variable => \$Part ); - $partbuttons{$_} -> pack (-padx => '2p',-pady => '2p','-side' => 'left' ); - } - # global top - $buttonframe -> pack ( -side => 'top' , -fill => 'x' , -expand => 0 ) ; - # top in left - $interfaceframe -> pack ( -side => 'top' , -fill => 'x' , -expand => 0 ) ; - $textwindow -> pack ( -side => 'top' , -fill => 'both' , -expand => 1 ) ; - $leftframe -> pack ( -side => 'left' , -fill => 'both' , -expand => 1 ) ; - # right - $request -> pack ( -side => 'top' , -fill => 'x' ) ; - $listbox -> pack ( -side => 'bottom' , -fill => 'both' , -expand => 1 ) ; - $rightframe -> pack ( -side => 'right' , -fill => 'both' , -expand => 1 ) ; - $listbox -> bind ('<B1-Motion>', \&show_command ) ; - $listbox -> bind ('<1>' , \&show_command ) ; - $listbox -> bind ('<Key>' , \&show_command ) ; - $textwindow -> tag ('configure', 'user' , -font => $userfont ) ; - $textwindow -> tag ('configure', 'optional' , -font => $userfont ) ; - $textwindow -> tag ('configure', 'command' , -foreground => 'green3' ) ; - $textwindow -> tag ('configure', 'variable' , -font => $userfont ) ; - $textwindow -> tag ('configure', 'default' , -underline => 1 ) ; - $textwindow -> tag ('configure', 'symbol' , -foreground => 'blue3' ) ; - $textwindow -> tag ('configure', 'or' , -foreground => 'yellow3' ) ; - $textwindow -> tag ('configure', 'argument' , -foreground => 'red3' ) ; - $textwindow -> tag ('configure', 'par' , -lmargin1 => '4m' , - -wrap => 'word' , - -lmargin2 => '6m' ) ; - foreach my $chr ('a'..'z','A'..'Z') { - $mainwindow -> bind ( "<KeyPress-$chr>", sub { insert_request(shift, $chr) } ); - } - $request -> bind ('<Return>', sub { handle_request() } ) ; - $mainwindow -> bind ( "<backslash>", sub { insert_request(shift, "\\") } ) ; - $mainwindow -> bind ( "<space>", sub { new_request() } ) ; - $mainwindow -> bind ( "<BackSpace>", sub { delete_request() } ) ; - $mainwindow -> bind ( "<Prior>", sub { prev_command() } ) ; - $mainwindow -> bind ( "<Next>", sub { next_command() } ) ; - $mainwindow -> bind ( "<Control-x>", sub { exit(0) } ) ; - $mainwindow -> bind ( "<Control-X>", sub { exit(0) } ) ; - $mainwindow -> bind ( "<Control-q>", sub { exit(0) } ) ; - $mainwindow -> bind ( "<Control-Q>", sub { exit(0) } ) ; - $mainwindow -> bind ( "<Alt-F4>", sub { exit(0) } ) ; -} - -sub show { - my ($pre,$post,$sep) = @_; - unless ($pre eq 'debug' && !$Debug) { - $sep = ':' unless defined $sep; - print sprintf("%22s $sep %+s\n",$pre,$post); - } -} - -sub change_setup { - # switches to another setup file - if ($current_interface ne $Interface ) { - my $loc = 0; - if ($current_command) { - $loc = $positions{$Interface}{$current_command} || 0; - } - my @list = sort {lc $a cmp lc $b} keys %{$setups{$Interface}} ; - my $num = 0; - map { $locations{$Interface}{$_} = $num++; } @list; - $listbox -> delete ('0.0', 'end') ; - $listbox -> insert ('end', @list) ; - # try to switch to other command as well, here. - if ($current_command ne '') { - show_command($crosslinks{$Interface}[$loc] || ''); - } else { - $listbox -> selectionSet ('0.0', '0.0') ; - $listbox -> activate ('0.0') ; - } - } - $current_interface = $Interface; - $mainwindow -> focus ; -} - -sub change_part { - if ($Part ne $current_part) { - if($Part eq 'Command') { - show_command(); - } elsif ($Part eq 'Description') { - show_description(); - } elsif ($Part eq 'Comments') { - show_comments(); - } elsif ($Part eq 'Examples') { - show_examples(); - } - } - $current_part = $Part; -} - - -sub setups_found { - # find the setup files - my ($primary) = @_; - $setup_path = `kpsewhich --progname=context cont-en.xml` ; - chomp $setup_path; - show ('debug', "path = '$setup_path'"); - if ($setup_path) { - $setup_path =~ s/cont-en\.xml.*// ; - @setup_files = glob ("${setup_path}cont\-??.xml") ; # HH: pattern patched, too greedy - show ('debug', "globbed path into '@setup_files'"); - if (@setup_files) { - my $found = 0; - foreach (@setup_files) { - s/\.xml.*$//; - s/^.*?cont-/cont-/; - if ($_ eq $primary) { - $found = 1; - show ('debug', "found primary setup '$primary'"); - } else { - show ('debug', "found non-primary setup '$_'"); - } - } - if ($found) { - return 1; - } else { - show('error',"setup file for '$primary' not found, using 'cont-en'"); - $Interface = 'cont-en'; - return 1; - } - } else { - show('error',"setup file glob failed"); - } - } elsif ($!) { - show('error','kpsewhich not found'); - } else { - show('error','setup files not found'); - } - return 0; -} - -sub load_setup { - my ($path,$filename) = @_; - my $localdefs = {}; - unless (keys %{$setups{$filename}}) { - if (open(IN,"<${path}$filename.xml")) { - my $position = 0 ; - local $/ = '</cd:command>'; - while (my $data= <IN>) { - next if $data =~ /\<\/cd:interface/; - if ($data =~ /\<cd:interface/) { - $data =~ s/^(.*?)\<cd:command/\<cd:command/sm; - my $meta = $1; - while ($meta =~ s!<cd:define name=(['"])(.*?)\1>(.*?)</cd:define>!!sm) { - my $localdef = $2; - my $localval = $3; - $localdefs->{$localdef} = $localval; - } - } - # - if (keys %$localdefs) { - while ($data =~ /<cd:resolve/) { - $data =~ s/<cd:resolve name=(['"])(.*?)\1\/>/$localdefs->{$2}/ms; - } - } - $data =~ s/\s*\n\s*//g; - $data =~ /\<cd:command(.*?)\>/; - my $info = $1; - my ($name,$environment) = ('',''); - while ($info =~ s/^\s*(.*?)\s*=\s*(["'])(.*?)\2\s*//) { - my $a = $1; my $b = $3; - if ($a eq 'name') { - $name = $b; - } elsif ($a eq 'type') { - $environment = $b; - } - } - my $cmd = $name; - if ($environment) { - $cmd = "start" . $name; - } - $setups {$filename}{$cmd} = $data ; - $trees {$filename}{$cmd} = undef; - $positions {$filename}{$cmd} = ++$position ; - $crosslinks{$filename}[$position] = $cmd ; - } - close IN; - # now get explanations as well ... - my $explname = $filename; - $explname =~ s/cont-/expl-/; - my $extras = 0 ; - if (open(IN,"<${path}$explname.xml")) { - local $/ = '</cd:explanation>'; - while (my $data= <IN>) { - if ($data =~ /\<\/cd:explanations/) { - next; - } - if ($data =~ /\<cd:explanations/) { - $data =~ s/^(.*?)\<cd:explanation /\<cd:explanation /sm; - my $meta = $1; - } - # - $extras++; - $data =~ /\<cd:explanation(.*?)\>/; - my $info = $1; - my ($name,$environment) = ('',''); - while ($info =~ s/^\s*(.*?)\s*=\s*(["'])(.*?)\2\s*//) { - my $a = $1; my $b = $3; - if ($a eq 'name') { - $name = $b; - } elsif ($a eq 'type') { - $environment = $b; - } - } - my $cmd = $name; - if ($environment) { - $cmd = "start" . $name; - } - my $comment = ''; - my $description = ''; - my @examples = (); - $data =~ /\<cd:description\>(.*)\<\/cd:description\>/s and $description = $1; - $data =~ /\<cd:comment\>(.*)\<\/cd:comment\>/s and $comment = $1; - while ($data =~ s/\<cd:example\>(.*?)\<\/cd:example\>//s) { - push @examples, $1; - } - if (length($comment) && $comment =~ /\S/) { - $commes {$filename}{$cmd} = $comment; - } - if (length($description) && $description =~ /\S/) { - $descrs {$filename}{$cmd} = $description; - } - my $testex = "@examples"; - if (length($testex) && $testex =~ /\S/) { - $examps {$filename}{$cmd} = [@examples]; - } - } - } - if ($extras) { - show('debug',"interface '$filename', $position\&$extras commands"); - } else { - show('debug',"interface '$filename', $position commands"); - } - } else { - show ('debug',"open() of ${path}$filename.xml failed"); - } - } - $Interface = $filename ; -} - -sub load_setups { - my ($primary) = @_; - # load all setup files, but default to $primary - my $t0 = [Time::HiRes::gettimeofday()]; - foreach my $setup (@setup_files) { - if ($setup ne $primary) { - load_setup ($setup_path,$setup); - show('status',"loading '$setup' took " .Time::HiRes::tv_interval($t0) . " seconds"); - $t0 = [Time::HiRes::gettimeofday()]; - }; - }; - load_setup ($setup_path,$primary); - show('status',"loading '$primary' took " .Time::HiRes::tv_interval($t0) . " seconds"); -} - -my @history = (); -my $current_history = 0; - -sub show_command { - my ($command,$nofix) = @_; - if (keys %{$setups{$Interface}}) { - my $key = ''; - if (defined $command && $command && - (defined $setups{$Interface}{$command} || - defined $setups{$Interface}{"start" . $command})) { - $key = $command; - my $whence =$locations{$Interface}{$command}; - $listbox -> selectionClear ('0.0','end') ; - $listbox -> selectionSet($whence,$whence); - $listbox -> activate($whence); - $listbox -> see($whence); - } else { - $listbox -> selectionSet('0.0','0.0') unless $listbox->curselection(); - $key = $listbox -> get($listbox->curselection()) ; - } - show('debug',"current command: $current_command"); - show('debug'," new command: $key"); - $current_command = $key ; - $textwindow -> delete ('1.0', 'end' ) ; - $partbuttons{"Command"}->select(); - $partbuttons{"Command"}->configure('-state' => 'normal'); - $partbuttons{"Description"}->configure('-state' => 'disabled'); - $partbuttons{"Comments"}->configure('-state' => 'disabled'); - $partbuttons{"Examples"}->configure('-state' => 'disabled'); - if (defined $commes{$Interface}{$key}) { - $partbuttons{"Comments"}->configure('-state' => 'normal'); - } - if (defined $descrs{$Interface}{$key}) { - $partbuttons{"Description"}->configure('-state' => 'normal'); - } - if (defined $examps{$Interface}{$key}) { - $partbuttons{"Examples"}->configure('-state' => 'normal'); - } - unless (defined $nofix && $nofix) { - push @history, $key; - $current_history = $#history; - } - do_update_command ($key) ; - $mainwindow -> update(); - $mainwindow -> focus() ; - } -} - -sub prev_command { - if ($current_history > 0) { - $current_history--; - show_command($history[$current_history],1); - } -} - -sub next_command { - unless ($current_history == $#history) { - $current_history++; - show_command($history[$current_history],1); - } -} - -sub show_description { - $textwindow -> delete ('1.0', 'end' ) ; - if (defined $descrs{$current_interface}{$current_command}) { - $textwindow-> insert ('end',$descrs{$current_interface}{$current_command}); - } - $mainwindow -> update(); - $mainwindow -> focus() ; -} - -sub show_comments { - $textwindow -> delete ('1.0', 'end' ) ; - if (defined $commes{$current_interface}{$current_command}) { - $textwindow-> insert ('end',$commes{$current_interface}{$current_command}); - } - $mainwindow -> update(); - $mainwindow -> focus() ; -} - - -sub show_examples { - $textwindow -> delete ('1.0', 'end' ) ; - if (defined $examps{$current_interface}{$current_command}) { - $textwindow-> insert ('end',join("\n\n",@{$examps{$current_interface}{$current_command}})); - } - $mainwindow -> update(); - $mainwindow -> focus() ; -} - - - - -sub has_attr { - my ($elem,$att,$val) = @_; - return 1 if (attribute($elem,$att) eq $val); - return 0; -} - - -sub view_post { - my ($stuff,$extra) = @_; - $extra = '' unless defined $extra; - $stuff =~ /^(.)(.*?)(.)$/; - my ($l,$c,$r) = ($1,$2,$3); - if ($l eq '[' || $l eq '(') { - return ($l,['symbol','par',$extra],$c,['par',$extra],$r,['symbol','par',$extra],"\n",'par'); - } else { - return ($l,['argument','par',$extra],$c,['par',$extra],$r,['argument','par',$extra],"\n",'par'); - } -} - -sub view_pre { - my ($stuff) = @_; - $stuff =~ /^(.)(.*?)(.)$/; - my ($l,$c,$r) = ($1,$2,$3); - if ($l eq '[' || $l eq '(') { - return ($l,['symbol'],$c,'',$r,['symbol']); - } else { - return ($l,['argument'],$c,'',$r,['argument']); - } -} - -sub create_setup_arguments { - my $argx = shift; - my @predisp = (); - my @postdisp = (); - foreach my $arg (children($argx)) { - if (name($arg) eq 'cd:keywords') { - # children are Constant* & Inherit? & Variable* - my @children = children($arg); - my $optional = (attribute($arg,'optional') eq 'yes' ? 'optional' : ''); - if (@children){ - push @predisp,'[', ['symbol',$optional]; - if (has_attr($arg,'list', 'yes')) { - if (has_attr($arg,'interactive', 'exclusive')) { - push @predisp, '...', ''; - } else { - push @predisp, '..,...,..', ''; - } - } else { - push @predisp,'...', ''; - } - push @predisp,']', ['symbol',$optional]; - } - push @postdisp,'[', ['symbol','par',$optional]; - my $firsttrue = 1; - foreach my $kwd (@children) { - if ($firsttrue) { - $firsttrue = 0; - } else { - push @postdisp,', ', ['symbol','par']; - } - if (name($kwd) eq 'cd:constant' || - name($kwd) eq 'cd:variable') { - my $v = attribute($kwd,'type'); - my $def = ''; - my $var = ''; - $var = 'variable' if (name($kwd) eq 'cd:variable') ; - $def = 'default' if (has_attr($kwd,'default', 'yes')); - if ($v =~ /^cd:/) { - $v =~ s/^cd://; - $v .= "s" if (has_attr($arg,'list', 'yes')); - push @postdisp, $v, ['user',$def,'par',$var]; - } else { - push @postdisp, $v, [$def,'par',$var]; - } - } elsif (name($kwd) eq 'cd:inherit') { - my $v = attribute($kwd,'name'); - $textwindow -> tag ('configure', $v , -foreground => 'blue3',-underline => 1 ) ; - $textwindow -> tagBind($v,'<ButtonPress>',sub {show_command($v)} ); - push @postdisp,"see ","par", "$v", [$v,'par']; - } - } - push @postdisp,']', ['symbol','par',$optional]; - push @postdisp,"\n", 'par'; - } elsif (name($arg) eq 'cd:assignments') { - # children are Parameter* & Inherit? - my @children = children($arg); - my $optional = (attribute($arg,'optional') eq 'yes' ? 'optional' : ''); - if (@children) { - push @predisp,'[', ['symbol',$optional]; - if (has_attr($arg,'list', 'yes')) { - push @predisp, '..,..=..,..', ''; - } else { - push @predisp,'..=..', ''; - } - push @predisp,']', ['symbol',$optional]; - push @postdisp,'[', ['symbol','par',$optional]; - my $isfirst = 1; - foreach my $assn (@children) { - if ($isfirst) { - $isfirst = 0; - } else { - push @postdisp,",\n ", ['symbol','par']; - } - if (name($assn) eq 'cd:parameter') { - push @postdisp,attribute($assn,'name'), 'par'; - push @postdisp,'=', ['symbol','par']; - my $firstxtrue = 1; - foreach my $par (children($assn)) { - if ($firstxtrue) { - $firstxtrue = 0; - } else { - push @postdisp,'|', ['or','par']; - } - if (name($par) eq 'cd:constant' || name($par) eq 'cd:variable') { - my $var = ''; - $var = 'variable' if name($par) eq 'cd:variable'; - my $v = attribute($par,'type'); - if ($v =~ /^cd:/) { - $v =~ s/^cd://; - push @postdisp,$v, ['user','par',$var]; - } else { - push @postdisp,$v, ['par',$var]; - } - } - } - } elsif (name($assn) eq 'cd:inherit') { - my $v = attribute($assn,'name'); - $textwindow -> tag ('configure', $v , -foreground => 'blue3',-underline => 1 ) ; - $textwindow -> tagBind($v,'<ButtonPress>',sub {show_command($v)} ); - push @postdisp,"see ","par", "$v", [$v,'par']; - } - } - push @postdisp,"]", ['symbol','par',$optional], "\n", ''; - } - } elsif (name($arg) eq 'cd:content') { - push @predisp, view_pre('{...}'); - push @postdisp, view_post('{...}'); - } elsif (name($arg) eq 'cd:triplet') { - if (has_attr($arg,'list','yes')) { - push @predisp, view_pre('[x:y:z=,..]'); - push @postdisp,view_post('[x:y:z=,..]'); - } else { - push @predisp, view_pre('[x:y:z=]'); - push @postdisp,view_post('[x:y:z=]'); - } - } elsif (name($arg) eq 'cd:reference') { - my $optional = (attribute($arg,'optional') eq 'yes' ? 'optional' : ''); - if (has_attr($arg,'list','yes')) { - push @postdisp, view_post('[ref,..]',$optional); - push @predisp, view_pre('[ref,..]'); - } else { - push @postdisp, view_post('[ref]',$optional); - push @predisp, view_pre('[ref]');; - } - } elsif (name($arg) eq 'cd:word') { - if (has_attr($arg,'list','yes')) { - push @predisp, view_pre ('{...,...}'); - push @postdisp,view_post('{...,...}'); - } else { - push @predisp, view_pre('{...}'); - push @postdisp, view_post('{...}'); - } - } elsif (name($arg) eq 'cd:nothing') { - my $sep = attribute($arg,'separator'); - if ($sep) { - if($sep eq 'backslash') { -# push @postdisp,'\\\\','par'; - push @predisp,'\\\\',''; - } else { -# push @postdisp,$sep,'par'; - push @predisp,$sep,''; - } - } - push @predisp,'...',''; - push @postdisp,'text',['variable','par'],"\n",'par'; - } elsif (name($arg) eq 'cd:file') { - push @predisp,'...',['default']; - push @postdisp,'...',['default','par'],"\n",'par'; - } elsif (name($arg) eq 'cd:csname') { - push @predisp,'\command',['command']; - push @postdisp,'\command',['command','par'],"\n",'par'; - } elsif (name($arg) eq 'cd:index') { - if (has_attr($arg,'list','yes')) { - push @predisp,view_pre('{..+...+..}'); - push @postdisp,view_post('{..+...+..}'); - } else { - push @predisp, view_pre('{...}'); - push @postdisp,view_post('{...}'); - } - } elsif (name($arg) eq 'cd:position') { - if (has_attr($arg,'list','yes')) { - push @predisp,view_pre('(...,...)'); - push @postdisp,view_post('(...,...)'); - } else { - push @predisp,view_pre('(...)'); - push @postdisp,view_post('(...)'); - } - } elsif (name($arg) eq 'cd:displaymath') { - push @predisp, ('$$',['argument'],'...','','$$',['argument']); - push @postdisp, ('$$',['argument','par'],'...',['par'],'$$',['argument','par']); - } elsif (name($arg) eq 'cd:tex') { - my $sep = attribute($arg,'separator'); - if ($sep) { - if($sep eq 'backslash') { -# push @postdisp,'\\\\','par'; - push @predisp,'\\\\',''; - } else { -# push @postdisp,$sep,'par'; - push @predisp,$sep,''; - } - } - my $cmd = "\\" . attribute($arg,'command'); - push @predisp,$cmd,''; -# push @postdisp,$cmd,['command','par'],"\n",'par'; - } - } - return (\@predisp,\@postdisp); -} - - -# <foo><head id="a">Hello <em>there</em></head><bar>Howdy<ref/></bar>do</foo> -# -# would be: -# -# Tag Content -# ================================================================== -# [foo, [{}, head, [{id => "a"}, 0, "Hello ", em, [{}, 0, "there"]], -# bar, [ {}, 0, "Howdy", ref, [{}]], -# 0, "do" -# ] -# ] - -sub attribute { - my ($elem,$att) = @_; - if (defined $elem->[1] && defined $elem->[1]->[0] && defined $elem->[1]->[0]->{$att}) { - my $ret = $elem->[1]->[0]->{$att}; - show ('debug',"returning attribute $att=$ret"); - return $elem->[1]->[0]->{$att}; - } else { - return ''; - } -} - -sub name { - my ($elem) = @_; - if (defined $elem->[0] ) { - return $elem->[0]; - } else { - return ''; - } -} - -# return all children at a certain level -sub children { - my ($elem) = @_; - if (defined $elem->[1] && defined $elem->[1]->[1]) { - my @items = @{$elem->[1]}; - shift @items ; # deletes the attribute. - my @ret = (); - while (@items) { - push @ret, [shift @items, shift @items]; - } - return @ret; - } else { - return (); - } -} - -# return the first child with the right name -sub find { - my ($elem,$name) = @_; - if ($elem->[0] eq $name) { - return $elem; - } - if (ref($elem->[1]) eq 'ARRAY') { - my @contents = @{$elem->[1]}; - shift @contents; - while (my $ename = shift @contents) { - my $con = shift @contents; - if ($ename eq $name) { - return [$ename,$con]; - } - } - } - return []; -} - -sub do_update_command # type: 0=display, 1=compute only - { my ($command, $type) = @_ ; - $type = 0 unless defined $type; - my $setup; - if (!defined $trees{$Interface}{$command}) { - my $parser = XML::Parser->new('Style' => 'Tree'); - $trees{$Interface}{$command} = $parser->parse($setups{$Interface}{$command}); - } - $setup = $trees{$Interface}{$command} ; - my $predisp = undef; - my $postdisp = undef; - my @cmddisp = (); - my @cmddispafter = (); - my $pradisp = undef; - my $altdisp = undef; - if (attribute($setup,'file')) { - my $filename = attribute($setup,'file'); - my $fileline = attribute($setup,'line') || 0; - $textwindow->insert ('end',"$filename:${fileline}::\n\n", '' ); - } - # start with backslash - push @cmddisp, "\\", 'command' ; - my $env = 0; - if (has_attr($setup,'type','environment')) { - $env = 1; - } - if ($env) { push @cmddisp, "start", 'command' ; } - if ($env) { push @cmddispafter, " ... ", '', "\\stop", 'command' ; } - my $seq = find($setup,'cd:sequence'); - # display rest of command name - foreach my $seqpart (children($seq)) { - my $text = attribute($seqpart,'value'); - if (name($seqpart) eq 'cd:variable') { - push @cmddisp, $text, ['command','user']; - if ($env) { push @cmddispafter, $text, ['command','user']; } - } elsif (name($seqpart) eq 'cd:string') { - push @cmddisp, $text, 'command'; - if ($env) { push @cmddispafter, $text, 'command'; } - } - } - # - my $args = find($setup,'cd:arguments'); - # display commands - if ($args) { - my $curarg = 0; - foreach my $arg (children($args)) { - if (name($arg) eq 'cd:choice') { - my ($a,$b) = children($arg); - ($predisp,$postdisp) = create_setup_arguments(['cd:arguments',[{}, @$a]]); - ($pradisp,$altdisp) = create_setup_arguments(['cd:arguments',[{}, @$b]]); - } else { - ($predisp,$postdisp) = create_setup_arguments($args); - } - $curarg++; - } - } - return if $type; - if(defined $postdisp) { - if(defined $altdisp) { - $textwindow->insert('end',@cmddisp,@$predisp,@cmddispafter, "\n",'', - @cmddisp,@$pradisp,@cmddispafter, "\n\n",'', - @cmddisp, "\n",'', - @$postdisp, "\n",'', - @cmddisp, "\n",'', - @$altdisp); - } else { - $textwindow->insert('end',@cmddisp,@$predisp, @cmddispafter ,"\n\n",'', - @cmddisp,"\n",'', - @$postdisp); - } - } else { - $textwindow->insert('end',@cmddisp); - } -} - - -#D The next feature is dedicated to Tobias, who suggested -#D it, and Taco, who saw it as yet another proof of the -#D speed of \PERL. It's also dedicated to Ton, who needs it -#D for translating the big manual. - -sub handle_request { - my $index = $listbox -> index('end') ; - return unless $index; - my $req = $request -> get ; - return unless $req; - $req =~ s/\\//o ; - $req =~ s/\s//go ; - $request -> delete('0','end') ; - $request -> insert('0',$req) ; - return unless $req; - my ($l,$c) = split (/\./,$index) ; - for (my $i=0;$i<=$l;$i++) { - $index = "$i.0" ; - my $str = $listbox -> get ($index, $index) ; - if (defined $str && ref($str) eq 'ARRAY') { - $str = "@{$str}"; - } - if (defined $str && $str =~ /^$req/) { - show_command($str) ; - return ; - } - } -} - -sub insert_request { - my ($self, $chr) = @_ ; - # don't echo duplicate if $chr was keyed in in the (focussed) entrybox - $request -> insert ('end', $chr) unless $self eq $request; - handle_request(); -} - -sub delete_request { - my $self = shift ; - # delete last character, carefully - if ($self ne $request) { - my $to = $request -> index ('end') ; - my $from = $to - 1 ; - if ($from<0) { $from = 0 } - $request -> delete ($from,$to); - } - handle_request(); -} - -sub new_request { - $request -> delete (0,'end') ; - handle_request(); -} - diff --git a/Master/texmf-dist/scripts/context/ruby/base/ctx.rb b/Master/texmf-dist/scripts/context/ruby/base/ctx.rb index a077297f243..13b4045afe0 100644 --- a/Master/texmf-dist/scripts/context/ruby/base/ctx.rb +++ b/Master/texmf-dist/scripts/context/ruby/base/ctx.rb @@ -124,6 +124,11 @@ class CtxRunner report("loading ctx file #{@ctxname}") end + if @xmldata then + # out if a sudden rexml started to be picky about namespaces + @xmldata.gsub!(/<ctx:job>/,"<ctx:job xmlns:ctx='http://www.pragma-ade.com/rng/ctx.rng'>") + end + begin @xmldata = REXML::Document.new(@xmldata) rescue diff --git a/Master/texmf-dist/scripts/context/ruby/base/tex.rb b/Master/texmf-dist/scripts/context/ruby/base/tex.rb index 23db7f1e872..84025693b01 100644 --- a/Master/texmf-dist/scripts/context/ruby/base/tex.rb +++ b/Master/texmf-dist/scripts/context/ruby/base/tex.rb @@ -680,6 +680,11 @@ class TEX texformatpath = '' setvariable('error','no permissions to write') end + if not mpsformats then + # we want metafun to be in sync + setvariable('mpsformats',defaultmpsformats) + mpsformats = validmpsformat(getarrayvariable('mpsformats')) + end else texformatpath = '' end @@ -756,7 +761,8 @@ class TEX # utilities report('start of analysis') results = Array.new - ['texexec','texutil','ctxtools'].each do |program| + # ['texexec','texutil','ctxtools'].each do |program| + ['texexec'].each do |program| result = `texmfstart #{program} --help` result.sub!(/.*?(#{program}[^\n]+)\n.*/mi) do $1 end results.push("#{result}") @@ -1718,10 +1724,10 @@ end def fixbackendvars(backend) if backend then - report("fixing backend map path for #{backend}") if getvariable('verbose') ENV['backend'] = backend ; ENV['progname'] = backend unless validtexengine(backend) - ENV['TEXFONTMAPS'] = ['.',"\$TEXMF/fonts/map/{#{backend},pdftex,dvips,}//",'./fonts//'].join_path + ENV['TEXFONTMAPS'] = ['.',"\$TEXMF/fonts/{data,map}/{#{backend},pdftex,dvips,}//",'./fonts//'].join_path + report("fixing backend map path for #{backend}: #{ENV['TEXFONTMAPS']}") if getvariable('verbose') else report("unable to fix backend map path") if getvariable('verbose') end @@ -2047,7 +2053,10 @@ end Kpse.runscript('ctxtools',rawbase,'--purge') if getvariable('purge') Kpse.runscript('ctxtools',rawbase,'--purge --all') if getvariable('purgeall') -# till here + + # runcommand('mtxrun','--script','ctxtools',rawbase,'--purge') if getvariable('purge') + # runcommand('mtxrun','--script','ctxtools',rawbase,'--purge --all') if getvariable('purgeall') + when 'latex' then ok = runtex(rawname) diff --git a/Master/texmf-dist/scripts/context/ruby/newimgtopdf.rb b/Master/texmf-dist/scripts/context/ruby/newimgtopdf.rb deleted file mode 100644 index 563ae5b80a0..00000000000 --- a/Master/texmf-dist/scripts/context/ruby/newimgtopdf.rb +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env ruby - -# program : imgtopdf -# copyright : PRAGMA Advanced Document Engineering -# version : 2002-2005 -# author : Hans Hagen - -load(File.join(File.expand_path(File.dirname($0)),'imgtopdf.rb')) diff --git a/Master/texmf-dist/scripts/context/ruby/newpstopdf.rb b/Master/texmf-dist/scripts/context/ruby/newpstopdf.rb deleted file mode 100644 index a45b4cab896..00000000000 --- a/Master/texmf-dist/scripts/context/ruby/newpstopdf.rb +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env ruby - -# program : pstopdf -# copyright : PRAGMA Advanced Document Engineering -# version : 2002-2005 -# author : Hans Hagen - -load(File.join(File.expand_path(File.dirname($0)),'pstopdf.rb')) diff --git a/Master/texmf-dist/scripts/context/ruby/newtexexec.rb b/Master/texmf-dist/scripts/context/ruby/newtexexec.rb deleted file mode 100644 index 6b4db52dc51..00000000000 --- a/Master/texmf-dist/scripts/context/ruby/newtexexec.rb +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env ruby - -# program : texexec -# copyright : PRAGMA Advanced Document Engineering -# version : 1997-2006 -# author : Hans Hagen - -load(File.join(File.expand_path(File.dirname($0)),'texexec.rb')) diff --git a/Master/texmf-dist/scripts/context/ruby/newtexutil.rb b/Master/texmf-dist/scripts/context/ruby/newtexutil.rb deleted file mode 100644 index d6dd06a71ee..00000000000 --- a/Master/texmf-dist/scripts/context/ruby/newtexutil.rb +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env ruby - -# program : texexec -# copyright : PRAGMA Advanced Document Engineering -# version : 1997-2006 -# author : Hans Hagen - -load(File.join(File.expand_path(File.dirname($0)),'texutil.rb')) diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/context.cmd b/Master/texmf-dist/scripts/context/stubs/mswin/context.cmd deleted file mode 100755 index 11303c2714e..00000000000 --- a/Master/texmf-dist/scripts/context/stubs/mswin/context.cmd +++ /dev/null @@ -1,5 +0,0 @@ -@echo off
-setlocal
-set ownpath=%~dp0%
-texlua "%ownpath%mtxrun.lua" --script context %*
-endlocal
diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/context.exe b/Master/texmf-dist/scripts/context/stubs/mswin/context.exe Binary files differnew file mode 100644 index 00000000000..2d45f27494d --- /dev/null +++ b/Master/texmf-dist/scripts/context/stubs/mswin/context.exe diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/ctxtools.bat b/Master/texmf-dist/scripts/context/stubs/mswin/ctxtools.bat deleted file mode 100755 index f502b6750bd..00000000000 --- a/Master/texmf-dist/scripts/context/stubs/mswin/ctxtools.bat +++ /dev/null @@ -1,5 +0,0 @@ -@echo off
-setlocal
-set ownpath=%~dp0%
-texlua "%ownpath%mtxrun.lua" --usekpse --execute ctxtools.rb %*
-endlocal
diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/luatools.cmd b/Master/texmf-dist/scripts/context/stubs/mswin/luatools.cmd deleted file mode 100755 index 635ee0db3d5..00000000000 --- a/Master/texmf-dist/scripts/context/stubs/mswin/luatools.cmd +++ /dev/null @@ -1,5 +0,0 @@ -@echo off
-setlocal
-set ownpath=%~dp0%
-texlua "%ownpath%luatools.lua" %*
-endlocal
diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/luatools.exe b/Master/texmf-dist/scripts/context/stubs/mswin/luatools.exe Binary files differnew file mode 100644 index 00000000000..2d45f27494d --- /dev/null +++ b/Master/texmf-dist/scripts/context/stubs/mswin/luatools.exe diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/luatools.lua b/Master/texmf-dist/scripts/context/stubs/mswin/luatools.lua index 433d1b8dc0a..1d87322c108 100644 --- a/Master/texmf-dist/scripts/context/stubs/mswin/luatools.lua +++ b/Master/texmf-dist/scripts/context/stubs/mswin/luatools.lua @@ -39,13 +39,16 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-string'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -local sub, gsub, find, match, gmatch, format, char, byte, rep = string.sub, string.gsub, string.find, string.match, string.gmatch, string.format, string.char, string.byte, string.rep +local sub, gsub, find, match, gmatch, format, char, byte, rep, lower = string.sub, string.gsub, string.find, string.match, string.gmatch, string.format, string.char, string.byte, string.rep, string.lower +local lpegmatch = lpeg.match + +-- some functions may disappear as they are not used anywhere if not string.split then @@ -85,8 +88,16 @@ function string:unquote() return (gsub(self,"^([\"\'])(.*)%1$","%2")) end +--~ function string:unquote() +--~ if find(self,"^[\'\"]") then +--~ return sub(self,2,-2) +--~ else +--~ return self +--~ end +--~ end + function string:quote() -- we could use format("%q") - return '"' .. self:unquote() .. '"' + return format("%q",self) end function string:count(pattern) -- variant 3 @@ -106,12 +117,23 @@ function string:limit(n,sentinel) end end -function string:strip() - return (gsub(self,"^%s*(.-)%s*$", "%1")) +--~ function string:strip() -- the .- is quite efficient +--~ -- return match(self,"^%s*(.-)%s*$") or "" +--~ -- return match(self,'^%s*(.*%S)') or '' -- posted on lua list +--~ return find(s,'^%s*$') and '' or match(s,'^%s*(.*%S)') +--~ end + +do -- roberto's variant: + local space = lpeg.S(" \t\v\n") + local nospace = 1 - space + local stripper = space^0 * lpeg.C((space^0 * nospace^1)^0) + function string.strip(str) + return lpegmatch(stripper,str) or "" + end end function string:is_empty() - return not find(find,"%S") + return not find(self,"%S") end function string:enhance(pattern,action) @@ -145,14 +167,14 @@ if not string.characters then local function nextchar(str, index) index = index + 1 - return (index <= #str) and index or nil, str:sub(index,index) + return (index <= #str) and index or nil, sub(str,index,index) end function string:characters() return nextchar, self, 0 end local function nextbyte(str, index) index = index + 1 - return (index <= #str) and index or nil, byte(str:sub(index,index)) + return (index <= #str) and index or nil, byte(sub(str,index,index)) end function string:bytes() return nextbyte, self, 0 @@ -165,7 +187,7 @@ end function string:rpadd(n,chr) local m = n-#self if m > 0 then - return self .. self.rep(chr or " ",m) + return self .. rep(chr or " ",m) else return self end @@ -174,7 +196,7 @@ end function string:lpadd(n,chr) local m = n-#self if m > 0 then - return self.rep(chr or " ",m) .. self + return rep(chr or " ",m) .. self else return self end @@ -222,6 +244,17 @@ function string:pattesc() return (gsub(self,".",patterns_escapes)) end +local simple_escapes = { + ["-"] = "%-", + ["."] = "%.", + ["?"] = ".", + ["*"] = ".*", +} + +function string:simpleesc() + return (gsub(self,".",simple_escapes)) +end + function string:tohash() local t = { } for s in gmatch(self,"([^, ]+)") do -- lpeg @@ -233,10 +266,10 @@ end local pattern = lpeg.Ct(lpeg.C(1)^0) function string:totable() - return pattern:match(self) + return lpegmatch(pattern,self) end ---~ for _, str in ipairs { +--~ local t = { --~ "1234567123456712345671234567", --~ "a\tb\tc", --~ "aa\tbb\tcc", @@ -244,7 +277,10 @@ end --~ "aaaa\tbbbb\tcccc", --~ "aaaaa\tbbbbb\tccccc", --~ "aaaaaa\tbbbbbb\tcccccc", ---~ } do print(string.tabtospace(str)) end +--~ } +--~ for k,v do +--~ print(string.tabtospace(t[k])) +--~ end function string.tabtospace(str,tab) -- we don't handle embedded newlines @@ -252,7 +288,7 @@ function string.tabtospace(str,tab) local s = find(str,"\t") if s then if not tab then tab = 7 end -- only when found - local d = tab-(s-1)%tab + local d = tab-(s-1) % tab if d > 0 then str = gsub(str,"\t",rep(" ",d),1) else @@ -271,6 +307,25 @@ function string:compactlong() -- strips newlines and leading spaces return self end +function string:striplong() -- strips newlines and leading spaces + self = gsub(self,"^%s*","") + self = gsub(self,"[\n\r]+ *","\n") + return self +end + +function string:topattern(lowercase,strict) + if lowercase then + self = lower(self) + end + self = gsub(self,".",simple_escapes) + if self == "" then + self = ".*" + elseif strict then + self = "^" .. self .. "$" + end + return self +end + end -- of closure @@ -278,58 +333,64 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-lpeg'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -local P, S, Ct, C, Cs, Cc = lpeg.P, lpeg.S, lpeg.Ct, lpeg.C, lpeg.Cs, lpeg.Cc - ---~ l-lpeg.lua : - ---~ lpeg.digit = lpeg.R('09')^1 ---~ lpeg.sign = lpeg.S('+-')^1 ---~ lpeg.cardinal = lpeg.P(lpeg.sign^0 * lpeg.digit^1) ---~ lpeg.integer = lpeg.P(lpeg.sign^0 * lpeg.digit^1) ---~ lpeg.float = lpeg.P(lpeg.sign^0 * lpeg.digit^0 * lpeg.P('.') * lpeg.digit^1) ---~ lpeg.number = lpeg.float + lpeg.integer ---~ lpeg.oct = lpeg.P("0") * lpeg.R('07')^1 ---~ lpeg.hex = lpeg.P("0x") * (lpeg.R('09') + lpeg.R('AF'))^1 ---~ lpeg.uppercase = lpeg.P("AZ") ---~ lpeg.lowercase = lpeg.P("az") - ---~ lpeg.eol = lpeg.S('\r\n\f')^1 -- includes formfeed ---~ lpeg.space = lpeg.S(' ')^1 ---~ lpeg.nonspace = lpeg.P(1-lpeg.space)^1 ---~ lpeg.whitespace = lpeg.S(' \r\n\f\t')^1 ---~ lpeg.nonwhitespace = lpeg.P(1-lpeg.whitespace)^1 - -local hash = { } +local lpeg = require("lpeg") + +lpeg.patterns = lpeg.patterns or { } -- so that we can share +local patterns = lpeg.patterns + +local P, R, S, Ct, C, Cs, Cc, V = lpeg.P, lpeg.R, lpeg.S, lpeg.Ct, lpeg.C, lpeg.Cs, lpeg.Cc, lpeg.V +local match = lpeg.match + +local digit, sign = R('09'), S('+-') +local cr, lf, crlf = P("\r"), P("\n"), P("\r\n") +local utf8byte = R("\128\191") + +patterns.utf8byte = utf8byte +patterns.utf8one = R("\000\127") +patterns.utf8two = R("\194\223") * utf8byte +patterns.utf8three = R("\224\239") * utf8byte * utf8byte +patterns.utf8four = R("\240\244") * utf8byte * utf8byte * utf8byte + +patterns.digit = digit +patterns.sign = sign +patterns.cardinal = sign^0 * digit^1 +patterns.integer = sign^0 * digit^1 +patterns.float = sign^0 * digit^0 * P('.') * digit^1 +patterns.number = patterns.float + patterns.integer +patterns.oct = P("0") * R("07")^1 +patterns.octal = patterns.oct +patterns.HEX = P("0x") * R("09","AF")^1 +patterns.hex = P("0x") * R("09","af")^1 +patterns.hexadecimal = P("0x") * R("09","AF","af")^1 +patterns.lowercase = R("az") +patterns.uppercase = R("AZ") +patterns.letter = patterns.lowercase + patterns.uppercase +patterns.space = S(" ") +patterns.eol = S("\n\r") +patterns.spacer = S(" \t\f\v") -- + string.char(0xc2, 0xa0) if we want utf (cf mail roberto) +patterns.newline = crlf + cr + lf +patterns.nonspace = 1 - patterns.space +patterns.nonspacer = 1 - patterns.spacer +patterns.whitespace = patterns.eol + patterns.spacer +patterns.nonwhitespace = 1 - patterns.whitespace +patterns.utf8 = patterns.utf8one + patterns.utf8two + patterns.utf8three + patterns.utf8four +patterns.utfbom = P('\000\000\254\255') + P('\255\254\000\000') + P('\255\254') + P('\254\255') + P('\239\187\191') function lpeg.anywhere(pattern) --slightly adapted from website - return P { P(pattern) + 1 * lpeg.V(1) } -end - -function lpeg.startswith(pattern) --slightly adapted - return P(pattern) + return P { P(pattern) + 1 * V(1) } -- why so complex? end function lpeg.splitter(pattern, action) return (((1-P(pattern))^1)/action+1)^0 end --- variant: - ---~ local parser = lpeg.Ct(lpeg.splitat(newline)) - -local crlf = P("\r\n") -local cr = P("\r") -local lf = P("\n") -local space = S(" \t\f\v") -- + string.char(0xc2, 0xa0) if we want utf (cf mail roberto) -local newline = crlf + cr + lf -local spacing = space^0 * newline - +local spacing = patterns.spacer^0 * patterns.newline -- sort of strip local empty = spacing * Cc("") local nonempty = Cs((1-spacing)^1) * spacing^-1 local content = (empty + nonempty)^1 @@ -337,15 +398,15 @@ local content = (empty + nonempty)^1 local capture = Ct(content^0) function string:splitlines() - return capture:match(self) + return match(capture,self) end -lpeg.linebyline = content -- better make a sublibrary +patterns.textline = content ---~ local p = lpeg.splitat("->",false) print(p:match("oeps->what->more")) -- oeps what more ---~ local p = lpeg.splitat("->",true) print(p:match("oeps->what->more")) -- oeps what->more ---~ local p = lpeg.splitat("->",false) print(p:match("oeps")) -- oeps ---~ local p = lpeg.splitat("->",true) print(p:match("oeps")) -- oeps +--~ local p = lpeg.splitat("->",false) print(match(p,"oeps->what->more")) -- oeps what more +--~ local p = lpeg.splitat("->",true) print(match(p,"oeps->what->more")) -- oeps what->more +--~ local p = lpeg.splitat("->",false) print(match(p,"oeps")) -- oeps +--~ local p = lpeg.splitat("->",true) print(match(p,"oeps")) -- oeps local splitters_s, splitters_m = { }, { } @@ -355,7 +416,7 @@ local function splitat(separator,single) separator = P(separator) if single then local other, any = C((1 - separator)^0), P(1) - splitter = other * (separator * C(any^0) + "") + splitter = other * (separator * C(any^0) + "") -- ? splitters_s[separator] = splitter else local other = C((1 - separator)^0) @@ -370,15 +431,72 @@ lpeg.splitat = splitat local cache = { } +function lpeg.split(separator,str) + local c = cache[separator] + if not c then + c = Ct(splitat(separator)) + cache[separator] = c + end + return match(c,str) +end + function string:split(separator) local c = cache[separator] if not c then c = Ct(splitat(separator)) cache[separator] = c end - return c:match(self) + return match(c,self) +end + +lpeg.splitters = cache + +local cache = { } + +function lpeg.checkedsplit(separator,str) + local c = cache[separator] + if not c then + separator = P(separator) + local other = C((1 - separator)^0) + c = Ct(separator^0 * other * (separator^1 * other)^0) + cache[separator] = c + end + return match(c,str) +end + +function string:checkedsplit(separator) + local c = cache[separator] + if not c then + separator = P(separator) + local other = C((1 - separator)^0) + c = Ct(separator^0 * other * (separator^1 * other)^0) + cache[separator] = c + end + return match(c,self) end +--~ function lpeg.append(list,pp) +--~ local p = pp +--~ for l=1,#list do +--~ if p then +--~ p = p + P(list[l]) +--~ else +--~ p = P(list[l]) +--~ end +--~ end +--~ return p +--~ end + +--~ from roberto's site: + +local f1 = string.byte + +local function f2(s) local c1, c2 = f1(s,1,2) return c1 * 64 + c2 - 12416 end +local function f3(s) local c1, c2, c3 = f1(s,1,3) return (c1 * 64 + c2) * 64 + c3 - 925824 end +local function f4(s) local c1, c2, c3, c4 = f1(s,1,4) return ((c1 * 64 + c2) * 64 + c3) * 64 + c4 - 63447168 end + +patterns.utf8byte = patterns.utf8one/f1 + patterns.utf8two/f2 + patterns.utf8three/f3 + patterns.utf8four/f4 + end -- of closure @@ -386,7 +504,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-table'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -395,9 +513,10 @@ if not modules then modules = { } end modules ['l-table'] = { table.join = table.concat local concat, sort, insert, remove = table.concat, table.sort, table.insert, table.remove -local format, find, gsub, lower, dump = string.format, string.find, string.gsub, string.lower, string.dump +local format, find, gsub, lower, dump, match = string.format, string.find, string.gsub, string.lower, string.dump, string.match local getmetatable, setmetatable = getmetatable, setmetatable -local type, next, tostring, ipairs = type, next, tostring, ipairs +local type, next, tostring, tonumber, ipairs = type, next, tostring, tonumber, ipairs +local unpack = unpack or table.unpack function table.strip(tab) local lst = { } @@ -412,6 +531,14 @@ function table.strip(tab) return lst end +function table.keys(t) + local k = { } + for key, _ in next, t do + k[#k+1] = key + end + return k +end + local function compare(a,b) return (tostring(a) < tostring(b)) end @@ -455,7 +582,7 @@ end table.sortedkeys = sortedkeys table.sortedhashkeys = sortedhashkeys -function table.sortedpairs(t) +function table.sortedhash(t) local s = sortedhashkeys(t) -- maybe just sortedkeys local n = 0 local function kv(s) @@ -466,6 +593,8 @@ function table.sortedpairs(t) return kv, s end +table.sortedpairs = table.sortedhash + function table.append(t, list) for _,v in next, list do insert(t,v) @@ -588,18 +717,18 @@ end -- slower than #t on indexed tables (#t only returns the size of the numerically indexed slice) -function table.is_empty(t) +function table.is_empty(t) -- obolete, use inline code instead return not t or not next(t) end -function table.one_entry(t) +function table.one_entry(t) -- obolete, use inline code instead local n = next(t) return n and not next(t,n) end -function table.starts_at(t) - return ipairs(t,1)(t,0) -end +--~ function table.starts_at(t) -- obsolete, not nice +--~ return ipairs(t,1)(t,0) +--~ end function table.tohash(t,value) local h = { } @@ -677,6 +806,8 @@ end -- -- local propername = lpeg.P(lpeg.R("AZ","az","__") * lpeg.R("09","AZ","az", "__")^0 * lpeg.P(-1) ) +-- problem: there no good number_to_string converter with the best resolution + local function do_serialize(root,name,depth,level,indexed) if level > 0 then depth = depth .. " " @@ -699,8 +830,9 @@ local function do_serialize(root,name,depth,level,indexed) handle(format("%s{",depth)) end end + -- we could check for k (index) being number (cardinal) if root and next(root) then - local first, last = nil, 0 -- #root cannot be trusted here + local first, last = nil, 0 -- #root cannot be trusted here (will be ok in 5.2 when ipairs is gone) if compact then -- NOT: for k=1,#root do (we need to quit at nil) for k,v in ipairs(root) do -- can we use next? @@ -721,10 +853,10 @@ local function do_serialize(root,name,depth,level,indexed) if hexify then handle(format("%s 0x%04X,",depth,v)) else - handle(format("%s %s,",depth,v)) + handle(format("%s %s,",depth,v)) -- %.99g end elseif t == "string" then - if reduce and (find(v,"^[%-%+]?[%d]-%.?[%d+]$") == 1) then + if reduce and tonumber(v) then handle(format("%s %s,",depth,v)) else handle(format("%s %q,",depth,v)) @@ -761,29 +893,29 @@ local function do_serialize(root,name,depth,level,indexed) --~ if hexify then --~ handle(format("%s %s=0x%04X,",depth,key(k),v)) --~ else - --~ handle(format("%s %s=%s,",depth,key(k),v)) + --~ handle(format("%s %s=%s,",depth,key(k),v)) -- %.99g --~ end if type(k) == "number" then -- or find(k,"^%d+$") then if hexify then handle(format("%s [0x%04X]=0x%04X,",depth,k,v)) else - handle(format("%s [%s]=%s,",depth,k,v)) + handle(format("%s [%s]=%s,",depth,k,v)) -- %.99g end elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then if hexify then handle(format("%s %s=0x%04X,",depth,k,v)) else - handle(format("%s %s=%s,",depth,k,v)) + handle(format("%s %s=%s,",depth,k,v)) -- %.99g end else if hexify then handle(format("%s [%q]=0x%04X,",depth,k,v)) else - handle(format("%s [%q]=%s,",depth,k,v)) + handle(format("%s [%q]=%s,",depth,k,v)) -- %.99g end end elseif t == "string" then - if reduce and (find(v,"^[%-%+]?[%d]-%.?[%d+]$") == 1) then + if reduce and tonumber(v) then --~ handle(format("%s %s=%s,",depth,key(k),v)) if type(k) == "number" then -- or find(k,"^%d+$") then if hexify then @@ -992,7 +1124,7 @@ function table.tofile(filename,root,name,reduce,noquotes,hexify) end end -local function flatten(t,f,complete) +local function flatten(t,f,complete) -- is this used? meybe a variant with next, ... for i=1,#t do local v = t[i] if type(v) == "table" then @@ -1021,6 +1153,24 @@ end table.flatten_one_level = table.unnest +-- a better one: + +local function flattened(t,f) + if not f then + f = { } + end + for k, v in next, t do + if type(v) == "table" then + flattened(v,f) + else + f[k] = v + end + end + return f +end + +table.flattened = flattened + -- the next three may disappear function table.remove_value(t,value) -- todo: n @@ -1156,7 +1306,7 @@ function table.clone(t,p) -- t is optional or nil or table elseif not t then t = { } end - setmetatable(t, { __index = function(_,key) return p[key] end }) + setmetatable(t, { __index = function(_,key) return p[key] end }) -- why not __index = p ? return t end @@ -1184,21 +1334,35 @@ function table.reverse(t) return tt end ---~ function table.keys(t) ---~ local k = { } ---~ for k,_ in next, t do ---~ k[#k+1] = k ---~ end ---~ return k ---~ end +function table.insert_before_value(t,value,extra) + for i=1,#t do + if t[i] == extra then + remove(t,i) + end + end + for i=1,#t do + if t[i] == value then + insert(t,i,extra) + return + end + end + insert(t,1,extra) +end ---~ function table.keys_as_string(t) ---~ local k = { } ---~ for k,_ in next, t do ---~ k[#k+1] = k ---~ end ---~ return concat(k,"") ---~ end +function table.insert_after_value(t,value,extra) + for i=1,#t do + if t[i] == extra then + remove(t,i) + end + end + for i=1,#t do + if t[i] == value then + insert(t,i+1,extra) + return + end + end + insert(t,#t+1,extra) +end end -- of closure @@ -1207,13 +1371,13 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-io'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -local byte = string.byte +local byte, find, gsub = string.byte, string.find, string.gsub if string.find(os.getenv("PATH"),";") then io.fileseparator, io.pathseparator = "\\", ";" @@ -1242,7 +1406,7 @@ function io.savedata(filename,data,joiner) elseif type(data) == "function" then data(f) else - f:write(data) + f:write(data or "") end f:close() return true @@ -1371,20 +1535,21 @@ function io.ask(question,default,options) end io.write(string.format(" ")) local answer = io.read() - answer = answer:gsub("^%s*(.*)%s*$","%1") + answer = gsub(answer,"^%s*(.*)%s*$","%1") if answer == "" and default then return default elseif not options then return answer else - for _,v in pairs(options) do - if v == answer then + for k=1,#options do + if options[k] == answer then return answer end end local pattern = "^" .. answer - for _,v in pairs(options) do - if v:find(pattern) then + for k=1,#options do + local v = options[k] + if find(v,pattern) then return v end end @@ -1399,20 +1564,22 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-number'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -local format = string.format +local tostring = tostring +local format, floor, insert, match = string.format, math.floor, table.insert, string.match +local lpegmatch = lpeg.match number = number or { } -- a,b,c,d,e,f = number.toset(100101) function number.toset(n) - return (tostring(n)):match("(.?)(.?)(.?)(.?)(.?)(.?)(.?)(.?)") + return match(tostring(n),"(.?)(.?)(.?)(.?)(.?)(.?)(.?)(.?)") end function number.toevenhex(n) @@ -1438,10 +1605,21 @@ end local one = lpeg.C(1-lpeg.S(''))^1 function number.toset(n) - return one:match(tostring(n)) + return lpegmatch(one,tostring(n)) end - +function number.bits(n,zero) + local t, i = { }, (zero and 0) or 1 + while n > 0 do + local m = n % 2 + if m > 0 then + insert(t,1,i) + end + n = floor(n/2) + i = i + 1 + end + return t +end end -- of closure @@ -1450,7 +1628,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-set'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -1540,46 +1718,63 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-os'] = { version = 1.001, - comment = "companion to luat-lub.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -local find = string.find +-- maybe build io.flush in os.execute + +local find, format, gsub = string.find, string.format, string.gsub +local random, ceil = math.random, math.ceil + +local execute, spawn, exec, ioflush = os.execute, os.spawn or os.execute, os.exec or os.execute, io.flush + +function os.execute(...) ioflush() return execute(...) end +function os.spawn (...) ioflush() return spawn (...) end +function os.exec (...) ioflush() return exec (...) end function os.resultof(command) - return io.popen(command,"r"):read("*all") + ioflush() -- else messed up logging + local handle = io.popen(command,"r") + if not handle then + -- print("unknown command '".. command .. "' in os.resultof") + return "" + else + return handle:read("*all") or "" + end end -if not os.exec then os.exec = os.execute end -if not os.spawn then os.spawn = os.execute end - ---~ os.type : windows | unix (new, we already guessed os.platform) ---~ os.name : windows | msdos | linux | macosx | solaris | .. | generic (new) +--~ os.type : windows | unix (new, we already guessed os.platform) +--~ os.name : windows | msdos | linux | macosx | solaris | .. | generic (new) +--~ os.platform : extended os.name with architecture if not io.fileseparator then if find(os.getenv("PATH"),";") then - io.fileseparator, io.pathseparator, os.platform = "\\", ";", os.type or "windows" + io.fileseparator, io.pathseparator, os.type = "\\", ";", os.type or "mswin" else - io.fileseparator, io.pathseparator, os.platform = "/" , ":", os.type or "unix" + io.fileseparator, io.pathseparator, os.type = "/" , ":", os.type or "unix" end end -os.platform = os.platform or os.type or (io.pathseparator == ";" and "windows") or "unix" +os.type = os.type or (io.pathseparator == ";" and "windows") or "unix" +os.name = os.name or (os.type == "windows" and "mswin" ) or "linux" + +if os.type == "windows" then + os.libsuffix, os.binsuffix = 'dll', 'exe' +else + os.libsuffix, os.binsuffix = 'so', '' +end function os.launch(str) - if os.platform == "windows" then + if os.type == "windows" then os.execute("start " .. str) -- os.spawn ? else os.execute(str .. " &") -- os.spawn ? end end -if not os.setenv then - function os.setenv() return false end -end - if not os.times then -- utime = user time -- stime = system time @@ -1609,64 +1804,218 @@ end --~ print(os.date("%H:%M:%S",os.gettimeofday())) --~ print(os.date("%H:%M:%S",os.time())) -os.arch = os.arch or function() - local a = os.resultof("uname -m") or "linux" - os.arch = function() - return a +-- no need for function anymore as we have more clever code and helpers now +-- this metatable trickery might as well disappear + +os.resolvers = os.resolvers or { } + +local resolvers = os.resolvers + +local osmt = getmetatable(os) or { __index = function(t,k) t[k] = "unset" return "unset" end } -- maybe nil +local osix = osmt.__index + +osmt.__index = function(t,k) + return (resolvers[k] or osix)(t,k) +end + +setmetatable(os,osmt) + +if not os.setenv then + + -- we still store them but they won't be seen in + -- child processes although we might pass them some day + -- using command concatination + + local env, getenv = { }, os.getenv + + function os.setenv(k,v) + env[k] = v end - return a + + function os.getenv(k) + return env[k] or getenv(k) + end + end -local platform +-- we can use HOSTTYPE on some platforms -function os.currentplatform(name,default) - if not platform then - local name = os.name or os.platform or name -- os.name is built in, os.platform is mine - if not name then - platform = default or "linux" - elseif name == "windows" or name == "mswin" or name == "win32" or name == "msdos" then - if os.getenv("PROCESSOR_ARCHITECTURE") == "AMD64" then - platform = "mswin-64" - else - platform = "mswin" - end +local name, platform = os.name or "linux", os.getenv("MTX_PLATFORM") or "" + +local function guess() + local architecture = os.resultof("uname -m") or "" + if architecture ~= "" then + return architecture + end + architecture = os.getenv("HOSTTYPE") or "" + if architecture ~= "" then + return architecture + end + return os.resultof("echo $HOSTTYPE") or "" +end + +if platform ~= "" then + + os.platform = platform + +elseif os.type == "windows" then + + -- we could set the variable directly, no function needed here + + function os.resolvers.platform(t,k) + local platform, architecture = "", os.getenv("PROCESSOR_ARCHITECTURE") or "" + if find(architecture,"AMD64") then + platform = "mswin-64" else - local architecture = os.arch() - if name == "linux" then - if find(architecture,"x86_64") then - platform = "linux-64" - elseif find(architecture,"ppc") then - platform = "linux-ppc" - else - platform = "linux" - end - elseif name == "macosx" then - if find(architecture,"i386") then - platform = "osx-intel" - else - platform = "osx-ppc" - end - elseif name == "sunos" then - if find(architecture,"sparc") then - platform = "solaris-sparc" - else -- if architecture == 'i86pc' - platform = "solaris-intel" - end - elseif name == "freebsd" then - if find(architecture,"amd64") then - platform = "freebsd-amd64" - else - platform = "freebsd" - end - else - platform = default or name - end + platform = "mswin" end - function os.currentplatform() - return platform + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +elseif name == "linux" then + + function os.resolvers.platform(t,k) + -- we sometims have HOSTTYPE set so let's check that first + local platform, architecture = "", os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" + if find(architecture,"x86_64") then + platform = "linux-64" + elseif find(architecture,"ppc") then + platform = "linux-ppc" + else + platform = "linux" + end + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +elseif name == "macosx" then + + --[[ + Identifying the architecture of OSX is quite a mess and this + is the best we can come up with. For some reason $HOSTTYPE is + a kind of pseudo environment variable, not known to the current + environment. And yes, uname cannot be trusted either, so there + is a change that you end up with a 32 bit run on a 64 bit system. + Also, some proper 64 bit intel macs are too cheap (low-end) and + therefore not permitted to run the 64 bit kernel. + ]]-- + + function os.resolvers.platform(t,k) + -- local platform, architecture = "", os.getenv("HOSTTYPE") or "" + -- if architecture == "" then + -- architecture = os.resultof("echo $HOSTTYPE") or "" + -- end + local platform, architecture = "", os.resultof("echo $HOSTTYPE") or "" + if architecture == "" then + -- print("\nI have no clue what kind of OSX you're running so let's assume an 32 bit intel.\n") + platform = "osx-intel" + elseif find(architecture,"i386") then + platform = "osx-intel" + elseif find(architecture,"x86_64") then + platform = "osx-64" + else + platform = "osx-ppc" + end + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +elseif name == "sunos" then + + function os.resolvers.platform(t,k) + local platform, architecture = "", os.resultof("uname -m") or "" + if find(architecture,"sparc") then + platform = "solaris-sparc" + else -- if architecture == 'i86pc' + platform = "solaris-intel" + end + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +elseif name == "freebsd" then + + function os.resolvers.platform(t,k) + local platform, architecture = "", os.resultof("uname -m") or "" + if find(architecture,"amd64") then + platform = "freebsd-amd64" + else + platform = "freebsd" + end + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +elseif name == "kfreebsd" then + + function os.resolvers.platform(t,k) + -- we sometims have HOSTTYPE set so let's check that first + local platform, architecture = "", os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" + if find(architecture,"x86_64") then + platform = "kfreebsd-64" + else + platform = "kfreebsd-i386" + end + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +else + + -- platform = "linux" + -- os.setenv("MTX_PLATFORM",platform) + -- os.platform = platform + + function os.resolvers.platform(t,k) + local platform = "linux" + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +end + +-- beware, we set the randomseed + +-- from wikipedia: Version 4 UUIDs use a scheme relying only on random numbers. This algorithm sets the +-- version number as well as two reserved bits. All other bits are set using a random or pseudorandom +-- data source. Version 4 UUIDs have the form xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx with hexadecimal +-- digits x and hexadecimal digits 8, 9, A, or B for y. e.g. f47ac10b-58cc-4372-a567-0e02b2c3d479. +-- +-- as we don't call this function too often there is not so much risk on repetition + +local t = { 8, 9, "a", "b" } + +function os.uuid() + return format("%04x%04x-4%03x-%s%03x-%04x-%04x%04x%04x", + random(0xFFFF),random(0xFFFF), + random(0x0FFF), + t[ceil(random(4))] or 8,random(0x0FFF), + random(0xFFFF), + random(0xFFFF),random(0xFFFF),random(0xFFFF) + ) +end + +local d + +function os.timezone(delta) + d = d or tonumber(tonumber(os.date("%H")-os.date("!%H"))) + if delta then + if d > 0 then + return format("+%02i:00",d) + else + return format("-%02i:00",-d) end + else + return 1 end - return platform end @@ -1676,7 +2025,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-file'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -1687,14 +2036,17 @@ if not modules then modules = { } end modules ['l-file'] = { file = file or { } local concat = table.concat -local find, gmatch, match, gsub = string.find, string.gmatch, string.match, string.gsub +local find, gmatch, match, gsub, sub, char = string.find, string.gmatch, string.match, string.gsub, string.sub, string.char +local lpegmatch = lpeg.match function file.removesuffix(filename) return (gsub(filename,"%.[%a%d]+$","")) end function file.addsuffix(filename, suffix) - if not find(filename,"%.[%a%d]+$") then + if not suffix or suffix == "" then + return filename + elseif not find(filename,"%.[%a%d]+$") then return filename .. "." .. suffix else return filename @@ -1717,20 +2069,39 @@ function file.nameonly(name) return (gsub(match(name,"^.+[/\\](.-)$") or name,"%..*$","")) end -function file.extname(name) - return match(name,"^.+%.([^/\\]-)$") or "" +function file.extname(name,default) + return match(name,"^.+%.([^/\\]-)$") or default or "" end file.suffix = file.extname ---~ print(file.join("x/","/y")) ---~ print(file.join("http://","/y")) ---~ print(file.join("http://a","/y")) ---~ print(file.join("http:///a","/y")) ---~ print(file.join("//nas-1","/y")) +--~ function file.join(...) +--~ local pth = concat({...},"/") +--~ pth = gsub(pth,"\\","/") +--~ local a, b = match(pth,"^(.*://)(.*)$") +--~ if a and b then +--~ return a .. gsub(b,"//+","/") +--~ end +--~ a, b = match(pth,"^(//)(.*)$") +--~ if a and b then +--~ return a .. gsub(b,"//+","/") +--~ end +--~ return (gsub(pth,"//+","/")) +--~ end + +local trick_1 = char(1) +local trick_2 = "^" .. trick_1 .. "/+" function file.join(...) - local pth = concat({...},"/") + local lst = { ... } + local a, b = lst[1], lst[2] + if a == "" then + lst[1] = trick_1 + elseif b and find(a,"^/+$") and find(b,"^/") then + lst[1] = "" + lst[2] = gsub(b,"^/+","") + end + local pth = concat(lst,"/") pth = gsub(pth,"\\","/") local a, b = match(pth,"^(.*://)(.*)$") if a and b then @@ -1740,17 +2111,28 @@ function file.join(...) if a and b then return a .. gsub(b,"//+","/") end + pth = gsub(pth,trick_2,"") return (gsub(pth,"//+","/")) end +--~ print(file.join("//","/y")) +--~ print(file.join("/","/y")) +--~ print(file.join("","/y")) +--~ print(file.join("/x/","/y")) +--~ print(file.join("x/","/y")) +--~ print(file.join("http://","/y")) +--~ print(file.join("http://a","/y")) +--~ print(file.join("http:///a","/y")) +--~ print(file.join("//nas-1","/y")) + function file.iswritable(name) local a = lfs.attributes(name) or lfs.attributes(file.dirname(name,".")) - return a and a.permissions:sub(2,2) == "w" + return a and sub(a.permissions,2,2) == "w" end function file.isreadable(name) local a = lfs.attributes(name) - return a and a.permissions:sub(1,1) == "r" + return a and sub(a.permissions,1,1) == "r" end file.is_readable = file.isreadable @@ -1758,36 +2140,50 @@ file.is_writable = file.iswritable -- todo: lpeg -function file.split_path(str) - local t = { } - str = gsub(str,"\\", "/") - str = gsub(str,"(%a):([;/])", "%1\001%2") - for name in gmatch(str,"([^;:]+)") do - if name ~= "" then - t[#t+1] = gsub(name,"\001",":") - end - end - return t +--~ function file.split_path(str) +--~ local t = { } +--~ str = gsub(str,"\\", "/") +--~ str = gsub(str,"(%a):([;/])", "%1\001%2") +--~ for name in gmatch(str,"([^;:]+)") do +--~ if name ~= "" then +--~ t[#t+1] = gsub(name,"\001",":") +--~ end +--~ end +--~ return t +--~ end + +local checkedsplit = string.checkedsplit + +function file.split_path(str,separator) + str = gsub(str,"\\","/") + return checkedsplit(str,separator or io.pathseparator) end function file.join_path(tab) return concat(tab,io.pathseparator) -- can have trailing // end +-- we can hash them weakly + function file.collapse_path(str) - str = gsub(str,"/%./","/") - local n, m = 1, 1 - while n > 0 or m > 0 do - str, n = gsub(str,"[^/%.]+/%.%.$","") - str, m = gsub(str,"[^/%.]+/%.%./","") - end - str = gsub(str,"([^/])/$","%1") - str = gsub(str,"^%./","") - str = gsub(str,"/%.$","") + str = gsub(str,"\\","/") + if find(str,"/") then + str = gsub(str,"^%./",(gsub(lfs.currentdir(),"\\","/")) .. "/") -- ./xx in qualified + str = gsub(str,"/%./","/") + local n, m = 1, 1 + while n > 0 or m > 0 do + str, n = gsub(str,"[^/%.]+/%.%.$","") + str, m = gsub(str,"[^/%.]+/%.%./","") + end + str = gsub(str,"([^/])/$","%1") + -- str = gsub(str,"^%./","") -- ./xx in qualified + str = gsub(str,"/%.$","") + end if str == "" then str = "." end return str end +--~ print(file.collapse_path("/a")) --~ print(file.collapse_path("a/./b/..")) --~ print(file.collapse_path("a/aa/../b/bb")) --~ print(file.collapse_path("a/../..")) @@ -1817,27 +2213,27 @@ end --~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * lpeg.C(noperiod^1) * -1 --~ function file.extname(name) ---~ return pattern:match(name) or "" +--~ return lpegmatch(pattern,name) or "" --~ end --~ local pattern = lpeg.Cs(((period * noperiod^1 * -1)/"" + 1)^1) --~ function file.removesuffix(name) ---~ return pattern:match(name) +--~ return lpegmatch(pattern,name) --~ end --~ local pattern = (noslashes^0 * slashes)^1 * lpeg.C(noslashes^1) * -1 --~ function file.basename(name) ---~ return pattern:match(name) or name +--~ return lpegmatch(pattern,name) or name --~ end --~ local pattern = (noslashes^0 * slashes)^1 * lpeg.Cp() * noslashes^1 * -1 --~ function file.dirname(name) ---~ local p = pattern:match(name) +--~ local p = lpegmatch(pattern,name) --~ if p then ---~ return name:sub(1,p-2) +--~ return sub(name,1,p-2) --~ else --~ return "" --~ end @@ -1846,7 +2242,7 @@ end --~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * lpeg.Cp() * noperiod^1 * -1 --~ function file.addsuffix(name, suffix) ---~ local p = pattern:match(name) +--~ local p = lpegmatch(pattern,name) --~ if p then --~ return name --~ else @@ -1857,9 +2253,9 @@ end --~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * lpeg.Cp() * noperiod^1 * -1 --~ function file.replacesuffix(name,suffix) ---~ local p = pattern:match(name) +--~ local p = lpegmatch(pattern,name) --~ if p then ---~ return name:sub(1,p-2) .. "." .. suffix +--~ return sub(name,1,p-2) .. "." .. suffix --~ else --~ return name .. "." .. suffix --~ end @@ -1868,11 +2264,11 @@ end --~ local pattern = (noslashes^0 * slashes)^0 * lpeg.Cp() * ((noperiod^1 * period)^1 * lpeg.Cp() + lpeg.P(true)) * noperiod^1 * -1 --~ function file.nameonly(name) ---~ local a, b = pattern:match(name) +--~ local a, b = lpegmatch(pattern,name) --~ if b then ---~ return name:sub(a,b-2) +--~ return sub(name,a,b-2) --~ elseif a then ---~ return name:sub(a) +--~ return sub(name,a) --~ else --~ return name --~ end @@ -1906,11 +2302,11 @@ local rootbased = lpeg.P("/") + letter*lpeg.P(":") -- ./name ../name /name c: :// name/name function file.is_qualified_path(filename) - return qualified:match(filename) + return lpegmatch(qualified,filename) ~= nil end function file.is_rootbased_path(filename) - return rootbased:match(filename) + return lpegmatch(rootbased,filename) ~= nil end local slash = lpeg.S("\\/") @@ -1923,16 +2319,25 @@ local base = lpeg.C((1-suffix)^0) local pattern = (drive + lpeg.Cc("")) * (path + lpeg.Cc("")) * (base + lpeg.Cc("")) * (suffix + lpeg.Cc("")) function file.splitname(str) -- returns drive, path, base, suffix - return pattern:match(str) + return lpegmatch(pattern,str) end --- function test(t) for k, v in pairs(t) do print(v, "=>", file.splitname(v)) end end +-- function test(t) for k, v in next, t do print(v, "=>", file.splitname(v)) end end -- -- test { "c:", "c:/aa", "c:/aa/bb", "c:/aa/bb/cc", "c:/aa/bb/cc.dd", "c:/aa/bb/cc.dd.ee" } -- test { "c:", "c:aa", "c:aa/bb", "c:aa/bb/cc", "c:aa/bb/cc.dd", "c:aa/bb/cc.dd.ee" } -- test { "/aa", "/aa/bb", "/aa/bb/cc", "/aa/bb/cc.dd", "/aa/bb/cc.dd.ee" } -- test { "aa", "aa/bb", "aa/bb/cc", "aa/bb/cc.dd", "aa/bb/cc.dd.ee" } +--~ -- todo: +--~ +--~ if os.type == "windows" then +--~ local currentdir = lfs.currentdir +--~ function lfs.currentdir() +--~ return (gsub(currentdir(),"\\","/")) +--~ end +--~ end + end -- of closure @@ -1997,7 +2402,7 @@ end function file.loadchecksum(name) if md5 then local data = io.loaddata(name .. ".md5") - return data and data:gsub("%s","") + return data and (gsub(data,"%s","")) end return nil end @@ -2018,14 +2423,15 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-url'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -local char, gmatch = string.char, string.gmatch +local char, gmatch, gsub = string.char, string.gmatch, string.gsub local tonumber, type = tonumber, type +local lpegmatch = lpeg.match -- from the spec (on the web): -- @@ -2049,7 +2455,9 @@ local hexdigit = lpeg.R("09","AF","af") local plus = lpeg.P("+") local escaped = (plus / " ") + (percent * lpeg.C(hexdigit * hexdigit) / tochar) -local scheme = lpeg.Cs((escaped+(1-colon-slash-qmark-hash))^0) * colon + lpeg.Cc("") +-- we assume schemes with more than 1 character (in order to avoid problems with windows disks) + +local scheme = lpeg.Cs((escaped+(1-colon-slash-qmark-hash))^2) * colon + lpeg.Cc("") local authority = slash * slash * lpeg.Cs((escaped+(1- slash-qmark-hash))^0) + lpeg.Cc("") local path = slash * lpeg.Cs((escaped+(1- qmark-hash))^0) + lpeg.Cc("") local query = qmark * lpeg.Cs((escaped+(1- hash))^0) + lpeg.Cc("") @@ -2057,25 +2465,51 @@ local fragment = hash * lpeg.Cs((escaped+(1- endofstring))^0 local parser = lpeg.Ct(scheme * authority * path * query * fragment) +-- todo: reconsider Ct as we can as well have five return values (saves a table) +-- so we can have two parsers, one with and one without + function url.split(str) - return (type(str) == "string" and parser:match(str)) or str + return (type(str) == "string" and lpegmatch(parser,str)) or str end +-- todo: cache them + function url.hashed(str) local s = url.split(str) + local somescheme = s[1] ~= "" return { - scheme = (s[1] ~= "" and s[1]) or "file", + scheme = (somescheme and s[1]) or "file", authority = s[2], - path = s[3], - query = s[4], - fragment = s[5], - original = str + path = s[3], + query = s[4], + fragment = s[5], + original = str, + noscheme = not somescheme, } end +function url.hasscheme(str) + return url.split(str)[1] ~= "" +end + +function url.addscheme(str,scheme) + return (url.hasscheme(str) and str) or ((scheme or "file:///") .. str) +end + +function url.construct(hash) + local fullurl = hash.sheme .. "://".. hash.authority .. hash.path + if hash.query then + fullurl = fullurl .. "?".. hash.query + end + if hash.fragment then + fullurl = fullurl .. "?".. hash.fragment + end + return fullurl +end + function url.filename(filename) local t = url.hashed(filename) - return (t.scheme == "file" and t.path:gsub("^/([a-zA-Z])([:|])/)","%1:")) or filename + return (t.scheme == "file" and (gsub(t.path,"^/([a-zA-Z])([:|])/)","%1:"))) or filename end function url.query(str) @@ -2129,17 +2563,26 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-dir'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } +-- dir.expand_name will be merged with cleanpath and collapsepath + local type = type -local find, gmatch = string.find, string.gmatch +local find, gmatch, match, gsub = string.find, string.gmatch, string.match, string.gsub +local lpegmatch = lpeg.match dir = dir or { } +-- handy + +function dir.current() + return (gsub(lfs.currentdir(),"\\","/")) +end + -- optimizing for no string.find (*) does not save time local attributes = lfs.attributes @@ -2170,6 +2613,35 @@ end dir.glob_pattern = glob_pattern +local function collect_pattern(path,patt,recurse,result) + local ok, scanner + result = result or { } + if path == "/" then + ok, scanner = xpcall(function() return walkdir(path..".") end, function() end) -- kepler safe + else + ok, scanner = xpcall(function() return walkdir(path) end, function() end) -- kepler safe + end + if ok and type(scanner) == "function" then + if not find(path,"/$") then path = path .. '/' end + for name in scanner do + local full = path .. name + local attr = attributes(full) + local mode = attr.mode + if mode == 'file' then + if find(full,patt) then + result[name] = attr + end + elseif recurse and (mode == "directory") and (name ~= '.') and (name ~= "..") then + attr.list = collect_pattern(full,patt,recurse) + result[name] = attr + end + end + end + return result +end + +dir.collect_pattern = collect_pattern + local P, S, R, C, Cc, Cs, Ct, Cv, V = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.Cc, lpeg.Cs, lpeg.Ct, lpeg.Cv, lpeg.V local pattern = Ct { @@ -2189,29 +2661,48 @@ local filter = Cs ( ( )^0 ) local function glob(str,t) - if type(str) == "table" then - local t = t or { } - for s=1,#str do - glob(str[s],t) + if type(t) == "function" then + if type(str) == "table" then + for s=1,#str do + glob(str[s],t) + end + elseif lfs.isfile(str) then + t(str) + else + local split = lpegmatch(pattern,str) + if split then + local root, path, base = split[1], split[2], split[3] + local recurse = find(base,"%*%*") + local start = root .. path + local result = lpegmatch(filter,start .. base) + glob_pattern(start,result,recurse,t) + end end - return t - elseif lfs.isfile(str) then - local t = t or { } - t[#t+1] = str - return t else - local split = pattern:match(str) - if split then + if type(str) == "table" then local t = t or { } - local action = action or function(name) t[#t+1] = name end - local root, path, base = split[1], split[2], split[3] - local recurse = find(base,"%*%*") - local start = root .. path - local result = filter:match(start .. base) - glob_pattern(start,result,recurse,action) + for s=1,#str do + glob(str[s],t) + end + return t + elseif lfs.isfile(str) then + local t = t or { } + t[#t+1] = str return t else - return { } + local split = lpegmatch(pattern,str) + if split then + local t = t or { } + local action = action or function(name) t[#t+1] = name end + local root, path, base = split[1], split[2], split[3] + local recurse = find(base,"%*%*") + local start = root .. path + local result = lpegmatch(filter,start .. base) + glob_pattern(start,result,recurse,action) + return t + else + return { } + end end end end @@ -2273,11 +2764,12 @@ end local make_indeed = true -- false -if string.find(os.getenv("PATH"),";") then +if string.find(os.getenv("PATH"),";") then -- os.type == "windows" function dir.mkdirs(...) - local str, pth = "", "" - for _, s in ipairs({...}) do + local str, pth, t = "", "", { ... } + for i=1,#t do + local s = t[i] if s ~= "" then if str ~= "" then str = str .. "/" .. s @@ -2288,13 +2780,13 @@ if string.find(os.getenv("PATH"),";") then end local first, middle, last local drive = false - first, middle, last = str:match("^(//)(//*)(.*)$") + first, middle, last = match(str,"^(//)(//*)(.*)$") if first then -- empty network path == local path else - first, last = str:match("^(//)/*(.-)$") + first, last = match(str,"^(//)/*(.-)$") if first then - middle, last = str:match("([^/]+)/+(.-)$") + middle, last = match(str,"([^/]+)/+(.-)$") if middle then pth = "//" .. middle else @@ -2302,11 +2794,11 @@ if string.find(os.getenv("PATH"),";") then last = "" end else - first, middle, last = str:match("^([a-zA-Z]:)(/*)(.-)$") + first, middle, last = match(str,"^([a-zA-Z]:)(/*)(.-)$") if first then pth, drive = first .. middle, true else - middle, last = str:match("^(/*)(.-)$") + middle, last = match(str,"^(/*)(.-)$") if not middle then last = str end @@ -2340,34 +2832,31 @@ if string.find(os.getenv("PATH"),";") then --~ print(dir.mkdirs("///a/b/c")) --~ print(dir.mkdirs("a/bbb//ccc/")) - function dir.expand_name(str) - local first, nothing, last = str:match("^(//)(//*)(.*)$") + function dir.expand_name(str) -- will be merged with cleanpath and collapsepath + local first, nothing, last = match(str,"^(//)(//*)(.*)$") if first then - first = lfs.currentdir() .. "/" - first = first:gsub("\\","/") + first = dir.current() .. "/" end if not first then - first, last = str:match("^(//)/*(.*)$") + first, last = match(str,"^(//)/*(.*)$") end if not first then - first, last = str:match("^([a-zA-Z]:)(.*)$") + first, last = match(str,"^([a-zA-Z]:)(.*)$") if first and not find(last,"^/") then local d = lfs.currentdir() if lfs.chdir(first) then - first = lfs.currentdir() - first = first:gsub("\\","/") + first = dir.current() end lfs.chdir(d) end end if not first then - first, last = lfs.currentdir(), str - first = first:gsub("\\","/") + first, last = dir.current(), str end - last = last:gsub("//","/") - last = last:gsub("/%./","/") - last = last:gsub("^/*","") - first = first:gsub("/*$","") + last = gsub(last,"//","/") + last = gsub(last,"/%./","/") + last = gsub(last,"^/*","") + first = gsub(first,"/*$","") if last == "" then return first else @@ -2378,8 +2867,9 @@ if string.find(os.getenv("PATH"),";") then else function dir.mkdirs(...) - local str, pth = "", "" - for _, s in ipairs({...}) do + local str, pth, t = "", "", { ... } + for i=1,#t do + local s = t[i] if s ~= "" then if str ~= "" then str = str .. "/" .. s @@ -2388,7 +2878,7 @@ else end end end - str = str:gsub("/+","/") + str = gsub(str,"/+","/") if find(str,"^/") then pth = "/" for s in gmatch(str,"[^/]+") do @@ -2422,12 +2912,12 @@ else --~ print(dir.mkdirs("///a/b/c")) --~ print(dir.mkdirs("a/bbb//ccc/")) - function dir.expand_name(str) + function dir.expand_name(str) -- will be merged with cleanpath and collapsepath if not find(str,"^/") then str = lfs.currentdir() .. "/" .. str end - str = str:gsub("//","/") - str = str:gsub("/%./","/") + str = gsub(str,"//","/") + str = gsub(str,"/%./","/") return str end @@ -2442,7 +2932,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-boolean'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -2503,19 +2993,40 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-unicode'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } +if not unicode then + + unicode = { utf8 = { } } + + local floor, char = math.floor, string.char + + function unicode.utf8.utfchar(n) + if n < 0x80 then + return char(n) + elseif n < 0x800 then + return char(0xC0 + floor(n/0x40)) .. char(0x80 + (n % 0x40)) + elseif n < 0x10000 then + return char(0xE0 + floor(n/0x1000)) .. char(0x80 + (floor(n/0x40) % 0x40)) .. char(0x80 + (n % 0x40)) + elseif n < 0x40000 then + return char(0xF0 + floor(n/0x40000)) .. char(0x80 + floor(n/0x1000)) .. char(0x80 + (floor(n/0x40) % 0x40)) .. char(0x80 + (n % 0x40)) + else -- wrong: + -- return char(0xF1 + floor(n/0x1000000)) .. char(0x80 + floor(n/0x40000)) .. char(0x80 + floor(n/0x1000)) .. char(0x80 + (floor(n/0x40) % 0x40)) .. char(0x80 + (n % 0x40)) + return "?" + end + end + +end + utf = utf or unicode.utf8 local concat, utfchar, utfgsub = table.concat, utf.char, utf.gsub local char, byte, find, bytepairs = string.char, string.byte, string.find, string.bytepairs -unicode = unicode or { } - -- 0 EF BB BF UTF-8 -- 1 FF FE UTF-16-little-endian -- 2 FE FF UTF-16-big-endian @@ -2530,14 +3041,20 @@ unicode.utfname = { [4] = 'utf-32-be' } -function unicode.utftype(f) -- \000 fails ! +-- \000 fails in <= 5.0 but is valid in >=5.1 where %z is depricated + +function unicode.utftype(f) local str = f:read(4) if not str then f:seek('set') return 0 - elseif find(str,"^%z%z\254\255") then + -- elseif find(str,"^%z%z\254\255") then -- depricated + -- elseif find(str,"^\000\000\254\255") then -- not permitted and bugged + elseif find(str,"\000\000\254\255",1,true) then -- seems to work okay (TH) return 4 - elseif find(str,"^\255\254%z%z") then + -- elseif find(str,"^\255\254%z%z") then -- depricated + -- elseif find(str,"^\255\254\000\000") then -- not permitted and bugged + elseif find(str,"\255\254\000\000",1,true) then -- seems to work okay (TH) return 3 elseif find(str,"^\254\255") then f:seek('set',2) @@ -2681,7 +3198,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-math'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -2728,7 +3245,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-utils'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -2736,6 +3253,10 @@ if not modules then modules = { } end modules ['l-utils'] = { -- hm, quite unreadable +local gsub = string.gsub +local concat = table.concat +local type, next = type, next + if not utils then utils = { } end if not utils.merger then utils.merger = { } end if not utils.lua then utils.lua = { } end @@ -2773,7 +3294,7 @@ function utils.merger._self_load_(name) end if data and utils.merger.strip_comment then -- saves some 20K - data = data:gsub("%-%-~[^\n\r]*[\r\n]", "") + data = gsub(data,"%-%-~[^\n\r]*[\r\n]", "") end return data or "" end @@ -2791,7 +3312,7 @@ end function utils.merger._self_swap_(data,code) if data ~= "" then - return (data:gsub(utils.merger.pattern, function(s) + return (gsub(data,utils.merger.pattern, function(s) return "\n\n" .. "-- "..utils.merger.m_begin .. "\n" .. code .. "\n" .. "-- "..utils.merger.m_end .. "\n\n" end, 1)) else @@ -2801,8 +3322,8 @@ end --~ stripper: --~ ---~ data = string.gsub(data,"%-%-~[^\n]*\n","") ---~ data = string.gsub(data,"\n\n+","\n") +--~ data = gsub(data,"%-%-~[^\n]*\n","") +--~ data = gsub(data,"\n\n+","\n") function utils.merger._self_libs_(libs,list) local result, f, frozen = { }, nil, false @@ -2810,9 +3331,10 @@ function utils.merger._self_libs_(libs,list) if type(libs) == 'string' then libs = { libs } end if type(list) == 'string' then list = { list } end local foundpath = nil - for _, lib in ipairs(libs) do - for _, pth in ipairs(list) do - pth = string.gsub(pth,"\\","/") -- file.clean_path + for i=1,#libs do + local lib = libs[i] + for j=1,#list do + local pth = gsub(list[j],"\\","/") -- file.clean_path utils.report("checking library path %s",pth) local name = pth .. "/" .. lib if lfs.isfile(name) then @@ -2824,7 +3346,8 @@ function utils.merger._self_libs_(libs,list) if foundpath then utils.report("using library path %s",foundpath) local right, wrong = { }, { } - for _, lib in ipairs(libs) do + for i=1,#libs do + local lib = libs[i] local fullname = foundpath .. "/" .. lib if lfs.isfile(fullname) then -- right[#right+1] = lib @@ -2838,15 +3361,15 @@ function utils.merger._self_libs_(libs,list) end end if #right > 0 then - utils.report("merged libraries: %s",table.concat(right," ")) + utils.report("merged libraries: %s",concat(right," ")) end if #wrong > 0 then - utils.report("skipped libraries: %s",table.concat(wrong," ")) + utils.report("skipped libraries: %s",concat(wrong," ")) end else utils.report("no valid library path found") end - return table.concat(result, "\n\n") + return concat(result, "\n\n") end function utils.merger.selfcreate(libs,list,target) @@ -2904,16 +3427,28 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-aux'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } +-- for inline, no store split : for s in string.gmatch(str,",* *([^,]+)") do .. end + aux = aux or { } local concat, format, gmatch = table.concat, string.format, string.gmatch local tostring, type = tostring, type +local lpegmatch = lpeg.match + +local P, R, V = lpeg.P, lpeg.R, lpeg.V + +local escape, left, right = P("\\"), P('{'), P('}') + +lpeg.patterns.balanced = P { + [1] = ((escape * (left+right)) + (1 - (left+right)) + V(2))^0, + [2] = left * V(1) * right +} local space = lpeg.P(' ') local equal = lpeg.P("=") @@ -2921,7 +3456,7 @@ local comma = lpeg.P(",") local lbrace = lpeg.P("{") local rbrace = lpeg.P("}") local nobrace = 1 - (lbrace+rbrace) -local nested = lpeg.P{ lbrace * (nobrace + lpeg.V(1))^0 * rbrace } +local nested = lpeg.P { lbrace * (nobrace + lpeg.V(1))^0 * rbrace } local spaces = space^0 local value = lpeg.P(lbrace * lpeg.C((nobrace + nested)^0) * rbrace) + lpeg.C((nested + (1-comma))^0) @@ -2959,13 +3494,13 @@ function aux.make_settings_to_hash_pattern(set,how) end end -function aux.settings_to_hash(str) +function aux.settings_to_hash(str,existing) if str and str ~= "" then - hash = { } + hash = existing or { } if moretolerant then - pattern_b_s:match(str) + lpegmatch(pattern_b_s,str) else - pattern_a_s:match(str) + lpegmatch(pattern_a_s,str) end return hash else @@ -2973,39 +3508,41 @@ function aux.settings_to_hash(str) end end -function aux.settings_to_hash_tolerant(str) +function aux.settings_to_hash_tolerant(str,existing) if str and str ~= "" then - hash = { } - pattern_b_s:match(str) + hash = existing or { } + lpegmatch(pattern_b_s,str) return hash else return { } end end -function aux.settings_to_hash_strict(str) +function aux.settings_to_hash_strict(str,existing) if str and str ~= "" then - hash = { } - pattern_c_s:match(str) + hash = existing or { } + lpegmatch(pattern_c_s,str) return next(hash) and hash else return nil end end -local seperator = comma * space^0 +local separator = comma * space^0 local value = lpeg.P(lbrace * lpeg.C((nobrace + nested)^0) * rbrace) + lpeg.C((nested + (1-comma))^0) -local pattern = lpeg.Ct(value*(seperator*value)^0) +local pattern = lpeg.Ct(value*(separator*value)^0) -- "aap, {noot}, mies" : outer {} removes, leading spaces ignored aux.settings_to_array_pattern = pattern +-- we could use a weak table as cache + function aux.settings_to_array(str) if not str or str == "" then return { } else - return pattern:match(str) + return lpegmatch(pattern,str) end end @@ -3014,10 +3551,10 @@ local function set(t,v) end local value = lpeg.P(lpeg.Carg(1)*value) / set -local pattern = value*(seperator*value)^0 * lpeg.Carg(1) +local pattern = value*(separator*value)^0 * lpeg.Carg(1) function aux.add_settings_to_array(t,str) - return pattern:match(str, nil, t) + return lpegmatch(pattern,str,nil,t) end function aux.hash_to_string(h,separator,yes,no,strict,omit) @@ -3065,6 +3602,13 @@ function aux.settings_to_set(str,t) return t end +local value = lbrace * lpeg.C((nobrace + nested)^0) * rbrace +local pattern = lpeg.Ct((space + value)^0) + +function aux.arguments_to_table(str) + return lpegmatch(pattern,str) +end + -- temporary here function aux.getparameters(self,class,parentclass,settings) @@ -3073,36 +3617,31 @@ function aux.getparameters(self,class,parentclass,settings) sc = table.clone(self[parent]) self[class] = sc end - aux.add_settings_to_array(sc, settings) + aux.settings_to_hash(settings,sc) end -- temporary here -local digit = lpeg.R("09") -local period = lpeg.P(".") -local zero = lpeg.P("0") - ---~ local finish = lpeg.P(-1) ---~ local nodigit = (1-digit) + finish ---~ local case_1 = (period * zero^1 * #nodigit)/"" -- .000 ---~ local case_2 = (period * (1-(zero^0/"") * #nodigit)^1 * (zero^0/"") * nodigit) -- .010 .10 .100100 - +local digit = lpeg.R("09") +local period = lpeg.P(".") +local zero = lpeg.P("0") local trailingzeros = zero^0 * -digit -- suggested by Roberto R -local case_1 = period * trailingzeros / "" -local case_2 = period * (digit - trailingzeros)^1 * (trailingzeros / "") - -local number = digit^1 * (case_1 + case_2) -local stripper = lpeg.Cs((number + 1)^0) +local case_1 = period * trailingzeros / "" +local case_2 = period * (digit - trailingzeros)^1 * (trailingzeros / "") +local number = digit^1 * (case_1 + case_2) +local stripper = lpeg.Cs((number + 1)^0) --~ local sample = "bla 11.00 bla 11 bla 0.1100 bla 1.00100 bla 0.00 bla 0.001 bla 1.1100 bla 0.100100100 bla 0.00100100100" --~ collectgarbage("collect") --~ str = string.rep(sample,10000) --~ local ts = os.clock() ---~ stripper:match(str) ---~ print(#str, os.clock()-ts, stripper:match(sample)) +--~ lpegmatch(stripper,str) +--~ print(#str, os.clock()-ts, lpegmatch(stripper,sample)) + +lpeg.patterns.strip_zeros = stripper function aux.strip_zeros(str) - return stripper:match(str) + return lpegmatch(stripper,str) end function aux.definetable(target) -- defines undefined tables @@ -3126,6 +3665,24 @@ function aux.accesstable(target) return t end +-- as we use this a lot ... + +--~ function aux.cachefunction(action,weak) +--~ local cache = { } +--~ if weak then +--~ setmetatable(cache, { __mode = "kv" } ) +--~ end +--~ local function reminder(str) +--~ local found = cache[str] +--~ if not found then +--~ found = action(str) +--~ cache[str] = found +--~ end +--~ return found +--~ end +--~ return reminder, cache +--~ end + end -- of closure @@ -3133,7 +3690,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['trac-tra'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to trac-tra.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -3143,12 +3700,17 @@ if not modules then modules = { } end modules ['trac-tra'] = { -- bound to a variable, like node.new, node.copy etc (contrary to for instance -- node.has_attribute which is bound to a has_attribute local variable in mkiv) +local debug = require "debug" + +local getinfo = debug.getinfo +local type, next = type, next +local concat = table.concat +local format, find, lower, gmatch, gsub = string.format, string.find, string.lower, string.gmatch, string.gsub + debugger = debugger or { } local counters = { } local names = { } -local getinfo = debug.getinfo -local format, find, lower, gmatch = string.format, string.find, string.lower, string.gmatch -- one @@ -3187,10 +3749,10 @@ function debugger.showstats(printer,threshold) local total, grandtotal, functions = 0, 0, 0 printer("\n") -- ugly but ok -- table.sort(counters) - for func, count in pairs(counters) do + for func, count in next, counters do if count > threshold then local name = getname(func) - if not name:find("for generator") then + if not find(name,"for generator") then printer(format("%8i %s", count, name)) total = total + count end @@ -3222,7 +3784,7 @@ end --~ local total, grandtotal, functions = 0, 0, 0 --~ printer("\n") -- ugly but ok --~ -- table.sort(counters) ---~ for func, count in pairs(counters) do +--~ for func, count in next, counters do --~ if count > threshold then --~ printer(format("%8i %s", count, func)) --~ total = total + count @@ -3276,38 +3838,77 @@ end --~ print("") --~ debugger.showstats(print,3) -trackers = trackers or { } +setters = setters or { } +setters.data = setters.data or { } -local data, done = { }, { } +--~ local function set(t,what,value) +--~ local data, done = t.data, t.done +--~ if type(what) == "string" then +--~ what = aux.settings_to_array(what) -- inefficient but ok +--~ end +--~ for i=1,#what do +--~ local w = what[i] +--~ for d, f in next, data do +--~ if done[d] then +--~ -- prevent recursion due to wildcards +--~ elseif find(d,w) then +--~ done[d] = true +--~ for i=1,#f do +--~ f[i](value) +--~ end +--~ end +--~ end +--~ end +--~ end -local function set(what,value) +local function set(t,what,value) + local data, done = t.data, t.done if type(what) == "string" then - what = aux.settings_to_array(what) + what = aux.settings_to_hash(what) -- inefficient but ok end - for i=1,#what do - local w = what[i] + for w, v in next, what do + if v == "" then + v = value + else + v = toboolean(v) + end for d, f in next, data do if done[d] then -- prevent recursion due to wildcards elseif find(d,w) then done[d] = true for i=1,#f do - f[i](value) + f[i](v) end end end end end -local function reset() - for d, f in next, data do +local function reset(t) + for d, f in next, t.data do for i=1,#f do f[i](false) end end end -function trackers.register(what,...) +local function enable(t,what) + set(t,what,true) +end + +local function disable(t,what) + local data = t.data + if not what or what == "" then + t.done = { } + reset(t) + else + set(t,what,false) + end +end + +function setters.register(t,what,...) + local data = t.data what = lower(what) local w = data[what] if not w then @@ -3319,32 +3920,32 @@ function trackers.register(what,...) if typ == "function" then w[#w+1] = fnc elseif typ == "string" then - w[#w+1] = function(value) set(fnc,value,nesting) end + w[#w+1] = function(value) set(t,fnc,value,nesting) end end end end -function trackers.enable(what) - done = { } - set(what,true) +function setters.enable(t,what) + local e = t.enable + t.enable, t.done = enable, { } + enable(t,string.simpleesc(tostring(what))) + t.enable, t.done = e, { } end -function trackers.disable(what) - done = { } - if not what or what == "" then - trackers.reset(what) - else - set(what,false) - end +function setters.disable(t,what) + local e = t.disable + t.disable, t.done = disable, { } + disable(t,string.simpleesc(tostring(what))) + t.disable, t.done = e, { } end -function trackers.reset(what) - done = { } - reset() +function setters.reset(t) + t.done = { } + reset(t) end -function trackers.list() -- pattern - local list = table.sortedkeys(data) +function setters.list(t) -- pattern + local list = table.sortedkeys(t.data) local user, system = { }, { } for l=1,#list do local what = list[l] @@ -3357,6 +3958,78 @@ function trackers.list() -- pattern return user, system end +function setters.show(t) + commands.writestatus("","") + local list = setters.list(t) + for k=1,#list do + commands.writestatus(t.name,list[k]) + end + commands.writestatus("","") +end + +-- we could have used a bit of oo and the trackers:enable syntax but +-- there is already a lot of code around using the singular tracker + +-- we could make this into a module + +function setters.new(name) + local t + t = { + data = { }, + name = name, + enable = function(...) setters.enable (t,...) end, + disable = function(...) setters.disable (t,...) end, + register = function(...) setters.register(t,...) end, + list = function(...) setters.list (t,...) end, + show = function(...) setters.show (t,...) end, + } + setters.data[name] = t + return t +end + +trackers = setters.new("trackers") +directives = setters.new("directives") +experiments = setters.new("experiments") + +-- nice trick: we overload two of the directives related functions with variants that +-- do tracing (itself using a tracker) .. proof of concept + +local trace_directives = false local trace_directives = false trackers.register("system.directives", function(v) trace_directives = v end) +local trace_experiments = false local trace_experiments = false trackers.register("system.experiments", function(v) trace_experiments = v end) + +local e = directives.enable +local d = directives.disable + +function directives.enable(...) + commands.writestatus("directives","enabling: %s",concat({...}," ")) + e(...) +end + +function directives.disable(...) + commands.writestatus("directives","disabling: %s",concat({...}," ")) + d(...) +end + +local e = experiments.enable +local d = experiments.disable + +function experiments.enable(...) + commands.writestatus("experiments","enabling: %s",concat({...}," ")) + e(...) +end + +function experiments.disable(...) + commands.writestatus("experiments","disabling: %s",concat({...}," ")) + d(...) +end + +-- a useful example + +directives.register("system.nostatistics", function(v) + statistics.enable = not v +end) + + end -- of closure @@ -3364,7 +4037,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['luat-env'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -3376,10 +4049,10 @@ if not modules then modules = { } end modules ['luat-env'] = { -- evolved before bytecode arrays were available and so a lot of -- code has disappeared already. -local trace_verbose = false trackers.register("resolvers.verbose", function(v) trace_verbose = v end) -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v trackers.enable("resolvers.verbose") end) +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) -local format = string.format +local format, sub, match, gsub, find = string.format, string.sub, string.match, string.gsub, string.find +local unquote, quote = string.unquote, string.quote -- precautions @@ -3413,13 +4086,14 @@ if not environment.jobname then environ function environment.initialize_arguments(arg) local arguments, files = { }, { } environment.arguments, environment.files, environment.sortedflags = arguments, files, nil - for index, argument in pairs(arg) do + for index=1,#arg do + local argument = arg[index] if index > 0 then - local flag, value = argument:match("^%-+(.+)=(.-)$") + local flag, value = match(argument,"^%-+(.-)=(.-)$") if flag then - arguments[flag] = string.unquote(value or "") + arguments[flag] = unquote(value or "") else - flag = argument:match("^%-+(.+)") + flag = match(argument,"^%-+(.+)") if flag then arguments[flag] = true else @@ -3446,25 +4120,30 @@ function environment.argument(name,partial) return arguments[name] elseif partial then if not sortedflags then - sortedflags = { } - for _,v in pairs(table.sortedkeys(arguments)) do - sortedflags[#sortedflags+1] = "^" .. v + sortedflags = table.sortedkeys(arguments) + for k=1,#sortedflags do + sortedflags[k] = "^" .. sortedflags[k] end environment.sortedflags = sortedflags end -- example of potential clash: ^mode ^modefile - for _,v in ipairs(sortedflags) do - if name:find(v) then - return arguments[v:sub(2,#v)] + for k=1,#sortedflags do + local v = sortedflags[k] + if find(name,v) then + return arguments[sub(v,2,#v)] end end end return nil end +environment.argument("x",true) + function environment.split_arguments(separator) -- rather special, cut-off before separator local done, before, after = false, { }, { } - for _,v in ipairs(environment.original_arguments) do + local original_arguments = environment.original_arguments + for k=1,#original_arguments do + local v = original_arguments[k] if not done and v == separator then done = true elseif done then @@ -3481,16 +4160,17 @@ function environment.reconstruct_commandline(arg,noquote) if noquote and #arg == 1 then local a = arg[1] a = resolvers.resolve(a) - a = a:unquote() + a = unquote(a) return a - elseif next(arg) then + elseif #arg > 0 then local result = { } - for _,a in ipairs(arg) do -- ipairs 1 .. #n + for i=1,#arg do + local a = arg[i] a = resolvers.resolve(a) - a = a:unquote() - a = a:gsub('"','\\"') -- tricky - if a:find(" ") then - result[#result+1] = a:quote() + a = unquote(a) + a = gsub(a,'"','\\"') -- tricky + if find(a," ") then + result[#result+1] = quote(a) else result[#result+1] = a end @@ -3503,17 +4183,18 @@ end if arg then - -- new, reconstruct quoted snippets (maybe better just remnove the " then and add them later) + -- new, reconstruct quoted snippets (maybe better just remove the " then and add them later) local newarg, instring = { }, false - for index, argument in ipairs(arg) do - if argument:find("^\"") then - newarg[#newarg+1] = argument:gsub("^\"","") - if not argument:find("\"$") then + for index=1,#arg do + local argument = arg[index] + if find(argument,"^\"") then + newarg[#newarg+1] = gsub(argument,"^\"","") + if not find(argument,"\"$") then instring = true end - elseif argument:find("\"$") then - newarg[#newarg] = newarg[#newarg] .. " " .. argument:gsub("\"$","") + elseif find(argument,"\"$") then + newarg[#newarg] = newarg[#newarg] .. " " .. gsub(argument,"\"$","") instring = false elseif instring then newarg[#newarg] = newarg[#newarg] .. " " .. argument @@ -3568,12 +4249,12 @@ function environment.luafilechunk(filename) -- used for loading lua bytecode in filename = file.replacesuffix(filename, "lua") local fullname = environment.luafile(filename) if fullname and fullname ~= "" then - if trace_verbose then + if trace_locating then logs.report("fileio","loading file %s", fullname) end return environment.loadedluacode(fullname) else - if trace_verbose then + if trace_locating then logs.report("fileio","unknown file %s", filename) end return nil @@ -3593,7 +4274,7 @@ function environment.loadluafile(filename, version) -- when not overloaded by explicit suffix we look for a luc file first local fullname = (lucname and environment.luafile(lucname)) or "" if fullname ~= "" then - if trace_verbose then + if trace_locating then logs.report("fileio","loading %s", fullname) end chunk = loadfile(fullname) -- this way we don't need a file exists check @@ -3611,7 +4292,7 @@ function environment.loadluafile(filename, version) if v == version then return true else - if trace_verbose then + if trace_locating then logs.report("fileio","version mismatch for %s: lua=%s, luc=%s", filename, v, version) end environment.loadluafile(filename) @@ -3622,12 +4303,12 @@ function environment.loadluafile(filename, version) end fullname = (luaname and environment.luafile(luaname)) or "" if fullname ~= "" then - if trace_verbose then + if trace_locating then logs.report("fileio","loading %s", fullname) end chunk = loadfile(fullname) -- this way we don't need a file exists check if not chunk then - if verbose then + if trace_locating then logs.report("fileio","unknown file %s", filename) end else @@ -3645,7 +4326,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['trac-inf'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to trac-inf.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -3670,6 +4351,14 @@ function statistics.hastimer(instance) return instance and instance.starttime end +function statistics.resettiming(instance) + if not instance then + notimer = { timing = 0, loadtime = 0 } + else + instance.timing, instance.loadtime = 0, 0 + end +end + function statistics.starttiming(instance) if not instance then notimer = { } @@ -3684,6 +4373,8 @@ function statistics.starttiming(instance) if not instance.loadtime then instance.loadtime = 0 end + else +--~ logs.report("system","nested timing (%s)",tostring(instance)) end instance.timing = it + 1 end @@ -3729,6 +4420,12 @@ function statistics.elapsedindeed(instance) return t > statistics.threshold end +function statistics.elapsedseconds(instance,rest) -- returns nil if 0 seconds + if statistics.elapsedindeed(instance) then + return format("%s seconds %s", statistics.elapsedtime(instance),rest or "") + end +end + -- general function function statistics.register(tag,fnc) @@ -3807,14 +4504,32 @@ function statistics.timed(action,report) report("total runtime: %s",statistics.elapsedtime(timer)) end +-- where, not really the best spot for this: + +commands = commands or { } + +local timer + +function commands.resettimer() + statistics.resettiming(timer) + statistics.starttiming(timer) +end + +function commands.elapsedtime() + statistics.stoptiming(timer) + tex.sprint(statistics.elapsedtime(timer)) +end + +commands.resettimer() + end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['luat-log'] = { +if not modules then modules = { } end modules ['trac-log'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to trac-log.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -3822,7 +4537,11 @@ if not modules then modules = { } end modules ['luat-log'] = { -- this is old code that needs an overhaul -local write_nl, write, format = texio.write_nl or print, texio.write or io.write, string.format +--~ io.stdout:setvbuf("no") +--~ io.stderr:setvbuf("no") + +local write_nl, write = texio.write_nl or print, texio.write or io.write +local format, gmatch = string.format, string.gmatch local texcount = tex and tex.count if texlua then @@ -3903,25 +4622,48 @@ function logs.tex.line(fmt,...) -- new end end +--~ function logs.tex.start_page_number() +--~ local real, user, sub = texcount.realpageno, texcount.userpageno, texcount.subpageno +--~ if real > 0 then +--~ if user > 0 then +--~ if sub > 0 then +--~ write(format("[%s.%s.%s",real,user,sub)) +--~ else +--~ write(format("[%s.%s",real,user)) +--~ end +--~ else +--~ write(format("[%s",real)) +--~ end +--~ else +--~ write("[-") +--~ end +--~ end + +--~ function logs.tex.stop_page_number() +--~ write("]") +--~ end + +local real, user, sub + function logs.tex.start_page_number() - local real, user, sub = texcount.realpageno, texcount.userpageno, texcount.subpageno + real, user, sub = texcount.realpageno, texcount.userpageno, texcount.subpageno +end + +function logs.tex.stop_page_number() if real > 0 then if user > 0 then if sub > 0 then - write(format("[%s.%s.%s",real,user,sub)) + logs.report("pages", "flushing realpage %s, userpage %s, subpage %s",real,user,sub) else - write(format("[%s.%s",real,user)) + logs.report("pages", "flushing realpage %s, userpage %s",real,user) end else - write(format("[%s",real)) + logs.report("pages", "flushing realpage %s",real) end else - write("[-") + logs.report("pages", "flushing page") end -end - -function logs.tex.stop_page_number() - write("]") + io.flush() end logs.tex.report_job_stat = statistics.show_job_stat @@ -4021,7 +4763,7 @@ end function logs.setprogram(_name_,_banner_,_verbose_) name, banner = _name_, _banner_ if _verbose_ then - trackers.enable("resolvers.verbose") + trackers.enable("resolvers.locating") end logs.set_method("tex") logs.report = report -- also used in libraries @@ -4034,9 +4776,9 @@ end function logs.setverbose(what) if what then - trackers.enable("resolvers.verbose") + trackers.enable("resolvers.locating") else - trackers.disable("resolvers.verbose") + trackers.disable("resolvers.locating") end logs.verbose = what or false end @@ -4053,7 +4795,7 @@ logs.report = logs.tex.report logs.simple = logs.tex.report function logs.reportlines(str) -- todo: <lines></lines> - for line in str:gmatch("(.-)[\n\r]") do + for line in gmatch(str,"(.-)[\n\r]") do logs.report(line) end end @@ -4064,8 +4806,12 @@ end logs.simpleline = logs.reportline -function logs.help(message,option) +function logs.reportbanner() -- for scripts too logs.report(banner) +end + +function logs.help(message,option) + logs.reportbanner() logs.reportline() logs.reportlines(message) local moreinfo = logs.moreinfo or "" @@ -4097,6 +4843,11 @@ end --~ logs.system(syslogname,"context","test","fonts","font %s recached due to newer version (%s)","blabla","123") --~ end +function logs.fatal(where,...) + logs.report(where,"fatal error: %s, aborting now",format(...)) + os.exit() +end + end -- of closure @@ -4104,10 +4855,10 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-inp'] = { version = 1.001, + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files", - comment = "companion to luat-lib.tex", } -- After a few years using the code the large luat-inp.lua file @@ -4119,7 +4870,7 @@ if not modules then modules = { } end modules ['data-inp'] = { -- * some public auxiliary functions were made private -- -- TODO: os.getenv -> os.env[] --- TODO: instances.[hashes,cnffiles,configurations,522] -> ipairs (alles check, sneller) +-- TODO: instances.[hashes,cnffiles,configurations,522] -- TODO: check escaping in find etc, too much, too slow -- This lib is multi-purpose and can be loaded again later on so that @@ -4140,12 +4891,13 @@ if not modules then modules = { } end modules ['data-inp'] = { local format, gsub, find, lower, upper, match, gmatch = string.format, string.gsub, string.find, string.lower, string.upper, string.match, string.gmatch local concat, insert, sortedkeys = table.concat, table.insert, table.sortedkeys local next, type = next, type +local lpegmatch = lpeg.match -local trace_locating, trace_detail, trace_verbose = false, false, false +local trace_locating, trace_detail, trace_expansions = false, false, false -trackers.register("resolvers.verbose", function(v) trace_verbose = v end) -trackers.register("resolvers.locating", function(v) trace_locating = v trackers.enable("resolvers.verbose") end) -trackers.register("resolvers.detail", function(v) trace_detail = v trackers.enable("resolvers.verbose,resolvers.detail") end) +trackers.register("resolvers.locating", function(v) trace_locating = v end) +trackers.register("resolvers.details", function(v) trace_detail = v end) +trackers.register("resolvers.expansions", function(v) trace_expansions = v end) -- todo if not resolvers then resolvers = { @@ -4169,7 +4921,7 @@ resolvers.generators.notfound = { nil } resolvers.cacheversion = '1.0.1' resolvers.cnfname = 'texmf.cnf' resolvers.luaname = 'texmfcnf.lua' -resolvers.homedir = os.env[os.platform == "windows" and 'USERPROFILE'] or os.env['HOME'] or '~' +resolvers.homedir = os.env[os.type == "windows" and 'USERPROFILE'] or os.env['HOME'] or '~' resolvers.cnfdefault = '{$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c}' local dummy_path_expr = "^!*unset/*$" @@ -4211,8 +4963,8 @@ suffixes['lua'] = { 'lua', 'luc', 'tma', 'tmc' } alternatives['map files'] = 'map' alternatives['enc files'] = 'enc' -alternatives['cid files'] = 'cid' -alternatives['fea files'] = 'fea' +alternatives['cid maps'] = 'cid' -- great, why no cid files +alternatives['font feature files'] = 'fea' -- and fea files here alternatives['opentype fonts'] = 'otf' alternatives['truetype fonts'] = 'ttf' alternatives['truetype collections'] = 'ttc' @@ -4228,6 +4980,11 @@ formats ['sfd'] = 'SFDFONTS' suffixes ['sfd'] = { 'sfd' } alternatives['subfont definition files'] = 'sfd' +-- lib paths + +formats ['lib'] = 'CLUAINPUTS' -- new (needs checking) +suffixes['lib'] = (os.libsuffix and { os.libsuffix }) or { 'dll', 'so' } + -- In practice we will work within one tds tree, but i want to keep -- the option open to build tools that look at multiple trees, which is -- why we keep the tree specific data in a table. We used to pass the @@ -4350,8 +5107,10 @@ local function check_configuration() -- not yet ok, no time for debugging now -- bad luck end fix("LUAINPUTS" , ".;$TEXINPUTS;$TEXMFSCRIPTS") -- no progname, hm - fix("FONTFEATURES", ".;$TEXMF/fonts/fea//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") - fix("FONTCIDMAPS" , ".;$TEXMF/fonts/cid//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") + -- this will go away some day + fix("FONTFEATURES", ".;$TEXMF/fonts/{data,fea}//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") + fix("FONTCIDMAPS" , ".;$TEXMF/fonts/{data,cid}//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") + -- fix("LUATEXLIBS" , ".;$TEXMF/luatex/lua//") end @@ -4366,7 +5125,7 @@ function resolvers.settrace(n) -- no longer number but: 'locating' or 'detail' end end -resolvers.settrace(os.getenv("MTX.resolvers.TRACE") or os.getenv("MTX_INPUT_TRACE")) +resolvers.settrace(os.getenv("MTX_INPUT_TRACE")) function resolvers.osenv(key) local ie = instance.environment @@ -4454,37 +5213,43 @@ end -- work that well; the parsing is ok, but dealing with the resulting -- table is a pain because we need to work inside-out recursively +local function do_first(a,b) + local t = { } + for s in gmatch(b,"[^,]+") do t[#t+1] = a .. s end + return "{" .. concat(t,",") .. "}" +end + +local function do_second(a,b) + local t = { } + for s in gmatch(a,"[^,]+") do t[#t+1] = s .. b end + return "{" .. concat(t,",") .. "}" +end + +local function do_both(a,b) + local t = { } + for sa in gmatch(a,"[^,]+") do + for sb in gmatch(b,"[^,]+") do + t[#t+1] = sa .. sb + end + end + return "{" .. concat(t,",") .. "}" +end + +local function do_three(a,b,c) + return a .. b.. c +end + local function splitpathexpr(str, t, validate) -- no need for further optimization as it is only called a - -- few times, we can use lpeg for the sub; we could move - -- the local functions outside the body + -- few times, we can use lpeg for the sub + if trace_expansions then + logs.report("fileio","expanding variable '%s'",str) + end t = t or { } str = gsub(str,",}",",@}") str = gsub(str,"{,","{@,") -- str = "@" .. str .. "@" local ok, done - local function do_first(a,b) - local t = { } - for s in gmatch(b,"[^,]+") do t[#t+1] = a .. s end - return "{" .. concat(t,",") .. "}" - end - local function do_second(a,b) - local t = { } - for s in gmatch(a,"[^,]+") do t[#t+1] = s .. b end - return "{" .. concat(t,",") .. "}" - end - local function do_both(a,b) - local t = { } - for sa in gmatch(a,"[^,]+") do - for sb in gmatch(b,"[^,]+") do - t[#t+1] = sa .. sb - end - end - return "{" .. concat(t,",") .. "}" - end - local function do_three(a,b,c) - return a .. b.. c - end while true do done = false while true do @@ -4515,6 +5280,11 @@ local function splitpathexpr(str, t, validate) t[#t+1] = s end end + if trace_expansions then + for k=1,#t do + logs.report("fileio","% 4i: %s",k,t[k]) + end + end return t end @@ -4554,18 +5324,27 @@ end -- also we now follow the stupid route: if not set then just assume *one* -- cnf file under texmf (i.e. distribution) -resolvers.ownpath = resolvers.ownpath or nil -resolvers.ownbin = resolvers.ownbin or arg[-2] or arg[-1] or arg[0] or "luatex" -resolvers.autoselfdir = true -- false may be handy for debugging +local args = environment and environment.original_arguments or arg -- this needs a cleanup + +resolvers.ownbin = resolvers.ownbin or args[-2] or arg[-2] or args[-1] or arg[-1] or arg[0] or "luatex" +resolvers.ownbin = gsub(resolvers.ownbin,"\\","/") function resolvers.getownpath() - if not resolvers.ownpath then - if resolvers.autoselfdir and os.selfdir then - resolvers.ownpath = os.selfdir - else - local binary = resolvers.ownbin - if os.platform == "windows" then - binary = file.replacesuffix(binary,"exe") + local ownpath = resolvers.ownpath or os.selfdir + if not ownpath or ownpath == "" or ownpath == "unset" then + ownpath = args[-1] or arg[-1] + ownpath = ownpath and file.dirname(gsub(ownpath,"\\","/")) + if not ownpath or ownpath == "" then + ownpath = args[-0] or arg[-0] + ownpath = ownpath and file.dirname(gsub(ownpath,"\\","/")) + end + local binary = resolvers.ownbin + if not ownpath or ownpath == "" then + ownpath = ownpath and file.dirname(binary) + end + if not ownpath or ownpath == "" then + if os.binsuffix ~= "" then + binary = file.replacesuffix(binary,os.binsuffix) end for p in gmatch(os.getenv("PATH"),"[^"..io.pathseparator.."]+") do local b = file.join(p,binary) @@ -4577,30 +5356,39 @@ function resolvers.getownpath() local olddir = lfs.currentdir() if lfs.chdir(p) then local pp = lfs.currentdir() - if trace_verbose and p ~= pp then - logs.report("fileio","following symlink %s to %s",p,pp) + if trace_locating and p ~= pp then + logs.report("fileio","following symlink '%s' to '%s'",p,pp) end - resolvers.ownpath = pp + ownpath = pp lfs.chdir(olddir) else - if trace_verbose then - logs.report("fileio","unable to check path %s",p) + if trace_locating then + logs.report("fileio","unable to check path '%s'",p) end - resolvers.ownpath = p + ownpath = p end break end end end - if not resolvers.ownpath then resolvers.ownpath = '.' end + if not ownpath or ownpath == "" then + ownpath = "." + logs.report("fileio","forcing fallback ownpath .") + elseif trace_locating then + logs.report("fileio","using ownpath '%s'",ownpath) + end end - return resolvers.ownpath + resolvers.ownpath = ownpath + function resolvers.getownpath() + return resolvers.ownpath + end + return ownpath end local own_places = { "SELFAUTOLOC", "SELFAUTODIR", "SELFAUTOPARENT", "TEXMFCNF" } local function identify_own() - local ownpath = resolvers.getownpath() or lfs.currentdir() + local ownpath = resolvers.getownpath() or dir.current() local ie = instance.environment if ownpath then if resolvers.env('SELFAUTOLOC') == "" then os.env['SELFAUTOLOC'] = file.collapse_path(ownpath) end @@ -4613,10 +5401,10 @@ local function identify_own() if resolvers.env('TEXMFCNF') == "" then os.env['TEXMFCNF'] = resolvers.cnfdefault end if resolvers.env('TEXOS') == "" then os.env['TEXOS'] = resolvers.env('SELFAUTODIR') end if resolvers.env('TEXROOT') == "" then os.env['TEXROOT'] = resolvers.env('SELFAUTOPARENT') end - if trace_verbose then + if trace_locating then for i=1,#own_places do local v = own_places[i] - logs.report("fileio","variable %s set to %s",v,resolvers.env(v) or "unknown") + logs.report("fileio","variable '%s' set to '%s'",v,resolvers.env(v) or "unknown") end end identify_own = function() end @@ -4648,10 +5436,8 @@ end local function load_cnf_file(fname) fname = resolvers.clean_path(fname) local lname = file.replacesuffix(fname,'lua') - local f = io.open(lname) - if f then -- this will go - f:close() - local dname = file.dirname(fname) + if lfs.isfile(lname) then + local dname = file.dirname(fname) -- fname ? if not instance.configuration[dname] then resolvers.load_data(dname,'configuration',lname and file.basename(lname)) instance.order[#instance.order+1] = instance.configuration[dname] @@ -4659,8 +5445,8 @@ local function load_cnf_file(fname) else f = io.open(fname) if f then - if trace_verbose then - logs.report("fileio","loading %s", fname) + if trace_locating then + logs.report("fileio","loading configuration file %s", fname) end local line, data, n, k, v local dname = file.dirname(fname) @@ -4694,14 +5480,16 @@ local function load_cnf_file(fname) end end f:close() - elseif trace_verbose then - logs.report("fileio","skipping %s", fname) + elseif trace_locating then + logs.report("fileio","skipping configuration file '%s'", fname) end end end local function collapse_cnf_data() -- potential optimization: pass start index (setup and configuration are shared) - for _,c in ipairs(instance.order) do + local order = instance.order + for i=1,#order do + local c = order[i] for k,v in next, c do if not instance.variables[k] then if instance.environment[k] then @@ -4717,19 +5505,24 @@ end function resolvers.load_cnf() local function loadoldconfigdata() - for _, fname in ipairs(instance.cnffiles) do - load_cnf_file(fname) + local cnffiles = instance.cnffiles + for i=1,#cnffiles do + load_cnf_file(cnffiles[i]) end end -- instance.cnffiles contain complete names now ! + -- we still use a funny mix of cnf and new but soon + -- we will switch to lua exclusively as we only use + -- the file to collect the tree roots if #instance.cnffiles == 0 then - if trace_verbose then + if trace_locating then logs.report("fileio","no cnf files found (TEXMFCNF may not be set/known)") end else - instance.rootpath = instance.cnffiles[1] - for k,fname in ipairs(instance.cnffiles) do - instance.cnffiles[k] = file.collapse_path(gsub(fname,"\\",'/')) + local cnffiles = instance.cnffiles + instance.rootpath = cnffiles[1] + for k=1,#cnffiles do + instance.cnffiles[k] = file.collapse_path(cnffiles[k]) end for i=1,3 do instance.rootpath = file.dirname(instance.rootpath) @@ -4757,8 +5550,9 @@ function resolvers.load_lua() -- yet harmless else instance.rootpath = instance.luafiles[1] - for k,fname in ipairs(instance.luafiles) do - instance.luafiles[k] = file.collapse_path(gsub(fname,"\\",'/')) + local luafiles = instance.luafiles + for k=1,#luafiles do + instance.luafiles[k] = file.collapse_path(luafiles[k]) end for i=1,3 do instance.rootpath = file.dirname(instance.rootpath) @@ -4790,14 +5584,14 @@ end function resolvers.append_hash(type,tag,name) if trace_locating then - logs.report("fileio","= hash append: %s",tag) + logs.report("fileio","hash '%s' appended",tag) end insert(instance.hashes, { ['type']=type, ['tag']=tag, ['name']=name } ) end function resolvers.prepend_hash(type,tag,name) if trace_locating then - logs.report("fileio","= hash prepend: %s",tag) + logs.report("fileio","hash '%s' prepended",tag) end insert(instance.hashes, 1, { ['type']=type, ['tag']=tag, ['name']=name } ) end @@ -4821,9 +5615,11 @@ end -- locators function resolvers.locatelists() - for _, path in ipairs(resolvers.clean_path_list('TEXMF')) do - if trace_verbose then - logs.report("fileio","locating list of %s",path) + local texmfpaths = resolvers.clean_path_list('TEXMF') + for i=1,#texmfpaths do + local path = texmfpaths[i] + if trace_locating then + logs.report("fileio","locating list of '%s'",path) end resolvers.locatedatabase(file.collapse_path(path)) end @@ -4836,11 +5632,11 @@ end function resolvers.locators.tex(specification) if specification and specification ~= '' and lfs.isdir(specification) then if trace_locating then - logs.report("fileio",'! tex locator found: %s',specification) + logs.report("fileio","tex locator '%s' found",specification) end resolvers.append_hash('file',specification,filename) elseif trace_locating then - logs.report("fileio",'? tex locator not found: %s',specification) + logs.report("fileio","tex locator '%s' not found",specification) end end @@ -4854,7 +5650,9 @@ function resolvers.loadfiles() instance.loaderror = false instance.files = { } if not instance.renewcache then - for _, hash in ipairs(instance.hashes) do + local hashes = instance.hashes + for k=1,#hashes do + local hash = hashes[k] resolvers.hashdatabase(hash.tag,hash.name) if instance.loaderror then break end end @@ -4868,8 +5666,9 @@ end -- generators: function resolvers.loadlists() - for _, hash in ipairs(instance.hashes) do - resolvers.generatedatabase(hash.tag) + local hashes = instance.hashes + for i=1,#hashes do + resolvers.generatedatabase(hashes[i].tag) end end @@ -4881,10 +5680,27 @@ end local weird = lpeg.P(".")^1 + lpeg.anywhere(lpeg.S("~`!#$%^&*()={}[]:;\"\'||<>,?\n\r\t")) +--~ local l_forbidden = lpeg.S("~`!#$%^&*()={}[]:;\"\'||\\/<>,?\n\r\t") +--~ local l_confusing = lpeg.P(" ") +--~ local l_character = lpeg.patterns.utf8 +--~ local l_dangerous = lpeg.P(".") + +--~ local l_normal = (l_character - l_forbidden - l_confusing - l_dangerous) * (l_character - l_forbidden - l_confusing^2)^0 * lpeg.P(-1) +--~ ----- l_normal = l_normal * lpeg.Cc(true) + lpeg.Cc(false) + +--~ local function test(str) +--~ print(str,lpeg.match(l_normal,str)) +--~ end +--~ test("ヒラギノ明朝 Pro W3") +--~ test("..ヒラギノ明朝 Pro W3") +--~ test(":ヒラギノ明朝 Pro W3;") +--~ test("ヒラギノ明朝 /Pro W3;") +--~ test("ヒラギノ明朝 Pro W3") + function resolvers.generators.tex(specification) local tag = specification - if trace_verbose then - logs.report("fileio","scanning path %s",specification) + if trace_locating then + logs.report("fileio","scanning path '%s'",specification) end instance.files[tag] = { } local files = instance.files[tag] @@ -4900,7 +5716,8 @@ function resolvers.generators.tex(specification) full = spec end for name in directory(full) do - if not weird:match(name) then + if not lpegmatch(weird,name) then + -- if lpegmatch(l_normal,name) then local mode = attributes(full..name,'mode') if mode == 'file' then if path then @@ -4933,7 +5750,7 @@ function resolvers.generators.tex(specification) end end action() - if trace_verbose then + if trace_locating then logs.report("fileio","%s files found on %s directories with %s uppercase remappings",n,m,r) end end @@ -4948,11 +5765,48 @@ end -- we join them and split them after the expansion has taken place. This -- is more convenient. +--~ local checkedsplit = string.checkedsplit + +local cache = { } + +local splitter = lpeg.Ct(lpeg.splitat(lpeg.S(os.type == "windows" and ";" or ":;"))) + +local function split_kpse_path(str) -- beware, this can be either a path or a {specification} + local found = cache[str] + if not found then + if str == "" then + found = { } + else + str = gsub(str,"\\","/") +--~ local split = (find(str,";") and checkedsplit(str,";")) or checkedsplit(str,io.pathseparator) +local split = lpegmatch(splitter,str) + found = { } + for i=1,#split do + local s = split[i] + if not find(s,"^{*unset}*") then + found[#found+1] = s + end + end + if trace_expansions then + logs.report("fileio","splitting path specification '%s'",str) + for k=1,#found do + logs.report("fileio","% 4i: %s",k,found[k]) + end + end + cache[str] = found + end + end + return found +end + +resolvers.split_kpse_path = split_kpse_path + function resolvers.splitconfig() - for i,c in ipairs(instance) do - for k,v in pairs(c) do + for i=1,#instance do + local c = instance[i] + for k,v in next, c do if type(v) == 'string' then - local t = file.split_path(v) + local t = split_kpse_path(v) if #t > 1 then c[k] = t end @@ -4962,21 +5816,25 @@ function resolvers.splitconfig() end function resolvers.joinconfig() - for i,c in ipairs(instance.order) do - for k,v in pairs(c) do -- ipairs? + local order = instance.order + for i=1,#order do + local c = order[i] + for k,v in next, c do -- indexed? if type(v) == 'table' then c[k] = file.join_path(v) end end end end + function resolvers.split_path(str) if type(str) == 'table' then return str else - return file.split_path(str) + return split_kpse_path(str) end end + function resolvers.join_path(str) if type(str) == 'table' then return file.join_path(str) @@ -4988,8 +5846,9 @@ end function resolvers.splitexpansions() local ie = instance.expansions for k,v in next, ie do - local t, h = { }, { } - for _,vv in ipairs(file.split_path(v)) do + local t, h, p = { }, { }, split_kpse_path(v) + for kk=1,#p do + local vv = p[kk] if vv ~= "" and not h[vv] then t[#t+1] = vv h[vv] = true @@ -5036,11 +5895,15 @@ function resolvers.serialize(files) end t[#t+1] = "return {" if instance.sortdata then - for _, k in pairs(sortedkeys(files)) do -- ipairs + local sortedfiles = sortedkeys(files) + for i=1,#sortedfiles do + local k = sortedfiles[i] local fk = files[k] if type(fk) == 'table' then t[#t+1] = "\t['" .. k .. "']={" - for _, kk in pairs(sortedkeys(fk)) do -- ipairs + local sortedfk = sortedkeys(fk) + for j=1,#sortedfk do + local kk = sortedfk[j] t[#t+1] = dump(kk,fk[kk],"\t\t") end t[#t+1] = "\t}," @@ -5065,12 +5928,18 @@ function resolvers.serialize(files) return concat(t,"\n") end +local data_state = { } + +function resolvers.data_state() + return data_state or { } +end + function resolvers.save_data(dataname, makename) -- untested without cache overload for cachename, files in next, instance[dataname] do local name = (makename or file.join)(cachename,dataname) local luaname, lucname = name .. ".lua", name .. ".luc" - if trace_verbose then - logs.report("fileio","preparing %s for %s",dataname,cachename) + if trace_locating then + logs.report("fileio","preparing '%s' for '%s'",dataname,cachename) end for k, v in next, files do if type(v) == "table" and #v == 1 then @@ -5084,24 +5953,25 @@ function resolvers.save_data(dataname, makename) -- untested without cache overl date = os.date("%Y-%m-%d"), time = os.date("%H:%M:%S"), content = files, + uuid = os.uuid(), } local ok = io.savedata(luaname,resolvers.serialize(data)) if ok then - if trace_verbose then - logs.report("fileio","%s saved in %s",dataname,luaname) + if trace_locating then + logs.report("fileio","'%s' saved in '%s'",dataname,luaname) end if utils.lua.compile(luaname,lucname,false,true) then -- no cleanup but strip - if trace_verbose then - logs.report("fileio","%s compiled to %s",dataname,lucname) + if trace_locating then + logs.report("fileio","'%s' compiled to '%s'",dataname,lucname) end else - if trace_verbose then - logs.report("fileio","compiling failed for %s, deleting file %s",dataname,lucname) + if trace_locating then + logs.report("fileio","compiling failed for '%s', deleting file '%s'",dataname,lucname) end os.remove(lucname) end - elseif trace_verbose then - logs.report("fileio","unable to save %s in %s (access error)",dataname,luaname) + elseif trace_locating then + logs.report("fileio","unable to save '%s' in '%s' (access error)",dataname,luaname) end end end @@ -5113,19 +5983,20 @@ function resolvers.load_data(pathname,dataname,filename,makename) -- untested wi if blob then local data = blob() if data and data.content and data.type == dataname and data.version == resolvers.cacheversion then - if trace_verbose then - logs.report("fileio","loading %s for %s from %s",dataname,pathname,filename) + data_state[#data_state+1] = data.uuid + if trace_locating then + logs.report("fileio","loading '%s' for '%s' from '%s'",dataname,pathname,filename) end instance[dataname][pathname] = data.content else - if trace_verbose then - logs.report("fileio","skipping %s for %s from %s",dataname,pathname,filename) + if trace_locating then + logs.report("fileio","skipping '%s' for '%s' from '%s'",dataname,pathname,filename) end instance[dataname][pathname] = { } instance.loaderror = true end - elseif trace_verbose then - logs.report("fileio","skipping %s for %s from %s",dataname,pathname,filename) + elseif trace_locating then + logs.report("fileio","skipping '%s' for '%s' from '%s'",dataname,pathname,filename) end end @@ -5144,15 +6015,17 @@ function resolvers.resetconfig() end function resolvers.loadnewconfig() - for _, cnf in ipairs(instance.luafiles) do + local luafiles = instance.luafiles + for i=1,#luafiles do + local cnf = luafiles[i] local pathname = file.dirname(cnf) local filename = file.join(pathname,resolvers.luaname) local blob = loadfile(filename) if blob then local data = blob() if data then - if trace_verbose then - logs.report("fileio","loading configuration file %s",filename) + if trace_locating then + logs.report("fileio","loading configuration file '%s'",filename) end if true then -- flatten to variable.progname @@ -5173,14 +6046,14 @@ function resolvers.loadnewconfig() instance['setup'][pathname] = data end else - if trace_verbose then - logs.report("fileio","skipping configuration file %s",filename) + if trace_locating then + logs.report("fileio","skipping configuration file '%s'",filename) end instance['setup'][pathname] = { } instance.loaderror = true end - elseif trace_verbose then - logs.report("fileio","skipping configuration file %s",filename) + elseif trace_locating then + logs.report("fileio","skipping configuration file '%s'",filename) end instance.order[#instance.order+1] = instance.setup[pathname] if instance.loaderror then break end @@ -5189,7 +6062,9 @@ end function resolvers.loadoldconfig() if not instance.renewcache then - for _, cnf in ipairs(instance.cnffiles) do + local cnffiles = instance.cnffiles + for i=1,#cnffiles do + local cnf = cnffiles[i] local dname = file.dirname(cnf) resolvers.load_data(dname,'configuration') instance.order[#instance.order+1] = instance.configuration[dname] @@ -5379,7 +6254,7 @@ end function resolvers.expanded_path_list(str) if not str then - return ep or { } + return ep or { } -- ep ? elseif instance.savelists then -- engine+progname hash str = gsub(str,"%$","") @@ -5397,9 +6272,9 @@ end function resolvers.expanded_path_list_from_var(str) -- brrr local tmp = resolvers.var_of_format_or_suffix(gsub(str,"%$","")) if tmp ~= "" then - return resolvers.expanded_path_list(str) - else return resolvers.expanded_path_list(tmp) + else + return resolvers.expanded_path_list(str) end end @@ -5446,9 +6321,9 @@ function resolvers.isreadable.file(name) local readable = lfs.isfile(name) -- brrr if trace_detail then if readable then - logs.report("fileio","+ readable: %s",name) + logs.report("fileio","file '%s' is readable",name) else - logs.report("fileio","- readable: %s", name) + logs.report("fileio","file '%s' is not readable", name) end end return readable @@ -5464,7 +6339,7 @@ local function collect_files(names) for k=1,#names do local fname = names[k] if trace_detail then - logs.report("fileio","? blobpath asked: %s",fname) + logs.report("fileio","checking name '%s'",fname) end local bname = file.basename(fname) local dname = file.dirname(fname) @@ -5480,7 +6355,7 @@ local function collect_files(names) local files = blobpath and instance.files[blobpath] if files then if trace_detail then - logs.report("fileio",'? blobpath do: %s (%s)',blobpath,bname) + logs.report("fileio","deep checking '%s' (%s)",blobpath,bname) end local blobfile = files[bname] if not blobfile then @@ -5514,7 +6389,7 @@ local function collect_files(names) end end elseif trace_locating then - logs.report("fileio",'! blobpath no: %s (%s)',blobpath,bname) + logs.report("fileio","no match in '%s' (%s)",blobpath,bname) end end end @@ -5564,14 +6439,13 @@ end local function collect_instance_files(filename,collected) -- todo : plugin (scanners, checkers etc) local result = collected or { } local stamp = nil - filename = file.collapse_path(filename) -- elsewhere - filename = file.collapse_path(gsub(filename,"\\","/")) -- elsewhere + filename = file.collapse_path(filename) -- speed up / beware: format problem if instance.remember then stamp = filename .. "--" .. instance.engine .. "--" .. instance.progname .. "--" .. instance.format if instance.found[stamp] then if trace_locating then - logs.report("fileio",'! remembered: %s',filename) + logs.report("fileio","remembering file '%s'",filename) end return instance.found[stamp] end @@ -5579,7 +6453,7 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan if not dangerous[instance.format or "?"] then if resolvers.isreadable.file(filename) then if trace_detail then - logs.report("fileio",'= found directly: %s',filename) + logs.report("fileio","file '%s' found directly",filename) end instance.found[stamp] = { filename } return { filename } @@ -5587,13 +6461,13 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan end if find(filename,'%*') then if trace_locating then - logs.report("fileio",'! wildcard: %s', filename) + logs.report("fileio","checking wildcard '%s'", filename) end result = resolvers.find_wildcard_files(filename) elseif file.is_qualified_path(filename) then if resolvers.isreadable.file(filename) then if trace_locating then - logs.report("fileio",'! qualified: %s', filename) + logs.report("fileio","qualified name '%s'", filename) end result = { filename } else @@ -5603,7 +6477,7 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan forcedname = filename .. ".tex" if resolvers.isreadable.file(forcedname) then if trace_locating then - logs.report("fileio",'! no suffix, forcing standard filetype: tex') + logs.report("fileio","no suffix, forcing standard filetype 'tex'") end result, ok = { forcedname }, true end @@ -5613,7 +6487,7 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan forcedname = filename .. "." .. s if resolvers.isreadable.file(forcedname) then if trace_locating then - logs.report("fileio",'! no suffix, forcing format filetype: %s', s) + logs.report("fileio","no suffix, forcing format filetype '%s'", s) end result, ok = { forcedname }, true break @@ -5625,7 +6499,7 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan -- try to find in tree (no suffix manipulation), here we search for the -- matching last part of the name local basename = file.basename(filename) - local pattern = (filename .. "$"):gsub("([%.%-])","%%%1") + local pattern = gsub(filename .. "$","([%.%-])","%%%1") local savedformat = instance.format local format = savedformat or "" if format == "" then @@ -5635,19 +6509,21 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan instance.format = "othertextfiles" -- kind of everything, maybe texinput is better end -- - local resolved = collect_instance_files(basename) - if #result == 0 then - local lowered = lower(basename) - if filename ~= lowered then - resolved = collect_instance_files(lowered) + if basename ~= filename then + local resolved = collect_instance_files(basename) + if #result == 0 then + local lowered = lower(basename) + if filename ~= lowered then + resolved = collect_instance_files(lowered) + end end - end - resolvers.format = savedformat - -- - for r=1,#resolved do - local rr = resolved[r] - if rr:find(pattern) then - result[#result+1], ok = rr, true + resolvers.format = savedformat + -- + for r=1,#resolved do + local rr = resolved[r] + if find(rr,pattern) then + result[#result+1], ok = rr, true + end end end -- a real wildcard: @@ -5656,14 +6532,14 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan -- local filelist = collect_files({basename}) -- for f=1,#filelist do -- local ff = filelist[f][3] or "" - -- if ff:find(pattern) then + -- if find(ff,pattern) then -- result[#result+1], ok = ff, true -- end -- end -- end end if not ok and trace_locating then - logs.report("fileio",'? qualified: %s', filename) + logs.report("fileio","qualified name '%s'", filename) end end else @@ -5682,12 +6558,12 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan wantedfiles[#wantedfiles+1] = forcedname filetype = resolvers.format_of_suffix(forcedname) if trace_locating then - logs.report("fileio",'! forcing filetype: %s',filetype) + logs.report("fileio","forcing filetype '%s'",filetype) end else filetype = resolvers.format_of_suffix(filename) if trace_locating then - logs.report("fileio",'! using suffix based filetype: %s',filetype) + logs.report("fileio","using suffix based filetype '%s'",filetype) end end else @@ -5699,7 +6575,7 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan end filetype = instance.format if trace_locating then - logs.report("fileio",'! using given filetype: %s',filetype) + logs.report("fileio","using given filetype '%s'",filetype) end end local typespec = resolvers.variable_of_format(filetype) @@ -5707,9 +6583,7 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan if not pathlist or #pathlist == 0 then -- no pathlist, access check only / todo == wildcard if trace_detail then - logs.report("fileio",'? filename: %s',filename) - logs.report("fileio",'? filetype: %s',filetype or '?') - logs.report("fileio",'? wanted files: %s',concat(wantedfiles," | ")) + logs.report("fileio","checking filename '%s', filetype '%s', wanted files '%s'",filename, filetype or '?',concat(wantedfiles," | ")) end for k=1,#wantedfiles do local fname = wantedfiles[k] @@ -5730,36 +6604,59 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan else -- list search local filelist = collect_files(wantedfiles) - local doscan, recurse + local dirlist = { } + if filelist then + for i=1,#filelist do + dirlist[i] = file.dirname(filelist[i][2]) .. "/" + end + end if trace_detail then - logs.report("fileio",'? filename: %s',filename) + logs.report("fileio","checking filename '%s'",filename) end -- a bit messy ... esp the doscan setting here + local doscan for k=1,#pathlist do local path = pathlist[k] if find(path,"^!!") then doscan = false else doscan = true end - if find(path,"//$") then recurse = true else recurse = false end local pathname = gsub(path,"^!+", '') done = false -- using file list - if filelist and not (done and not instance.allresults) and recurse then - -- compare list entries with permitted pattern - pathname = gsub(pathname,"([%-%.])","%%%1") -- this also influences - pathname = gsub(pathname,"/+$", '/.*') -- later usage of pathname - pathname = gsub(pathname,"//", '/.-/') -- not ok for /// but harmless - local expr = "^" .. pathname + if filelist then + local expression + -- compare list entries with permitted pattern -- /xx /xx// + if not find(pathname,"/$") then + expression = pathname .. "/" + else + expression = pathname + end + expression = gsub(expression,"([%-%.])","%%%1") -- this also influences + expression = gsub(expression,"//+$", '/.*') -- later usage of pathname + expression = gsub(expression,"//", '/.-/') -- not ok for /// but harmless + expression = "^" .. expression .. "$" + if trace_detail then + logs.report("fileio","using pattern '%s' for path '%s'",expression,pathname) + end for k=1,#filelist do local fl = filelist[k] local f = fl[2] - if find(f,expr) then - if trace_detail then - logs.report("fileio",'= found in hash: %s',f) - end + local d = dirlist[k] + if find(d,expression) then --- todo, test for readable result[#result+1] = fl[3] resolvers.register_in_trees(f) -- for tracing used files done = true - if not instance.allresults then break end + if instance.allresults then + if trace_detail then + logs.report("fileio","match in hash for file '%s' on path '%s', continue scanning",f,d) + end + else + if trace_detail then + logs.report("fileio","match in hash for file '%s' on path '%s', quit scanning",f,d) + end + break + end + elseif trace_detail then + logs.report("fileio","no match in hash for file '%s' on path '%s'",f,d) end end end @@ -5775,7 +6672,7 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan local fname = file.join(ppname,w) if resolvers.isreadable.file(fname) then if trace_detail then - logs.report("fileio",'= found by scanning: %s',fname) + logs.report("fileio","found '%s' by scanning",fname) end result[#result+1] = fname done = true @@ -5838,7 +6735,7 @@ function resolvers.find_given_files(filename) local hashes = instance.hashes for k=1,#hashes do local hash = hashes[k] - local files = instance.files[hash.tag] + local files = instance.files[hash.tag] or { } local blist = files[bname] if not blist then local rname = "remap:"..bname @@ -5948,9 +6845,9 @@ function resolvers.load(option) statistics.starttiming(instance) resolvers.resetconfig() resolvers.identify_cnf() - resolvers.load_lua() + resolvers.load_lua() -- will become the new method resolvers.expand_variables() - resolvers.load_cnf() + resolvers.load_cnf() -- will be skipped when we have a lua file resolvers.expand_variables() if option ~= "nofiles" then resolvers.load_hash() @@ -5962,22 +6859,23 @@ end function resolvers.for_files(command, files, filetype, mustexist) if files and #files > 0 then local function report(str) - if trace_verbose then + if trace_locating then logs.report("fileio",str) -- has already verbose else print(str) end end - if trace_verbose then - report('') + if trace_locating then + report('') -- ? end - for _, file in ipairs(files) do + for f=1,#files do + local file = files[f] local result = command(file,filetype,mustexist) if type(result) == 'string' then report(result) else - for _,v in ipairs(result) do - report(v) + for i=1,#result do + report(result[i]) -- could be unpack end end end @@ -6024,18 +6922,19 @@ end function table.sequenced(t,sep) -- temp here local s = { } - for k, v in pairs(t) do -- pairs? - s[#s+1] = k .. "=" .. v + for k, v in next, t do -- indexed? + s[#s+1] = k .. "=" .. tostring(v) end return concat(s, sep or " | ") end function resolvers.methodhandler(what, filename, filetype) -- ... + filename = file.collapse_path(filename) local specification = (type(filename) == "string" and resolvers.splitmethod(filename)) or filename -- no or { }, let it bomb local scheme = specification.scheme if resolvers[what][scheme] then if trace_locating then - logs.report("fileio",'= handler: %s -> %s -> %s',specification.original,what,table.sequenced(specification)) + logs.report("fileio","handler '%s' -> '%s' -> '%s'",specification.original,what,table.sequenced(specification)) end return resolvers[what][scheme](filename,filetype) -- todo: specification else @@ -6055,8 +6954,9 @@ function resolvers.clean_path(str) end function resolvers.do_with_path(name,func) - for _, v in pairs(resolvers.expanded_path_list(name)) do -- pairs? - func("^"..resolvers.clean_path(v)) + local pathlist = resolvers.expanded_path_list(name) + for i=1,#pathlist do + func("^"..resolvers.clean_path(pathlist[i])) end end @@ -6065,7 +6965,9 @@ function resolvers.do_with_var(name,func) end function resolvers.with_files(pattern,handle) - for _, hash in ipairs(instance.hashes) do + local hashes = instance.hashes + for i=1,#hashes do + local hash = hashes[i] local blobpath = hash.tag local blobtype = hash.type if blobpath then @@ -6080,7 +6982,7 @@ function resolvers.with_files(pattern,handle) if type(v) == "string" then handle(blobtype,blobpath,v,k) else - for _,vv in pairs(v) do -- ipairs? + for _,vv in next, v do -- indexed handle(blobtype,blobpath,vv,k) end end @@ -6092,7 +6994,7 @@ function resolvers.with_files(pattern,handle) end function resolvers.locate_format(name) - local barename, fmtname = name:gsub("%.%a+$",""), "" + local barename, fmtname = gsub(name,"%.%a+$",""), "" if resolvers.usecache then local path = file.join(caches.setpath("formats")) -- maybe platform fmtname = file.join(path,barename..".fmt") or "" @@ -6140,7 +7042,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-tmp'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -6164,7 +7066,7 @@ luatools with a recache feature.</p> local format, lower, gsub = string.format, string.lower, string.gsub -local trace_cache = false trackers.register("resolvers.cache", function(v) trace_cache = v end) +local trace_cache = false trackers.register("resolvers.cache", function(v) trace_cache = v end) -- not used yet caches = caches or { } @@ -6251,7 +7153,8 @@ function caches.setpath(...) caches.path = '.' end caches.path = resolvers.clean_path(caches.path) - if not table.is_empty({...}) then + local dirs = { ... } + if #dirs > 0 then local pth = dir.mkdirs(caches.path,...) return pth end @@ -6297,6 +7200,7 @@ function caches.savedata(filepath,filename,data,raw) if raw then reduce, simplify = false, false end + data.cache_uuid = os.uuid() if caches.direct then file.savedata(tmaname, table.serialize(data,'return',false,true,false)) -- no hex else @@ -6322,7 +7226,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-inp'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -6343,7 +7247,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-out'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -6359,7 +7263,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-con'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -6370,8 +7274,6 @@ local format, lower, gsub = string.format, string.lower, string.gsub local trace_cache = false trackers.register("resolvers.cache", function(v) trace_cache = v end) local trace_containers = false trackers.register("resolvers.containers", function(v) trace_containers = v end) local trace_storage = false trackers.register("resolvers.storage", function(v) trace_storage = v end) -local trace_verbose = false trackers.register("resolvers.verbose", function(v) trace_verbose = v end) -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v trackers.enable("resolvers.verbose") end) --[[ldx-- <p>Once we found ourselves defining similar cache constructs @@ -6435,7 +7337,7 @@ end function containers.is_valid(container, name) if name and name ~= "" then local storage = container.storage[name] - return storage and not table.is_empty(storage) and storage.cache_version == container.version + return storage and storage.cache_version == container.version else return false end @@ -6487,16 +7389,15 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-use'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -local format, lower, gsub = string.format, string.lower, string.gsub +local format, lower, gsub, find = string.format, string.lower, string.gsub, string.find -local trace_verbose = false trackers.register("resolvers.verbose", function(v) trace_verbose = v end) -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v trackers.enable("resolvers.verbose") end) +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) -- since we want to use the cache instead of the tree, we will now -- reimplement the saver. @@ -6540,19 +7441,20 @@ resolvers.automounted = resolvers.automounted or { } function resolvers.automount(usecache) local mountpaths = resolvers.clean_path_list(resolvers.expansion('TEXMFMOUNT')) - if table.is_empty(mountpaths) and usecache then + if (not mountpaths or #mountpaths == 0) and usecache then mountpaths = { caches.setpath("mount") } end - if not table.is_empty(mountpaths) then + if mountpaths and #mountpaths > 0 then statistics.starttiming(resolvers.instance) - for k, root in pairs(mountpaths) do + for k=1,#mountpaths do + local root = mountpaths[k] local f = io.open(root.."/url.tmi") if f then for line in f:lines() do if line then - if line:find("^[%%#%-]") then -- or %W + if find(line,"^[%%#%-]") then -- or %W -- skip - elseif line:find("^zip://") then + elseif find(line,"^zip://") then if trace_locating then logs.report("fileio","mounting %s",line) end @@ -6597,11 +7499,13 @@ function statistics.check_fmt_status(texname) local luv = dofile(luvname) if luv and luv.sourcefile then local sourcehash = md5.hex(io.loaddata(resolvers.find_file(luv.sourcefile)) or "unknown") - if luv.enginebanner and luv.enginebanner ~= enginebanner then - return "engine mismatch" + local luvbanner = luv.enginebanner or "?" + if luvbanner ~= enginebanner then + return string.format("engine mismatch (luv:%s <> bin:%s)",luvbanner,enginebanner) end - if luv.sourcehash and luv.sourcehash ~= sourcehash then - return "source mismatch" + local luvhash = luv.sourcehash or "?" + if luvhash ~= sourcehash then + return string.format("source mismatch (luv:%s <> bin:%s)",luvhash,sourcehash) end else return "invalid status file" @@ -6727,7 +7631,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-aux'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -6735,47 +7639,47 @@ if not modules then modules = { } end modules ['data-aux'] = { local find = string.find -local trace_verbose = false trackers.register("resolvers.verbose", function(v) trace_verbose = v end) +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) function resolvers.update_script(oldname,newname) -- oldname -> own.name, not per se a suffix local scriptpath = "scripts/context/lua" newname = file.addsuffix(newname,"lua") local oldscript = resolvers.clean_path(oldname) - if trace_verbose then + if trace_locating then logs.report("fileio","to be replaced old script %s", oldscript) end local newscripts = resolvers.find_files(newname) or { } if #newscripts == 0 then - if trace_verbose then + if trace_locating then logs.report("fileio","unable to locate new script") end else for i=1,#newscripts do local newscript = resolvers.clean_path(newscripts[i]) - if trace_verbose then + if trace_locating then logs.report("fileio","checking new script %s", newscript) end if oldscript == newscript then - if trace_verbose then + if trace_locating then logs.report("fileio","old and new script are the same") end elseif not find(newscript,scriptpath) then - if trace_verbose then + if trace_locating then logs.report("fileio","new script should come from %s",scriptpath) end elseif not (find(oldscript,file.removesuffix(newname).."$") or find(oldscript,newname.."$")) then - if trace_verbose then + if trace_locating then logs.report("fileio","invalid new script name") end else local newdata = io.loaddata(newscript) if newdata then - if trace_verbose then + if trace_locating then logs.report("fileio","old script content replaced by new content") end io.savedata(oldscript,newdata) break - elseif trace_verbose then + elseif trace_locating then logs.report("fileio","unable to load new script") end end @@ -6790,7 +7694,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-lst'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -6814,7 +7718,9 @@ local function list(list,report) local instance = resolvers.instance local pat = upper(pattern or "","") local report = report or texio.write_nl - for _,key in pairs(table.sortedkeys(list)) do + local sorted = table.sortedkeys(list) + for i=1,#sorted do + local key = sorted[i] if instance.pattern == "" or find(upper(key),pat) then if instance.kpseonly then if instance.kpsevars[key] then @@ -6833,11 +7739,14 @@ function resolvers.listers.expansions() list(resolvers.instance.expansions) end function resolvers.listers.configurations(report) local report = report or texio.write_nl local instance = resolvers.instance - for _,key in ipairs(table.sortedkeys(instance.kpsevars)) do + local sorted = table.sortedkeys(instance.kpsevars) + for i=1,#sorted do + local key = sorted[i] if not instance.pattern or (instance.pattern=="") or find(key,instance.pattern) then report(format("%s\n",key)) - for i,c in ipairs(instance.order) do - local str = c[key] + local order = instance.order + for i=1,#order do + local str = order[i][key] if str then report(format("\t%s\t%s",i,str)) end @@ -6943,7 +7852,7 @@ if not resolvers then os.exit() end -logs.setprogram('LuaTools',"TDS Management Tool 1.31",environment.arguments["verbose"] or false) +logs.setprogram('LuaTools',"TDS Management Tool 1.32",environment.arguments["verbose"] or false) local instance = resolvers.reset() @@ -7000,6 +7909,12 @@ end if environment.arguments["trace"] then resolvers.settrace(environment.arguments["trace"]) end +local trackspec = environment.argument("trackers") or environment.argument("track") + +if trackspec then + trackers.enable(trackspec) +end + runners = runners or { } messages = messages or { } @@ -7033,6 +7948,7 @@ messages.help = [[ --engine=str target engine --progname=str format or backend --pattern=str filter variables +--trackers=list enable given trackers ]] function runners.make_format(texname) @@ -7091,8 +8007,9 @@ function runners.make_format(texname) logs.simple("using uncompiled initialization file: %s",luaname) end else - for _, v in pairs({instance.luaname, instance.progname, barename}) do - v = string.gsub(v..".lua","%.lua%.lua$",".lua") + local what = { instance.luaname, instance.progname, barename } + for k=1,#what do + local v = string.gsub(what[k]..".lua","%.lua%.lua$",".lua") if v and (v ~= "") then luaname = resolvers.find_files(v)[1] or "" if luaname ~= "" then @@ -7116,7 +8033,8 @@ function runners.make_format(texname) logs.simple("using lua initialization file: %s",luaname) local mp = dir.glob(file.removesuffix(file.basename(luaname)).."-*.mem") if mp and #mp > 0 then - for _, name in ipairs(mp) do + for i=1,#mp do + local name = mp[i] logs.simple("removing related mplib format %s", file.basename(name)) os.remove(name) end diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/makempy.bat b/Master/texmf-dist/scripts/context/stubs/mswin/makempy.bat deleted file mode 100755 index d02bbf7ca7e..00000000000 --- a/Master/texmf-dist/scripts/context/stubs/mswin/makempy.bat +++ /dev/null @@ -1,5 +0,0 @@ -@echo off
-setlocal
-set ownpath=%~dp0%
-texlua "%ownpath%mtxrun.lua" --usekpse --execute makempy.pl %*
-endlocal
diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/metatex.cmd b/Master/texmf-dist/scripts/context/stubs/mswin/metatex.cmd deleted file mode 100644 index 858f28f8fd5..00000000000 --- a/Master/texmf-dist/scripts/context/stubs/mswin/metatex.cmd +++ /dev/null @@ -1,5 +0,0 @@ -@echo off -setlocal -set ownpath=%~dp0% -texlua "%ownpath%mtxrun.lua" --script metatex %* -endlocal diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/metatex.exe b/Master/texmf-dist/scripts/context/stubs/mswin/metatex.exe Binary files differnew file mode 100644 index 00000000000..2d45f27494d --- /dev/null +++ b/Master/texmf-dist/scripts/context/stubs/mswin/metatex.exe diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/mpstools.bat b/Master/texmf-dist/scripts/context/stubs/mswin/mpstools.bat deleted file mode 100755 index 8fa0c9639c2..00000000000 --- a/Master/texmf-dist/scripts/context/stubs/mswin/mpstools.bat +++ /dev/null @@ -1,5 +0,0 @@ -@echo off
-setlocal
-set ownpath=%~dp0%
-texlua "%ownpath%mtxrun.lua" --usekpse --execute mpstools.rb %*
-endlocal
diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/mptopdf.bat b/Master/texmf-dist/scripts/context/stubs/mswin/mptopdf.bat deleted file mode 100755 index 3cab9669f7e..00000000000 --- a/Master/texmf-dist/scripts/context/stubs/mswin/mptopdf.bat +++ /dev/null @@ -1,5 +0,0 @@ -@echo off
-setlocal
-set ownpath=%~dp0%
-texlua "%ownpath%mtxrun.lua" --usekpse --execute mptopdf.pl %*
-endlocal
diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/mtxrun.cmd b/Master/texmf-dist/scripts/context/stubs/mswin/mtxrun.cmd deleted file mode 100755 index 858bab87fed..00000000000 --- a/Master/texmf-dist/scripts/context/stubs/mswin/mtxrun.cmd +++ /dev/null @@ -1,5 +0,0 @@ -@echo off
-setlocal
-set ownpath=%~dp0%
-texlua "%ownpath%mtxrun.lua" %*
-endlocal
diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/mtxrun.dll b/Master/texmf-dist/scripts/context/stubs/mswin/mtxrun.dll Binary files differnew file mode 100644 index 00000000000..23e476cac49 --- /dev/null +++ b/Master/texmf-dist/scripts/context/stubs/mswin/mtxrun.dll diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/mtxrun.exe b/Master/texmf-dist/scripts/context/stubs/mswin/mtxrun.exe Binary files differnew file mode 100644 index 00000000000..745eaf22464 --- /dev/null +++ b/Master/texmf-dist/scripts/context/stubs/mswin/mtxrun.exe diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/mtxrun.lua b/Master/texmf-dist/scripts/context/stubs/mswin/mtxrun.lua index 82d1edecbc5..b99327692d7 100644 --- a/Master/texmf-dist/scripts/context/stubs/mswin/mtxrun.lua +++ b/Master/texmf-dist/scripts/context/stubs/mswin/mtxrun.lua @@ -48,13 +48,16 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-string'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -local sub, gsub, find, match, gmatch, format, char, byte, rep = string.sub, string.gsub, string.find, string.match, string.gmatch, string.format, string.char, string.byte, string.rep +local sub, gsub, find, match, gmatch, format, char, byte, rep, lower = string.sub, string.gsub, string.find, string.match, string.gmatch, string.format, string.char, string.byte, string.rep, string.lower +local lpegmatch = lpeg.match + +-- some functions may disappear as they are not used anywhere if not string.split then @@ -94,8 +97,16 @@ function string:unquote() return (gsub(self,"^([\"\'])(.*)%1$","%2")) end +--~ function string:unquote() +--~ if find(self,"^[\'\"]") then +--~ return sub(self,2,-2) +--~ else +--~ return self +--~ end +--~ end + function string:quote() -- we could use format("%q") - return '"' .. self:unquote() .. '"' + return format("%q",self) end function string:count(pattern) -- variant 3 @@ -115,12 +126,23 @@ function string:limit(n,sentinel) end end -function string:strip() - return (gsub(self,"^%s*(.-)%s*$", "%1")) +--~ function string:strip() -- the .- is quite efficient +--~ -- return match(self,"^%s*(.-)%s*$") or "" +--~ -- return match(self,'^%s*(.*%S)') or '' -- posted on lua list +--~ return find(s,'^%s*$') and '' or match(s,'^%s*(.*%S)') +--~ end + +do -- roberto's variant: + local space = lpeg.S(" \t\v\n") + local nospace = 1 - space + local stripper = space^0 * lpeg.C((space^0 * nospace^1)^0) + function string.strip(str) + return lpegmatch(stripper,str) or "" + end end function string:is_empty() - return not find(find,"%S") + return not find(self,"%S") end function string:enhance(pattern,action) @@ -154,14 +176,14 @@ if not string.characters then local function nextchar(str, index) index = index + 1 - return (index <= #str) and index or nil, str:sub(index,index) + return (index <= #str) and index or nil, sub(str,index,index) end function string:characters() return nextchar, self, 0 end local function nextbyte(str, index) index = index + 1 - return (index <= #str) and index or nil, byte(str:sub(index,index)) + return (index <= #str) and index or nil, byte(sub(str,index,index)) end function string:bytes() return nextbyte, self, 0 @@ -174,7 +196,7 @@ end function string:rpadd(n,chr) local m = n-#self if m > 0 then - return self .. self.rep(chr or " ",m) + return self .. rep(chr or " ",m) else return self end @@ -183,7 +205,7 @@ end function string:lpadd(n,chr) local m = n-#self if m > 0 then - return self.rep(chr or " ",m) .. self + return rep(chr or " ",m) .. self else return self end @@ -231,6 +253,17 @@ function string:pattesc() return (gsub(self,".",patterns_escapes)) end +local simple_escapes = { + ["-"] = "%-", + ["."] = "%.", + ["?"] = ".", + ["*"] = ".*", +} + +function string:simpleesc() + return (gsub(self,".",simple_escapes)) +end + function string:tohash() local t = { } for s in gmatch(self,"([^, ]+)") do -- lpeg @@ -242,10 +275,10 @@ end local pattern = lpeg.Ct(lpeg.C(1)^0) function string:totable() - return pattern:match(self) + return lpegmatch(pattern,self) end ---~ for _, str in ipairs { +--~ local t = { --~ "1234567123456712345671234567", --~ "a\tb\tc", --~ "aa\tbb\tcc", @@ -253,7 +286,10 @@ end --~ "aaaa\tbbbb\tcccc", --~ "aaaaa\tbbbbb\tccccc", --~ "aaaaaa\tbbbbbb\tcccccc", ---~ } do print(string.tabtospace(str)) end +--~ } +--~ for k,v do +--~ print(string.tabtospace(t[k])) +--~ end function string.tabtospace(str,tab) -- we don't handle embedded newlines @@ -261,7 +297,7 @@ function string.tabtospace(str,tab) local s = find(str,"\t") if s then if not tab then tab = 7 end -- only when found - local d = tab-(s-1)%tab + local d = tab-(s-1) % tab if d > 0 then str = gsub(str,"\t",rep(" ",d),1) else @@ -280,6 +316,25 @@ function string:compactlong() -- strips newlines and leading spaces return self end +function string:striplong() -- strips newlines and leading spaces + self = gsub(self,"^%s*","") + self = gsub(self,"[\n\r]+ *","\n") + return self +end + +function string:topattern(lowercase,strict) + if lowercase then + self = lower(self) + end + self = gsub(self,".",simple_escapes) + if self == "" then + self = ".*" + elseif strict then + self = "^" .. self .. "$" + end + return self +end + end -- of closure @@ -287,58 +342,64 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-lpeg'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -local P, S, Ct, C, Cs, Cc = lpeg.P, lpeg.S, lpeg.Ct, lpeg.C, lpeg.Cs, lpeg.Cc - ---~ l-lpeg.lua : - ---~ lpeg.digit = lpeg.R('09')^1 ---~ lpeg.sign = lpeg.S('+-')^1 ---~ lpeg.cardinal = lpeg.P(lpeg.sign^0 * lpeg.digit^1) ---~ lpeg.integer = lpeg.P(lpeg.sign^0 * lpeg.digit^1) ---~ lpeg.float = lpeg.P(lpeg.sign^0 * lpeg.digit^0 * lpeg.P('.') * lpeg.digit^1) ---~ lpeg.number = lpeg.float + lpeg.integer ---~ lpeg.oct = lpeg.P("0") * lpeg.R('07')^1 ---~ lpeg.hex = lpeg.P("0x") * (lpeg.R('09') + lpeg.R('AF'))^1 ---~ lpeg.uppercase = lpeg.P("AZ") ---~ lpeg.lowercase = lpeg.P("az") - ---~ lpeg.eol = lpeg.S('\r\n\f')^1 -- includes formfeed ---~ lpeg.space = lpeg.S(' ')^1 ---~ lpeg.nonspace = lpeg.P(1-lpeg.space)^1 ---~ lpeg.whitespace = lpeg.S(' \r\n\f\t')^1 ---~ lpeg.nonwhitespace = lpeg.P(1-lpeg.whitespace)^1 - -local hash = { } +local lpeg = require("lpeg") + +lpeg.patterns = lpeg.patterns or { } -- so that we can share +local patterns = lpeg.patterns + +local P, R, S, Ct, C, Cs, Cc, V = lpeg.P, lpeg.R, lpeg.S, lpeg.Ct, lpeg.C, lpeg.Cs, lpeg.Cc, lpeg.V +local match = lpeg.match + +local digit, sign = R('09'), S('+-') +local cr, lf, crlf = P("\r"), P("\n"), P("\r\n") +local utf8byte = R("\128\191") + +patterns.utf8byte = utf8byte +patterns.utf8one = R("\000\127") +patterns.utf8two = R("\194\223") * utf8byte +patterns.utf8three = R("\224\239") * utf8byte * utf8byte +patterns.utf8four = R("\240\244") * utf8byte * utf8byte * utf8byte + +patterns.digit = digit +patterns.sign = sign +patterns.cardinal = sign^0 * digit^1 +patterns.integer = sign^0 * digit^1 +patterns.float = sign^0 * digit^0 * P('.') * digit^1 +patterns.number = patterns.float + patterns.integer +patterns.oct = P("0") * R("07")^1 +patterns.octal = patterns.oct +patterns.HEX = P("0x") * R("09","AF")^1 +patterns.hex = P("0x") * R("09","af")^1 +patterns.hexadecimal = P("0x") * R("09","AF","af")^1 +patterns.lowercase = R("az") +patterns.uppercase = R("AZ") +patterns.letter = patterns.lowercase + patterns.uppercase +patterns.space = S(" ") +patterns.eol = S("\n\r") +patterns.spacer = S(" \t\f\v") -- + string.char(0xc2, 0xa0) if we want utf (cf mail roberto) +patterns.newline = crlf + cr + lf +patterns.nonspace = 1 - patterns.space +patterns.nonspacer = 1 - patterns.spacer +patterns.whitespace = patterns.eol + patterns.spacer +patterns.nonwhitespace = 1 - patterns.whitespace +patterns.utf8 = patterns.utf8one + patterns.utf8two + patterns.utf8three + patterns.utf8four +patterns.utfbom = P('\000\000\254\255') + P('\255\254\000\000') + P('\255\254') + P('\254\255') + P('\239\187\191') function lpeg.anywhere(pattern) --slightly adapted from website - return P { P(pattern) + 1 * lpeg.V(1) } -end - -function lpeg.startswith(pattern) --slightly adapted - return P(pattern) + return P { P(pattern) + 1 * V(1) } -- why so complex? end function lpeg.splitter(pattern, action) return (((1-P(pattern))^1)/action+1)^0 end --- variant: - ---~ local parser = lpeg.Ct(lpeg.splitat(newline)) - -local crlf = P("\r\n") -local cr = P("\r") -local lf = P("\n") -local space = S(" \t\f\v") -- + string.char(0xc2, 0xa0) if we want utf (cf mail roberto) -local newline = crlf + cr + lf -local spacing = space^0 * newline - +local spacing = patterns.spacer^0 * patterns.newline -- sort of strip local empty = spacing * Cc("") local nonempty = Cs((1-spacing)^1) * spacing^-1 local content = (empty + nonempty)^1 @@ -346,15 +407,15 @@ local content = (empty + nonempty)^1 local capture = Ct(content^0) function string:splitlines() - return capture:match(self) + return match(capture,self) end -lpeg.linebyline = content -- better make a sublibrary +patterns.textline = content ---~ local p = lpeg.splitat("->",false) print(p:match("oeps->what->more")) -- oeps what more ---~ local p = lpeg.splitat("->",true) print(p:match("oeps->what->more")) -- oeps what->more ---~ local p = lpeg.splitat("->",false) print(p:match("oeps")) -- oeps ---~ local p = lpeg.splitat("->",true) print(p:match("oeps")) -- oeps +--~ local p = lpeg.splitat("->",false) print(match(p,"oeps->what->more")) -- oeps what more +--~ local p = lpeg.splitat("->",true) print(match(p,"oeps->what->more")) -- oeps what->more +--~ local p = lpeg.splitat("->",false) print(match(p,"oeps")) -- oeps +--~ local p = lpeg.splitat("->",true) print(match(p,"oeps")) -- oeps local splitters_s, splitters_m = { }, { } @@ -364,7 +425,7 @@ local function splitat(separator,single) separator = P(separator) if single then local other, any = C((1 - separator)^0), P(1) - splitter = other * (separator * C(any^0) + "") + splitter = other * (separator * C(any^0) + "") -- ? splitters_s[separator] = splitter else local other = C((1 - separator)^0) @@ -379,15 +440,72 @@ lpeg.splitat = splitat local cache = { } +function lpeg.split(separator,str) + local c = cache[separator] + if not c then + c = Ct(splitat(separator)) + cache[separator] = c + end + return match(c,str) +end + function string:split(separator) local c = cache[separator] if not c then c = Ct(splitat(separator)) cache[separator] = c end - return c:match(self) + return match(c,self) +end + +lpeg.splitters = cache + +local cache = { } + +function lpeg.checkedsplit(separator,str) + local c = cache[separator] + if not c then + separator = P(separator) + local other = C((1 - separator)^0) + c = Ct(separator^0 * other * (separator^1 * other)^0) + cache[separator] = c + end + return match(c,str) +end + +function string:checkedsplit(separator) + local c = cache[separator] + if not c then + separator = P(separator) + local other = C((1 - separator)^0) + c = Ct(separator^0 * other * (separator^1 * other)^0) + cache[separator] = c + end + return match(c,self) end +--~ function lpeg.append(list,pp) +--~ local p = pp +--~ for l=1,#list do +--~ if p then +--~ p = p + P(list[l]) +--~ else +--~ p = P(list[l]) +--~ end +--~ end +--~ return p +--~ end + +--~ from roberto's site: + +local f1 = string.byte + +local function f2(s) local c1, c2 = f1(s,1,2) return c1 * 64 + c2 - 12416 end +local function f3(s) local c1, c2, c3 = f1(s,1,3) return (c1 * 64 + c2) * 64 + c3 - 925824 end +local function f4(s) local c1, c2, c3, c4 = f1(s,1,4) return ((c1 * 64 + c2) * 64 + c3) * 64 + c4 - 63447168 end + +patterns.utf8byte = patterns.utf8one/f1 + patterns.utf8two/f2 + patterns.utf8three/f3 + patterns.utf8four/f4 + end -- of closure @@ -395,7 +513,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-table'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -404,9 +522,58 @@ if not modules then modules = { } end modules ['l-table'] = { table.join = table.concat local concat, sort, insert, remove = table.concat, table.sort, table.insert, table.remove -local format, find, gsub, lower, dump = string.format, string.find, string.gsub, string.lower, string.dump +local format, find, gsub, lower, dump, match = string.format, string.find, string.gsub, string.lower, string.dump, string.match local getmetatable, setmetatable = getmetatable, setmetatable -local type, next, tostring, ipairs = type, next, tostring, ipairs +local type, next, tostring, tonumber, ipairs = type, next, tostring, tonumber, ipairs + +-- Starting with version 5.2 Lua no longer provide ipairs, which makes +-- sense. As we already used the for loop and # in most places the +-- impact on ConTeXt was not that large; the remaining ipairs already +-- have been replaced. In a similar fashio we also hardly used pairs. +-- +-- Just in case, we provide the fallbacks as discussed in Programming +-- in Lua (http://www.lua.org/pil/7.3.html): + +if not ipairs then + + -- for k, v in ipairs(t) do ... end + -- for k=1,#t do local v = t[k] ... end + + local function iterate(a,i) + i = i + 1 + local v = a[i] + if v ~= nil then + return i, v --, nil + end + end + + function ipairs(a) + return iterate, a, 0 + end + +end + +if not pairs then + + -- for k, v in pairs(t) do ... end + -- for k, v in next, t do ... end + + function pairs(t) + return next, t -- , nil + end + +end + +-- Also, unpack has been moved to the table table, and for compatiility +-- reasons we provide both now. + +if not table.unpack then + table.unpack = _G.unpack +elseif not unpack then + _G.unpack = table.unpack +end + +-- extra functions, some might go (when not used) function table.strip(tab) local lst = { } @@ -421,6 +588,14 @@ function table.strip(tab) return lst end +function table.keys(t) + local k = { } + for key, _ in next, t do + k[#k+1] = key + end + return k +end + local function compare(a,b) return (tostring(a) < tostring(b)) end @@ -464,7 +639,7 @@ end table.sortedkeys = sortedkeys table.sortedhashkeys = sortedhashkeys -function table.sortedpairs(t) +function table.sortedhash(t) local s = sortedhashkeys(t) -- maybe just sortedkeys local n = 0 local function kv(s) @@ -475,6 +650,8 @@ function table.sortedpairs(t) return kv, s end +table.sortedpairs = table.sortedhash + function table.append(t, list) for _,v in next, list do insert(t,v) @@ -583,7 +760,7 @@ end table.fastcopy = fastcopy table.copy = copy --- rougly: copy-loop : unpack : sub == 0.9 : 0.4 : 0.45 (so in critical apps, use unpack) +-- roughly: copy-loop : unpack : sub == 0.9 : 0.4 : 0.45 (so in critical apps, use unpack) function table.sub(t,i,j) return { unpack(t,i,j) } @@ -597,18 +774,18 @@ end -- slower than #t on indexed tables (#t only returns the size of the numerically indexed slice) -function table.is_empty(t) +function table.is_empty(t) -- obolete, use inline code instead return not t or not next(t) end -function table.one_entry(t) +function table.one_entry(t) -- obolete, use inline code instead local n = next(t) return n and not next(t,n) end -function table.starts_at(t) - return ipairs(t,1)(t,0) -end +--~ function table.starts_at(t) -- obsolete, not nice anyway +--~ return ipairs(t,1)(t,0) +--~ end function table.tohash(t,value) local h = { } @@ -686,6 +863,8 @@ end -- -- local propername = lpeg.P(lpeg.R("AZ","az","__") * lpeg.R("09","AZ","az", "__")^0 * lpeg.P(-1) ) +-- problem: there no good number_to_string converter with the best resolution + local function do_serialize(root,name,depth,level,indexed) if level > 0 then depth = depth .. " " @@ -708,8 +887,9 @@ local function do_serialize(root,name,depth,level,indexed) handle(format("%s{",depth)) end end + -- we could check for k (index) being number (cardinal) if root and next(root) then - local first, last = nil, 0 -- #root cannot be trusted here + local first, last = nil, 0 -- #root cannot be trusted here (will be ok in 5.2 when ipairs is gone) if compact then -- NOT: for k=1,#root do (we need to quit at nil) for k,v in ipairs(root) do -- can we use next? @@ -730,10 +910,10 @@ local function do_serialize(root,name,depth,level,indexed) if hexify then handle(format("%s 0x%04X,",depth,v)) else - handle(format("%s %s,",depth,v)) + handle(format("%s %s,",depth,v)) -- %.99g end elseif t == "string" then - if reduce and (find(v,"^[%-%+]?[%d]-%.?[%d+]$") == 1) then + if reduce and tonumber(v) then handle(format("%s %s,",depth,v)) else handle(format("%s %q,",depth,v)) @@ -770,29 +950,29 @@ local function do_serialize(root,name,depth,level,indexed) --~ if hexify then --~ handle(format("%s %s=0x%04X,",depth,key(k),v)) --~ else - --~ handle(format("%s %s=%s,",depth,key(k),v)) + --~ handle(format("%s %s=%s,",depth,key(k),v)) -- %.99g --~ end if type(k) == "number" then -- or find(k,"^%d+$") then if hexify then handle(format("%s [0x%04X]=0x%04X,",depth,k,v)) else - handle(format("%s [%s]=%s,",depth,k,v)) + handle(format("%s [%s]=%s,",depth,k,v)) -- %.99g end elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then if hexify then handle(format("%s %s=0x%04X,",depth,k,v)) else - handle(format("%s %s=%s,",depth,k,v)) + handle(format("%s %s=%s,",depth,k,v)) -- %.99g end else if hexify then handle(format("%s [%q]=0x%04X,",depth,k,v)) else - handle(format("%s [%q]=%s,",depth,k,v)) + handle(format("%s [%q]=%s,",depth,k,v)) -- %.99g end end elseif t == "string" then - if reduce and (find(v,"^[%-%+]?[%d]-%.?[%d+]$") == 1) then + if reduce and tonumber(v) then --~ handle(format("%s %s=%s,",depth,key(k),v)) if type(k) == "number" then -- or find(k,"^%d+$") then if hexify then @@ -1001,7 +1181,7 @@ function table.tofile(filename,root,name,reduce,noquotes,hexify) end end -local function flatten(t,f,complete) +local function flatten(t,f,complete) -- is this used? meybe a variant with next, ... for i=1,#t do local v = t[i] if type(v) == "table" then @@ -1030,6 +1210,24 @@ end table.flatten_one_level = table.unnest +-- a better one: + +local function flattened(t,f) + if not f then + f = { } + end + for k, v in next, t do + if type(v) == "table" then + flattened(v,f) + else + f[k] = v + end + end + return f +end + +table.flattened = flattened + -- the next three may disappear function table.remove_value(t,value) -- todo: n @@ -1165,7 +1363,7 @@ function table.clone(t,p) -- t is optional or nil or table elseif not t then t = { } end - setmetatable(t, { __index = function(_,key) return p[key] end }) + setmetatable(t, { __index = function(_,key) return p[key] end }) -- why not __index = p ? return t end @@ -1193,21 +1391,36 @@ function table.reverse(t) return tt end ---~ function table.keys(t) ---~ local k = { } ---~ for k,_ in next, t do ---~ k[#k+1] = k ---~ end ---~ return k ---~ end +function table.insert_before_value(t,value,extra) + for i=1,#t do + if t[i] == extra then + remove(t,i) + end + end + for i=1,#t do + if t[i] == value then + insert(t,i,extra) + return + end + end + insert(t,1,extra) +end + +function table.insert_after_value(t,value,extra) + for i=1,#t do + if t[i] == extra then + remove(t,i) + end + end + for i=1,#t do + if t[i] == value then + insert(t,i+1,extra) + return + end + end + insert(t,#t+1,extra) +end ---~ function table.keys_as_string(t) ---~ local k = { } ---~ for k,_ in next, t do ---~ k[#k+1] = k ---~ end ---~ return concat(k,"") ---~ end end -- of closure @@ -1216,13 +1429,13 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-io'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -local byte = string.byte +local byte, find, gsub = string.byte, string.find, string.gsub if string.find(os.getenv("PATH"),";") then io.fileseparator, io.pathseparator = "\\", ";" @@ -1251,7 +1464,7 @@ function io.savedata(filename,data,joiner) elseif type(data) == "function" then data(f) else - f:write(data) + f:write(data or "") end f:close() return true @@ -1380,20 +1593,21 @@ function io.ask(question,default,options) end io.write(string.format(" ")) local answer = io.read() - answer = answer:gsub("^%s*(.*)%s*$","%1") + answer = gsub(answer,"^%s*(.*)%s*$","%1") if answer == "" and default then return default elseif not options then return answer else - for _,v in pairs(options) do - if v == answer then + for k=1,#options do + if options[k] == answer then return answer end end local pattern = "^" .. answer - for _,v in pairs(options) do - if v:find(pattern) then + for k=1,#options do + local v = options[k] + if find(v,pattern) then return v end end @@ -1408,20 +1622,22 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-number'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -local format = string.format +local tostring = tostring +local format, floor, insert, match = string.format, math.floor, table.insert, string.match +local lpegmatch = lpeg.match number = number or { } -- a,b,c,d,e,f = number.toset(100101) function number.toset(n) - return (tostring(n)):match("(.?)(.?)(.?)(.?)(.?)(.?)(.?)(.?)") + return match(tostring(n),"(.?)(.?)(.?)(.?)(.?)(.?)(.?)(.?)") end function number.toevenhex(n) @@ -1447,10 +1663,21 @@ end local one = lpeg.C(1-lpeg.S(''))^1 function number.toset(n) - return one:match(tostring(n)) + return lpegmatch(one,tostring(n)) end - +function number.bits(n,zero) + local t, i = { }, (zero and 0) or 1 + while n > 0 do + local m = n % 2 + if m > 0 then + insert(t,1,i) + end + n = floor(n/2) + i = i + 1 + end + return t +end end -- of closure @@ -1459,7 +1686,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-set'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -1549,46 +1776,63 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-os'] = { version = 1.001, - comment = "companion to luat-lub.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -local find = string.find +-- maybe build io.flush in os.execute + +local find, format, gsub = string.find, string.format, string.gsub +local random, ceil = math.random, math.ceil + +local execute, spawn, exec, ioflush = os.execute, os.spawn or os.execute, os.exec or os.execute, io.flush + +function os.execute(...) ioflush() return execute(...) end +function os.spawn (...) ioflush() return spawn (...) end +function os.exec (...) ioflush() return exec (...) end function os.resultof(command) - return io.popen(command,"r"):read("*all") + ioflush() -- else messed up logging + local handle = io.popen(command,"r") + if not handle then + -- print("unknown command '".. command .. "' in os.resultof") + return "" + else + return handle:read("*all") or "" + end end -if not os.exec then os.exec = os.execute end -if not os.spawn then os.spawn = os.execute end - ---~ os.type : windows | unix (new, we already guessed os.platform) ---~ os.name : windows | msdos | linux | macosx | solaris | .. | generic (new) +--~ os.type : windows | unix (new, we already guessed os.platform) +--~ os.name : windows | msdos | linux | macosx | solaris | .. | generic (new) +--~ os.platform : extended os.name with architecture if not io.fileseparator then if find(os.getenv("PATH"),";") then - io.fileseparator, io.pathseparator, os.platform = "\\", ";", os.type or "windows" + io.fileseparator, io.pathseparator, os.type = "\\", ";", os.type or "mswin" else - io.fileseparator, io.pathseparator, os.platform = "/" , ":", os.type or "unix" + io.fileseparator, io.pathseparator, os.type = "/" , ":", os.type or "unix" end end -os.platform = os.platform or os.type or (io.pathseparator == ";" and "windows") or "unix" +os.type = os.type or (io.pathseparator == ";" and "windows") or "unix" +os.name = os.name or (os.type == "windows" and "mswin" ) or "linux" + +if os.type == "windows" then + os.libsuffix, os.binsuffix = 'dll', 'exe' +else + os.libsuffix, os.binsuffix = 'so', '' +end function os.launch(str) - if os.platform == "windows" then + if os.type == "windows" then os.execute("start " .. str) -- os.spawn ? else os.execute(str .. " &") -- os.spawn ? end end -if not os.setenv then - function os.setenv() return false end -end - if not os.times then -- utime = user time -- stime = system time @@ -1618,64 +1862,218 @@ end --~ print(os.date("%H:%M:%S",os.gettimeofday())) --~ print(os.date("%H:%M:%S",os.time())) -os.arch = os.arch or function() - local a = os.resultof("uname -m") or "linux" - os.arch = function() - return a +-- no need for function anymore as we have more clever code and helpers now +-- this metatable trickery might as well disappear + +os.resolvers = os.resolvers or { } + +local resolvers = os.resolvers + +local osmt = getmetatable(os) or { __index = function(t,k) t[k] = "unset" return "unset" end } -- maybe nil +local osix = osmt.__index + +osmt.__index = function(t,k) + return (resolvers[k] or osix)(t,k) +end + +setmetatable(os,osmt) + +if not os.setenv then + + -- we still store them but they won't be seen in + -- child processes although we might pass them some day + -- using command concatination + + local env, getenv = { }, os.getenv + + function os.setenv(k,v) + env[k] = v + end + + function os.getenv(k) + return env[k] or getenv(k) end - return a + end -local platform +-- we can use HOSTTYPE on some platforms -function os.currentplatform(name,default) - if not platform then - local name = os.name or os.platform or name -- os.name is built in, os.platform is mine - if not name then - platform = default or "linux" - elseif name == "windows" or name == "mswin" or name == "win32" or name == "msdos" then - if os.getenv("PROCESSOR_ARCHITECTURE") == "AMD64" then - platform = "mswin-64" - else - platform = "mswin" - end +local name, platform = os.name or "linux", os.getenv("MTX_PLATFORM") or "" + +local function guess() + local architecture = os.resultof("uname -m") or "" + if architecture ~= "" then + return architecture + end + architecture = os.getenv("HOSTTYPE") or "" + if architecture ~= "" then + return architecture + end + return os.resultof("echo $HOSTTYPE") or "" +end + +if platform ~= "" then + + os.platform = platform + +elseif os.type == "windows" then + + -- we could set the variable directly, no function needed here + + function os.resolvers.platform(t,k) + local platform, architecture = "", os.getenv("PROCESSOR_ARCHITECTURE") or "" + if find(architecture,"AMD64") then + platform = "mswin-64" else - local architecture = os.arch() - if name == "linux" then - if find(architecture,"x86_64") then - platform = "linux-64" - elseif find(architecture,"ppc") then - platform = "linux-ppc" - else - platform = "linux" - end - elseif name == "macosx" then - if find(architecture,"i386") then - platform = "osx-intel" - else - platform = "osx-ppc" - end - elseif name == "sunos" then - if find(architecture,"sparc") then - platform = "solaris-sparc" - else -- if architecture == 'i86pc' - platform = "solaris-intel" - end - elseif name == "freebsd" then - if find(architecture,"amd64") then - platform = "freebsd-amd64" - else - platform = "freebsd" - end - else - platform = default or name - end + platform = "mswin" end - function os.currentplatform() - return platform + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +elseif name == "linux" then + + function os.resolvers.platform(t,k) + -- we sometims have HOSTTYPE set so let's check that first + local platform, architecture = "", os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" + if find(architecture,"x86_64") then + platform = "linux-64" + elseif find(architecture,"ppc") then + platform = "linux-ppc" + else + platform = "linux" + end + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +elseif name == "macosx" then + + --[[ + Identifying the architecture of OSX is quite a mess and this + is the best we can come up with. For some reason $HOSTTYPE is + a kind of pseudo environment variable, not known to the current + environment. And yes, uname cannot be trusted either, so there + is a change that you end up with a 32 bit run on a 64 bit system. + Also, some proper 64 bit intel macs are too cheap (low-end) and + therefore not permitted to run the 64 bit kernel. + ]]-- + + function os.resolvers.platform(t,k) + -- local platform, architecture = "", os.getenv("HOSTTYPE") or "" + -- if architecture == "" then + -- architecture = os.resultof("echo $HOSTTYPE") or "" + -- end + local platform, architecture = "", os.resultof("echo $HOSTTYPE") or "" + if architecture == "" then + -- print("\nI have no clue what kind of OSX you're running so let's assume an 32 bit intel.\n") + platform = "osx-intel" + elseif find(architecture,"i386") then + platform = "osx-intel" + elseif find(architecture,"x86_64") then + platform = "osx-64" + else + platform = "osx-ppc" end + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +elseif name == "sunos" then + + function os.resolvers.platform(t,k) + local platform, architecture = "", os.resultof("uname -m") or "" + if find(architecture,"sparc") then + platform = "solaris-sparc" + else -- if architecture == 'i86pc' + platform = "solaris-intel" + end + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +elseif name == "freebsd" then + + function os.resolvers.platform(t,k) + local platform, architecture = "", os.resultof("uname -m") or "" + if find(architecture,"amd64") then + platform = "freebsd-amd64" + else + platform = "freebsd" + end + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +elseif name == "kfreebsd" then + + function os.resolvers.platform(t,k) + -- we sometims have HOSTTYPE set so let's check that first + local platform, architecture = "", os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" + if find(architecture,"x86_64") then + platform = "kfreebsd-64" + else + platform = "kfreebsd-i386" + end + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +else + + -- platform = "linux" + -- os.setenv("MTX_PLATFORM",platform) + -- os.platform = platform + + function os.resolvers.platform(t,k) + local platform = "linux" + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +end + +-- beware, we set the randomseed + +-- from wikipedia: Version 4 UUIDs use a scheme relying only on random numbers. This algorithm sets the +-- version number as well as two reserved bits. All other bits are set using a random or pseudorandom +-- data source. Version 4 UUIDs have the form xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx with hexadecimal +-- digits x and hexadecimal digits 8, 9, A, or B for y. e.g. f47ac10b-58cc-4372-a567-0e02b2c3d479. +-- +-- as we don't call this function too often there is not so much risk on repetition + +local t = { 8, 9, "a", "b" } + +function os.uuid() + return format("%04x%04x-4%03x-%s%03x-%04x-%04x%04x%04x", + random(0xFFFF),random(0xFFFF), + random(0x0FFF), + t[ceil(random(4))] or 8,random(0x0FFF), + random(0xFFFF), + random(0xFFFF),random(0xFFFF),random(0xFFFF) + ) +end + +local d + +function os.timezone(delta) + d = d or tonumber(tonumber(os.date("%H")-os.date("!%H"))) + if delta then + if d > 0 then + return format("+%02i:00",d) + else + return format("-%02i:00",-d) + end + else + return 1 end - return platform end @@ -1685,7 +2083,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-file'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -1696,14 +2094,17 @@ if not modules then modules = { } end modules ['l-file'] = { file = file or { } local concat = table.concat -local find, gmatch, match, gsub = string.find, string.gmatch, string.match, string.gsub +local find, gmatch, match, gsub, sub, char = string.find, string.gmatch, string.match, string.gsub, string.sub, string.char +local lpegmatch = lpeg.match function file.removesuffix(filename) return (gsub(filename,"%.[%a%d]+$","")) end function file.addsuffix(filename, suffix) - if not find(filename,"%.[%a%d]+$") then + if not suffix or suffix == "" then + return filename + elseif not find(filename,"%.[%a%d]+$") then return filename .. "." .. suffix else return filename @@ -1726,20 +2127,39 @@ function file.nameonly(name) return (gsub(match(name,"^.+[/\\](.-)$") or name,"%..*$","")) end -function file.extname(name) - return match(name,"^.+%.([^/\\]-)$") or "" +function file.extname(name,default) + return match(name,"^.+%.([^/\\]-)$") or default or "" end file.suffix = file.extname ---~ print(file.join("x/","/y")) ---~ print(file.join("http://","/y")) ---~ print(file.join("http://a","/y")) ---~ print(file.join("http:///a","/y")) ---~ print(file.join("//nas-1","/y")) +--~ function file.join(...) +--~ local pth = concat({...},"/") +--~ pth = gsub(pth,"\\","/") +--~ local a, b = match(pth,"^(.*://)(.*)$") +--~ if a and b then +--~ return a .. gsub(b,"//+","/") +--~ end +--~ a, b = match(pth,"^(//)(.*)$") +--~ if a and b then +--~ return a .. gsub(b,"//+","/") +--~ end +--~ return (gsub(pth,"//+","/")) +--~ end + +local trick_1 = char(1) +local trick_2 = "^" .. trick_1 .. "/+" function file.join(...) - local pth = concat({...},"/") + local lst = { ... } + local a, b = lst[1], lst[2] + if a == "" then + lst[1] = trick_1 + elseif b and find(a,"^/+$") and find(b,"^/") then + lst[1] = "" + lst[2] = gsub(b,"^/+","") + end + local pth = concat(lst,"/") pth = gsub(pth,"\\","/") local a, b = match(pth,"^(.*://)(.*)$") if a and b then @@ -1749,17 +2169,28 @@ function file.join(...) if a and b then return a .. gsub(b,"//+","/") end + pth = gsub(pth,trick_2,"") return (gsub(pth,"//+","/")) end +--~ print(file.join("//","/y")) +--~ print(file.join("/","/y")) +--~ print(file.join("","/y")) +--~ print(file.join("/x/","/y")) +--~ print(file.join("x/","/y")) +--~ print(file.join("http://","/y")) +--~ print(file.join("http://a","/y")) +--~ print(file.join("http:///a","/y")) +--~ print(file.join("//nas-1","/y")) + function file.iswritable(name) local a = lfs.attributes(name) or lfs.attributes(file.dirname(name,".")) - return a and a.permissions:sub(2,2) == "w" + return a and sub(a.permissions,2,2) == "w" end function file.isreadable(name) local a = lfs.attributes(name) - return a and a.permissions:sub(1,1) == "r" + return a and sub(a.permissions,1,1) == "r" end file.is_readable = file.isreadable @@ -1767,36 +2198,50 @@ file.is_writable = file.iswritable -- todo: lpeg -function file.split_path(str) - local t = { } - str = gsub(str,"\\", "/") - str = gsub(str,"(%a):([;/])", "%1\001%2") - for name in gmatch(str,"([^;:]+)") do - if name ~= "" then - t[#t+1] = gsub(name,"\001",":") - end - end - return t +--~ function file.split_path(str) +--~ local t = { } +--~ str = gsub(str,"\\", "/") +--~ str = gsub(str,"(%a):([;/])", "%1\001%2") +--~ for name in gmatch(str,"([^;:]+)") do +--~ if name ~= "" then +--~ t[#t+1] = gsub(name,"\001",":") +--~ end +--~ end +--~ return t +--~ end + +local checkedsplit = string.checkedsplit + +function file.split_path(str,separator) + str = gsub(str,"\\","/") + return checkedsplit(str,separator or io.pathseparator) end function file.join_path(tab) return concat(tab,io.pathseparator) -- can have trailing // end +-- we can hash them weakly + function file.collapse_path(str) - str = gsub(str,"/%./","/") - local n, m = 1, 1 - while n > 0 or m > 0 do - str, n = gsub(str,"[^/%.]+/%.%.$","") - str, m = gsub(str,"[^/%.]+/%.%./","") - end - str = gsub(str,"([^/])/$","%1") - str = gsub(str,"^%./","") - str = gsub(str,"/%.$","") + str = gsub(str,"\\","/") + if find(str,"/") then + str = gsub(str,"^%./",(gsub(lfs.currentdir(),"\\","/")) .. "/") -- ./xx in qualified + str = gsub(str,"/%./","/") + local n, m = 1, 1 + while n > 0 or m > 0 do + str, n = gsub(str,"[^/%.]+/%.%.$","") + str, m = gsub(str,"[^/%.]+/%.%./","") + end + str = gsub(str,"([^/])/$","%1") + -- str = gsub(str,"^%./","") -- ./xx in qualified + str = gsub(str,"/%.$","") + end if str == "" then str = "." end return str end +--~ print(file.collapse_path("/a")) --~ print(file.collapse_path("a/./b/..")) --~ print(file.collapse_path("a/aa/../b/bb")) --~ print(file.collapse_path("a/../..")) @@ -1826,27 +2271,27 @@ end --~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * lpeg.C(noperiod^1) * -1 --~ function file.extname(name) ---~ return pattern:match(name) or "" +--~ return lpegmatch(pattern,name) or "" --~ end --~ local pattern = lpeg.Cs(((period * noperiod^1 * -1)/"" + 1)^1) --~ function file.removesuffix(name) ---~ return pattern:match(name) +--~ return lpegmatch(pattern,name) --~ end --~ local pattern = (noslashes^0 * slashes)^1 * lpeg.C(noslashes^1) * -1 --~ function file.basename(name) ---~ return pattern:match(name) or name +--~ return lpegmatch(pattern,name) or name --~ end --~ local pattern = (noslashes^0 * slashes)^1 * lpeg.Cp() * noslashes^1 * -1 --~ function file.dirname(name) ---~ local p = pattern:match(name) +--~ local p = lpegmatch(pattern,name) --~ if p then ---~ return name:sub(1,p-2) +--~ return sub(name,1,p-2) --~ else --~ return "" --~ end @@ -1855,7 +2300,7 @@ end --~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * lpeg.Cp() * noperiod^1 * -1 --~ function file.addsuffix(name, suffix) ---~ local p = pattern:match(name) +--~ local p = lpegmatch(pattern,name) --~ if p then --~ return name --~ else @@ -1866,9 +2311,9 @@ end --~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * lpeg.Cp() * noperiod^1 * -1 --~ function file.replacesuffix(name,suffix) ---~ local p = pattern:match(name) +--~ local p = lpegmatch(pattern,name) --~ if p then ---~ return name:sub(1,p-2) .. "." .. suffix +--~ return sub(name,1,p-2) .. "." .. suffix --~ else --~ return name .. "." .. suffix --~ end @@ -1877,11 +2322,11 @@ end --~ local pattern = (noslashes^0 * slashes)^0 * lpeg.Cp() * ((noperiod^1 * period)^1 * lpeg.Cp() + lpeg.P(true)) * noperiod^1 * -1 --~ function file.nameonly(name) ---~ local a, b = pattern:match(name) +--~ local a, b = lpegmatch(pattern,name) --~ if b then ---~ return name:sub(a,b-2) +--~ return sub(name,a,b-2) --~ elseif a then ---~ return name:sub(a) +--~ return sub(name,a) --~ else --~ return name --~ end @@ -1915,11 +2360,11 @@ local rootbased = lpeg.P("/") + letter*lpeg.P(":") -- ./name ../name /name c: :// name/name function file.is_qualified_path(filename) - return qualified:match(filename) + return lpegmatch(qualified,filename) ~= nil end function file.is_rootbased_path(filename) - return rootbased:match(filename) + return lpegmatch(rootbased,filename) ~= nil end local slash = lpeg.S("\\/") @@ -1932,16 +2377,25 @@ local base = lpeg.C((1-suffix)^0) local pattern = (drive + lpeg.Cc("")) * (path + lpeg.Cc("")) * (base + lpeg.Cc("")) * (suffix + lpeg.Cc("")) function file.splitname(str) -- returns drive, path, base, suffix - return pattern:match(str) + return lpegmatch(pattern,str) end --- function test(t) for k, v in pairs(t) do print(v, "=>", file.splitname(v)) end end +-- function test(t) for k, v in next, t do print(v, "=>", file.splitname(v)) end end -- -- test { "c:", "c:/aa", "c:/aa/bb", "c:/aa/bb/cc", "c:/aa/bb/cc.dd", "c:/aa/bb/cc.dd.ee" } -- test { "c:", "c:aa", "c:aa/bb", "c:aa/bb/cc", "c:aa/bb/cc.dd", "c:aa/bb/cc.dd.ee" } -- test { "/aa", "/aa/bb", "/aa/bb/cc", "/aa/bb/cc.dd", "/aa/bb/cc.dd.ee" } -- test { "aa", "aa/bb", "aa/bb/cc", "aa/bb/cc.dd", "aa/bb/cc.dd.ee" } +--~ -- todo: +--~ +--~ if os.type == "windows" then +--~ local currentdir = lfs.currentdir +--~ function lfs.currentdir() +--~ return (gsub(currentdir(),"\\","/")) +--~ end +--~ end + end -- of closure @@ -2006,7 +2460,7 @@ end function file.loadchecksum(name) if md5 then local data = io.loaddata(name .. ".md5") - return data and data:gsub("%s","") + return data and (gsub(data,"%s","")) end return nil end @@ -2025,19 +2479,168 @@ end -- of closure do -- create closure to overcome 200 locals limit +if not modules then modules = { } end modules ['l-url'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local char, gmatch, gsub = string.char, string.gmatch, string.gsub +local tonumber, type = tonumber, type +local lpegmatch = lpeg.match + +-- from the spec (on the web): +-- +-- foo://example.com:8042/over/there?name=ferret#nose +-- \_/ \______________/\_________/ \_________/ \__/ +-- | | | | | +-- scheme authority path query fragment +-- | _____________________|__ +-- / \ / \ +-- urn:example:animal:ferret:nose + +url = url or { } + +local function tochar(s) + return char(tonumber(s,16)) +end + +local colon, qmark, hash, slash, percent, endofstring = lpeg.P(":"), lpeg.P("?"), lpeg.P("#"), lpeg.P("/"), lpeg.P("%"), lpeg.P(-1) + +local hexdigit = lpeg.R("09","AF","af") +local plus = lpeg.P("+") +local escaped = (plus / " ") + (percent * lpeg.C(hexdigit * hexdigit) / tochar) + +-- we assume schemes with more than 1 character (in order to avoid problems with windows disks) + +local scheme = lpeg.Cs((escaped+(1-colon-slash-qmark-hash))^2) * colon + lpeg.Cc("") +local authority = slash * slash * lpeg.Cs((escaped+(1- slash-qmark-hash))^0) + lpeg.Cc("") +local path = slash * lpeg.Cs((escaped+(1- qmark-hash))^0) + lpeg.Cc("") +local query = qmark * lpeg.Cs((escaped+(1- hash))^0) + lpeg.Cc("") +local fragment = hash * lpeg.Cs((escaped+(1- endofstring))^0) + lpeg.Cc("") + +local parser = lpeg.Ct(scheme * authority * path * query * fragment) + +-- todo: reconsider Ct as we can as well have five return values (saves a table) +-- so we can have two parsers, one with and one without + +function url.split(str) + return (type(str) == "string" and lpegmatch(parser,str)) or str +end + +-- todo: cache them + +function url.hashed(str) + local s = url.split(str) + local somescheme = s[1] ~= "" + return { + scheme = (somescheme and s[1]) or "file", + authority = s[2], + path = s[3], + query = s[4], + fragment = s[5], + original = str, + noscheme = not somescheme, + } +end + +function url.hasscheme(str) + return url.split(str)[1] ~= "" +end + +function url.addscheme(str,scheme) + return (url.hasscheme(str) and str) or ((scheme or "file:///") .. str) +end + +function url.construct(hash) + local fullurl = hash.sheme .. "://".. hash.authority .. hash.path + if hash.query then + fullurl = fullurl .. "?".. hash.query + end + if hash.fragment then + fullurl = fullurl .. "?".. hash.fragment + end + return fullurl +end + +function url.filename(filename) + local t = url.hashed(filename) + return (t.scheme == "file" and (gsub(t.path,"^/([a-zA-Z])([:|])/)","%1:"))) or filename +end + +function url.query(str) + if type(str) == "string" then + local t = { } + for k, v in gmatch(str,"([^&=]*)=([^&=]*)") do + t[k] = v + end + return t + else + return str + end +end + +--~ print(url.filename("file:///c:/oeps.txt")) +--~ print(url.filename("c:/oeps.txt")) +--~ print(url.filename("file:///oeps.txt")) +--~ print(url.filename("file:///etc/test.txt")) +--~ print(url.filename("/oeps.txt")) + +--~ from the spec on the web (sort of): +--~ +--~ function test(str) +--~ print(table.serialize(url.hashed(str))) +--~ end +--~ +--~ test("%56pass%20words") +--~ test("file:///c:/oeps.txt") +--~ test("file:///c|/oeps.txt") +--~ test("file:///etc/oeps.txt") +--~ test("file://./etc/oeps.txt") +--~ test("file:////etc/oeps.txt") +--~ test("ftp://ftp.is.co.za/rfc/rfc1808.txt") +--~ test("http://www.ietf.org/rfc/rfc2396.txt") +--~ test("ldap://[2001:db8::7]/c=GB?objectClass?one#what") +--~ test("mailto:John.Doe@example.com") +--~ test("news:comp.infosystems.www.servers.unix") +--~ test("tel:+1-816-555-1212") +--~ test("telnet://192.0.2.16:80/") +--~ test("urn:oasis:names:specification:docbook:dtd:xml:4.1.2") +--~ test("/etc/passwords") +--~ test("http://www.pragma-ade.com/spaced%20name") + +--~ test("zip:///oeps/oeps.zip#bla/bla.tex") +--~ test("zip:///oeps/oeps.zip?bla/bla.tex") + + +end -- of closure + +do -- create closure to overcome 200 locals limit + if not modules then modules = { } end modules ['l-dir'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } +-- dir.expand_name will be merged with cleanpath and collapsepath + local type = type -local find, gmatch = string.find, string.gmatch +local find, gmatch, match, gsub = string.find, string.gmatch, string.match, string.gsub +local lpegmatch = lpeg.match dir = dir or { } +-- handy + +function dir.current() + return (gsub(lfs.currentdir(),"\\","/")) +end + -- optimizing for no string.find (*) does not save time local attributes = lfs.attributes @@ -2068,6 +2671,35 @@ end dir.glob_pattern = glob_pattern +local function collect_pattern(path,patt,recurse,result) + local ok, scanner + result = result or { } + if path == "/" then + ok, scanner = xpcall(function() return walkdir(path..".") end, function() end) -- kepler safe + else + ok, scanner = xpcall(function() return walkdir(path) end, function() end) -- kepler safe + end + if ok and type(scanner) == "function" then + if not find(path,"/$") then path = path .. '/' end + for name in scanner do + local full = path .. name + local attr = attributes(full) + local mode = attr.mode + if mode == 'file' then + if find(full,patt) then + result[name] = attr + end + elseif recurse and (mode == "directory") and (name ~= '.') and (name ~= "..") then + attr.list = collect_pattern(full,patt,recurse) + result[name] = attr + end + end + end + return result +end + +dir.collect_pattern = collect_pattern + local P, S, R, C, Cc, Cs, Ct, Cv, V = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.Cc, lpeg.Cs, lpeg.Ct, lpeg.Cv, lpeg.V local pattern = Ct { @@ -2087,29 +2719,48 @@ local filter = Cs ( ( )^0 ) local function glob(str,t) - if type(str) == "table" then - local t = t or { } - for s=1,#str do - glob(str[s],t) + if type(t) == "function" then + if type(str) == "table" then + for s=1,#str do + glob(str[s],t) + end + elseif lfs.isfile(str) then + t(str) + else + local split = lpegmatch(pattern,str) + if split then + local root, path, base = split[1], split[2], split[3] + local recurse = find(base,"%*%*") + local start = root .. path + local result = lpegmatch(filter,start .. base) + glob_pattern(start,result,recurse,t) + end end - return t - elseif lfs.isfile(str) then - local t = t or { } - t[#t+1] = str - return t else - local split = pattern:match(str) - if split then + if type(str) == "table" then local t = t or { } - local action = action or function(name) t[#t+1] = name end - local root, path, base = split[1], split[2], split[3] - local recurse = find(base,"%*%*") - local start = root .. path - local result = filter:match(start .. base) - glob_pattern(start,result,recurse,action) + for s=1,#str do + glob(str[s],t) + end + return t + elseif lfs.isfile(str) then + local t = t or { } + t[#t+1] = str return t else - return { } + local split = lpegmatch(pattern,str) + if split then + local t = t or { } + local action = action or function(name) t[#t+1] = name end + local root, path, base = split[1], split[2], split[3] + local recurse = find(base,"%*%*") + local start = root .. path + local result = lpegmatch(filter,start .. base) + glob_pattern(start,result,recurse,action) + return t + else + return { } + end end end end @@ -2171,11 +2822,12 @@ end local make_indeed = true -- false -if string.find(os.getenv("PATH"),";") then +if string.find(os.getenv("PATH"),";") then -- os.type == "windows" function dir.mkdirs(...) - local str, pth = "", "" - for _, s in ipairs({...}) do + local str, pth, t = "", "", { ... } + for i=1,#t do + local s = t[i] if s ~= "" then if str ~= "" then str = str .. "/" .. s @@ -2186,13 +2838,13 @@ if string.find(os.getenv("PATH"),";") then end local first, middle, last local drive = false - first, middle, last = str:match("^(//)(//*)(.*)$") + first, middle, last = match(str,"^(//)(//*)(.*)$") if first then -- empty network path == local path else - first, last = str:match("^(//)/*(.-)$") + first, last = match(str,"^(//)/*(.-)$") if first then - middle, last = str:match("([^/]+)/+(.-)$") + middle, last = match(str,"([^/]+)/+(.-)$") if middle then pth = "//" .. middle else @@ -2200,11 +2852,11 @@ if string.find(os.getenv("PATH"),";") then last = "" end else - first, middle, last = str:match("^([a-zA-Z]:)(/*)(.-)$") + first, middle, last = match(str,"^([a-zA-Z]:)(/*)(.-)$") if first then pth, drive = first .. middle, true else - middle, last = str:match("^(/*)(.-)$") + middle, last = match(str,"^(/*)(.-)$") if not middle then last = str end @@ -2238,34 +2890,31 @@ if string.find(os.getenv("PATH"),";") then --~ print(dir.mkdirs("///a/b/c")) --~ print(dir.mkdirs("a/bbb//ccc/")) - function dir.expand_name(str) - local first, nothing, last = str:match("^(//)(//*)(.*)$") + function dir.expand_name(str) -- will be merged with cleanpath and collapsepath + local first, nothing, last = match(str,"^(//)(//*)(.*)$") if first then - first = lfs.currentdir() .. "/" - first = first:gsub("\\","/") + first = dir.current() .. "/" end if not first then - first, last = str:match("^(//)/*(.*)$") + first, last = match(str,"^(//)/*(.*)$") end if not first then - first, last = str:match("^([a-zA-Z]:)(.*)$") + first, last = match(str,"^([a-zA-Z]:)(.*)$") if first and not find(last,"^/") then local d = lfs.currentdir() if lfs.chdir(first) then - first = lfs.currentdir() - first = first:gsub("\\","/") + first = dir.current() end lfs.chdir(d) end end if not first then - first, last = lfs.currentdir(), str - first = first:gsub("\\","/") + first, last = dir.current(), str end - last = last:gsub("//","/") - last = last:gsub("/%./","/") - last = last:gsub("^/*","") - first = first:gsub("/*$","") + last = gsub(last,"//","/") + last = gsub(last,"/%./","/") + last = gsub(last,"^/*","") + first = gsub(first,"/*$","") if last == "" then return first else @@ -2276,8 +2925,9 @@ if string.find(os.getenv("PATH"),";") then else function dir.mkdirs(...) - local str, pth = "", "" - for _, s in ipairs({...}) do + local str, pth, t = "", "", { ... } + for i=1,#t do + local s = t[i] if s ~= "" then if str ~= "" then str = str .. "/" .. s @@ -2286,7 +2936,7 @@ else end end end - str = str:gsub("/+","/") + str = gsub(str,"/+","/") if find(str,"^/") then pth = "/" for s in gmatch(str,"[^/]+") do @@ -2320,12 +2970,12 @@ else --~ print(dir.mkdirs("///a/b/c")) --~ print(dir.mkdirs("a/bbb//ccc/")) - function dir.expand_name(str) + function dir.expand_name(str) -- will be merged with cleanpath and collapsepath if not find(str,"^/") then str = lfs.currentdir() .. "/" .. str end - str = str:gsub("//","/") - str = str:gsub("/%./","/") + str = gsub(str,"//","/") + str = gsub(str,"/%./","/") return str end @@ -2340,7 +2990,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-boolean'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -2401,7 +3051,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-math'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -2448,7 +3098,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-utils'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -2456,6 +3106,10 @@ if not modules then modules = { } end modules ['l-utils'] = { -- hm, quite unreadable +local gsub = string.gsub +local concat = table.concat +local type, next = type, next + if not utils then utils = { } end if not utils.merger then utils.merger = { } end if not utils.lua then utils.lua = { } end @@ -2493,7 +3147,7 @@ function utils.merger._self_load_(name) end if data and utils.merger.strip_comment then -- saves some 20K - data = data:gsub("%-%-~[^\n\r]*[\r\n]", "") + data = gsub(data,"%-%-~[^\n\r]*[\r\n]", "") end return data or "" end @@ -2511,7 +3165,7 @@ end function utils.merger._self_swap_(data,code) if data ~= "" then - return (data:gsub(utils.merger.pattern, function(s) + return (gsub(data,utils.merger.pattern, function(s) return "\n\n" .. "-- "..utils.merger.m_begin .. "\n" .. code .. "\n" .. "-- "..utils.merger.m_end .. "\n\n" end, 1)) else @@ -2521,8 +3175,8 @@ end --~ stripper: --~ ---~ data = string.gsub(data,"%-%-~[^\n]*\n","") ---~ data = string.gsub(data,"\n\n+","\n") +--~ data = gsub(data,"%-%-~[^\n]*\n","") +--~ data = gsub(data,"\n\n+","\n") function utils.merger._self_libs_(libs,list) local result, f, frozen = { }, nil, false @@ -2530,9 +3184,10 @@ function utils.merger._self_libs_(libs,list) if type(libs) == 'string' then libs = { libs } end if type(list) == 'string' then list = { list } end local foundpath = nil - for _, lib in ipairs(libs) do - for _, pth in ipairs(list) do - pth = string.gsub(pth,"\\","/") -- file.clean_path + for i=1,#libs do + local lib = libs[i] + for j=1,#list do + local pth = gsub(list[j],"\\","/") -- file.clean_path utils.report("checking library path %s",pth) local name = pth .. "/" .. lib if lfs.isfile(name) then @@ -2544,7 +3199,8 @@ function utils.merger._self_libs_(libs,list) if foundpath then utils.report("using library path %s",foundpath) local right, wrong = { }, { } - for _, lib in ipairs(libs) do + for i=1,#libs do + local lib = libs[i] local fullname = foundpath .. "/" .. lib if lfs.isfile(fullname) then -- right[#right+1] = lib @@ -2558,15 +3214,15 @@ function utils.merger._self_libs_(libs,list) end end if #right > 0 then - utils.report("merged libraries: %s",table.concat(right," ")) + utils.report("merged libraries: %s",concat(right," ")) end if #wrong > 0 then - utils.report("skipped libraries: %s",table.concat(wrong," ")) + utils.report("skipped libraries: %s",concat(wrong," ")) end else utils.report("no valid library path found") end - return table.concat(result, "\n\n") + return concat(result, "\n\n") end function utils.merger.selfcreate(libs,list,target) @@ -2624,16 +3280,28 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-aux'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } +-- for inline, no store split : for s in string.gmatch(str,",* *([^,]+)") do .. end + aux = aux or { } local concat, format, gmatch = table.concat, string.format, string.gmatch local tostring, type = tostring, type +local lpegmatch = lpeg.match + +local P, R, V = lpeg.P, lpeg.R, lpeg.V + +local escape, left, right = P("\\"), P('{'), P('}') + +lpeg.patterns.balanced = P { + [1] = ((escape * (left+right)) + (1 - (left+right)) + V(2))^0, + [2] = left * V(1) * right +} local space = lpeg.P(' ') local equal = lpeg.P("=") @@ -2641,7 +3309,7 @@ local comma = lpeg.P(",") local lbrace = lpeg.P("{") local rbrace = lpeg.P("}") local nobrace = 1 - (lbrace+rbrace) -local nested = lpeg.P{ lbrace * (nobrace + lpeg.V(1))^0 * rbrace } +local nested = lpeg.P { lbrace * (nobrace + lpeg.V(1))^0 * rbrace } local spaces = space^0 local value = lpeg.P(lbrace * lpeg.C((nobrace + nested)^0) * rbrace) + lpeg.C((nested + (1-comma))^0) @@ -2679,13 +3347,13 @@ function aux.make_settings_to_hash_pattern(set,how) end end -function aux.settings_to_hash(str) +function aux.settings_to_hash(str,existing) if str and str ~= "" then - hash = { } + hash = existing or { } if moretolerant then - pattern_b_s:match(str) + lpegmatch(pattern_b_s,str) else - pattern_a_s:match(str) + lpegmatch(pattern_a_s,str) end return hash else @@ -2693,39 +3361,41 @@ function aux.settings_to_hash(str) end end -function aux.settings_to_hash_tolerant(str) +function aux.settings_to_hash_tolerant(str,existing) if str and str ~= "" then - hash = { } - pattern_b_s:match(str) + hash = existing or { } + lpegmatch(pattern_b_s,str) return hash else return { } end end -function aux.settings_to_hash_strict(str) +function aux.settings_to_hash_strict(str,existing) if str and str ~= "" then - hash = { } - pattern_c_s:match(str) + hash = existing or { } + lpegmatch(pattern_c_s,str) return next(hash) and hash else return nil end end -local seperator = comma * space^0 +local separator = comma * space^0 local value = lpeg.P(lbrace * lpeg.C((nobrace + nested)^0) * rbrace) + lpeg.C((nested + (1-comma))^0) -local pattern = lpeg.Ct(value*(seperator*value)^0) +local pattern = lpeg.Ct(value*(separator*value)^0) -- "aap, {noot}, mies" : outer {} removes, leading spaces ignored aux.settings_to_array_pattern = pattern +-- we could use a weak table as cache + function aux.settings_to_array(str) if not str or str == "" then return { } else - return pattern:match(str) + return lpegmatch(pattern,str) end end @@ -2734,10 +3404,10 @@ local function set(t,v) end local value = lpeg.P(lpeg.Carg(1)*value) / set -local pattern = value*(seperator*value)^0 * lpeg.Carg(1) +local pattern = value*(separator*value)^0 * lpeg.Carg(1) function aux.add_settings_to_array(t,str) - return pattern:match(str, nil, t) + return lpegmatch(pattern,str,nil,t) end function aux.hash_to_string(h,separator,yes,no,strict,omit) @@ -2785,6 +3455,13 @@ function aux.settings_to_set(str,t) return t end +local value = lbrace * lpeg.C((nobrace + nested)^0) * rbrace +local pattern = lpeg.Ct((space + value)^0) + +function aux.arguments_to_table(str) + return lpegmatch(pattern,str) +end + -- temporary here function aux.getparameters(self,class,parentclass,settings) @@ -2793,36 +3470,31 @@ function aux.getparameters(self,class,parentclass,settings) sc = table.clone(self[parent]) self[class] = sc end - aux.add_settings_to_array(sc, settings) + aux.settings_to_hash(settings,sc) end -- temporary here -local digit = lpeg.R("09") -local period = lpeg.P(".") -local zero = lpeg.P("0") - ---~ local finish = lpeg.P(-1) ---~ local nodigit = (1-digit) + finish ---~ local case_1 = (period * zero^1 * #nodigit)/"" -- .000 ---~ local case_2 = (period * (1-(zero^0/"") * #nodigit)^1 * (zero^0/"") * nodigit) -- .010 .10 .100100 - +local digit = lpeg.R("09") +local period = lpeg.P(".") +local zero = lpeg.P("0") local trailingzeros = zero^0 * -digit -- suggested by Roberto R -local case_1 = period * trailingzeros / "" -local case_2 = period * (digit - trailingzeros)^1 * (trailingzeros / "") - -local number = digit^1 * (case_1 + case_2) -local stripper = lpeg.Cs((number + 1)^0) +local case_1 = period * trailingzeros / "" +local case_2 = period * (digit - trailingzeros)^1 * (trailingzeros / "") +local number = digit^1 * (case_1 + case_2) +local stripper = lpeg.Cs((number + 1)^0) --~ local sample = "bla 11.00 bla 11 bla 0.1100 bla 1.00100 bla 0.00 bla 0.001 bla 1.1100 bla 0.100100100 bla 0.00100100100" --~ collectgarbage("collect") --~ str = string.rep(sample,10000) --~ local ts = os.clock() ---~ stripper:match(str) ---~ print(#str, os.clock()-ts, stripper:match(sample)) +--~ lpegmatch(stripper,str) +--~ print(#str, os.clock()-ts, lpegmatch(stripper,sample)) + +lpeg.patterns.strip_zeros = stripper function aux.strip_zeros(str) - return stripper:match(str) + return lpegmatch(stripper,str) end function aux.definetable(target) -- defines undefined tables @@ -2846,6 +3518,375 @@ function aux.accesstable(target) return t end +--~ function string.commaseparated(str) +--~ return gmatch(str,"([^,%s]+)") +--~ end + +-- as we use this a lot ... + +--~ function aux.cachefunction(action,weak) +--~ local cache = { } +--~ if weak then +--~ setmetatable(cache, { __mode = "kv" } ) +--~ end +--~ local function reminder(str) +--~ local found = cache[str] +--~ if not found then +--~ found = action(str) +--~ cache[str] = found +--~ end +--~ return found +--~ end +--~ return reminder, cache +--~ end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['trac-tra'] = { + version = 1.001, + comment = "companion to trac-tra.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- the <anonymous> tag is kind of generic and used for functions that are not +-- bound to a variable, like node.new, node.copy etc (contrary to for instance +-- node.has_attribute which is bound to a has_attribute local variable in mkiv) + +local debug = require "debug" + +local getinfo = debug.getinfo +local type, next = type, next +local concat = table.concat +local format, find, lower, gmatch, gsub = string.format, string.find, string.lower, string.gmatch, string.gsub + +debugger = debugger or { } + +local counters = { } +local names = { } + +-- one + +local function hook() + local f = getinfo(2,"f").func + local n = getinfo(2,"Sn") +-- if n.what == "C" and n.name then print (n.namewhat .. ': ' .. n.name) end + if f then + local cf = counters[f] + if cf == nil then + counters[f] = 1 + names[f] = n + else + counters[f] = cf + 1 + end + end +end +local function getname(func) + local n = names[func] + if n then + if n.what == "C" then + return n.name or '<anonymous>' + else + -- source short_src linedefined what name namewhat nups func + local name = n.name or n.namewhat or n.what + if not name or name == "" then name = "?" end + return format("%s : %s : %s", n.short_src or "unknown source", n.linedefined or "--", name) + end + else + return "unknown" + end +end +function debugger.showstats(printer,threshold) + printer = printer or texio.write or print + threshold = threshold or 0 + local total, grandtotal, functions = 0, 0, 0 + printer("\n") -- ugly but ok + -- table.sort(counters) + for func, count in next, counters do + if count > threshold then + local name = getname(func) + if not find(name,"for generator") then + printer(format("%8i %s", count, name)) + total = total + count + end + end + grandtotal = grandtotal + count + functions = functions + 1 + end + printer(format("functions: %s, total: %s, grand total: %s, threshold: %s\n", functions, total, grandtotal, threshold)) +end + +-- two + +--~ local function hook() +--~ local n = getinfo(2) +--~ if n.what=="C" and not n.name then +--~ local f = tostring(debug.traceback()) +--~ local cf = counters[f] +--~ if cf == nil then +--~ counters[f] = 1 +--~ names[f] = n +--~ else +--~ counters[f] = cf + 1 +--~ end +--~ end +--~ end +--~ function debugger.showstats(printer,threshold) +--~ printer = printer or texio.write or print +--~ threshold = threshold or 0 +--~ local total, grandtotal, functions = 0, 0, 0 +--~ printer("\n") -- ugly but ok +--~ -- table.sort(counters) +--~ for func, count in next, counters do +--~ if count > threshold then +--~ printer(format("%8i %s", count, func)) +--~ total = total + count +--~ end +--~ grandtotal = grandtotal + count +--~ functions = functions + 1 +--~ end +--~ printer(format("functions: %s, total: %s, grand total: %s, threshold: %s\n", functions, total, grandtotal, threshold)) +--~ end + +-- rest + +function debugger.savestats(filename,threshold) + local f = io.open(filename,'w') + if f then + debugger.showstats(function(str) f:write(str) end,threshold) + f:close() + end +end + +function debugger.enable() + debug.sethook(hook,"c") +end + +function debugger.disable() + debug.sethook() +--~ counters[debug.getinfo(2,"f").func] = nil +end + +function debugger.tracing() + local n = tonumber(os.env['MTX.TRACE.CALLS']) or tonumber(os.env['MTX_TRACE_CALLS']) or 0 + if n > 0 then + function debugger.tracing() return true end ; return true + else + function debugger.tracing() return false end ; return false + end +end + +--~ debugger.enable() + +--~ print(math.sin(1*.5)) +--~ print(math.sin(1*.5)) +--~ print(math.sin(1*.5)) +--~ print(math.sin(1*.5)) +--~ print(math.sin(1*.5)) + +--~ debugger.disable() + +--~ print("") +--~ debugger.showstats() +--~ print("") +--~ debugger.showstats(print,3) + +setters = setters or { } +setters.data = setters.data or { } + +--~ local function set(t,what,value) +--~ local data, done = t.data, t.done +--~ if type(what) == "string" then +--~ what = aux.settings_to_array(what) -- inefficient but ok +--~ end +--~ for i=1,#what do +--~ local w = what[i] +--~ for d, f in next, data do +--~ if done[d] then +--~ -- prevent recursion due to wildcards +--~ elseif find(d,w) then +--~ done[d] = true +--~ for i=1,#f do +--~ f[i](value) +--~ end +--~ end +--~ end +--~ end +--~ end + +local function set(t,what,value) + local data, done = t.data, t.done + if type(what) == "string" then + what = aux.settings_to_hash(what) -- inefficient but ok + end + for w, v in next, what do + if v == "" then + v = value + else + v = toboolean(v) + end + for d, f in next, data do + if done[d] then + -- prevent recursion due to wildcards + elseif find(d,w) then + done[d] = true + for i=1,#f do + f[i](v) + end + end + end + end +end + +local function reset(t) + for d, f in next, t.data do + for i=1,#f do + f[i](false) + end + end +end + +local function enable(t,what) + set(t,what,true) +end + +local function disable(t,what) + local data = t.data + if not what or what == "" then + t.done = { } + reset(t) + else + set(t,what,false) + end +end + +function setters.register(t,what,...) + local data = t.data + what = lower(what) + local w = data[what] + if not w then + w = { } + data[what] = w + end + for _, fnc in next, { ... } do + local typ = type(fnc) + if typ == "function" then + w[#w+1] = fnc + elseif typ == "string" then + w[#w+1] = function(value) set(t,fnc,value,nesting) end + end + end +end + +function setters.enable(t,what) + local e = t.enable + t.enable, t.done = enable, { } + enable(t,string.simpleesc(tostring(what))) + t.enable, t.done = e, { } +end + +function setters.disable(t,what) + local e = t.disable + t.disable, t.done = disable, { } + disable(t,string.simpleesc(tostring(what))) + t.disable, t.done = e, { } +end + +function setters.reset(t) + t.done = { } + reset(t) +end + +function setters.list(t) -- pattern + local list = table.sortedkeys(t.data) + local user, system = { }, { } + for l=1,#list do + local what = list[l] + if find(what,"^%*") then + system[#system+1] = what + else + user[#user+1] = what + end + end + return user, system +end + +function setters.show(t) + commands.writestatus("","") + local list = setters.list(t) + for k=1,#list do + commands.writestatus(t.name,list[k]) + end + commands.writestatus("","") +end + +-- we could have used a bit of oo and the trackers:enable syntax but +-- there is already a lot of code around using the singular tracker + +-- we could make this into a module + +function setters.new(name) + local t + t = { + data = { }, + name = name, + enable = function(...) setters.enable (t,...) end, + disable = function(...) setters.disable (t,...) end, + register = function(...) setters.register(t,...) end, + list = function(...) setters.list (t,...) end, + show = function(...) setters.show (t,...) end, + } + setters.data[name] = t + return t +end + +trackers = setters.new("trackers") +directives = setters.new("directives") +experiments = setters.new("experiments") + +-- nice trick: we overload two of the directives related functions with variants that +-- do tracing (itself using a tracker) .. proof of concept + +local trace_directives = false local trace_directives = false trackers.register("system.directives", function(v) trace_directives = v end) +local trace_experiments = false local trace_experiments = false trackers.register("system.experiments", function(v) trace_experiments = v end) + +local e = directives.enable +local d = directives.disable + +function directives.enable(...) + commands.writestatus("directives","enabling: %s",concat({...}," ")) + e(...) +end + +function directives.disable(...) + commands.writestatus("directives","disabling: %s",concat({...}," ")) + d(...) +end + +local e = experiments.enable +local d = experiments.disable + +function experiments.enable(...) + commands.writestatus("experiments","enabling: %s",concat({...}," ")) + e(...) +end + +function experiments.disable(...) + commands.writestatus("experiments","disabling: %s",concat({...}," ")) + d(...) +end + +-- a useful example + +directives.register("system.nostatistics", function(v) + statistics.enable = not v +end) + + end -- of closure @@ -2859,6 +3900,12 @@ if not modules then modules = { } end modules ['lxml-tab'] = { license = "see context related readme files" } +-- this module needs a cleanup: check latest lpeg, passing args, (sub)grammar, etc etc +-- stripping spaces from e.g. cont-en.xml saves .2 sec runtime so it's not worth the +-- trouble + +local trace_entities = false trackers.register("xml.entities", function(v) trace_entities = v end) + --[[ldx-- <p>The parser used here is inspired by the variant discussed in the lua book, but handles comment and processing instructions, has a different structure, provides @@ -2866,18 +3913,6 @@ parent access; a first version used different trickery but was less optimized to went this route. First we had a find based parser, now we have an <l n='lpeg'/> based one. The find based parser can be found in l-xml-edu.lua along with other older code.</p> -<p>Expecially the lpath code is experimental, we will support some of xpath, but -only things that make sense for us; as compensation it is possible to hook in your -own functions. Apart from preprocessing content for <l n='context'/> we also need -this module for process management, like handling <l n='ctx'/> and <l n='rlx'/> -files.</p> - -<typing> -a/b/c /*/c -a/b/c/first() a/b/c/last() a/b/c/index(n) a/b/c/index(-n) -a/b/c/text() a/b/c/text(1) a/b/c/text(-1) a/b/c/text(n) -</typing> - <p>Beware, the interface may change. For instance at, ns, tg, dt may get more verbose names. Once the code is stable we will also remove some tracing and optimize the code.</p> @@ -2888,26 +3923,11 @@ xml = xml or { } --~ local xml = xml local concat, remove, insert = table.concat, table.remove, table.insert -local type, next, setmetatable = type, next, setmetatable -local format, lower, find = string.format, string.lower, string.find - ---[[ldx-- -<p>This module can be used stand alone but also inside <l n='mkiv'/> in -which case it hooks into the tracker code. Therefore we provide a few -functions that set the tracers.</p> ---ldx]]-- - -local trace_remap = false - -if trackers then - trackers.register("xml.remap", function(v) trace_remap = v end) -end - -function xml.settrace(str,value) - if str == "remap" then - trace_remap = value or false - end -end +local type, next, setmetatable, getmetatable, tonumber = type, next, setmetatable, getmetatable, tonumber +local format, lower, find, match, gsub = string.format, string.lower, string.find, string.match, string.gsub +local utfchar = unicode.utf8.char +local lpegmatch = lpeg.match +local P, S, R, C, V, C, Cs = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.C, lpeg.Cs --[[ldx-- <p>First a hack to enable namespace resolving. A namespace is characterized by @@ -2919,7 +3939,7 @@ much cleaner.</p> xml.xmlns = xml.xmlns or { } -local check = lpeg.P(false) +local check = P(false) local parse = check --[[ldx-- @@ -2932,8 +3952,8 @@ xml.registerns("mml","mathml") --ldx]]-- function xml.registerns(namespace, pattern) -- pattern can be an lpeg - check = check + lpeg.C(lpeg.P(lower(pattern))) / namespace - parse = lpeg.P { lpeg.P(check) + 1 * lpeg.V(1) } + check = check + C(P(lower(pattern))) / namespace + parse = P { P(check) + 1 * V(1) } end --[[ldx-- @@ -2947,7 +3967,7 @@ xml.checkns("m","http://www.w3.org/mathml") --ldx]]-- function xml.checkns(namespace,url) - local ns = parse:match(lower(url)) + local ns = lpegmatch(parse,lower(url)) if ns and namespace ~= ns then xml.xmlns[namespace] = ns end @@ -2965,7 +3985,7 @@ This returns <t>mml</t>. --ldx]]-- function xml.resolvens(url) - return parse:match(lower(url)) or "" + return lpegmatch(parse,lower(url)) or "" end --[[ldx-- @@ -3004,27 +4024,36 @@ local x = xml.convert(somestring) <p>An optional second boolean argument tells this function not to create a root element.</p> ---ldx]]-- -xml.strip_cm_and_dt = false -- an extra global flag, in case we have many includes +<p>Valid entities are:</p> + +<typing> +<!ENTITY xxxx SYSTEM "yyyy" NDATA zzzz> +<!ENTITY xxxx PUBLIC "yyyy" > +<!ENTITY xxxx "yyyy" > +</typing> +--ldx]]-- -- not just one big nested table capture (lpeg overflow) local nsremap, resolvens = xml.xmlns, xml.resolvens -local stack, top, dt, at, xmlns, errorstr, entities = {}, {}, {}, {}, {}, nil, {} +local stack, top, dt, at, xmlns, errorstr, entities = { }, { }, { }, { }, { }, nil, { } +local strip, cleanup, utfize, resolve, resolve_predefined, unify_predefined = false, false, false, false, false, false +local dcache, hcache, acache = { }, { }, { } -local mt = { __tostring = xml.text } +local mt = { } -function xml.check_error(top,toclose) - return "" +function initialize_mt(root) + mt = { __index = root } -- will be redefined later end -local strip = false -local cleanup = false +function xml.setproperty(root,k,v) + getmetatable(root).__index[k] = v +end -function xml.set_text_cleanup(fnc) - cleanup = fnc +function xml.check_error(top,toclose) + return "" end local function add_attribute(namespace,tag,value) @@ -3034,12 +4063,31 @@ local function add_attribute(namespace,tag,value) if tag == "xmlns" then xmlns[#xmlns+1] = resolvens(value) at[tag] = value + elseif namespace == "" then + at[tag] = value elseif namespace == "xmlns" then xml.checkns(tag,value) at["xmlns:" .. tag] = value else - at[tag] = value + -- for the moment this way: + at[namespace .. ":" .. tag] = value + end +end + +local function add_empty(spacing, namespace, tag) + if #spacing > 0 then + dt[#dt+1] = spacing + end + local resolved = (namespace == "" and xmlns[#xmlns]) or nsremap[namespace] or namespace + top = stack[#stack] + dt = top.dt + local t = { ns=namespace or "", rn=resolved, tg=tag, at=at, dt={}, __p__ = top } + dt[#dt+1] = t + setmetatable(t, mt) + if at.xmlns then + remove(xmlns) end + at = { } end local function add_begin(spacing, namespace, tag) @@ -3067,28 +4115,12 @@ local function add_end(spacing, namespace, tag) end dt = top.dt dt[#dt+1] = toclose - dt[0] = top + -- dt[0] = top -- nasty circular reference when serializing table if toclose.at.xmlns then remove(xmlns) end end -local function add_empty(spacing, namespace, tag) - if #spacing > 0 then - dt[#dt+1] = spacing - end - local resolved = (namespace == "" and xmlns[#xmlns]) or nsremap[namespace] or namespace - top = stack[#stack] - dt = top.dt - local t = { ns=namespace or "", rn=resolved, tg=tag, at=at, dt={}, __p__ = top } - dt[#dt+1] = t - setmetatable(t, mt) - if at.xmlns then - remove(xmlns) - end - at = { } -end - local function add_text(text) if cleanup and #text > 0 then dt[#dt+1] = cleanup(text) @@ -3104,7 +4136,7 @@ local function add_special(what, spacing, text) if strip and (what == "@cm@" or what == "@dt@") then -- forget it else - dt[#dt+1] = { special=true, ns="", tg=what, dt={text} } + dt[#dt+1] = { special=true, ns="", tg=what, dt={ text } } end end @@ -3112,7 +4144,199 @@ local function set_message(txt) errorstr = "garbage at the end of the file: " .. gsub(txt,"([ \n\r\t]*)","") end -local P, S, R, C, V = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V +local reported_attribute_errors = { } + +local function attribute_value_error(str) + if not reported_attribute_errors[str] then + logs.report("xml","invalid attribute value: %q",str) + reported_attribute_errors[str] = true + at._error_ = str + end + return str +end +local function attribute_specification_error(str) + if not reported_attribute_errors[str] then + logs.report("xml","invalid attribute specification: %q",str) + reported_attribute_errors[str] = true + at._error_ = str + end + return str +end + +function xml.unknown_dec_entity_format(str) return (str == "" and "&error;") or format("&%s;",str) end +function xml.unknown_hex_entity_format(str) return format("&#x%s;",str) end +function xml.unknown_any_entity_format(str) return format("&#x%s;",str) end + +local function fromhex(s) + local n = tonumber(s,16) + if n then + return utfchar(n) + else + return format("h:%s",s), true + end +end + +local function fromdec(s) + local n = tonumber(s) + if n then + return utfchar(n) + else + return format("d:%s",s), true + end +end + +-- one level expansion (simple case), no checking done + +local rest = (1-P(";"))^0 +local many = P(1)^0 + +local parsedentity = + P("&") * (P("#x")*(rest/fromhex) + P("#")*(rest/fromdec)) * P(";") * P(-1) + + (P("#x")*(many/fromhex) + P("#")*(many/fromdec)) + +-- parsing in the xml file + +local predefined_unified = { + [38] = "&", + [42] = """, + [47] = "'", + [74] = "<", + [76] = "&gr;", +} + +local predefined_simplified = { + [38] = "&", amp = "&", + [42] = '"', quot = '"', + [47] = "'", apos = "'", + [74] = "<", lt = "<", + [76] = ">", gt = ">", +} + +local function handle_hex_entity(str) + local h = hcache[str] + if not h then + local n = tonumber(str,16) + h = unify_predefined and predefined_unified[n] + if h then + if trace_entities then + logs.report("xml","utfize, converting hex entity &#x%s; into %s",str,h) + end + elseif utfize then + h = (n and utfchar(n)) or xml.unknown_hex_entity_format(str) or "" + if not n then + logs.report("xml","utfize, ignoring hex entity &#x%s;",str) + elseif trace_entities then + logs.report("xml","utfize, converting hex entity &#x%s; into %s",str,h) + end + else + if trace_entities then + logs.report("xml","found entity &#x%s;",str) + end + h = "&#x" .. str .. ";" + end + hcache[str] = h + end + return h +end + +local function handle_dec_entity(str) + local d = dcache[str] + if not d then + local n = tonumber(str) + d = unify_predefined and predefined_unified[n] + if d then + if trace_entities then + logs.report("xml","utfize, converting dec entity &#%s; into %s",str,d) + end + elseif utfize then + d = (n and utfchar(n)) or xml.unknown_dec_entity_format(str) or "" + if not n then + logs.report("xml","utfize, ignoring dec entity &#%s;",str) + elseif trace_entities then + logs.report("xml","utfize, converting dec entity &#%s; into %s",str,h) + end + else + if trace_entities then + logs.report("xml","found entity &#%s;",str) + end + d = "&#" .. str .. ";" + end + dcache[str] = d + end + return d +end + +xml.parsedentitylpeg = parsedentity + +local function handle_any_entity(str) + if resolve then + local a = acache[str] -- per instance ! todo + if not a then + a = resolve_predefined and predefined_simplified[str] + if a then + -- one of the predefined + elseif type(resolve) == "function" then + a = resolve(str) or entities[str] + else + a = entities[str] + end + if a then + if trace_entities then + logs.report("xml","resolved entity &%s; -> %s (internal)",str,a) + end + a = lpegmatch(parsedentity,a) or a + else + if xml.unknown_any_entity_format then + a = xml.unknown_any_entity_format(str) or "" + end + if a then + if trace_entities then + logs.report("xml","resolved entity &%s; -> %s (external)",str,a) + end + else + if trace_entities then + logs.report("xml","keeping entity &%s;",str) + end + if str == "" then + a = "&error;" + else + a = "&" .. str .. ";" + end + end + end + acache[str] = a + elseif trace_entities then + if not acache[str] then + logs.report("xml","converting entity &%s; into %s",str,a) + acache[str] = a + end + end + return a + else + local a = acache[str] + if not a then + if trace_entities then + logs.report("xml","found entity &%s;",str) + end + a = resolve_predefined and predefined_simplified[str] + if a then + -- one of the predefined + acache[str] = a + elseif str == "" then + a = "&error;" + acache[str] = a + else + a = "&" .. str .. ";" + acache[str] = a + end + end + return a + end +end + +local function handle_end_entity(chr) + logs.report("xml","error in entity, %q found instead of ';'",chr) +end local space = S(' \r\n\t') local open = P('<') @@ -3122,24 +4346,50 @@ local dquote = S('"') local equal = P('=') local slash = P('/') local colon = P(':') +local semicolon = P(';') +local ampersand = P('&') local valid = R('az', 'AZ', '09') + S('_-.') local name_yes = C(valid^1) * colon * C(valid^1) local name_nop = C(P(true)) * C(valid^1) local name = name_yes + name_nop +local utfbom = lpeg.patterns.utfbom -- no capture +local spacing = C(space^0) -local utfbom = P('\000\000\254\255') + P('\255\254\000\000') + - P('\255\254') + P('\254\255') + P('\239\187\191') -- no capture +----- entitycontent = (1-open-semicolon)^0 +local anyentitycontent = (1-open-semicolon-space-close)^0 +local hexentitycontent = R("AF","af","09")^0 +local decentitycontent = R("09")^0 +local parsedentity = P("#")/"" * ( + P("x")/"" * (hexentitycontent/handle_hex_entity) + + (decentitycontent/handle_dec_entity) + ) + (anyentitycontent/handle_any_entity) +local entity = ampersand/"" * parsedentity * ( (semicolon/"") + #(P(1)/handle_end_entity)) + +local text_unparsed = C((1-open)^1) +local text_parsed = Cs(((1-open-ampersand)^1 + entity)^1) -local spacing = C(space^0) -local justtext = C((1-open)^1) local somespace = space^1 local optionalspace = space^0 -local value = (squote * C((1 - squote)^0) * squote) + (dquote * C((1 - dquote)^0) * dquote) -local attribute = (somespace * name * optionalspace * equal * optionalspace * value) / add_attribute -local attributes = attribute^0 +----- value = (squote * C((1 - squote)^0) * squote) + (dquote * C((1 - dquote)^0) * dquote) -- ampersand and < also invalid in value +local value = (squote * Cs((entity + (1 - squote))^0) * squote) + (dquote * Cs((entity + (1 - dquote))^0) * dquote) -- ampersand and < also invalid in value + +local endofattributes = slash * close + close -- recovery of flacky html +local whatever = space * name * optionalspace * equal +local wrongvalue = C(P(1-whatever-close)^1 + P(1-close)^1) / attribute_value_error +----- wrongvalue = C(P(1-whatever-endofattributes)^1 + P(1-endofattributes)^1) / attribute_value_error +----- wrongvalue = C(P(1-space-endofattributes)^1) / attribute_value_error +local wrongvalue = Cs(P(entity + (1-space-endofattributes))^1) / attribute_value_error -local text = justtext / add_text +local attributevalue = value + wrongvalue + +local attribute = (somespace * name * optionalspace * equal * optionalspace * attributevalue) / add_attribute +----- attributes = (attribute)^0 + +local attributes = (attribute + somespace^-1 * (((1-endofattributes)^1)/attribute_specification_error))^0 + +local parsedtext = text_parsed / add_text +local unparsedtext = text_unparsed / add_text local balanced = P { "[" * ((1 - S"[]") + V(1))^0 * "]" } -- taken from lpeg manual, () example local emptyelement = (spacing * open * name * attributes * optionalspace * slash * close) / add_empty @@ -3157,19 +4407,27 @@ local someinstruction = C((1 - endinstruction)^0) local somecomment = C((1 - endcomment )^0) local somecdata = C((1 - endcdata )^0) -local function entity(k,v) entities[k] = v end +local function normalentity(k,v ) entities[k] = v end +local function systementity(k,v,n) entities[k] = v end +local function publicentity(k,v,n) entities[k] = v end local begindoctype = open * P("!DOCTYPE") local enddoctype = close local beginset = P("[") local endset = P("]") -local doctypename = C((1-somespace)^0) +local doctypename = C((1-somespace-close)^0) local elementdoctype = optionalspace * P("<!ELEMENT") * (1-close)^0 * close -local entitydoctype = optionalspace * P("<!ENTITY") * somespace * (doctypename * somespace * value)/entity * optionalspace * close -local publicdoctype = doctypename * somespace * P("PUBLIC") * somespace * value * somespace * value * somespace -local systemdoctype = doctypename * somespace * P("SYSTEM") * somespace * value * somespace -local definitiondoctype= doctypename * somespace * beginset * P(elementdoctype + entitydoctype)^0 * optionalspace * endset -local simpledoctype = (1-close)^1 -- * balanced^0 + +local normalentitytype = (doctypename * somespace * value)/normalentity +local publicentitytype = (doctypename * somespace * P("PUBLIC") * somespace * value)/publicentity +local systementitytype = (doctypename * somespace * P("SYSTEM") * somespace * value * somespace * P("NDATA") * somespace * doctypename)/systementity +local entitydoctype = optionalspace * P("<!ENTITY") * somespace * (systementitytype + publicentitytype + normalentitytype) * optionalspace * close + +local doctypeset = beginset * optionalspace * P(elementdoctype + entitydoctype + space)^0 * optionalspace * endset +local definitiondoctype= doctypename * somespace * doctypeset +local publicdoctype = doctypename * somespace * P("PUBLIC") * somespace * value * somespace * value * somespace * doctypeset +local systemdoctype = doctypename * somespace * P("SYSTEM") * somespace * value * somespace * doctypeset +local simpledoctype = (1-close)^1 -- * balanced^0 local somedoctype = C((somespace * (publicdoctype + systemdoctype + definitiondoctype + simpledoctype) * optionalspace)^0) local instruction = (spacing * begininstruction * someinstruction * endinstruction) / function(...) add_special("@pi@",...) end @@ -3179,60 +4437,116 @@ local doctype = (spacing * begindoctype * somedoctype * enddoct -- nicer but slower: -- --- local instruction = (lpeg.Cc("@pi@") * spacing * begininstruction * someinstruction * endinstruction) / add_special --- local comment = (lpeg.Cc("@cm@") * spacing * begincomment * somecomment * endcomment ) / add_special --- local cdata = (lpeg.Cc("@cd@") * spacing * begincdata * somecdata * endcdata ) / add_special --- local doctype = (lpeg.Cc("@dt@") * spacing * begindoctype * somedoctype * enddoctype ) / add_special +-- local instruction = (Cc("@pi@") * spacing * begininstruction * someinstruction * endinstruction) / add_special +-- local comment = (Cc("@cm@") * spacing * begincomment * somecomment * endcomment ) / add_special +-- local cdata = (Cc("@cd@") * spacing * begincdata * somecdata * endcdata ) / add_special +-- local doctype = (Cc("@dt@") * spacing * begindoctype * somedoctype * enddoctype ) / add_special -local trailer = space^0 * (justtext/set_message)^0 +local trailer = space^0 * (text_unparsed/set_message)^0 -- comment + emptyelement + text + cdata + instruction + V("parent"), -- 6.5 seconds on 40 MB database file -- text + comment + emptyelement + cdata + instruction + V("parent"), -- 5.8 -- text + V("parent") + emptyelement + comment + cdata + instruction, -- 5.5 -local grammar = P { "preamble", +local grammar_parsed_text = P { "preamble", preamble = utfbom^0 * instruction^0 * (doctype + comment + instruction)^0 * V("parent") * trailer, parent = beginelement * V("children")^0 * endelement, - children = text + V("parent") + emptyelement + comment + cdata + instruction, + children = parsedtext + V("parent") + emptyelement + comment + cdata + instruction, } --- todo: xml.new + properties like entities and strip and such (store in root) +local grammar_unparsed_text = P { "preamble", + preamble = utfbom^0 * instruction^0 * (doctype + comment + instruction)^0 * V("parent") * trailer, + parent = beginelement * V("children")^0 * endelement, + children = unparsedtext + V("parent") + emptyelement + comment + cdata + instruction, +} -function xml.convert(data, no_root, strip_cm_and_dt, given_entities) -- maybe use table met k/v (given_entities may disapear) - strip = strip_cm_and_dt or xml.strip_cm_and_dt - stack, top, at, xmlns, errorstr, result, entities = {}, {}, {}, {}, nil, nil, given_entities or {} +-- maybe we will add settinsg to result as well + +local function xmlconvert(data, settings) + settings = settings or { } -- no_root strip_cm_and_dt given_entities parent_root error_handler + strip = settings.strip_cm_and_dt + utfize = settings.utfize_entities + resolve = settings.resolve_entities + resolve_predefined = settings.resolve_predefined_entities -- in case we have escaped entities + unify_predefined = settings.unify_predefined_entities -- & -> & + cleanup = settings.text_cleanup + stack, top, at, xmlns, errorstr, result, entities = { }, { }, { }, { }, nil, nil, settings.entities or { } + acache, hcache, dcache = { }, { }, { } -- not stored + reported_attribute_errors = { } + if settings.parent_root then + mt = getmetatable(settings.parent_root) + else + initialize_mt(top) + end stack[#stack+1] = top top.dt = { } dt = top.dt if not data or data == "" then errorstr = "empty xml file" - elseif not grammar:match(data) then - errorstr = "invalid xml file" + elseif utfize or resolve then + if lpegmatch(grammar_parsed_text,data) then + errorstr = "" + else + errorstr = "invalid xml file - parsed text" + end + elseif type(data) == "string" then + if lpegmatch(grammar_unparsed_text,data) then + errorstr = "" + else + errorstr = "invalid xml file - unparsed text" + end else - errorstr = "" + errorstr = "invalid xml file - no text at all" end if errorstr and errorstr ~= "" then - result = { dt = { { ns = "", tg = "error", dt = { errorstr }, at={}, er = true } }, error = true } + result = { dt = { { ns = "", tg = "error", dt = { errorstr }, at={ }, er = true } } } setmetatable(stack, mt) - if xml.error_handler then xml.error_handler("load",errorstr) end + local error_handler = settings.error_handler + if error_handler == false then + -- no error message + else + error_handler = error_handler or xml.error_handler + if error_handler then + xml.error_handler("load",errorstr) + end + end else result = stack[1] end - if not no_root then - result = { special = true, ns = "", tg = '@rt@', dt = result.dt, at={}, entities = entities } + if not settings.no_root then + result = { special = true, ns = "", tg = '@rt@', dt = result.dt, at={ }, entities = entities, settings = settings } setmetatable(result, mt) local rdt = result.dt for k=1,#rdt do local v = rdt[k] if type(v) == "table" and not v.special then -- always table -) result.ri = k -- rootindex +v.__p__ = result -- new, experiment, else we cannot go back to settings, we need to test this ! break end end end + if errorstr and errorstr ~= "" then + result.error = true + end return result end +xml.convert = xmlconvert + +function xml.inheritedconvert(data,xmldata) + local settings = xmldata.settings + settings.parent_root = xmldata -- to be tested + -- settings.no_root = true + local xc = xmlconvert(data,settings) + -- xc.settings = nil + -- xc.entities = nil + -- xc.special = nil + -- xc.ri = nil + -- print(xc.tg) + return xc +end + --[[ldx-- <p>Packaging data in an xml like table is done with the following function. Maybe it will go away (when not used).</p> @@ -3243,7 +4557,7 @@ function xml.is_valid(root) end function xml.package(tag,attributes,data) - local ns, tg = tag:match("^(.-):?([^:]+)$") + local ns, tg = match(tag,"^(.-):?([^:]+)$") local t = { ns = ns, tg = tg, dt = data or "", at = attributes or {} } setmetatable(t, mt) return t @@ -3261,21 +4575,19 @@ the whole file first. The function accepts a string representing a filename or a file handle.</p> --ldx]]-- -function xml.load(filename) +function xml.load(filename,settings) + local data = "" if type(filename) == "string" then + -- local data = io.loaddata(filename) - -todo: check type in io.loaddata local f = io.open(filename,'r') if f then - local root = xml.convert(f:read("*all")) + data = f:read("*all") f:close() - return root - else - return xml.convert("") end elseif filename then -- filehandle - return xml.convert(filename:read("*all")) - else - return xml.convert("") + data = filename:read("*all") end + return xmlconvert(data,settings) end --[[ldx-- @@ -3283,9 +4595,11 @@ end valid trees, which is what the next function does.</p> --ldx]]-- +local no_root = { no_root = true } + function xml.toxml(data) if type(data) == "string" then - local root = { xml.convert(data,true) } + local root = { xmlconvert(data,no_root) } return (#root > 1 and root) or root[1] else return data @@ -3305,7 +4619,7 @@ local function copy(old,tables) if not tables[old] then tables[old] = new end - for k,v in pairs(old) do + for k,v in next, old do new[k] = (type(v) == "table" and (tables[v] or copy(v, tables))) or v end local mt = getmetatable(old) @@ -3330,217 +4644,304 @@ alternative.</p> -- todo: add <?xml version='1.0' standalone='yes'?> when not present -local fallbackhandle = (tex and tex.sprint) or io.write - -local function serialize(e, handle, textconverter, attributeconverter, specialconverter, nocommands) - if not e then - return - elseif not nocommands then - local ec = e.command - if ec ~= nil then -- we can have all kind of types - if e.special then - local etg, edt = e.tg, e.dt - local spc = specialconverter and specialconverter[etg] - if spc then - local result = spc(edt[1]) - if result then - handle(result) - return - else - -- no need to handle any further - end - end - end - local xc = xml.command - if xc then - xc(e,ec) - return +function xml.checkbom(root) -- can be made faster + if root.ri then + local dt, found = root.dt, false + for k=1,#dt do + local v = dt[k] + if type(v) == "table" and v.special and v.tg == "@pi@" and find(v.dt[1],"xml.*version=") then + found = true + break end end + if not found then + insert(dt, 1, { special=true, ns="", tg="@pi@", dt = { "xml version='1.0' standalone='yes'"} } ) + insert(dt, 2, "\n" ) + end end - handle = handle or fallbackhandle - local etg = e.tg - if etg then - if e.special then - local edt = e.dt - local spc = specialconverter and specialconverter[etg] - if spc then - local result = spc(edt[1]) - if result then - handle(result) +end + +--[[ldx-- +<p>At the cost of some 25% runtime overhead you can first convert the tree to a string +and then handle the lot.</p> +--ldx]]-- + +-- new experimental reorganized serialize + +local function verbose_element(e,handlers) + local handle = handlers.handle + local serialize = handlers.serialize + local ens, etg, eat, edt, ern = e.ns, e.tg, e.at, e.dt, e.rn + local ats = eat and next(eat) and { } + if ats then + for k,v in next, eat do + ats[#ats+1] = format('%s=%q',k,v) + end + end + if ern and trace_remap and ern ~= ens then + ens = ern + end + if ens ~= "" then + if edt and #edt > 0 then + if ats then + handle("<",ens,":",etg," ",concat(ats," "),">") + else + handle("<",ens,":",etg,">") + end + for i=1,#edt do + local e = edt[i] + if type(e) == "string" then + handle(e) else - -- no need to handle any further - end - elseif etg == "@pi@" then - -- handle(format("<?%s?>",edt[1])) - handle("<?" .. edt[1] .. "?>") - elseif etg == "@cm@" then - -- handle(format("<!--%s-->",edt[1])) - handle("<!--" .. edt[1] .. "-->") - elseif etg == "@cd@" then - -- handle(format("<![CDATA[%s]]>",edt[1])) - handle("<![CDATA[" .. edt[1] .. "]]>") - elseif etg == "@dt@" then - -- handle(format("<!DOCTYPE %s>",edt[1])) - handle("<!DOCTYPE " .. edt[1] .. ">") - elseif etg == "@rt@" then - serialize(edt,handle,textconverter,attributeconverter,specialconverter,nocommands) + serialize(e,handlers) + end end + handle("</",ens,":",etg,">") else - local ens, eat, edt, ern = e.ns, e.at, e.dt, e.rn - local ats = eat and next(eat) and { } -- type test maybe faster if ats then - if attributeconverter then - for k,v in next, eat do - ats[#ats+1] = format('%s=%q',k,attributeconverter(v)) - end - else - for k,v in next, eat do - ats[#ats+1] = format('%s=%q',k,v) - end - end + handle("<",ens,":",etg," ",concat(ats," "),"/>") + else + handle("<",ens,":",etg,"/>") end - if ern and trace_remap and ern ~= ens then - ens = ern + end + else + if edt and #edt > 0 then + if ats then + handle("<",etg," ",concat(ats," "),">") + else + handle("<",etg,">") end - if ens ~= "" then - if edt and #edt > 0 then - if ats then - -- handle(format("<%s:%s %s>",ens,etg,concat(ats," "))) - handle("<" .. ens .. ":" .. etg .. " " .. concat(ats," ") .. ">") - else - -- handle(format("<%s:%s>",ens,etg)) - handle("<" .. ens .. ":" .. etg .. ">") - end - for i=1,#edt do - local e = edt[i] - if type(e) == "string" then - if textconverter then - handle(textconverter(e)) - else - handle(e) - end - else - serialize(e,handle,textconverter,attributeconverter,specialconverter,nocommands) - end - end - -- handle(format("</%s:%s>",ens,etg)) - handle("</" .. ens .. ":" .. etg .. ">") + for i=1,#edt do + local ei = edt[i] + if type(ei) == "string" then + handle(ei) else - if ats then - -- handle(format("<%s:%s %s/>",ens,etg,concat(ats," "))) - handle("<" .. ens .. ":" .. etg .. " " .. concat(ats," ") .. "/>") - else - -- handle(format("<%s:%s/>",ens,etg)) - handle("<" .. ens .. ":" .. etg .. "/>") - end + serialize(ei,handlers) end + end + handle("</",etg,">") + else + if ats then + handle("<",etg," ",concat(ats," "),"/>") else - if edt and #edt > 0 then - if ats then - -- handle(format("<%s %s>",etg,concat(ats," "))) - handle("<" .. etg .. " " .. concat(ats," ") .. ">") - else - -- handle(format("<%s>",etg)) - handle("<" .. etg .. ">") - end - for i=1,#edt do - local ei = edt[i] - if type(ei) == "string" then - if textconverter then - handle(textconverter(ei)) - else - handle(ei) - end - else - serialize(ei,handle,textconverter,attributeconverter,specialconverter,nocommands) - end - end - -- handle(format("</%s>",etg)) - handle("</" .. etg .. ">") - else - if ats then - -- handle(format("<%s %s/>",etg,concat(ats," "))) - handle("<" .. etg .. " " .. concat(ats," ") .. "/>") - else - -- handle(format("<%s/>",etg)) - handle("<" .. etg .. "/>") - end - end + handle("<",etg,"/>") end end - elseif type(e) == "string" then - if textconverter then - handle(textconverter(e)) + end +end + +local function verbose_pi(e,handlers) + handlers.handle("<?",e.dt[1],"?>") +end + +local function verbose_comment(e,handlers) + handlers.handle("<!--",e.dt[1],"-->") +end + +local function verbose_cdata(e,handlers) + handlers.handle("<![CDATA[", e.dt[1],"]]>") +end + +local function verbose_doctype(e,handlers) + handlers.handle("<!DOCTYPE ",e.dt[1],">") +end + +local function verbose_root(e,handlers) + handlers.serialize(e.dt,handlers) +end + +local function verbose_text(e,handlers) + handlers.handle(e) +end + +local function verbose_document(e,handlers) + local serialize = handlers.serialize + local functions = handlers.functions + for i=1,#e do + local ei = e[i] + if type(ei) == "string" then + functions["@tx@"](ei,handlers) else - handle(e) + serialize(ei,handlers) end - else - for i=1,#e do - local ei = e[i] - if type(ei) == "string" then - if textconverter then - handle(textconverter(ei)) - else - handle(ei) - end - else - serialize(ei,handle,textconverter,attributeconverter,specialconverter,nocommands) - end + end +end + +local function serialize(e,handlers,...) + local initialize = handlers.initialize + local finalize = handlers.finalize + local functions = handlers.functions + if initialize then + local state = initialize(...) + if not state == true then + return state end end + local etg = e.tg + if etg then + (functions[etg] or functions["@el@"])(e,handlers) + -- elseif type(e) == "string" then + -- functions["@tx@"](e,handlers) + else + functions["@dc@"](e,handlers) + end + if finalize then + return finalize() + end end -xml.serialize = serialize +local function xserialize(e,handlers) + local functions = handlers.functions + local etg = e.tg + if etg then + (functions[etg] or functions["@el@"])(e,handlers) + -- elseif type(e) == "string" then + -- functions["@tx@"](e,handlers) + else + functions["@dc@"](e,handlers) + end +end -function xml.checkbom(root) -- can be made faster - if root.ri then - local dt, found = root.dt, false - for k=1,#dt do - local v = dt[k] - if type(v) == "table" and v.special and v.tg == "@pi" and find(v.dt,"xml.*version=") then - found = true - break +local handlers = { } + +local function newhandlers(settings) + local t = table.copy(handlers.verbose or { }) -- merge + if settings then + for k,v in next, settings do + if type(v) == "table" then + tk = t[k] if not tk then tk = { } t[k] = tk end + for kk,vv in next, v do + tk[kk] = vv + end + else + t[k] = v end end - if not found then - insert(dt, 1, { special=true, ns="", tg="@pi@", dt = { "xml version='1.0' standalone='yes'"} } ) - insert(dt, 2, "\n" ) + if settings.name then + handlers[settings.name] = t end end + return t end +local nofunction = function() end + +function xml.sethandlersfunction(handler,name,fnc) + handler.functions[name] = fnc or nofunction +end + +function xml.gethandlersfunction(handler,name) + return handler.functions[name] +end + +function xml.gethandlers(name) + return handlers[name] +end + +newhandlers { + name = "verbose", + initialize = false, -- faster than nil and mt lookup + finalize = false, -- faster than nil and mt lookup + serialize = xserialize, + handle = print, + functions = { + ["@dc@"] = verbose_document, + ["@dt@"] = verbose_doctype, + ["@rt@"] = verbose_root, + ["@el@"] = verbose_element, + ["@pi@"] = verbose_pi, + ["@cm@"] = verbose_comment, + ["@cd@"] = verbose_cdata, + ["@tx@"] = verbose_text, + } +} + --[[ldx-- -<p>At the cost of some 25% runtime overhead you can first convert the tree to a string -and then handle the lot.</p> +<p>How you deal with saving data depends on your preferences. For a 40 MB database +file the timing on a 2.3 Core Duo are as follows (time in seconds):</p> + +<lines> +1.3 : load data from file to string +6.1 : convert string into tree +5.3 : saving in file using xmlsave +6.8 : converting to string using xml.tostring +3.6 : saving converted string in file +</lines> + +<p>Beware, these were timing with the old routine but measurements will not be that +much different I guess.</p> --ldx]]-- -function xml.tostring(root) -- 25% overhead due to collecting +-- maybe this will move to lxml-xml + +local result + +local xmlfilehandler = newhandlers { + name = "file", + initialize = function(name) result = io.open(name,"wb") return result end, + finalize = function() result:close() return true end, + handle = function(...) result:write(...) end, +} + +-- no checking on writeability here but not faster either +-- +-- local xmlfilehandler = newhandlers { +-- initialize = function(name) io.output(name,"wb") return true end, +-- finalize = function() io.close() return true end, +-- handle = io.write, +-- } + + +function xml.save(root,name) + serialize(root,xmlfilehandler,name) +end + +local result + +local xmlstringhandler = newhandlers { + name = "string", + initialize = function() result = { } return result end, + finalize = function() return concat(result) end, + handle = function(...) result[#result+1] = concat { ... } end +} + +local function xmltostring(root) -- 25% overhead due to collecting if root then if type(root) == 'string' then return root - elseif next(root) then -- next is faster than type (and >0 test) - local result = { } - serialize(root,function(s) result[#result+1] = s end) -- brrr, slow (direct printing is faster) - return concat(result,"") + else -- if next(root) then -- next is faster than type (and >0 test) + return serialize(root,xmlstringhandler) or "" end end return "" end +local function xmltext(root) -- inline + return (root and xmltostring(root)) or "" +end + +function initialize_mt(root) + mt = { __tostring = xmltext, __index = root } +end + +xml.defaulthandlers = handlers +xml.newhandlers = newhandlers +xml.serialize = serialize +xml.tostring = xmltostring + --[[ldx-- <p>The next function operated on the content only and needs a handle function that accepts a string.</p> --ldx]]-- -function xml.string(e,handle) +local function xmlstring(e,handle) if not handle or (e.special and e.tg ~= "@rt@") then -- nothing elseif e.tg then local edt = e.dt if edt then for i=1,#edt do - xml.string(edt[i],handle) + xmlstring(edt[i],handle) end end else @@ -3548,54 +4949,52 @@ function xml.string(e,handle) end end ---[[ldx-- -<p>How you deal with saving data depends on your preferences. For a 40 MB database -file the timing on a 2.3 Core Duo are as follows (time in seconds):</p> +xml.string = xmlstring -<lines> -1.3 : load data from file to string -6.1 : convert string into tree -5.3 : saving in file using xmlsave -6.8 : converting to string using xml.tostring -3.6 : saving converted string in file -</lines> - -<p>The save function is given below.</p> +--[[ldx-- +<p>A few helpers:</p> --ldx]]-- -function xml.save(root,name) - local f = io.open(name,"w") - if f then - xml.serialize(root,function(s) f:write(s) end) - f:close() +--~ xmlsetproperty(root,"settings",settings) + +function xml.settings(e) + while e do + local s = e.settings + if s then + return s + else + e = e.__p__ + end end + return nil end ---[[ldx-- -<p>A few helpers:</p> ---ldx]]-- - -function xml.body(root) - return (root.ri and root.dt[root.ri]) or root +function xml.root(e) + local r = e + while e do + e = e.__p__ + if e then + r = e + end + end + return r end -function xml.text(root) - return (root and xml.tostring(root)) or "" +function xml.parent(root) + return root.__p__ end -function xml.content(root) -- bugged - return (root and root.dt and xml.tostring(root.dt)) or "" +function xml.body(root) + return (root.ri and root.dt[root.ri]) or root -- not ok yet end -function xml.isempty(root, pattern) - if pattern == "" or pattern == "*" then - pattern = nil - end - if pattern then - -- todo - return false +function xml.name(root) + if not root then + return "" + elseif root.ns == "" then + return root.tg else - return not root or not root.dt or #root.dt == 0 or root.dt == "" + return root.ns .. ":" .. root.tg end end @@ -3603,18 +5002,15 @@ end <p>The next helper erases an element but keeps the table as it is, and since empty strings are not serialized (effectively) it does not harm. Copying the table would take more time. Usage:</p> - -<typing> -dt[k] = xml.empty() or xml.empty(dt,k) -</typing> --ldx]]-- -function xml.empty(dt,k) - if dt and k then - dt[k] = "" - return dt[k] - else - return "" +function xml.erase(dt,k) + if dt then + if k then + dt[k] = "" + else for k=1,#dt do + dt[1] = { "" } + end end end end @@ -3635,6 +5031,42 @@ function xml.assign(dt,k,root) end end +-- the following helpers may move + +--[[ldx-- +<p>The next helper assigns a tree (or string). Usage:</p> +<typing> +xml.tocdata(e) +xml.tocdata(e,"error") +</typing> +--ldx]]-- + +function xml.tocdata(e,wrapper) + local whatever = xmltostring(e.dt) + if wrapper then + whatever = format("<%s>%s</%s>",wrapper,whatever,wrapper) + end + local t = { special = true, ns = "", tg = "@cd@", at = {}, rn = "", dt = { whatever }, __p__ = e } + setmetatable(t,getmetatable(e)) + e.dt = { t } +end + +function xml.makestandalone(root) + if root.ri then + local dt = root.dt + for k=1,#dt do + local v = dt[k] + if type(v) == "table" and v.special and v.tg == "@pi@" then + local txt = v.dt[1] + if find(txt,"xml.*version=") then + v.dt[1] = txt .. " standalone='yes'" + break + end + end + end + end +end + end -- of closure @@ -3648,1420 +5080,1285 @@ if not modules then modules = { } end modules ['lxml-pth'] = { license = "see context related readme files" } +-- e.ni is only valid after a filter run + local concat, remove, insert = table.concat, table.remove, table.insert local type, next, tonumber, tostring, setmetatable, loadstring = type, next, tonumber, tostring, setmetatable, loadstring -local format, lower, gmatch, gsub, find = string.format, string.lower, string.gmatch, string.gsub, string.find +local format, upper, lower, gmatch, gsub, find, rep = string.format, string.upper, string.lower, string.gmatch, string.gsub, string.find, string.rep +local lpegmatch = lpeg.match + +-- beware, this is not xpath ... e.g. position is different (currently) and +-- we have reverse-sibling as reversed preceding sibling --[[ldx-- <p>This module can be used stand alone but also inside <l n='mkiv'/> in which case it hooks into the tracker code. Therefore we provide a few functions that set the tracers. Here we overload a previously defined function.</p> +<p>If I can get in the mood I will make a variant that is XSLT compliant +but I wonder if it makes sense.</P> --ldx]]-- -local trace_lpath = false - -if trackers then - trackers.register("xml.lpath", function(v) trace_lpath = v end) -end +--[[ldx-- +<p>Expecially the lpath code is experimental, we will support some of xpath, but +only things that make sense for us; as compensation it is possible to hook in your +own functions. Apart from preprocessing content for <l n='context'/> we also need +this module for process management, like handling <l n='ctx'/> and <l n='rlx'/> +files.</p> -local settrace = xml.settrace -- lxml-tab +<typing> +a/b/c /*/c +a/b/c/first() a/b/c/last() a/b/c/index(n) a/b/c/index(-n) +a/b/c/text() a/b/c/text(1) a/b/c/text(-1) a/b/c/text(n) +</typing> +--ldx]]-- -function xml.settrace(str,value) - if str == "lpath" then - trace_lpath = value or false - else - settrace(str,value) -- lxml-tab - end -end +local trace_lpath = false if trackers then trackers.register("xml.path", function(v) trace_lpath = v end) end +local trace_lparse = false if trackers then trackers.register("xml.parse", function(v) trace_lparse = v end) end +local trace_lprofile = false if trackers then trackers.register("xml.profile", function(v) trace_lpath = v trace_lparse = v trace_lprofile = v end) end --[[ldx-- -<p>We've now arrived at an intersting part: accessing the tree using a subset +<p>We've now arrived at an interesting part: accessing the tree using a subset of <l n='xpath'/> and since we're not compatible we call it <l n='lpath'/>. We will explain more about its usage in other documents.</p> --ldx]]-- -local lpathcalls = 0 -- statistics -local lpathcached = 0 -- statistics +local lpathcalls = 0 function xml.lpathcalls () return lpathcalls end +local lpathcached = 0 function xml.lpathcached() return lpathcached end -xml.functions = xml.functions or { } -xml.expressions = xml.expressions or { } +xml.functions = xml.functions or { } -- internal +xml.expressions = xml.expressions or { } -- in expressions +xml.finalizers = xml.finalizers or { } -- fast do-with ... (with return value other than collection) +xml.specialhandler = xml.specialhandler or { } local functions = xml.functions local expressions = xml.expressions +local finalizers = xml.finalizers -local actions = { - [10] = "stay", - [11] = "parent", - [12] = "subtree root", - [13] = "document root", - [14] = "any", - [15] = "many", - [16] = "initial", - [20] = "match", - [21] = "match one of", - [22] = "match and attribute eq", - [23] = "match and attribute ne", - [24] = "match one of and attribute eq", - [25] = "match one of and attribute ne", - [27] = "has attribute", - [28] = "has value", - [29] = "fast match", - [30] = "select", - [31] = "expression", - [40] = "processing instruction", -} - --- a rather dumb lpeg +finalizers.xml = finalizers.xml or { } +finalizers.tex = finalizers.tex or { } -local P, S, R, C, V, Cc = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc +local function fallback (t, name) + local fn = finalizers[name] + if fn then + t[name] = fn + else + logs.report("xml","unknown sub finalizer '%s'",tostring(name)) + fn = function() end + end + return fn +end --- instead of using functions we just parse a few names which saves a call --- later on +setmetatable(finalizers.xml, { __index = fallback }) +setmetatable(finalizers.tex, { __index = fallback }) -local lp_position = P("position()") / "ps" -local lp_index = P("index()") / "id" -local lp_text = P("text()") / "tx" -local lp_name = P("name()") / "(ns~='' and ns..':'..tg)" -- "((rt.ns~='' and rt.ns..':'..rt.tg) or '')" -local lp_tag = P("tag()") / "tg" -- (rt.tg or '') -local lp_ns = P("ns()") / "ns" -- (rt.ns or '') -local lp_noequal = P("!=") / "~=" + P("<=") + P(">=") + P("==") -local lp_doequal = P("=") / "==" -local lp_attribute = P("@") / "" * Cc("(at['") * R("az","AZ","--","__")^1 * Cc("'] or '')") +xml.defaultprotocol = "xml" -local lp_lua_function = C(R("az","AZ","--","__")^1 * (P(".") * R("az","AZ","--","__")^1)^1) * P("(") / function(t) -- todo: better . handling - return t .. "(" -end +-- as xsl does not follow xpath completely here we will also +-- be more liberal especially with regards to the use of | and +-- the rootpath: +-- +-- test : all 'test' under current +-- /test : 'test' relative to current +-- a|b|c : set of names +-- (a|b|c) : idem +-- ! : not +-- +-- after all, we're not doing transformations but filtering. in +-- addition we provide filter functions (last bit) +-- +-- todo: optimizer +-- +-- .. : parent +-- * : all kids +-- / : anchor here +-- // : /**/ +-- ** : all in between +-- +-- so far we had (more practical as we don't transform) +-- +-- {/test} : kids 'test' under current node +-- {test} : any kid with tag 'test' +-- {//test} : same as above -local lp_function = C(R("az","AZ","--","__")^1) * P("(") / function(t) -- todo: better . handling - if expressions[t] then - return "expressions." .. t .. "(" - else - return "expressions.error(" - end -end +-- evaluator (needs to be redone, for the moment copied) -local lparent = lpeg.P("(") -local rparent = lpeg.P(")") -local noparent = 1 - (lparent+rparent) -local nested = lpeg.P{lparent * (noparent + lpeg.V(1))^0 * rparent} -local value = lpeg.P(lparent * lpeg.C((noparent + nested)^0) * rparent) -- lpeg.P{"("*C(((1-S("()"))+V(1))^0)*")"} +-- todo: apply_axis(list,notable) and collection vs single --- if we use a dedicated namespace then we don't need to pass rt and k +local apply_axis = { } -local lp_special = (C(P("name")+P("text")+P("tag"))) * value / function(t,s) - if expressions[t] then - if s then - return "expressions." .. t .. "(r,k," .. s ..")" - else - return "expressions." .. t .. "(r,k)" +apply_axis['root'] = function(list) + local collected = { } + for l=1,#list do + local ll = list[l] + local rt = ll + while ll do + ll = ll.__p__ + if ll then + rt = ll + end end - else - return "expressions.error(" .. t .. ")" + collected[#collected+1] = rt end + return collected end -local converter = lpeg.Cs ( ( - lp_position + - lp_index + - lp_text + lp_name + -- fast one - lp_special + - lp_noequal + lp_doequal + - lp_attribute + - lp_lua_function + - lp_function + -1 )^1 ) - --- expressions,root,rootdt,k,e,edt,ns,tg,idx,hsh[tg] or 1 +apply_axis['self'] = function(list) +--~ local collected = { } +--~ for l=1,#list do +--~ collected[#collected+1] = list[l] +--~ end +--~ return collected + return list +end -local template = [[ - return function(expressions,r,d,k,e,dt,ns,tg,id,ps) - local at, tx = e.at or { }, dt[1] or "" - return %s +apply_axis['child'] = function(list) + local collected = { } + for l=1,#list do + local ll = list[l] + local dt = ll.dt + local en = 0 + for k=1,#dt do + local dk = dt[k] + if dk.tg then + collected[#collected+1] = dk + dk.ni = k -- refresh + en = en + 1 + dk.ei = en + end + end + ll.en = en end -]] - -local function make_expression(str) - str = converter:match(str) - return str, loadstring(format(template,str))() -end - -local map = { } - -local space = S(' \r\n\t') -local squote = S("'") -local dquote = S('"') -local lparent = P('(') -local rparent = P(')') -local atsign = P('@') -local lbracket = P('[') -local rbracket = P(']') -local exclam = P('!') -local period = P('.') -local eq = P('==') + P('=') -local ne = P('<>') + P('!=') -local star = P('*') -local slash = P('/') -local colon = P(':') -local bar = P('|') -local hat = P('^') -local valid = R('az', 'AZ', '09') + S('_-') -local name_yes = C(valid^1 + star) * colon * C(valid^1 + star) -- permits ns:* *:tg *:* -local name_nop = Cc("*") * C(valid^1) -local name = name_yes + name_nop -local number = C((S('+-')^0 * R('09')^1)) / tonumber -local names = (bar^0 * name)^1 -local morenames = name * (bar^0 * name)^1 -local instructiontag = P('pi::') -local spacing = C(space^0) -local somespace = space^1 -local optionalspace = space^0 -local text = C(valid^0) -local value = (squote * C((1 - squote)^0) * squote) + (dquote * C((1 - dquote)^0) * dquote) -local empty = 1-slash - -local is_eq = lbracket * atsign * name * eq * value * rbracket -local is_ne = lbracket * atsign * name * ne * value * rbracket -local is_attribute = lbracket * atsign * name * rbracket -local is_value = lbracket * value * rbracket -local is_number = lbracket * number * rbracket - -local nobracket = 1-(lbracket+rbracket) -- must be improved -local is_expression = lbracket * C(((C(nobracket^1))/make_expression)) * rbracket - -local is_expression = lbracket * (C(nobracket^1))/make_expression * rbracket - -local is_one = name -local is_none = exclam * name -local is_one_of = ((lparent * names * rparent) + morenames) -local is_none_of = exclam * ((lparent * names * rparent) + morenames) - -local stay = (period ) -local parent = (period * period ) / function( ) map[#map+1] = { 11 } end -local subtreeroot = (slash + hat ) / function( ) map[#map+1] = { 12 } end -local documentroot = (hat * hat ) / function( ) map[#map+1] = { 13 } end -local any = (star ) / function( ) map[#map+1] = { 14 } end -local many = (star * star ) / function( ) map[#map+1] = { 15 } end -local initial = (hat * hat * hat ) / function( ) map[#map+1] = { 16 } end - -local match = (is_one ) / function(...) map[#map+1] = { 20, true , ... } end -local match_one_of = (is_one_of ) / function(...) map[#map+1] = { 21, true , ... } end -local dont_match = (is_none ) / function(...) map[#map+1] = { 20, false, ... } end -local dont_match_one_of = (is_none_of ) / function(...) map[#map+1] = { 21, false, ... } end - -local match_and_eq = (is_one * is_eq ) / function(...) map[#map+1] = { 22, true , ... } end -local match_and_ne = (is_one * is_ne ) / function(...) map[#map+1] = { 23, true , ... } end -local dont_match_and_eq = (is_none * is_eq ) / function(...) map[#map+1] = { 22, false, ... } end -local dont_match_and_ne = (is_none * is_ne ) / function(...) map[#map+1] = { 23, false, ... } end - -local match_one_of_and_eq = (is_one_of * is_eq ) / function(...) map[#map+1] = { 24, true , ... } end -local match_one_of_and_ne = (is_one_of * is_ne ) / function(...) map[#map+1] = { 25, true , ... } end -local dont_match_one_of_and_eq = (is_none_of * is_eq ) / function(...) map[#map+1] = { 24, false, ... } end -local dont_match_one_of_and_ne = (is_none_of * is_ne ) / function(...) map[#map+1] = { 25, false, ... } end - -local has_attribute = (is_one * is_attribute) / function(...) map[#map+1] = { 27, true , ... } end -local has_value = (is_one * is_value ) / function(...) map[#map+1] = { 28, true , ... } end -local dont_has_attribute = (is_none * is_attribute) / function(...) map[#map+1] = { 27, false, ... } end -local dont_has_value = (is_none * is_value ) / function(...) map[#map+1] = { 28, false, ... } end -local position = (is_one * is_number ) / function(...) map[#map+1] = { 30, true, ... } end -local dont_position = (is_none * is_number ) / function(...) map[#map+1] = { 30, false, ... } end - -local expression = (is_one * is_expression)/ function(...) map[#map+1] = { 31, true, ... } end -local dont_expression = (is_none * is_expression)/ function(...) map[#map+1] = { 31, false, ... } end - -local self_expression = ( is_expression) / function(...) if #map == 0 then map[#map+1] = { 11 } end - map[#map+1] = { 31, true, "*", "*", ... } end -local dont_self_expression = (exclam * is_expression) / function(...) if #map == 0 then map[#map+1] = { 11 } end - map[#map+1] = { 31, false, "*", "*", ... } end - -local instruction = (instructiontag * text ) / function(...) map[#map+1] = { 40, ... } end -local nothing = (empty ) / function( ) map[#map+1] = { 15 } end -- 15 ? -local crap = (1-slash)^1 - --- a few ugly goodies: - -local docroottag = P('^^') / function( ) map[#map+1] = { 12 } end -local subroottag = P('^') / function( ) map[#map+1] = { 13 } end -local roottag = P('root::') / function( ) map[#map+1] = { 12 } end -local parenttag = P('parent::') / function( ) map[#map+1] = { 11 } end -local childtag = P('child::') -local selftag = P('self::') - --- there will be more and order will be optimized - -local selector = ( - instruction + --- many + any + -- brrr, not here ! - parent + stay + - dont_position + position + - dont_match_one_of_and_eq + dont_match_one_of_and_ne + - match_one_of_and_eq + match_one_of_and_ne + - dont_match_and_eq + dont_match_and_ne + - match_and_eq + match_and_ne + - dont_expression + expression + - dont_self_expression + self_expression + - has_attribute + has_value + - dont_match_one_of + match_one_of + - dont_match + match + - many + any + - crap + empty -) - -local grammar = P { "startup", - startup = (initial + documentroot + subtreeroot + roottag + docroottag + subroottag)^0 * V("followup"), - followup = ((slash + parenttag + childtag + selftag)^0 * selector)^1, -} + return collected +end -local function compose(str) - if not str or str == "" then - -- wildcard - return true - elseif str == '/' then - -- root - return false - else - map = { } - grammar:match(str) - if #map == 0 then - return true - else - local m = map[1][1] - if #map == 1 then - if m == 14 or m == 15 then - -- wildcard - return true - elseif m == 12 then - -- root - return false - end - elseif #map == 2 and m == 12 and map[2][1] == 20 then - -- return { { 29, map[2][2], map[2][3], map[2][4], map[2][5] } } - map[2][1] = 29 - return { map[2] } - end - if m ~= 11 and m ~= 12 and m ~= 13 and m ~= 14 and m ~= 15 and m ~= 16 then - insert(map, 1, { 16 }) +local function collect(list,collected) + local dt = list.dt + if dt then + local en = 0 + for k=1,#dt do + local dk = dt[k] + if dk.tg then + collected[#collected+1] = dk + dk.ni = k -- refresh + en = en + 1 + dk.ei = en + collect(dk,collected) end - -- print(gsub(table.serialize(map),"[ \n]+"," ")) - return map end + list.en = en end end +apply_axis['descendant'] = function(list) + local collected = { } + for l=1,#list do + collect(list[l],collected) + end + return collected +end -local cache = { } - -function xml.lpath(pattern,trace) - lpathcalls = lpathcalls + 1 - if type(pattern) == "string" then - local result = cache[pattern] - if result == nil then -- can be false which is valid -) - result = compose(pattern) - cache[pattern] = result - lpathcached = lpathcached + 1 +local function collect(list,collected) + local dt = list.dt + if dt then + local en = 0 + for k=1,#dt do + local dk = dt[k] + if dk.tg then + collected[#collected+1] = dk + dk.ni = k -- refresh + en = en + 1 + dk.ei = en + collect(dk,collected) + end end - if trace or trace_lpath then - xml.lshow(result) + list.en = en + end +end +apply_axis['descendant-or-self'] = function(list) + local collected = { } + for l=1,#list do + local ll = list[l] + if ll.special ~= true then -- catch double root + collected[#collected+1] = ll end - return result - else - return pattern + collect(ll,collected) end + return collected end -function xml.cached_patterns() - return cache +apply_axis['ancestor'] = function(list) + local collected = { } + for l=1,#list do + local ll = list[l] + while ll do + ll = ll.__p__ + if ll then + collected[#collected+1] = ll + end + end + end + return collected end --- we run out of locals (limited to 200) --- --- local fallbackreport = (texio and texio.write) or io.write - -function xml.lshow(pattern,report) --- report = report or fallbackreport - report = report or (texio and texio.write) or io.write - local lp = xml.lpath(pattern) - if lp == false then - report(" -: root\n") - elseif lp == true then - report(" -: wildcard\n") - else - if type(pattern) == "string" then - report(format("pattern: %s\n",pattern)) - end - for k=1,#lp do - local v = lp[k] - if #v > 1 then - local t = { } - for i=2,#v do - local vv = v[i] - if type(vv) == "string" then - t[#t+1] = (vv ~= "" and vv) or "#" - elseif type(vv) == "boolean" then - t[#t+1] = (vv and "==") or "<>" - end - end - report(format("%2i: %s %s -> %s\n", k,v[1],actions[v[1]],concat(t," "))) - else - report(format("%2i: %s %s\n", k,v[1],actions[v[1]])) +apply_axis['ancestor-or-self'] = function(list) + local collected = { } + for l=1,#list do + local ll = list[l] + collected[#collected+1] = ll + while ll do + ll = ll.__p__ + if ll then + collected[#collected+1] = ll end end end + return collected end -function xml.xshow(e,...) -- also handy when report is given, use () to isolate first e - local t = { ... } --- local report = (type(t[#t]) == "function" and t[#t]) or fallbackreport - local report = (type(t[#t]) == "function" and t[#t]) or (texio and texio.write) or io.write - if e == nil then - report("<!-- no element -->\n") - elseif type(e) ~= "table" then - report(tostring(e)) - elseif e.tg then - report(tostring(e) .. "\n") - else - for i=1,#e do - report(tostring(e[i]) .. "\n") +apply_axis['parent'] = function(list) + local collected = { } + for l=1,#list do + local pl = list[l].__p__ + if pl then + collected[#collected+1] = pl end end + return collected end ---[[ldx-- -<p>An <l n='lpath'/> is converted to a table with instructions for traversing the -tree. Hoever, simple cases are signaled by booleans. Because we don't know in -advance what we want to do with the found element the handle gets three arguments:</p> - -<lines> -<t>r</t> : the root element of the data table -<t>d</t> : the data table of the result -<t>t</t> : the index in the data table of the result -</lines> - -<p> Access to the root and data table makes it possible to construct insert and delete -functions.</p> ---ldx]]-- - -local functions = xml.functions -local expressions = xml.expressions - -expressions.contains = string.find -expressions.find = string.find -expressions.upper = string.upper -expressions.lower = string.lower -expressions.number = tonumber -expressions.boolean = toboolean - -expressions.oneof = function(s,...) -- slow - local t = {...} for i=1,#t do if s == t[i] then return true end end return false +apply_axis['attribute'] = function(list) + return { } end -expressions.error = function(str) - xml.error_handler("unknown function in lpath expression",str or "?") - return false +apply_axis['namespace'] = function(list) + return { } end -functions.text = function(root,k,n) -- unchecked, maybe one deeper - local t = type(t) - if t == "string" then - return t - else -- todo n - local rdt = root.dt - return (rdt and rdt[k]) or root[k] or "" - end +apply_axis['following'] = function(list) -- incomplete +--~ local collected = { } +--~ for l=1,#list do +--~ local ll = list[l] +--~ local p = ll.__p__ +--~ local d = p.dt +--~ for i=ll.ni+1,#d do +--~ local di = d[i] +--~ if type(di) == "table" then +--~ collected[#collected+1] = di +--~ break +--~ end +--~ end +--~ end +--~ return collected + return { } +end + +apply_axis['preceding'] = function(list) -- incomplete +--~ local collected = { } +--~ for l=1,#list do +--~ local ll = list[l] +--~ local p = ll.__p__ +--~ local d = p.dt +--~ for i=ll.ni-1,1,-1 do +--~ local di = d[i] +--~ if type(di) == "table" then +--~ collected[#collected+1] = di +--~ break +--~ end +--~ end +--~ end +--~ return collected + return { } end -functions.name = function(d,k,n) -- ns + tg - local found = false - n = n or 0 - if not k then - -- not found - elseif n == 0 then - local dk = d[k] - found = dk and (type(dk) == "table") and dk - elseif n < 0 then - for i=k-1,1,-1 do - local di = d[i] - if type(di) == "table" then - if n == -1 then - found = di - break - else - n = n + 1 - end - end - end - else - for i=k+1,#d,1 do +apply_axis['following-sibling'] = function(list) + local collected = { } + for l=1,#list do + local ll = list[l] + local p = ll.__p__ + local d = p.dt + for i=ll.ni+1,#d do local di = d[i] if type(di) == "table" then - if n == 1 then - found = di - break - else - n = n - 1 - end + collected[#collected+1] = di end end end - if found then - local ns, tg = found.rn or found.ns or "", found.tg - if ns ~= "" then - return ns .. ":" .. tg - else - return tg - end - else - return "" - end + return collected end -functions.tag = function(d,k,n) -- only tg - local found = false - n = n or 0 - if not k then - -- not found - elseif n == 0 then - local dk = d[k] - found = dk and (type(dk) == "table") and dk - elseif n < 0 then - for i=k-1,1,-1 do +apply_axis['preceding-sibling'] = function(list) + local collected = { } + for l=1,#list do + local ll = list[l] + local p = ll.__p__ + local d = p.dt + for i=1,ll.ni-1 do local di = d[i] if type(di) == "table" then - if n == -1 then - found = di - break - else - n = n + 1 - end + collected[#collected+1] = di end end - else - for i=k+1,#d,1 do + end + return collected +end + +apply_axis['reverse-sibling'] = function(list) -- reverse preceding + local collected = { } + for l=1,#list do + local ll = list[l] + local p = ll.__p__ + local d = p.dt + for i=ll.ni-1,1,-1 do local di = d[i] if type(di) == "table" then - if n == 1 then - found = di - break - else - n = n - 1 - end + collected[#collected+1] = di end end end - return (found and found.tg) or "" + return collected end -expressions.text = functions.text -expressions.name = functions.name -expressions.tag = functions.tag +apply_axis['auto-descendant-or-self'] = apply_axis['descendant-or-self'] +apply_axis['auto-descendant'] = apply_axis['descendant'] +apply_axis['auto-child'] = apply_axis['child'] +apply_axis['auto-self'] = apply_axis['self'] +apply_axis['initial-child'] = apply_axis['child'] -local function traverse(root,pattern,handle,reverse,index,parent,wildcard) -- multiple only for tags, not for namespaces - if not root then -- error - return false - elseif pattern == false then -- root - handle(root,root.dt,root.ri) - return false - elseif pattern == true then -- wildcard - local rootdt = root.dt - if rootdt then - local start, stop, step = 1, #rootdt, 1 - if reverse then - start, stop, step = stop, start, -1 - end - for k=start,stop,step do - if handle(root,rootdt,root.ri or k) then return false end - if not traverse(rootdt[k],true,handle,reverse) then return false end - end - end - return false - elseif root.dt then - index = index or 1 - local action = pattern[index] - local command = action[1] - if command == 29 then -- fast case /oeps - local rootdt = root.dt - for k=1,#rootdt do - local e = rootdt[k] - local tg = e.tg - if e.tg then - local ns = e.rn or e.ns - local ns_a, tg_a = action[3], action[4] - local matched = (ns_a == "*" or ns == ns_a) and (tg_a == "*" or tg == tg_a) - if not action[2] then matched = not matched end - if matched then - if handle(root,rootdt,k) then return false end - end - end - end - elseif command == 11 then -- parent - local ep = root.__p__ or parent - if index < #pattern then - if not traverse(ep,pattern,handle,reverse,index+1,root) then return false end - elseif handle(root,rootdt,k) then - return false +local function apply_nodes(list,directive,nodes) + -- todo: nodes[1] etc ... negated node name in set ... when needed + -- ... currently ignored + local maxn = #nodes + if maxn == 3 then --optimized loop + local nns, ntg = nodes[2], nodes[3] + if not nns and not ntg then -- wildcard + if directive then + return list + else + return { } end else - if (command == 16 or command == 12) and index == 1 then -- initial - -- wildcard = true - wildcard = command == 16 -- ok? - index = index + 1 - action = pattern[index] - command = action and action[1] or 0 -- something is wrong - end - if command == 11 then -- parent - local ep = root.__p__ or parent - if index < #pattern then - if not traverse(ep,pattern,handle,reverse,index+1,root) then return false end - elseif handle(root,rootdt,k) then - return false - end - else - local rootdt = root.dt - local start, stop, step, n, dn = 1, #rootdt, 1, 0, 1 - if command == 30 then - if action[5] < 0 then - start, stop, step = stop, start, -1 - dn = -1 - end - elseif reverse and index == #pattern then - start, stop, step = stop, start, -1 - end - local idx = 0 - local hsh = { } -- this will slooow down the lot - for k=start,stop,step do -- we used to have functions for all but a case is faster - local e = rootdt[k] - local ns, tg = e.rn or e.ns, e.tg - if tg then - -- we can optimize this for simple searches, but it probably does not pay off - hsh[tg] = (hsh[tg] or 0) + 1 - idx = idx + 1 - if command == 30 then - local ns_a, tg_a = action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - if not action[2] then matched = not matched end - if matched then - n = n + dn - if n == action[5] then - if index == #pattern then - if handle(root,rootdt,root.ri or k) then return false end - else - if not traverse(e,pattern,handle,reverse,index+1,root) then return false end - end - break - end - elseif wildcard then - if not traverse(e,pattern,handle,reverse,index,root,true) then return false end + local collected, m, p = { }, 0, nil + if not nns then -- only check tag + for l=1,#list do + local ll = list[l] + local ltg = ll.tg + if ltg then + if directive then + if ntg == ltg then + local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end + collected[#collected+1], ll.mi = ll, m end - else - local matched, multiple = false, false - if command == 20 then -- match - local ns_a, tg_a = action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - if not action[2] then matched = not matched end - elseif command == 21 then -- match one of - multiple = true - for i=3,#action,2 do - local ns_a, tg_a = action[i], action[i+1] - if (ns_a == "*" or ns == ns_a) and (tg == "*" or tg == tg_a) then - matched = true - break - end - end - if not action[2] then matched = not matched end - elseif command == 22 then -- eq - local ns_a, tg_a = action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - matched = matched and e.at[action[6]] == action[7] - elseif command == 23 then -- ne - local ns_a, tg_a = action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - if not action[2] then matched = not matched end - matched = mached and e.at[action[6]] ~= action[7] - elseif command == 24 then -- one of eq - multiple = true - for i=3,#action-2,2 do - local ns_a, tg_a = action[i], action[i+1] - if (ns_a == "*" or ns == ns_a) and (tg == "*" or tg == tg_a) then - matched = true - break - end - end - if not action[2] then matched = not matched end - matched = matched and e.at[action[#action-1]] == action[#action] - elseif command == 25 then -- one of ne - multiple = true - for i=3,#action-2,2 do - local ns_a, tg_a = action[i], action[i+1] - if (ns_a == "*" or ns == ns_a) and (tg == "*" or tg == tg_a) then - matched = true - break - end - end - if not action[2] then matched = not matched end - matched = matched and e.at[action[#action-1]] ~= action[#action] - elseif command == 27 then -- has attribute - local ns_a, tg_a = action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - if not action[2] then matched = not matched end - matched = matched and e.at[action[5]] - elseif command == 28 then -- has value - local edt, ns_a, tg_a = e.dt, action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - if not action[2] then matched = not matched end - matched = matched and edt and edt[1] == action[5] - elseif command == 31 then - local edt, ns_a, tg_a = e.dt, action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - if not action[2] then matched = not matched end - if matched then - matched = action[6](expressions,root,rootdt,k,e,edt,ns,tg,idx,hsh[tg] or 1) - end - end - if matched then -- combine tg test and at test - if index == #pattern then - if handle(root,rootdt,root.ri or k) then return false end - if wildcard then - if multiple then - if not traverse(e,pattern,handle,reverse,index,root,true) then return false end - else - -- maybe or multiple; anyhow, check on (section|title) vs just section and title in example in lxml - if not traverse(e,pattern,handle,reverse,index,root) then return false end - end - end - else - if not traverse(e,pattern,handle,reverse,index+1,root) then return false end - end - elseif command == 14 then -- any - if index == #pattern then - if handle(root,rootdt,root.ri or k) then return false end - else - if not traverse(e,pattern,handle,reverse,index+1,root) then return false end - end - elseif command == 15 then -- many - if index == #pattern then - if handle(root,rootdt,root.ri or k) then return false end - else - if not traverse(e,pattern,handle,reverse,index+1,root,true) then return false end - end - -- not here : 11 - elseif command == 11 then -- parent - local ep = e.__p__ or parent - if index < #pattern then - if not traverse(ep,pattern,handle,reverse,root,index+1) then return false end - elseif handle(root,rootdt,k) then - return false - end - elseif command == 40 and e.special and tg == "@pi@" then -- pi - local pi = action[2] - if pi ~= "" then - local pt = e.dt[1] - if pt and pt:find(pi) then - if handle(root,rootdt,k) then - return false - end - end - elseif handle(root,rootdt,k) then - return false - end - elseif wildcard then - if not traverse(e,pattern,handle,reverse,index,root,true) then return false end + elseif ntg ~= ltg then + local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end + collected[#collected+1], ll.mi = ll, m + end + end + end + elseif not ntg then -- only check namespace + for l=1,#list do + local ll = list[l] + local lns = ll.rn or ll.ns + if lns then + if directive then + if lns == nns then + local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end + collected[#collected+1], ll.mi = ll, m end + elseif lns ~= nns then + local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end + collected[#collected+1], ll.mi = ll, m end - else - -- not here : 11 - if command == 11 then -- parent - local ep = e.__p__ or parent - if index < #pattern then - if not traverse(ep,pattern,handle,reverse,index+1,root) then return false end - elseif handle(root,rootdt,k) then - return false + end + end + else -- check both + for l=1,#list do + local ll = list[l] + local ltg = ll.tg + if ltg then + local lns = ll.rn or ll.ns + local ok = ltg == ntg and lns == nns + if directive then + if ok then + local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end + collected[#collected+1], ll.mi = ll, m end - break -- else loop + elseif not ok then + local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end + collected[#collected+1], ll.mi = ll, m end end end end + return collected end + else + local collected, m, p = { }, 0, nil + for l=1,#list do + local ll = list[l] + local ltg = ll.tg + if ltg then + local lns = ll.rn or ll.ns + local ok = false + for n=1,maxn,3 do + local nns, ntg = nodes[n+1], nodes[n+2] + ok = (not ntg or ltg == ntg) and (not nns or lns == nns) + if ok then + break + end + end + if directive then + if ok then + local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end + collected[#collected+1], ll.mi = ll, m + end + elseif not ok then + local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end + collected[#collected+1], ll.mi = ll, m + end + end + end + return collected end - return true end -xml.traverse = traverse +local quit_expression = false ---[[ldx-- -<p>Next come all kind of locators and manipulators. The most generic function here -is <t>xml.filter(root,pattern)</t>. All registers functions in the filters namespace -can be path of a search path, as in:</p> +local function apply_expression(list,expression,order) + local collected = { } + quit_expression = false + for l=1,#list do + local ll = list[l] + if expression(list,ll,l,order) then -- nasty, order alleen valid als n=1 + collected[#collected+1] = ll + end + if quit_expression then + break + end + end + return collected +end + +local P, V, C, Cs, Cc, Ct, R, S, Cg, Cb = lpeg.P, lpeg.V, lpeg.C, lpeg.Cs, lpeg.Cc, lpeg.Ct, lpeg.R, lpeg.S, lpeg.Cg, lpeg.Cb + +local spaces = S(" \n\r\t\f")^0 +local lp_space = S(" \n\r\t\f") +local lp_any = P(1) +local lp_noequal = P("!=") / "~=" + P("<=") + P(">=") + P("==") +local lp_doequal = P("=") / "==" +local lp_or = P("|") / " or " +local lp_and = P("&") / " and " + +local lp_builtin = P ( + P("firstindex") / "1" + + P("lastindex") / "(#ll.__p__.dt or 1)" + + P("firstelement") / "1" + + P("lastelement") / "(ll.__p__.en or 1)" + + P("first") / "1" + + P("last") / "#list" + + P("rootposition") / "order" + + P("position") / "l" + -- is element in finalizer + P("order") / "order" + + P("element") / "(ll.ei or 1)" + + P("index") / "(ll.ni or 1)" + + P("match") / "(ll.mi or 1)" + + P("text") / "(ll.dt[1] or '')" + + -- P("name") / "(ll.ns~='' and ll.ns..':'..ll.tg)" + + P("name") / "((ll.ns~='' and ll.ns..':'..ll.tg) or ll.tg)" + + P("tag") / "ll.tg" + + P("ns") / "ll.ns" + ) * ((spaces * P("(") * spaces * P(")"))/"") + +local lp_attribute = (P("@") + P("attribute::")) / "" * Cc("(ll.at and ll.at['") * R("az","AZ","--","__")^1 * Cc("'])") +local lp_fastpos_p = ((P("+")^0 * R("09")^1 * P(-1)) / function(s) return "l==" .. s end) +local lp_fastpos_n = ((P("-") * R("09")^1 * P(-1)) / function(s) return "(" .. s .. "<0 and (#list+".. s .. "==l))" end) +local lp_fastpos = lp_fastpos_n + lp_fastpos_p +local lp_reserved = C("and") + C("or") + C("not") + C("div") + C("mod") + C("true") + C("false") + +local lp_lua_function = C(R("az","AZ","__")^1 * (P(".") * R("az","AZ","__")^1)^1) * ("(") / function(t) -- todo: better . handling + return t .. "(" +end -<typing> -local r, d, k = xml.filter(root,"/a/b/c/position(4)" -</typing> ---ldx]]-- +local lp_function = C(R("az","AZ","__")^1) * P("(") / function(t) -- todo: better . handling + if expressions[t] then + return "expr." .. t .. "(" + else + return "expr.error(" + end +end -local traverse, lpath, convert = xml.traverse, xml.lpath, xml.convert +local lparent = lpeg.P("(") +local rparent = lpeg.P(")") +local noparent = 1 - (lparent+rparent) +local nested = lpeg.P{lparent * (noparent + lpeg.V(1))^0 * rparent} +local value = lpeg.P(lparent * lpeg.C((noparent + nested)^0) * rparent) -- lpeg.P{"("*C(((1-S("()"))+V(1))^0)*")"} -xml.filters = { } +local lp_child = Cc("expr.child(ll,'") * R("az","AZ","--","__")^1 * Cc("')") +local lp_number = S("+-") * R("09")^1 +local lp_string = Cc("'") * R("az","AZ","--","__")^1 * Cc("'") +local lp_content = (P("'") * (1-P("'"))^0 * P("'") + P('"') * (1-P('"'))^0 * P('"')) -function xml.filters.default(root,pattern) - local rt, dt, dk - traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end) - return dt and dt[dk], rt, dt, dk -end +local cleaner -function xml.filters.attributes(root,pattern,arguments) - local rt, dt, dk - traverse(root, lpath(pattern), function(r,d,k) rt, dt, dk = r, d, k return true end) - local ekat = (dt and dt[dk] and dt[dk].at) or (rt and rt.at) - if ekat then - if arguments then - return ekat[arguments] or "", rt, dt, dk +local lp_special = (C(P("name")+P("text")+P("tag")+P("count")+P("child"))) * value / function(t,s) + if expressions[t] then + s = s and s ~= "" and lpegmatch(cleaner,s) + if s and s ~= "" then + return "expr." .. t .. "(ll," .. s ..")" else - return ekat, rt, dt, dk + return "expr." .. t .. "(ll)" end else - return { }, rt, dt, dk + return "expr.error(" .. t .. ")" end end -function xml.filters.reverse(root,pattern) - local rt, dt, dk - traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end, 'reverse') - return dt and dt[dk], rt, dt, dk -end +local content = + lp_builtin + + lp_attribute + + lp_special + + lp_noequal + lp_doequal + + lp_or + lp_and + + lp_reserved + + lp_lua_function + lp_function + + lp_content + -- too fragile + lp_child + + lp_any + +local converter = Cs ( + lp_fastpos + (P { lparent * (V(1))^0 * rparent + content } )^0 +) -function xml.filters.count(root,pattern,everything) - local n = 0 - traverse(root, lpath(pattern), function(r,d,t) - if everything or type(d[t]) == "table" then - n = n + 1 - end - end) - return n -end +cleaner = Cs ( ( +--~ lp_fastpos + + lp_reserved + + lp_number + + lp_string + +1 )^1 ) -function xml.filters.elements(root, pattern) -- == all - local t = { } - traverse(root, lpath(pattern), function(r,d,k) - local e = d[k] - if e then - t[#t+1] = e - end - end) - return t -end -function xml.filters.texts(root, pattern) - local t = { } - traverse(root, lpath(pattern), function(r,d,k) - local e = d[k] - if e and e.dt then - t[#t+1] = e.dt - end - end) - return t -end +--~ expr -function xml.filters.first(root,pattern) - local rt, dt, dk - traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end) - return dt and dt[dk], rt, dt, dk -end +local template_e = [[ + local expr = xml.expressions + return function(list,ll,l,order) + return %s + end +]] -function xml.filters.last(root,pattern) - local rt, dt, dk - traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end, 'reverse') - return dt and dt[dk], rt, dt, dk -end +local template_f_y = [[ + local finalizer = xml.finalizers['%s']['%s'] + return function(collection) + return finalizer(collection,%s) + end +]] -function xml.filters.index(root,pattern,arguments) - local rt, dt, dk, reverse, i = nil, nil, nil, false, tonumber(arguments or '1') or 1 - if i and i ~= 0 then - if i < 0 then - reverse, i = true, -i - end - traverse(root, lpath(pattern), function(r,d,k) rt, dt, dk, i = r, d, k, i-1 return i == 0 end, reverse) - if i == 0 then - return dt and dt[dk], rt, dt, dk - end +local template_f_n = [[ + return xml.finalizers['%s']['%s'] +]] + +-- + +local register_self = { kind = "axis", axis = "self" } -- , apply = apply_axis["self"] } +local register_parent = { kind = "axis", axis = "parent" } -- , apply = apply_axis["parent"] } +local register_descendant = { kind = "axis", axis = "descendant" } -- , apply = apply_axis["descendant"] } +local register_child = { kind = "axis", axis = "child" } -- , apply = apply_axis["child"] } +local register_descendant_or_self = { kind = "axis", axis = "descendant-or-self" } -- , apply = apply_axis["descendant-or-self"] } +local register_root = { kind = "axis", axis = "root" } -- , apply = apply_axis["root"] } +local register_ancestor = { kind = "axis", axis = "ancestor" } -- , apply = apply_axis["ancestor"] } +local register_ancestor_or_self = { kind = "axis", axis = "ancestor-or-self" } -- , apply = apply_axis["ancestor-or-self"] } +local register_attribute = { kind = "axis", axis = "attribute" } -- , apply = apply_axis["attribute"] } +local register_namespace = { kind = "axis", axis = "namespace" } -- , apply = apply_axis["namespace"] } +local register_following = { kind = "axis", axis = "following" } -- , apply = apply_axis["following"] } +local register_following_sibling = { kind = "axis", axis = "following-sibling" } -- , apply = apply_axis["following-sibling"] } +local register_preceding = { kind = "axis", axis = "preceding" } -- , apply = apply_axis["preceding"] } +local register_preceding_sibling = { kind = "axis", axis = "preceding-sibling" } -- , apply = apply_axis["preceding-sibling"] } +local register_reverse_sibling = { kind = "axis", axis = "reverse-sibling" } -- , apply = apply_axis["reverse-sibling"] } + +local register_auto_descendant_or_self = { kind = "axis", axis = "auto-descendant-or-self" } -- , apply = apply_axis["auto-descendant-or-self"] } +local register_auto_descendant = { kind = "axis", axis = "auto-descendant" } -- , apply = apply_axis["auto-descendant"] } +local register_auto_self = { kind = "axis", axis = "auto-self" } -- , apply = apply_axis["auto-self"] } +local register_auto_child = { kind = "axis", axis = "auto-child" } -- , apply = apply_axis["auto-child"] } + +local register_initial_child = { kind = "axis", axis = "initial-child" } -- , apply = apply_axis["initial-child"] } + +local register_all_nodes = { kind = "nodes", nodetest = true, nodes = { true, false, false } } + +local skip = { } + +local function errorrunner_e(str,cnv) + if not skip[str] then + logs.report("lpath","error in expression: %s => %s",str,cnv) + skip[str] = cnv or str end - return nil, nil, nil, nil + return false +end +local function errorrunner_f(str,arg) + logs.report("lpath","error in finalizer: %s(%s)",str,arg or "") + return false end -function xml.filters.attribute(root,pattern,arguments) - local rt, dt, dk - traverse(root, lpath(pattern), function(r,d,k) rt, dt, dk = r, d, k return true end) - local ekat = (dt and dt[dk] and dt[dk].at) or (rt and rt.at) - return (ekat and (ekat[arguments] or ekat[gsub(arguments,"^([\"\'])(.*)%1$","%2")])) or "" +local function register_nodes(nodetest,nodes) + return { kind = "nodes", nodetest = nodetest, nodes = nodes } end -function xml.filters.text(root,pattern,arguments) -- ?? why index, tostring slow - local dtk, rt, dt, dk = xml.filters.index(root,pattern,arguments) - if dtk then -- n - local dtkdt = dtk.dt - if not dtkdt then - return "", rt, dt, dk - elseif #dtkdt == 1 and type(dtkdt[1]) == "string" then - return dtkdt[1], rt, dt, dk - else - return xml.tostring(dtkdt), rt, dt, dk - end +local function register_expression(expression) + local converted = lpegmatch(converter,expression) + local runner = loadstring(format(template_e,converted)) + runner = (runner and runner()) or function() errorrunner_e(expression,converted) end + return { kind = "expression", expression = expression, converted = converted, evaluator = runner } +end + +local function register_finalizer(protocol,name,arguments) + local runner + if arguments and arguments ~= "" then + runner = loadstring(format(template_f_y,protocol or xml.defaultprotocol,name,arguments)) else - return "", rt, dt, dk + runner = loadstring(format(template_f_n,protocol or xml.defaultprotocol,name)) end + runner = (runner and runner()) or function() errorrunner_f(name,arguments) end + return { kind = "finalizer", name = name, arguments = arguments, finalizer = runner } end -function xml.filters.tag(root,pattern,n) - local tag = "" - traverse(root, lpath(pattern), function(r,d,k) - tag = xml.functions.tag(d,k,n and tonumber(n)) - return true - end) - return tag -end +local expression = P { "ex", + ex = "[" * C((V("sq") + V("dq") + (1 - S("[]")) + V("ex"))^0) * "]", + sq = "'" * (1 - S("'"))^0 * "'", + dq = '"' * (1 - S('"'))^0 * '"', +} -function xml.filters.name(root,pattern,n) - local tag = "" - traverse(root, lpath(pattern), function(r,d,k) - tag = xml.functions.name(d,k,n and tonumber(n)) - return true - end) - return tag +local arguments = P { "ar", + ar = "(" * Cs((V("sq") + V("dq") + V("nq") + P(1-P(")")))^0) * ")", + nq = ((1 - S("),'\""))^1) / function(s) return format("%q",s) end, + sq = P("'") * (1 - P("'"))^0 * P("'"), + dq = P('"') * (1 - P('"'))^0 * P('"'), +} + +-- todo: better arg parser + +local function register_error(str) + return { kind = "error", error = format("unparsed: %s",str) } end ---[[ldx-- -<p>For splitting the filter function from the path specification, we can -use string matching or lpeg matching. Here the difference in speed is -neglectable but the lpeg variant is more robust.</p> ---ldx]]-- +-- there is a difference in * and /*/ and so we need to catch a few special cases + +local special_1 = P("*") * Cc(register_auto_descendant) * Cc(register_all_nodes) -- last one not needed +local special_2 = P("/") * Cc(register_auto_self) +local special_3 = P("") * Cc(register_auto_self) + +local parser = Ct { "patterns", -- can be made a bit faster by moving pattern outside + + patterns = spaces * V("protocol") * spaces * ( + ( V("special") * spaces * P(-1) ) + + ( V("initial") * spaces * V("step") * spaces * (P("/") * spaces * V("step") * spaces)^0 ) + ), + + protocol = Cg(V("letters"),"protocol") * P("://") + Cg(Cc(nil),"protocol"), + + -- the / is needed for // as descendant or self is somewhat special + -- step = (V("shortcuts") + V("axis") * spaces * V("nodes")^0 + V("error")) * spaces * V("expressions")^0 * spaces * V("finalizer")^0, + step = ((V("shortcuts") + P("/") + V("axis")) * spaces * V("nodes")^0 + V("error")) * spaces * V("expressions")^0 * spaces * V("finalizer")^0, + + axis = V("descendant") + V("child") + V("parent") + V("self") + V("root") + V("ancestor") + + V("descendant_or_self") + V("following_sibling") + V("following") + + V("reverse_sibling") + V("preceding_sibling") + V("preceding") + V("ancestor_or_self") + + #(1-P(-1)) * Cc(register_auto_child), + + special = special_1 + special_2 + special_3, --- not faster but hipper ... although ... i can't get rid of the trailing / in the path + initial = (P("/") * spaces * Cc(register_initial_child))^-1, -local P, S, R, C, V, Cc = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc + error = (P(1)^1) / register_error, -local slash = P('/') -local name = (R("az","AZ","--","__"))^1 -local path = C(((1-slash)^0 * slash)^1) -local argument = P { "(" * C(((1 - S("()")) + V(1))^0) * ")" } -local action = Cc(1) * path * C(name) * argument -local attribute = Cc(2) * path * P('@') * C(name) -local direct = Cc(3) * Cc("../*") * slash^0 * C(name) * argument + shortcuts_a = V("s_descendant_or_self") + V("s_descendant") + V("s_child") + V("s_parent") + V("s_self") + V("s_root") + V("s_ancestor"), -local parser = direct + action + attribute + shortcuts = V("shortcuts_a") * (spaces * "/" * spaces * V("shortcuts_a"))^0, -local filters = xml.filters -local attribute_filter = xml.filters.attributes -local default_filter = xml.filters.default + s_descendant_or_self = (P("***/") + P("/")) * Cc(register_descendant_or_self), --- *** is a bonus + -- s_descendant_or_self = P("/") * Cc(register_descendant_or_self), + s_descendant = P("**") * Cc(register_descendant), + s_child = P("*") * #(1-P(":")) * Cc(register_child ), +-- s_child = P("*") * #(P("/")+P(-1)) * Cc(register_child ), + s_parent = P("..") * Cc(register_parent ), + s_self = P("." ) * Cc(register_self ), + s_root = P("^^") * Cc(register_root ), + s_ancestor = P("^") * Cc(register_ancestor ), --- todo: also hash, could be gc'd + descendant = P("descendant::") * Cc(register_descendant ), + child = P("child::") * Cc(register_child ), + parent = P("parent::") * Cc(register_parent ), + self = P("self::") * Cc(register_self ), + root = P('root::') * Cc(register_root ), + ancestor = P('ancestor::') * Cc(register_ancestor ), + descendant_or_self = P('descendant-or-self::') * Cc(register_descendant_or_self ), + ancestor_or_self = P('ancestor-or-self::') * Cc(register_ancestor_or_self ), + -- attribute = P('attribute::') * Cc(register_attribute ), + -- namespace = P('namespace::') * Cc(register_namespace ), + following = P('following::') * Cc(register_following ), + following_sibling = P('following-sibling::') * Cc(register_following_sibling ), + preceding = P('preceding::') * Cc(register_preceding ), + preceding_sibling = P('preceding-sibling::') * Cc(register_preceding_sibling ), + reverse_sibling = P('reverse-sibling::') * Cc(register_reverse_sibling ), + + nodes = (V("nodefunction") * spaces * P("(") * V("nodeset") * P(")") + V("nodetest") * V("nodeset")) / register_nodes, + + expressions = expression / register_expression, + + letters = R("az")^1, + name = (1-lpeg.S("/[]()|:*!"))^1, + negate = P("!") * Cc(false), + + nodefunction = V("negate") + P("not") * Cc(false) + Cc(true), + nodetest = V("negate") + Cc(true), + nodename = (V("negate") + Cc(true)) * spaces * ((V("wildnodename") * P(":") * V("wildnodename")) + (Cc(false) * V("wildnodename"))), + wildnodename = (C(V("name")) + P("*") * Cc(false)) * #(1-P("(")), + nodeset = spaces * Ct(V("nodename") * (spaces * P("|") * spaces * V("nodename"))^0) * spaces, + + finalizer = (Cb("protocol") * P("/")^-1 * C(V("name")) * arguments * P(-1)) / register_finalizer, + +} + +local cache = { } -function xml.filter(root,pattern) - local kind, a, b, c = parser:match(pattern) - if kind == 1 or kind == 3 then - return (filters[b] or default_filter)(root,a,c) - elseif kind == 2 then - return attribute_filter(root,a,b) +local function nodesettostring(set,nodetest) + local t = { } + for i=1,#set,3 do + local directive, ns, tg = set[i], set[i+1], set[i+2] + if not ns or ns == "" then ns = "*" end + if not tg or tg == "" then tg = "*" end + tg = (tg == "@rt@" and "[root]") or format("%s:%s",ns,tg) + t[#t+1] = (directive and tg) or format("not(%s)",tg) + end + if nodetest == false then + return format("not(%s)",concat(t,"|")) else - return default_filter(root,pattern) + return concat(t,"|") end end ---~ slightly faster, but first we need a proper test file ---~ ---~ local hash = { } ---~ ---~ function xml.filter(root,pattern) ---~ local h = hash[pattern] ---~ if not h then ---~ local kind, a, b, c = parser:match(pattern) ---~ if kind == 1 then ---~ h = { kind, filters[b] or default_filter, a, b, c } ---~ elseif kind == 2 then ---~ h = { kind, attribute_filter, a, b, c } ---~ else ---~ h = { kind, default_filter, a, b, c } ---~ end ---~ hash[pattern] = h ---~ end ---~ local kind = h[1] ---~ if kind == 1 then ---~ return h[2](root,h[2],h[4]) ---~ elseif kind == 2 then ---~ return h[2](root,h[2],h[3]) ---~ else ---~ return h[2](root,pattern) ---~ end ---~ end +local function tagstostring(list) + if #list == 0 then + return "no elements" + else + local t = { } + for i=1, #list do + local li = list[i] + local ns, tg = li.ns, li.tg + if not ns or ns == "" then ns = "*" end + if not tg or tg == "" then tg = "*" end + t[#t+1] = (tg == "@rt@" and "[root]") or format("%s:%s",ns,tg) + end + return concat(t," ") + end +end ---[[ldx-- -<p>The following functions collect elements and texts.</p> ---ldx]]-- +xml.nodesettostring = nodesettostring --- still somewhat bugged +local parse_pattern -- we have a harmless kind of circular reference -function xml.collect_elements(root, pattern, ignorespaces) - local rr, dd = { }, { } - traverse(root, lpath(pattern), function(r,d,k) - local dk = d and d[k] - if dk then - if ignorespaces and type(dk) == "string" and dk:find("[^%S]") then - -- ignore - else - local n = #rr+1 - rr[n], dd[n] = r, dk - end - end - end) - return dd, rr +local function lshow(parsed) + if type(parsed) == "string" then + parsed = parse_pattern(parsed) + end + local s = table.serialize_functions -- ugly + table.serialize_functions = false -- ugly + logs.report("lpath","%s://%s => %s",parsed.protocol or xml.defaultprotocol,parsed.pattern,table.serialize(parsed,false)) + table.serialize_functions = s -- ugly end -function xml.collect_texts(root, pattern, flatten) - local t = { } -- no r collector - traverse(root, lpath(pattern), function(r,d,k) - if d then - local ek = d[k] - local tx = ek and ek.dt - if flatten then - if tx then - t[#t+1] = xml.tostring(tx) or "" +xml.lshow = lshow + +local function add_comment(p,str) + local pc = p.comment + if not pc then + p.comment = { str } + else + pc[#pc+1] = str + end +end + +parse_pattern = function (pattern) -- the gain of caching is rather minimal + lpathcalls = lpathcalls + 1 + if type(pattern) == "table" then + return pattern + else + local parsed = cache[pattern] + if parsed then + lpathcached = lpathcached + 1 + else + parsed = lpegmatch(parser,pattern) + if parsed then + parsed.pattern = pattern + local np = #parsed + if np == 0 then + parsed = { pattern = pattern, register_self, state = "parsing error" } + logs.report("lpath","parsing error in '%s'",pattern) + lshow(parsed) else - t[#t+1] = "" + -- we could have done this with a more complex parser but this + -- is cleaner + local pi = parsed[1] + if pi.axis == "auto-child" then + if false then + add_comment(parsed, "auto-child replaced by auto-descendant-or-self") + parsed[1] = register_auto_descendant_or_self + else + add_comment(parsed, "auto-child replaced by auto-descendant") + parsed[1] = register_auto_descendant + end + elseif pi.axis == "initial-child" and np > 1 and parsed[2].axis then + add_comment(parsed, "initial-child removed") -- we could also make it a auto-self + remove(parsed,1) + end + local np = #parsed -- can have changed + if np > 1 then + local pnp = parsed[np] + if pnp.kind == "nodes" and pnp.nodetest == true then + local nodes = pnp.nodes + if nodes[1] == true and nodes[2] == false and nodes[3] == false then + add_comment(parsed, "redundant final wildcard filter removed") + remove(parsed,np) + end + end + end end else - t[#t+1] = tx or "" + parsed = { pattern = pattern } + end + cache[pattern] = parsed + if trace_lparse and not trace_lprofile then + lshow(parsed) end - else - t[#t+1] = "" end - end) - return t + return parsed + end end -function xml.collect_tags(root, pattern, nonamespace) - local t = { } - xml.traverse(root, xml.lpath(pattern), function(r,d,k) - local dk = d and d[k] - if dk and type(dk) == "table" then - local ns, tg = e.ns, e.tg - if nonamespace then - t[#t+1] = tg -- if needed we can return an extra table - elseif ns == "" then - t[#t+1] = tg - else - t[#t+1] = ns .. ":" .. tg +-- we can move all calls inline and then merge the trace back +-- technically we can combine axis and the next nodes which is +-- what we did before but this a bit cleaner (but slower too) +-- but interesting is that it's not that much faster when we +-- go inline +-- +-- beware: we need to return a collection even when we filter +-- else the (simple) cache gets messed up + +-- caching found lookups saves not that much (max .1 sec on a 8 sec run) +-- and it also messes up finalizers + +-- watch out: when there is a finalizer, it's always called as there +-- can be cases that a finalizer returns (or does) something in case +-- there is no match; an example of this is count() + +local profiled = { } xml.profiled = profiled + +local function profiled_apply(list,parsed,nofparsed,order) + local p = profiled[parsed.pattern] + if p then + p.tested = p.tested + 1 + else + p = { tested = 1, matched = 0, finalized = 0 } + profiled[parsed.pattern] = p + end + local collected = list + for i=1,nofparsed do + local pi = parsed[i] + local kind = pi.kind + if kind == "axis" then + collected = apply_axis[pi.axis](collected) + elseif kind == "nodes" then + collected = apply_nodes(collected,pi.nodetest,pi.nodes) + elseif kind == "expression" then + collected = apply_expression(collected,pi.evaluator,order) + elseif kind == "finalizer" then + collected = pi.finalizer(collected) + p.matched = p.matched + 1 + p.finalized = p.finalized + 1 + return collected + end + if not collected or #collected == 0 then + local pn = i < nofparsed and parsed[nofparsed] + if pn and pn.kind == "finalizer" then + collected = pn.finalizer(collected) + p.finalized = p.finalized + 1 + return collected end + return nil end - end) - return #t > 0 and {} + end + if collected then + p.matched = p.matched + 1 + end + return collected +end + +local function traced_apply(list,parsed,nofparsed,order) + if trace_lparse then + lshow(parsed) + end + logs.report("lpath", "collecting : %s",parsed.pattern) + logs.report("lpath", " root tags : %s",tagstostring(list)) + logs.report("lpath", " order : %s",order or "unset") + local collected = list + for i=1,nofparsed do + local pi = parsed[i] + local kind = pi.kind + if kind == "axis" then + collected = apply_axis[pi.axis](collected) + logs.report("lpath", "% 10i : ax : %s",(collected and #collected) or 0,pi.axis) + elseif kind == "nodes" then + collected = apply_nodes(collected,pi.nodetest,pi.nodes) + logs.report("lpath", "% 10i : ns : %s",(collected and #collected) or 0,nodesettostring(pi.nodes,pi.nodetest)) + elseif kind == "expression" then + collected = apply_expression(collected,pi.evaluator,order) + logs.report("lpath", "% 10i : ex : %s -> %s",(collected and #collected) or 0,pi.expression,pi.converted) + elseif kind == "finalizer" then + collected = pi.finalizer(collected) + logs.report("lpath", "% 10i : fi : %s : %s(%s)",(type(collected) == "table" and #collected) or 0,parsed.protocol or xml.defaultprotocol,pi.name,pi.arguments or "") + return collected + end + if not collected or #collected == 0 then + local pn = i < nofparsed and parsed[nofparsed] + if pn and pn.kind == "finalizer" then + collected = pn.finalizer(collected) + logs.report("lpath", "% 10i : fi : %s : %s(%s)",(type(collected) == "table" and #collected) or 0,parsed.protocol or xml.defaultprotocol,pn.name,pn.arguments or "") + return collected + end + return nil + end + end + return collected end ---[[ldx-- -<p>Often using an iterators looks nicer in the code than passing handler -functions. The <l n='lua'/> book describes how to use coroutines for that -purpose (<url href='http://www.lua.org/pil/9.3.html'/>). This permits -code like:</p> - -<typing> -for r, d, k in xml.elements(xml.load('text.xml'),"title") do - print(d[k]) +local function normal_apply(list,parsed,nofparsed,order) + local collected = list + for i=1,nofparsed do + local pi = parsed[i] + local kind = pi.kind + if kind == "axis" then + local axis = pi.axis + if axis ~= "self" then + collected = apply_axis[axis](collected) + end + elseif kind == "nodes" then + collected = apply_nodes(collected,pi.nodetest,pi.nodes) + elseif kind == "expression" then + collected = apply_expression(collected,pi.evaluator,order) + elseif kind == "finalizer" then + return pi.finalizer(collected) + end + if not collected or #collected == 0 then + local pf = i < nofparsed and parsed[nofparsed].finalizer + if pf then + return pf(collected) -- can be anything + end + return nil + end + end + return collected end -</typing> - -<p>Which will print all the titles in the document. The iterator variant takes -1.5 times the runtime of the function variant which is due to the overhead in -creating the wrapper. So, instead of:</p> -<typing> -function xml.filters.first(root,pattern) - for rt,dt,dk in xml.elements(root,pattern) - return dt and dt[dk], rt, dt, dk +local function parse_apply(list,pattern) + -- we avoid an extra call + local parsed = cache[pattern] + if parsed then + lpathcalls = lpathcalls + 1 + lpathcached = lpathcached + 1 + elseif type(pattern) == "table" then + lpathcalls = lpathcalls + 1 + parsed = pattern + else + parsed = parse_pattern(pattern) or pattern + end + if not parsed then + return + end + local nofparsed = #parsed + if nofparsed == 0 then + return -- something is wrong + end + local one = list[1] + if not one then + return -- something is wrong + elseif not trace_lpath then + return normal_apply(list,parsed,nofparsed,one.mi) + elseif trace_lprofile then + return profiled_apply(list,parsed,nofparsed,one.mi) + else + return traced_apply(list,parsed,nofparsed,one.mi) end - return nil, nil, nil, nil end -</typing> -<p>We use the function variants in the filters.</p> ---ldx]]-- - -local wrap, yield = coroutine.wrap, coroutine.yield +-- internal (parsed) -function xml.elements(root,pattern,reverse) - return wrap(function() traverse(root, lpath(pattern), yield, reverse) end) +expressions.child = function(e,pattern) + return parse_apply({ e },pattern) -- todo: cache end - -function xml.elements_only(root,pattern,reverse) - return wrap(function() traverse(root, lpath(pattern), function(r,d,k) yield(d[k]) end, reverse) end) +expressions.count = function(e,pattern) + local collected = parse_apply({ e },pattern) -- todo: cache + return (collected and #collected) or 0 end -function xml.each_element(root, pattern, handle, reverse) - local ok - traverse(root, lpath(pattern), function(r,d,k) ok = true handle(r,d,k) end, reverse) - return ok +-- external + +expressions.oneof = function(s,...) -- slow + local t = {...} for i=1,#t do if s == t[i] then return true end end return false +end +expressions.error = function(str) + xml.error_handler("unknown function in lpath expression",tostring(str or "?")) + return false +end +expressions.undefined = function(s) + return s == nil end -function xml.process_elements(root, pattern, handle) - traverse(root, lpath(pattern), function(r,d,k) - local dkdt = d[k].dt - if dkdt then - for i=1,#dkdt do - local v = dkdt[i] - if v.tg then handle(v) end - end - end - end) +expressions.quit = function(s) + if s or s == nil then + quit_expression = true + end + return true end -function xml.process_attributes(root, pattern, handle) - traverse(root, lpath(pattern), function(r,d,k) - local ek = d[k] - local a = ek.at or { } - handle(a) - if next(a) then -- next is faster than type (and >0 test) - ek.at = a - else - ek.at = nil - end - end) +expressions.print = function(...) + print(...) + return true end ---[[ldx-- -<p>We've now arrives at the functions that manipulate the tree.</p> ---ldx]]-- +expressions.contains = find +expressions.find = find +expressions.upper = upper +expressions.lower = lower +expressions.number = tonumber +expressions.boolean = toboolean -function xml.inject_element(root, pattern, element, prepend) - if root and element then - local matches, collect = { }, nil - if type(element) == "string" then - element = convert(element,true) - end - if element then - collect = function(r,d,k) matches[#matches+1] = { r, d, k, element } end - traverse(root, lpath(pattern), collect) - for i=1,#matches do - local m = matches[i] - local r, d, k, element, edt = m[1], m[2], m[3], m[4], nil - if element.ri then - element = element.dt[element.ri].dt - else - element = element.dt - end - if r.ri then - edt = r.dt[r.ri].dt - else - edt = d and d[k] and d[k].dt - end - if edt then - local be, af - if prepend then - be, af = xml.copy(element), edt - else - be, af = edt, xml.copy(element) - end - for i=1,#af do - be[#be+1] = af[i] - end - if r.ri then - r.dt[r.ri].dt = be - else - d[k].dt = be - end - else - -- r.dt = element.dt -- todo - end - end +-- user interface + +local function traverse(root,pattern,handle) + logs.report("xml","use 'xml.selection' instead for '%s'",pattern) + local collected = parse_apply({ root },pattern) + if collected then + for c=1,#collected do + local e = collected[c] + local r = e.__p__ + handle(r,r.dt,e.ni) end end end --- todo: copy ! - -function xml.insert_element(root, pattern, element, before) -- todo: element als functie - if root and element then - if pattern == "/" then - xml.inject_element(root, pattern, element, before) - else - local matches, collect = { }, nil - if type(element) == "string" then - element = convert(element,true) - end - if element and element.ri then - element = element.dt[element.ri] - end - if element then - collect = function(r,d,k) matches[#matches+1] = { r, d, k, element } end - traverse(root, lpath(pattern), collect) - for i=#matches,1,-1 do - local m = matches[i] - local r, d, k, element = m[1], m[2], m[3], m[4] - if not before then k = k + 1 end - if element.tg then - insert(d,k,element) -- untested ---~ elseif element.dt then ---~ for _,v in ipairs(element.dt) do -- i added ---~ insert(d,k,v) ---~ k = k + 1 ---~ end ---~ end - else - local edt = element.dt - if edt then - for i=1,#edt do - insert(d,k,edt[i]) - k = k + 1 - end - end - end - end +local function selection(root,pattern,handle) + local collected = parse_apply({ root },pattern) + if collected then + if handle then + for c=1,#collected do + handle(collected[c]) end + else + return collected end end end -xml.insert_element_after = xml.insert_element -xml.insert_element_before = function(r,p,e) xml.insert_element(r,p,e,true) end -xml.inject_element_after = xml.inject_element -xml.inject_element_before = function(r,p,e) xml.inject_element(r,p,e,true) end +xml.parse_parser = parser +xml.parse_pattern = parse_pattern +xml.parse_apply = parse_apply +xml.traverse = traverse -- old method, r, d, k +xml.selection = selection -- new method, simple handle -function xml.delete_element(root, pattern) - local matches, deleted = { }, { } - local collect = function(r,d,k) matches[#matches+1] = { r, d, k } end - traverse(root, lpath(pattern), collect) - for i=#matches,1,-1 do - local m = matches[i] - deleted[#deleted+1] = remove(m[2],m[3]) - end - return deleted +local lpath = parse_pattern + +xml.lpath = lpath + +function xml.cached_patterns() + return cache end -function xml.replace_element(root, pattern, element) - if type(element) == "string" then - element = convert(element,true) - end - if element and element.ri then - element = element.dt[element.ri] - end - if element then - traverse(root, lpath(pattern), function(rm, d, k) - d[k] = element.dt -- maybe not clever enough - end) +-- generic function finalizer (independant namespace) + +local function dofunction(collected,fnc) + if collected then + local f = functions[fnc] + if f then + for c=1,#collected do + f(collected[c]) + end + else + logs.report("xml","unknown function '%s'",fnc) + end end end -local function load_data(name) -- == io.loaddata - local f, data = io.open(name), "" - if f then - data = f:read("*all",'b') -- 'b' ? - f:close() - end - return data +xml.finalizers.xml["function"] = dofunction +xml.finalizers.tex["function"] = dofunction + +-- functions + +expressions.text = function(e,n) + local rdt = e.__p__.dt + return (rdt and rdt[n]) or "" end -function xml.include(xmldata,pattern,attribute,recursive,loaddata) - -- parse="text" (default: xml), encoding="" (todo) - -- attribute = attribute or 'href' - pattern = pattern or 'include' - loaddata = loaddata or load_data - local function include(r,d,k) - local ek, name = d[k], nil - if not attribute or attribute == "" then - local ekdt = ek.dt - name = (type(ekdt) == "table" and ekdt[1]) or ekdt - end - if not name then - if ek.at then - for a in gmatch(attribute or "href","([^|]+)") do - name = ek.at[a] - if name then break end +expressions.name = function(e,n) -- ns + tg + local found = false + n = tonumber(n) or 0 + if n == 0 then + found = type(e) == "table" and e + elseif n < 0 then + local d, k = e.__p__.dt, e.ni + for i=k-1,1,-1 do + local di = d[i] + if type(di) == "table" then + if n == -1 then + found = di + break + else + n = n + 1 end end end - local data = (name and name ~= "" and loaddata(name)) or "" - if data == "" then - xml.empty(d,k) - elseif ek.at["parse"] == "text" then -- for the moment hard coded - d[k] = xml.escaped(data) - else - local xi = xml.convert(data) - if not xi then - xml.empty(d,k) - else - if recursive then - xml.include(xi,pattern,attribute,recursive,loaddata) + else + local d, k = e.__p__.dt, e.ni + for i=k+1,#d,1 do + local di = d[i] + if type(di) == "table" then + if n == 1 then + found = di + break + else + n = n - 1 end - xml.assign(d,k,xi) end end end - xml.each_element(xmldata, pattern, include) + if found then + local ns, tg = found.rn or found.ns or "", found.tg + if ns ~= "" then + return ns .. ":" .. tg + else + return tg + end + else + return "" + end end -function xml.strip_whitespace(root, pattern, nolines) -- strips all leading and trailing space ! - traverse(root, lpath(pattern), function(r,d,k) - local dkdt = d[k].dt - if dkdt then -- can be optimized - local t = { } - for i=1,#dkdt do - local str = dkdt[i] - if type(str) == "string" then - - if str == "" then - -- stripped +expressions.tag = function(e,n) -- only tg + if not e then + return "" + else + local found = false + n = tonumber(n) or 0 + if n == 0 then + found = (type(e) == "table") and e -- seems to fail + elseif n < 0 then + local d, k = e.__p__.dt, e.ni + for i=k-1,1,-1 do + local di = d[i] + if type(di) == "table" then + if n == -1 then + found = di + break else - if nolines then - str = gsub(str,"[ \n\r\t]+"," ") - end - if str == "" then - -- stripped - else - t[#t+1] = str - end + n = n + 1 end - else - t[#t+1] = str end end - d[k].dt = t - end - end) -end - -local function rename_space(root, oldspace, newspace) -- fast variant - local ndt = #root.dt - for i=1,ndt or 0 do - local e = root[i] - if type(e) == "table" then - if e.ns == oldspace then - e.ns = newspace - if e.rn then - e.rn = newspace + else + local d, k = e.__p__.dt, e.ni + for i=k+1,#d,1 do + local di = d[i] + if type(di) == "table" then + if n == 1 then + found = di + break + else + n = n - 1 + end end end - local edt = e.dt - if edt then - rename_space(edt, oldspace, newspace) - end end + return (found and found.tg) or "" end end -xml.rename_space = rename_space +--[[ldx-- +<p>This is the main filter function. It returns whatever is asked for.</p> +--ldx]]-- -function xml.remap_tag(root, pattern, newtg) - traverse(root, lpath(pattern), function(r,d,k) - d[k].tg = newtg - end) +function xml.filter(root,pattern) -- no longer funny attribute handling here + return parse_apply({ root },pattern) end -function xml.remap_namespace(root, pattern, newns) - traverse(root, lpath(pattern), function(r,d,k) - d[k].ns = newns - end) + +--[[ldx-- +<p>Often using an iterators looks nicer in the code than passing handler +functions. The <l n='lua'/> book describes how to use coroutines for that +purpose (<url href='http://www.lua.org/pil/9.3.html'/>). This permits +code like:</p> + +<typing> +for r, d, k in xml.elements(xml.load('text.xml'),"title") do + print(d[k]) -- old method end -function xml.check_namespace(root, pattern, newns) - traverse(root, lpath(pattern), function(r,d,k) - local dk = d[k] - if (not dk.rn or dk.rn == "") and dk.ns == "" then - dk.rn = newns - end - end) +for e in xml.collected(xml.load('text.xml'),"title") do + print(e) -- new one end -function xml.remap_name(root, pattern, newtg, newns, newrn) - traverse(root, lpath(pattern), function(r,d,k) - local dk = d[k] - dk.tg = newtg - dk.ns = newns - dk.rn = newrn - end) +</typing> +--ldx]]-- + +local wrap, yield = coroutine.wrap, coroutine.yield + +function xml.elements(root,pattern,reverse) -- r, d, k + local collected = parse_apply({ root },pattern) + if collected then + if reverse then + return wrap(function() for c=#collected,1,-1 do + local e = collected[c] local r = e.__p__ yield(r,r.dt,e.ni) + end end) + else + return wrap(function() for c=1,#collected do + local e = collected[c] local r = e.__p__ yield(r,r.dt,e.ni) + end end) + end + end + return wrap(function() end) end -function xml.filters.found(root,pattern,check_content) - local found = false - traverse(root, lpath(pattern), function(r,d,k) - if check_content then - local dk = d and d[k] - found = dk and dk.dt and next(dk.dt) and true +function xml.collected(root,pattern,reverse) -- e + local collected = parse_apply({ root },pattern) + if collected then + if reverse then + return wrap(function() for c=#collected,1,-1 do yield(collected[c]) end end) else - found = true + return wrap(function() for c=1,#collected do yield(collected[c]) end end) end - return true - end) - return found + end + return wrap(function() end) end ---[[ldx-- -<p>Here are a few synonyms.</p> ---ldx]]-- -xml.filters.position = xml.filters.index +end -- of closure -xml.count = xml.filters.count -xml.index = xml.filters.index -xml.position = xml.filters.index -xml.first = xml.filters.first -xml.last = xml.filters.last -xml.found = xml.filters.found +do -- create closure to overcome 200 locals limit -xml.each = xml.each_element -xml.process = xml.process_element -xml.strip = xml.strip_whitespace -xml.collect = xml.collect_elements -xml.all = xml.collect_elements +if not modules then modules = { } end modules ['lxml-mis'] = { + version = 1.001, + comment = "this module is the basis for the lxml-* ones", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} -xml.insert = xml.insert_element_after -xml.inject = xml.inject_element_after -xml.after = xml.insert_element_after -xml.before = xml.insert_element_before -xml.delete = xml.delete_element -xml.replace = xml.replace_element +local concat = table.concat +local type, next, tonumber, tostring, setmetatable, loadstring = type, next, tonumber, tostring, setmetatable, loadstring +local format, gsub, match = string.format, string.gsub, string.match +local lpegmatch = lpeg.match --[[ldx-- -<p>The following helper functions best belong to the <t>lmxl-ini</t> +<p>The following helper functions best belong to the <t>lxml-ini</t> module. Some are here because we need then in the <t>mk</t> document and other manuals, others came up when playing with this module. Since this module is also used in <l n='mtxrun'/> we've put them here instead of loading mode modules there then needed.</p> --ldx]]-- -function xml.gsub(t,old,new) +local function xmlgsub(t,old,new) -- will be replaced local dt = t.dt if dt then for k=1,#dt do @@ -5069,28 +6366,26 @@ function xml.gsub(t,old,new) if type(v) == "string" then dt[k] = gsub(v,old,new) else - xml.gsub(v,old,new) + xmlgsub(v,old,new) end end end end +--~ xml.gsub = xmlgsub + function xml.strip_leading_spaces(dk,d,k) -- cosmetic, for manual - if d and k and d[k-1] and type(d[k-1]) == "string" then - local s = d[k-1]:match("\n(%s+)") - xml.gsub(dk,"\n"..string.rep(" ",#s),"\n") + if d and k then + local dkm = d[k-1] + if dkm and type(dkm) == "string" then + local s = match(dkm,"\n(%s+)") + xmlgsub(dk,"\n"..rep(" ",#s),"\n") + end end end -function xml.serialize_path(root,lpath,handle) - local dk, r, d, k = xml.first(root,lpath) - dk = xml.copy(dk) - xml.strip_leading_spaces(dk,d,k) - xml.serialize(dk,handle) -end - --~ xml.escapes = { ['&'] = '&', ['<'] = '<', ['>'] = '>', ['"'] = '"' } ---~ xml.unescapes = { } for k,v in pairs(xml.escapes) do xml.unescapes[v] = k end +--~ xml.unescapes = { } for k,v in next, xml.escapes do xml.unescapes[v] = k end --~ function xml.escaped (str) return (gsub(str,"(.)" , xml.escapes )) end --~ function xml.unescaped(str) return (gsub(str,"(&.-;)", xml.unescapes)) end @@ -5114,8 +6409,6 @@ local escaped = Cs(normal * (special * normal)^0) -- 100 * 1000 * "oeps< oeps> oeps&" : gsub:lpeg == 0153:0280:0151:0080 (last one by roberto) --- unescaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0) --- unescaped = Cs((((P("&")/"") * (P("lt")/"<" + P("gt")/">" + P("amp")/"&") * (P(";")/"")) + 1)^0) local normal = (1 - S"&")^0 local special = P("<")/"<" + P(">")/">" + P("&")/"&" local unescaped = Cs(normal * (special * normal)^0) @@ -5124,84 +6417,32 @@ local unescaped = Cs(normal * (special * normal)^0) local cleansed = Cs(((P("<") * (1-P(">"))^0 * P(">"))/"" + 1)^0) -function xml.escaped (str) return escaped :match(str) end -function xml.unescaped(str) return unescaped:match(str) end -function xml.cleansed (str) return cleansed :match(str) end +xml.escaped_pattern = escaped +xml.unescaped_pattern = unescaped +xml.cleansed_pattern = cleansed -function xml.join(t,separator,lastseparator) - if #t > 0 then - local result = { } - for k,v in pairs(t) do - result[k] = xml.tostring(v) - end - if lastseparator then - return concat(result,separator or "",1,#result-1) .. (lastseparator or "") .. result[#result] - else - return concat(result,separator) +function xml.escaped (str) return lpegmatch(escaped,str) end +function xml.unescaped(str) return lpegmatch(unescaped,str) end +function xml.cleansed (str) return lpegmatch(cleansed,str) end + +-- this might move + +function xml.fillin(root,pattern,str,check) + local e = xml.first(root,pattern) + if e then + local n = #e.dt + if not check or n == 0 or (n == 1 and e.dt[1] == "") then + e.dt = { str } end - else - return "" end end -function xml.statistics() - return { - lpathcalls = lpathcalls, - lpathcached = lpathcached, - } -end - --- xml.set_text_cleanup(xml.show_text_entities) --- xml.set_text_cleanup(xml.resolve_text_entities) - ---~ xml.lshow("/../../../a/(b|c)[@d='e']/f") ---~ xml.lshow("/../../../a/!(b|c)[@d='e']/f") ---~ xml.lshow("/../../../a/!b[@d!='e']/f") - ---~ x = xml.convert([[ ---~ <a> ---~ <b n='01'>01</b> ---~ <b n='02'>02</b> ---~ <b n='03'>03</b> ---~ <b n='04'>OK</b> ---~ <b n='05'>05</b> ---~ <b n='06'>06</b> ---~ <b n='07'>ALSO OK</b> ---~ </a> ---~ ]]) - ---~ xml.settrace("lpath",true) - ---~ xml.xshow(xml.first(x,"b[position() > 2 and position() < 5 and text() == 'ok']")) ---~ xml.xshow(xml.first(x,"b[position() > 2 and position() < 5 and text() == upper('ok')]")) ---~ xml.xshow(xml.first(x,"b[@n=='03' or @n=='08']")) ---~ xml.xshow(xml.all (x,"b[number(@n)>2 and number(@n)<6]")) ---~ xml.xshow(xml.first(x,"b[find(text(),'ALSO')]")) - ---~ str = [[ ---~ <?xml version="1.0" encoding="utf-8"?> ---~ <story line='mojca'> ---~ <windows>my secret</mouse> ---~ </story> ---~ ]] - ---~ x = xml.convert([[ ---~ <a><b n='01'>01</b><b n='02'>02</b><x>xx</x><b n='03'>03</b><b n='04'>OK</b></a> ---~ ]]) ---~ xml.xshow(xml.first(x,"b[tag(2) == 'x']")) ---~ xml.xshow(xml.first(x,"b[tag(1) == 'x']")) ---~ xml.xshow(xml.first(x,"b[tag(-1) == 'x']")) ---~ xml.xshow(xml.first(x,"b[tag(-2) == 'x']")) - ---~ print(xml.filter(x,"b/tag(2)")) ---~ print(xml.filter(x,"b/tag(1)")) - end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['lxml-ent'] = { +if not modules then modules = { } end modules ['lxml-aux'] = { version = 1.001, comment = "this module is the basis for the lxml-* ones", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", @@ -5209,457 +6450,836 @@ if not modules then modules = { } end modules ['lxml-ent'] = { license = "see context related readme files" } -local type, next, tonumber, tostring, setmetatable, loadstring = type, next, tonumber, tostring, setmetatable, loadstring -local format, gsub, find = string.format, string.gsub, string.find -local utfchar = unicode.utf8.char +-- not all functions here make sense anymore vbut we keep them for +-- compatibility reasons ---[[ldx-- -<p>We provide (at least here) two entity handlers. The more extensive -resolver consults a hash first, tries to convert to <l n='utf'/> next, -and finaly calls a handler when defines. When this all fails, the -original entity is returned.</p> ---ldx]]-- +local trace_manipulations = false trackers.register("lxml.manipulations", function(v) trace_manipulations = v end) -xml.entities = xml.entities or { } -- xml.entity_handler == function +local xmlparseapply, xmlconvert, xmlcopy, xmlname = xml.parse_apply, xml.convert, xml.copy, xml.name +local xmlinheritedconvert = xml.inheritedconvert -function xml.entity_handler(e) - return format("[%s]",e) -end +local type = type +local insert, remove = table.insert, table.remove +local gmatch, gsub = string.gmatch, string.gsub -local function toutf(s) - return utfchar(tonumber(s,16)) +local function report(what,pattern,c,e) + logs.report("xml","%s element '%s' (root: '%s', position: %s, index: %s, pattern: %s)",what,xmlname(e),xmlname(e.__p__),c,e.ni,pattern) end -local function utfize(root) - local d = root.dt - for k=1,#d do - local dk = d[k] - if type(dk) == "string" then - -- test prevents copying if no match - if find(dk,"&#x.-;") then - d[k] = gsub(dk,"&#x(.-);",toutf) +local function withelements(e,handle,depth) + if e and handle then + local edt = e.dt + if edt then + depth = depth or 0 + for i=1,#edt do + local e = edt[i] + if type(e) == "table" then + handle(e,depth) + withelements(e,handle,depth+1) + end end - else - utfize(dk) end end end -xml.utfize = utfize +xml.withelements = withelements -local function resolve(e) -- hex encoded always first, just to avoid mkii fallbacks - if find(e,"^#x") then - return utfchar(tonumber(e:sub(3),16)) - elseif find(e,"^#") then - return utfchar(tonumber(e:sub(2))) - else - local ee = xml.entities[e] -- we cannot shortcut this one (is reloaded) - if ee then - return ee - else - local h = xml.entity_handler - return (h and h(e)) or "&" .. e .. ";" +function xml.withelement(e,n,handle) -- slow + if e and n ~= 0 and handle then + local edt = e.dt + if edt then + if n > 0 then + for i=1,#edt do + local ei = edt[i] + if type(ei) == "table" then + if n == 1 then + handle(ei) + return + else + n = n - 1 + end + end + end + elseif n < 0 then + for i=#edt,1,-1 do + local ei = edt[i] + if type(ei) == "table" then + if n == -1 then + handle(ei) + return + else + n = n + 1 + end + end + end + end end end end -local function resolve_entities(root) - if not root.special or root.tg == "@rt@" then - local d = root.dt - for k=1,#d do - local dk = d[k] - if type(dk) == "string" then - if find(dk,"&.-;") then - d[k] = gsub(dk,"&(.-);",resolve) - end - else - resolve_entities(dk) +xml.elements_only = xml.collected + +function xml.each_element(root,pattern,handle,reverse) + local collected = xmlparseapply({ root },pattern) + if collected then + if reverse then + for c=#collected,1,-1 do + handle(collected[c]) + end + else + for c=1,#collected do + handle(collected[c]) end end + return collected end end -xml.resolve_entities = resolve_entities +xml.process_elements = xml.each_element -function xml.utfize_text(str) - if find(str,"&#") then - return (gsub(str,"&#x(.-);",toutf)) - else - return str +function xml.process_attributes(root,pattern,handle) + local collected = xmlparseapply({ root },pattern) + if collected and handle then + for c=1,#collected do + handle(collected[c].at) + end end + return collected end -function xml.resolve_text_entities(str) -- maybe an lpeg. maybe resolve inline - if find(str,"&") then - return (gsub(str,"&(.-);",resolve)) - else - return str - end +--[[ldx-- +<p>The following functions collect elements and texts.</p> +--ldx]]-- + +-- are these still needed -> lxml-cmp.lua + +function xml.collect_elements(root, pattern) + return xmlparseapply({ root },pattern) end -function xml.show_text_entities(str) - if find(str,"&") then - return (gsub(str,"&(.-);","[%1]")) - else - return str +function xml.collect_texts(root, pattern, flatten) -- todo: variant with handle + local collected = xmlparseapply({ root },pattern) + if collected and flatten then + local xmltostring = xml.tostring + for c=1,#collected do + collected[c] = xmltostring(collected[c].dt) + end end + return collected or { } end --- experimental, this will be done differently - -function xml.merge_entities(root) - local documententities = root.entities - local allentities = xml.entities - if documententities then - for k, v in next, documententities do - allentities[k] = v +function xml.collect_tags(root, pattern, nonamespace) + local collected = xmlparseapply({ root },pattern) + if collected then + local t = { } + for c=1,#collected do + local e = collected[c] + local ns, tg = e.ns, e.tg + if nonamespace then + t[#t+1] = tg + elseif ns == "" then + t[#t+1] = tg + else + t[#t+1] = ns .. ":" .. tg + end end + return t end end +--[[ldx-- +<p>We've now arrived at the functions that manipulate the tree.</p> +--ldx]]-- -end -- of closure +local no_root = { no_root = true } -do -- create closure to overcome 200 locals limit +function xml.redo_ni(d) + for k=1,#d do + local dk = d[k] + if type(dk) == "table" then + dk.ni = k + end + end +end -if not modules then modules = { } end modules ['lxml-mis'] = { - version = 1.001, - comment = "this module is the basis for the lxml-* ones", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} +local function xmltoelement(whatever,root) + if not whatever then + return nil + end + local element + if type(whatever) == "string" then + element = xmlinheritedconvert(whatever,root) + else + element = whatever -- we assume a table + end + if element.error then + return whatever -- string + end + if element then + --~ if element.ri then + --~ element = element.dt[element.ri].dt + --~ else + --~ element = element.dt + --~ end + end + return element +end -local concat = table.concat -local type, next, tonumber, tostring, setmetatable, loadstring = type, next, tonumber, tostring, setmetatable, loadstring -local format, gsub = string.format, string.gsub +xml.toelement = xmltoelement ---[[ldx-- -<p>The following helper functions best belong to the <t>lmxl-ini</t> -module. Some are here because we need then in the <t>mk</t> -document and other manuals, others came up when playing with -this module. Since this module is also used in <l n='mtxrun'/> we've -put them here instead of loading mode modules there then needed.</p> ---ldx]]-- +local function copiedelement(element,newparent) + if type(element) == "string" then + return element + else + element = xmlcopy(element).dt + if newparent and type(element) == "table" then + element.__p__ = newparent + end + return element + end +end -function xml.gsub(t,old,new) - local dt = t.dt - if dt then - for k=1,#dt do - local v = dt[k] - if type(v) == "string" then - dt[k] = gsub(v,old,new) - else - xml.gsub(v,old,new) +function xml.delete_element(root,pattern) + local collected = xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + local e = collected[c] + local p = e.__p__ + if p then + if trace_manipulations then + report('deleting',pattern,c,e) + end + local d = p.dt + remove(d,e.ni) + xml.redo_ni(d) -- can be made faster and inlined end end end end -function xml.strip_leading_spaces(dk,d,k) -- cosmetic, for manual - if d and k and d[k-1] and type(d[k-1]) == "string" then - local s = d[k-1]:match("\n(%s+)") - xml.gsub(dk,"\n"..string.rep(" ",#s),"\n") +function xml.replace_element(root,pattern,whatever) + local element = root and xmltoelement(whatever,root) + local collected = element and xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + local e = collected[c] + local p = e.__p__ + if p then + if trace_manipulations then + report('replacing',pattern,c,e) + end + local d = p.dt + d[e.ni] = copiedelement(element,p) + xml.redo_ni(d) -- probably not needed + end + end end end -function xml.serialize_path(root,lpath,handle) - local dk, r, d, k = xml.first(root,lpath) - dk = xml.copy(dk) - xml.strip_leading_spaces(dk,d,k) - xml.serialize(dk,handle) +local function inject_element(root,pattern,whatever,prepend) + local element = root and xmltoelement(whatever,root) + local collected = element and xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + local e = collected[c] + local r = e.__p__ + local d, k, rri = r.dt, e.ni, r.ri + local edt = (rri and d[rri].dt) or (d and d[k] and d[k].dt) + if edt then + local be, af + local cp = copiedelement(element,e) + if prepend then + be, af = cp, edt + else + be, af = edt, cp + end + for i=1,#af do + be[#be+1] = af[i] + end + if rri then + r.dt[rri].dt = be + else + d[k].dt = be + end + xml.redo_ni(d) + end + end + end end ---~ xml.escapes = { ['&'] = '&', ['<'] = '<', ['>'] = '>', ['"'] = '"' } ---~ xml.unescapes = { } for k,v in pairs(xml.escapes) do xml.unescapes[v] = k end +local function insert_element(root,pattern,whatever,before) -- todo: element als functie + local element = root and xmltoelement(whatever,root) + local collected = element and xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + local e = collected[c] + local r = e.__p__ + local d, k = r.dt, e.ni + if not before then + k = k + 1 + end + insert(d,k,copiedelement(element,r)) + xml.redo_ni(d) + end + end +end ---~ function xml.escaped (str) return (gsub(str,"(.)" , xml.escapes )) end ---~ function xml.unescaped(str) return (gsub(str,"(&.-;)", xml.unescapes)) end ---~ function xml.cleansed (str) return (gsub(str,"<.->" , '' )) end -- "%b<>" +xml.insert_element = insert_element +xml.insert_element_after = insert_element +xml.insert_element_before = function(r,p,e) insert_element(r,p,e,true) end +xml.inject_element = inject_element +xml.inject_element_after = inject_element +xml.inject_element_before = function(r,p,e) inject_element(r,p,e,true) end -local P, S, R, C, V, Cc, Cs = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc, lpeg.Cs +local function include(xmldata,pattern,attribute,recursive,loaddata) + -- parse="text" (default: xml), encoding="" (todo) + -- attribute = attribute or 'href' + pattern = pattern or 'include' + loaddata = loaddata or io.loaddata + local collected = xmlparseapply({ xmldata },pattern) + if collected then + for c=1,#collected do + local ek = collected[c] + local name = nil + local ekdt = ek.dt + local ekat = ek.at + local epdt = ek.__p__.dt + if not attribute or attribute == "" then + name = (type(ekdt) == "table" and ekdt[1]) or ekdt -- ckeck, probably always tab or str + end + if not name then + for a in gmatch(attribute or "href","([^|]+)") do + name = ekat[a] + if name then break end + end + end + local data = (name and name ~= "" and loaddata(name)) or "" + if data == "" then + epdt[ek.ni] = "" -- xml.empty(d,k) + elseif ekat["parse"] == "text" then + -- for the moment hard coded + epdt[ek.ni] = xml.escaped(data) -- d[k] = xml.escaped(data) + else +--~ local settings = xmldata.settings +--~ settings.parent_root = xmldata -- to be tested +--~ local xi = xmlconvert(data,settings) + local xi = xmlinheritedconvert(data,xmldata) + if not xi then + epdt[ek.ni] = "" -- xml.empty(d,k) + else + if recursive then + include(xi,pattern,attribute,recursive,loaddata) + end + epdt[ek.ni] = xml.body(xi) -- xml.assign(d,k,xi) + end + end + end + end +end --- 100 * 2500 * "oeps< oeps> oeps&" : gsub:lpeg|lpeg|lpeg --- --- 1021:0335:0287:0247 +xml.include = include --- 10 * 1000 * "oeps< oeps> oeps& asfjhalskfjh alskfjh alskfjh alskfjh ;al J;LSFDJ" --- --- 1559:0257:0288:0190 (last one suggested by roberto) +--~ local function manipulate(xmldata,pattern,manipulator) -- untested and might go away +--~ local collected = xmlparseapply({ xmldata },pattern) +--~ if collected then +--~ local xmltostring = xml.tostring +--~ for c=1,#collected do +--~ local e = collected[c] +--~ local data = manipulator(xmltostring(e)) +--~ if data == "" then +--~ epdt[e.ni] = "" +--~ else +--~ local xi = xmlinheritedconvert(data,xmldata) +--~ if not xi then +--~ epdt[e.ni] = "" +--~ else +--~ epdt[e.ni] = xml.body(xi) -- xml.assign(d,k,xi) +--~ end +--~ end +--~ end +--~ end +--~ end --- escaped = Cs((S("<&>") / xml.escapes + 1)^0) --- escaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0) -local normal = (1 - S("<&>"))^0 -local special = P("<")/"<" + P(">")/">" + P("&")/"&" -local escaped = Cs(normal * (special * normal)^0) +--~ xml.manipulate = manipulate --- 100 * 1000 * "oeps< oeps> oeps&" : gsub:lpeg == 0153:0280:0151:0080 (last one by roberto) +function xml.strip_whitespace(root, pattern, nolines) -- strips all leading and trailing space ! + local collected = xmlparseapply({ root },pattern) + if collected then + for i=1,#collected do + local e = collected[i] + local edt = e.dt + if edt then + local t = { } + for i=1,#edt do + local str = edt[i] + if type(str) == "string" then + if str == "" then + -- stripped + else + if nolines then + str = gsub(str,"[ \n\r\t]+"," ") + end + if str == "" then + -- stripped + else + t[#t+1] = str + end + end + else + --~ str.ni = i + t[#t+1] = str + end + end + e.dt = t + end + end + end +end --- unescaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0) --- unescaped = Cs((((P("&")/"") * (P("lt")/"<" + P("gt")/">" + P("amp")/"&") * (P(";")/"")) + 1)^0) -local normal = (1 - S"&")^0 -local special = P("<")/"<" + P(">")/">" + P("&")/"&" -local unescaped = Cs(normal * (special * normal)^0) +function xml.strip_whitespace(root, pattern, nolines, anywhere) -- strips all leading and trailing spacing + local collected = xmlparseapply({ root },pattern) -- beware, indices no longer are valid now + if collected then + for i=1,#collected do + local e = collected[i] + local edt = e.dt + if edt then + if anywhere then + local t = { } + for e=1,#edt do + local str = edt[e] + if type(str) ~= "string" then + t[#t+1] = str + elseif str ~= "" then + -- todo: lpeg for each case + if nolines then + str = gsub(str,"%s+"," ") + end + str = gsub(str,"^%s*(.-)%s*$","%1") + if str ~= "" then + t[#t+1] = str + end + end + end + e.dt = t + else + -- we can assume a regular sparse xml table with no successive strings + -- otherwise we should use a while loop + if #edt > 0 then + -- strip front + local str = edt[1] + if type(str) ~= "string" then + -- nothing + elseif str == "" then + remove(edt,1) + else + if nolines then + str = gsub(str,"%s+"," ") + end + str = gsub(str,"^%s+","") + if str == "" then + remove(edt,1) + else + edt[1] = str + end + end + end + if #edt > 1 then + -- strip end + local str = edt[#edt] + if type(str) ~= "string" then + -- nothing + elseif str == "" then + remove(edt) + else + if nolines then + str = gsub(str,"%s+"," ") + end + str = gsub(str,"%s+$","") + if str == "" then + remove(edt) + else + edt[#edt] = str + end + end + end + end + end + end + end +end --- 100 * 5000 * "oeps <oeps bla='oeps' foo='bar'> oeps </oeps> oeps " : gsub:lpeg == 623:501 msec (short tags, less difference) +local function rename_space(root, oldspace, newspace) -- fast variant + local ndt = #root.dt + for i=1,ndt or 0 do + local e = root[i] + if type(e) == "table" then + if e.ns == oldspace then + e.ns = newspace + if e.rn then + e.rn = newspace + end + end + local edt = e.dt + if edt then + rename_space(edt, oldspace, newspace) + end + end + end +end -local cleansed = Cs(((P("<") * (1-P(">"))^0 * P(">"))/"" + 1)^0) +xml.rename_space = rename_space -xml.escaped_pattern = escaped -xml.unescaped_pattern = unescaped -xml.cleansed_pattern = cleansed +function xml.remap_tag(root, pattern, newtg) + local collected = xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + collected[c].tg = newtg + end + end +end -function xml.escaped (str) return escaped :match(str) end -function xml.unescaped(str) return unescaped:match(str) end -function xml.cleansed (str) return cleansed :match(str) end +function xml.remap_namespace(root, pattern, newns) + local collected = xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + collected[c].ns = newns + end + end +end -function xml.join(t,separator,lastseparator) - if #t > 0 then - local result = { } - for k,v in pairs(t) do - result[k] = xml.tostring(v) +function xml.check_namespace(root, pattern, newns) + local collected = xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + local e = collected[c] + if (not e.rn or e.rn == "") and e.ns == "" then + e.rn = newns + end end - if lastseparator then - return concat(result,separator or "",1,#result-1) .. (lastseparator or "") .. result[#result] - else - return concat(result,separator) + end +end + +function xml.remap_name(root, pattern, newtg, newns, newrn) + local collected = xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + local e = collected[c] + e.tg, e.ns, e.rn = newtg, newns, newrn end - else - return "" end end +--[[ldx-- +<p>Here are a few synonyms.</p> +--ldx]]-- + +xml.each = xml.each_element +xml.process = xml.process_element +xml.strip = xml.strip_whitespace +xml.collect = xml.collect_elements +xml.all = xml.collect_elements + +xml.insert = xml.insert_element_after +xml.inject = xml.inject_element_after +xml.after = xml.insert_element_after +xml.before = xml.insert_element_before +xml.delete = xml.delete_element +xml.replace = xml.replace_element + end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['trac-tra'] = { +if not modules then modules = { } end modules ['lxml-xml'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "this module is the basis for the lxml-* ones", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } --- the <anonymous> tag is kind of generic and used for functions that are not --- bound to a variable, like node.new, node.copy etc (contrary to for instance --- node.has_attribute which is bound to a has_attribute local variable in mkiv) +local finalizers = xml.finalizers.xml +local xmlfilter = xml.filter -- we could inline this one for speed +local xmltostring = xml.tostring +local xmlserialize = xml.serialize -debugger = debugger or { } +local function first(collected) -- wrong ? + return collected and collected[1] +end -local counters = { } -local names = { } -local getinfo = debug.getinfo -local format, find, lower, gmatch = string.format, string.find, string.lower, string.gmatch +local function last(collected) + return collected and collected[#collected] +end --- one +local function all(collected) + return collected +end -local function hook() - local f = getinfo(2,"f").func - local n = getinfo(2,"Sn") --- if n.what == "C" and n.name then print (n.namewhat .. ': ' .. n.name) end - if f then - local cf = counters[f] - if cf == nil then - counters[f] = 1 - names[f] = n - else - counters[f] = cf + 1 +local function reverse(collected) + if collected then + local reversed = { } + for c=#collected,1,-1 do + reversed[#reversed+1] = collected[c] end + return reversed end end -local function getname(func) - local n = names[func] - if n then - if n.what == "C" then - return n.name or '<anonymous>' - else - -- source short_src linedefined what name namewhat nups func - local name = n.name or n.namewhat or n.what - if not name or name == "" then name = "?" end - return format("%s : %s : %s", n.short_src or "unknown source", n.linedefined or "--", name) - end - else - return "unknown" + +local function attribute(collected,name) + if collected and #collected > 0 then + local at = collected[1].at + return at and at[name] end end -function debugger.showstats(printer,threshold) - printer = printer or texio.write or print - threshold = threshold or 0 - local total, grandtotal, functions = 0, 0, 0 - printer("\n") -- ugly but ok - -- table.sort(counters) - for func, count in pairs(counters) do - if count > threshold then - local name = getname(func) - if not name:find("for generator") then - printer(format("%8i %s", count, name)) - total = total + count - end - end - grandtotal = grandtotal + count - functions = functions + 1 - end - printer(format("functions: %s, total: %s, grand total: %s, threshold: %s\n", functions, total, grandtotal, threshold)) + +local function att(id,name) + local at = id.at + return at and at[name] end --- two +local function count(collected) + return (collected and #collected) or 0 +end ---~ local function hook() ---~ local n = getinfo(2) ---~ if n.what=="C" and not n.name then ---~ local f = tostring(debug.traceback()) ---~ local cf = counters[f] ---~ if cf == nil then ---~ counters[f] = 1 ---~ names[f] = n ---~ else ---~ counters[f] = cf + 1 ---~ end ---~ end ---~ end ---~ function debugger.showstats(printer,threshold) ---~ printer = printer or texio.write or print ---~ threshold = threshold or 0 ---~ local total, grandtotal, functions = 0, 0, 0 ---~ printer("\n") -- ugly but ok ---~ -- table.sort(counters) ---~ for func, count in pairs(counters) do ---~ if count > threshold then ---~ printer(format("%8i %s", count, func)) ---~ total = total + count ---~ end ---~ grandtotal = grandtotal + count ---~ functions = functions + 1 ---~ end ---~ printer(format("functions: %s, total: %s, grand total: %s, threshold: %s\n", functions, total, grandtotal, threshold)) ---~ end +local function position(collected,n) + if collected then + n = tonumber(n) or 0 + if n < 0 then + return collected[#collected + n + 1] + elseif n > 0 then + return collected[n] + else + return collected[1].mi or 0 + end + end +end --- rest +local function match(collected) + return (collected and collected[1].mi) or 0 -- match +end -function debugger.savestats(filename,threshold) - local f = io.open(filename,'w') - if f then - debugger.showstats(function(str) f:write(str) end,threshold) - f:close() +local function index(collected) + if collected then + return collected[1].ni end end -function debugger.enable() - debug.sethook(hook,"c") +local function attributes(collected,arguments) + if collected then + local at = collected[1].at + if arguments then + return at[arguments] + elseif next(at) then + return at -- all of them + end + end end -function debugger.disable() - debug.sethook() ---~ counters[debug.getinfo(2,"f").func] = nil +local function chainattribute(collected,arguments) -- todo: optional levels + if collected then + local e = collected[1] + while e do + local at = e.at + if at then + local a = at[arguments] + if a then + return a + end + else + break -- error + end + e = e.__p__ + end + end + return "" end -function debugger.tracing() - local n = tonumber(os.env['MTX.TRACE.CALLS']) or tonumber(os.env['MTX_TRACE_CALLS']) or 0 - if n > 0 then - function debugger.tracing() return true end ; return true +local function raw(collected) -- hybrid + if collected then + local e = collected[1] or collected + return (e and xmlserialize(e)) or "" -- only first as we cannot concat function else - function debugger.tracing() return false end ; return false + return "" end end ---~ debugger.enable() - ---~ print(math.sin(1*.5)) ---~ print(math.sin(1*.5)) ---~ print(math.sin(1*.5)) ---~ print(math.sin(1*.5)) ---~ print(math.sin(1*.5)) - ---~ debugger.disable() - ---~ print("") ---~ debugger.showstats() ---~ print("") ---~ debugger.showstats(print,3) - -trackers = trackers or { } +local function text(collected) -- hybrid + if collected then + local e = collected[1] or collected + return (e and xmltostring(e.dt)) or "" + else + return "" + end +end -local data, done = { }, { } +local function texts(collected) + if collected then + local t = { } + for c=1,#collected do + local e = collection[c] + if e and e.dt then + t[#t+1] = e.dt + end + end + return t + end +end -local function set(what,value) - if type(what) == "string" then - what = aux.settings_to_array(what) +local function tag(collected,n) + if collected then + local c + if n == 0 or not n then + c = collected[1] + elseif n > 1 then + c = collected[n] + else + c = collected[#collected-n+1] + end + return c and c.tg end - for i=1,#what do - local w = what[i] - for d, f in next, data do - if done[d] then - -- prevent recursion due to wildcards - elseif find(d,w) then - done[d] = true - for i=1,#f do - f[i](value) - end +end + +local function name(collected,n) + if collected then + local c + if n == 0 or not n then + c = collected[1] + elseif n > 1 then + c = collected[n] + else + c = collected[#collected-n+1] + end + if c then + if c.ns == "" then + return c.tg + else + return c.ns .. ":" .. c.tg end end end end -local function reset() - for d, f in next, data do - for i=1,#f do - f[i](false) +local function tags(collected,nonamespace) + if collected then + local t = { } + for c=1,#collected do + local e = collected[c] + local ns, tg = e.ns, e.tg + if nonamespace or ns == "" then + t[#t+1] = tg + else + t[#t+1] = ns .. ":" .. tg + end end + return t end end -function trackers.register(what,...) - what = lower(what) - local w = data[what] - if not w then - w = { } - data[what] = w - end - for _, fnc in next, { ... } do - local typ = type(fnc) - if typ == "function" then - w[#w+1] = fnc - elseif typ == "string" then - w[#w+1] = function(value) set(fnc,value,nesting) end +local function empty(collected) + if collected then + for c=1,#collected do + local e = collected[c] + if e then + local edt = e.dt + if edt then + local n = #edt + if n == 1 then + local edk = edt[1] + local typ = type(edk) + if typ == "table" then + return false + elseif edk ~= "" then -- maybe an extra tester for spacing only + return false + end + elseif n > 1 then + return false + end + end + end end end + return true end -function trackers.enable(what) - done = { } - set(what,true) +finalizers.first = first +finalizers.last = last +finalizers.all = all +finalizers.reverse = reverse +finalizers.elements = all +finalizers.default = all +finalizers.attribute = attribute +finalizers.att = att +finalizers.count = count +finalizers.position = position +finalizers.match = match +finalizers.index = index +finalizers.attributes = attributes +finalizers.chainattribute = chainattribute +finalizers.text = text +finalizers.texts = texts +finalizers.tag = tag +finalizers.name = name +finalizers.tags = tags +finalizers.empty = empty + +-- shortcuts -- we could support xmlfilter(id,pattern,first) + +function xml.first(id,pattern) + return first(xmlfilter(id,pattern)) end -function trackers.disable(what) - done = { } - if not what or what == "" then - trackers.reset(what) +function xml.last(id,pattern) + return last(xmlfilter(id,pattern)) +end + +function xml.count(id,pattern) + return count(xmlfilter(id,pattern)) +end + +function xml.attribute(id,pattern,a,default) + return attribute(xmlfilter(id,pattern),a,default) +end + +function xml.raw(id,pattern) + if pattern then + return raw(xmlfilter(id,pattern)) else - set(what,false) + return raw(id) end end -function trackers.reset(what) - done = { } - reset() +function xml.text(id,pattern) + if pattern then + -- return text(xmlfilter(id,pattern)) + local collected = xmlfilter(id,pattern) + return (collected and xmltostring(collected[1].dt)) or "" + elseif id then + -- return text(id) + return xmltostring(id.dt) or "" + else + return "" + end end -function trackers.list() -- pattern - local list = table.sortedkeys(data) - local user, system = { }, { } - for l=1,#list do - local what = list[l] - if find(what,"^%*") then - system[#system+1] = what - else - user[#user+1] = what - end - end - return user, system +xml.content = text + +function xml.position(id,pattern,n) -- element + return position(xmlfilter(id,pattern),n) +end + +function xml.match(id,pattern) -- number + return match(xmlfilter(id,pattern)) +end + +function xml.empty(id,pattern) + return empty(xmlfilter(id,pattern)) end +xml.all = xml.filter +xml.index = xml.position +xml.found = xml.filter + end -- of closure @@ -5667,7 +7287,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['luat-env'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -5679,10 +7299,10 @@ if not modules then modules = { } end modules ['luat-env'] = { -- evolved before bytecode arrays were available and so a lot of -- code has disappeared already. -local trace_verbose = false trackers.register("resolvers.verbose", function(v) trace_verbose = v end) -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v trackers.enable("resolvers.verbose") end) +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) -local format = string.format +local format, sub, match, gsub, find = string.format, string.sub, string.match, string.gsub, string.find +local unquote, quote = string.unquote, string.quote -- precautions @@ -5716,13 +7336,14 @@ if not environment.jobname then environ function environment.initialize_arguments(arg) local arguments, files = { }, { } environment.arguments, environment.files, environment.sortedflags = arguments, files, nil - for index, argument in pairs(arg) do + for index=1,#arg do + local argument = arg[index] if index > 0 then - local flag, value = argument:match("^%-+(.+)=(.-)$") + local flag, value = match(argument,"^%-+(.-)=(.-)$") if flag then - arguments[flag] = string.unquote(value or "") + arguments[flag] = unquote(value or "") else - flag = argument:match("^%-+(.+)") + flag = match(argument,"^%-+(.+)") if flag then arguments[flag] = true else @@ -5749,25 +7370,30 @@ function environment.argument(name,partial) return arguments[name] elseif partial then if not sortedflags then - sortedflags = { } - for _,v in pairs(table.sortedkeys(arguments)) do - sortedflags[#sortedflags+1] = "^" .. v + sortedflags = table.sortedkeys(arguments) + for k=1,#sortedflags do + sortedflags[k] = "^" .. sortedflags[k] end environment.sortedflags = sortedflags end -- example of potential clash: ^mode ^modefile - for _,v in ipairs(sortedflags) do - if name:find(v) then - return arguments[v:sub(2,#v)] + for k=1,#sortedflags do + local v = sortedflags[k] + if find(name,v) then + return arguments[sub(v,2,#v)] end end end return nil end +environment.argument("x",true) + function environment.split_arguments(separator) -- rather special, cut-off before separator local done, before, after = false, { }, { } - for _,v in ipairs(environment.original_arguments) do + local original_arguments = environment.original_arguments + for k=1,#original_arguments do + local v = original_arguments[k] if not done and v == separator then done = true elseif done then @@ -5784,16 +7410,17 @@ function environment.reconstruct_commandline(arg,noquote) if noquote and #arg == 1 then local a = arg[1] a = resolvers.resolve(a) - a = a:unquote() + a = unquote(a) return a - elseif next(arg) then + elseif #arg > 0 then local result = { } - for _,a in ipairs(arg) do -- ipairs 1 .. #n + for i=1,#arg do + local a = arg[i] a = resolvers.resolve(a) - a = a:unquote() - a = a:gsub('"','\\"') -- tricky - if a:find(" ") then - result[#result+1] = a:quote() + a = unquote(a) + a = gsub(a,'"','\\"') -- tricky + if find(a," ") then + result[#result+1] = quote(a) else result[#result+1] = a end @@ -5806,17 +7433,18 @@ end if arg then - -- new, reconstruct quoted snippets (maybe better just remnove the " then and add them later) + -- new, reconstruct quoted snippets (maybe better just remove the " then and add them later) local newarg, instring = { }, false - for index, argument in ipairs(arg) do - if argument:find("^\"") then - newarg[#newarg+1] = argument:gsub("^\"","") - if not argument:find("\"$") then + for index=1,#arg do + local argument = arg[index] + if find(argument,"^\"") then + newarg[#newarg+1] = gsub(argument,"^\"","") + if not find(argument,"\"$") then instring = true end - elseif argument:find("\"$") then - newarg[#newarg] = newarg[#newarg] .. " " .. argument:gsub("\"$","") + elseif find(argument,"\"$") then + newarg[#newarg] = newarg[#newarg] .. " " .. gsub(argument,"\"$","") instring = false elseif instring then newarg[#newarg] = newarg[#newarg] .. " " .. argument @@ -5871,12 +7499,12 @@ function environment.luafilechunk(filename) -- used for loading lua bytecode in filename = file.replacesuffix(filename, "lua") local fullname = environment.luafile(filename) if fullname and fullname ~= "" then - if trace_verbose then + if trace_locating then logs.report("fileio","loading file %s", fullname) end return environment.loadedluacode(fullname) else - if trace_verbose then + if trace_locating then logs.report("fileio","unknown file %s", filename) end return nil @@ -5896,7 +7524,7 @@ function environment.loadluafile(filename, version) -- when not overloaded by explicit suffix we look for a luc file first local fullname = (lucname and environment.luafile(lucname)) or "" if fullname ~= "" then - if trace_verbose then + if trace_locating then logs.report("fileio","loading %s", fullname) end chunk = loadfile(fullname) -- this way we don't need a file exists check @@ -5914,7 +7542,7 @@ function environment.loadluafile(filename, version) if v == version then return true else - if trace_verbose then + if trace_locating then logs.report("fileio","version mismatch for %s: lua=%s, luc=%s", filename, v, version) end environment.loadluafile(filename) @@ -5925,12 +7553,12 @@ function environment.loadluafile(filename, version) end fullname = (luaname and environment.luafile(luaname)) or "" if fullname ~= "" then - if trace_verbose then + if trace_locating then logs.report("fileio","loading %s", fullname) end chunk = loadfile(fullname) -- this way we don't need a file exists check if not chunk then - if verbose then + if trace_locating then logs.report("fileio","unknown file %s", filename) end else @@ -5948,7 +7576,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['trac-inf'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to trac-inf.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -5973,6 +7601,14 @@ function statistics.hastimer(instance) return instance and instance.starttime end +function statistics.resettiming(instance) + if not instance then + notimer = { timing = 0, loadtime = 0 } + else + instance.timing, instance.loadtime = 0, 0 + end +end + function statistics.starttiming(instance) if not instance then notimer = { } @@ -5987,6 +7623,8 @@ function statistics.starttiming(instance) if not instance.loadtime then instance.loadtime = 0 end + else +--~ logs.report("system","nested timing (%s)",tostring(instance)) end instance.timing = it + 1 end @@ -6032,6 +7670,12 @@ function statistics.elapsedindeed(instance) return t > statistics.threshold end +function statistics.elapsedseconds(instance,rest) -- returns nil if 0 seconds + if statistics.elapsedindeed(instance) then + return format("%s seconds %s", statistics.elapsedtime(instance),rest or "") + end +end + -- general function function statistics.register(tag,fnc) @@ -6110,14 +7754,32 @@ function statistics.timed(action,report) report("total runtime: %s",statistics.elapsedtime(timer)) end +-- where, not really the best spot for this: + +commands = commands or { } + +local timer + +function commands.resettimer() + statistics.resettiming(timer) + statistics.starttiming(timer) +end + +function commands.elapsedtime() + statistics.stoptiming(timer) + tex.sprint(statistics.elapsedtime(timer)) +end + +commands.resettimer() + end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['luat-log'] = { +if not modules then modules = { } end modules ['trac-log'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to trac-log.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -6125,7 +7787,11 @@ if not modules then modules = { } end modules ['luat-log'] = { -- this is old code that needs an overhaul -local write_nl, write, format = texio.write_nl or print, texio.write or io.write, string.format +--~ io.stdout:setvbuf("no") +--~ io.stderr:setvbuf("no") + +local write_nl, write = texio.write_nl or print, texio.write or io.write +local format, gmatch = string.format, string.gmatch local texcount = tex and tex.count if texlua then @@ -6206,25 +7872,48 @@ function logs.tex.line(fmt,...) -- new end end +--~ function logs.tex.start_page_number() +--~ local real, user, sub = texcount.realpageno, texcount.userpageno, texcount.subpageno +--~ if real > 0 then +--~ if user > 0 then +--~ if sub > 0 then +--~ write(format("[%s.%s.%s",real,user,sub)) +--~ else +--~ write(format("[%s.%s",real,user)) +--~ end +--~ else +--~ write(format("[%s",real)) +--~ end +--~ else +--~ write("[-") +--~ end +--~ end + +--~ function logs.tex.stop_page_number() +--~ write("]") +--~ end + +local real, user, sub + function logs.tex.start_page_number() - local real, user, sub = texcount.realpageno, texcount.userpageno, texcount.subpageno + real, user, sub = texcount.realpageno, texcount.userpageno, texcount.subpageno +end + +function logs.tex.stop_page_number() if real > 0 then if user > 0 then if sub > 0 then - write(format("[%s.%s.%s",real,user,sub)) + logs.report("pages", "flushing realpage %s, userpage %s, subpage %s",real,user,sub) else - write(format("[%s.%s",real,user)) + logs.report("pages", "flushing realpage %s, userpage %s",real,user) end else - write(format("[%s",real)) + logs.report("pages", "flushing realpage %s",real) end else - write("[-") + logs.report("pages", "flushing page") end -end - -function logs.tex.stop_page_number() - write("]") + io.flush() end logs.tex.report_job_stat = statistics.show_job_stat @@ -6324,7 +8013,7 @@ end function logs.setprogram(_name_,_banner_,_verbose_) name, banner = _name_, _banner_ if _verbose_ then - trackers.enable("resolvers.verbose") + trackers.enable("resolvers.locating") end logs.set_method("tex") logs.report = report -- also used in libraries @@ -6337,9 +8026,9 @@ end function logs.setverbose(what) if what then - trackers.enable("resolvers.verbose") + trackers.enable("resolvers.locating") else - trackers.disable("resolvers.verbose") + trackers.disable("resolvers.locating") end logs.verbose = what or false end @@ -6356,7 +8045,7 @@ logs.report = logs.tex.report logs.simple = logs.tex.report function logs.reportlines(str) -- todo: <lines></lines> - for line in str:gmatch("(.-)[\n\r]") do + for line in gmatch(str,"(.-)[\n\r]") do logs.report(line) end end @@ -6367,8 +8056,12 @@ end logs.simpleline = logs.reportline -function logs.help(message,option) +function logs.reportbanner() -- for scripts too logs.report(banner) +end + +function logs.help(message,option) + logs.reportbanner() logs.reportline() logs.reportlines(message) local moreinfo = logs.moreinfo or "" @@ -6400,6 +8093,11 @@ end --~ logs.system(syslogname,"context","test","fonts","font %s recached due to newer version (%s)","blabla","123") --~ end +function logs.fatal(where,...) + logs.report(where,"fatal error: %s, aborting now",format(...)) + os.exit() +end + end -- of closure @@ -6407,10 +8105,10 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-inp'] = { version = 1.001, + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files", - comment = "companion to luat-lib.tex", } -- After a few years using the code the large luat-inp.lua file @@ -6422,7 +8120,7 @@ if not modules then modules = { } end modules ['data-inp'] = { -- * some public auxiliary functions were made private -- -- TODO: os.getenv -> os.env[] --- TODO: instances.[hashes,cnffiles,configurations,522] -> ipairs (alles check, sneller) +-- TODO: instances.[hashes,cnffiles,configurations,522] -- TODO: check escaping in find etc, too much, too slow -- This lib is multi-purpose and can be loaded again later on so that @@ -6443,12 +8141,13 @@ if not modules then modules = { } end modules ['data-inp'] = { local format, gsub, find, lower, upper, match, gmatch = string.format, string.gsub, string.find, string.lower, string.upper, string.match, string.gmatch local concat, insert, sortedkeys = table.concat, table.insert, table.sortedkeys local next, type = next, type +local lpegmatch = lpeg.match -local trace_locating, trace_detail, trace_verbose = false, false, false +local trace_locating, trace_detail, trace_expansions = false, false, false -trackers.register("resolvers.verbose", function(v) trace_verbose = v end) -trackers.register("resolvers.locating", function(v) trace_locating = v trackers.enable("resolvers.verbose") end) -trackers.register("resolvers.detail", function(v) trace_detail = v trackers.enable("resolvers.verbose,resolvers.detail") end) +trackers.register("resolvers.locating", function(v) trace_locating = v end) +trackers.register("resolvers.details", function(v) trace_detail = v end) +trackers.register("resolvers.expansions", function(v) trace_expansions = v end) -- todo if not resolvers then resolvers = { @@ -6472,7 +8171,7 @@ resolvers.generators.notfound = { nil } resolvers.cacheversion = '1.0.1' resolvers.cnfname = 'texmf.cnf' resolvers.luaname = 'texmfcnf.lua' -resolvers.homedir = os.env[os.platform == "windows" and 'USERPROFILE'] or os.env['HOME'] or '~' +resolvers.homedir = os.env[os.type == "windows" and 'USERPROFILE'] or os.env['HOME'] or '~' resolvers.cnfdefault = '{$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c}' local dummy_path_expr = "^!*unset/*$" @@ -6514,8 +8213,8 @@ suffixes['lua'] = { 'lua', 'luc', 'tma', 'tmc' } alternatives['map files'] = 'map' alternatives['enc files'] = 'enc' -alternatives['cid files'] = 'cid' -alternatives['fea files'] = 'fea' +alternatives['cid maps'] = 'cid' -- great, why no cid files +alternatives['font feature files'] = 'fea' -- and fea files here alternatives['opentype fonts'] = 'otf' alternatives['truetype fonts'] = 'ttf' alternatives['truetype collections'] = 'ttc' @@ -6531,6 +8230,11 @@ formats ['sfd'] = 'SFDFONTS' suffixes ['sfd'] = { 'sfd' } alternatives['subfont definition files'] = 'sfd' +-- lib paths + +formats ['lib'] = 'CLUAINPUTS' -- new (needs checking) +suffixes['lib'] = (os.libsuffix and { os.libsuffix }) or { 'dll', 'so' } + -- In practice we will work within one tds tree, but i want to keep -- the option open to build tools that look at multiple trees, which is -- why we keep the tree specific data in a table. We used to pass the @@ -6653,8 +8357,10 @@ local function check_configuration() -- not yet ok, no time for debugging now -- bad luck end fix("LUAINPUTS" , ".;$TEXINPUTS;$TEXMFSCRIPTS") -- no progname, hm - fix("FONTFEATURES", ".;$TEXMF/fonts/fea//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") - fix("FONTCIDMAPS" , ".;$TEXMF/fonts/cid//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") + -- this will go away some day + fix("FONTFEATURES", ".;$TEXMF/fonts/{data,fea}//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") + fix("FONTCIDMAPS" , ".;$TEXMF/fonts/{data,cid}//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") + -- fix("LUATEXLIBS" , ".;$TEXMF/luatex/lua//") end @@ -6669,7 +8375,7 @@ function resolvers.settrace(n) -- no longer number but: 'locating' or 'detail' end end -resolvers.settrace(os.getenv("MTX.resolvers.TRACE") or os.getenv("MTX_INPUT_TRACE")) +resolvers.settrace(os.getenv("MTX_INPUT_TRACE")) function resolvers.osenv(key) local ie = instance.environment @@ -6757,37 +8463,43 @@ end -- work that well; the parsing is ok, but dealing with the resulting -- table is a pain because we need to work inside-out recursively +local function do_first(a,b) + local t = { } + for s in gmatch(b,"[^,]+") do t[#t+1] = a .. s end + return "{" .. concat(t,",") .. "}" +end + +local function do_second(a,b) + local t = { } + for s in gmatch(a,"[^,]+") do t[#t+1] = s .. b end + return "{" .. concat(t,",") .. "}" +end + +local function do_both(a,b) + local t = { } + for sa in gmatch(a,"[^,]+") do + for sb in gmatch(b,"[^,]+") do + t[#t+1] = sa .. sb + end + end + return "{" .. concat(t,",") .. "}" +end + +local function do_three(a,b,c) + return a .. b.. c +end + local function splitpathexpr(str, t, validate) -- no need for further optimization as it is only called a - -- few times, we can use lpeg for the sub; we could move - -- the local functions outside the body + -- few times, we can use lpeg for the sub + if trace_expansions then + logs.report("fileio","expanding variable '%s'",str) + end t = t or { } str = gsub(str,",}",",@}") str = gsub(str,"{,","{@,") -- str = "@" .. str .. "@" local ok, done - local function do_first(a,b) - local t = { } - for s in gmatch(b,"[^,]+") do t[#t+1] = a .. s end - return "{" .. concat(t,",") .. "}" - end - local function do_second(a,b) - local t = { } - for s in gmatch(a,"[^,]+") do t[#t+1] = s .. b end - return "{" .. concat(t,",") .. "}" - end - local function do_both(a,b) - local t = { } - for sa in gmatch(a,"[^,]+") do - for sb in gmatch(b,"[^,]+") do - t[#t+1] = sa .. sb - end - end - return "{" .. concat(t,",") .. "}" - end - local function do_three(a,b,c) - return a .. b.. c - end while true do done = false while true do @@ -6818,6 +8530,11 @@ local function splitpathexpr(str, t, validate) t[#t+1] = s end end + if trace_expansions then + for k=1,#t do + logs.report("fileio","% 4i: %s",k,t[k]) + end + end return t end @@ -6857,18 +8574,27 @@ end -- also we now follow the stupid route: if not set then just assume *one* -- cnf file under texmf (i.e. distribution) -resolvers.ownpath = resolvers.ownpath or nil -resolvers.ownbin = resolvers.ownbin or arg[-2] or arg[-1] or arg[0] or "luatex" -resolvers.autoselfdir = true -- false may be handy for debugging +local args = environment and environment.original_arguments or arg -- this needs a cleanup + +resolvers.ownbin = resolvers.ownbin or args[-2] or arg[-2] or args[-1] or arg[-1] or arg[0] or "luatex" +resolvers.ownbin = gsub(resolvers.ownbin,"\\","/") function resolvers.getownpath() - if not resolvers.ownpath then - if resolvers.autoselfdir and os.selfdir then - resolvers.ownpath = os.selfdir - else - local binary = resolvers.ownbin - if os.platform == "windows" then - binary = file.replacesuffix(binary,"exe") + local ownpath = resolvers.ownpath or os.selfdir + if not ownpath or ownpath == "" or ownpath == "unset" then + ownpath = args[-1] or arg[-1] + ownpath = ownpath and file.dirname(gsub(ownpath,"\\","/")) + if not ownpath or ownpath == "" then + ownpath = args[-0] or arg[-0] + ownpath = ownpath and file.dirname(gsub(ownpath,"\\","/")) + end + local binary = resolvers.ownbin + if not ownpath or ownpath == "" then + ownpath = ownpath and file.dirname(binary) + end + if not ownpath or ownpath == "" then + if os.binsuffix ~= "" then + binary = file.replacesuffix(binary,os.binsuffix) end for p in gmatch(os.getenv("PATH"),"[^"..io.pathseparator.."]+") do local b = file.join(p,binary) @@ -6880,30 +8606,39 @@ function resolvers.getownpath() local olddir = lfs.currentdir() if lfs.chdir(p) then local pp = lfs.currentdir() - if trace_verbose and p ~= pp then - logs.report("fileio","following symlink %s to %s",p,pp) + if trace_locating and p ~= pp then + logs.report("fileio","following symlink '%s' to '%s'",p,pp) end - resolvers.ownpath = pp + ownpath = pp lfs.chdir(olddir) else - if trace_verbose then - logs.report("fileio","unable to check path %s",p) + if trace_locating then + logs.report("fileio","unable to check path '%s'",p) end - resolvers.ownpath = p + ownpath = p end break end end end - if not resolvers.ownpath then resolvers.ownpath = '.' end + if not ownpath or ownpath == "" then + ownpath = "." + logs.report("fileio","forcing fallback ownpath .") + elseif trace_locating then + logs.report("fileio","using ownpath '%s'",ownpath) + end + end + resolvers.ownpath = ownpath + function resolvers.getownpath() + return resolvers.ownpath end - return resolvers.ownpath + return ownpath end local own_places = { "SELFAUTOLOC", "SELFAUTODIR", "SELFAUTOPARENT", "TEXMFCNF" } local function identify_own() - local ownpath = resolvers.getownpath() or lfs.currentdir() + local ownpath = resolvers.getownpath() or dir.current() local ie = instance.environment if ownpath then if resolvers.env('SELFAUTOLOC') == "" then os.env['SELFAUTOLOC'] = file.collapse_path(ownpath) end @@ -6916,10 +8651,10 @@ local function identify_own() if resolvers.env('TEXMFCNF') == "" then os.env['TEXMFCNF'] = resolvers.cnfdefault end if resolvers.env('TEXOS') == "" then os.env['TEXOS'] = resolvers.env('SELFAUTODIR') end if resolvers.env('TEXROOT') == "" then os.env['TEXROOT'] = resolvers.env('SELFAUTOPARENT') end - if trace_verbose then + if trace_locating then for i=1,#own_places do local v = own_places[i] - logs.report("fileio","variable %s set to %s",v,resolvers.env(v) or "unknown") + logs.report("fileio","variable '%s' set to '%s'",v,resolvers.env(v) or "unknown") end end identify_own = function() end @@ -6951,10 +8686,8 @@ end local function load_cnf_file(fname) fname = resolvers.clean_path(fname) local lname = file.replacesuffix(fname,'lua') - local f = io.open(lname) - if f then -- this will go - f:close() - local dname = file.dirname(fname) + if lfs.isfile(lname) then + local dname = file.dirname(fname) -- fname ? if not instance.configuration[dname] then resolvers.load_data(dname,'configuration',lname and file.basename(lname)) instance.order[#instance.order+1] = instance.configuration[dname] @@ -6962,8 +8695,8 @@ local function load_cnf_file(fname) else f = io.open(fname) if f then - if trace_verbose then - logs.report("fileio","loading %s", fname) + if trace_locating then + logs.report("fileio","loading configuration file %s", fname) end local line, data, n, k, v local dname = file.dirname(fname) @@ -6997,14 +8730,16 @@ local function load_cnf_file(fname) end end f:close() - elseif trace_verbose then - logs.report("fileio","skipping %s", fname) + elseif trace_locating then + logs.report("fileio","skipping configuration file '%s'", fname) end end end local function collapse_cnf_data() -- potential optimization: pass start index (setup and configuration are shared) - for _,c in ipairs(instance.order) do + local order = instance.order + for i=1,#order do + local c = order[i] for k,v in next, c do if not instance.variables[k] then if instance.environment[k] then @@ -7020,19 +8755,24 @@ end function resolvers.load_cnf() local function loadoldconfigdata() - for _, fname in ipairs(instance.cnffiles) do - load_cnf_file(fname) + local cnffiles = instance.cnffiles + for i=1,#cnffiles do + load_cnf_file(cnffiles[i]) end end -- instance.cnffiles contain complete names now ! + -- we still use a funny mix of cnf and new but soon + -- we will switch to lua exclusively as we only use + -- the file to collect the tree roots if #instance.cnffiles == 0 then - if trace_verbose then + if trace_locating then logs.report("fileio","no cnf files found (TEXMFCNF may not be set/known)") end else - instance.rootpath = instance.cnffiles[1] - for k,fname in ipairs(instance.cnffiles) do - instance.cnffiles[k] = file.collapse_path(gsub(fname,"\\",'/')) + local cnffiles = instance.cnffiles + instance.rootpath = cnffiles[1] + for k=1,#cnffiles do + instance.cnffiles[k] = file.collapse_path(cnffiles[k]) end for i=1,3 do instance.rootpath = file.dirname(instance.rootpath) @@ -7060,8 +8800,9 @@ function resolvers.load_lua() -- yet harmless else instance.rootpath = instance.luafiles[1] - for k,fname in ipairs(instance.luafiles) do - instance.luafiles[k] = file.collapse_path(gsub(fname,"\\",'/')) + local luafiles = instance.luafiles + for k=1,#luafiles do + instance.luafiles[k] = file.collapse_path(luafiles[k]) end for i=1,3 do instance.rootpath = file.dirname(instance.rootpath) @@ -7093,14 +8834,14 @@ end function resolvers.append_hash(type,tag,name) if trace_locating then - logs.report("fileio","= hash append: %s",tag) + logs.report("fileio","hash '%s' appended",tag) end insert(instance.hashes, { ['type']=type, ['tag']=tag, ['name']=name } ) end function resolvers.prepend_hash(type,tag,name) if trace_locating then - logs.report("fileio","= hash prepend: %s",tag) + logs.report("fileio","hash '%s' prepended",tag) end insert(instance.hashes, 1, { ['type']=type, ['tag']=tag, ['name']=name } ) end @@ -7124,9 +8865,11 @@ end -- locators function resolvers.locatelists() - for _, path in ipairs(resolvers.clean_path_list('TEXMF')) do - if trace_verbose then - logs.report("fileio","locating list of %s",path) + local texmfpaths = resolvers.clean_path_list('TEXMF') + for i=1,#texmfpaths do + local path = texmfpaths[i] + if trace_locating then + logs.report("fileio","locating list of '%s'",path) end resolvers.locatedatabase(file.collapse_path(path)) end @@ -7139,11 +8882,11 @@ end function resolvers.locators.tex(specification) if specification and specification ~= '' and lfs.isdir(specification) then if trace_locating then - logs.report("fileio",'! tex locator found: %s',specification) + logs.report("fileio","tex locator '%s' found",specification) end resolvers.append_hash('file',specification,filename) elseif trace_locating then - logs.report("fileio",'? tex locator not found: %s',specification) + logs.report("fileio","tex locator '%s' not found",specification) end end @@ -7157,7 +8900,9 @@ function resolvers.loadfiles() instance.loaderror = false instance.files = { } if not instance.renewcache then - for _, hash in ipairs(instance.hashes) do + local hashes = instance.hashes + for k=1,#hashes do + local hash = hashes[k] resolvers.hashdatabase(hash.tag,hash.name) if instance.loaderror then break end end @@ -7171,8 +8916,9 @@ end -- generators: function resolvers.loadlists() - for _, hash in ipairs(instance.hashes) do - resolvers.generatedatabase(hash.tag) + local hashes = instance.hashes + for i=1,#hashes do + resolvers.generatedatabase(hashes[i].tag) end end @@ -7184,10 +8930,27 @@ end local weird = lpeg.P(".")^1 + lpeg.anywhere(lpeg.S("~`!#$%^&*()={}[]:;\"\'||<>,?\n\r\t")) +--~ local l_forbidden = lpeg.S("~`!#$%^&*()={}[]:;\"\'||\\/<>,?\n\r\t") +--~ local l_confusing = lpeg.P(" ") +--~ local l_character = lpeg.patterns.utf8 +--~ local l_dangerous = lpeg.P(".") + +--~ local l_normal = (l_character - l_forbidden - l_confusing - l_dangerous) * (l_character - l_forbidden - l_confusing^2)^0 * lpeg.P(-1) +--~ ----- l_normal = l_normal * lpeg.Cc(true) + lpeg.Cc(false) + +--~ local function test(str) +--~ print(str,lpeg.match(l_normal,str)) +--~ end +--~ test("ヒラギノ明朝 Pro W3") +--~ test("..ヒラギノ明朝 Pro W3") +--~ test(":ヒラギノ明朝 Pro W3;") +--~ test("ヒラギノ明朝 /Pro W3;") +--~ test("ヒラギノ明朝 Pro W3") + function resolvers.generators.tex(specification) local tag = specification - if trace_verbose then - logs.report("fileio","scanning path %s",specification) + if trace_locating then + logs.report("fileio","scanning path '%s'",specification) end instance.files[tag] = { } local files = instance.files[tag] @@ -7203,7 +8966,8 @@ function resolvers.generators.tex(specification) full = spec end for name in directory(full) do - if not weird:match(name) then + if not lpegmatch(weird,name) then + -- if lpegmatch(l_normal,name) then local mode = attributes(full..name,'mode') if mode == 'file' then if path then @@ -7236,7 +9000,7 @@ function resolvers.generators.tex(specification) end end action() - if trace_verbose then + if trace_locating then logs.report("fileio","%s files found on %s directories with %s uppercase remappings",n,m,r) end end @@ -7251,11 +9015,48 @@ end -- we join them and split them after the expansion has taken place. This -- is more convenient. +--~ local checkedsplit = string.checkedsplit + +local cache = { } + +local splitter = lpeg.Ct(lpeg.splitat(lpeg.S(os.type == "windows" and ";" or ":;"))) + +local function split_kpse_path(str) -- beware, this can be either a path or a {specification} + local found = cache[str] + if not found then + if str == "" then + found = { } + else + str = gsub(str,"\\","/") +--~ local split = (find(str,";") and checkedsplit(str,";")) or checkedsplit(str,io.pathseparator) +local split = lpegmatch(splitter,str) + found = { } + for i=1,#split do + local s = split[i] + if not find(s,"^{*unset}*") then + found[#found+1] = s + end + end + if trace_expansions then + logs.report("fileio","splitting path specification '%s'",str) + for k=1,#found do + logs.report("fileio","% 4i: %s",k,found[k]) + end + end + cache[str] = found + end + end + return found +end + +resolvers.split_kpse_path = split_kpse_path + function resolvers.splitconfig() - for i,c in ipairs(instance) do - for k,v in pairs(c) do + for i=1,#instance do + local c = instance[i] + for k,v in next, c do if type(v) == 'string' then - local t = file.split_path(v) + local t = split_kpse_path(v) if #t > 1 then c[k] = t end @@ -7265,21 +9066,25 @@ function resolvers.splitconfig() end function resolvers.joinconfig() - for i,c in ipairs(instance.order) do - for k,v in pairs(c) do -- ipairs? + local order = instance.order + for i=1,#order do + local c = order[i] + for k,v in next, c do -- indexed? if type(v) == 'table' then c[k] = file.join_path(v) end end end end + function resolvers.split_path(str) if type(str) == 'table' then return str else - return file.split_path(str) + return split_kpse_path(str) end end + function resolvers.join_path(str) if type(str) == 'table' then return file.join_path(str) @@ -7291,8 +9096,9 @@ end function resolvers.splitexpansions() local ie = instance.expansions for k,v in next, ie do - local t, h = { }, { } - for _,vv in ipairs(file.split_path(v)) do + local t, h, p = { }, { }, split_kpse_path(v) + for kk=1,#p do + local vv = p[kk] if vv ~= "" and not h[vv] then t[#t+1] = vv h[vv] = true @@ -7339,11 +9145,15 @@ function resolvers.serialize(files) end t[#t+1] = "return {" if instance.sortdata then - for _, k in pairs(sortedkeys(files)) do -- ipairs + local sortedfiles = sortedkeys(files) + for i=1,#sortedfiles do + local k = sortedfiles[i] local fk = files[k] if type(fk) == 'table' then t[#t+1] = "\t['" .. k .. "']={" - for _, kk in pairs(sortedkeys(fk)) do -- ipairs + local sortedfk = sortedkeys(fk) + for j=1,#sortedfk do + local kk = sortedfk[j] t[#t+1] = dump(kk,fk[kk],"\t\t") end t[#t+1] = "\t}," @@ -7368,12 +9178,18 @@ function resolvers.serialize(files) return concat(t,"\n") end +local data_state = { } + +function resolvers.data_state() + return data_state or { } +end + function resolvers.save_data(dataname, makename) -- untested without cache overload for cachename, files in next, instance[dataname] do local name = (makename or file.join)(cachename,dataname) local luaname, lucname = name .. ".lua", name .. ".luc" - if trace_verbose then - logs.report("fileio","preparing %s for %s",dataname,cachename) + if trace_locating then + logs.report("fileio","preparing '%s' for '%s'",dataname,cachename) end for k, v in next, files do if type(v) == "table" and #v == 1 then @@ -7387,24 +9203,25 @@ function resolvers.save_data(dataname, makename) -- untested without cache overl date = os.date("%Y-%m-%d"), time = os.date("%H:%M:%S"), content = files, + uuid = os.uuid(), } local ok = io.savedata(luaname,resolvers.serialize(data)) if ok then - if trace_verbose then - logs.report("fileio","%s saved in %s",dataname,luaname) + if trace_locating then + logs.report("fileio","'%s' saved in '%s'",dataname,luaname) end if utils.lua.compile(luaname,lucname,false,true) then -- no cleanup but strip - if trace_verbose then - logs.report("fileio","%s compiled to %s",dataname,lucname) + if trace_locating then + logs.report("fileio","'%s' compiled to '%s'",dataname,lucname) end else - if trace_verbose then - logs.report("fileio","compiling failed for %s, deleting file %s",dataname,lucname) + if trace_locating then + logs.report("fileio","compiling failed for '%s', deleting file '%s'",dataname,lucname) end os.remove(lucname) end - elseif trace_verbose then - logs.report("fileio","unable to save %s in %s (access error)",dataname,luaname) + elseif trace_locating then + logs.report("fileio","unable to save '%s' in '%s' (access error)",dataname,luaname) end end end @@ -7416,19 +9233,20 @@ function resolvers.load_data(pathname,dataname,filename,makename) -- untested wi if blob then local data = blob() if data and data.content and data.type == dataname and data.version == resolvers.cacheversion then - if trace_verbose then - logs.report("fileio","loading %s for %s from %s",dataname,pathname,filename) + data_state[#data_state+1] = data.uuid + if trace_locating then + logs.report("fileio","loading '%s' for '%s' from '%s'",dataname,pathname,filename) end instance[dataname][pathname] = data.content else - if trace_verbose then - logs.report("fileio","skipping %s for %s from %s",dataname,pathname,filename) + if trace_locating then + logs.report("fileio","skipping '%s' for '%s' from '%s'",dataname,pathname,filename) end instance[dataname][pathname] = { } instance.loaderror = true end - elseif trace_verbose then - logs.report("fileio","skipping %s for %s from %s",dataname,pathname,filename) + elseif trace_locating then + logs.report("fileio","skipping '%s' for '%s' from '%s'",dataname,pathname,filename) end end @@ -7447,15 +9265,17 @@ function resolvers.resetconfig() end function resolvers.loadnewconfig() - for _, cnf in ipairs(instance.luafiles) do + local luafiles = instance.luafiles + for i=1,#luafiles do + local cnf = luafiles[i] local pathname = file.dirname(cnf) local filename = file.join(pathname,resolvers.luaname) local blob = loadfile(filename) if blob then local data = blob() if data then - if trace_verbose then - logs.report("fileio","loading configuration file %s",filename) + if trace_locating then + logs.report("fileio","loading configuration file '%s'",filename) end if true then -- flatten to variable.progname @@ -7476,14 +9296,14 @@ function resolvers.loadnewconfig() instance['setup'][pathname] = data end else - if trace_verbose then - logs.report("fileio","skipping configuration file %s",filename) + if trace_locating then + logs.report("fileio","skipping configuration file '%s'",filename) end instance['setup'][pathname] = { } instance.loaderror = true end - elseif trace_verbose then - logs.report("fileio","skipping configuration file %s",filename) + elseif trace_locating then + logs.report("fileio","skipping configuration file '%s'",filename) end instance.order[#instance.order+1] = instance.setup[pathname] if instance.loaderror then break end @@ -7492,7 +9312,9 @@ end function resolvers.loadoldconfig() if not instance.renewcache then - for _, cnf in ipairs(instance.cnffiles) do + local cnffiles = instance.cnffiles + for i=1,#cnffiles do + local cnf = cnffiles[i] local dname = file.dirname(cnf) resolvers.load_data(dname,'configuration') instance.order[#instance.order+1] = instance.configuration[dname] @@ -7682,7 +9504,7 @@ end function resolvers.expanded_path_list(str) if not str then - return ep or { } + return ep or { } -- ep ? elseif instance.savelists then -- engine+progname hash str = gsub(str,"%$","") @@ -7700,9 +9522,9 @@ end function resolvers.expanded_path_list_from_var(str) -- brrr local tmp = resolvers.var_of_format_or_suffix(gsub(str,"%$","")) if tmp ~= "" then - return resolvers.expanded_path_list(str) - else return resolvers.expanded_path_list(tmp) + else + return resolvers.expanded_path_list(str) end end @@ -7749,9 +9571,9 @@ function resolvers.isreadable.file(name) local readable = lfs.isfile(name) -- brrr if trace_detail then if readable then - logs.report("fileio","+ readable: %s",name) + logs.report("fileio","file '%s' is readable",name) else - logs.report("fileio","- readable: %s", name) + logs.report("fileio","file '%s' is not readable", name) end end return readable @@ -7767,7 +9589,7 @@ local function collect_files(names) for k=1,#names do local fname = names[k] if trace_detail then - logs.report("fileio","? blobpath asked: %s",fname) + logs.report("fileio","checking name '%s'",fname) end local bname = file.basename(fname) local dname = file.dirname(fname) @@ -7783,7 +9605,7 @@ local function collect_files(names) local files = blobpath and instance.files[blobpath] if files then if trace_detail then - logs.report("fileio",'? blobpath do: %s (%s)',blobpath,bname) + logs.report("fileio","deep checking '%s' (%s)",blobpath,bname) end local blobfile = files[bname] if not blobfile then @@ -7817,7 +9639,7 @@ local function collect_files(names) end end elseif trace_locating then - logs.report("fileio",'! blobpath no: %s (%s)',blobpath,bname) + logs.report("fileio","no match in '%s' (%s)",blobpath,bname) end end end @@ -7867,14 +9689,13 @@ end local function collect_instance_files(filename,collected) -- todo : plugin (scanners, checkers etc) local result = collected or { } local stamp = nil - filename = file.collapse_path(filename) -- elsewhere - filename = file.collapse_path(gsub(filename,"\\","/")) -- elsewhere + filename = file.collapse_path(filename) -- speed up / beware: format problem if instance.remember then stamp = filename .. "--" .. instance.engine .. "--" .. instance.progname .. "--" .. instance.format if instance.found[stamp] then if trace_locating then - logs.report("fileio",'! remembered: %s',filename) + logs.report("fileio","remembering file '%s'",filename) end return instance.found[stamp] end @@ -7882,7 +9703,7 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan if not dangerous[instance.format or "?"] then if resolvers.isreadable.file(filename) then if trace_detail then - logs.report("fileio",'= found directly: %s',filename) + logs.report("fileio","file '%s' found directly",filename) end instance.found[stamp] = { filename } return { filename } @@ -7890,13 +9711,13 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan end if find(filename,'%*') then if trace_locating then - logs.report("fileio",'! wildcard: %s', filename) + logs.report("fileio","checking wildcard '%s'", filename) end result = resolvers.find_wildcard_files(filename) elseif file.is_qualified_path(filename) then if resolvers.isreadable.file(filename) then if trace_locating then - logs.report("fileio",'! qualified: %s', filename) + logs.report("fileio","qualified name '%s'", filename) end result = { filename } else @@ -7906,7 +9727,7 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan forcedname = filename .. ".tex" if resolvers.isreadable.file(forcedname) then if trace_locating then - logs.report("fileio",'! no suffix, forcing standard filetype: tex') + logs.report("fileio","no suffix, forcing standard filetype 'tex'") end result, ok = { forcedname }, true end @@ -7916,7 +9737,7 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan forcedname = filename .. "." .. s if resolvers.isreadable.file(forcedname) then if trace_locating then - logs.report("fileio",'! no suffix, forcing format filetype: %s', s) + logs.report("fileio","no suffix, forcing format filetype '%s'", s) end result, ok = { forcedname }, true break @@ -7928,7 +9749,7 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan -- try to find in tree (no suffix manipulation), here we search for the -- matching last part of the name local basename = file.basename(filename) - local pattern = (filename .. "$"):gsub("([%.%-])","%%%1") + local pattern = gsub(filename .. "$","([%.%-])","%%%1") local savedformat = instance.format local format = savedformat or "" if format == "" then @@ -7938,19 +9759,21 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan instance.format = "othertextfiles" -- kind of everything, maybe texinput is better end -- - local resolved = collect_instance_files(basename) - if #result == 0 then - local lowered = lower(basename) - if filename ~= lowered then - resolved = collect_instance_files(lowered) + if basename ~= filename then + local resolved = collect_instance_files(basename) + if #result == 0 then + local lowered = lower(basename) + if filename ~= lowered then + resolved = collect_instance_files(lowered) + end end - end - resolvers.format = savedformat - -- - for r=1,#resolved do - local rr = resolved[r] - if rr:find(pattern) then - result[#result+1], ok = rr, true + resolvers.format = savedformat + -- + for r=1,#resolved do + local rr = resolved[r] + if find(rr,pattern) then + result[#result+1], ok = rr, true + end end end -- a real wildcard: @@ -7959,14 +9782,14 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan -- local filelist = collect_files({basename}) -- for f=1,#filelist do -- local ff = filelist[f][3] or "" - -- if ff:find(pattern) then + -- if find(ff,pattern) then -- result[#result+1], ok = ff, true -- end -- end -- end end if not ok and trace_locating then - logs.report("fileio",'? qualified: %s', filename) + logs.report("fileio","qualified name '%s'", filename) end end else @@ -7985,12 +9808,12 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan wantedfiles[#wantedfiles+1] = forcedname filetype = resolvers.format_of_suffix(forcedname) if trace_locating then - logs.report("fileio",'! forcing filetype: %s',filetype) + logs.report("fileio","forcing filetype '%s'",filetype) end else filetype = resolvers.format_of_suffix(filename) if trace_locating then - logs.report("fileio",'! using suffix based filetype: %s',filetype) + logs.report("fileio","using suffix based filetype '%s'",filetype) end end else @@ -8002,7 +9825,7 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan end filetype = instance.format if trace_locating then - logs.report("fileio",'! using given filetype: %s',filetype) + logs.report("fileio","using given filetype '%s'",filetype) end end local typespec = resolvers.variable_of_format(filetype) @@ -8010,9 +9833,7 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan if not pathlist or #pathlist == 0 then -- no pathlist, access check only / todo == wildcard if trace_detail then - logs.report("fileio",'? filename: %s',filename) - logs.report("fileio",'? filetype: %s',filetype or '?') - logs.report("fileio",'? wanted files: %s',concat(wantedfiles," | ")) + logs.report("fileio","checking filename '%s', filetype '%s', wanted files '%s'",filename, filetype or '?',concat(wantedfiles," | ")) end for k=1,#wantedfiles do local fname = wantedfiles[k] @@ -8033,36 +9854,59 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan else -- list search local filelist = collect_files(wantedfiles) - local doscan, recurse + local dirlist = { } + if filelist then + for i=1,#filelist do + dirlist[i] = file.dirname(filelist[i][2]) .. "/" + end + end if trace_detail then - logs.report("fileio",'? filename: %s',filename) + logs.report("fileio","checking filename '%s'",filename) end -- a bit messy ... esp the doscan setting here + local doscan for k=1,#pathlist do local path = pathlist[k] if find(path,"^!!") then doscan = false else doscan = true end - if find(path,"//$") then recurse = true else recurse = false end local pathname = gsub(path,"^!+", '') done = false -- using file list - if filelist and not (done and not instance.allresults) and recurse then - -- compare list entries with permitted pattern - pathname = gsub(pathname,"([%-%.])","%%%1") -- this also influences - pathname = gsub(pathname,"/+$", '/.*') -- later usage of pathname - pathname = gsub(pathname,"//", '/.-/') -- not ok for /// but harmless - local expr = "^" .. pathname + if filelist then + local expression + -- compare list entries with permitted pattern -- /xx /xx// + if not find(pathname,"/$") then + expression = pathname .. "/" + else + expression = pathname + end + expression = gsub(expression,"([%-%.])","%%%1") -- this also influences + expression = gsub(expression,"//+$", '/.*') -- later usage of pathname + expression = gsub(expression,"//", '/.-/') -- not ok for /// but harmless + expression = "^" .. expression .. "$" + if trace_detail then + logs.report("fileio","using pattern '%s' for path '%s'",expression,pathname) + end for k=1,#filelist do local fl = filelist[k] local f = fl[2] - if find(f,expr) then - if trace_detail then - logs.report("fileio",'= found in hash: %s',f) - end + local d = dirlist[k] + if find(d,expression) then --- todo, test for readable result[#result+1] = fl[3] resolvers.register_in_trees(f) -- for tracing used files done = true - if not instance.allresults then break end + if instance.allresults then + if trace_detail then + logs.report("fileio","match in hash for file '%s' on path '%s', continue scanning",f,d) + end + else + if trace_detail then + logs.report("fileio","match in hash for file '%s' on path '%s', quit scanning",f,d) + end + break + end + elseif trace_detail then + logs.report("fileio","no match in hash for file '%s' on path '%s'",f,d) end end end @@ -8078,7 +9922,7 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan local fname = file.join(ppname,w) if resolvers.isreadable.file(fname) then if trace_detail then - logs.report("fileio",'= found by scanning: %s',fname) + logs.report("fileio","found '%s' by scanning",fname) end result[#result+1] = fname done = true @@ -8141,7 +9985,7 @@ function resolvers.find_given_files(filename) local hashes = instance.hashes for k=1,#hashes do local hash = hashes[k] - local files = instance.files[hash.tag] + local files = instance.files[hash.tag] or { } local blist = files[bname] if not blist then local rname = "remap:"..bname @@ -8251,9 +10095,9 @@ function resolvers.load(option) statistics.starttiming(instance) resolvers.resetconfig() resolvers.identify_cnf() - resolvers.load_lua() + resolvers.load_lua() -- will become the new method resolvers.expand_variables() - resolvers.load_cnf() + resolvers.load_cnf() -- will be skipped when we have a lua file resolvers.expand_variables() if option ~= "nofiles" then resolvers.load_hash() @@ -8265,22 +10109,23 @@ end function resolvers.for_files(command, files, filetype, mustexist) if files and #files > 0 then local function report(str) - if trace_verbose then + if trace_locating then logs.report("fileio",str) -- has already verbose else print(str) end end - if trace_verbose then - report('') + if trace_locating then + report('') -- ? end - for _, file in ipairs(files) do + for f=1,#files do + local file = files[f] local result = command(file,filetype,mustexist) if type(result) == 'string' then report(result) else - for _,v in ipairs(result) do - report(v) + for i=1,#result do + report(result[i]) -- could be unpack end end end @@ -8327,18 +10172,19 @@ end function table.sequenced(t,sep) -- temp here local s = { } - for k, v in pairs(t) do -- pairs? - s[#s+1] = k .. "=" .. v + for k, v in next, t do -- indexed? + s[#s+1] = k .. "=" .. tostring(v) end return concat(s, sep or " | ") end function resolvers.methodhandler(what, filename, filetype) -- ... + filename = file.collapse_path(filename) local specification = (type(filename) == "string" and resolvers.splitmethod(filename)) or filename -- no or { }, let it bomb local scheme = specification.scheme if resolvers[what][scheme] then if trace_locating then - logs.report("fileio",'= handler: %s -> %s -> %s',specification.original,what,table.sequenced(specification)) + logs.report("fileio","handler '%s' -> '%s' -> '%s'",specification.original,what,table.sequenced(specification)) end return resolvers[what][scheme](filename,filetype) -- todo: specification else @@ -8358,8 +10204,9 @@ function resolvers.clean_path(str) end function resolvers.do_with_path(name,func) - for _, v in pairs(resolvers.expanded_path_list(name)) do -- pairs? - func("^"..resolvers.clean_path(v)) + local pathlist = resolvers.expanded_path_list(name) + for i=1,#pathlist do + func("^"..resolvers.clean_path(pathlist[i])) end end @@ -8368,7 +10215,9 @@ function resolvers.do_with_var(name,func) end function resolvers.with_files(pattern,handle) - for _, hash in ipairs(instance.hashes) do + local hashes = instance.hashes + for i=1,#hashes do + local hash = hashes[i] local blobpath = hash.tag local blobtype = hash.type if blobpath then @@ -8383,7 +10232,7 @@ function resolvers.with_files(pattern,handle) if type(v) == "string" then handle(blobtype,blobpath,v,k) else - for _,vv in pairs(v) do -- ipairs? + for _,vv in next, v do -- indexed handle(blobtype,blobpath,vv,k) end end @@ -8395,7 +10244,7 @@ function resolvers.with_files(pattern,handle) end function resolvers.locate_format(name) - local barename, fmtname = name:gsub("%.%a+$",""), "" + local barename, fmtname = gsub(name,"%.%a+$",""), "" if resolvers.usecache then local path = file.join(caches.setpath("formats")) -- maybe platform fmtname = file.join(path,barename..".fmt") or "" @@ -8443,7 +10292,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-tmp'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -8467,7 +10316,7 @@ luatools with a recache feature.</p> local format, lower, gsub = string.format, string.lower, string.gsub -local trace_cache = false trackers.register("resolvers.cache", function(v) trace_cache = v end) +local trace_cache = false trackers.register("resolvers.cache", function(v) trace_cache = v end) -- not used yet caches = caches or { } @@ -8554,7 +10403,8 @@ function caches.setpath(...) caches.path = '.' end caches.path = resolvers.clean_path(caches.path) - if not table.is_empty({...}) then + local dirs = { ... } + if #dirs > 0 then local pth = dir.mkdirs(caches.path,...) return pth end @@ -8600,6 +10450,7 @@ function caches.savedata(filepath,filename,data,raw) if raw then reduce, simplify = false, false end + data.cache_uuid = os.uuid() if caches.direct then file.savedata(tmaname, table.serialize(data,'return',false,true,false)) -- no hex else @@ -8625,7 +10476,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-res'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -8660,6 +10511,14 @@ prefixes.relative = function(str,n) return resolvers.clean_path(str) end +prefixes.auto = function(str) + local fullname = prefixes.relative(str) + if not lfs.isfile(fullname) then + fullname = prefixes.locate(str) + end + return fullname +end + prefixes.locate = function(str) local fullname = resolvers.find_given_file(str) or "" return resolvers.clean_path((fullname ~= "" and fullname) or str) @@ -8683,6 +10542,16 @@ prefixes.full = prefixes.locate prefixes.file = prefixes.filename prefixes.path = prefixes.pathname +function resolvers.allprefixes(separator) + local all = table.sortedkeys(prefixes) + if separator then + for i=1,#all do + all[i] = all[i] .. ":" + end + end + return all +end + local function _resolve_(method,target) if prefixes[method] then return prefixes[method](target) @@ -8693,7 +10562,8 @@ end local function resolve(str) if type(str) == "table" then - for k, v in pairs(str) do -- ipairs + for k=1,#str do + local v = str[k] str[k] = resolve(v) or v end elseif str and str ~= "" then @@ -8706,7 +10576,7 @@ resolvers.resolve = resolve if os.uname then - for k, v in pairs(os.uname()) do + for k, v in next, os.uname() do if not prefixes[k] then prefixes[k] = function() return v end end @@ -8721,7 +10591,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-inp'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -8742,7 +10612,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-out'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -8758,7 +10628,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-con'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -8769,8 +10639,6 @@ local format, lower, gsub = string.format, string.lower, string.gsub local trace_cache = false trackers.register("resolvers.cache", function(v) trace_cache = v end) local trace_containers = false trackers.register("resolvers.containers", function(v) trace_containers = v end) local trace_storage = false trackers.register("resolvers.storage", function(v) trace_storage = v end) -local trace_verbose = false trackers.register("resolvers.verbose", function(v) trace_verbose = v end) -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v trackers.enable("resolvers.verbose") end) --[[ldx-- <p>Once we found ourselves defining similar cache constructs @@ -8834,7 +10702,7 @@ end function containers.is_valid(container, name) if name and name ~= "" then local storage = container.storage[name] - return storage and not table.is_empty(storage) and storage.cache_version == container.version + return storage and storage.cache_version == container.version else return false end @@ -8886,16 +10754,15 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-use'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -local format, lower, gsub = string.format, string.lower, string.gsub +local format, lower, gsub, find = string.format, string.lower, string.gsub, string.find -local trace_verbose = false trackers.register("resolvers.verbose", function(v) trace_verbose = v end) -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v trackers.enable("resolvers.verbose") end) +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) -- since we want to use the cache instead of the tree, we will now -- reimplement the saver. @@ -8939,19 +10806,20 @@ resolvers.automounted = resolvers.automounted or { } function resolvers.automount(usecache) local mountpaths = resolvers.clean_path_list(resolvers.expansion('TEXMFMOUNT')) - if table.is_empty(mountpaths) and usecache then + if (not mountpaths or #mountpaths == 0) and usecache then mountpaths = { caches.setpath("mount") } end - if not table.is_empty(mountpaths) then + if mountpaths and #mountpaths > 0 then statistics.starttiming(resolvers.instance) - for k, root in pairs(mountpaths) do + for k=1,#mountpaths do + local root = mountpaths[k] local f = io.open(root.."/url.tmi") if f then for line in f:lines() do if line then - if line:find("^[%%#%-]") then -- or %W + if find(line,"^[%%#%-]") then -- or %W -- skip - elseif line:find("^zip://") then + elseif find(line,"^zip://") then if trace_locating then logs.report("fileio","mounting %s",line) end @@ -8996,11 +10864,13 @@ function statistics.check_fmt_status(texname) local luv = dofile(luvname) if luv and luv.sourcefile then local sourcehash = md5.hex(io.loaddata(resolvers.find_file(luv.sourcefile)) or "unknown") - if luv.enginebanner and luv.enginebanner ~= enginebanner then - return "engine mismatch" + local luvbanner = luv.enginebanner or "?" + if luvbanner ~= enginebanner then + return string.format("engine mismatch (luv:%s <> bin:%s)",luvbanner,enginebanner) end - if luv.sourcehash and luv.sourcehash ~= sourcehash then - return "source mismatch" + local luvhash = luv.sourcehash or "?" + if luvhash ~= sourcehash then + return string.format("source mismatch (luv:%s <> bin:%s)",luvhash,sourcehash) end else return "invalid status file" @@ -9019,18 +10889,22 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-zip'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -local format, find = string.format, string.find +local format, find, match = string.format, string.find, string.match +local unpack = unpack or table.unpack -local trace_locating, trace_verbose = false, false +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) -trackers.register("resolvers.verbose", function(v) trace_verbose = v end) -trackers.register("resolvers.locating", function(v) trace_locating = v trace_verbose = v end) +-- zip:///oeps.zip?name=bla/bla.tex +-- zip:///oeps.zip?tree=tex/texmf-local +-- zip:///texmf.zip?tree=/tex/texmf +-- zip:///texmf.zip?tree=/tex/texmf-local +-- zip:///texmf-mine.zip?tree=/tex/texmf-projects zip = zip or { } zip.archives = zip.archives or { } @@ -9041,9 +10915,6 @@ local locators, hashers, concatinators = resolvers.locators, resolvers.hashers, local archives = zip.archives --- zip:///oeps.zip?name=bla/bla.tex --- zip:///oeps.zip?tree=tex/texmf-local - local function validzip(str) -- todo: use url splitter if not find(str,"^zip://") then return "zip:///" .. str @@ -9073,26 +10944,22 @@ function zip.closearchive(name) end end --- zip:///texmf.zip?tree=/tex/texmf --- zip:///texmf.zip?tree=/tex/texmf-local --- zip:///texmf-mine.zip?tree=/tex/texmf-projects - function locators.zip(specification) -- where is this used? startup zips (untested) specification = resolvers.splitmethod(specification) local zipfile = specification.path local zfile = zip.openarchive(name) -- tricky, could be in to be initialized tree if trace_locating then if zfile then - logs.report("fileio",'! zip locator, found: %s',specification.original) + logs.report("fileio","zip locator, archive '%s' found",specification.original) else - logs.report("fileio",'? zip locator, not found: %s',specification.original) + logs.report("fileio","zip locator, archive '%s' not found",specification.original) end end end function hashers.zip(tag,name) - if trace_verbose then - logs.report("fileio","loading zip file %s as %s",name,tag) + if trace_locating then + logs.report("fileio","loading zip file '%s' as '%s'",name,tag) end resolvers.usezipfile(format("%s?tree=%s",tag,name)) end @@ -9117,23 +10984,25 @@ function finders.zip(specification,filetype) local zfile = zip.openarchive(specification.path) if zfile then if trace_locating then - logs.report("fileio",'! zip finder, path: %s',specification.path) + logs.report("fileio","zip finder, archive '%s' found",specification.path) end local dfile = zfile:open(q.name) if dfile then dfile = zfile:close() if trace_locating then - logs.report("fileio",'+ zip finder, name: %s',q.name) + logs.report("fileio","zip finder, file '%s' found",q.name) end return specification.original + elseif trace_locating then + logs.report("fileio","zip finder, file '%s' not found",q.name) end elseif trace_locating then - logs.report("fileio",'? zip finder, path %s',specification.path) + logs.report("fileio","zip finder, unknown archive '%s'",specification.path) end end end if trace_locating then - logs.report("fileio",'- zip finder, name: %s',filename) + logs.report("fileio","zip finder, '%s' not found",filename) end return unpack(finders.notfound) end @@ -9146,20 +11015,25 @@ function openers.zip(specification) local zfile = zip.openarchive(zipspecification.path) if zfile then if trace_locating then - logs.report("fileio",'+ zip starter, path: %s',zipspecification.path) + logs.report("fileio","zip opener, archive '%s' opened",zipspecification.path) end local dfile = zfile:open(q.name) if dfile then logs.show_open(specification) + if trace_locating then + logs.report("fileio","zip opener, file '%s' found",q.name) + end return openers.text_opener(specification,dfile,'zip') + elseif trace_locating then + logs.report("fileio","zip opener, file '%s' not found",q.name) end elseif trace_locating then - logs.report("fileio",'- zip starter, path %s',zipspecification.path) + logs.report("fileio","zip opener, unknown archive '%s'",zipspecification.path) end end end if trace_locating then - logs.report("fileio",'- zip opener, name: %s',filename) + logs.report("fileio","zip opener, '%s' not found",filename) end return unpack(openers.notfound) end @@ -9172,25 +11046,27 @@ function loaders.zip(specification) local zfile = zip.openarchive(specification.path) if zfile then if trace_locating then - logs.report("fileio",'+ zip starter, path: %s',specification.path) + logs.report("fileio","zip loader, archive '%s' opened",specification.path) end local dfile = zfile:open(q.name) if dfile then logs.show_load(filename) if trace_locating then - logs.report("fileio",'+ zip loader, name: %s',filename) + logs.report("fileio","zip loader, file '%s' loaded",filename) end local s = dfile:read("*all") dfile:close() return true, s, #s + elseif trace_locating then + logs.report("fileio","zip loader, file '%s' not found",q.name) end elseif trace_locating then - logs.report("fileio",'- zip starter, path: %s',specification.path) + logs.report("fileio","zip loader, unknown archive '%s'",specification.path) end end end if trace_locating then - logs.report("fileio",'- zip loader, name: %s',filename) + logs.report("fileio","zip loader, '%s' not found",filename) end return unpack(openers.notfound) end @@ -9200,21 +11076,15 @@ end function resolvers.usezipfile(zipname) zipname = validzip(zipname) - if trace_locating then - logs.report("fileio",'! zip use, file: %s',zipname) - end local specification = resolvers.splitmethod(zipname) local zipfile = specification.path if zipfile and not zip.registeredfiles[zipname] then local tree = url.query(specification.query).tree or "" - if trace_locating then - logs.report("fileio",'! zip register, file: %s',zipname) - end local z = zip.openarchive(zipfile) if z then local instance = resolvers.instance if trace_locating then - logs.report("fileio","= zipfile, registering: %s",zipname) + logs.report("fileio","zip registering, registering archive '%s'",zipname) end statistics.starttiming(instance) resolvers.prepend_hash('zip',zipname,zipfile) @@ -9223,10 +11093,10 @@ function resolvers.usezipfile(zipname) instance.files[zipname] = resolvers.register_zip_file(z,tree or "") statistics.stoptiming(instance) elseif trace_locating then - logs.report("fileio","? zipfile, unknown: %s",zipname) + logs.report("fileio","zip registering, unknown archive '%s'",zipname) end elseif trace_locating then - logs.report("fileio",'! zip register, no file: %s',zipname) + logs.report("fileio","zip registering, '%s' not found",zipname) end end @@ -9238,11 +11108,11 @@ function resolvers.register_zip_file(z,tree) filter = format("^%s/(.+)/(.-)$",tree) end if trace_locating then - logs.report("fileio",'= zip filter: %s',filter) + logs.report("fileio","zip registering, using filter '%s'",filter) end local register, n = resolvers.register_file, 0 for i in z:files() do - local path, name = i.filename:match(filter) + local path, name = match(i.filename,filter) if path then if name and name ~= '' then register(files, name, path) @@ -9255,7 +11125,7 @@ function resolvers.register_zip_file(z,tree) n = n + 1 end end - logs.report("fileio",'= zip entries: %s',n) + logs.report("fileio","zip registering, %s files registered",n) return files end @@ -9266,12 +11136,14 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-crl'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } +local gsub = string.gsub + curl = curl or { } curl.cached = { } @@ -9280,9 +11152,9 @@ curl.cachepath = caches.definepath("curl") local finders, openers, loaders = resolvers.finders, resolvers.openers, resolvers.loaders function curl.fetch(protocol, name) - local cachename = curl.cachepath() .. "/" .. name:gsub("[^%a%d%.]+","-") --- cachename = cachename:gsub("[\\/]", io.fileseparator) - cachename = cachename:gsub("[\\]", "/") -- cleanup + local cachename = curl.cachepath() .. "/" .. gsub(name,"[^%a%d%.]+","-") +-- cachename = gsub(cachename,"[\\/]", io.fileseparator) + cachename = gsub(cachename,"[\\]", "/") -- cleanup if not curl.cached[name] then if not io.exists(cachename) then curl.cached[name] = cachename @@ -9328,6 +11200,164 @@ end -- of closure do -- create closure to overcome 200 locals limit +if not modules then modules = { } end modules ['data-lua'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- some loading stuff ... we might move this one to slot 2 depending +-- on the developments (the loaders must not trigger kpse); we could +-- of course use a more extensive lib path spec + +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) + +local gsub, insert = string.gsub, table.insert +local unpack = unpack or table.unpack + +local libformats = { 'luatexlibs', 'tex', 'texmfscripts', 'othertextfiles' } -- 'luainputs' +local clibformats = { 'lib' } + +local _path_, libpaths, _cpath_, clibpaths + +function package.libpaths() + if not _path_ or package.path ~= _path_ then + _path_ = package.path + libpaths = file.split_path(_path_,";") + end + return libpaths +end + +function package.clibpaths() + if not _cpath_ or package.cpath ~= _cpath_ then + _cpath_ = package.cpath + clibpaths = file.split_path(_cpath_,";") + end + return clibpaths +end + +local function thepath(...) + local t = { ... } t[#t+1] = "?.lua" + local path = file.join(unpack(t)) + if trace_locating then + logs.report("fileio","! appending '%s' to 'package.path'",path) + end + return path +end + +local p_libpaths, a_libpaths = { }, { } + +function package.append_libpath(...) + insert(a_libpath,thepath(...)) +end + +function package.prepend_libpath(...) + insert(p_libpaths,1,thepath(...)) +end + +-- beware, we need to return a loadfile result ! + +local function loaded(libpaths,name,simple) + for i=1,#libpaths do -- package.path, might become option + local libpath = libpaths[i] + local resolved = gsub(libpath,"%?",simple) + if trace_locating then -- more detail + logs.report("fileio","! checking for '%s' on 'package.path': '%s' => '%s'",simple,libpath,resolved) + end + if resolvers.isreadable.file(resolved) then + if trace_locating then + logs.report("fileio","! lib '%s' located via 'package.path': '%s'",name,resolved) + end + return loadfile(resolved) + end + end +end + + +package.loaders[2] = function(name) -- was [#package.loaders+1] + if trace_locating then -- mode detail + logs.report("fileio","! locating '%s'",name) + end + for i=1,#libformats do + local format = libformats[i] + local resolved = resolvers.find_file(name,format) or "" + if trace_locating then -- mode detail + logs.report("fileio","! checking for '%s' using 'libformat path': '%s'",name,format) + end + if resolved ~= "" then + if trace_locating then + logs.report("fileio","! lib '%s' located via environment: '%s'",name,resolved) + end + return loadfile(resolved) + end + end + -- libpaths + local libpaths, clibpaths = package.libpaths(), package.clibpaths() + local simple = gsub(name,"%.lua$","") + local simple = gsub(simple,"%.","/") + local resolved = loaded(p_libpaths,name,simple) or loaded(libpaths,name,simple) or loaded(a_libpaths,name,simple) + if resolved then + return resolved + end + -- + local libname = file.addsuffix(simple,os.libsuffix) + for i=1,#clibformats do + -- better have a dedicated loop + local format = clibformats[i] + local paths = resolvers.expanded_path_list_from_var(format) + for p=1,#paths do + local path = paths[p] + local resolved = file.join(path,libname) + if trace_locating then -- mode detail + logs.report("fileio","! checking for '%s' using 'clibformat path': '%s'",libname,path) + end + if resolvers.isreadable.file(resolved) then + if trace_locating then + logs.report("fileio","! lib '%s' located via 'clibformat': '%s'",libname,resolved) + end + return package.loadlib(resolved,name) + end + end + end + for i=1,#clibpaths do -- package.path, might become option + local libpath = clibpaths[i] + local resolved = gsub(libpath,"?",simple) + if trace_locating then -- more detail + logs.report("fileio","! checking for '%s' on 'package.cpath': '%s'",simple,libpath) + end + if resolvers.isreadable.file(resolved) then + if trace_locating then + logs.report("fileio","! lib '%s' located via 'package.cpath': '%s'",name,resolved) + end + return package.loadlib(resolved,name) + end + end + -- just in case the distribution is messed up + if trace_loading then -- more detail + logs.report("fileio","! checking for '%s' using 'luatexlibs': '%s'",name) + end + local resolved = resolvers.find_file(file.basename(name),'luatexlibs') or "" + if resolved ~= "" then + if trace_locating then + logs.report("fileio","! lib '%s' located by basename via environment: '%s'",name,resolved) + end + return loadfile(resolved) + end + if trace_locating then + logs.report("fileio",'? unable to locate lib: %s',name) + end +-- return "unable to locate " .. name +end + +resolvers.loadlualib = require + + +end -- of closure + +do -- create closure to overcome 200 locals limit + if not modules then modules = { } end modules ['luat-kps'] = { version = 1.001, comment = "companion to luatools.lua", @@ -9437,7 +11467,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-aux'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -9445,47 +11475,47 @@ if not modules then modules = { } end modules ['data-aux'] = { local find = string.find -local trace_verbose = false trackers.register("resolvers.verbose", function(v) trace_verbose = v end) +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) function resolvers.update_script(oldname,newname) -- oldname -> own.name, not per se a suffix local scriptpath = "scripts/context/lua" newname = file.addsuffix(newname,"lua") local oldscript = resolvers.clean_path(oldname) - if trace_verbose then + if trace_locating then logs.report("fileio","to be replaced old script %s", oldscript) end local newscripts = resolvers.find_files(newname) or { } if #newscripts == 0 then - if trace_verbose then + if trace_locating then logs.report("fileio","unable to locate new script") end else for i=1,#newscripts do local newscript = resolvers.clean_path(newscripts[i]) - if trace_verbose then + if trace_locating then logs.report("fileio","checking new script %s", newscript) end if oldscript == newscript then - if trace_verbose then + if trace_locating then logs.report("fileio","old and new script are the same") end elseif not find(newscript,scriptpath) then - if trace_verbose then + if trace_locating then logs.report("fileio","new script should come from %s",scriptpath) end elseif not (find(oldscript,file.removesuffix(newname).."$") or find(oldscript,newname.."$")) then - if trace_verbose then + if trace_locating then logs.report("fileio","invalid new script name") end else local newdata = io.loaddata(newscript) if newdata then - if trace_verbose then + if trace_locating then logs.report("fileio","old script content replaced by new content") end io.savedata(oldscript,newdata) break - elseif trace_verbose then + elseif trace_locating then logs.report("fileio","unable to load new script") end end @@ -9500,25 +11530,28 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-tmf'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } +local find, gsub, match = string.find, string.gsub, string.match +local getenv, setenv = os.getenv, os.setenv + -- loads *.tmf files in minimal tree roots (to be optimized and documented) function resolvers.check_environment(tree) logs.simpleline() - os.setenv('TMP', os.getenv('TMP') or os.getenv('TEMP') or os.getenv('TMPDIR') or os.getenv('HOME')) - os.setenv('TEXOS', os.getenv('TEXOS') or ("texmf-" .. os.currentplatform())) - os.setenv('TEXPATH', (tree or "tex"):gsub("\/+$",'')) - os.setenv('TEXMFOS', os.getenv('TEXPATH') .. "/" .. os.getenv('TEXOS')) + setenv('TMP', getenv('TMP') or getenv('TEMP') or getenv('TMPDIR') or getenv('HOME')) + setenv('TEXOS', getenv('TEXOS') or ("texmf-" .. os.platform)) + setenv('TEXPATH', gsub(tree or "tex","\/+$",'')) + setenv('TEXMFOS', getenv('TEXPATH') .. "/" .. getenv('TEXOS')) logs.simpleline() - logs.simple("preset : TEXPATH => %s", os.getenv('TEXPATH')) - logs.simple("preset : TEXOS => %s", os.getenv('TEXOS')) - logs.simple("preset : TEXMFOS => %s", os.getenv('TEXMFOS')) - logs.simple("preset : TMP => %s", os.getenv('TMP')) + logs.simple("preset : TEXPATH => %s", getenv('TEXPATH')) + logs.simple("preset : TEXOS => %s", getenv('TEXOS')) + logs.simple("preset : TEXMFOS => %s", getenv('TEXMFOS')) + logs.simple("preset : TMP => %s", getenv('TMP')) logs.simple('') end @@ -9526,27 +11559,27 @@ function resolvers.load_environment(name) -- todo: key=value as well as lua local f = io.open(name) if f then for line in f:lines() do - if line:find("^[%%%#]") then + if find(line,"^[%%%#]") then -- skip comment else - local key, how, value = line:match("^(.-)%s*([<=>%?]+)%s*(.*)%s*$") + local key, how, value = match(line,"^(.-)%s*([<=>%?]+)%s*(.*)%s*$") if how then - value = value:gsub("%%(.-)%%", function(v) return os.getenv(v) or "" end) + value = gsub(value,"%%(.-)%%", function(v) return getenv(v) or "" end) if how == "=" or how == "<<" then - os.setenv(key,value) + setenv(key,value) elseif how == "?" or how == "??" then - os.setenv(key,os.getenv(key) or value) + setenv(key,getenv(key) or value) elseif how == "<" or how == "+=" then - if os.getenv(key) then - os.setenv(key,os.getenv(key) .. io.fileseparator .. value) + if getenv(key) then + setenv(key,getenv(key) .. io.fileseparator .. value) else - os.setenv(key,value) + setenv(key,value) end elseif how == ">" or how == "=+" then - if os.getenv(key) then - os.setenv(key,value .. io.pathseparator .. os.getenv(key)) + if getenv(key) then + setenv(key,value .. io.pathseparator .. getenv(key)) else - os.setenv(key,value) + setenv(key,value) end end end @@ -9585,6 +11618,9 @@ if not modules then modules = { } end modules ['luat-sta'] = { -- this code is used in the updater +local gmatch, match = string.gmatch, string.match +local type = type + states = states or { } states.data = states.data or { } states.hash = states.hash or { } @@ -9613,13 +11649,17 @@ function states.set_by_tag(tag,key,value,default,persistent) if d then if type(d) == "table" then local dkey, hkey = key, key - local pre, post = key:match("(.+)%.([^%.]+)$") + local pre, post = match(key,"(.+)%.([^%.]+)$") if pre and post then - for k in pre:gmatch("[^%.]+") do + for k in gmatch(pre,"[^%.]+") do local dk = d[k] if not dk then dk = { } d[k] = dk + elseif type(dk) == "string" then + -- invalid table, unable to upgrade structure + -- hope for the best or delete the state file + break end d = dk end @@ -9647,7 +11687,7 @@ function states.get_by_tag(tag,key,default) else local d = states.data[tag] if d then - for k in key:gmatch("[^%.]+") do + for k in gmatch(key,"[^%.]+") do local dk = d[k] if dk then d = dk @@ -9782,6 +11822,7 @@ own.libs = { -- todo: check which ones are really needed 'l-os.lua', 'l-file.lua', 'l-md5.lua', + 'l-url.lua', 'l-dir.lua', 'l-boolean.lua', 'l-math.lua', @@ -9790,11 +11831,13 @@ own.libs = { -- todo: check which ones are really needed 'l-utils.lua', 'l-aux.lua', -- 'l-xml.lua', + 'trac-tra.lua', 'lxml-tab.lua', - 'lxml-pth.lua', - 'lxml-ent.lua', + 'lxml-lpt.lua', +-- 'lxml-ent.lua', 'lxml-mis.lua', - 'trac-tra.lua', + 'lxml-aux.lua', + 'lxml-xml.lua', 'luat-env.lua', 'trac-inf.lua', 'trac-log.lua', @@ -9809,7 +11852,7 @@ own.libs = { -- todo: check which ones are really needed -- 'data-bin.lua', 'data-zip.lua', 'data-crl.lua', --- 'data-lua.lua', + 'data-lua.lua', 'data-kps.lua', -- so that we can replace kpsewhich 'data-aux.lua', -- updater 'data-tmf.lua', -- tree files @@ -9827,7 +11870,8 @@ end -- End of hack. -own.name = (environment and environment.ownname) or arg[0] or 'luatools.lua' +own.name = (environment and environment.ownname) or arg[0] or 'luatools.lua' + own.path = string.match(own.name,"^(.+)[\\/].-$") or "." own.list = { '.' } @@ -9865,18 +11909,25 @@ if not resolvers then os.exit() end -logs.setprogram('MTXrun',"TDS Runner Tool 1.22",environment.arguments["verbose"] or false) +logs.setprogram('MTXrun',"TDS Runner Tool 1.24",environment.arguments["verbose"] or false) local instance = resolvers.reset() +local trackspec = environment.argument("trackers") or environment.argument("track") + +if trackspec then + trackers.enable(trackspec) +end + runners = runners or { } -- global messages = messages or { } messages.help = [[ ---script run an mtx script (--noquotes) ---execute run a script or program (--noquotes) +--script run an mtx script (lua prefered method) (--noquotes), no script gives list +--execute run a script or program (texmfstart method) (--noquotes) --resolve resolve prefixed arguments --ctxlua run internally (using preloaded libs) +--internal run script using built in libraries (same as --ctxlua) --locate locate given filename --autotree use texmf tree cf. env 'texmfstart_tree' or 'texmfstarttree' @@ -9893,16 +11944,20 @@ messages.help = [[ --unix create unix (linux) stubs --verbose give a bit more info +--trackers=list enable given trackers --engine=str target engine --progname=str format or backend --edit launch editor with found file --launch (--all) launch files like manuals, assumes os support ---intern run script using built in libraries +--timedrun run a script an time its run +--autogenerate regenerate databases if needed (handy when used to run context in an editor) + +--usekpse use kpse as fallback (when no mkiv and cache installed, often slower) +--forcekpse force using kpse (handy when no mkiv and cache installed but less functionality) ---usekpse use kpse as fallback (when no mkiv and cache installed, often slower) ---forcekpse force using kpse (handy when no mkiv and cache installed but less functionality) +--prefixes show supported prefixes ]] runners.applications = { @@ -9918,20 +11973,17 @@ runners.suffixes = { } runners.registered = { - texexec = { 'texexec.rb', true }, -- context mkii runner (only tool not to be luafied) + texexec = { 'texexec.rb', false }, -- context mkii runner (only tool not to be luafied) texutil = { 'texutil.rb', true }, -- old perl based index sorter for mkii (old versions need it) texfont = { 'texfont.pl', true }, -- perl script that makes mkii font metric files texfind = { 'texfind.pl', false }, -- perltk based tex searching tool, mostly used at pragma texshow = { 'texshow.pl', false }, -- perltk based context help system, will be luafied - -- texwork = { \texwork.pl', false }, -- perltk based editing environment, only used at pragma - + -- texwork = { 'texwork.pl', false }, -- perltk based editing environment, only used at pragma makempy = { 'makempy.pl', true }, mptopdf = { 'mptopdf.pl', true }, pstopdf = { 'pstopdf.rb', true }, -- converts ps (and some more) images, does some cleaning (replaced) - -- examplex = { 'examplex.rb', false }, concheck = { 'concheck.rb', false }, - runtools = { 'runtools.rb', true }, textools = { 'textools.rb', true }, tmftools = { 'tmftools.rb', true }, @@ -9943,7 +11995,6 @@ runners.registered = { xmltools = { 'xmltools.rb', true }, -- luatools = { 'luatools.lua', true }, mtxtools = { 'mtxtools.rb', true }, - pdftrimwhite = { 'pdftrimwhite.pl', false } } @@ -9952,6 +12003,13 @@ runners.launchers = { unix = { } } +-- like runners.libpath("framework"): looks on script's subpath + +function runners.libpath(...) + package.prepend_libpath(file.dirname(environment.ownscript),...) + package.prepend_libpath(file.dirname(environment.ownname) ,...) +end + function runners.prepare() local checkname = environment.argument("ifchanged") if checkname and checkname ~= "" then @@ -9996,7 +12054,7 @@ function runners.prepare() return "run" end -function runners.execute_script(fullname,internal) +function runners.execute_script(fullname,internal,nosplit) local noquote = environment.argument("noquotes") if fullname and fullname ~= "" then local state = runners.prepare() @@ -10036,17 +12094,20 @@ function runners.execute_script(fullname,internal) end end if result and result ~= "" then - local before, after = environment.split_arguments(fullname) -- already done - environment.arguments_before, environment.arguments_after = before, after + if not no_split then + local before, after = environment.split_arguments(fullname) -- already done + environment.arguments_before, environment.arguments_after = before, after + end if internal then - arg = { } for _,v in pairs(after) do arg[#arg+1] = v end + arg = { } for _,v in pairs(environment.arguments_after) do arg[#arg+1] = v end + environment.ownscript = result dofile(result) else local binary = runners.applications[file.extname(result)] if binary and binary ~= "" then result = binary .. " " .. result end - local command = result .. " " .. environment.reconstruct_commandline(after,noquote) + local command = result .. " " .. environment.reconstruct_commandline(environment.arguments_after,noquote) if logs.verbose then logs.simpleline() logs.simple("executing: %s",command) @@ -10054,8 +12115,24 @@ function runners.execute_script(fullname,internal) logs.simpleline() io.flush() end - local code = os.exec(command) -- maybe spawn - return code == 0 + -- no os.exec because otherwise we get the wrong return value + local code = os.execute(command) -- maybe spawn + if code == 0 then + return true + else + if binary then + binary = file.addsuffix(binary,os.binsuffix) + for p in string.gmatch(os.getenv("PATH"),"[^"..io.pathseparator.."]+") do + if lfs.isfile(file.join(p,binary)) then + return false + end + end + logs.simpleline() + logs.simple("This script needs '%s' which seems not to be installed.",binary) + logs.simpleline() + end + return false + end end end end @@ -10088,7 +12165,7 @@ function runners.execute_program(fullname) return false end --- the --usekpse flag will fallback on kpse +-- the --usekpse flag will fallback on kpse (hm, we can better update mtx-stubs) local windows_stub = '@echo off\013\010setlocal\013\010set ownpath=%%~dp0%%\013\010texlua "%%ownpath%%mtxrun.lua" --usekpse --execute %s %%*\013\010endlocal\013\010' local unix_stub = '#!/bin/sh\010mtxrun --usekpse --execute %s \"$@\"\010' @@ -10143,7 +12220,7 @@ function runners.locate_file(filename) end function runners.locate_platform() - runners.report_location(os.currentplatform()) + runners.report_location(os.platform) end function runners.report_location(result) @@ -10176,7 +12253,8 @@ end function runners.save_script_session(filename, list) local t = { } - for _, key in ipairs(list) do + for i=1,#list do + local key = list[i] t[key] = environment.arguments[key] end io.savedata(filename,table.serialize(t,true)) @@ -10265,20 +12343,22 @@ function runners.find_mtx_script(filename) if fullname and fullname ~= "" then return fullname end + -- mtx- prefix checking + local mtxprefix = (filename:find("^mtx%-") and "") or "mtx-" -- context namespace, mtx-<filename> - fullname = "mtx-" .. filename + fullname = mtxprefix .. filename fullname = found(fullname) or resolvers.find_file(fullname) if fullname and fullname ~= "" then return fullname end -- context namespace, mtx-<filename>s - fullname = "mtx-" .. basename .. "s" .. "." .. suffix + fullname = mtxprefix .. basename .. "s" .. "." .. suffix fullname = found(fullname) or resolvers.find_file(fullname) if fullname and fullname ~= "" then return fullname end -- context namespace, mtx-<filename minus trailing s> - fullname = "mtx-" .. basename:gsub("s$","") .. "." .. suffix + fullname = mtxprefix .. basename:gsub("s$","") .. "." .. suffix fullname = found(fullname) or resolvers.find_file(fullname) if fullname and fullname ~= "" then return fullname @@ -10288,9 +12368,17 @@ function runners.find_mtx_script(filename) return fullname end -function runners.execute_ctx_script(filename,arguments) +function runners.execute_ctx_script(filename) + local arguments = environment.arguments_after local fullname = runners.find_mtx_script(filename) or "" - -- retyr after generate but only if --autogenerate + if file.extname(fullname) == "cld" then + -- handy in editors where we force --autopdf + logs.simple("running cld script: %s",filename) + table.insert(arguments,1,fullname) + table.insert(arguments,"--autopdf") + fullname = runners.find_mtx_script("context") or "" + end + -- retry after generate but only if --autogenerate if fullname == "" and environment.argument("autogenerate") then -- might become the default instance.renewcache = true logs.setverbose(true) @@ -10319,32 +12407,51 @@ function runners.execute_ctx_script(filename,arguments) if logs.verbose then logs.simple("using script: %s\n",fullname) end + environment.ownscript = fullname dofile(fullname) local savename = environment.arguments['save'] - if savename and runners.save_list and not table.is_empty(runners.save_list or { }) then - if type(savename) ~= "string" then savename = file.basename(fullname) end - savename = file.replacesuffix(savename,"cfg") - runners.save_script_session(savename, runners.save_list) + if savename then + local save_list = runners.save_list + if save_list and next(save_list) then + if type(savename) ~= "string" then savename = file.basename(fullname) end + savename = file.replacesuffix(savename,"cfg") + runners.save_script_session(savename,save_list) + end end return true end else - logs.setverbose(true) - if filename == "" then - logs.simple("unknown script, no name given") + -- logs.setverbose(true) + if filename == "" or filename == "help" then local context = resolvers.find_file("mtx-context.lua") + logs.setverbose(true) if context ~= "" then local result = dir.glob((string.gsub(context,"mtx%-context","mtx-*"))) -- () needed local valid = { } - for _, scriptname in ipairs(result) do - scriptname = string.match(scriptname,".*mtx%-([^%-]-)%.lua") - if scriptname then - valid[#valid+1] = scriptname + table.sort(result) + for i=1,#result do + local scriptname = result[i] + local scriptbase = string.match(scriptname,".*mtx%-([^%-]-)%.lua") + if scriptbase then + local data = io.loaddata(scriptname) + local banner, version = string.match(data,"[\n\r]logs%.extendbanner%s*%(%s*[\"\']([^\n\r]+)%s*(%d+%.%d+)") + if banner then + valid[#valid+1] = { scriptbase, version, banner } + end end end if #valid > 0 then - logs.simple("known scripts: %s",table.concat(valid,", ")) + logs.reportbanner() + logs.reportline() + logs.simple("no script name given, known scripts:") + logs.simple() + for k=1,#valid do + local v = valid[k] + logs.simple("%-12s %4s %s",v[1],v[2],v[3]) + end end + else + logs.simple("no script name given") end else filename = file.addsuffix(filename,"lua") @@ -10358,6 +12465,12 @@ function runners.execute_ctx_script(filename,arguments) end end +function runners.prefixes() + logs.reportbanner() + logs.reportline() + logs.simple(table.concat(resolvers.allprefixes(true)," ")) +end + function runners.timedrun(filename) -- just for me if filename and filename ~= "" then runners.timed(function() os.execute(filename) end) @@ -10385,7 +12498,9 @@ instance.lsrmode = environment.argument("lsr") or false -- maybe the unset has to go to this level -if environment.argument("usekpse") or environment.argument("forcekpse") then +local is_mkii_stub = runners.registered[file.removesuffix(file.basename(filename))] + +if environment.argument("usekpse") or environment.argument("forcekpse") or is_mkii_stub then os.setenv("engine","") os.setenv("progname","") @@ -10420,7 +12535,7 @@ if environment.argument("usekpse") or environment.argument("forcekpse") then return (kpse_initialized():show_path(name)) or "" end - elseif environment.argument("usekpse") then + elseif environment.argument("usekpse") or is_mkii_stub then resolvers.load() @@ -10449,7 +12564,6 @@ else end - if environment.argument("selfmerge") then -- embed used libraries utils.merger.selfmerge(own.name,own.libs,own.list) @@ -10462,9 +12576,14 @@ elseif environment.argument("selfupdate") then elseif environment.argument("ctxlua") or environment.argument("internal") then -- run a script by loading it (using libs) ok = runners.execute_script(filename,true) -elseif environment.argument("script") or environment.argument("s") then +elseif environment.argument("script") or environment.argument("scripts") then -- run a script by loading it (using libs), pass args - ok = runners.execute_ctx_script(filename,after) + if is_mkii_stub then + -- execute mkii script + ok = runners.execute_script(filename,false,true) + else + ok = runners.execute_ctx_script(filename) + end elseif environment.argument("execute") then -- execute script ok = runners.execute_script(filename) @@ -10491,6 +12610,8 @@ elseif environment.argument("locate") then elseif environment.argument("platform")then -- locate platform runners.locate_platform() +elseif environment.argument("prefixes") then + runners.prefixes() elseif environment.argument("timedrun") then -- locate platform runners.timedrun(filename) @@ -10499,8 +12620,14 @@ elseif environment.argument("help") or filename=='help' or filename == "" then -- execute script elseif filename:find("^bin:") then ok = runners.execute_program(filename) +elseif is_mkii_stub then + -- execute mkii script + ok = runners.execute_script(filename,false,true) else - ok = runners.execute_script(filename) + ok = runners.execute_ctx_script(filename) + if not ok then + ok = runners.execute_script(filename) + end end if os.platform == "unix" then diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/mtxtools.bat b/Master/texmf-dist/scripts/context/stubs/mswin/mtxtools.bat deleted file mode 100755 index 2e0fd35080d..00000000000 --- a/Master/texmf-dist/scripts/context/stubs/mswin/mtxtools.bat +++ /dev/null @@ -1,5 +0,0 @@ -@echo off
-setlocal
-set ownpath=%~dp0%
-texlua "%ownpath%mtxrun.lua" --usekpse --execute mtxtools.rb %*
-endlocal
diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/mtxworks.cmd b/Master/texmf-dist/scripts/context/stubs/mswin/mtxworks.cmd deleted file mode 100644 index 322d9464dd5..00000000000 --- a/Master/texmf-dist/scripts/context/stubs/mswin/mtxworks.cmd +++ /dev/null @@ -1 +0,0 @@ -mtxrun --script texworks --start diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/pdftools.bat b/Master/texmf-dist/scripts/context/stubs/mswin/pdftools.bat deleted file mode 100755 index 94c8cf3bd79..00000000000 --- a/Master/texmf-dist/scripts/context/stubs/mswin/pdftools.bat +++ /dev/null @@ -1,5 +0,0 @@ -@echo off
-setlocal
-set ownpath=%~dp0%
-texlua "%ownpath%mtxrun.lua" --usekpse --execute pdftools.rb %*
-endlocal
diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/pstopdf.bat b/Master/texmf-dist/scripts/context/stubs/mswin/pstopdf.bat deleted file mode 100755 index ebc3cb690a7..00000000000 --- a/Master/texmf-dist/scripts/context/stubs/mswin/pstopdf.bat +++ /dev/null @@ -1,5 +0,0 @@ -@echo off
-setlocal
-set ownpath=%~dp0%
-texlua "%ownpath%mtxrun.lua" --usekpse --execute pstopdf.rb %*
-endlocal
diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/rlxtools.bat b/Master/texmf-dist/scripts/context/stubs/mswin/rlxtools.bat deleted file mode 100755 index 5069c935f3c..00000000000 --- a/Master/texmf-dist/scripts/context/stubs/mswin/rlxtools.bat +++ /dev/null @@ -1,5 +0,0 @@ -@echo off
-setlocal
-set ownpath=%~dp0%
-texlua "%ownpath%mtxrun.lua" --usekpse --execute rlxtools.rb %*
-endlocal
diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/runtools.bat b/Master/texmf-dist/scripts/context/stubs/mswin/runtools.bat deleted file mode 100755 index 2f89413d436..00000000000 --- a/Master/texmf-dist/scripts/context/stubs/mswin/runtools.bat +++ /dev/null @@ -1,5 +0,0 @@ -@echo off
-setlocal
-set ownpath=%~dp0%
-texlua "%ownpath%mtxrun.lua" --usekpse --execute runtools.rb %*
-endlocal
diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/texexec.bat b/Master/texmf-dist/scripts/context/stubs/mswin/texexec.bat deleted file mode 100755 index 6e8b80aabce..00000000000 --- a/Master/texmf-dist/scripts/context/stubs/mswin/texexec.bat +++ /dev/null @@ -1,5 +0,0 @@ -@echo off
-setlocal
-set ownpath=%~dp0%
-texlua "%ownpath%mtxrun.lua" --usekpse --execute texexec.rb %*
-endlocal
diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/texexec.cmd b/Master/texmf-dist/scripts/context/stubs/mswin/texexec.cmd deleted file mode 100644 index acbe41fd82b..00000000000 --- a/Master/texmf-dist/scripts/context/stubs/mswin/texexec.cmd +++ /dev/null @@ -1,5 +0,0 @@ -@echo off -setlocal -set ownpath=%~dp0% -texlua "%ownpath%mtxrun.lua" --usekpse --execute texexec.rb %* -endlocal diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/texexec.exe b/Master/texmf-dist/scripts/context/stubs/mswin/texexec.exe Binary files differnew file mode 100644 index 00000000000..2d45f27494d --- /dev/null +++ b/Master/texmf-dist/scripts/context/stubs/mswin/texexec.exe diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/texfont.bat b/Master/texmf-dist/scripts/context/stubs/mswin/texfont.bat deleted file mode 100755 index 498d4d85b51..00000000000 --- a/Master/texmf-dist/scripts/context/stubs/mswin/texfont.bat +++ /dev/null @@ -1,5 +0,0 @@ -@echo off
-setlocal
-set ownpath=%~dp0%
-texlua "%ownpath%mtxrun.lua" --usekpse --execute texfont.pl %*
-endlocal
diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/texmfstart.cmd b/Master/texmf-dist/scripts/context/stubs/mswin/texmfstart.cmd deleted file mode 100644 index 47a10cc54cd..00000000000 --- a/Master/texmf-dist/scripts/context/stubs/mswin/texmfstart.cmd +++ /dev/null @@ -1,5 +0,0 @@ -@echo off -setlocal -set ownpath=%~dp0% -texlua "%ownpath%mtxrun.lua" --usekpse %* -endlocal diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/texmfstart.exe b/Master/texmf-dist/scripts/context/stubs/mswin/texmfstart.exe Binary files differnew file mode 100644 index 00000000000..2d45f27494d --- /dev/null +++ b/Master/texmf-dist/scripts/context/stubs/mswin/texmfstart.exe diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/textools.bat b/Master/texmf-dist/scripts/context/stubs/mswin/textools.bat deleted file mode 100755 index b4cdfa1b332..00000000000 --- a/Master/texmf-dist/scripts/context/stubs/mswin/textools.bat +++ /dev/null @@ -1,5 +0,0 @@ -@echo off
-setlocal
-set ownpath=%~dp0%
-texlua "%ownpath%mtxrun.lua" --usekpse --execute textools.rb %*
-endlocal
diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/texutil.bat b/Master/texmf-dist/scripts/context/stubs/mswin/texutil.bat deleted file mode 100755 index 8b718640dcf..00000000000 --- a/Master/texmf-dist/scripts/context/stubs/mswin/texutil.bat +++ /dev/null @@ -1,5 +0,0 @@ -@echo off
-setlocal
-set ownpath=%~dp0%
-texlua "%ownpath%mtxrun.lua" --usekpse --execute texutil.rb %*
-endlocal
diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/tmftools.bat b/Master/texmf-dist/scripts/context/stubs/mswin/tmftools.bat deleted file mode 100755 index 1f5d8022280..00000000000 --- a/Master/texmf-dist/scripts/context/stubs/mswin/tmftools.bat +++ /dev/null @@ -1,5 +0,0 @@ -@echo off
-setlocal
-set ownpath=%~dp0%
-texlua "%ownpath%mtxrun.lua" --usekpse --execute tmftools.rb %*
-endlocal
diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/xmltools.bat b/Master/texmf-dist/scripts/context/stubs/mswin/xmltools.bat deleted file mode 100755 index 41f4ec30465..00000000000 --- a/Master/texmf-dist/scripts/context/stubs/mswin/xmltools.bat +++ /dev/null @@ -1,5 +0,0 @@ -@echo off
-setlocal
-set ownpath=%~dp0%
-texlua "%ownpath%mtxrun.lua" --usekpse --execute xmltools.rb %*
-endlocal
diff --git a/Master/texmf-dist/scripts/context/stubs/source/mtxrun_dll.c b/Master/texmf-dist/scripts/context/stubs/source/mtxrun_dll.c new file mode 100644 index 00000000000..5b7cd31a07b --- /dev/null +++ b/Master/texmf-dist/scripts/context/stubs/source/mtxrun_dll.c @@ -0,0 +1,221 @@ +/************************************************************************ + + Copyright: + + Public Domain + Originally written in 2010 by Tomasz M. Trzeciak and Hans Hagen + + This program is derived from the 'runscript' program originally + written in 2009 by T.M. Trzeciak. It has been adapted for use in + ConTeXt MkIV. + + Comment: + + In ConTeXt MkIV we have two core scripts: luatools.lua and + mtxrun.lua where the second one is used to launch other scripts. + Normally a user will use a call like: + + mtxrun --script font --reload + + Here mtxrun is a lua script. In order to avoid the usage of a cmd + file on windows this runner will start texlua directly. If the + shared library luatex.dll is available, texlua will be started in + the same process avoiding thus any additional overhead. Otherwise + it will be spawned in a new proces. + + We also don't want to use other runners, like those that use kpse + to locate the script as this is exactly what mtxrun itself is doing + already. Therefore the runscript program is adapted to a more direct + approach suitable for mtxrun. + + Compilation: + + with gcc (size optimized): + + gcc -Os -s -shared -o mtxrun.dll mtxrun_dll.c + gcc -Os -s -o mtxrun.exe mtxrun_exe.c -L./ -lmtxrun + + with tcc (extra small size): + + tcc -shared -o mtxrun.dll mtxrun_dll.c + tcc -o mtxrun.exe mtxrun_exe.c mtxrun.def + +************************************************************************/ + +#include <stdio.h> +#include <stdlib.h> +#include <windows.h> + +//#define STATIC +#define IS_WHITESPACE(c) ((c == ' ') || (c == '\t')) +#define MAX_CMD 32768 +#define DIE(...) { \ + fprintf( stderr, "mtxrun: " ); \ + fprintf( stderr, __VA_ARGS__ ); \ + return 1; \ +} + +char texlua_name[] = "texlua"; // just a bare name, luatex strips the rest anyway +static char cmdline[MAX_CMD]; +static char dirpath[MAX_PATH]; +static char progname[MAX_PATH]; +static char scriptpath[MAX_PATH]; +static char luatexpath[MAX_PATH]; +HMODULE dllluatex = NULL; +typedef int ( *mainlikeproc )( int, char ** ); + +#ifdef STATIC +int main( int argc, char *argv[] ) +#else +__declspec(dllexport) int dllrunscript( int argc, char *argv[] ) +#endif +{ + char *s, *luatexfname, *argstr, **lua_argv; + int k, quoted, lua_argc; + int passprogname = 0; + + // directory of this module/executable + + HMODULE module_handle = GetModuleHandle( "mtxrun.dll" ); + // if ( module_handle == NULL ) exe path will be used, which is OK too + k = (int) GetModuleFileName( module_handle, dirpath, MAX_PATH ); + if ( !k || ( k == MAX_PATH ) ) + DIE( "unable to determine a valid module name\n" ); + s = strrchr(dirpath, '\\'); + if ( s == NULL ) DIE( "no directory part in module path: %s\n", dirpath ); + *(++s) = '\0'; //remove file name, leave trailing backslash + + // program name + + k = strlen(argv[0]); + while ( k && (argv[0][k-1] != '/') && (argv[0][k-1] != '\\') ) k--; + strcpy(progname, &argv[0][k]); + s = progname; + if ( s = strrchr(s, '.') ) *s = '\0'; // remove file extension part + + // script path + + strcpy( scriptpath, dirpath ); + k = strlen(progname); + if ( k < 6 ) k = 6; // in case the program name is shorter than "mtxrun" + if ( strlen(dirpath) + k + 4 >= MAX_PATH ) + DIE( "path too long: %s%s\n", dirpath, progname ); + if ( ( strcmpi(progname,"mtxrun") == 0 ) || ( strcmpi(progname,"luatools") == 0 ) ) { + strcat( scriptpath, progname ); + strcat( scriptpath, ".lua" ); + } else { + strcat( scriptpath, "mtxrun.lua" ); + if ( strcmpi(progname,"texmfstart") != 0 ) passprogname = 1; + } + if ( GetFileAttributes(scriptpath) == INVALID_FILE_ATTRIBUTES ) + DIE( "file not found: %s\n", scriptpath ); + + // find texlua.exe + + if ( !SearchPath( + getenv( "PATH" ), // path to search (optional) + "texlua.exe", // file name to search + NULL, // file extension to add (optional) + MAX_PATH, // output buffer size + luatexpath, // output buffer pointer + &luatexfname ) // pointer to a file part in the output buffer (optional) + ) DIE( "unable to locate texlua.exe on the search path" ); + + // link directly with luatex.dll if available in texlua's dir + + strcpy( luatexfname, "luatex.dll" ); + if ( dllluatex = LoadLibrary(luatexpath) ) + { + mainlikeproc dllluatexmain = (mainlikeproc) GetProcAddress( dllluatex, "dllluatexmain" ); + if ( dllluatexmain == NULL ) + DIE( "unable to locate dllluatexmain procedure in luatex.dll" ); + + // set up argument list for texlua script + + lua_argv = (char **)malloc( (argc + 4) * sizeof(char *) ); + if ( lua_argv == NULL ) DIE( "out of memory\n" ); + lua_argv[lua_argc=0] = texlua_name; + lua_argv[++lua_argc] = scriptpath; // script to execute + if (passprogname) { + lua_argv[++lua_argc] = "--script"; + lua_argv[++lua_argc] = progname; + } + for ( k = 1; k < argc; k++ ) lua_argv[++lua_argc] = argv[k]; + lua_argv[++lua_argc] = NULL; + + // call texlua interpreter + // dllluatexmain never returns, but we pretend that it does + + k = dllluatexmain( lua_argc, lua_argv ); + if (lua_argv) free( lua_argv ); + return k; + } + + // we are still here, so no luatex.dll; spawn texlua.exe instead + + strcpy( luatexfname, "texlua.exe" ); + strcpy( cmdline, "\"" ); + strcat( cmdline, luatexpath ); + strcat( cmdline, "\" \"" ); + strcat( cmdline, scriptpath ); + strcat( cmdline, "\"" ); + if (passprogname) { + strcat( cmdline, " --script " ); + strcat( cmdline, progname ); + } + + argstr = GetCommandLine(); // get the command line of this process + if ( argstr == NULL ) DIE( "unable to retrieve the command line string\n" ); + + // skip over argv[0] in the argument string + // (it can contain embedded double quotes if launched from cmd.exe!) + + for ( quoted = 0; (*argstr) && ( !IS_WHITESPACE(*argstr) || quoted ); argstr++ ) + if (*argstr == '"') quoted = !quoted; + + // pass through all the arguments + + if ( strlen(cmdline) + strlen(argstr) >= MAX_CMD ) + DIE( "command line string too long:\n%s%s\n", cmdline, argstr ); + strcat( cmdline, argstr ); + + // create child process + + STARTUPINFO si; + PROCESS_INFORMATION pi; + ZeroMemory( &si, sizeof(si) ); + si.cb = sizeof(si); + si.dwFlags = STARTF_USESTDHANDLES;// | STARTF_USESHOWWINDOW; + //si.dwFlags = STARTF_USESHOWWINDOW; + //si.wShowWindow = SW_HIDE ; // can be used to hide console window (requires STARTF_USESHOWWINDOW flag) + si.hStdInput = GetStdHandle( STD_INPUT_HANDLE ); + si.hStdOutput = GetStdHandle( STD_OUTPUT_HANDLE ); + si.hStdError = GetStdHandle( STD_ERROR_HANDLE ); + ZeroMemory( &pi, sizeof(pi) ); + + if( !CreateProcess( + NULL, // module name (uses command line if NULL) + cmdline, // command line + NULL, // process security atrributes + NULL, // thread security atrributes + TRUE, // handle inheritance + 0, // creation flags, e.g. CREATE_NEW_CONSOLE, CREATE_NO_WINDOW, DETACHED_PROCESS + NULL, // pointer to environment block (uses parent if NULL) + NULL, // starting directory (uses parent if NULL) + &si, // STARTUPINFO structure + &pi ) // PROCESS_INFORMATION structure + ) DIE( "command execution failed: %s\n", cmdline ); + + DWORD ret = 0; + CloseHandle( pi.hThread ); // thread handle is not needed + if ( WaitForSingleObject( pi.hProcess, INFINITE ) == WAIT_OBJECT_0 ) { + if ( !GetExitCodeProcess( pi.hProcess, &ret) ) + DIE( "unable to retrieve process exit code: %s\n", cmdline ); + } else DIE( "failed to wait for process termination: %s\n", cmdline ); + CloseHandle( pi.hProcess ); + + // propagate exit code from the child process + + return ret; + +} diff --git a/Master/texmf-dist/scripts/context/stubs/source/mtxrun_exe.c b/Master/texmf-dist/scripts/context/stubs/source/mtxrun_exe.c new file mode 100644 index 00000000000..0c27c272e37 --- /dev/null +++ b/Master/texmf-dist/scripts/context/stubs/source/mtxrun_exe.c @@ -0,0 +1,8 @@ +// This is the .exe part of the mtxrun program, see mtxrun_dll.c +// for more details. + +#include <windows.h> + +__declspec(dllimport) int dllrunscript( int argc, char *argv[] ); + +int main( int argc, char *argv[] ) { return dllrunscript( argc, argv ); } diff --git a/Master/texmf-dist/scripts/context/stubs/source/readme.txt b/Master/texmf-dist/scripts/context/stubs/source/readme.txt new file mode 100644 index 00000000000..354d85b0920 --- /dev/null +++ b/Master/texmf-dist/scripts/context/stubs/source/readme.txt @@ -0,0 +1,36 @@ +Copyright: + +The originally 'runscript' program was written by in 2009 by +T.M.Trzeciak and is public domain. This derived mtxrun program +is an adapted version by Hans Hagen. + +Comment: + +In ConTeXt MkIV we have two core scripts: luatools.lua and +mtxrun.lua where the second one is used to launch other scripts. +Normally a user will use a call like: + +mtxrun --script font --reload + +Here mtxrun is a lua script. In order to avoid the usage of a cmd +file on windows this runner will start texlua directly. In TeXlive +a runner is added for each cmd file but we don't want that overhead +(and extra files). By using an exe we can call these scripts in +batch files without the need for using call. + +We also don't want to use other runners, like those that use kpse +to locate the script as this is exactly what mtxrun itself is doing +already. Therefore the runscript program is adapted to a more direct +approach suitable for mtxrun. + +Compilation: + +with gcc (size optimized): + +gcc -Os -s -shared -o mtxrun.dll mtxrun_dll.c +gcc -Os -s -o mtxrun.exe mtxrun_exe.c -L./ -lmtxrun + +with tcc (ver. 0.9.24), extra small size + +tcc -shared -o runscript.dll runscript_dll.c +tcc -o runscript.exe runscript_exe.c runscript.def diff --git a/Master/texmf-dist/scripts/context/stubs/unix/ctxtools b/Master/texmf-dist/scripts/context/stubs/unix/ctxtools deleted file mode 100755 index 4658a345a8c..00000000000 --- a/Master/texmf-dist/scripts/context/stubs/unix/ctxtools +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -mtxrun --usekpse --execute ctxtools.rb "$@" diff --git a/Master/texmf-dist/scripts/context/stubs/unix/luatools b/Master/texmf-dist/scripts/context/stubs/unix/luatools index 433d1b8dc0a..1d87322c108 100644 --- a/Master/texmf-dist/scripts/context/stubs/unix/luatools +++ b/Master/texmf-dist/scripts/context/stubs/unix/luatools @@ -39,13 +39,16 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-string'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -local sub, gsub, find, match, gmatch, format, char, byte, rep = string.sub, string.gsub, string.find, string.match, string.gmatch, string.format, string.char, string.byte, string.rep +local sub, gsub, find, match, gmatch, format, char, byte, rep, lower = string.sub, string.gsub, string.find, string.match, string.gmatch, string.format, string.char, string.byte, string.rep, string.lower +local lpegmatch = lpeg.match + +-- some functions may disappear as they are not used anywhere if not string.split then @@ -85,8 +88,16 @@ function string:unquote() return (gsub(self,"^([\"\'])(.*)%1$","%2")) end +--~ function string:unquote() +--~ if find(self,"^[\'\"]") then +--~ return sub(self,2,-2) +--~ else +--~ return self +--~ end +--~ end + function string:quote() -- we could use format("%q") - return '"' .. self:unquote() .. '"' + return format("%q",self) end function string:count(pattern) -- variant 3 @@ -106,12 +117,23 @@ function string:limit(n,sentinel) end end -function string:strip() - return (gsub(self,"^%s*(.-)%s*$", "%1")) +--~ function string:strip() -- the .- is quite efficient +--~ -- return match(self,"^%s*(.-)%s*$") or "" +--~ -- return match(self,'^%s*(.*%S)') or '' -- posted on lua list +--~ return find(s,'^%s*$') and '' or match(s,'^%s*(.*%S)') +--~ end + +do -- roberto's variant: + local space = lpeg.S(" \t\v\n") + local nospace = 1 - space + local stripper = space^0 * lpeg.C((space^0 * nospace^1)^0) + function string.strip(str) + return lpegmatch(stripper,str) or "" + end end function string:is_empty() - return not find(find,"%S") + return not find(self,"%S") end function string:enhance(pattern,action) @@ -145,14 +167,14 @@ if not string.characters then local function nextchar(str, index) index = index + 1 - return (index <= #str) and index or nil, str:sub(index,index) + return (index <= #str) and index or nil, sub(str,index,index) end function string:characters() return nextchar, self, 0 end local function nextbyte(str, index) index = index + 1 - return (index <= #str) and index or nil, byte(str:sub(index,index)) + return (index <= #str) and index or nil, byte(sub(str,index,index)) end function string:bytes() return nextbyte, self, 0 @@ -165,7 +187,7 @@ end function string:rpadd(n,chr) local m = n-#self if m > 0 then - return self .. self.rep(chr or " ",m) + return self .. rep(chr or " ",m) else return self end @@ -174,7 +196,7 @@ end function string:lpadd(n,chr) local m = n-#self if m > 0 then - return self.rep(chr or " ",m) .. self + return rep(chr or " ",m) .. self else return self end @@ -222,6 +244,17 @@ function string:pattesc() return (gsub(self,".",patterns_escapes)) end +local simple_escapes = { + ["-"] = "%-", + ["."] = "%.", + ["?"] = ".", + ["*"] = ".*", +} + +function string:simpleesc() + return (gsub(self,".",simple_escapes)) +end + function string:tohash() local t = { } for s in gmatch(self,"([^, ]+)") do -- lpeg @@ -233,10 +266,10 @@ end local pattern = lpeg.Ct(lpeg.C(1)^0) function string:totable() - return pattern:match(self) + return lpegmatch(pattern,self) end ---~ for _, str in ipairs { +--~ local t = { --~ "1234567123456712345671234567", --~ "a\tb\tc", --~ "aa\tbb\tcc", @@ -244,7 +277,10 @@ end --~ "aaaa\tbbbb\tcccc", --~ "aaaaa\tbbbbb\tccccc", --~ "aaaaaa\tbbbbbb\tcccccc", ---~ } do print(string.tabtospace(str)) end +--~ } +--~ for k,v do +--~ print(string.tabtospace(t[k])) +--~ end function string.tabtospace(str,tab) -- we don't handle embedded newlines @@ -252,7 +288,7 @@ function string.tabtospace(str,tab) local s = find(str,"\t") if s then if not tab then tab = 7 end -- only when found - local d = tab-(s-1)%tab + local d = tab-(s-1) % tab if d > 0 then str = gsub(str,"\t",rep(" ",d),1) else @@ -271,6 +307,25 @@ function string:compactlong() -- strips newlines and leading spaces return self end +function string:striplong() -- strips newlines and leading spaces + self = gsub(self,"^%s*","") + self = gsub(self,"[\n\r]+ *","\n") + return self +end + +function string:topattern(lowercase,strict) + if lowercase then + self = lower(self) + end + self = gsub(self,".",simple_escapes) + if self == "" then + self = ".*" + elseif strict then + self = "^" .. self .. "$" + end + return self +end + end -- of closure @@ -278,58 +333,64 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-lpeg'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -local P, S, Ct, C, Cs, Cc = lpeg.P, lpeg.S, lpeg.Ct, lpeg.C, lpeg.Cs, lpeg.Cc - ---~ l-lpeg.lua : - ---~ lpeg.digit = lpeg.R('09')^1 ---~ lpeg.sign = lpeg.S('+-')^1 ---~ lpeg.cardinal = lpeg.P(lpeg.sign^0 * lpeg.digit^1) ---~ lpeg.integer = lpeg.P(lpeg.sign^0 * lpeg.digit^1) ---~ lpeg.float = lpeg.P(lpeg.sign^0 * lpeg.digit^0 * lpeg.P('.') * lpeg.digit^1) ---~ lpeg.number = lpeg.float + lpeg.integer ---~ lpeg.oct = lpeg.P("0") * lpeg.R('07')^1 ---~ lpeg.hex = lpeg.P("0x") * (lpeg.R('09') + lpeg.R('AF'))^1 ---~ lpeg.uppercase = lpeg.P("AZ") ---~ lpeg.lowercase = lpeg.P("az") - ---~ lpeg.eol = lpeg.S('\r\n\f')^1 -- includes formfeed ---~ lpeg.space = lpeg.S(' ')^1 ---~ lpeg.nonspace = lpeg.P(1-lpeg.space)^1 ---~ lpeg.whitespace = lpeg.S(' \r\n\f\t')^1 ---~ lpeg.nonwhitespace = lpeg.P(1-lpeg.whitespace)^1 - -local hash = { } +local lpeg = require("lpeg") + +lpeg.patterns = lpeg.patterns or { } -- so that we can share +local patterns = lpeg.patterns + +local P, R, S, Ct, C, Cs, Cc, V = lpeg.P, lpeg.R, lpeg.S, lpeg.Ct, lpeg.C, lpeg.Cs, lpeg.Cc, lpeg.V +local match = lpeg.match + +local digit, sign = R('09'), S('+-') +local cr, lf, crlf = P("\r"), P("\n"), P("\r\n") +local utf8byte = R("\128\191") + +patterns.utf8byte = utf8byte +patterns.utf8one = R("\000\127") +patterns.utf8two = R("\194\223") * utf8byte +patterns.utf8three = R("\224\239") * utf8byte * utf8byte +patterns.utf8four = R("\240\244") * utf8byte * utf8byte * utf8byte + +patterns.digit = digit +patterns.sign = sign +patterns.cardinal = sign^0 * digit^1 +patterns.integer = sign^0 * digit^1 +patterns.float = sign^0 * digit^0 * P('.') * digit^1 +patterns.number = patterns.float + patterns.integer +patterns.oct = P("0") * R("07")^1 +patterns.octal = patterns.oct +patterns.HEX = P("0x") * R("09","AF")^1 +patterns.hex = P("0x") * R("09","af")^1 +patterns.hexadecimal = P("0x") * R("09","AF","af")^1 +patterns.lowercase = R("az") +patterns.uppercase = R("AZ") +patterns.letter = patterns.lowercase + patterns.uppercase +patterns.space = S(" ") +patterns.eol = S("\n\r") +patterns.spacer = S(" \t\f\v") -- + string.char(0xc2, 0xa0) if we want utf (cf mail roberto) +patterns.newline = crlf + cr + lf +patterns.nonspace = 1 - patterns.space +patterns.nonspacer = 1 - patterns.spacer +patterns.whitespace = patterns.eol + patterns.spacer +patterns.nonwhitespace = 1 - patterns.whitespace +patterns.utf8 = patterns.utf8one + patterns.utf8two + patterns.utf8three + patterns.utf8four +patterns.utfbom = P('\000\000\254\255') + P('\255\254\000\000') + P('\255\254') + P('\254\255') + P('\239\187\191') function lpeg.anywhere(pattern) --slightly adapted from website - return P { P(pattern) + 1 * lpeg.V(1) } -end - -function lpeg.startswith(pattern) --slightly adapted - return P(pattern) + return P { P(pattern) + 1 * V(1) } -- why so complex? end function lpeg.splitter(pattern, action) return (((1-P(pattern))^1)/action+1)^0 end --- variant: - ---~ local parser = lpeg.Ct(lpeg.splitat(newline)) - -local crlf = P("\r\n") -local cr = P("\r") -local lf = P("\n") -local space = S(" \t\f\v") -- + string.char(0xc2, 0xa0) if we want utf (cf mail roberto) -local newline = crlf + cr + lf -local spacing = space^0 * newline - +local spacing = patterns.spacer^0 * patterns.newline -- sort of strip local empty = spacing * Cc("") local nonempty = Cs((1-spacing)^1) * spacing^-1 local content = (empty + nonempty)^1 @@ -337,15 +398,15 @@ local content = (empty + nonempty)^1 local capture = Ct(content^0) function string:splitlines() - return capture:match(self) + return match(capture,self) end -lpeg.linebyline = content -- better make a sublibrary +patterns.textline = content ---~ local p = lpeg.splitat("->",false) print(p:match("oeps->what->more")) -- oeps what more ---~ local p = lpeg.splitat("->",true) print(p:match("oeps->what->more")) -- oeps what->more ---~ local p = lpeg.splitat("->",false) print(p:match("oeps")) -- oeps ---~ local p = lpeg.splitat("->",true) print(p:match("oeps")) -- oeps +--~ local p = lpeg.splitat("->",false) print(match(p,"oeps->what->more")) -- oeps what more +--~ local p = lpeg.splitat("->",true) print(match(p,"oeps->what->more")) -- oeps what->more +--~ local p = lpeg.splitat("->",false) print(match(p,"oeps")) -- oeps +--~ local p = lpeg.splitat("->",true) print(match(p,"oeps")) -- oeps local splitters_s, splitters_m = { }, { } @@ -355,7 +416,7 @@ local function splitat(separator,single) separator = P(separator) if single then local other, any = C((1 - separator)^0), P(1) - splitter = other * (separator * C(any^0) + "") + splitter = other * (separator * C(any^0) + "") -- ? splitters_s[separator] = splitter else local other = C((1 - separator)^0) @@ -370,15 +431,72 @@ lpeg.splitat = splitat local cache = { } +function lpeg.split(separator,str) + local c = cache[separator] + if not c then + c = Ct(splitat(separator)) + cache[separator] = c + end + return match(c,str) +end + function string:split(separator) local c = cache[separator] if not c then c = Ct(splitat(separator)) cache[separator] = c end - return c:match(self) + return match(c,self) +end + +lpeg.splitters = cache + +local cache = { } + +function lpeg.checkedsplit(separator,str) + local c = cache[separator] + if not c then + separator = P(separator) + local other = C((1 - separator)^0) + c = Ct(separator^0 * other * (separator^1 * other)^0) + cache[separator] = c + end + return match(c,str) +end + +function string:checkedsplit(separator) + local c = cache[separator] + if not c then + separator = P(separator) + local other = C((1 - separator)^0) + c = Ct(separator^0 * other * (separator^1 * other)^0) + cache[separator] = c + end + return match(c,self) end +--~ function lpeg.append(list,pp) +--~ local p = pp +--~ for l=1,#list do +--~ if p then +--~ p = p + P(list[l]) +--~ else +--~ p = P(list[l]) +--~ end +--~ end +--~ return p +--~ end + +--~ from roberto's site: + +local f1 = string.byte + +local function f2(s) local c1, c2 = f1(s,1,2) return c1 * 64 + c2 - 12416 end +local function f3(s) local c1, c2, c3 = f1(s,1,3) return (c1 * 64 + c2) * 64 + c3 - 925824 end +local function f4(s) local c1, c2, c3, c4 = f1(s,1,4) return ((c1 * 64 + c2) * 64 + c3) * 64 + c4 - 63447168 end + +patterns.utf8byte = patterns.utf8one/f1 + patterns.utf8two/f2 + patterns.utf8three/f3 + patterns.utf8four/f4 + end -- of closure @@ -386,7 +504,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-table'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -395,9 +513,10 @@ if not modules then modules = { } end modules ['l-table'] = { table.join = table.concat local concat, sort, insert, remove = table.concat, table.sort, table.insert, table.remove -local format, find, gsub, lower, dump = string.format, string.find, string.gsub, string.lower, string.dump +local format, find, gsub, lower, dump, match = string.format, string.find, string.gsub, string.lower, string.dump, string.match local getmetatable, setmetatable = getmetatable, setmetatable -local type, next, tostring, ipairs = type, next, tostring, ipairs +local type, next, tostring, tonumber, ipairs = type, next, tostring, tonumber, ipairs +local unpack = unpack or table.unpack function table.strip(tab) local lst = { } @@ -412,6 +531,14 @@ function table.strip(tab) return lst end +function table.keys(t) + local k = { } + for key, _ in next, t do + k[#k+1] = key + end + return k +end + local function compare(a,b) return (tostring(a) < tostring(b)) end @@ -455,7 +582,7 @@ end table.sortedkeys = sortedkeys table.sortedhashkeys = sortedhashkeys -function table.sortedpairs(t) +function table.sortedhash(t) local s = sortedhashkeys(t) -- maybe just sortedkeys local n = 0 local function kv(s) @@ -466,6 +593,8 @@ function table.sortedpairs(t) return kv, s end +table.sortedpairs = table.sortedhash + function table.append(t, list) for _,v in next, list do insert(t,v) @@ -588,18 +717,18 @@ end -- slower than #t on indexed tables (#t only returns the size of the numerically indexed slice) -function table.is_empty(t) +function table.is_empty(t) -- obolete, use inline code instead return not t or not next(t) end -function table.one_entry(t) +function table.one_entry(t) -- obolete, use inline code instead local n = next(t) return n and not next(t,n) end -function table.starts_at(t) - return ipairs(t,1)(t,0) -end +--~ function table.starts_at(t) -- obsolete, not nice +--~ return ipairs(t,1)(t,0) +--~ end function table.tohash(t,value) local h = { } @@ -677,6 +806,8 @@ end -- -- local propername = lpeg.P(lpeg.R("AZ","az","__") * lpeg.R("09","AZ","az", "__")^0 * lpeg.P(-1) ) +-- problem: there no good number_to_string converter with the best resolution + local function do_serialize(root,name,depth,level,indexed) if level > 0 then depth = depth .. " " @@ -699,8 +830,9 @@ local function do_serialize(root,name,depth,level,indexed) handle(format("%s{",depth)) end end + -- we could check for k (index) being number (cardinal) if root and next(root) then - local first, last = nil, 0 -- #root cannot be trusted here + local first, last = nil, 0 -- #root cannot be trusted here (will be ok in 5.2 when ipairs is gone) if compact then -- NOT: for k=1,#root do (we need to quit at nil) for k,v in ipairs(root) do -- can we use next? @@ -721,10 +853,10 @@ local function do_serialize(root,name,depth,level,indexed) if hexify then handle(format("%s 0x%04X,",depth,v)) else - handle(format("%s %s,",depth,v)) + handle(format("%s %s,",depth,v)) -- %.99g end elseif t == "string" then - if reduce and (find(v,"^[%-%+]?[%d]-%.?[%d+]$") == 1) then + if reduce and tonumber(v) then handle(format("%s %s,",depth,v)) else handle(format("%s %q,",depth,v)) @@ -761,29 +893,29 @@ local function do_serialize(root,name,depth,level,indexed) --~ if hexify then --~ handle(format("%s %s=0x%04X,",depth,key(k),v)) --~ else - --~ handle(format("%s %s=%s,",depth,key(k),v)) + --~ handle(format("%s %s=%s,",depth,key(k),v)) -- %.99g --~ end if type(k) == "number" then -- or find(k,"^%d+$") then if hexify then handle(format("%s [0x%04X]=0x%04X,",depth,k,v)) else - handle(format("%s [%s]=%s,",depth,k,v)) + handle(format("%s [%s]=%s,",depth,k,v)) -- %.99g end elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then if hexify then handle(format("%s %s=0x%04X,",depth,k,v)) else - handle(format("%s %s=%s,",depth,k,v)) + handle(format("%s %s=%s,",depth,k,v)) -- %.99g end else if hexify then handle(format("%s [%q]=0x%04X,",depth,k,v)) else - handle(format("%s [%q]=%s,",depth,k,v)) + handle(format("%s [%q]=%s,",depth,k,v)) -- %.99g end end elseif t == "string" then - if reduce and (find(v,"^[%-%+]?[%d]-%.?[%d+]$") == 1) then + if reduce and tonumber(v) then --~ handle(format("%s %s=%s,",depth,key(k),v)) if type(k) == "number" then -- or find(k,"^%d+$") then if hexify then @@ -992,7 +1124,7 @@ function table.tofile(filename,root,name,reduce,noquotes,hexify) end end -local function flatten(t,f,complete) +local function flatten(t,f,complete) -- is this used? meybe a variant with next, ... for i=1,#t do local v = t[i] if type(v) == "table" then @@ -1021,6 +1153,24 @@ end table.flatten_one_level = table.unnest +-- a better one: + +local function flattened(t,f) + if not f then + f = { } + end + for k, v in next, t do + if type(v) == "table" then + flattened(v,f) + else + f[k] = v + end + end + return f +end + +table.flattened = flattened + -- the next three may disappear function table.remove_value(t,value) -- todo: n @@ -1156,7 +1306,7 @@ function table.clone(t,p) -- t is optional or nil or table elseif not t then t = { } end - setmetatable(t, { __index = function(_,key) return p[key] end }) + setmetatable(t, { __index = function(_,key) return p[key] end }) -- why not __index = p ? return t end @@ -1184,21 +1334,35 @@ function table.reverse(t) return tt end ---~ function table.keys(t) ---~ local k = { } ---~ for k,_ in next, t do ---~ k[#k+1] = k ---~ end ---~ return k ---~ end +function table.insert_before_value(t,value,extra) + for i=1,#t do + if t[i] == extra then + remove(t,i) + end + end + for i=1,#t do + if t[i] == value then + insert(t,i,extra) + return + end + end + insert(t,1,extra) +end ---~ function table.keys_as_string(t) ---~ local k = { } ---~ for k,_ in next, t do ---~ k[#k+1] = k ---~ end ---~ return concat(k,"") ---~ end +function table.insert_after_value(t,value,extra) + for i=1,#t do + if t[i] == extra then + remove(t,i) + end + end + for i=1,#t do + if t[i] == value then + insert(t,i+1,extra) + return + end + end + insert(t,#t+1,extra) +end end -- of closure @@ -1207,13 +1371,13 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-io'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -local byte = string.byte +local byte, find, gsub = string.byte, string.find, string.gsub if string.find(os.getenv("PATH"),";") then io.fileseparator, io.pathseparator = "\\", ";" @@ -1242,7 +1406,7 @@ function io.savedata(filename,data,joiner) elseif type(data) == "function" then data(f) else - f:write(data) + f:write(data or "") end f:close() return true @@ -1371,20 +1535,21 @@ function io.ask(question,default,options) end io.write(string.format(" ")) local answer = io.read() - answer = answer:gsub("^%s*(.*)%s*$","%1") + answer = gsub(answer,"^%s*(.*)%s*$","%1") if answer == "" and default then return default elseif not options then return answer else - for _,v in pairs(options) do - if v == answer then + for k=1,#options do + if options[k] == answer then return answer end end local pattern = "^" .. answer - for _,v in pairs(options) do - if v:find(pattern) then + for k=1,#options do + local v = options[k] + if find(v,pattern) then return v end end @@ -1399,20 +1564,22 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-number'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -local format = string.format +local tostring = tostring +local format, floor, insert, match = string.format, math.floor, table.insert, string.match +local lpegmatch = lpeg.match number = number or { } -- a,b,c,d,e,f = number.toset(100101) function number.toset(n) - return (tostring(n)):match("(.?)(.?)(.?)(.?)(.?)(.?)(.?)(.?)") + return match(tostring(n),"(.?)(.?)(.?)(.?)(.?)(.?)(.?)(.?)") end function number.toevenhex(n) @@ -1438,10 +1605,21 @@ end local one = lpeg.C(1-lpeg.S(''))^1 function number.toset(n) - return one:match(tostring(n)) + return lpegmatch(one,tostring(n)) end - +function number.bits(n,zero) + local t, i = { }, (zero and 0) or 1 + while n > 0 do + local m = n % 2 + if m > 0 then + insert(t,1,i) + end + n = floor(n/2) + i = i + 1 + end + return t +end end -- of closure @@ -1450,7 +1628,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-set'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -1540,46 +1718,63 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-os'] = { version = 1.001, - comment = "companion to luat-lub.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -local find = string.find +-- maybe build io.flush in os.execute + +local find, format, gsub = string.find, string.format, string.gsub +local random, ceil = math.random, math.ceil + +local execute, spawn, exec, ioflush = os.execute, os.spawn or os.execute, os.exec or os.execute, io.flush + +function os.execute(...) ioflush() return execute(...) end +function os.spawn (...) ioflush() return spawn (...) end +function os.exec (...) ioflush() return exec (...) end function os.resultof(command) - return io.popen(command,"r"):read("*all") + ioflush() -- else messed up logging + local handle = io.popen(command,"r") + if not handle then + -- print("unknown command '".. command .. "' in os.resultof") + return "" + else + return handle:read("*all") or "" + end end -if not os.exec then os.exec = os.execute end -if not os.spawn then os.spawn = os.execute end - ---~ os.type : windows | unix (new, we already guessed os.platform) ---~ os.name : windows | msdos | linux | macosx | solaris | .. | generic (new) +--~ os.type : windows | unix (new, we already guessed os.platform) +--~ os.name : windows | msdos | linux | macosx | solaris | .. | generic (new) +--~ os.platform : extended os.name with architecture if not io.fileseparator then if find(os.getenv("PATH"),";") then - io.fileseparator, io.pathseparator, os.platform = "\\", ";", os.type or "windows" + io.fileseparator, io.pathseparator, os.type = "\\", ";", os.type or "mswin" else - io.fileseparator, io.pathseparator, os.platform = "/" , ":", os.type or "unix" + io.fileseparator, io.pathseparator, os.type = "/" , ":", os.type or "unix" end end -os.platform = os.platform or os.type or (io.pathseparator == ";" and "windows") or "unix" +os.type = os.type or (io.pathseparator == ";" and "windows") or "unix" +os.name = os.name or (os.type == "windows" and "mswin" ) or "linux" + +if os.type == "windows" then + os.libsuffix, os.binsuffix = 'dll', 'exe' +else + os.libsuffix, os.binsuffix = 'so', '' +end function os.launch(str) - if os.platform == "windows" then + if os.type == "windows" then os.execute("start " .. str) -- os.spawn ? else os.execute(str .. " &") -- os.spawn ? end end -if not os.setenv then - function os.setenv() return false end -end - if not os.times then -- utime = user time -- stime = system time @@ -1609,64 +1804,218 @@ end --~ print(os.date("%H:%M:%S",os.gettimeofday())) --~ print(os.date("%H:%M:%S",os.time())) -os.arch = os.arch or function() - local a = os.resultof("uname -m") or "linux" - os.arch = function() - return a +-- no need for function anymore as we have more clever code and helpers now +-- this metatable trickery might as well disappear + +os.resolvers = os.resolvers or { } + +local resolvers = os.resolvers + +local osmt = getmetatable(os) or { __index = function(t,k) t[k] = "unset" return "unset" end } -- maybe nil +local osix = osmt.__index + +osmt.__index = function(t,k) + return (resolvers[k] or osix)(t,k) +end + +setmetatable(os,osmt) + +if not os.setenv then + + -- we still store them but they won't be seen in + -- child processes although we might pass them some day + -- using command concatination + + local env, getenv = { }, os.getenv + + function os.setenv(k,v) + env[k] = v end - return a + + function os.getenv(k) + return env[k] or getenv(k) + end + end -local platform +-- we can use HOSTTYPE on some platforms -function os.currentplatform(name,default) - if not platform then - local name = os.name or os.platform or name -- os.name is built in, os.platform is mine - if not name then - platform = default or "linux" - elseif name == "windows" or name == "mswin" or name == "win32" or name == "msdos" then - if os.getenv("PROCESSOR_ARCHITECTURE") == "AMD64" then - platform = "mswin-64" - else - platform = "mswin" - end +local name, platform = os.name or "linux", os.getenv("MTX_PLATFORM") or "" + +local function guess() + local architecture = os.resultof("uname -m") or "" + if architecture ~= "" then + return architecture + end + architecture = os.getenv("HOSTTYPE") or "" + if architecture ~= "" then + return architecture + end + return os.resultof("echo $HOSTTYPE") or "" +end + +if platform ~= "" then + + os.platform = platform + +elseif os.type == "windows" then + + -- we could set the variable directly, no function needed here + + function os.resolvers.platform(t,k) + local platform, architecture = "", os.getenv("PROCESSOR_ARCHITECTURE") or "" + if find(architecture,"AMD64") then + platform = "mswin-64" else - local architecture = os.arch() - if name == "linux" then - if find(architecture,"x86_64") then - platform = "linux-64" - elseif find(architecture,"ppc") then - platform = "linux-ppc" - else - platform = "linux" - end - elseif name == "macosx" then - if find(architecture,"i386") then - platform = "osx-intel" - else - platform = "osx-ppc" - end - elseif name == "sunos" then - if find(architecture,"sparc") then - platform = "solaris-sparc" - else -- if architecture == 'i86pc' - platform = "solaris-intel" - end - elseif name == "freebsd" then - if find(architecture,"amd64") then - platform = "freebsd-amd64" - else - platform = "freebsd" - end - else - platform = default or name - end + platform = "mswin" end - function os.currentplatform() - return platform + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +elseif name == "linux" then + + function os.resolvers.platform(t,k) + -- we sometims have HOSTTYPE set so let's check that first + local platform, architecture = "", os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" + if find(architecture,"x86_64") then + platform = "linux-64" + elseif find(architecture,"ppc") then + platform = "linux-ppc" + else + platform = "linux" + end + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +elseif name == "macosx" then + + --[[ + Identifying the architecture of OSX is quite a mess and this + is the best we can come up with. For some reason $HOSTTYPE is + a kind of pseudo environment variable, not known to the current + environment. And yes, uname cannot be trusted either, so there + is a change that you end up with a 32 bit run on a 64 bit system. + Also, some proper 64 bit intel macs are too cheap (low-end) and + therefore not permitted to run the 64 bit kernel. + ]]-- + + function os.resolvers.platform(t,k) + -- local platform, architecture = "", os.getenv("HOSTTYPE") or "" + -- if architecture == "" then + -- architecture = os.resultof("echo $HOSTTYPE") or "" + -- end + local platform, architecture = "", os.resultof("echo $HOSTTYPE") or "" + if architecture == "" then + -- print("\nI have no clue what kind of OSX you're running so let's assume an 32 bit intel.\n") + platform = "osx-intel" + elseif find(architecture,"i386") then + platform = "osx-intel" + elseif find(architecture,"x86_64") then + platform = "osx-64" + else + platform = "osx-ppc" + end + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +elseif name == "sunos" then + + function os.resolvers.platform(t,k) + local platform, architecture = "", os.resultof("uname -m") or "" + if find(architecture,"sparc") then + platform = "solaris-sparc" + else -- if architecture == 'i86pc' + platform = "solaris-intel" + end + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +elseif name == "freebsd" then + + function os.resolvers.platform(t,k) + local platform, architecture = "", os.resultof("uname -m") or "" + if find(architecture,"amd64") then + platform = "freebsd-amd64" + else + platform = "freebsd" + end + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +elseif name == "kfreebsd" then + + function os.resolvers.platform(t,k) + -- we sometims have HOSTTYPE set so let's check that first + local platform, architecture = "", os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" + if find(architecture,"x86_64") then + platform = "kfreebsd-64" + else + platform = "kfreebsd-i386" + end + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +else + + -- platform = "linux" + -- os.setenv("MTX_PLATFORM",platform) + -- os.platform = platform + + function os.resolvers.platform(t,k) + local platform = "linux" + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +end + +-- beware, we set the randomseed + +-- from wikipedia: Version 4 UUIDs use a scheme relying only on random numbers. This algorithm sets the +-- version number as well as two reserved bits. All other bits are set using a random or pseudorandom +-- data source. Version 4 UUIDs have the form xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx with hexadecimal +-- digits x and hexadecimal digits 8, 9, A, or B for y. e.g. f47ac10b-58cc-4372-a567-0e02b2c3d479. +-- +-- as we don't call this function too often there is not so much risk on repetition + +local t = { 8, 9, "a", "b" } + +function os.uuid() + return format("%04x%04x-4%03x-%s%03x-%04x-%04x%04x%04x", + random(0xFFFF),random(0xFFFF), + random(0x0FFF), + t[ceil(random(4))] or 8,random(0x0FFF), + random(0xFFFF), + random(0xFFFF),random(0xFFFF),random(0xFFFF) + ) +end + +local d + +function os.timezone(delta) + d = d or tonumber(tonumber(os.date("%H")-os.date("!%H"))) + if delta then + if d > 0 then + return format("+%02i:00",d) + else + return format("-%02i:00",-d) end + else + return 1 end - return platform end @@ -1676,7 +2025,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-file'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -1687,14 +2036,17 @@ if not modules then modules = { } end modules ['l-file'] = { file = file or { } local concat = table.concat -local find, gmatch, match, gsub = string.find, string.gmatch, string.match, string.gsub +local find, gmatch, match, gsub, sub, char = string.find, string.gmatch, string.match, string.gsub, string.sub, string.char +local lpegmatch = lpeg.match function file.removesuffix(filename) return (gsub(filename,"%.[%a%d]+$","")) end function file.addsuffix(filename, suffix) - if not find(filename,"%.[%a%d]+$") then + if not suffix or suffix == "" then + return filename + elseif not find(filename,"%.[%a%d]+$") then return filename .. "." .. suffix else return filename @@ -1717,20 +2069,39 @@ function file.nameonly(name) return (gsub(match(name,"^.+[/\\](.-)$") or name,"%..*$","")) end -function file.extname(name) - return match(name,"^.+%.([^/\\]-)$") or "" +function file.extname(name,default) + return match(name,"^.+%.([^/\\]-)$") or default or "" end file.suffix = file.extname ---~ print(file.join("x/","/y")) ---~ print(file.join("http://","/y")) ---~ print(file.join("http://a","/y")) ---~ print(file.join("http:///a","/y")) ---~ print(file.join("//nas-1","/y")) +--~ function file.join(...) +--~ local pth = concat({...},"/") +--~ pth = gsub(pth,"\\","/") +--~ local a, b = match(pth,"^(.*://)(.*)$") +--~ if a and b then +--~ return a .. gsub(b,"//+","/") +--~ end +--~ a, b = match(pth,"^(//)(.*)$") +--~ if a and b then +--~ return a .. gsub(b,"//+","/") +--~ end +--~ return (gsub(pth,"//+","/")) +--~ end + +local trick_1 = char(1) +local trick_2 = "^" .. trick_1 .. "/+" function file.join(...) - local pth = concat({...},"/") + local lst = { ... } + local a, b = lst[1], lst[2] + if a == "" then + lst[1] = trick_1 + elseif b and find(a,"^/+$") and find(b,"^/") then + lst[1] = "" + lst[2] = gsub(b,"^/+","") + end + local pth = concat(lst,"/") pth = gsub(pth,"\\","/") local a, b = match(pth,"^(.*://)(.*)$") if a and b then @@ -1740,17 +2111,28 @@ function file.join(...) if a and b then return a .. gsub(b,"//+","/") end + pth = gsub(pth,trick_2,"") return (gsub(pth,"//+","/")) end +--~ print(file.join("//","/y")) +--~ print(file.join("/","/y")) +--~ print(file.join("","/y")) +--~ print(file.join("/x/","/y")) +--~ print(file.join("x/","/y")) +--~ print(file.join("http://","/y")) +--~ print(file.join("http://a","/y")) +--~ print(file.join("http:///a","/y")) +--~ print(file.join("//nas-1","/y")) + function file.iswritable(name) local a = lfs.attributes(name) or lfs.attributes(file.dirname(name,".")) - return a and a.permissions:sub(2,2) == "w" + return a and sub(a.permissions,2,2) == "w" end function file.isreadable(name) local a = lfs.attributes(name) - return a and a.permissions:sub(1,1) == "r" + return a and sub(a.permissions,1,1) == "r" end file.is_readable = file.isreadable @@ -1758,36 +2140,50 @@ file.is_writable = file.iswritable -- todo: lpeg -function file.split_path(str) - local t = { } - str = gsub(str,"\\", "/") - str = gsub(str,"(%a):([;/])", "%1\001%2") - for name in gmatch(str,"([^;:]+)") do - if name ~= "" then - t[#t+1] = gsub(name,"\001",":") - end - end - return t +--~ function file.split_path(str) +--~ local t = { } +--~ str = gsub(str,"\\", "/") +--~ str = gsub(str,"(%a):([;/])", "%1\001%2") +--~ for name in gmatch(str,"([^;:]+)") do +--~ if name ~= "" then +--~ t[#t+1] = gsub(name,"\001",":") +--~ end +--~ end +--~ return t +--~ end + +local checkedsplit = string.checkedsplit + +function file.split_path(str,separator) + str = gsub(str,"\\","/") + return checkedsplit(str,separator or io.pathseparator) end function file.join_path(tab) return concat(tab,io.pathseparator) -- can have trailing // end +-- we can hash them weakly + function file.collapse_path(str) - str = gsub(str,"/%./","/") - local n, m = 1, 1 - while n > 0 or m > 0 do - str, n = gsub(str,"[^/%.]+/%.%.$","") - str, m = gsub(str,"[^/%.]+/%.%./","") - end - str = gsub(str,"([^/])/$","%1") - str = gsub(str,"^%./","") - str = gsub(str,"/%.$","") + str = gsub(str,"\\","/") + if find(str,"/") then + str = gsub(str,"^%./",(gsub(lfs.currentdir(),"\\","/")) .. "/") -- ./xx in qualified + str = gsub(str,"/%./","/") + local n, m = 1, 1 + while n > 0 or m > 0 do + str, n = gsub(str,"[^/%.]+/%.%.$","") + str, m = gsub(str,"[^/%.]+/%.%./","") + end + str = gsub(str,"([^/])/$","%1") + -- str = gsub(str,"^%./","") -- ./xx in qualified + str = gsub(str,"/%.$","") + end if str == "" then str = "." end return str end +--~ print(file.collapse_path("/a")) --~ print(file.collapse_path("a/./b/..")) --~ print(file.collapse_path("a/aa/../b/bb")) --~ print(file.collapse_path("a/../..")) @@ -1817,27 +2213,27 @@ end --~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * lpeg.C(noperiod^1) * -1 --~ function file.extname(name) ---~ return pattern:match(name) or "" +--~ return lpegmatch(pattern,name) or "" --~ end --~ local pattern = lpeg.Cs(((period * noperiod^1 * -1)/"" + 1)^1) --~ function file.removesuffix(name) ---~ return pattern:match(name) +--~ return lpegmatch(pattern,name) --~ end --~ local pattern = (noslashes^0 * slashes)^1 * lpeg.C(noslashes^1) * -1 --~ function file.basename(name) ---~ return pattern:match(name) or name +--~ return lpegmatch(pattern,name) or name --~ end --~ local pattern = (noslashes^0 * slashes)^1 * lpeg.Cp() * noslashes^1 * -1 --~ function file.dirname(name) ---~ local p = pattern:match(name) +--~ local p = lpegmatch(pattern,name) --~ if p then ---~ return name:sub(1,p-2) +--~ return sub(name,1,p-2) --~ else --~ return "" --~ end @@ -1846,7 +2242,7 @@ end --~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * lpeg.Cp() * noperiod^1 * -1 --~ function file.addsuffix(name, suffix) ---~ local p = pattern:match(name) +--~ local p = lpegmatch(pattern,name) --~ if p then --~ return name --~ else @@ -1857,9 +2253,9 @@ end --~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * lpeg.Cp() * noperiod^1 * -1 --~ function file.replacesuffix(name,suffix) ---~ local p = pattern:match(name) +--~ local p = lpegmatch(pattern,name) --~ if p then ---~ return name:sub(1,p-2) .. "." .. suffix +--~ return sub(name,1,p-2) .. "." .. suffix --~ else --~ return name .. "." .. suffix --~ end @@ -1868,11 +2264,11 @@ end --~ local pattern = (noslashes^0 * slashes)^0 * lpeg.Cp() * ((noperiod^1 * period)^1 * lpeg.Cp() + lpeg.P(true)) * noperiod^1 * -1 --~ function file.nameonly(name) ---~ local a, b = pattern:match(name) +--~ local a, b = lpegmatch(pattern,name) --~ if b then ---~ return name:sub(a,b-2) +--~ return sub(name,a,b-2) --~ elseif a then ---~ return name:sub(a) +--~ return sub(name,a) --~ else --~ return name --~ end @@ -1906,11 +2302,11 @@ local rootbased = lpeg.P("/") + letter*lpeg.P(":") -- ./name ../name /name c: :// name/name function file.is_qualified_path(filename) - return qualified:match(filename) + return lpegmatch(qualified,filename) ~= nil end function file.is_rootbased_path(filename) - return rootbased:match(filename) + return lpegmatch(rootbased,filename) ~= nil end local slash = lpeg.S("\\/") @@ -1923,16 +2319,25 @@ local base = lpeg.C((1-suffix)^0) local pattern = (drive + lpeg.Cc("")) * (path + lpeg.Cc("")) * (base + lpeg.Cc("")) * (suffix + lpeg.Cc("")) function file.splitname(str) -- returns drive, path, base, suffix - return pattern:match(str) + return lpegmatch(pattern,str) end --- function test(t) for k, v in pairs(t) do print(v, "=>", file.splitname(v)) end end +-- function test(t) for k, v in next, t do print(v, "=>", file.splitname(v)) end end -- -- test { "c:", "c:/aa", "c:/aa/bb", "c:/aa/bb/cc", "c:/aa/bb/cc.dd", "c:/aa/bb/cc.dd.ee" } -- test { "c:", "c:aa", "c:aa/bb", "c:aa/bb/cc", "c:aa/bb/cc.dd", "c:aa/bb/cc.dd.ee" } -- test { "/aa", "/aa/bb", "/aa/bb/cc", "/aa/bb/cc.dd", "/aa/bb/cc.dd.ee" } -- test { "aa", "aa/bb", "aa/bb/cc", "aa/bb/cc.dd", "aa/bb/cc.dd.ee" } +--~ -- todo: +--~ +--~ if os.type == "windows" then +--~ local currentdir = lfs.currentdir +--~ function lfs.currentdir() +--~ return (gsub(currentdir(),"\\","/")) +--~ end +--~ end + end -- of closure @@ -1997,7 +2402,7 @@ end function file.loadchecksum(name) if md5 then local data = io.loaddata(name .. ".md5") - return data and data:gsub("%s","") + return data and (gsub(data,"%s","")) end return nil end @@ -2018,14 +2423,15 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-url'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -local char, gmatch = string.char, string.gmatch +local char, gmatch, gsub = string.char, string.gmatch, string.gsub local tonumber, type = tonumber, type +local lpegmatch = lpeg.match -- from the spec (on the web): -- @@ -2049,7 +2455,9 @@ local hexdigit = lpeg.R("09","AF","af") local plus = lpeg.P("+") local escaped = (plus / " ") + (percent * lpeg.C(hexdigit * hexdigit) / tochar) -local scheme = lpeg.Cs((escaped+(1-colon-slash-qmark-hash))^0) * colon + lpeg.Cc("") +-- we assume schemes with more than 1 character (in order to avoid problems with windows disks) + +local scheme = lpeg.Cs((escaped+(1-colon-slash-qmark-hash))^2) * colon + lpeg.Cc("") local authority = slash * slash * lpeg.Cs((escaped+(1- slash-qmark-hash))^0) + lpeg.Cc("") local path = slash * lpeg.Cs((escaped+(1- qmark-hash))^0) + lpeg.Cc("") local query = qmark * lpeg.Cs((escaped+(1- hash))^0) + lpeg.Cc("") @@ -2057,25 +2465,51 @@ local fragment = hash * lpeg.Cs((escaped+(1- endofstring))^0 local parser = lpeg.Ct(scheme * authority * path * query * fragment) +-- todo: reconsider Ct as we can as well have five return values (saves a table) +-- so we can have two parsers, one with and one without + function url.split(str) - return (type(str) == "string" and parser:match(str)) or str + return (type(str) == "string" and lpegmatch(parser,str)) or str end +-- todo: cache them + function url.hashed(str) local s = url.split(str) + local somescheme = s[1] ~= "" return { - scheme = (s[1] ~= "" and s[1]) or "file", + scheme = (somescheme and s[1]) or "file", authority = s[2], - path = s[3], - query = s[4], - fragment = s[5], - original = str + path = s[3], + query = s[4], + fragment = s[5], + original = str, + noscheme = not somescheme, } end +function url.hasscheme(str) + return url.split(str)[1] ~= "" +end + +function url.addscheme(str,scheme) + return (url.hasscheme(str) and str) or ((scheme or "file:///") .. str) +end + +function url.construct(hash) + local fullurl = hash.sheme .. "://".. hash.authority .. hash.path + if hash.query then + fullurl = fullurl .. "?".. hash.query + end + if hash.fragment then + fullurl = fullurl .. "?".. hash.fragment + end + return fullurl +end + function url.filename(filename) local t = url.hashed(filename) - return (t.scheme == "file" and t.path:gsub("^/([a-zA-Z])([:|])/)","%1:")) or filename + return (t.scheme == "file" and (gsub(t.path,"^/([a-zA-Z])([:|])/)","%1:"))) or filename end function url.query(str) @@ -2129,17 +2563,26 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-dir'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } +-- dir.expand_name will be merged with cleanpath and collapsepath + local type = type -local find, gmatch = string.find, string.gmatch +local find, gmatch, match, gsub = string.find, string.gmatch, string.match, string.gsub +local lpegmatch = lpeg.match dir = dir or { } +-- handy + +function dir.current() + return (gsub(lfs.currentdir(),"\\","/")) +end + -- optimizing for no string.find (*) does not save time local attributes = lfs.attributes @@ -2170,6 +2613,35 @@ end dir.glob_pattern = glob_pattern +local function collect_pattern(path,patt,recurse,result) + local ok, scanner + result = result or { } + if path == "/" then + ok, scanner = xpcall(function() return walkdir(path..".") end, function() end) -- kepler safe + else + ok, scanner = xpcall(function() return walkdir(path) end, function() end) -- kepler safe + end + if ok and type(scanner) == "function" then + if not find(path,"/$") then path = path .. '/' end + for name in scanner do + local full = path .. name + local attr = attributes(full) + local mode = attr.mode + if mode == 'file' then + if find(full,patt) then + result[name] = attr + end + elseif recurse and (mode == "directory") and (name ~= '.') and (name ~= "..") then + attr.list = collect_pattern(full,patt,recurse) + result[name] = attr + end + end + end + return result +end + +dir.collect_pattern = collect_pattern + local P, S, R, C, Cc, Cs, Ct, Cv, V = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.Cc, lpeg.Cs, lpeg.Ct, lpeg.Cv, lpeg.V local pattern = Ct { @@ -2189,29 +2661,48 @@ local filter = Cs ( ( )^0 ) local function glob(str,t) - if type(str) == "table" then - local t = t or { } - for s=1,#str do - glob(str[s],t) + if type(t) == "function" then + if type(str) == "table" then + for s=1,#str do + glob(str[s],t) + end + elseif lfs.isfile(str) then + t(str) + else + local split = lpegmatch(pattern,str) + if split then + local root, path, base = split[1], split[2], split[3] + local recurse = find(base,"%*%*") + local start = root .. path + local result = lpegmatch(filter,start .. base) + glob_pattern(start,result,recurse,t) + end end - return t - elseif lfs.isfile(str) then - local t = t or { } - t[#t+1] = str - return t else - local split = pattern:match(str) - if split then + if type(str) == "table" then local t = t or { } - local action = action or function(name) t[#t+1] = name end - local root, path, base = split[1], split[2], split[3] - local recurse = find(base,"%*%*") - local start = root .. path - local result = filter:match(start .. base) - glob_pattern(start,result,recurse,action) + for s=1,#str do + glob(str[s],t) + end + return t + elseif lfs.isfile(str) then + local t = t or { } + t[#t+1] = str return t else - return { } + local split = lpegmatch(pattern,str) + if split then + local t = t or { } + local action = action or function(name) t[#t+1] = name end + local root, path, base = split[1], split[2], split[3] + local recurse = find(base,"%*%*") + local start = root .. path + local result = lpegmatch(filter,start .. base) + glob_pattern(start,result,recurse,action) + return t + else + return { } + end end end end @@ -2273,11 +2764,12 @@ end local make_indeed = true -- false -if string.find(os.getenv("PATH"),";") then +if string.find(os.getenv("PATH"),";") then -- os.type == "windows" function dir.mkdirs(...) - local str, pth = "", "" - for _, s in ipairs({...}) do + local str, pth, t = "", "", { ... } + for i=1,#t do + local s = t[i] if s ~= "" then if str ~= "" then str = str .. "/" .. s @@ -2288,13 +2780,13 @@ if string.find(os.getenv("PATH"),";") then end local first, middle, last local drive = false - first, middle, last = str:match("^(//)(//*)(.*)$") + first, middle, last = match(str,"^(//)(//*)(.*)$") if first then -- empty network path == local path else - first, last = str:match("^(//)/*(.-)$") + first, last = match(str,"^(//)/*(.-)$") if first then - middle, last = str:match("([^/]+)/+(.-)$") + middle, last = match(str,"([^/]+)/+(.-)$") if middle then pth = "//" .. middle else @@ -2302,11 +2794,11 @@ if string.find(os.getenv("PATH"),";") then last = "" end else - first, middle, last = str:match("^([a-zA-Z]:)(/*)(.-)$") + first, middle, last = match(str,"^([a-zA-Z]:)(/*)(.-)$") if first then pth, drive = first .. middle, true else - middle, last = str:match("^(/*)(.-)$") + middle, last = match(str,"^(/*)(.-)$") if not middle then last = str end @@ -2340,34 +2832,31 @@ if string.find(os.getenv("PATH"),";") then --~ print(dir.mkdirs("///a/b/c")) --~ print(dir.mkdirs("a/bbb//ccc/")) - function dir.expand_name(str) - local first, nothing, last = str:match("^(//)(//*)(.*)$") + function dir.expand_name(str) -- will be merged with cleanpath and collapsepath + local first, nothing, last = match(str,"^(//)(//*)(.*)$") if first then - first = lfs.currentdir() .. "/" - first = first:gsub("\\","/") + first = dir.current() .. "/" end if not first then - first, last = str:match("^(//)/*(.*)$") + first, last = match(str,"^(//)/*(.*)$") end if not first then - first, last = str:match("^([a-zA-Z]:)(.*)$") + first, last = match(str,"^([a-zA-Z]:)(.*)$") if first and not find(last,"^/") then local d = lfs.currentdir() if lfs.chdir(first) then - first = lfs.currentdir() - first = first:gsub("\\","/") + first = dir.current() end lfs.chdir(d) end end if not first then - first, last = lfs.currentdir(), str - first = first:gsub("\\","/") + first, last = dir.current(), str end - last = last:gsub("//","/") - last = last:gsub("/%./","/") - last = last:gsub("^/*","") - first = first:gsub("/*$","") + last = gsub(last,"//","/") + last = gsub(last,"/%./","/") + last = gsub(last,"^/*","") + first = gsub(first,"/*$","") if last == "" then return first else @@ -2378,8 +2867,9 @@ if string.find(os.getenv("PATH"),";") then else function dir.mkdirs(...) - local str, pth = "", "" - for _, s in ipairs({...}) do + local str, pth, t = "", "", { ... } + for i=1,#t do + local s = t[i] if s ~= "" then if str ~= "" then str = str .. "/" .. s @@ -2388,7 +2878,7 @@ else end end end - str = str:gsub("/+","/") + str = gsub(str,"/+","/") if find(str,"^/") then pth = "/" for s in gmatch(str,"[^/]+") do @@ -2422,12 +2912,12 @@ else --~ print(dir.mkdirs("///a/b/c")) --~ print(dir.mkdirs("a/bbb//ccc/")) - function dir.expand_name(str) + function dir.expand_name(str) -- will be merged with cleanpath and collapsepath if not find(str,"^/") then str = lfs.currentdir() .. "/" .. str end - str = str:gsub("//","/") - str = str:gsub("/%./","/") + str = gsub(str,"//","/") + str = gsub(str,"/%./","/") return str end @@ -2442,7 +2932,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-boolean'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -2503,19 +2993,40 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-unicode'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } +if not unicode then + + unicode = { utf8 = { } } + + local floor, char = math.floor, string.char + + function unicode.utf8.utfchar(n) + if n < 0x80 then + return char(n) + elseif n < 0x800 then + return char(0xC0 + floor(n/0x40)) .. char(0x80 + (n % 0x40)) + elseif n < 0x10000 then + return char(0xE0 + floor(n/0x1000)) .. char(0x80 + (floor(n/0x40) % 0x40)) .. char(0x80 + (n % 0x40)) + elseif n < 0x40000 then + return char(0xF0 + floor(n/0x40000)) .. char(0x80 + floor(n/0x1000)) .. char(0x80 + (floor(n/0x40) % 0x40)) .. char(0x80 + (n % 0x40)) + else -- wrong: + -- return char(0xF1 + floor(n/0x1000000)) .. char(0x80 + floor(n/0x40000)) .. char(0x80 + floor(n/0x1000)) .. char(0x80 + (floor(n/0x40) % 0x40)) .. char(0x80 + (n % 0x40)) + return "?" + end + end + +end + utf = utf or unicode.utf8 local concat, utfchar, utfgsub = table.concat, utf.char, utf.gsub local char, byte, find, bytepairs = string.char, string.byte, string.find, string.bytepairs -unicode = unicode or { } - -- 0 EF BB BF UTF-8 -- 1 FF FE UTF-16-little-endian -- 2 FE FF UTF-16-big-endian @@ -2530,14 +3041,20 @@ unicode.utfname = { [4] = 'utf-32-be' } -function unicode.utftype(f) -- \000 fails ! +-- \000 fails in <= 5.0 but is valid in >=5.1 where %z is depricated + +function unicode.utftype(f) local str = f:read(4) if not str then f:seek('set') return 0 - elseif find(str,"^%z%z\254\255") then + -- elseif find(str,"^%z%z\254\255") then -- depricated + -- elseif find(str,"^\000\000\254\255") then -- not permitted and bugged + elseif find(str,"\000\000\254\255",1,true) then -- seems to work okay (TH) return 4 - elseif find(str,"^\255\254%z%z") then + -- elseif find(str,"^\255\254%z%z") then -- depricated + -- elseif find(str,"^\255\254\000\000") then -- not permitted and bugged + elseif find(str,"\255\254\000\000",1,true) then -- seems to work okay (TH) return 3 elseif find(str,"^\254\255") then f:seek('set',2) @@ -2681,7 +3198,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-math'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -2728,7 +3245,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-utils'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -2736,6 +3253,10 @@ if not modules then modules = { } end modules ['l-utils'] = { -- hm, quite unreadable +local gsub = string.gsub +local concat = table.concat +local type, next = type, next + if not utils then utils = { } end if not utils.merger then utils.merger = { } end if not utils.lua then utils.lua = { } end @@ -2773,7 +3294,7 @@ function utils.merger._self_load_(name) end if data and utils.merger.strip_comment then -- saves some 20K - data = data:gsub("%-%-~[^\n\r]*[\r\n]", "") + data = gsub(data,"%-%-~[^\n\r]*[\r\n]", "") end return data or "" end @@ -2791,7 +3312,7 @@ end function utils.merger._self_swap_(data,code) if data ~= "" then - return (data:gsub(utils.merger.pattern, function(s) + return (gsub(data,utils.merger.pattern, function(s) return "\n\n" .. "-- "..utils.merger.m_begin .. "\n" .. code .. "\n" .. "-- "..utils.merger.m_end .. "\n\n" end, 1)) else @@ -2801,8 +3322,8 @@ end --~ stripper: --~ ---~ data = string.gsub(data,"%-%-~[^\n]*\n","") ---~ data = string.gsub(data,"\n\n+","\n") +--~ data = gsub(data,"%-%-~[^\n]*\n","") +--~ data = gsub(data,"\n\n+","\n") function utils.merger._self_libs_(libs,list) local result, f, frozen = { }, nil, false @@ -2810,9 +3331,10 @@ function utils.merger._self_libs_(libs,list) if type(libs) == 'string' then libs = { libs } end if type(list) == 'string' then list = { list } end local foundpath = nil - for _, lib in ipairs(libs) do - for _, pth in ipairs(list) do - pth = string.gsub(pth,"\\","/") -- file.clean_path + for i=1,#libs do + local lib = libs[i] + for j=1,#list do + local pth = gsub(list[j],"\\","/") -- file.clean_path utils.report("checking library path %s",pth) local name = pth .. "/" .. lib if lfs.isfile(name) then @@ -2824,7 +3346,8 @@ function utils.merger._self_libs_(libs,list) if foundpath then utils.report("using library path %s",foundpath) local right, wrong = { }, { } - for _, lib in ipairs(libs) do + for i=1,#libs do + local lib = libs[i] local fullname = foundpath .. "/" .. lib if lfs.isfile(fullname) then -- right[#right+1] = lib @@ -2838,15 +3361,15 @@ function utils.merger._self_libs_(libs,list) end end if #right > 0 then - utils.report("merged libraries: %s",table.concat(right," ")) + utils.report("merged libraries: %s",concat(right," ")) end if #wrong > 0 then - utils.report("skipped libraries: %s",table.concat(wrong," ")) + utils.report("skipped libraries: %s",concat(wrong," ")) end else utils.report("no valid library path found") end - return table.concat(result, "\n\n") + return concat(result, "\n\n") end function utils.merger.selfcreate(libs,list,target) @@ -2904,16 +3427,28 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-aux'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } +-- for inline, no store split : for s in string.gmatch(str,",* *([^,]+)") do .. end + aux = aux or { } local concat, format, gmatch = table.concat, string.format, string.gmatch local tostring, type = tostring, type +local lpegmatch = lpeg.match + +local P, R, V = lpeg.P, lpeg.R, lpeg.V + +local escape, left, right = P("\\"), P('{'), P('}') + +lpeg.patterns.balanced = P { + [1] = ((escape * (left+right)) + (1 - (left+right)) + V(2))^0, + [2] = left * V(1) * right +} local space = lpeg.P(' ') local equal = lpeg.P("=") @@ -2921,7 +3456,7 @@ local comma = lpeg.P(",") local lbrace = lpeg.P("{") local rbrace = lpeg.P("}") local nobrace = 1 - (lbrace+rbrace) -local nested = lpeg.P{ lbrace * (nobrace + lpeg.V(1))^0 * rbrace } +local nested = lpeg.P { lbrace * (nobrace + lpeg.V(1))^0 * rbrace } local spaces = space^0 local value = lpeg.P(lbrace * lpeg.C((nobrace + nested)^0) * rbrace) + lpeg.C((nested + (1-comma))^0) @@ -2959,13 +3494,13 @@ function aux.make_settings_to_hash_pattern(set,how) end end -function aux.settings_to_hash(str) +function aux.settings_to_hash(str,existing) if str and str ~= "" then - hash = { } + hash = existing or { } if moretolerant then - pattern_b_s:match(str) + lpegmatch(pattern_b_s,str) else - pattern_a_s:match(str) + lpegmatch(pattern_a_s,str) end return hash else @@ -2973,39 +3508,41 @@ function aux.settings_to_hash(str) end end -function aux.settings_to_hash_tolerant(str) +function aux.settings_to_hash_tolerant(str,existing) if str and str ~= "" then - hash = { } - pattern_b_s:match(str) + hash = existing or { } + lpegmatch(pattern_b_s,str) return hash else return { } end end -function aux.settings_to_hash_strict(str) +function aux.settings_to_hash_strict(str,existing) if str and str ~= "" then - hash = { } - pattern_c_s:match(str) + hash = existing or { } + lpegmatch(pattern_c_s,str) return next(hash) and hash else return nil end end -local seperator = comma * space^0 +local separator = comma * space^0 local value = lpeg.P(lbrace * lpeg.C((nobrace + nested)^0) * rbrace) + lpeg.C((nested + (1-comma))^0) -local pattern = lpeg.Ct(value*(seperator*value)^0) +local pattern = lpeg.Ct(value*(separator*value)^0) -- "aap, {noot}, mies" : outer {} removes, leading spaces ignored aux.settings_to_array_pattern = pattern +-- we could use a weak table as cache + function aux.settings_to_array(str) if not str or str == "" then return { } else - return pattern:match(str) + return lpegmatch(pattern,str) end end @@ -3014,10 +3551,10 @@ local function set(t,v) end local value = lpeg.P(lpeg.Carg(1)*value) / set -local pattern = value*(seperator*value)^0 * lpeg.Carg(1) +local pattern = value*(separator*value)^0 * lpeg.Carg(1) function aux.add_settings_to_array(t,str) - return pattern:match(str, nil, t) + return lpegmatch(pattern,str,nil,t) end function aux.hash_to_string(h,separator,yes,no,strict,omit) @@ -3065,6 +3602,13 @@ function aux.settings_to_set(str,t) return t end +local value = lbrace * lpeg.C((nobrace + nested)^0) * rbrace +local pattern = lpeg.Ct((space + value)^0) + +function aux.arguments_to_table(str) + return lpegmatch(pattern,str) +end + -- temporary here function aux.getparameters(self,class,parentclass,settings) @@ -3073,36 +3617,31 @@ function aux.getparameters(self,class,parentclass,settings) sc = table.clone(self[parent]) self[class] = sc end - aux.add_settings_to_array(sc, settings) + aux.settings_to_hash(settings,sc) end -- temporary here -local digit = lpeg.R("09") -local period = lpeg.P(".") -local zero = lpeg.P("0") - ---~ local finish = lpeg.P(-1) ---~ local nodigit = (1-digit) + finish ---~ local case_1 = (period * zero^1 * #nodigit)/"" -- .000 ---~ local case_2 = (period * (1-(zero^0/"") * #nodigit)^1 * (zero^0/"") * nodigit) -- .010 .10 .100100 - +local digit = lpeg.R("09") +local period = lpeg.P(".") +local zero = lpeg.P("0") local trailingzeros = zero^0 * -digit -- suggested by Roberto R -local case_1 = period * trailingzeros / "" -local case_2 = period * (digit - trailingzeros)^1 * (trailingzeros / "") - -local number = digit^1 * (case_1 + case_2) -local stripper = lpeg.Cs((number + 1)^0) +local case_1 = period * trailingzeros / "" +local case_2 = period * (digit - trailingzeros)^1 * (trailingzeros / "") +local number = digit^1 * (case_1 + case_2) +local stripper = lpeg.Cs((number + 1)^0) --~ local sample = "bla 11.00 bla 11 bla 0.1100 bla 1.00100 bla 0.00 bla 0.001 bla 1.1100 bla 0.100100100 bla 0.00100100100" --~ collectgarbage("collect") --~ str = string.rep(sample,10000) --~ local ts = os.clock() ---~ stripper:match(str) ---~ print(#str, os.clock()-ts, stripper:match(sample)) +--~ lpegmatch(stripper,str) +--~ print(#str, os.clock()-ts, lpegmatch(stripper,sample)) + +lpeg.patterns.strip_zeros = stripper function aux.strip_zeros(str) - return stripper:match(str) + return lpegmatch(stripper,str) end function aux.definetable(target) -- defines undefined tables @@ -3126,6 +3665,24 @@ function aux.accesstable(target) return t end +-- as we use this a lot ... + +--~ function aux.cachefunction(action,weak) +--~ local cache = { } +--~ if weak then +--~ setmetatable(cache, { __mode = "kv" } ) +--~ end +--~ local function reminder(str) +--~ local found = cache[str] +--~ if not found then +--~ found = action(str) +--~ cache[str] = found +--~ end +--~ return found +--~ end +--~ return reminder, cache +--~ end + end -- of closure @@ -3133,7 +3690,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['trac-tra'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to trac-tra.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -3143,12 +3700,17 @@ if not modules then modules = { } end modules ['trac-tra'] = { -- bound to a variable, like node.new, node.copy etc (contrary to for instance -- node.has_attribute which is bound to a has_attribute local variable in mkiv) +local debug = require "debug" + +local getinfo = debug.getinfo +local type, next = type, next +local concat = table.concat +local format, find, lower, gmatch, gsub = string.format, string.find, string.lower, string.gmatch, string.gsub + debugger = debugger or { } local counters = { } local names = { } -local getinfo = debug.getinfo -local format, find, lower, gmatch = string.format, string.find, string.lower, string.gmatch -- one @@ -3187,10 +3749,10 @@ function debugger.showstats(printer,threshold) local total, grandtotal, functions = 0, 0, 0 printer("\n") -- ugly but ok -- table.sort(counters) - for func, count in pairs(counters) do + for func, count in next, counters do if count > threshold then local name = getname(func) - if not name:find("for generator") then + if not find(name,"for generator") then printer(format("%8i %s", count, name)) total = total + count end @@ -3222,7 +3784,7 @@ end --~ local total, grandtotal, functions = 0, 0, 0 --~ printer("\n") -- ugly but ok --~ -- table.sort(counters) ---~ for func, count in pairs(counters) do +--~ for func, count in next, counters do --~ if count > threshold then --~ printer(format("%8i %s", count, func)) --~ total = total + count @@ -3276,38 +3838,77 @@ end --~ print("") --~ debugger.showstats(print,3) -trackers = trackers or { } +setters = setters or { } +setters.data = setters.data or { } -local data, done = { }, { } +--~ local function set(t,what,value) +--~ local data, done = t.data, t.done +--~ if type(what) == "string" then +--~ what = aux.settings_to_array(what) -- inefficient but ok +--~ end +--~ for i=1,#what do +--~ local w = what[i] +--~ for d, f in next, data do +--~ if done[d] then +--~ -- prevent recursion due to wildcards +--~ elseif find(d,w) then +--~ done[d] = true +--~ for i=1,#f do +--~ f[i](value) +--~ end +--~ end +--~ end +--~ end +--~ end -local function set(what,value) +local function set(t,what,value) + local data, done = t.data, t.done if type(what) == "string" then - what = aux.settings_to_array(what) + what = aux.settings_to_hash(what) -- inefficient but ok end - for i=1,#what do - local w = what[i] + for w, v in next, what do + if v == "" then + v = value + else + v = toboolean(v) + end for d, f in next, data do if done[d] then -- prevent recursion due to wildcards elseif find(d,w) then done[d] = true for i=1,#f do - f[i](value) + f[i](v) end end end end end -local function reset() - for d, f in next, data do +local function reset(t) + for d, f in next, t.data do for i=1,#f do f[i](false) end end end -function trackers.register(what,...) +local function enable(t,what) + set(t,what,true) +end + +local function disable(t,what) + local data = t.data + if not what or what == "" then + t.done = { } + reset(t) + else + set(t,what,false) + end +end + +function setters.register(t,what,...) + local data = t.data what = lower(what) local w = data[what] if not w then @@ -3319,32 +3920,32 @@ function trackers.register(what,...) if typ == "function" then w[#w+1] = fnc elseif typ == "string" then - w[#w+1] = function(value) set(fnc,value,nesting) end + w[#w+1] = function(value) set(t,fnc,value,nesting) end end end end -function trackers.enable(what) - done = { } - set(what,true) +function setters.enable(t,what) + local e = t.enable + t.enable, t.done = enable, { } + enable(t,string.simpleesc(tostring(what))) + t.enable, t.done = e, { } end -function trackers.disable(what) - done = { } - if not what or what == "" then - trackers.reset(what) - else - set(what,false) - end +function setters.disable(t,what) + local e = t.disable + t.disable, t.done = disable, { } + disable(t,string.simpleesc(tostring(what))) + t.disable, t.done = e, { } end -function trackers.reset(what) - done = { } - reset() +function setters.reset(t) + t.done = { } + reset(t) end -function trackers.list() -- pattern - local list = table.sortedkeys(data) +function setters.list(t) -- pattern + local list = table.sortedkeys(t.data) local user, system = { }, { } for l=1,#list do local what = list[l] @@ -3357,6 +3958,78 @@ function trackers.list() -- pattern return user, system end +function setters.show(t) + commands.writestatus("","") + local list = setters.list(t) + for k=1,#list do + commands.writestatus(t.name,list[k]) + end + commands.writestatus("","") +end + +-- we could have used a bit of oo and the trackers:enable syntax but +-- there is already a lot of code around using the singular tracker + +-- we could make this into a module + +function setters.new(name) + local t + t = { + data = { }, + name = name, + enable = function(...) setters.enable (t,...) end, + disable = function(...) setters.disable (t,...) end, + register = function(...) setters.register(t,...) end, + list = function(...) setters.list (t,...) end, + show = function(...) setters.show (t,...) end, + } + setters.data[name] = t + return t +end + +trackers = setters.new("trackers") +directives = setters.new("directives") +experiments = setters.new("experiments") + +-- nice trick: we overload two of the directives related functions with variants that +-- do tracing (itself using a tracker) .. proof of concept + +local trace_directives = false local trace_directives = false trackers.register("system.directives", function(v) trace_directives = v end) +local trace_experiments = false local trace_experiments = false trackers.register("system.experiments", function(v) trace_experiments = v end) + +local e = directives.enable +local d = directives.disable + +function directives.enable(...) + commands.writestatus("directives","enabling: %s",concat({...}," ")) + e(...) +end + +function directives.disable(...) + commands.writestatus("directives","disabling: %s",concat({...}," ")) + d(...) +end + +local e = experiments.enable +local d = experiments.disable + +function experiments.enable(...) + commands.writestatus("experiments","enabling: %s",concat({...}," ")) + e(...) +end + +function experiments.disable(...) + commands.writestatus("experiments","disabling: %s",concat({...}," ")) + d(...) +end + +-- a useful example + +directives.register("system.nostatistics", function(v) + statistics.enable = not v +end) + + end -- of closure @@ -3364,7 +4037,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['luat-env'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -3376,10 +4049,10 @@ if not modules then modules = { } end modules ['luat-env'] = { -- evolved before bytecode arrays were available and so a lot of -- code has disappeared already. -local trace_verbose = false trackers.register("resolvers.verbose", function(v) trace_verbose = v end) -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v trackers.enable("resolvers.verbose") end) +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) -local format = string.format +local format, sub, match, gsub, find = string.format, string.sub, string.match, string.gsub, string.find +local unquote, quote = string.unquote, string.quote -- precautions @@ -3413,13 +4086,14 @@ if not environment.jobname then environ function environment.initialize_arguments(arg) local arguments, files = { }, { } environment.arguments, environment.files, environment.sortedflags = arguments, files, nil - for index, argument in pairs(arg) do + for index=1,#arg do + local argument = arg[index] if index > 0 then - local flag, value = argument:match("^%-+(.+)=(.-)$") + local flag, value = match(argument,"^%-+(.-)=(.-)$") if flag then - arguments[flag] = string.unquote(value or "") + arguments[flag] = unquote(value or "") else - flag = argument:match("^%-+(.+)") + flag = match(argument,"^%-+(.+)") if flag then arguments[flag] = true else @@ -3446,25 +4120,30 @@ function environment.argument(name,partial) return arguments[name] elseif partial then if not sortedflags then - sortedflags = { } - for _,v in pairs(table.sortedkeys(arguments)) do - sortedflags[#sortedflags+1] = "^" .. v + sortedflags = table.sortedkeys(arguments) + for k=1,#sortedflags do + sortedflags[k] = "^" .. sortedflags[k] end environment.sortedflags = sortedflags end -- example of potential clash: ^mode ^modefile - for _,v in ipairs(sortedflags) do - if name:find(v) then - return arguments[v:sub(2,#v)] + for k=1,#sortedflags do + local v = sortedflags[k] + if find(name,v) then + return arguments[sub(v,2,#v)] end end end return nil end +environment.argument("x",true) + function environment.split_arguments(separator) -- rather special, cut-off before separator local done, before, after = false, { }, { } - for _,v in ipairs(environment.original_arguments) do + local original_arguments = environment.original_arguments + for k=1,#original_arguments do + local v = original_arguments[k] if not done and v == separator then done = true elseif done then @@ -3481,16 +4160,17 @@ function environment.reconstruct_commandline(arg,noquote) if noquote and #arg == 1 then local a = arg[1] a = resolvers.resolve(a) - a = a:unquote() + a = unquote(a) return a - elseif next(arg) then + elseif #arg > 0 then local result = { } - for _,a in ipairs(arg) do -- ipairs 1 .. #n + for i=1,#arg do + local a = arg[i] a = resolvers.resolve(a) - a = a:unquote() - a = a:gsub('"','\\"') -- tricky - if a:find(" ") then - result[#result+1] = a:quote() + a = unquote(a) + a = gsub(a,'"','\\"') -- tricky + if find(a," ") then + result[#result+1] = quote(a) else result[#result+1] = a end @@ -3503,17 +4183,18 @@ end if arg then - -- new, reconstruct quoted snippets (maybe better just remnove the " then and add them later) + -- new, reconstruct quoted snippets (maybe better just remove the " then and add them later) local newarg, instring = { }, false - for index, argument in ipairs(arg) do - if argument:find("^\"") then - newarg[#newarg+1] = argument:gsub("^\"","") - if not argument:find("\"$") then + for index=1,#arg do + local argument = arg[index] + if find(argument,"^\"") then + newarg[#newarg+1] = gsub(argument,"^\"","") + if not find(argument,"\"$") then instring = true end - elseif argument:find("\"$") then - newarg[#newarg] = newarg[#newarg] .. " " .. argument:gsub("\"$","") + elseif find(argument,"\"$") then + newarg[#newarg] = newarg[#newarg] .. " " .. gsub(argument,"\"$","") instring = false elseif instring then newarg[#newarg] = newarg[#newarg] .. " " .. argument @@ -3568,12 +4249,12 @@ function environment.luafilechunk(filename) -- used for loading lua bytecode in filename = file.replacesuffix(filename, "lua") local fullname = environment.luafile(filename) if fullname and fullname ~= "" then - if trace_verbose then + if trace_locating then logs.report("fileio","loading file %s", fullname) end return environment.loadedluacode(fullname) else - if trace_verbose then + if trace_locating then logs.report("fileio","unknown file %s", filename) end return nil @@ -3593,7 +4274,7 @@ function environment.loadluafile(filename, version) -- when not overloaded by explicit suffix we look for a luc file first local fullname = (lucname and environment.luafile(lucname)) or "" if fullname ~= "" then - if trace_verbose then + if trace_locating then logs.report("fileio","loading %s", fullname) end chunk = loadfile(fullname) -- this way we don't need a file exists check @@ -3611,7 +4292,7 @@ function environment.loadluafile(filename, version) if v == version then return true else - if trace_verbose then + if trace_locating then logs.report("fileio","version mismatch for %s: lua=%s, luc=%s", filename, v, version) end environment.loadluafile(filename) @@ -3622,12 +4303,12 @@ function environment.loadluafile(filename, version) end fullname = (luaname and environment.luafile(luaname)) or "" if fullname ~= "" then - if trace_verbose then + if trace_locating then logs.report("fileio","loading %s", fullname) end chunk = loadfile(fullname) -- this way we don't need a file exists check if not chunk then - if verbose then + if trace_locating then logs.report("fileio","unknown file %s", filename) end else @@ -3645,7 +4326,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['trac-inf'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to trac-inf.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -3670,6 +4351,14 @@ function statistics.hastimer(instance) return instance and instance.starttime end +function statistics.resettiming(instance) + if not instance then + notimer = { timing = 0, loadtime = 0 } + else + instance.timing, instance.loadtime = 0, 0 + end +end + function statistics.starttiming(instance) if not instance then notimer = { } @@ -3684,6 +4373,8 @@ function statistics.starttiming(instance) if not instance.loadtime then instance.loadtime = 0 end + else +--~ logs.report("system","nested timing (%s)",tostring(instance)) end instance.timing = it + 1 end @@ -3729,6 +4420,12 @@ function statistics.elapsedindeed(instance) return t > statistics.threshold end +function statistics.elapsedseconds(instance,rest) -- returns nil if 0 seconds + if statistics.elapsedindeed(instance) then + return format("%s seconds %s", statistics.elapsedtime(instance),rest or "") + end +end + -- general function function statistics.register(tag,fnc) @@ -3807,14 +4504,32 @@ function statistics.timed(action,report) report("total runtime: %s",statistics.elapsedtime(timer)) end +-- where, not really the best spot for this: + +commands = commands or { } + +local timer + +function commands.resettimer() + statistics.resettiming(timer) + statistics.starttiming(timer) +end + +function commands.elapsedtime() + statistics.stoptiming(timer) + tex.sprint(statistics.elapsedtime(timer)) +end + +commands.resettimer() + end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['luat-log'] = { +if not modules then modules = { } end modules ['trac-log'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to trac-log.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -3822,7 +4537,11 @@ if not modules then modules = { } end modules ['luat-log'] = { -- this is old code that needs an overhaul -local write_nl, write, format = texio.write_nl or print, texio.write or io.write, string.format +--~ io.stdout:setvbuf("no") +--~ io.stderr:setvbuf("no") + +local write_nl, write = texio.write_nl or print, texio.write or io.write +local format, gmatch = string.format, string.gmatch local texcount = tex and tex.count if texlua then @@ -3903,25 +4622,48 @@ function logs.tex.line(fmt,...) -- new end end +--~ function logs.tex.start_page_number() +--~ local real, user, sub = texcount.realpageno, texcount.userpageno, texcount.subpageno +--~ if real > 0 then +--~ if user > 0 then +--~ if sub > 0 then +--~ write(format("[%s.%s.%s",real,user,sub)) +--~ else +--~ write(format("[%s.%s",real,user)) +--~ end +--~ else +--~ write(format("[%s",real)) +--~ end +--~ else +--~ write("[-") +--~ end +--~ end + +--~ function logs.tex.stop_page_number() +--~ write("]") +--~ end + +local real, user, sub + function logs.tex.start_page_number() - local real, user, sub = texcount.realpageno, texcount.userpageno, texcount.subpageno + real, user, sub = texcount.realpageno, texcount.userpageno, texcount.subpageno +end + +function logs.tex.stop_page_number() if real > 0 then if user > 0 then if sub > 0 then - write(format("[%s.%s.%s",real,user,sub)) + logs.report("pages", "flushing realpage %s, userpage %s, subpage %s",real,user,sub) else - write(format("[%s.%s",real,user)) + logs.report("pages", "flushing realpage %s, userpage %s",real,user) end else - write(format("[%s",real)) + logs.report("pages", "flushing realpage %s",real) end else - write("[-") + logs.report("pages", "flushing page") end -end - -function logs.tex.stop_page_number() - write("]") + io.flush() end logs.tex.report_job_stat = statistics.show_job_stat @@ -4021,7 +4763,7 @@ end function logs.setprogram(_name_,_banner_,_verbose_) name, banner = _name_, _banner_ if _verbose_ then - trackers.enable("resolvers.verbose") + trackers.enable("resolvers.locating") end logs.set_method("tex") logs.report = report -- also used in libraries @@ -4034,9 +4776,9 @@ end function logs.setverbose(what) if what then - trackers.enable("resolvers.verbose") + trackers.enable("resolvers.locating") else - trackers.disable("resolvers.verbose") + trackers.disable("resolvers.locating") end logs.verbose = what or false end @@ -4053,7 +4795,7 @@ logs.report = logs.tex.report logs.simple = logs.tex.report function logs.reportlines(str) -- todo: <lines></lines> - for line in str:gmatch("(.-)[\n\r]") do + for line in gmatch(str,"(.-)[\n\r]") do logs.report(line) end end @@ -4064,8 +4806,12 @@ end logs.simpleline = logs.reportline -function logs.help(message,option) +function logs.reportbanner() -- for scripts too logs.report(banner) +end + +function logs.help(message,option) + logs.reportbanner() logs.reportline() logs.reportlines(message) local moreinfo = logs.moreinfo or "" @@ -4097,6 +4843,11 @@ end --~ logs.system(syslogname,"context","test","fonts","font %s recached due to newer version (%s)","blabla","123") --~ end +function logs.fatal(where,...) + logs.report(where,"fatal error: %s, aborting now",format(...)) + os.exit() +end + end -- of closure @@ -4104,10 +4855,10 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-inp'] = { version = 1.001, + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files", - comment = "companion to luat-lib.tex", } -- After a few years using the code the large luat-inp.lua file @@ -4119,7 +4870,7 @@ if not modules then modules = { } end modules ['data-inp'] = { -- * some public auxiliary functions were made private -- -- TODO: os.getenv -> os.env[] --- TODO: instances.[hashes,cnffiles,configurations,522] -> ipairs (alles check, sneller) +-- TODO: instances.[hashes,cnffiles,configurations,522] -- TODO: check escaping in find etc, too much, too slow -- This lib is multi-purpose and can be loaded again later on so that @@ -4140,12 +4891,13 @@ if not modules then modules = { } end modules ['data-inp'] = { local format, gsub, find, lower, upper, match, gmatch = string.format, string.gsub, string.find, string.lower, string.upper, string.match, string.gmatch local concat, insert, sortedkeys = table.concat, table.insert, table.sortedkeys local next, type = next, type +local lpegmatch = lpeg.match -local trace_locating, trace_detail, trace_verbose = false, false, false +local trace_locating, trace_detail, trace_expansions = false, false, false -trackers.register("resolvers.verbose", function(v) trace_verbose = v end) -trackers.register("resolvers.locating", function(v) trace_locating = v trackers.enable("resolvers.verbose") end) -trackers.register("resolvers.detail", function(v) trace_detail = v trackers.enable("resolvers.verbose,resolvers.detail") end) +trackers.register("resolvers.locating", function(v) trace_locating = v end) +trackers.register("resolvers.details", function(v) trace_detail = v end) +trackers.register("resolvers.expansions", function(v) trace_expansions = v end) -- todo if not resolvers then resolvers = { @@ -4169,7 +4921,7 @@ resolvers.generators.notfound = { nil } resolvers.cacheversion = '1.0.1' resolvers.cnfname = 'texmf.cnf' resolvers.luaname = 'texmfcnf.lua' -resolvers.homedir = os.env[os.platform == "windows" and 'USERPROFILE'] or os.env['HOME'] or '~' +resolvers.homedir = os.env[os.type == "windows" and 'USERPROFILE'] or os.env['HOME'] or '~' resolvers.cnfdefault = '{$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c}' local dummy_path_expr = "^!*unset/*$" @@ -4211,8 +4963,8 @@ suffixes['lua'] = { 'lua', 'luc', 'tma', 'tmc' } alternatives['map files'] = 'map' alternatives['enc files'] = 'enc' -alternatives['cid files'] = 'cid' -alternatives['fea files'] = 'fea' +alternatives['cid maps'] = 'cid' -- great, why no cid files +alternatives['font feature files'] = 'fea' -- and fea files here alternatives['opentype fonts'] = 'otf' alternatives['truetype fonts'] = 'ttf' alternatives['truetype collections'] = 'ttc' @@ -4228,6 +4980,11 @@ formats ['sfd'] = 'SFDFONTS' suffixes ['sfd'] = { 'sfd' } alternatives['subfont definition files'] = 'sfd' +-- lib paths + +formats ['lib'] = 'CLUAINPUTS' -- new (needs checking) +suffixes['lib'] = (os.libsuffix and { os.libsuffix }) or { 'dll', 'so' } + -- In practice we will work within one tds tree, but i want to keep -- the option open to build tools that look at multiple trees, which is -- why we keep the tree specific data in a table. We used to pass the @@ -4350,8 +5107,10 @@ local function check_configuration() -- not yet ok, no time for debugging now -- bad luck end fix("LUAINPUTS" , ".;$TEXINPUTS;$TEXMFSCRIPTS") -- no progname, hm - fix("FONTFEATURES", ".;$TEXMF/fonts/fea//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") - fix("FONTCIDMAPS" , ".;$TEXMF/fonts/cid//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") + -- this will go away some day + fix("FONTFEATURES", ".;$TEXMF/fonts/{data,fea}//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") + fix("FONTCIDMAPS" , ".;$TEXMF/fonts/{data,cid}//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") + -- fix("LUATEXLIBS" , ".;$TEXMF/luatex/lua//") end @@ -4366,7 +5125,7 @@ function resolvers.settrace(n) -- no longer number but: 'locating' or 'detail' end end -resolvers.settrace(os.getenv("MTX.resolvers.TRACE") or os.getenv("MTX_INPUT_TRACE")) +resolvers.settrace(os.getenv("MTX_INPUT_TRACE")) function resolvers.osenv(key) local ie = instance.environment @@ -4454,37 +5213,43 @@ end -- work that well; the parsing is ok, but dealing with the resulting -- table is a pain because we need to work inside-out recursively +local function do_first(a,b) + local t = { } + for s in gmatch(b,"[^,]+") do t[#t+1] = a .. s end + return "{" .. concat(t,",") .. "}" +end + +local function do_second(a,b) + local t = { } + for s in gmatch(a,"[^,]+") do t[#t+1] = s .. b end + return "{" .. concat(t,",") .. "}" +end + +local function do_both(a,b) + local t = { } + for sa in gmatch(a,"[^,]+") do + for sb in gmatch(b,"[^,]+") do + t[#t+1] = sa .. sb + end + end + return "{" .. concat(t,",") .. "}" +end + +local function do_three(a,b,c) + return a .. b.. c +end + local function splitpathexpr(str, t, validate) -- no need for further optimization as it is only called a - -- few times, we can use lpeg for the sub; we could move - -- the local functions outside the body + -- few times, we can use lpeg for the sub + if trace_expansions then + logs.report("fileio","expanding variable '%s'",str) + end t = t or { } str = gsub(str,",}",",@}") str = gsub(str,"{,","{@,") -- str = "@" .. str .. "@" local ok, done - local function do_first(a,b) - local t = { } - for s in gmatch(b,"[^,]+") do t[#t+1] = a .. s end - return "{" .. concat(t,",") .. "}" - end - local function do_second(a,b) - local t = { } - for s in gmatch(a,"[^,]+") do t[#t+1] = s .. b end - return "{" .. concat(t,",") .. "}" - end - local function do_both(a,b) - local t = { } - for sa in gmatch(a,"[^,]+") do - for sb in gmatch(b,"[^,]+") do - t[#t+1] = sa .. sb - end - end - return "{" .. concat(t,",") .. "}" - end - local function do_three(a,b,c) - return a .. b.. c - end while true do done = false while true do @@ -4515,6 +5280,11 @@ local function splitpathexpr(str, t, validate) t[#t+1] = s end end + if trace_expansions then + for k=1,#t do + logs.report("fileio","% 4i: %s",k,t[k]) + end + end return t end @@ -4554,18 +5324,27 @@ end -- also we now follow the stupid route: if not set then just assume *one* -- cnf file under texmf (i.e. distribution) -resolvers.ownpath = resolvers.ownpath or nil -resolvers.ownbin = resolvers.ownbin or arg[-2] or arg[-1] or arg[0] or "luatex" -resolvers.autoselfdir = true -- false may be handy for debugging +local args = environment and environment.original_arguments or arg -- this needs a cleanup + +resolvers.ownbin = resolvers.ownbin or args[-2] or arg[-2] or args[-1] or arg[-1] or arg[0] or "luatex" +resolvers.ownbin = gsub(resolvers.ownbin,"\\","/") function resolvers.getownpath() - if not resolvers.ownpath then - if resolvers.autoselfdir and os.selfdir then - resolvers.ownpath = os.selfdir - else - local binary = resolvers.ownbin - if os.platform == "windows" then - binary = file.replacesuffix(binary,"exe") + local ownpath = resolvers.ownpath or os.selfdir + if not ownpath or ownpath == "" or ownpath == "unset" then + ownpath = args[-1] or arg[-1] + ownpath = ownpath and file.dirname(gsub(ownpath,"\\","/")) + if not ownpath or ownpath == "" then + ownpath = args[-0] or arg[-0] + ownpath = ownpath and file.dirname(gsub(ownpath,"\\","/")) + end + local binary = resolvers.ownbin + if not ownpath or ownpath == "" then + ownpath = ownpath and file.dirname(binary) + end + if not ownpath or ownpath == "" then + if os.binsuffix ~= "" then + binary = file.replacesuffix(binary,os.binsuffix) end for p in gmatch(os.getenv("PATH"),"[^"..io.pathseparator.."]+") do local b = file.join(p,binary) @@ -4577,30 +5356,39 @@ function resolvers.getownpath() local olddir = lfs.currentdir() if lfs.chdir(p) then local pp = lfs.currentdir() - if trace_verbose and p ~= pp then - logs.report("fileio","following symlink %s to %s",p,pp) + if trace_locating and p ~= pp then + logs.report("fileio","following symlink '%s' to '%s'",p,pp) end - resolvers.ownpath = pp + ownpath = pp lfs.chdir(olddir) else - if trace_verbose then - logs.report("fileio","unable to check path %s",p) + if trace_locating then + logs.report("fileio","unable to check path '%s'",p) end - resolvers.ownpath = p + ownpath = p end break end end end - if not resolvers.ownpath then resolvers.ownpath = '.' end + if not ownpath or ownpath == "" then + ownpath = "." + logs.report("fileio","forcing fallback ownpath .") + elseif trace_locating then + logs.report("fileio","using ownpath '%s'",ownpath) + end end - return resolvers.ownpath + resolvers.ownpath = ownpath + function resolvers.getownpath() + return resolvers.ownpath + end + return ownpath end local own_places = { "SELFAUTOLOC", "SELFAUTODIR", "SELFAUTOPARENT", "TEXMFCNF" } local function identify_own() - local ownpath = resolvers.getownpath() or lfs.currentdir() + local ownpath = resolvers.getownpath() or dir.current() local ie = instance.environment if ownpath then if resolvers.env('SELFAUTOLOC') == "" then os.env['SELFAUTOLOC'] = file.collapse_path(ownpath) end @@ -4613,10 +5401,10 @@ local function identify_own() if resolvers.env('TEXMFCNF') == "" then os.env['TEXMFCNF'] = resolvers.cnfdefault end if resolvers.env('TEXOS') == "" then os.env['TEXOS'] = resolvers.env('SELFAUTODIR') end if resolvers.env('TEXROOT') == "" then os.env['TEXROOT'] = resolvers.env('SELFAUTOPARENT') end - if trace_verbose then + if trace_locating then for i=1,#own_places do local v = own_places[i] - logs.report("fileio","variable %s set to %s",v,resolvers.env(v) or "unknown") + logs.report("fileio","variable '%s' set to '%s'",v,resolvers.env(v) or "unknown") end end identify_own = function() end @@ -4648,10 +5436,8 @@ end local function load_cnf_file(fname) fname = resolvers.clean_path(fname) local lname = file.replacesuffix(fname,'lua') - local f = io.open(lname) - if f then -- this will go - f:close() - local dname = file.dirname(fname) + if lfs.isfile(lname) then + local dname = file.dirname(fname) -- fname ? if not instance.configuration[dname] then resolvers.load_data(dname,'configuration',lname and file.basename(lname)) instance.order[#instance.order+1] = instance.configuration[dname] @@ -4659,8 +5445,8 @@ local function load_cnf_file(fname) else f = io.open(fname) if f then - if trace_verbose then - logs.report("fileio","loading %s", fname) + if trace_locating then + logs.report("fileio","loading configuration file %s", fname) end local line, data, n, k, v local dname = file.dirname(fname) @@ -4694,14 +5480,16 @@ local function load_cnf_file(fname) end end f:close() - elseif trace_verbose then - logs.report("fileio","skipping %s", fname) + elseif trace_locating then + logs.report("fileio","skipping configuration file '%s'", fname) end end end local function collapse_cnf_data() -- potential optimization: pass start index (setup and configuration are shared) - for _,c in ipairs(instance.order) do + local order = instance.order + for i=1,#order do + local c = order[i] for k,v in next, c do if not instance.variables[k] then if instance.environment[k] then @@ -4717,19 +5505,24 @@ end function resolvers.load_cnf() local function loadoldconfigdata() - for _, fname in ipairs(instance.cnffiles) do - load_cnf_file(fname) + local cnffiles = instance.cnffiles + for i=1,#cnffiles do + load_cnf_file(cnffiles[i]) end end -- instance.cnffiles contain complete names now ! + -- we still use a funny mix of cnf and new but soon + -- we will switch to lua exclusively as we only use + -- the file to collect the tree roots if #instance.cnffiles == 0 then - if trace_verbose then + if trace_locating then logs.report("fileio","no cnf files found (TEXMFCNF may not be set/known)") end else - instance.rootpath = instance.cnffiles[1] - for k,fname in ipairs(instance.cnffiles) do - instance.cnffiles[k] = file.collapse_path(gsub(fname,"\\",'/')) + local cnffiles = instance.cnffiles + instance.rootpath = cnffiles[1] + for k=1,#cnffiles do + instance.cnffiles[k] = file.collapse_path(cnffiles[k]) end for i=1,3 do instance.rootpath = file.dirname(instance.rootpath) @@ -4757,8 +5550,9 @@ function resolvers.load_lua() -- yet harmless else instance.rootpath = instance.luafiles[1] - for k,fname in ipairs(instance.luafiles) do - instance.luafiles[k] = file.collapse_path(gsub(fname,"\\",'/')) + local luafiles = instance.luafiles + for k=1,#luafiles do + instance.luafiles[k] = file.collapse_path(luafiles[k]) end for i=1,3 do instance.rootpath = file.dirname(instance.rootpath) @@ -4790,14 +5584,14 @@ end function resolvers.append_hash(type,tag,name) if trace_locating then - logs.report("fileio","= hash append: %s",tag) + logs.report("fileio","hash '%s' appended",tag) end insert(instance.hashes, { ['type']=type, ['tag']=tag, ['name']=name } ) end function resolvers.prepend_hash(type,tag,name) if trace_locating then - logs.report("fileio","= hash prepend: %s",tag) + logs.report("fileio","hash '%s' prepended",tag) end insert(instance.hashes, 1, { ['type']=type, ['tag']=tag, ['name']=name } ) end @@ -4821,9 +5615,11 @@ end -- locators function resolvers.locatelists() - for _, path in ipairs(resolvers.clean_path_list('TEXMF')) do - if trace_verbose then - logs.report("fileio","locating list of %s",path) + local texmfpaths = resolvers.clean_path_list('TEXMF') + for i=1,#texmfpaths do + local path = texmfpaths[i] + if trace_locating then + logs.report("fileio","locating list of '%s'",path) end resolvers.locatedatabase(file.collapse_path(path)) end @@ -4836,11 +5632,11 @@ end function resolvers.locators.tex(specification) if specification and specification ~= '' and lfs.isdir(specification) then if trace_locating then - logs.report("fileio",'! tex locator found: %s',specification) + logs.report("fileio","tex locator '%s' found",specification) end resolvers.append_hash('file',specification,filename) elseif trace_locating then - logs.report("fileio",'? tex locator not found: %s',specification) + logs.report("fileio","tex locator '%s' not found",specification) end end @@ -4854,7 +5650,9 @@ function resolvers.loadfiles() instance.loaderror = false instance.files = { } if not instance.renewcache then - for _, hash in ipairs(instance.hashes) do + local hashes = instance.hashes + for k=1,#hashes do + local hash = hashes[k] resolvers.hashdatabase(hash.tag,hash.name) if instance.loaderror then break end end @@ -4868,8 +5666,9 @@ end -- generators: function resolvers.loadlists() - for _, hash in ipairs(instance.hashes) do - resolvers.generatedatabase(hash.tag) + local hashes = instance.hashes + for i=1,#hashes do + resolvers.generatedatabase(hashes[i].tag) end end @@ -4881,10 +5680,27 @@ end local weird = lpeg.P(".")^1 + lpeg.anywhere(lpeg.S("~`!#$%^&*()={}[]:;\"\'||<>,?\n\r\t")) +--~ local l_forbidden = lpeg.S("~`!#$%^&*()={}[]:;\"\'||\\/<>,?\n\r\t") +--~ local l_confusing = lpeg.P(" ") +--~ local l_character = lpeg.patterns.utf8 +--~ local l_dangerous = lpeg.P(".") + +--~ local l_normal = (l_character - l_forbidden - l_confusing - l_dangerous) * (l_character - l_forbidden - l_confusing^2)^0 * lpeg.P(-1) +--~ ----- l_normal = l_normal * lpeg.Cc(true) + lpeg.Cc(false) + +--~ local function test(str) +--~ print(str,lpeg.match(l_normal,str)) +--~ end +--~ test("ヒラギノ明朝 Pro W3") +--~ test("..ヒラギノ明朝 Pro W3") +--~ test(":ヒラギノ明朝 Pro W3;") +--~ test("ヒラギノ明朝 /Pro W3;") +--~ test("ヒラギノ明朝 Pro W3") + function resolvers.generators.tex(specification) local tag = specification - if trace_verbose then - logs.report("fileio","scanning path %s",specification) + if trace_locating then + logs.report("fileio","scanning path '%s'",specification) end instance.files[tag] = { } local files = instance.files[tag] @@ -4900,7 +5716,8 @@ function resolvers.generators.tex(specification) full = spec end for name in directory(full) do - if not weird:match(name) then + if not lpegmatch(weird,name) then + -- if lpegmatch(l_normal,name) then local mode = attributes(full..name,'mode') if mode == 'file' then if path then @@ -4933,7 +5750,7 @@ function resolvers.generators.tex(specification) end end action() - if trace_verbose then + if trace_locating then logs.report("fileio","%s files found on %s directories with %s uppercase remappings",n,m,r) end end @@ -4948,11 +5765,48 @@ end -- we join them and split them after the expansion has taken place. This -- is more convenient. +--~ local checkedsplit = string.checkedsplit + +local cache = { } + +local splitter = lpeg.Ct(lpeg.splitat(lpeg.S(os.type == "windows" and ";" or ":;"))) + +local function split_kpse_path(str) -- beware, this can be either a path or a {specification} + local found = cache[str] + if not found then + if str == "" then + found = { } + else + str = gsub(str,"\\","/") +--~ local split = (find(str,";") and checkedsplit(str,";")) or checkedsplit(str,io.pathseparator) +local split = lpegmatch(splitter,str) + found = { } + for i=1,#split do + local s = split[i] + if not find(s,"^{*unset}*") then + found[#found+1] = s + end + end + if trace_expansions then + logs.report("fileio","splitting path specification '%s'",str) + for k=1,#found do + logs.report("fileio","% 4i: %s",k,found[k]) + end + end + cache[str] = found + end + end + return found +end + +resolvers.split_kpse_path = split_kpse_path + function resolvers.splitconfig() - for i,c in ipairs(instance) do - for k,v in pairs(c) do + for i=1,#instance do + local c = instance[i] + for k,v in next, c do if type(v) == 'string' then - local t = file.split_path(v) + local t = split_kpse_path(v) if #t > 1 then c[k] = t end @@ -4962,21 +5816,25 @@ function resolvers.splitconfig() end function resolvers.joinconfig() - for i,c in ipairs(instance.order) do - for k,v in pairs(c) do -- ipairs? + local order = instance.order + for i=1,#order do + local c = order[i] + for k,v in next, c do -- indexed? if type(v) == 'table' then c[k] = file.join_path(v) end end end end + function resolvers.split_path(str) if type(str) == 'table' then return str else - return file.split_path(str) + return split_kpse_path(str) end end + function resolvers.join_path(str) if type(str) == 'table' then return file.join_path(str) @@ -4988,8 +5846,9 @@ end function resolvers.splitexpansions() local ie = instance.expansions for k,v in next, ie do - local t, h = { }, { } - for _,vv in ipairs(file.split_path(v)) do + local t, h, p = { }, { }, split_kpse_path(v) + for kk=1,#p do + local vv = p[kk] if vv ~= "" and not h[vv] then t[#t+1] = vv h[vv] = true @@ -5036,11 +5895,15 @@ function resolvers.serialize(files) end t[#t+1] = "return {" if instance.sortdata then - for _, k in pairs(sortedkeys(files)) do -- ipairs + local sortedfiles = sortedkeys(files) + for i=1,#sortedfiles do + local k = sortedfiles[i] local fk = files[k] if type(fk) == 'table' then t[#t+1] = "\t['" .. k .. "']={" - for _, kk in pairs(sortedkeys(fk)) do -- ipairs + local sortedfk = sortedkeys(fk) + for j=1,#sortedfk do + local kk = sortedfk[j] t[#t+1] = dump(kk,fk[kk],"\t\t") end t[#t+1] = "\t}," @@ -5065,12 +5928,18 @@ function resolvers.serialize(files) return concat(t,"\n") end +local data_state = { } + +function resolvers.data_state() + return data_state or { } +end + function resolvers.save_data(dataname, makename) -- untested without cache overload for cachename, files in next, instance[dataname] do local name = (makename or file.join)(cachename,dataname) local luaname, lucname = name .. ".lua", name .. ".luc" - if trace_verbose then - logs.report("fileio","preparing %s for %s",dataname,cachename) + if trace_locating then + logs.report("fileio","preparing '%s' for '%s'",dataname,cachename) end for k, v in next, files do if type(v) == "table" and #v == 1 then @@ -5084,24 +5953,25 @@ function resolvers.save_data(dataname, makename) -- untested without cache overl date = os.date("%Y-%m-%d"), time = os.date("%H:%M:%S"), content = files, + uuid = os.uuid(), } local ok = io.savedata(luaname,resolvers.serialize(data)) if ok then - if trace_verbose then - logs.report("fileio","%s saved in %s",dataname,luaname) + if trace_locating then + logs.report("fileio","'%s' saved in '%s'",dataname,luaname) end if utils.lua.compile(luaname,lucname,false,true) then -- no cleanup but strip - if trace_verbose then - logs.report("fileio","%s compiled to %s",dataname,lucname) + if trace_locating then + logs.report("fileio","'%s' compiled to '%s'",dataname,lucname) end else - if trace_verbose then - logs.report("fileio","compiling failed for %s, deleting file %s",dataname,lucname) + if trace_locating then + logs.report("fileio","compiling failed for '%s', deleting file '%s'",dataname,lucname) end os.remove(lucname) end - elseif trace_verbose then - logs.report("fileio","unable to save %s in %s (access error)",dataname,luaname) + elseif trace_locating then + logs.report("fileio","unable to save '%s' in '%s' (access error)",dataname,luaname) end end end @@ -5113,19 +5983,20 @@ function resolvers.load_data(pathname,dataname,filename,makename) -- untested wi if blob then local data = blob() if data and data.content and data.type == dataname and data.version == resolvers.cacheversion then - if trace_verbose then - logs.report("fileio","loading %s for %s from %s",dataname,pathname,filename) + data_state[#data_state+1] = data.uuid + if trace_locating then + logs.report("fileio","loading '%s' for '%s' from '%s'",dataname,pathname,filename) end instance[dataname][pathname] = data.content else - if trace_verbose then - logs.report("fileio","skipping %s for %s from %s",dataname,pathname,filename) + if trace_locating then + logs.report("fileio","skipping '%s' for '%s' from '%s'",dataname,pathname,filename) end instance[dataname][pathname] = { } instance.loaderror = true end - elseif trace_verbose then - logs.report("fileio","skipping %s for %s from %s",dataname,pathname,filename) + elseif trace_locating then + logs.report("fileio","skipping '%s' for '%s' from '%s'",dataname,pathname,filename) end end @@ -5144,15 +6015,17 @@ function resolvers.resetconfig() end function resolvers.loadnewconfig() - for _, cnf in ipairs(instance.luafiles) do + local luafiles = instance.luafiles + for i=1,#luafiles do + local cnf = luafiles[i] local pathname = file.dirname(cnf) local filename = file.join(pathname,resolvers.luaname) local blob = loadfile(filename) if blob then local data = blob() if data then - if trace_verbose then - logs.report("fileio","loading configuration file %s",filename) + if trace_locating then + logs.report("fileio","loading configuration file '%s'",filename) end if true then -- flatten to variable.progname @@ -5173,14 +6046,14 @@ function resolvers.loadnewconfig() instance['setup'][pathname] = data end else - if trace_verbose then - logs.report("fileio","skipping configuration file %s",filename) + if trace_locating then + logs.report("fileio","skipping configuration file '%s'",filename) end instance['setup'][pathname] = { } instance.loaderror = true end - elseif trace_verbose then - logs.report("fileio","skipping configuration file %s",filename) + elseif trace_locating then + logs.report("fileio","skipping configuration file '%s'",filename) end instance.order[#instance.order+1] = instance.setup[pathname] if instance.loaderror then break end @@ -5189,7 +6062,9 @@ end function resolvers.loadoldconfig() if not instance.renewcache then - for _, cnf in ipairs(instance.cnffiles) do + local cnffiles = instance.cnffiles + for i=1,#cnffiles do + local cnf = cnffiles[i] local dname = file.dirname(cnf) resolvers.load_data(dname,'configuration') instance.order[#instance.order+1] = instance.configuration[dname] @@ -5379,7 +6254,7 @@ end function resolvers.expanded_path_list(str) if not str then - return ep or { } + return ep or { } -- ep ? elseif instance.savelists then -- engine+progname hash str = gsub(str,"%$","") @@ -5397,9 +6272,9 @@ end function resolvers.expanded_path_list_from_var(str) -- brrr local tmp = resolvers.var_of_format_or_suffix(gsub(str,"%$","")) if tmp ~= "" then - return resolvers.expanded_path_list(str) - else return resolvers.expanded_path_list(tmp) + else + return resolvers.expanded_path_list(str) end end @@ -5446,9 +6321,9 @@ function resolvers.isreadable.file(name) local readable = lfs.isfile(name) -- brrr if trace_detail then if readable then - logs.report("fileio","+ readable: %s",name) + logs.report("fileio","file '%s' is readable",name) else - logs.report("fileio","- readable: %s", name) + logs.report("fileio","file '%s' is not readable", name) end end return readable @@ -5464,7 +6339,7 @@ local function collect_files(names) for k=1,#names do local fname = names[k] if trace_detail then - logs.report("fileio","? blobpath asked: %s",fname) + logs.report("fileio","checking name '%s'",fname) end local bname = file.basename(fname) local dname = file.dirname(fname) @@ -5480,7 +6355,7 @@ local function collect_files(names) local files = blobpath and instance.files[blobpath] if files then if trace_detail then - logs.report("fileio",'? blobpath do: %s (%s)',blobpath,bname) + logs.report("fileio","deep checking '%s' (%s)",blobpath,bname) end local blobfile = files[bname] if not blobfile then @@ -5514,7 +6389,7 @@ local function collect_files(names) end end elseif trace_locating then - logs.report("fileio",'! blobpath no: %s (%s)',blobpath,bname) + logs.report("fileio","no match in '%s' (%s)",blobpath,bname) end end end @@ -5564,14 +6439,13 @@ end local function collect_instance_files(filename,collected) -- todo : plugin (scanners, checkers etc) local result = collected or { } local stamp = nil - filename = file.collapse_path(filename) -- elsewhere - filename = file.collapse_path(gsub(filename,"\\","/")) -- elsewhere + filename = file.collapse_path(filename) -- speed up / beware: format problem if instance.remember then stamp = filename .. "--" .. instance.engine .. "--" .. instance.progname .. "--" .. instance.format if instance.found[stamp] then if trace_locating then - logs.report("fileio",'! remembered: %s',filename) + logs.report("fileio","remembering file '%s'",filename) end return instance.found[stamp] end @@ -5579,7 +6453,7 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan if not dangerous[instance.format or "?"] then if resolvers.isreadable.file(filename) then if trace_detail then - logs.report("fileio",'= found directly: %s',filename) + logs.report("fileio","file '%s' found directly",filename) end instance.found[stamp] = { filename } return { filename } @@ -5587,13 +6461,13 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan end if find(filename,'%*') then if trace_locating then - logs.report("fileio",'! wildcard: %s', filename) + logs.report("fileio","checking wildcard '%s'", filename) end result = resolvers.find_wildcard_files(filename) elseif file.is_qualified_path(filename) then if resolvers.isreadable.file(filename) then if trace_locating then - logs.report("fileio",'! qualified: %s', filename) + logs.report("fileio","qualified name '%s'", filename) end result = { filename } else @@ -5603,7 +6477,7 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan forcedname = filename .. ".tex" if resolvers.isreadable.file(forcedname) then if trace_locating then - logs.report("fileio",'! no suffix, forcing standard filetype: tex') + logs.report("fileio","no suffix, forcing standard filetype 'tex'") end result, ok = { forcedname }, true end @@ -5613,7 +6487,7 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan forcedname = filename .. "." .. s if resolvers.isreadable.file(forcedname) then if trace_locating then - logs.report("fileio",'! no suffix, forcing format filetype: %s', s) + logs.report("fileio","no suffix, forcing format filetype '%s'", s) end result, ok = { forcedname }, true break @@ -5625,7 +6499,7 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan -- try to find in tree (no suffix manipulation), here we search for the -- matching last part of the name local basename = file.basename(filename) - local pattern = (filename .. "$"):gsub("([%.%-])","%%%1") + local pattern = gsub(filename .. "$","([%.%-])","%%%1") local savedformat = instance.format local format = savedformat or "" if format == "" then @@ -5635,19 +6509,21 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan instance.format = "othertextfiles" -- kind of everything, maybe texinput is better end -- - local resolved = collect_instance_files(basename) - if #result == 0 then - local lowered = lower(basename) - if filename ~= lowered then - resolved = collect_instance_files(lowered) + if basename ~= filename then + local resolved = collect_instance_files(basename) + if #result == 0 then + local lowered = lower(basename) + if filename ~= lowered then + resolved = collect_instance_files(lowered) + end end - end - resolvers.format = savedformat - -- - for r=1,#resolved do - local rr = resolved[r] - if rr:find(pattern) then - result[#result+1], ok = rr, true + resolvers.format = savedformat + -- + for r=1,#resolved do + local rr = resolved[r] + if find(rr,pattern) then + result[#result+1], ok = rr, true + end end end -- a real wildcard: @@ -5656,14 +6532,14 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan -- local filelist = collect_files({basename}) -- for f=1,#filelist do -- local ff = filelist[f][3] or "" - -- if ff:find(pattern) then + -- if find(ff,pattern) then -- result[#result+1], ok = ff, true -- end -- end -- end end if not ok and trace_locating then - logs.report("fileio",'? qualified: %s', filename) + logs.report("fileio","qualified name '%s'", filename) end end else @@ -5682,12 +6558,12 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan wantedfiles[#wantedfiles+1] = forcedname filetype = resolvers.format_of_suffix(forcedname) if trace_locating then - logs.report("fileio",'! forcing filetype: %s',filetype) + logs.report("fileio","forcing filetype '%s'",filetype) end else filetype = resolvers.format_of_suffix(filename) if trace_locating then - logs.report("fileio",'! using suffix based filetype: %s',filetype) + logs.report("fileio","using suffix based filetype '%s'",filetype) end end else @@ -5699,7 +6575,7 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan end filetype = instance.format if trace_locating then - logs.report("fileio",'! using given filetype: %s',filetype) + logs.report("fileio","using given filetype '%s'",filetype) end end local typespec = resolvers.variable_of_format(filetype) @@ -5707,9 +6583,7 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan if not pathlist or #pathlist == 0 then -- no pathlist, access check only / todo == wildcard if trace_detail then - logs.report("fileio",'? filename: %s',filename) - logs.report("fileio",'? filetype: %s',filetype or '?') - logs.report("fileio",'? wanted files: %s',concat(wantedfiles," | ")) + logs.report("fileio","checking filename '%s', filetype '%s', wanted files '%s'",filename, filetype or '?',concat(wantedfiles," | ")) end for k=1,#wantedfiles do local fname = wantedfiles[k] @@ -5730,36 +6604,59 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan else -- list search local filelist = collect_files(wantedfiles) - local doscan, recurse + local dirlist = { } + if filelist then + for i=1,#filelist do + dirlist[i] = file.dirname(filelist[i][2]) .. "/" + end + end if trace_detail then - logs.report("fileio",'? filename: %s',filename) + logs.report("fileio","checking filename '%s'",filename) end -- a bit messy ... esp the doscan setting here + local doscan for k=1,#pathlist do local path = pathlist[k] if find(path,"^!!") then doscan = false else doscan = true end - if find(path,"//$") then recurse = true else recurse = false end local pathname = gsub(path,"^!+", '') done = false -- using file list - if filelist and not (done and not instance.allresults) and recurse then - -- compare list entries with permitted pattern - pathname = gsub(pathname,"([%-%.])","%%%1") -- this also influences - pathname = gsub(pathname,"/+$", '/.*') -- later usage of pathname - pathname = gsub(pathname,"//", '/.-/') -- not ok for /// but harmless - local expr = "^" .. pathname + if filelist then + local expression + -- compare list entries with permitted pattern -- /xx /xx// + if not find(pathname,"/$") then + expression = pathname .. "/" + else + expression = pathname + end + expression = gsub(expression,"([%-%.])","%%%1") -- this also influences + expression = gsub(expression,"//+$", '/.*') -- later usage of pathname + expression = gsub(expression,"//", '/.-/') -- not ok for /// but harmless + expression = "^" .. expression .. "$" + if trace_detail then + logs.report("fileio","using pattern '%s' for path '%s'",expression,pathname) + end for k=1,#filelist do local fl = filelist[k] local f = fl[2] - if find(f,expr) then - if trace_detail then - logs.report("fileio",'= found in hash: %s',f) - end + local d = dirlist[k] + if find(d,expression) then --- todo, test for readable result[#result+1] = fl[3] resolvers.register_in_trees(f) -- for tracing used files done = true - if not instance.allresults then break end + if instance.allresults then + if trace_detail then + logs.report("fileio","match in hash for file '%s' on path '%s', continue scanning",f,d) + end + else + if trace_detail then + logs.report("fileio","match in hash for file '%s' on path '%s', quit scanning",f,d) + end + break + end + elseif trace_detail then + logs.report("fileio","no match in hash for file '%s' on path '%s'",f,d) end end end @@ -5775,7 +6672,7 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan local fname = file.join(ppname,w) if resolvers.isreadable.file(fname) then if trace_detail then - logs.report("fileio",'= found by scanning: %s',fname) + logs.report("fileio","found '%s' by scanning",fname) end result[#result+1] = fname done = true @@ -5838,7 +6735,7 @@ function resolvers.find_given_files(filename) local hashes = instance.hashes for k=1,#hashes do local hash = hashes[k] - local files = instance.files[hash.tag] + local files = instance.files[hash.tag] or { } local blist = files[bname] if not blist then local rname = "remap:"..bname @@ -5948,9 +6845,9 @@ function resolvers.load(option) statistics.starttiming(instance) resolvers.resetconfig() resolvers.identify_cnf() - resolvers.load_lua() + resolvers.load_lua() -- will become the new method resolvers.expand_variables() - resolvers.load_cnf() + resolvers.load_cnf() -- will be skipped when we have a lua file resolvers.expand_variables() if option ~= "nofiles" then resolvers.load_hash() @@ -5962,22 +6859,23 @@ end function resolvers.for_files(command, files, filetype, mustexist) if files and #files > 0 then local function report(str) - if trace_verbose then + if trace_locating then logs.report("fileio",str) -- has already verbose else print(str) end end - if trace_verbose then - report('') + if trace_locating then + report('') -- ? end - for _, file in ipairs(files) do + for f=1,#files do + local file = files[f] local result = command(file,filetype,mustexist) if type(result) == 'string' then report(result) else - for _,v in ipairs(result) do - report(v) + for i=1,#result do + report(result[i]) -- could be unpack end end end @@ -6024,18 +6922,19 @@ end function table.sequenced(t,sep) -- temp here local s = { } - for k, v in pairs(t) do -- pairs? - s[#s+1] = k .. "=" .. v + for k, v in next, t do -- indexed? + s[#s+1] = k .. "=" .. tostring(v) end return concat(s, sep or " | ") end function resolvers.methodhandler(what, filename, filetype) -- ... + filename = file.collapse_path(filename) local specification = (type(filename) == "string" and resolvers.splitmethod(filename)) or filename -- no or { }, let it bomb local scheme = specification.scheme if resolvers[what][scheme] then if trace_locating then - logs.report("fileio",'= handler: %s -> %s -> %s',specification.original,what,table.sequenced(specification)) + logs.report("fileio","handler '%s' -> '%s' -> '%s'",specification.original,what,table.sequenced(specification)) end return resolvers[what][scheme](filename,filetype) -- todo: specification else @@ -6055,8 +6954,9 @@ function resolvers.clean_path(str) end function resolvers.do_with_path(name,func) - for _, v in pairs(resolvers.expanded_path_list(name)) do -- pairs? - func("^"..resolvers.clean_path(v)) + local pathlist = resolvers.expanded_path_list(name) + for i=1,#pathlist do + func("^"..resolvers.clean_path(pathlist[i])) end end @@ -6065,7 +6965,9 @@ function resolvers.do_with_var(name,func) end function resolvers.with_files(pattern,handle) - for _, hash in ipairs(instance.hashes) do + local hashes = instance.hashes + for i=1,#hashes do + local hash = hashes[i] local blobpath = hash.tag local blobtype = hash.type if blobpath then @@ -6080,7 +6982,7 @@ function resolvers.with_files(pattern,handle) if type(v) == "string" then handle(blobtype,blobpath,v,k) else - for _,vv in pairs(v) do -- ipairs? + for _,vv in next, v do -- indexed handle(blobtype,blobpath,vv,k) end end @@ -6092,7 +6994,7 @@ function resolvers.with_files(pattern,handle) end function resolvers.locate_format(name) - local barename, fmtname = name:gsub("%.%a+$",""), "" + local barename, fmtname = gsub(name,"%.%a+$",""), "" if resolvers.usecache then local path = file.join(caches.setpath("formats")) -- maybe platform fmtname = file.join(path,barename..".fmt") or "" @@ -6140,7 +7042,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-tmp'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -6164,7 +7066,7 @@ luatools with a recache feature.</p> local format, lower, gsub = string.format, string.lower, string.gsub -local trace_cache = false trackers.register("resolvers.cache", function(v) trace_cache = v end) +local trace_cache = false trackers.register("resolvers.cache", function(v) trace_cache = v end) -- not used yet caches = caches or { } @@ -6251,7 +7153,8 @@ function caches.setpath(...) caches.path = '.' end caches.path = resolvers.clean_path(caches.path) - if not table.is_empty({...}) then + local dirs = { ... } + if #dirs > 0 then local pth = dir.mkdirs(caches.path,...) return pth end @@ -6297,6 +7200,7 @@ function caches.savedata(filepath,filename,data,raw) if raw then reduce, simplify = false, false end + data.cache_uuid = os.uuid() if caches.direct then file.savedata(tmaname, table.serialize(data,'return',false,true,false)) -- no hex else @@ -6322,7 +7226,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-inp'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -6343,7 +7247,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-out'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -6359,7 +7263,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-con'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -6370,8 +7274,6 @@ local format, lower, gsub = string.format, string.lower, string.gsub local trace_cache = false trackers.register("resolvers.cache", function(v) trace_cache = v end) local trace_containers = false trackers.register("resolvers.containers", function(v) trace_containers = v end) local trace_storage = false trackers.register("resolvers.storage", function(v) trace_storage = v end) -local trace_verbose = false trackers.register("resolvers.verbose", function(v) trace_verbose = v end) -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v trackers.enable("resolvers.verbose") end) --[[ldx-- <p>Once we found ourselves defining similar cache constructs @@ -6435,7 +7337,7 @@ end function containers.is_valid(container, name) if name and name ~= "" then local storage = container.storage[name] - return storage and not table.is_empty(storage) and storage.cache_version == container.version + return storage and storage.cache_version == container.version else return false end @@ -6487,16 +7389,15 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-use'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -local format, lower, gsub = string.format, string.lower, string.gsub +local format, lower, gsub, find = string.format, string.lower, string.gsub, string.find -local trace_verbose = false trackers.register("resolvers.verbose", function(v) trace_verbose = v end) -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v trackers.enable("resolvers.verbose") end) +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) -- since we want to use the cache instead of the tree, we will now -- reimplement the saver. @@ -6540,19 +7441,20 @@ resolvers.automounted = resolvers.automounted or { } function resolvers.automount(usecache) local mountpaths = resolvers.clean_path_list(resolvers.expansion('TEXMFMOUNT')) - if table.is_empty(mountpaths) and usecache then + if (not mountpaths or #mountpaths == 0) and usecache then mountpaths = { caches.setpath("mount") } end - if not table.is_empty(mountpaths) then + if mountpaths and #mountpaths > 0 then statistics.starttiming(resolvers.instance) - for k, root in pairs(mountpaths) do + for k=1,#mountpaths do + local root = mountpaths[k] local f = io.open(root.."/url.tmi") if f then for line in f:lines() do if line then - if line:find("^[%%#%-]") then -- or %W + if find(line,"^[%%#%-]") then -- or %W -- skip - elseif line:find("^zip://") then + elseif find(line,"^zip://") then if trace_locating then logs.report("fileio","mounting %s",line) end @@ -6597,11 +7499,13 @@ function statistics.check_fmt_status(texname) local luv = dofile(luvname) if luv and luv.sourcefile then local sourcehash = md5.hex(io.loaddata(resolvers.find_file(luv.sourcefile)) or "unknown") - if luv.enginebanner and luv.enginebanner ~= enginebanner then - return "engine mismatch" + local luvbanner = luv.enginebanner or "?" + if luvbanner ~= enginebanner then + return string.format("engine mismatch (luv:%s <> bin:%s)",luvbanner,enginebanner) end - if luv.sourcehash and luv.sourcehash ~= sourcehash then - return "source mismatch" + local luvhash = luv.sourcehash or "?" + if luvhash ~= sourcehash then + return string.format("source mismatch (luv:%s <> bin:%s)",luvhash,sourcehash) end else return "invalid status file" @@ -6727,7 +7631,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-aux'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -6735,47 +7639,47 @@ if not modules then modules = { } end modules ['data-aux'] = { local find = string.find -local trace_verbose = false trackers.register("resolvers.verbose", function(v) trace_verbose = v end) +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) function resolvers.update_script(oldname,newname) -- oldname -> own.name, not per se a suffix local scriptpath = "scripts/context/lua" newname = file.addsuffix(newname,"lua") local oldscript = resolvers.clean_path(oldname) - if trace_verbose then + if trace_locating then logs.report("fileio","to be replaced old script %s", oldscript) end local newscripts = resolvers.find_files(newname) or { } if #newscripts == 0 then - if trace_verbose then + if trace_locating then logs.report("fileio","unable to locate new script") end else for i=1,#newscripts do local newscript = resolvers.clean_path(newscripts[i]) - if trace_verbose then + if trace_locating then logs.report("fileio","checking new script %s", newscript) end if oldscript == newscript then - if trace_verbose then + if trace_locating then logs.report("fileio","old and new script are the same") end elseif not find(newscript,scriptpath) then - if trace_verbose then + if trace_locating then logs.report("fileio","new script should come from %s",scriptpath) end elseif not (find(oldscript,file.removesuffix(newname).."$") or find(oldscript,newname.."$")) then - if trace_verbose then + if trace_locating then logs.report("fileio","invalid new script name") end else local newdata = io.loaddata(newscript) if newdata then - if trace_verbose then + if trace_locating then logs.report("fileio","old script content replaced by new content") end io.savedata(oldscript,newdata) break - elseif trace_verbose then + elseif trace_locating then logs.report("fileio","unable to load new script") end end @@ -6790,7 +7694,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-lst'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -6814,7 +7718,9 @@ local function list(list,report) local instance = resolvers.instance local pat = upper(pattern or "","") local report = report or texio.write_nl - for _,key in pairs(table.sortedkeys(list)) do + local sorted = table.sortedkeys(list) + for i=1,#sorted do + local key = sorted[i] if instance.pattern == "" or find(upper(key),pat) then if instance.kpseonly then if instance.kpsevars[key] then @@ -6833,11 +7739,14 @@ function resolvers.listers.expansions() list(resolvers.instance.expansions) end function resolvers.listers.configurations(report) local report = report or texio.write_nl local instance = resolvers.instance - for _,key in ipairs(table.sortedkeys(instance.kpsevars)) do + local sorted = table.sortedkeys(instance.kpsevars) + for i=1,#sorted do + local key = sorted[i] if not instance.pattern or (instance.pattern=="") or find(key,instance.pattern) then report(format("%s\n",key)) - for i,c in ipairs(instance.order) do - local str = c[key] + local order = instance.order + for i=1,#order do + local str = order[i][key] if str then report(format("\t%s\t%s",i,str)) end @@ -6943,7 +7852,7 @@ if not resolvers then os.exit() end -logs.setprogram('LuaTools',"TDS Management Tool 1.31",environment.arguments["verbose"] or false) +logs.setprogram('LuaTools',"TDS Management Tool 1.32",environment.arguments["verbose"] or false) local instance = resolvers.reset() @@ -7000,6 +7909,12 @@ end if environment.arguments["trace"] then resolvers.settrace(environment.arguments["trace"]) end +local trackspec = environment.argument("trackers") or environment.argument("track") + +if trackspec then + trackers.enable(trackspec) +end + runners = runners or { } messages = messages or { } @@ -7033,6 +7948,7 @@ messages.help = [[ --engine=str target engine --progname=str format or backend --pattern=str filter variables +--trackers=list enable given trackers ]] function runners.make_format(texname) @@ -7091,8 +8007,9 @@ function runners.make_format(texname) logs.simple("using uncompiled initialization file: %s",luaname) end else - for _, v in pairs({instance.luaname, instance.progname, barename}) do - v = string.gsub(v..".lua","%.lua%.lua$",".lua") + local what = { instance.luaname, instance.progname, barename } + for k=1,#what do + local v = string.gsub(what[k]..".lua","%.lua%.lua$",".lua") if v and (v ~= "") then luaname = resolvers.find_files(v)[1] or "" if luaname ~= "" then @@ -7116,7 +8033,8 @@ function runners.make_format(texname) logs.simple("using lua initialization file: %s",luaname) local mp = dir.glob(file.removesuffix(file.basename(luaname)).."-*.mem") if mp and #mp > 0 then - for _, name in ipairs(mp) do + for i=1,#mp do + local name = mp[i] logs.simple("removing related mplib format %s", file.basename(name)) os.remove(name) end diff --git a/Master/texmf-dist/scripts/context/stubs/unix/makempy b/Master/texmf-dist/scripts/context/stubs/unix/makempy deleted file mode 100755 index 34892b28460..00000000000 --- a/Master/texmf-dist/scripts/context/stubs/unix/makempy +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -mtxrun --usekpse --execute makempy.pl "$@" diff --git a/Master/texmf-dist/scripts/context/stubs/unix/mpstools b/Master/texmf-dist/scripts/context/stubs/unix/mpstools deleted file mode 100755 index 1a64d90b03f..00000000000 --- a/Master/texmf-dist/scripts/context/stubs/unix/mpstools +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -mtxrun --usekpse --execute mpstools.rb "$@" diff --git a/Master/texmf-dist/scripts/context/stubs/unix/mptopdf b/Master/texmf-dist/scripts/context/stubs/unix/mptopdf deleted file mode 100755 index f57a8b7a792..00000000000 --- a/Master/texmf-dist/scripts/context/stubs/unix/mptopdf +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -mtxrun --usekpse --execute mptopdf.pl "$@" diff --git a/Master/texmf-dist/scripts/context/stubs/unix/mtxrun b/Master/texmf-dist/scripts/context/stubs/unix/mtxrun index 82d1edecbc5..b99327692d7 100644 --- a/Master/texmf-dist/scripts/context/stubs/unix/mtxrun +++ b/Master/texmf-dist/scripts/context/stubs/unix/mtxrun @@ -48,13 +48,16 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-string'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -local sub, gsub, find, match, gmatch, format, char, byte, rep = string.sub, string.gsub, string.find, string.match, string.gmatch, string.format, string.char, string.byte, string.rep +local sub, gsub, find, match, gmatch, format, char, byte, rep, lower = string.sub, string.gsub, string.find, string.match, string.gmatch, string.format, string.char, string.byte, string.rep, string.lower +local lpegmatch = lpeg.match + +-- some functions may disappear as they are not used anywhere if not string.split then @@ -94,8 +97,16 @@ function string:unquote() return (gsub(self,"^([\"\'])(.*)%1$","%2")) end +--~ function string:unquote() +--~ if find(self,"^[\'\"]") then +--~ return sub(self,2,-2) +--~ else +--~ return self +--~ end +--~ end + function string:quote() -- we could use format("%q") - return '"' .. self:unquote() .. '"' + return format("%q",self) end function string:count(pattern) -- variant 3 @@ -115,12 +126,23 @@ function string:limit(n,sentinel) end end -function string:strip() - return (gsub(self,"^%s*(.-)%s*$", "%1")) +--~ function string:strip() -- the .- is quite efficient +--~ -- return match(self,"^%s*(.-)%s*$") or "" +--~ -- return match(self,'^%s*(.*%S)') or '' -- posted on lua list +--~ return find(s,'^%s*$') and '' or match(s,'^%s*(.*%S)') +--~ end + +do -- roberto's variant: + local space = lpeg.S(" \t\v\n") + local nospace = 1 - space + local stripper = space^0 * lpeg.C((space^0 * nospace^1)^0) + function string.strip(str) + return lpegmatch(stripper,str) or "" + end end function string:is_empty() - return not find(find,"%S") + return not find(self,"%S") end function string:enhance(pattern,action) @@ -154,14 +176,14 @@ if not string.characters then local function nextchar(str, index) index = index + 1 - return (index <= #str) and index or nil, str:sub(index,index) + return (index <= #str) and index or nil, sub(str,index,index) end function string:characters() return nextchar, self, 0 end local function nextbyte(str, index) index = index + 1 - return (index <= #str) and index or nil, byte(str:sub(index,index)) + return (index <= #str) and index or nil, byte(sub(str,index,index)) end function string:bytes() return nextbyte, self, 0 @@ -174,7 +196,7 @@ end function string:rpadd(n,chr) local m = n-#self if m > 0 then - return self .. self.rep(chr or " ",m) + return self .. rep(chr or " ",m) else return self end @@ -183,7 +205,7 @@ end function string:lpadd(n,chr) local m = n-#self if m > 0 then - return self.rep(chr or " ",m) .. self + return rep(chr or " ",m) .. self else return self end @@ -231,6 +253,17 @@ function string:pattesc() return (gsub(self,".",patterns_escapes)) end +local simple_escapes = { + ["-"] = "%-", + ["."] = "%.", + ["?"] = ".", + ["*"] = ".*", +} + +function string:simpleesc() + return (gsub(self,".",simple_escapes)) +end + function string:tohash() local t = { } for s in gmatch(self,"([^, ]+)") do -- lpeg @@ -242,10 +275,10 @@ end local pattern = lpeg.Ct(lpeg.C(1)^0) function string:totable() - return pattern:match(self) + return lpegmatch(pattern,self) end ---~ for _, str in ipairs { +--~ local t = { --~ "1234567123456712345671234567", --~ "a\tb\tc", --~ "aa\tbb\tcc", @@ -253,7 +286,10 @@ end --~ "aaaa\tbbbb\tcccc", --~ "aaaaa\tbbbbb\tccccc", --~ "aaaaaa\tbbbbbb\tcccccc", ---~ } do print(string.tabtospace(str)) end +--~ } +--~ for k,v do +--~ print(string.tabtospace(t[k])) +--~ end function string.tabtospace(str,tab) -- we don't handle embedded newlines @@ -261,7 +297,7 @@ function string.tabtospace(str,tab) local s = find(str,"\t") if s then if not tab then tab = 7 end -- only when found - local d = tab-(s-1)%tab + local d = tab-(s-1) % tab if d > 0 then str = gsub(str,"\t",rep(" ",d),1) else @@ -280,6 +316,25 @@ function string:compactlong() -- strips newlines and leading spaces return self end +function string:striplong() -- strips newlines and leading spaces + self = gsub(self,"^%s*","") + self = gsub(self,"[\n\r]+ *","\n") + return self +end + +function string:topattern(lowercase,strict) + if lowercase then + self = lower(self) + end + self = gsub(self,".",simple_escapes) + if self == "" then + self = ".*" + elseif strict then + self = "^" .. self .. "$" + end + return self +end + end -- of closure @@ -287,58 +342,64 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-lpeg'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -local P, S, Ct, C, Cs, Cc = lpeg.P, lpeg.S, lpeg.Ct, lpeg.C, lpeg.Cs, lpeg.Cc - ---~ l-lpeg.lua : - ---~ lpeg.digit = lpeg.R('09')^1 ---~ lpeg.sign = lpeg.S('+-')^1 ---~ lpeg.cardinal = lpeg.P(lpeg.sign^0 * lpeg.digit^1) ---~ lpeg.integer = lpeg.P(lpeg.sign^0 * lpeg.digit^1) ---~ lpeg.float = lpeg.P(lpeg.sign^0 * lpeg.digit^0 * lpeg.P('.') * lpeg.digit^1) ---~ lpeg.number = lpeg.float + lpeg.integer ---~ lpeg.oct = lpeg.P("0") * lpeg.R('07')^1 ---~ lpeg.hex = lpeg.P("0x") * (lpeg.R('09') + lpeg.R('AF'))^1 ---~ lpeg.uppercase = lpeg.P("AZ") ---~ lpeg.lowercase = lpeg.P("az") - ---~ lpeg.eol = lpeg.S('\r\n\f')^1 -- includes formfeed ---~ lpeg.space = lpeg.S(' ')^1 ---~ lpeg.nonspace = lpeg.P(1-lpeg.space)^1 ---~ lpeg.whitespace = lpeg.S(' \r\n\f\t')^1 ---~ lpeg.nonwhitespace = lpeg.P(1-lpeg.whitespace)^1 - -local hash = { } +local lpeg = require("lpeg") + +lpeg.patterns = lpeg.patterns or { } -- so that we can share +local patterns = lpeg.patterns + +local P, R, S, Ct, C, Cs, Cc, V = lpeg.P, lpeg.R, lpeg.S, lpeg.Ct, lpeg.C, lpeg.Cs, lpeg.Cc, lpeg.V +local match = lpeg.match + +local digit, sign = R('09'), S('+-') +local cr, lf, crlf = P("\r"), P("\n"), P("\r\n") +local utf8byte = R("\128\191") + +patterns.utf8byte = utf8byte +patterns.utf8one = R("\000\127") +patterns.utf8two = R("\194\223") * utf8byte +patterns.utf8three = R("\224\239") * utf8byte * utf8byte +patterns.utf8four = R("\240\244") * utf8byte * utf8byte * utf8byte + +patterns.digit = digit +patterns.sign = sign +patterns.cardinal = sign^0 * digit^1 +patterns.integer = sign^0 * digit^1 +patterns.float = sign^0 * digit^0 * P('.') * digit^1 +patterns.number = patterns.float + patterns.integer +patterns.oct = P("0") * R("07")^1 +patterns.octal = patterns.oct +patterns.HEX = P("0x") * R("09","AF")^1 +patterns.hex = P("0x") * R("09","af")^1 +patterns.hexadecimal = P("0x") * R("09","AF","af")^1 +patterns.lowercase = R("az") +patterns.uppercase = R("AZ") +patterns.letter = patterns.lowercase + patterns.uppercase +patterns.space = S(" ") +patterns.eol = S("\n\r") +patterns.spacer = S(" \t\f\v") -- + string.char(0xc2, 0xa0) if we want utf (cf mail roberto) +patterns.newline = crlf + cr + lf +patterns.nonspace = 1 - patterns.space +patterns.nonspacer = 1 - patterns.spacer +patterns.whitespace = patterns.eol + patterns.spacer +patterns.nonwhitespace = 1 - patterns.whitespace +patterns.utf8 = patterns.utf8one + patterns.utf8two + patterns.utf8three + patterns.utf8four +patterns.utfbom = P('\000\000\254\255') + P('\255\254\000\000') + P('\255\254') + P('\254\255') + P('\239\187\191') function lpeg.anywhere(pattern) --slightly adapted from website - return P { P(pattern) + 1 * lpeg.V(1) } -end - -function lpeg.startswith(pattern) --slightly adapted - return P(pattern) + return P { P(pattern) + 1 * V(1) } -- why so complex? end function lpeg.splitter(pattern, action) return (((1-P(pattern))^1)/action+1)^0 end --- variant: - ---~ local parser = lpeg.Ct(lpeg.splitat(newline)) - -local crlf = P("\r\n") -local cr = P("\r") -local lf = P("\n") -local space = S(" \t\f\v") -- + string.char(0xc2, 0xa0) if we want utf (cf mail roberto) -local newline = crlf + cr + lf -local spacing = space^0 * newline - +local spacing = patterns.spacer^0 * patterns.newline -- sort of strip local empty = spacing * Cc("") local nonempty = Cs((1-spacing)^1) * spacing^-1 local content = (empty + nonempty)^1 @@ -346,15 +407,15 @@ local content = (empty + nonempty)^1 local capture = Ct(content^0) function string:splitlines() - return capture:match(self) + return match(capture,self) end -lpeg.linebyline = content -- better make a sublibrary +patterns.textline = content ---~ local p = lpeg.splitat("->",false) print(p:match("oeps->what->more")) -- oeps what more ---~ local p = lpeg.splitat("->",true) print(p:match("oeps->what->more")) -- oeps what->more ---~ local p = lpeg.splitat("->",false) print(p:match("oeps")) -- oeps ---~ local p = lpeg.splitat("->",true) print(p:match("oeps")) -- oeps +--~ local p = lpeg.splitat("->",false) print(match(p,"oeps->what->more")) -- oeps what more +--~ local p = lpeg.splitat("->",true) print(match(p,"oeps->what->more")) -- oeps what->more +--~ local p = lpeg.splitat("->",false) print(match(p,"oeps")) -- oeps +--~ local p = lpeg.splitat("->",true) print(match(p,"oeps")) -- oeps local splitters_s, splitters_m = { }, { } @@ -364,7 +425,7 @@ local function splitat(separator,single) separator = P(separator) if single then local other, any = C((1 - separator)^0), P(1) - splitter = other * (separator * C(any^0) + "") + splitter = other * (separator * C(any^0) + "") -- ? splitters_s[separator] = splitter else local other = C((1 - separator)^0) @@ -379,15 +440,72 @@ lpeg.splitat = splitat local cache = { } +function lpeg.split(separator,str) + local c = cache[separator] + if not c then + c = Ct(splitat(separator)) + cache[separator] = c + end + return match(c,str) +end + function string:split(separator) local c = cache[separator] if not c then c = Ct(splitat(separator)) cache[separator] = c end - return c:match(self) + return match(c,self) +end + +lpeg.splitters = cache + +local cache = { } + +function lpeg.checkedsplit(separator,str) + local c = cache[separator] + if not c then + separator = P(separator) + local other = C((1 - separator)^0) + c = Ct(separator^0 * other * (separator^1 * other)^0) + cache[separator] = c + end + return match(c,str) +end + +function string:checkedsplit(separator) + local c = cache[separator] + if not c then + separator = P(separator) + local other = C((1 - separator)^0) + c = Ct(separator^0 * other * (separator^1 * other)^0) + cache[separator] = c + end + return match(c,self) end +--~ function lpeg.append(list,pp) +--~ local p = pp +--~ for l=1,#list do +--~ if p then +--~ p = p + P(list[l]) +--~ else +--~ p = P(list[l]) +--~ end +--~ end +--~ return p +--~ end + +--~ from roberto's site: + +local f1 = string.byte + +local function f2(s) local c1, c2 = f1(s,1,2) return c1 * 64 + c2 - 12416 end +local function f3(s) local c1, c2, c3 = f1(s,1,3) return (c1 * 64 + c2) * 64 + c3 - 925824 end +local function f4(s) local c1, c2, c3, c4 = f1(s,1,4) return ((c1 * 64 + c2) * 64 + c3) * 64 + c4 - 63447168 end + +patterns.utf8byte = patterns.utf8one/f1 + patterns.utf8two/f2 + patterns.utf8three/f3 + patterns.utf8four/f4 + end -- of closure @@ -395,7 +513,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-table'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -404,9 +522,58 @@ if not modules then modules = { } end modules ['l-table'] = { table.join = table.concat local concat, sort, insert, remove = table.concat, table.sort, table.insert, table.remove -local format, find, gsub, lower, dump = string.format, string.find, string.gsub, string.lower, string.dump +local format, find, gsub, lower, dump, match = string.format, string.find, string.gsub, string.lower, string.dump, string.match local getmetatable, setmetatable = getmetatable, setmetatable -local type, next, tostring, ipairs = type, next, tostring, ipairs +local type, next, tostring, tonumber, ipairs = type, next, tostring, tonumber, ipairs + +-- Starting with version 5.2 Lua no longer provide ipairs, which makes +-- sense. As we already used the for loop and # in most places the +-- impact on ConTeXt was not that large; the remaining ipairs already +-- have been replaced. In a similar fashio we also hardly used pairs. +-- +-- Just in case, we provide the fallbacks as discussed in Programming +-- in Lua (http://www.lua.org/pil/7.3.html): + +if not ipairs then + + -- for k, v in ipairs(t) do ... end + -- for k=1,#t do local v = t[k] ... end + + local function iterate(a,i) + i = i + 1 + local v = a[i] + if v ~= nil then + return i, v --, nil + end + end + + function ipairs(a) + return iterate, a, 0 + end + +end + +if not pairs then + + -- for k, v in pairs(t) do ... end + -- for k, v in next, t do ... end + + function pairs(t) + return next, t -- , nil + end + +end + +-- Also, unpack has been moved to the table table, and for compatiility +-- reasons we provide both now. + +if not table.unpack then + table.unpack = _G.unpack +elseif not unpack then + _G.unpack = table.unpack +end + +-- extra functions, some might go (when not used) function table.strip(tab) local lst = { } @@ -421,6 +588,14 @@ function table.strip(tab) return lst end +function table.keys(t) + local k = { } + for key, _ in next, t do + k[#k+1] = key + end + return k +end + local function compare(a,b) return (tostring(a) < tostring(b)) end @@ -464,7 +639,7 @@ end table.sortedkeys = sortedkeys table.sortedhashkeys = sortedhashkeys -function table.sortedpairs(t) +function table.sortedhash(t) local s = sortedhashkeys(t) -- maybe just sortedkeys local n = 0 local function kv(s) @@ -475,6 +650,8 @@ function table.sortedpairs(t) return kv, s end +table.sortedpairs = table.sortedhash + function table.append(t, list) for _,v in next, list do insert(t,v) @@ -583,7 +760,7 @@ end table.fastcopy = fastcopy table.copy = copy --- rougly: copy-loop : unpack : sub == 0.9 : 0.4 : 0.45 (so in critical apps, use unpack) +-- roughly: copy-loop : unpack : sub == 0.9 : 0.4 : 0.45 (so in critical apps, use unpack) function table.sub(t,i,j) return { unpack(t,i,j) } @@ -597,18 +774,18 @@ end -- slower than #t on indexed tables (#t only returns the size of the numerically indexed slice) -function table.is_empty(t) +function table.is_empty(t) -- obolete, use inline code instead return not t or not next(t) end -function table.one_entry(t) +function table.one_entry(t) -- obolete, use inline code instead local n = next(t) return n and not next(t,n) end -function table.starts_at(t) - return ipairs(t,1)(t,0) -end +--~ function table.starts_at(t) -- obsolete, not nice anyway +--~ return ipairs(t,1)(t,0) +--~ end function table.tohash(t,value) local h = { } @@ -686,6 +863,8 @@ end -- -- local propername = lpeg.P(lpeg.R("AZ","az","__") * lpeg.R("09","AZ","az", "__")^0 * lpeg.P(-1) ) +-- problem: there no good number_to_string converter with the best resolution + local function do_serialize(root,name,depth,level,indexed) if level > 0 then depth = depth .. " " @@ -708,8 +887,9 @@ local function do_serialize(root,name,depth,level,indexed) handle(format("%s{",depth)) end end + -- we could check for k (index) being number (cardinal) if root and next(root) then - local first, last = nil, 0 -- #root cannot be trusted here + local first, last = nil, 0 -- #root cannot be trusted here (will be ok in 5.2 when ipairs is gone) if compact then -- NOT: for k=1,#root do (we need to quit at nil) for k,v in ipairs(root) do -- can we use next? @@ -730,10 +910,10 @@ local function do_serialize(root,name,depth,level,indexed) if hexify then handle(format("%s 0x%04X,",depth,v)) else - handle(format("%s %s,",depth,v)) + handle(format("%s %s,",depth,v)) -- %.99g end elseif t == "string" then - if reduce and (find(v,"^[%-%+]?[%d]-%.?[%d+]$") == 1) then + if reduce and tonumber(v) then handle(format("%s %s,",depth,v)) else handle(format("%s %q,",depth,v)) @@ -770,29 +950,29 @@ local function do_serialize(root,name,depth,level,indexed) --~ if hexify then --~ handle(format("%s %s=0x%04X,",depth,key(k),v)) --~ else - --~ handle(format("%s %s=%s,",depth,key(k),v)) + --~ handle(format("%s %s=%s,",depth,key(k),v)) -- %.99g --~ end if type(k) == "number" then -- or find(k,"^%d+$") then if hexify then handle(format("%s [0x%04X]=0x%04X,",depth,k,v)) else - handle(format("%s [%s]=%s,",depth,k,v)) + handle(format("%s [%s]=%s,",depth,k,v)) -- %.99g end elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then if hexify then handle(format("%s %s=0x%04X,",depth,k,v)) else - handle(format("%s %s=%s,",depth,k,v)) + handle(format("%s %s=%s,",depth,k,v)) -- %.99g end else if hexify then handle(format("%s [%q]=0x%04X,",depth,k,v)) else - handle(format("%s [%q]=%s,",depth,k,v)) + handle(format("%s [%q]=%s,",depth,k,v)) -- %.99g end end elseif t == "string" then - if reduce and (find(v,"^[%-%+]?[%d]-%.?[%d+]$") == 1) then + if reduce and tonumber(v) then --~ handle(format("%s %s=%s,",depth,key(k),v)) if type(k) == "number" then -- or find(k,"^%d+$") then if hexify then @@ -1001,7 +1181,7 @@ function table.tofile(filename,root,name,reduce,noquotes,hexify) end end -local function flatten(t,f,complete) +local function flatten(t,f,complete) -- is this used? meybe a variant with next, ... for i=1,#t do local v = t[i] if type(v) == "table" then @@ -1030,6 +1210,24 @@ end table.flatten_one_level = table.unnest +-- a better one: + +local function flattened(t,f) + if not f then + f = { } + end + for k, v in next, t do + if type(v) == "table" then + flattened(v,f) + else + f[k] = v + end + end + return f +end + +table.flattened = flattened + -- the next three may disappear function table.remove_value(t,value) -- todo: n @@ -1165,7 +1363,7 @@ function table.clone(t,p) -- t is optional or nil or table elseif not t then t = { } end - setmetatable(t, { __index = function(_,key) return p[key] end }) + setmetatable(t, { __index = function(_,key) return p[key] end }) -- why not __index = p ? return t end @@ -1193,21 +1391,36 @@ function table.reverse(t) return tt end ---~ function table.keys(t) ---~ local k = { } ---~ for k,_ in next, t do ---~ k[#k+1] = k ---~ end ---~ return k ---~ end +function table.insert_before_value(t,value,extra) + for i=1,#t do + if t[i] == extra then + remove(t,i) + end + end + for i=1,#t do + if t[i] == value then + insert(t,i,extra) + return + end + end + insert(t,1,extra) +end + +function table.insert_after_value(t,value,extra) + for i=1,#t do + if t[i] == extra then + remove(t,i) + end + end + for i=1,#t do + if t[i] == value then + insert(t,i+1,extra) + return + end + end + insert(t,#t+1,extra) +end ---~ function table.keys_as_string(t) ---~ local k = { } ---~ for k,_ in next, t do ---~ k[#k+1] = k ---~ end ---~ return concat(k,"") ---~ end end -- of closure @@ -1216,13 +1429,13 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-io'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -local byte = string.byte +local byte, find, gsub = string.byte, string.find, string.gsub if string.find(os.getenv("PATH"),";") then io.fileseparator, io.pathseparator = "\\", ";" @@ -1251,7 +1464,7 @@ function io.savedata(filename,data,joiner) elseif type(data) == "function" then data(f) else - f:write(data) + f:write(data or "") end f:close() return true @@ -1380,20 +1593,21 @@ function io.ask(question,default,options) end io.write(string.format(" ")) local answer = io.read() - answer = answer:gsub("^%s*(.*)%s*$","%1") + answer = gsub(answer,"^%s*(.*)%s*$","%1") if answer == "" and default then return default elseif not options then return answer else - for _,v in pairs(options) do - if v == answer then + for k=1,#options do + if options[k] == answer then return answer end end local pattern = "^" .. answer - for _,v in pairs(options) do - if v:find(pattern) then + for k=1,#options do + local v = options[k] + if find(v,pattern) then return v end end @@ -1408,20 +1622,22 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-number'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -local format = string.format +local tostring = tostring +local format, floor, insert, match = string.format, math.floor, table.insert, string.match +local lpegmatch = lpeg.match number = number or { } -- a,b,c,d,e,f = number.toset(100101) function number.toset(n) - return (tostring(n)):match("(.?)(.?)(.?)(.?)(.?)(.?)(.?)(.?)") + return match(tostring(n),"(.?)(.?)(.?)(.?)(.?)(.?)(.?)(.?)") end function number.toevenhex(n) @@ -1447,10 +1663,21 @@ end local one = lpeg.C(1-lpeg.S(''))^1 function number.toset(n) - return one:match(tostring(n)) + return lpegmatch(one,tostring(n)) end - +function number.bits(n,zero) + local t, i = { }, (zero and 0) or 1 + while n > 0 do + local m = n % 2 + if m > 0 then + insert(t,1,i) + end + n = floor(n/2) + i = i + 1 + end + return t +end end -- of closure @@ -1459,7 +1686,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-set'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -1549,46 +1776,63 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-os'] = { version = 1.001, - comment = "companion to luat-lub.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -local find = string.find +-- maybe build io.flush in os.execute + +local find, format, gsub = string.find, string.format, string.gsub +local random, ceil = math.random, math.ceil + +local execute, spawn, exec, ioflush = os.execute, os.spawn or os.execute, os.exec or os.execute, io.flush + +function os.execute(...) ioflush() return execute(...) end +function os.spawn (...) ioflush() return spawn (...) end +function os.exec (...) ioflush() return exec (...) end function os.resultof(command) - return io.popen(command,"r"):read("*all") + ioflush() -- else messed up logging + local handle = io.popen(command,"r") + if not handle then + -- print("unknown command '".. command .. "' in os.resultof") + return "" + else + return handle:read("*all") or "" + end end -if not os.exec then os.exec = os.execute end -if not os.spawn then os.spawn = os.execute end - ---~ os.type : windows | unix (new, we already guessed os.platform) ---~ os.name : windows | msdos | linux | macosx | solaris | .. | generic (new) +--~ os.type : windows | unix (new, we already guessed os.platform) +--~ os.name : windows | msdos | linux | macosx | solaris | .. | generic (new) +--~ os.platform : extended os.name with architecture if not io.fileseparator then if find(os.getenv("PATH"),";") then - io.fileseparator, io.pathseparator, os.platform = "\\", ";", os.type or "windows" + io.fileseparator, io.pathseparator, os.type = "\\", ";", os.type or "mswin" else - io.fileseparator, io.pathseparator, os.platform = "/" , ":", os.type or "unix" + io.fileseparator, io.pathseparator, os.type = "/" , ":", os.type or "unix" end end -os.platform = os.platform or os.type or (io.pathseparator == ";" and "windows") or "unix" +os.type = os.type or (io.pathseparator == ";" and "windows") or "unix" +os.name = os.name or (os.type == "windows" and "mswin" ) or "linux" + +if os.type == "windows" then + os.libsuffix, os.binsuffix = 'dll', 'exe' +else + os.libsuffix, os.binsuffix = 'so', '' +end function os.launch(str) - if os.platform == "windows" then + if os.type == "windows" then os.execute("start " .. str) -- os.spawn ? else os.execute(str .. " &") -- os.spawn ? end end -if not os.setenv then - function os.setenv() return false end -end - if not os.times then -- utime = user time -- stime = system time @@ -1618,64 +1862,218 @@ end --~ print(os.date("%H:%M:%S",os.gettimeofday())) --~ print(os.date("%H:%M:%S",os.time())) -os.arch = os.arch or function() - local a = os.resultof("uname -m") or "linux" - os.arch = function() - return a +-- no need for function anymore as we have more clever code and helpers now +-- this metatable trickery might as well disappear + +os.resolvers = os.resolvers or { } + +local resolvers = os.resolvers + +local osmt = getmetatable(os) or { __index = function(t,k) t[k] = "unset" return "unset" end } -- maybe nil +local osix = osmt.__index + +osmt.__index = function(t,k) + return (resolvers[k] or osix)(t,k) +end + +setmetatable(os,osmt) + +if not os.setenv then + + -- we still store them but they won't be seen in + -- child processes although we might pass them some day + -- using command concatination + + local env, getenv = { }, os.getenv + + function os.setenv(k,v) + env[k] = v + end + + function os.getenv(k) + return env[k] or getenv(k) end - return a + end -local platform +-- we can use HOSTTYPE on some platforms -function os.currentplatform(name,default) - if not platform then - local name = os.name or os.platform or name -- os.name is built in, os.platform is mine - if not name then - platform = default or "linux" - elseif name == "windows" or name == "mswin" or name == "win32" or name == "msdos" then - if os.getenv("PROCESSOR_ARCHITECTURE") == "AMD64" then - platform = "mswin-64" - else - platform = "mswin" - end +local name, platform = os.name or "linux", os.getenv("MTX_PLATFORM") or "" + +local function guess() + local architecture = os.resultof("uname -m") or "" + if architecture ~= "" then + return architecture + end + architecture = os.getenv("HOSTTYPE") or "" + if architecture ~= "" then + return architecture + end + return os.resultof("echo $HOSTTYPE") or "" +end + +if platform ~= "" then + + os.platform = platform + +elseif os.type == "windows" then + + -- we could set the variable directly, no function needed here + + function os.resolvers.platform(t,k) + local platform, architecture = "", os.getenv("PROCESSOR_ARCHITECTURE") or "" + if find(architecture,"AMD64") then + platform = "mswin-64" else - local architecture = os.arch() - if name == "linux" then - if find(architecture,"x86_64") then - platform = "linux-64" - elseif find(architecture,"ppc") then - platform = "linux-ppc" - else - platform = "linux" - end - elseif name == "macosx" then - if find(architecture,"i386") then - platform = "osx-intel" - else - platform = "osx-ppc" - end - elseif name == "sunos" then - if find(architecture,"sparc") then - platform = "solaris-sparc" - else -- if architecture == 'i86pc' - platform = "solaris-intel" - end - elseif name == "freebsd" then - if find(architecture,"amd64") then - platform = "freebsd-amd64" - else - platform = "freebsd" - end - else - platform = default or name - end + platform = "mswin" end - function os.currentplatform() - return platform + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +elseif name == "linux" then + + function os.resolvers.platform(t,k) + -- we sometims have HOSTTYPE set so let's check that first + local platform, architecture = "", os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" + if find(architecture,"x86_64") then + platform = "linux-64" + elseif find(architecture,"ppc") then + platform = "linux-ppc" + else + platform = "linux" + end + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +elseif name == "macosx" then + + --[[ + Identifying the architecture of OSX is quite a mess and this + is the best we can come up with. For some reason $HOSTTYPE is + a kind of pseudo environment variable, not known to the current + environment. And yes, uname cannot be trusted either, so there + is a change that you end up with a 32 bit run on a 64 bit system. + Also, some proper 64 bit intel macs are too cheap (low-end) and + therefore not permitted to run the 64 bit kernel. + ]]-- + + function os.resolvers.platform(t,k) + -- local platform, architecture = "", os.getenv("HOSTTYPE") or "" + -- if architecture == "" then + -- architecture = os.resultof("echo $HOSTTYPE") or "" + -- end + local platform, architecture = "", os.resultof("echo $HOSTTYPE") or "" + if architecture == "" then + -- print("\nI have no clue what kind of OSX you're running so let's assume an 32 bit intel.\n") + platform = "osx-intel" + elseif find(architecture,"i386") then + platform = "osx-intel" + elseif find(architecture,"x86_64") then + platform = "osx-64" + else + platform = "osx-ppc" end + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +elseif name == "sunos" then + + function os.resolvers.platform(t,k) + local platform, architecture = "", os.resultof("uname -m") or "" + if find(architecture,"sparc") then + platform = "solaris-sparc" + else -- if architecture == 'i86pc' + platform = "solaris-intel" + end + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +elseif name == "freebsd" then + + function os.resolvers.platform(t,k) + local platform, architecture = "", os.resultof("uname -m") or "" + if find(architecture,"amd64") then + platform = "freebsd-amd64" + else + platform = "freebsd" + end + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +elseif name == "kfreebsd" then + + function os.resolvers.platform(t,k) + -- we sometims have HOSTTYPE set so let's check that first + local platform, architecture = "", os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" + if find(architecture,"x86_64") then + platform = "kfreebsd-64" + else + platform = "kfreebsd-i386" + end + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +else + + -- platform = "linux" + -- os.setenv("MTX_PLATFORM",platform) + -- os.platform = platform + + function os.resolvers.platform(t,k) + local platform = "linux" + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +end + +-- beware, we set the randomseed + +-- from wikipedia: Version 4 UUIDs use a scheme relying only on random numbers. This algorithm sets the +-- version number as well as two reserved bits. All other bits are set using a random or pseudorandom +-- data source. Version 4 UUIDs have the form xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx with hexadecimal +-- digits x and hexadecimal digits 8, 9, A, or B for y. e.g. f47ac10b-58cc-4372-a567-0e02b2c3d479. +-- +-- as we don't call this function too often there is not so much risk on repetition + +local t = { 8, 9, "a", "b" } + +function os.uuid() + return format("%04x%04x-4%03x-%s%03x-%04x-%04x%04x%04x", + random(0xFFFF),random(0xFFFF), + random(0x0FFF), + t[ceil(random(4))] or 8,random(0x0FFF), + random(0xFFFF), + random(0xFFFF),random(0xFFFF),random(0xFFFF) + ) +end + +local d + +function os.timezone(delta) + d = d or tonumber(tonumber(os.date("%H")-os.date("!%H"))) + if delta then + if d > 0 then + return format("+%02i:00",d) + else + return format("-%02i:00",-d) + end + else + return 1 end - return platform end @@ -1685,7 +2083,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-file'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -1696,14 +2094,17 @@ if not modules then modules = { } end modules ['l-file'] = { file = file or { } local concat = table.concat -local find, gmatch, match, gsub = string.find, string.gmatch, string.match, string.gsub +local find, gmatch, match, gsub, sub, char = string.find, string.gmatch, string.match, string.gsub, string.sub, string.char +local lpegmatch = lpeg.match function file.removesuffix(filename) return (gsub(filename,"%.[%a%d]+$","")) end function file.addsuffix(filename, suffix) - if not find(filename,"%.[%a%d]+$") then + if not suffix or suffix == "" then + return filename + elseif not find(filename,"%.[%a%d]+$") then return filename .. "." .. suffix else return filename @@ -1726,20 +2127,39 @@ function file.nameonly(name) return (gsub(match(name,"^.+[/\\](.-)$") or name,"%..*$","")) end -function file.extname(name) - return match(name,"^.+%.([^/\\]-)$") or "" +function file.extname(name,default) + return match(name,"^.+%.([^/\\]-)$") or default or "" end file.suffix = file.extname ---~ print(file.join("x/","/y")) ---~ print(file.join("http://","/y")) ---~ print(file.join("http://a","/y")) ---~ print(file.join("http:///a","/y")) ---~ print(file.join("//nas-1","/y")) +--~ function file.join(...) +--~ local pth = concat({...},"/") +--~ pth = gsub(pth,"\\","/") +--~ local a, b = match(pth,"^(.*://)(.*)$") +--~ if a and b then +--~ return a .. gsub(b,"//+","/") +--~ end +--~ a, b = match(pth,"^(//)(.*)$") +--~ if a and b then +--~ return a .. gsub(b,"//+","/") +--~ end +--~ return (gsub(pth,"//+","/")) +--~ end + +local trick_1 = char(1) +local trick_2 = "^" .. trick_1 .. "/+" function file.join(...) - local pth = concat({...},"/") + local lst = { ... } + local a, b = lst[1], lst[2] + if a == "" then + lst[1] = trick_1 + elseif b and find(a,"^/+$") and find(b,"^/") then + lst[1] = "" + lst[2] = gsub(b,"^/+","") + end + local pth = concat(lst,"/") pth = gsub(pth,"\\","/") local a, b = match(pth,"^(.*://)(.*)$") if a and b then @@ -1749,17 +2169,28 @@ function file.join(...) if a and b then return a .. gsub(b,"//+","/") end + pth = gsub(pth,trick_2,"") return (gsub(pth,"//+","/")) end +--~ print(file.join("//","/y")) +--~ print(file.join("/","/y")) +--~ print(file.join("","/y")) +--~ print(file.join("/x/","/y")) +--~ print(file.join("x/","/y")) +--~ print(file.join("http://","/y")) +--~ print(file.join("http://a","/y")) +--~ print(file.join("http:///a","/y")) +--~ print(file.join("//nas-1","/y")) + function file.iswritable(name) local a = lfs.attributes(name) or lfs.attributes(file.dirname(name,".")) - return a and a.permissions:sub(2,2) == "w" + return a and sub(a.permissions,2,2) == "w" end function file.isreadable(name) local a = lfs.attributes(name) - return a and a.permissions:sub(1,1) == "r" + return a and sub(a.permissions,1,1) == "r" end file.is_readable = file.isreadable @@ -1767,36 +2198,50 @@ file.is_writable = file.iswritable -- todo: lpeg -function file.split_path(str) - local t = { } - str = gsub(str,"\\", "/") - str = gsub(str,"(%a):([;/])", "%1\001%2") - for name in gmatch(str,"([^;:]+)") do - if name ~= "" then - t[#t+1] = gsub(name,"\001",":") - end - end - return t +--~ function file.split_path(str) +--~ local t = { } +--~ str = gsub(str,"\\", "/") +--~ str = gsub(str,"(%a):([;/])", "%1\001%2") +--~ for name in gmatch(str,"([^;:]+)") do +--~ if name ~= "" then +--~ t[#t+1] = gsub(name,"\001",":") +--~ end +--~ end +--~ return t +--~ end + +local checkedsplit = string.checkedsplit + +function file.split_path(str,separator) + str = gsub(str,"\\","/") + return checkedsplit(str,separator or io.pathseparator) end function file.join_path(tab) return concat(tab,io.pathseparator) -- can have trailing // end +-- we can hash them weakly + function file.collapse_path(str) - str = gsub(str,"/%./","/") - local n, m = 1, 1 - while n > 0 or m > 0 do - str, n = gsub(str,"[^/%.]+/%.%.$","") - str, m = gsub(str,"[^/%.]+/%.%./","") - end - str = gsub(str,"([^/])/$","%1") - str = gsub(str,"^%./","") - str = gsub(str,"/%.$","") + str = gsub(str,"\\","/") + if find(str,"/") then + str = gsub(str,"^%./",(gsub(lfs.currentdir(),"\\","/")) .. "/") -- ./xx in qualified + str = gsub(str,"/%./","/") + local n, m = 1, 1 + while n > 0 or m > 0 do + str, n = gsub(str,"[^/%.]+/%.%.$","") + str, m = gsub(str,"[^/%.]+/%.%./","") + end + str = gsub(str,"([^/])/$","%1") + -- str = gsub(str,"^%./","") -- ./xx in qualified + str = gsub(str,"/%.$","") + end if str == "" then str = "." end return str end +--~ print(file.collapse_path("/a")) --~ print(file.collapse_path("a/./b/..")) --~ print(file.collapse_path("a/aa/../b/bb")) --~ print(file.collapse_path("a/../..")) @@ -1826,27 +2271,27 @@ end --~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * lpeg.C(noperiod^1) * -1 --~ function file.extname(name) ---~ return pattern:match(name) or "" +--~ return lpegmatch(pattern,name) or "" --~ end --~ local pattern = lpeg.Cs(((period * noperiod^1 * -1)/"" + 1)^1) --~ function file.removesuffix(name) ---~ return pattern:match(name) +--~ return lpegmatch(pattern,name) --~ end --~ local pattern = (noslashes^0 * slashes)^1 * lpeg.C(noslashes^1) * -1 --~ function file.basename(name) ---~ return pattern:match(name) or name +--~ return lpegmatch(pattern,name) or name --~ end --~ local pattern = (noslashes^0 * slashes)^1 * lpeg.Cp() * noslashes^1 * -1 --~ function file.dirname(name) ---~ local p = pattern:match(name) +--~ local p = lpegmatch(pattern,name) --~ if p then ---~ return name:sub(1,p-2) +--~ return sub(name,1,p-2) --~ else --~ return "" --~ end @@ -1855,7 +2300,7 @@ end --~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * lpeg.Cp() * noperiod^1 * -1 --~ function file.addsuffix(name, suffix) ---~ local p = pattern:match(name) +--~ local p = lpegmatch(pattern,name) --~ if p then --~ return name --~ else @@ -1866,9 +2311,9 @@ end --~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * lpeg.Cp() * noperiod^1 * -1 --~ function file.replacesuffix(name,suffix) ---~ local p = pattern:match(name) +--~ local p = lpegmatch(pattern,name) --~ if p then ---~ return name:sub(1,p-2) .. "." .. suffix +--~ return sub(name,1,p-2) .. "." .. suffix --~ else --~ return name .. "." .. suffix --~ end @@ -1877,11 +2322,11 @@ end --~ local pattern = (noslashes^0 * slashes)^0 * lpeg.Cp() * ((noperiod^1 * period)^1 * lpeg.Cp() + lpeg.P(true)) * noperiod^1 * -1 --~ function file.nameonly(name) ---~ local a, b = pattern:match(name) +--~ local a, b = lpegmatch(pattern,name) --~ if b then ---~ return name:sub(a,b-2) +--~ return sub(name,a,b-2) --~ elseif a then ---~ return name:sub(a) +--~ return sub(name,a) --~ else --~ return name --~ end @@ -1915,11 +2360,11 @@ local rootbased = lpeg.P("/") + letter*lpeg.P(":") -- ./name ../name /name c: :// name/name function file.is_qualified_path(filename) - return qualified:match(filename) + return lpegmatch(qualified,filename) ~= nil end function file.is_rootbased_path(filename) - return rootbased:match(filename) + return lpegmatch(rootbased,filename) ~= nil end local slash = lpeg.S("\\/") @@ -1932,16 +2377,25 @@ local base = lpeg.C((1-suffix)^0) local pattern = (drive + lpeg.Cc("")) * (path + lpeg.Cc("")) * (base + lpeg.Cc("")) * (suffix + lpeg.Cc("")) function file.splitname(str) -- returns drive, path, base, suffix - return pattern:match(str) + return lpegmatch(pattern,str) end --- function test(t) for k, v in pairs(t) do print(v, "=>", file.splitname(v)) end end +-- function test(t) for k, v in next, t do print(v, "=>", file.splitname(v)) end end -- -- test { "c:", "c:/aa", "c:/aa/bb", "c:/aa/bb/cc", "c:/aa/bb/cc.dd", "c:/aa/bb/cc.dd.ee" } -- test { "c:", "c:aa", "c:aa/bb", "c:aa/bb/cc", "c:aa/bb/cc.dd", "c:aa/bb/cc.dd.ee" } -- test { "/aa", "/aa/bb", "/aa/bb/cc", "/aa/bb/cc.dd", "/aa/bb/cc.dd.ee" } -- test { "aa", "aa/bb", "aa/bb/cc", "aa/bb/cc.dd", "aa/bb/cc.dd.ee" } +--~ -- todo: +--~ +--~ if os.type == "windows" then +--~ local currentdir = lfs.currentdir +--~ function lfs.currentdir() +--~ return (gsub(currentdir(),"\\","/")) +--~ end +--~ end + end -- of closure @@ -2006,7 +2460,7 @@ end function file.loadchecksum(name) if md5 then local data = io.loaddata(name .. ".md5") - return data and data:gsub("%s","") + return data and (gsub(data,"%s","")) end return nil end @@ -2025,19 +2479,168 @@ end -- of closure do -- create closure to overcome 200 locals limit +if not modules then modules = { } end modules ['l-url'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local char, gmatch, gsub = string.char, string.gmatch, string.gsub +local tonumber, type = tonumber, type +local lpegmatch = lpeg.match + +-- from the spec (on the web): +-- +-- foo://example.com:8042/over/there?name=ferret#nose +-- \_/ \______________/\_________/ \_________/ \__/ +-- | | | | | +-- scheme authority path query fragment +-- | _____________________|__ +-- / \ / \ +-- urn:example:animal:ferret:nose + +url = url or { } + +local function tochar(s) + return char(tonumber(s,16)) +end + +local colon, qmark, hash, slash, percent, endofstring = lpeg.P(":"), lpeg.P("?"), lpeg.P("#"), lpeg.P("/"), lpeg.P("%"), lpeg.P(-1) + +local hexdigit = lpeg.R("09","AF","af") +local plus = lpeg.P("+") +local escaped = (plus / " ") + (percent * lpeg.C(hexdigit * hexdigit) / tochar) + +-- we assume schemes with more than 1 character (in order to avoid problems with windows disks) + +local scheme = lpeg.Cs((escaped+(1-colon-slash-qmark-hash))^2) * colon + lpeg.Cc("") +local authority = slash * slash * lpeg.Cs((escaped+(1- slash-qmark-hash))^0) + lpeg.Cc("") +local path = slash * lpeg.Cs((escaped+(1- qmark-hash))^0) + lpeg.Cc("") +local query = qmark * lpeg.Cs((escaped+(1- hash))^0) + lpeg.Cc("") +local fragment = hash * lpeg.Cs((escaped+(1- endofstring))^0) + lpeg.Cc("") + +local parser = lpeg.Ct(scheme * authority * path * query * fragment) + +-- todo: reconsider Ct as we can as well have five return values (saves a table) +-- so we can have two parsers, one with and one without + +function url.split(str) + return (type(str) == "string" and lpegmatch(parser,str)) or str +end + +-- todo: cache them + +function url.hashed(str) + local s = url.split(str) + local somescheme = s[1] ~= "" + return { + scheme = (somescheme and s[1]) or "file", + authority = s[2], + path = s[3], + query = s[4], + fragment = s[5], + original = str, + noscheme = not somescheme, + } +end + +function url.hasscheme(str) + return url.split(str)[1] ~= "" +end + +function url.addscheme(str,scheme) + return (url.hasscheme(str) and str) or ((scheme or "file:///") .. str) +end + +function url.construct(hash) + local fullurl = hash.sheme .. "://".. hash.authority .. hash.path + if hash.query then + fullurl = fullurl .. "?".. hash.query + end + if hash.fragment then + fullurl = fullurl .. "?".. hash.fragment + end + return fullurl +end + +function url.filename(filename) + local t = url.hashed(filename) + return (t.scheme == "file" and (gsub(t.path,"^/([a-zA-Z])([:|])/)","%1:"))) or filename +end + +function url.query(str) + if type(str) == "string" then + local t = { } + for k, v in gmatch(str,"([^&=]*)=([^&=]*)") do + t[k] = v + end + return t + else + return str + end +end + +--~ print(url.filename("file:///c:/oeps.txt")) +--~ print(url.filename("c:/oeps.txt")) +--~ print(url.filename("file:///oeps.txt")) +--~ print(url.filename("file:///etc/test.txt")) +--~ print(url.filename("/oeps.txt")) + +--~ from the spec on the web (sort of): +--~ +--~ function test(str) +--~ print(table.serialize(url.hashed(str))) +--~ end +--~ +--~ test("%56pass%20words") +--~ test("file:///c:/oeps.txt") +--~ test("file:///c|/oeps.txt") +--~ test("file:///etc/oeps.txt") +--~ test("file://./etc/oeps.txt") +--~ test("file:////etc/oeps.txt") +--~ test("ftp://ftp.is.co.za/rfc/rfc1808.txt") +--~ test("http://www.ietf.org/rfc/rfc2396.txt") +--~ test("ldap://[2001:db8::7]/c=GB?objectClass?one#what") +--~ test("mailto:John.Doe@example.com") +--~ test("news:comp.infosystems.www.servers.unix") +--~ test("tel:+1-816-555-1212") +--~ test("telnet://192.0.2.16:80/") +--~ test("urn:oasis:names:specification:docbook:dtd:xml:4.1.2") +--~ test("/etc/passwords") +--~ test("http://www.pragma-ade.com/spaced%20name") + +--~ test("zip:///oeps/oeps.zip#bla/bla.tex") +--~ test("zip:///oeps/oeps.zip?bla/bla.tex") + + +end -- of closure + +do -- create closure to overcome 200 locals limit + if not modules then modules = { } end modules ['l-dir'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } +-- dir.expand_name will be merged with cleanpath and collapsepath + local type = type -local find, gmatch = string.find, string.gmatch +local find, gmatch, match, gsub = string.find, string.gmatch, string.match, string.gsub +local lpegmatch = lpeg.match dir = dir or { } +-- handy + +function dir.current() + return (gsub(lfs.currentdir(),"\\","/")) +end + -- optimizing for no string.find (*) does not save time local attributes = lfs.attributes @@ -2068,6 +2671,35 @@ end dir.glob_pattern = glob_pattern +local function collect_pattern(path,patt,recurse,result) + local ok, scanner + result = result or { } + if path == "/" then + ok, scanner = xpcall(function() return walkdir(path..".") end, function() end) -- kepler safe + else + ok, scanner = xpcall(function() return walkdir(path) end, function() end) -- kepler safe + end + if ok and type(scanner) == "function" then + if not find(path,"/$") then path = path .. '/' end + for name in scanner do + local full = path .. name + local attr = attributes(full) + local mode = attr.mode + if mode == 'file' then + if find(full,patt) then + result[name] = attr + end + elseif recurse and (mode == "directory") and (name ~= '.') and (name ~= "..") then + attr.list = collect_pattern(full,patt,recurse) + result[name] = attr + end + end + end + return result +end + +dir.collect_pattern = collect_pattern + local P, S, R, C, Cc, Cs, Ct, Cv, V = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.Cc, lpeg.Cs, lpeg.Ct, lpeg.Cv, lpeg.V local pattern = Ct { @@ -2087,29 +2719,48 @@ local filter = Cs ( ( )^0 ) local function glob(str,t) - if type(str) == "table" then - local t = t or { } - for s=1,#str do - glob(str[s],t) + if type(t) == "function" then + if type(str) == "table" then + for s=1,#str do + glob(str[s],t) + end + elseif lfs.isfile(str) then + t(str) + else + local split = lpegmatch(pattern,str) + if split then + local root, path, base = split[1], split[2], split[3] + local recurse = find(base,"%*%*") + local start = root .. path + local result = lpegmatch(filter,start .. base) + glob_pattern(start,result,recurse,t) + end end - return t - elseif lfs.isfile(str) then - local t = t or { } - t[#t+1] = str - return t else - local split = pattern:match(str) - if split then + if type(str) == "table" then local t = t or { } - local action = action or function(name) t[#t+1] = name end - local root, path, base = split[1], split[2], split[3] - local recurse = find(base,"%*%*") - local start = root .. path - local result = filter:match(start .. base) - glob_pattern(start,result,recurse,action) + for s=1,#str do + glob(str[s],t) + end + return t + elseif lfs.isfile(str) then + local t = t or { } + t[#t+1] = str return t else - return { } + local split = lpegmatch(pattern,str) + if split then + local t = t or { } + local action = action or function(name) t[#t+1] = name end + local root, path, base = split[1], split[2], split[3] + local recurse = find(base,"%*%*") + local start = root .. path + local result = lpegmatch(filter,start .. base) + glob_pattern(start,result,recurse,action) + return t + else + return { } + end end end end @@ -2171,11 +2822,12 @@ end local make_indeed = true -- false -if string.find(os.getenv("PATH"),";") then +if string.find(os.getenv("PATH"),";") then -- os.type == "windows" function dir.mkdirs(...) - local str, pth = "", "" - for _, s in ipairs({...}) do + local str, pth, t = "", "", { ... } + for i=1,#t do + local s = t[i] if s ~= "" then if str ~= "" then str = str .. "/" .. s @@ -2186,13 +2838,13 @@ if string.find(os.getenv("PATH"),";") then end local first, middle, last local drive = false - first, middle, last = str:match("^(//)(//*)(.*)$") + first, middle, last = match(str,"^(//)(//*)(.*)$") if first then -- empty network path == local path else - first, last = str:match("^(//)/*(.-)$") + first, last = match(str,"^(//)/*(.-)$") if first then - middle, last = str:match("([^/]+)/+(.-)$") + middle, last = match(str,"([^/]+)/+(.-)$") if middle then pth = "//" .. middle else @@ -2200,11 +2852,11 @@ if string.find(os.getenv("PATH"),";") then last = "" end else - first, middle, last = str:match("^([a-zA-Z]:)(/*)(.-)$") + first, middle, last = match(str,"^([a-zA-Z]:)(/*)(.-)$") if first then pth, drive = first .. middle, true else - middle, last = str:match("^(/*)(.-)$") + middle, last = match(str,"^(/*)(.-)$") if not middle then last = str end @@ -2238,34 +2890,31 @@ if string.find(os.getenv("PATH"),";") then --~ print(dir.mkdirs("///a/b/c")) --~ print(dir.mkdirs("a/bbb//ccc/")) - function dir.expand_name(str) - local first, nothing, last = str:match("^(//)(//*)(.*)$") + function dir.expand_name(str) -- will be merged with cleanpath and collapsepath + local first, nothing, last = match(str,"^(//)(//*)(.*)$") if first then - first = lfs.currentdir() .. "/" - first = first:gsub("\\","/") + first = dir.current() .. "/" end if not first then - first, last = str:match("^(//)/*(.*)$") + first, last = match(str,"^(//)/*(.*)$") end if not first then - first, last = str:match("^([a-zA-Z]:)(.*)$") + first, last = match(str,"^([a-zA-Z]:)(.*)$") if first and not find(last,"^/") then local d = lfs.currentdir() if lfs.chdir(first) then - first = lfs.currentdir() - first = first:gsub("\\","/") + first = dir.current() end lfs.chdir(d) end end if not first then - first, last = lfs.currentdir(), str - first = first:gsub("\\","/") + first, last = dir.current(), str end - last = last:gsub("//","/") - last = last:gsub("/%./","/") - last = last:gsub("^/*","") - first = first:gsub("/*$","") + last = gsub(last,"//","/") + last = gsub(last,"/%./","/") + last = gsub(last,"^/*","") + first = gsub(first,"/*$","") if last == "" then return first else @@ -2276,8 +2925,9 @@ if string.find(os.getenv("PATH"),";") then else function dir.mkdirs(...) - local str, pth = "", "" - for _, s in ipairs({...}) do + local str, pth, t = "", "", { ... } + for i=1,#t do + local s = t[i] if s ~= "" then if str ~= "" then str = str .. "/" .. s @@ -2286,7 +2936,7 @@ else end end end - str = str:gsub("/+","/") + str = gsub(str,"/+","/") if find(str,"^/") then pth = "/" for s in gmatch(str,"[^/]+") do @@ -2320,12 +2970,12 @@ else --~ print(dir.mkdirs("///a/b/c")) --~ print(dir.mkdirs("a/bbb//ccc/")) - function dir.expand_name(str) + function dir.expand_name(str) -- will be merged with cleanpath and collapsepath if not find(str,"^/") then str = lfs.currentdir() .. "/" .. str end - str = str:gsub("//","/") - str = str:gsub("/%./","/") + str = gsub(str,"//","/") + str = gsub(str,"/%./","/") return str end @@ -2340,7 +2990,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-boolean'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -2401,7 +3051,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-math'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -2448,7 +3098,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-utils'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -2456,6 +3106,10 @@ if not modules then modules = { } end modules ['l-utils'] = { -- hm, quite unreadable +local gsub = string.gsub +local concat = table.concat +local type, next = type, next + if not utils then utils = { } end if not utils.merger then utils.merger = { } end if not utils.lua then utils.lua = { } end @@ -2493,7 +3147,7 @@ function utils.merger._self_load_(name) end if data and utils.merger.strip_comment then -- saves some 20K - data = data:gsub("%-%-~[^\n\r]*[\r\n]", "") + data = gsub(data,"%-%-~[^\n\r]*[\r\n]", "") end return data or "" end @@ -2511,7 +3165,7 @@ end function utils.merger._self_swap_(data,code) if data ~= "" then - return (data:gsub(utils.merger.pattern, function(s) + return (gsub(data,utils.merger.pattern, function(s) return "\n\n" .. "-- "..utils.merger.m_begin .. "\n" .. code .. "\n" .. "-- "..utils.merger.m_end .. "\n\n" end, 1)) else @@ -2521,8 +3175,8 @@ end --~ stripper: --~ ---~ data = string.gsub(data,"%-%-~[^\n]*\n","") ---~ data = string.gsub(data,"\n\n+","\n") +--~ data = gsub(data,"%-%-~[^\n]*\n","") +--~ data = gsub(data,"\n\n+","\n") function utils.merger._self_libs_(libs,list) local result, f, frozen = { }, nil, false @@ -2530,9 +3184,10 @@ function utils.merger._self_libs_(libs,list) if type(libs) == 'string' then libs = { libs } end if type(list) == 'string' then list = { list } end local foundpath = nil - for _, lib in ipairs(libs) do - for _, pth in ipairs(list) do - pth = string.gsub(pth,"\\","/") -- file.clean_path + for i=1,#libs do + local lib = libs[i] + for j=1,#list do + local pth = gsub(list[j],"\\","/") -- file.clean_path utils.report("checking library path %s",pth) local name = pth .. "/" .. lib if lfs.isfile(name) then @@ -2544,7 +3199,8 @@ function utils.merger._self_libs_(libs,list) if foundpath then utils.report("using library path %s",foundpath) local right, wrong = { }, { } - for _, lib in ipairs(libs) do + for i=1,#libs do + local lib = libs[i] local fullname = foundpath .. "/" .. lib if lfs.isfile(fullname) then -- right[#right+1] = lib @@ -2558,15 +3214,15 @@ function utils.merger._self_libs_(libs,list) end end if #right > 0 then - utils.report("merged libraries: %s",table.concat(right," ")) + utils.report("merged libraries: %s",concat(right," ")) end if #wrong > 0 then - utils.report("skipped libraries: %s",table.concat(wrong," ")) + utils.report("skipped libraries: %s",concat(wrong," ")) end else utils.report("no valid library path found") end - return table.concat(result, "\n\n") + return concat(result, "\n\n") end function utils.merger.selfcreate(libs,list,target) @@ -2624,16 +3280,28 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-aux'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } +-- for inline, no store split : for s in string.gmatch(str,",* *([^,]+)") do .. end + aux = aux or { } local concat, format, gmatch = table.concat, string.format, string.gmatch local tostring, type = tostring, type +local lpegmatch = lpeg.match + +local P, R, V = lpeg.P, lpeg.R, lpeg.V + +local escape, left, right = P("\\"), P('{'), P('}') + +lpeg.patterns.balanced = P { + [1] = ((escape * (left+right)) + (1 - (left+right)) + V(2))^0, + [2] = left * V(1) * right +} local space = lpeg.P(' ') local equal = lpeg.P("=") @@ -2641,7 +3309,7 @@ local comma = lpeg.P(",") local lbrace = lpeg.P("{") local rbrace = lpeg.P("}") local nobrace = 1 - (lbrace+rbrace) -local nested = lpeg.P{ lbrace * (nobrace + lpeg.V(1))^0 * rbrace } +local nested = lpeg.P { lbrace * (nobrace + lpeg.V(1))^0 * rbrace } local spaces = space^0 local value = lpeg.P(lbrace * lpeg.C((nobrace + nested)^0) * rbrace) + lpeg.C((nested + (1-comma))^0) @@ -2679,13 +3347,13 @@ function aux.make_settings_to_hash_pattern(set,how) end end -function aux.settings_to_hash(str) +function aux.settings_to_hash(str,existing) if str and str ~= "" then - hash = { } + hash = existing or { } if moretolerant then - pattern_b_s:match(str) + lpegmatch(pattern_b_s,str) else - pattern_a_s:match(str) + lpegmatch(pattern_a_s,str) end return hash else @@ -2693,39 +3361,41 @@ function aux.settings_to_hash(str) end end -function aux.settings_to_hash_tolerant(str) +function aux.settings_to_hash_tolerant(str,existing) if str and str ~= "" then - hash = { } - pattern_b_s:match(str) + hash = existing or { } + lpegmatch(pattern_b_s,str) return hash else return { } end end -function aux.settings_to_hash_strict(str) +function aux.settings_to_hash_strict(str,existing) if str and str ~= "" then - hash = { } - pattern_c_s:match(str) + hash = existing or { } + lpegmatch(pattern_c_s,str) return next(hash) and hash else return nil end end -local seperator = comma * space^0 +local separator = comma * space^0 local value = lpeg.P(lbrace * lpeg.C((nobrace + nested)^0) * rbrace) + lpeg.C((nested + (1-comma))^0) -local pattern = lpeg.Ct(value*(seperator*value)^0) +local pattern = lpeg.Ct(value*(separator*value)^0) -- "aap, {noot}, mies" : outer {} removes, leading spaces ignored aux.settings_to_array_pattern = pattern +-- we could use a weak table as cache + function aux.settings_to_array(str) if not str or str == "" then return { } else - return pattern:match(str) + return lpegmatch(pattern,str) end end @@ -2734,10 +3404,10 @@ local function set(t,v) end local value = lpeg.P(lpeg.Carg(1)*value) / set -local pattern = value*(seperator*value)^0 * lpeg.Carg(1) +local pattern = value*(separator*value)^0 * lpeg.Carg(1) function aux.add_settings_to_array(t,str) - return pattern:match(str, nil, t) + return lpegmatch(pattern,str,nil,t) end function aux.hash_to_string(h,separator,yes,no,strict,omit) @@ -2785,6 +3455,13 @@ function aux.settings_to_set(str,t) return t end +local value = lbrace * lpeg.C((nobrace + nested)^0) * rbrace +local pattern = lpeg.Ct((space + value)^0) + +function aux.arguments_to_table(str) + return lpegmatch(pattern,str) +end + -- temporary here function aux.getparameters(self,class,parentclass,settings) @@ -2793,36 +3470,31 @@ function aux.getparameters(self,class,parentclass,settings) sc = table.clone(self[parent]) self[class] = sc end - aux.add_settings_to_array(sc, settings) + aux.settings_to_hash(settings,sc) end -- temporary here -local digit = lpeg.R("09") -local period = lpeg.P(".") -local zero = lpeg.P("0") - ---~ local finish = lpeg.P(-1) ---~ local nodigit = (1-digit) + finish ---~ local case_1 = (period * zero^1 * #nodigit)/"" -- .000 ---~ local case_2 = (period * (1-(zero^0/"") * #nodigit)^1 * (zero^0/"") * nodigit) -- .010 .10 .100100 - +local digit = lpeg.R("09") +local period = lpeg.P(".") +local zero = lpeg.P("0") local trailingzeros = zero^0 * -digit -- suggested by Roberto R -local case_1 = period * trailingzeros / "" -local case_2 = period * (digit - trailingzeros)^1 * (trailingzeros / "") - -local number = digit^1 * (case_1 + case_2) -local stripper = lpeg.Cs((number + 1)^0) +local case_1 = period * trailingzeros / "" +local case_2 = period * (digit - trailingzeros)^1 * (trailingzeros / "") +local number = digit^1 * (case_1 + case_2) +local stripper = lpeg.Cs((number + 1)^0) --~ local sample = "bla 11.00 bla 11 bla 0.1100 bla 1.00100 bla 0.00 bla 0.001 bla 1.1100 bla 0.100100100 bla 0.00100100100" --~ collectgarbage("collect") --~ str = string.rep(sample,10000) --~ local ts = os.clock() ---~ stripper:match(str) ---~ print(#str, os.clock()-ts, stripper:match(sample)) +--~ lpegmatch(stripper,str) +--~ print(#str, os.clock()-ts, lpegmatch(stripper,sample)) + +lpeg.patterns.strip_zeros = stripper function aux.strip_zeros(str) - return stripper:match(str) + return lpegmatch(stripper,str) end function aux.definetable(target) -- defines undefined tables @@ -2846,6 +3518,375 @@ function aux.accesstable(target) return t end +--~ function string.commaseparated(str) +--~ return gmatch(str,"([^,%s]+)") +--~ end + +-- as we use this a lot ... + +--~ function aux.cachefunction(action,weak) +--~ local cache = { } +--~ if weak then +--~ setmetatable(cache, { __mode = "kv" } ) +--~ end +--~ local function reminder(str) +--~ local found = cache[str] +--~ if not found then +--~ found = action(str) +--~ cache[str] = found +--~ end +--~ return found +--~ end +--~ return reminder, cache +--~ end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['trac-tra'] = { + version = 1.001, + comment = "companion to trac-tra.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- the <anonymous> tag is kind of generic and used for functions that are not +-- bound to a variable, like node.new, node.copy etc (contrary to for instance +-- node.has_attribute which is bound to a has_attribute local variable in mkiv) + +local debug = require "debug" + +local getinfo = debug.getinfo +local type, next = type, next +local concat = table.concat +local format, find, lower, gmatch, gsub = string.format, string.find, string.lower, string.gmatch, string.gsub + +debugger = debugger or { } + +local counters = { } +local names = { } + +-- one + +local function hook() + local f = getinfo(2,"f").func + local n = getinfo(2,"Sn") +-- if n.what == "C" and n.name then print (n.namewhat .. ': ' .. n.name) end + if f then + local cf = counters[f] + if cf == nil then + counters[f] = 1 + names[f] = n + else + counters[f] = cf + 1 + end + end +end +local function getname(func) + local n = names[func] + if n then + if n.what == "C" then + return n.name or '<anonymous>' + else + -- source short_src linedefined what name namewhat nups func + local name = n.name or n.namewhat or n.what + if not name or name == "" then name = "?" end + return format("%s : %s : %s", n.short_src or "unknown source", n.linedefined or "--", name) + end + else + return "unknown" + end +end +function debugger.showstats(printer,threshold) + printer = printer or texio.write or print + threshold = threshold or 0 + local total, grandtotal, functions = 0, 0, 0 + printer("\n") -- ugly but ok + -- table.sort(counters) + for func, count in next, counters do + if count > threshold then + local name = getname(func) + if not find(name,"for generator") then + printer(format("%8i %s", count, name)) + total = total + count + end + end + grandtotal = grandtotal + count + functions = functions + 1 + end + printer(format("functions: %s, total: %s, grand total: %s, threshold: %s\n", functions, total, grandtotal, threshold)) +end + +-- two + +--~ local function hook() +--~ local n = getinfo(2) +--~ if n.what=="C" and not n.name then +--~ local f = tostring(debug.traceback()) +--~ local cf = counters[f] +--~ if cf == nil then +--~ counters[f] = 1 +--~ names[f] = n +--~ else +--~ counters[f] = cf + 1 +--~ end +--~ end +--~ end +--~ function debugger.showstats(printer,threshold) +--~ printer = printer or texio.write or print +--~ threshold = threshold or 0 +--~ local total, grandtotal, functions = 0, 0, 0 +--~ printer("\n") -- ugly but ok +--~ -- table.sort(counters) +--~ for func, count in next, counters do +--~ if count > threshold then +--~ printer(format("%8i %s", count, func)) +--~ total = total + count +--~ end +--~ grandtotal = grandtotal + count +--~ functions = functions + 1 +--~ end +--~ printer(format("functions: %s, total: %s, grand total: %s, threshold: %s\n", functions, total, grandtotal, threshold)) +--~ end + +-- rest + +function debugger.savestats(filename,threshold) + local f = io.open(filename,'w') + if f then + debugger.showstats(function(str) f:write(str) end,threshold) + f:close() + end +end + +function debugger.enable() + debug.sethook(hook,"c") +end + +function debugger.disable() + debug.sethook() +--~ counters[debug.getinfo(2,"f").func] = nil +end + +function debugger.tracing() + local n = tonumber(os.env['MTX.TRACE.CALLS']) or tonumber(os.env['MTX_TRACE_CALLS']) or 0 + if n > 0 then + function debugger.tracing() return true end ; return true + else + function debugger.tracing() return false end ; return false + end +end + +--~ debugger.enable() + +--~ print(math.sin(1*.5)) +--~ print(math.sin(1*.5)) +--~ print(math.sin(1*.5)) +--~ print(math.sin(1*.5)) +--~ print(math.sin(1*.5)) + +--~ debugger.disable() + +--~ print("") +--~ debugger.showstats() +--~ print("") +--~ debugger.showstats(print,3) + +setters = setters or { } +setters.data = setters.data or { } + +--~ local function set(t,what,value) +--~ local data, done = t.data, t.done +--~ if type(what) == "string" then +--~ what = aux.settings_to_array(what) -- inefficient but ok +--~ end +--~ for i=1,#what do +--~ local w = what[i] +--~ for d, f in next, data do +--~ if done[d] then +--~ -- prevent recursion due to wildcards +--~ elseif find(d,w) then +--~ done[d] = true +--~ for i=1,#f do +--~ f[i](value) +--~ end +--~ end +--~ end +--~ end +--~ end + +local function set(t,what,value) + local data, done = t.data, t.done + if type(what) == "string" then + what = aux.settings_to_hash(what) -- inefficient but ok + end + for w, v in next, what do + if v == "" then + v = value + else + v = toboolean(v) + end + for d, f in next, data do + if done[d] then + -- prevent recursion due to wildcards + elseif find(d,w) then + done[d] = true + for i=1,#f do + f[i](v) + end + end + end + end +end + +local function reset(t) + for d, f in next, t.data do + for i=1,#f do + f[i](false) + end + end +end + +local function enable(t,what) + set(t,what,true) +end + +local function disable(t,what) + local data = t.data + if not what or what == "" then + t.done = { } + reset(t) + else + set(t,what,false) + end +end + +function setters.register(t,what,...) + local data = t.data + what = lower(what) + local w = data[what] + if not w then + w = { } + data[what] = w + end + for _, fnc in next, { ... } do + local typ = type(fnc) + if typ == "function" then + w[#w+1] = fnc + elseif typ == "string" then + w[#w+1] = function(value) set(t,fnc,value,nesting) end + end + end +end + +function setters.enable(t,what) + local e = t.enable + t.enable, t.done = enable, { } + enable(t,string.simpleesc(tostring(what))) + t.enable, t.done = e, { } +end + +function setters.disable(t,what) + local e = t.disable + t.disable, t.done = disable, { } + disable(t,string.simpleesc(tostring(what))) + t.disable, t.done = e, { } +end + +function setters.reset(t) + t.done = { } + reset(t) +end + +function setters.list(t) -- pattern + local list = table.sortedkeys(t.data) + local user, system = { }, { } + for l=1,#list do + local what = list[l] + if find(what,"^%*") then + system[#system+1] = what + else + user[#user+1] = what + end + end + return user, system +end + +function setters.show(t) + commands.writestatus("","") + local list = setters.list(t) + for k=1,#list do + commands.writestatus(t.name,list[k]) + end + commands.writestatus("","") +end + +-- we could have used a bit of oo and the trackers:enable syntax but +-- there is already a lot of code around using the singular tracker + +-- we could make this into a module + +function setters.new(name) + local t + t = { + data = { }, + name = name, + enable = function(...) setters.enable (t,...) end, + disable = function(...) setters.disable (t,...) end, + register = function(...) setters.register(t,...) end, + list = function(...) setters.list (t,...) end, + show = function(...) setters.show (t,...) end, + } + setters.data[name] = t + return t +end + +trackers = setters.new("trackers") +directives = setters.new("directives") +experiments = setters.new("experiments") + +-- nice trick: we overload two of the directives related functions with variants that +-- do tracing (itself using a tracker) .. proof of concept + +local trace_directives = false local trace_directives = false trackers.register("system.directives", function(v) trace_directives = v end) +local trace_experiments = false local trace_experiments = false trackers.register("system.experiments", function(v) trace_experiments = v end) + +local e = directives.enable +local d = directives.disable + +function directives.enable(...) + commands.writestatus("directives","enabling: %s",concat({...}," ")) + e(...) +end + +function directives.disable(...) + commands.writestatus("directives","disabling: %s",concat({...}," ")) + d(...) +end + +local e = experiments.enable +local d = experiments.disable + +function experiments.enable(...) + commands.writestatus("experiments","enabling: %s",concat({...}," ")) + e(...) +end + +function experiments.disable(...) + commands.writestatus("experiments","disabling: %s",concat({...}," ")) + d(...) +end + +-- a useful example + +directives.register("system.nostatistics", function(v) + statistics.enable = not v +end) + + end -- of closure @@ -2859,6 +3900,12 @@ if not modules then modules = { } end modules ['lxml-tab'] = { license = "see context related readme files" } +-- this module needs a cleanup: check latest lpeg, passing args, (sub)grammar, etc etc +-- stripping spaces from e.g. cont-en.xml saves .2 sec runtime so it's not worth the +-- trouble + +local trace_entities = false trackers.register("xml.entities", function(v) trace_entities = v end) + --[[ldx-- <p>The parser used here is inspired by the variant discussed in the lua book, but handles comment and processing instructions, has a different structure, provides @@ -2866,18 +3913,6 @@ parent access; a first version used different trickery but was less optimized to went this route. First we had a find based parser, now we have an <l n='lpeg'/> based one. The find based parser can be found in l-xml-edu.lua along with other older code.</p> -<p>Expecially the lpath code is experimental, we will support some of xpath, but -only things that make sense for us; as compensation it is possible to hook in your -own functions. Apart from preprocessing content for <l n='context'/> we also need -this module for process management, like handling <l n='ctx'/> and <l n='rlx'/> -files.</p> - -<typing> -a/b/c /*/c -a/b/c/first() a/b/c/last() a/b/c/index(n) a/b/c/index(-n) -a/b/c/text() a/b/c/text(1) a/b/c/text(-1) a/b/c/text(n) -</typing> - <p>Beware, the interface may change. For instance at, ns, tg, dt may get more verbose names. Once the code is stable we will also remove some tracing and optimize the code.</p> @@ -2888,26 +3923,11 @@ xml = xml or { } --~ local xml = xml local concat, remove, insert = table.concat, table.remove, table.insert -local type, next, setmetatable = type, next, setmetatable -local format, lower, find = string.format, string.lower, string.find - ---[[ldx-- -<p>This module can be used stand alone but also inside <l n='mkiv'/> in -which case it hooks into the tracker code. Therefore we provide a few -functions that set the tracers.</p> ---ldx]]-- - -local trace_remap = false - -if trackers then - trackers.register("xml.remap", function(v) trace_remap = v end) -end - -function xml.settrace(str,value) - if str == "remap" then - trace_remap = value or false - end -end +local type, next, setmetatable, getmetatable, tonumber = type, next, setmetatable, getmetatable, tonumber +local format, lower, find, match, gsub = string.format, string.lower, string.find, string.match, string.gsub +local utfchar = unicode.utf8.char +local lpegmatch = lpeg.match +local P, S, R, C, V, C, Cs = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.C, lpeg.Cs --[[ldx-- <p>First a hack to enable namespace resolving. A namespace is characterized by @@ -2919,7 +3939,7 @@ much cleaner.</p> xml.xmlns = xml.xmlns or { } -local check = lpeg.P(false) +local check = P(false) local parse = check --[[ldx-- @@ -2932,8 +3952,8 @@ xml.registerns("mml","mathml") --ldx]]-- function xml.registerns(namespace, pattern) -- pattern can be an lpeg - check = check + lpeg.C(lpeg.P(lower(pattern))) / namespace - parse = lpeg.P { lpeg.P(check) + 1 * lpeg.V(1) } + check = check + C(P(lower(pattern))) / namespace + parse = P { P(check) + 1 * V(1) } end --[[ldx-- @@ -2947,7 +3967,7 @@ xml.checkns("m","http://www.w3.org/mathml") --ldx]]-- function xml.checkns(namespace,url) - local ns = parse:match(lower(url)) + local ns = lpegmatch(parse,lower(url)) if ns and namespace ~= ns then xml.xmlns[namespace] = ns end @@ -2965,7 +3985,7 @@ This returns <t>mml</t>. --ldx]]-- function xml.resolvens(url) - return parse:match(lower(url)) or "" + return lpegmatch(parse,lower(url)) or "" end --[[ldx-- @@ -3004,27 +4024,36 @@ local x = xml.convert(somestring) <p>An optional second boolean argument tells this function not to create a root element.</p> ---ldx]]-- -xml.strip_cm_and_dt = false -- an extra global flag, in case we have many includes +<p>Valid entities are:</p> + +<typing> +<!ENTITY xxxx SYSTEM "yyyy" NDATA zzzz> +<!ENTITY xxxx PUBLIC "yyyy" > +<!ENTITY xxxx "yyyy" > +</typing> +--ldx]]-- -- not just one big nested table capture (lpeg overflow) local nsremap, resolvens = xml.xmlns, xml.resolvens -local stack, top, dt, at, xmlns, errorstr, entities = {}, {}, {}, {}, {}, nil, {} +local stack, top, dt, at, xmlns, errorstr, entities = { }, { }, { }, { }, { }, nil, { } +local strip, cleanup, utfize, resolve, resolve_predefined, unify_predefined = false, false, false, false, false, false +local dcache, hcache, acache = { }, { }, { } -local mt = { __tostring = xml.text } +local mt = { } -function xml.check_error(top,toclose) - return "" +function initialize_mt(root) + mt = { __index = root } -- will be redefined later end -local strip = false -local cleanup = false +function xml.setproperty(root,k,v) + getmetatable(root).__index[k] = v +end -function xml.set_text_cleanup(fnc) - cleanup = fnc +function xml.check_error(top,toclose) + return "" end local function add_attribute(namespace,tag,value) @@ -3034,12 +4063,31 @@ local function add_attribute(namespace,tag,value) if tag == "xmlns" then xmlns[#xmlns+1] = resolvens(value) at[tag] = value + elseif namespace == "" then + at[tag] = value elseif namespace == "xmlns" then xml.checkns(tag,value) at["xmlns:" .. tag] = value else - at[tag] = value + -- for the moment this way: + at[namespace .. ":" .. tag] = value + end +end + +local function add_empty(spacing, namespace, tag) + if #spacing > 0 then + dt[#dt+1] = spacing + end + local resolved = (namespace == "" and xmlns[#xmlns]) or nsremap[namespace] or namespace + top = stack[#stack] + dt = top.dt + local t = { ns=namespace or "", rn=resolved, tg=tag, at=at, dt={}, __p__ = top } + dt[#dt+1] = t + setmetatable(t, mt) + if at.xmlns then + remove(xmlns) end + at = { } end local function add_begin(spacing, namespace, tag) @@ -3067,28 +4115,12 @@ local function add_end(spacing, namespace, tag) end dt = top.dt dt[#dt+1] = toclose - dt[0] = top + -- dt[0] = top -- nasty circular reference when serializing table if toclose.at.xmlns then remove(xmlns) end end -local function add_empty(spacing, namespace, tag) - if #spacing > 0 then - dt[#dt+1] = spacing - end - local resolved = (namespace == "" and xmlns[#xmlns]) or nsremap[namespace] or namespace - top = stack[#stack] - dt = top.dt - local t = { ns=namespace or "", rn=resolved, tg=tag, at=at, dt={}, __p__ = top } - dt[#dt+1] = t - setmetatable(t, mt) - if at.xmlns then - remove(xmlns) - end - at = { } -end - local function add_text(text) if cleanup and #text > 0 then dt[#dt+1] = cleanup(text) @@ -3104,7 +4136,7 @@ local function add_special(what, spacing, text) if strip and (what == "@cm@" or what == "@dt@") then -- forget it else - dt[#dt+1] = { special=true, ns="", tg=what, dt={text} } + dt[#dt+1] = { special=true, ns="", tg=what, dt={ text } } end end @@ -3112,7 +4144,199 @@ local function set_message(txt) errorstr = "garbage at the end of the file: " .. gsub(txt,"([ \n\r\t]*)","") end -local P, S, R, C, V = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V +local reported_attribute_errors = { } + +local function attribute_value_error(str) + if not reported_attribute_errors[str] then + logs.report("xml","invalid attribute value: %q",str) + reported_attribute_errors[str] = true + at._error_ = str + end + return str +end +local function attribute_specification_error(str) + if not reported_attribute_errors[str] then + logs.report("xml","invalid attribute specification: %q",str) + reported_attribute_errors[str] = true + at._error_ = str + end + return str +end + +function xml.unknown_dec_entity_format(str) return (str == "" and "&error;") or format("&%s;",str) end +function xml.unknown_hex_entity_format(str) return format("&#x%s;",str) end +function xml.unknown_any_entity_format(str) return format("&#x%s;",str) end + +local function fromhex(s) + local n = tonumber(s,16) + if n then + return utfchar(n) + else + return format("h:%s",s), true + end +end + +local function fromdec(s) + local n = tonumber(s) + if n then + return utfchar(n) + else + return format("d:%s",s), true + end +end + +-- one level expansion (simple case), no checking done + +local rest = (1-P(";"))^0 +local many = P(1)^0 + +local parsedentity = + P("&") * (P("#x")*(rest/fromhex) + P("#")*(rest/fromdec)) * P(";") * P(-1) + + (P("#x")*(many/fromhex) + P("#")*(many/fromdec)) + +-- parsing in the xml file + +local predefined_unified = { + [38] = "&", + [42] = """, + [47] = "'", + [74] = "<", + [76] = "&gr;", +} + +local predefined_simplified = { + [38] = "&", amp = "&", + [42] = '"', quot = '"', + [47] = "'", apos = "'", + [74] = "<", lt = "<", + [76] = ">", gt = ">", +} + +local function handle_hex_entity(str) + local h = hcache[str] + if not h then + local n = tonumber(str,16) + h = unify_predefined and predefined_unified[n] + if h then + if trace_entities then + logs.report("xml","utfize, converting hex entity &#x%s; into %s",str,h) + end + elseif utfize then + h = (n and utfchar(n)) or xml.unknown_hex_entity_format(str) or "" + if not n then + logs.report("xml","utfize, ignoring hex entity &#x%s;",str) + elseif trace_entities then + logs.report("xml","utfize, converting hex entity &#x%s; into %s",str,h) + end + else + if trace_entities then + logs.report("xml","found entity &#x%s;",str) + end + h = "&#x" .. str .. ";" + end + hcache[str] = h + end + return h +end + +local function handle_dec_entity(str) + local d = dcache[str] + if not d then + local n = tonumber(str) + d = unify_predefined and predefined_unified[n] + if d then + if trace_entities then + logs.report("xml","utfize, converting dec entity &#%s; into %s",str,d) + end + elseif utfize then + d = (n and utfchar(n)) or xml.unknown_dec_entity_format(str) or "" + if not n then + logs.report("xml","utfize, ignoring dec entity &#%s;",str) + elseif trace_entities then + logs.report("xml","utfize, converting dec entity &#%s; into %s",str,h) + end + else + if trace_entities then + logs.report("xml","found entity &#%s;",str) + end + d = "&#" .. str .. ";" + end + dcache[str] = d + end + return d +end + +xml.parsedentitylpeg = parsedentity + +local function handle_any_entity(str) + if resolve then + local a = acache[str] -- per instance ! todo + if not a then + a = resolve_predefined and predefined_simplified[str] + if a then + -- one of the predefined + elseif type(resolve) == "function" then + a = resolve(str) or entities[str] + else + a = entities[str] + end + if a then + if trace_entities then + logs.report("xml","resolved entity &%s; -> %s (internal)",str,a) + end + a = lpegmatch(parsedentity,a) or a + else + if xml.unknown_any_entity_format then + a = xml.unknown_any_entity_format(str) or "" + end + if a then + if trace_entities then + logs.report("xml","resolved entity &%s; -> %s (external)",str,a) + end + else + if trace_entities then + logs.report("xml","keeping entity &%s;",str) + end + if str == "" then + a = "&error;" + else + a = "&" .. str .. ";" + end + end + end + acache[str] = a + elseif trace_entities then + if not acache[str] then + logs.report("xml","converting entity &%s; into %s",str,a) + acache[str] = a + end + end + return a + else + local a = acache[str] + if not a then + if trace_entities then + logs.report("xml","found entity &%s;",str) + end + a = resolve_predefined and predefined_simplified[str] + if a then + -- one of the predefined + acache[str] = a + elseif str == "" then + a = "&error;" + acache[str] = a + else + a = "&" .. str .. ";" + acache[str] = a + end + end + return a + end +end + +local function handle_end_entity(chr) + logs.report("xml","error in entity, %q found instead of ';'",chr) +end local space = S(' \r\n\t') local open = P('<') @@ -3122,24 +4346,50 @@ local dquote = S('"') local equal = P('=') local slash = P('/') local colon = P(':') +local semicolon = P(';') +local ampersand = P('&') local valid = R('az', 'AZ', '09') + S('_-.') local name_yes = C(valid^1) * colon * C(valid^1) local name_nop = C(P(true)) * C(valid^1) local name = name_yes + name_nop +local utfbom = lpeg.patterns.utfbom -- no capture +local spacing = C(space^0) -local utfbom = P('\000\000\254\255') + P('\255\254\000\000') + - P('\255\254') + P('\254\255') + P('\239\187\191') -- no capture +----- entitycontent = (1-open-semicolon)^0 +local anyentitycontent = (1-open-semicolon-space-close)^0 +local hexentitycontent = R("AF","af","09")^0 +local decentitycontent = R("09")^0 +local parsedentity = P("#")/"" * ( + P("x")/"" * (hexentitycontent/handle_hex_entity) + + (decentitycontent/handle_dec_entity) + ) + (anyentitycontent/handle_any_entity) +local entity = ampersand/"" * parsedentity * ( (semicolon/"") + #(P(1)/handle_end_entity)) + +local text_unparsed = C((1-open)^1) +local text_parsed = Cs(((1-open-ampersand)^1 + entity)^1) -local spacing = C(space^0) -local justtext = C((1-open)^1) local somespace = space^1 local optionalspace = space^0 -local value = (squote * C((1 - squote)^0) * squote) + (dquote * C((1 - dquote)^0) * dquote) -local attribute = (somespace * name * optionalspace * equal * optionalspace * value) / add_attribute -local attributes = attribute^0 +----- value = (squote * C((1 - squote)^0) * squote) + (dquote * C((1 - dquote)^0) * dquote) -- ampersand and < also invalid in value +local value = (squote * Cs((entity + (1 - squote))^0) * squote) + (dquote * Cs((entity + (1 - dquote))^0) * dquote) -- ampersand and < also invalid in value + +local endofattributes = slash * close + close -- recovery of flacky html +local whatever = space * name * optionalspace * equal +local wrongvalue = C(P(1-whatever-close)^1 + P(1-close)^1) / attribute_value_error +----- wrongvalue = C(P(1-whatever-endofattributes)^1 + P(1-endofattributes)^1) / attribute_value_error +----- wrongvalue = C(P(1-space-endofattributes)^1) / attribute_value_error +local wrongvalue = Cs(P(entity + (1-space-endofattributes))^1) / attribute_value_error -local text = justtext / add_text +local attributevalue = value + wrongvalue + +local attribute = (somespace * name * optionalspace * equal * optionalspace * attributevalue) / add_attribute +----- attributes = (attribute)^0 + +local attributes = (attribute + somespace^-1 * (((1-endofattributes)^1)/attribute_specification_error))^0 + +local parsedtext = text_parsed / add_text +local unparsedtext = text_unparsed / add_text local balanced = P { "[" * ((1 - S"[]") + V(1))^0 * "]" } -- taken from lpeg manual, () example local emptyelement = (spacing * open * name * attributes * optionalspace * slash * close) / add_empty @@ -3157,19 +4407,27 @@ local someinstruction = C((1 - endinstruction)^0) local somecomment = C((1 - endcomment )^0) local somecdata = C((1 - endcdata )^0) -local function entity(k,v) entities[k] = v end +local function normalentity(k,v ) entities[k] = v end +local function systementity(k,v,n) entities[k] = v end +local function publicentity(k,v,n) entities[k] = v end local begindoctype = open * P("!DOCTYPE") local enddoctype = close local beginset = P("[") local endset = P("]") -local doctypename = C((1-somespace)^0) +local doctypename = C((1-somespace-close)^0) local elementdoctype = optionalspace * P("<!ELEMENT") * (1-close)^0 * close -local entitydoctype = optionalspace * P("<!ENTITY") * somespace * (doctypename * somespace * value)/entity * optionalspace * close -local publicdoctype = doctypename * somespace * P("PUBLIC") * somespace * value * somespace * value * somespace -local systemdoctype = doctypename * somespace * P("SYSTEM") * somespace * value * somespace -local definitiondoctype= doctypename * somespace * beginset * P(elementdoctype + entitydoctype)^0 * optionalspace * endset -local simpledoctype = (1-close)^1 -- * balanced^0 + +local normalentitytype = (doctypename * somespace * value)/normalentity +local publicentitytype = (doctypename * somespace * P("PUBLIC") * somespace * value)/publicentity +local systementitytype = (doctypename * somespace * P("SYSTEM") * somespace * value * somespace * P("NDATA") * somespace * doctypename)/systementity +local entitydoctype = optionalspace * P("<!ENTITY") * somespace * (systementitytype + publicentitytype + normalentitytype) * optionalspace * close + +local doctypeset = beginset * optionalspace * P(elementdoctype + entitydoctype + space)^0 * optionalspace * endset +local definitiondoctype= doctypename * somespace * doctypeset +local publicdoctype = doctypename * somespace * P("PUBLIC") * somespace * value * somespace * value * somespace * doctypeset +local systemdoctype = doctypename * somespace * P("SYSTEM") * somespace * value * somespace * doctypeset +local simpledoctype = (1-close)^1 -- * balanced^0 local somedoctype = C((somespace * (publicdoctype + systemdoctype + definitiondoctype + simpledoctype) * optionalspace)^0) local instruction = (spacing * begininstruction * someinstruction * endinstruction) / function(...) add_special("@pi@",...) end @@ -3179,60 +4437,116 @@ local doctype = (spacing * begindoctype * somedoctype * enddoct -- nicer but slower: -- --- local instruction = (lpeg.Cc("@pi@") * spacing * begininstruction * someinstruction * endinstruction) / add_special --- local comment = (lpeg.Cc("@cm@") * spacing * begincomment * somecomment * endcomment ) / add_special --- local cdata = (lpeg.Cc("@cd@") * spacing * begincdata * somecdata * endcdata ) / add_special --- local doctype = (lpeg.Cc("@dt@") * spacing * begindoctype * somedoctype * enddoctype ) / add_special +-- local instruction = (Cc("@pi@") * spacing * begininstruction * someinstruction * endinstruction) / add_special +-- local comment = (Cc("@cm@") * spacing * begincomment * somecomment * endcomment ) / add_special +-- local cdata = (Cc("@cd@") * spacing * begincdata * somecdata * endcdata ) / add_special +-- local doctype = (Cc("@dt@") * spacing * begindoctype * somedoctype * enddoctype ) / add_special -local trailer = space^0 * (justtext/set_message)^0 +local trailer = space^0 * (text_unparsed/set_message)^0 -- comment + emptyelement + text + cdata + instruction + V("parent"), -- 6.5 seconds on 40 MB database file -- text + comment + emptyelement + cdata + instruction + V("parent"), -- 5.8 -- text + V("parent") + emptyelement + comment + cdata + instruction, -- 5.5 -local grammar = P { "preamble", +local grammar_parsed_text = P { "preamble", preamble = utfbom^0 * instruction^0 * (doctype + comment + instruction)^0 * V("parent") * trailer, parent = beginelement * V("children")^0 * endelement, - children = text + V("parent") + emptyelement + comment + cdata + instruction, + children = parsedtext + V("parent") + emptyelement + comment + cdata + instruction, } --- todo: xml.new + properties like entities and strip and such (store in root) +local grammar_unparsed_text = P { "preamble", + preamble = utfbom^0 * instruction^0 * (doctype + comment + instruction)^0 * V("parent") * trailer, + parent = beginelement * V("children")^0 * endelement, + children = unparsedtext + V("parent") + emptyelement + comment + cdata + instruction, +} -function xml.convert(data, no_root, strip_cm_and_dt, given_entities) -- maybe use table met k/v (given_entities may disapear) - strip = strip_cm_and_dt or xml.strip_cm_and_dt - stack, top, at, xmlns, errorstr, result, entities = {}, {}, {}, {}, nil, nil, given_entities or {} +-- maybe we will add settinsg to result as well + +local function xmlconvert(data, settings) + settings = settings or { } -- no_root strip_cm_and_dt given_entities parent_root error_handler + strip = settings.strip_cm_and_dt + utfize = settings.utfize_entities + resolve = settings.resolve_entities + resolve_predefined = settings.resolve_predefined_entities -- in case we have escaped entities + unify_predefined = settings.unify_predefined_entities -- & -> & + cleanup = settings.text_cleanup + stack, top, at, xmlns, errorstr, result, entities = { }, { }, { }, { }, nil, nil, settings.entities or { } + acache, hcache, dcache = { }, { }, { } -- not stored + reported_attribute_errors = { } + if settings.parent_root then + mt = getmetatable(settings.parent_root) + else + initialize_mt(top) + end stack[#stack+1] = top top.dt = { } dt = top.dt if not data or data == "" then errorstr = "empty xml file" - elseif not grammar:match(data) then - errorstr = "invalid xml file" + elseif utfize or resolve then + if lpegmatch(grammar_parsed_text,data) then + errorstr = "" + else + errorstr = "invalid xml file - parsed text" + end + elseif type(data) == "string" then + if lpegmatch(grammar_unparsed_text,data) then + errorstr = "" + else + errorstr = "invalid xml file - unparsed text" + end else - errorstr = "" + errorstr = "invalid xml file - no text at all" end if errorstr and errorstr ~= "" then - result = { dt = { { ns = "", tg = "error", dt = { errorstr }, at={}, er = true } }, error = true } + result = { dt = { { ns = "", tg = "error", dt = { errorstr }, at={ }, er = true } } } setmetatable(stack, mt) - if xml.error_handler then xml.error_handler("load",errorstr) end + local error_handler = settings.error_handler + if error_handler == false then + -- no error message + else + error_handler = error_handler or xml.error_handler + if error_handler then + xml.error_handler("load",errorstr) + end + end else result = stack[1] end - if not no_root then - result = { special = true, ns = "", tg = '@rt@', dt = result.dt, at={}, entities = entities } + if not settings.no_root then + result = { special = true, ns = "", tg = '@rt@', dt = result.dt, at={ }, entities = entities, settings = settings } setmetatable(result, mt) local rdt = result.dt for k=1,#rdt do local v = rdt[k] if type(v) == "table" and not v.special then -- always table -) result.ri = k -- rootindex +v.__p__ = result -- new, experiment, else we cannot go back to settings, we need to test this ! break end end end + if errorstr and errorstr ~= "" then + result.error = true + end return result end +xml.convert = xmlconvert + +function xml.inheritedconvert(data,xmldata) + local settings = xmldata.settings + settings.parent_root = xmldata -- to be tested + -- settings.no_root = true + local xc = xmlconvert(data,settings) + -- xc.settings = nil + -- xc.entities = nil + -- xc.special = nil + -- xc.ri = nil + -- print(xc.tg) + return xc +end + --[[ldx-- <p>Packaging data in an xml like table is done with the following function. Maybe it will go away (when not used).</p> @@ -3243,7 +4557,7 @@ function xml.is_valid(root) end function xml.package(tag,attributes,data) - local ns, tg = tag:match("^(.-):?([^:]+)$") + local ns, tg = match(tag,"^(.-):?([^:]+)$") local t = { ns = ns, tg = tg, dt = data or "", at = attributes or {} } setmetatable(t, mt) return t @@ -3261,21 +4575,19 @@ the whole file first. The function accepts a string representing a filename or a file handle.</p> --ldx]]-- -function xml.load(filename) +function xml.load(filename,settings) + local data = "" if type(filename) == "string" then + -- local data = io.loaddata(filename) - -todo: check type in io.loaddata local f = io.open(filename,'r') if f then - local root = xml.convert(f:read("*all")) + data = f:read("*all") f:close() - return root - else - return xml.convert("") end elseif filename then -- filehandle - return xml.convert(filename:read("*all")) - else - return xml.convert("") + data = filename:read("*all") end + return xmlconvert(data,settings) end --[[ldx-- @@ -3283,9 +4595,11 @@ end valid trees, which is what the next function does.</p> --ldx]]-- +local no_root = { no_root = true } + function xml.toxml(data) if type(data) == "string" then - local root = { xml.convert(data,true) } + local root = { xmlconvert(data,no_root) } return (#root > 1 and root) or root[1] else return data @@ -3305,7 +4619,7 @@ local function copy(old,tables) if not tables[old] then tables[old] = new end - for k,v in pairs(old) do + for k,v in next, old do new[k] = (type(v) == "table" and (tables[v] or copy(v, tables))) or v end local mt = getmetatable(old) @@ -3330,217 +4644,304 @@ alternative.</p> -- todo: add <?xml version='1.0' standalone='yes'?> when not present -local fallbackhandle = (tex and tex.sprint) or io.write - -local function serialize(e, handle, textconverter, attributeconverter, specialconverter, nocommands) - if not e then - return - elseif not nocommands then - local ec = e.command - if ec ~= nil then -- we can have all kind of types - if e.special then - local etg, edt = e.tg, e.dt - local spc = specialconverter and specialconverter[etg] - if spc then - local result = spc(edt[1]) - if result then - handle(result) - return - else - -- no need to handle any further - end - end - end - local xc = xml.command - if xc then - xc(e,ec) - return +function xml.checkbom(root) -- can be made faster + if root.ri then + local dt, found = root.dt, false + for k=1,#dt do + local v = dt[k] + if type(v) == "table" and v.special and v.tg == "@pi@" and find(v.dt[1],"xml.*version=") then + found = true + break end end + if not found then + insert(dt, 1, { special=true, ns="", tg="@pi@", dt = { "xml version='1.0' standalone='yes'"} } ) + insert(dt, 2, "\n" ) + end end - handle = handle or fallbackhandle - local etg = e.tg - if etg then - if e.special then - local edt = e.dt - local spc = specialconverter and specialconverter[etg] - if spc then - local result = spc(edt[1]) - if result then - handle(result) +end + +--[[ldx-- +<p>At the cost of some 25% runtime overhead you can first convert the tree to a string +and then handle the lot.</p> +--ldx]]-- + +-- new experimental reorganized serialize + +local function verbose_element(e,handlers) + local handle = handlers.handle + local serialize = handlers.serialize + local ens, etg, eat, edt, ern = e.ns, e.tg, e.at, e.dt, e.rn + local ats = eat and next(eat) and { } + if ats then + for k,v in next, eat do + ats[#ats+1] = format('%s=%q',k,v) + end + end + if ern and trace_remap and ern ~= ens then + ens = ern + end + if ens ~= "" then + if edt and #edt > 0 then + if ats then + handle("<",ens,":",etg," ",concat(ats," "),">") + else + handle("<",ens,":",etg,">") + end + for i=1,#edt do + local e = edt[i] + if type(e) == "string" then + handle(e) else - -- no need to handle any further - end - elseif etg == "@pi@" then - -- handle(format("<?%s?>",edt[1])) - handle("<?" .. edt[1] .. "?>") - elseif etg == "@cm@" then - -- handle(format("<!--%s-->",edt[1])) - handle("<!--" .. edt[1] .. "-->") - elseif etg == "@cd@" then - -- handle(format("<![CDATA[%s]]>",edt[1])) - handle("<![CDATA[" .. edt[1] .. "]]>") - elseif etg == "@dt@" then - -- handle(format("<!DOCTYPE %s>",edt[1])) - handle("<!DOCTYPE " .. edt[1] .. ">") - elseif etg == "@rt@" then - serialize(edt,handle,textconverter,attributeconverter,specialconverter,nocommands) + serialize(e,handlers) + end end + handle("</",ens,":",etg,">") else - local ens, eat, edt, ern = e.ns, e.at, e.dt, e.rn - local ats = eat and next(eat) and { } -- type test maybe faster if ats then - if attributeconverter then - for k,v in next, eat do - ats[#ats+1] = format('%s=%q',k,attributeconverter(v)) - end - else - for k,v in next, eat do - ats[#ats+1] = format('%s=%q',k,v) - end - end + handle("<",ens,":",etg," ",concat(ats," "),"/>") + else + handle("<",ens,":",etg,"/>") end - if ern and trace_remap and ern ~= ens then - ens = ern + end + else + if edt and #edt > 0 then + if ats then + handle("<",etg," ",concat(ats," "),">") + else + handle("<",etg,">") end - if ens ~= "" then - if edt and #edt > 0 then - if ats then - -- handle(format("<%s:%s %s>",ens,etg,concat(ats," "))) - handle("<" .. ens .. ":" .. etg .. " " .. concat(ats," ") .. ">") - else - -- handle(format("<%s:%s>",ens,etg)) - handle("<" .. ens .. ":" .. etg .. ">") - end - for i=1,#edt do - local e = edt[i] - if type(e) == "string" then - if textconverter then - handle(textconverter(e)) - else - handle(e) - end - else - serialize(e,handle,textconverter,attributeconverter,specialconverter,nocommands) - end - end - -- handle(format("</%s:%s>",ens,etg)) - handle("</" .. ens .. ":" .. etg .. ">") + for i=1,#edt do + local ei = edt[i] + if type(ei) == "string" then + handle(ei) else - if ats then - -- handle(format("<%s:%s %s/>",ens,etg,concat(ats," "))) - handle("<" .. ens .. ":" .. etg .. " " .. concat(ats," ") .. "/>") - else - -- handle(format("<%s:%s/>",ens,etg)) - handle("<" .. ens .. ":" .. etg .. "/>") - end + serialize(ei,handlers) end + end + handle("</",etg,">") + else + if ats then + handle("<",etg," ",concat(ats," "),"/>") else - if edt and #edt > 0 then - if ats then - -- handle(format("<%s %s>",etg,concat(ats," "))) - handle("<" .. etg .. " " .. concat(ats," ") .. ">") - else - -- handle(format("<%s>",etg)) - handle("<" .. etg .. ">") - end - for i=1,#edt do - local ei = edt[i] - if type(ei) == "string" then - if textconverter then - handle(textconverter(ei)) - else - handle(ei) - end - else - serialize(ei,handle,textconverter,attributeconverter,specialconverter,nocommands) - end - end - -- handle(format("</%s>",etg)) - handle("</" .. etg .. ">") - else - if ats then - -- handle(format("<%s %s/>",etg,concat(ats," "))) - handle("<" .. etg .. " " .. concat(ats," ") .. "/>") - else - -- handle(format("<%s/>",etg)) - handle("<" .. etg .. "/>") - end - end + handle("<",etg,"/>") end end - elseif type(e) == "string" then - if textconverter then - handle(textconverter(e)) + end +end + +local function verbose_pi(e,handlers) + handlers.handle("<?",e.dt[1],"?>") +end + +local function verbose_comment(e,handlers) + handlers.handle("<!--",e.dt[1],"-->") +end + +local function verbose_cdata(e,handlers) + handlers.handle("<![CDATA[", e.dt[1],"]]>") +end + +local function verbose_doctype(e,handlers) + handlers.handle("<!DOCTYPE ",e.dt[1],">") +end + +local function verbose_root(e,handlers) + handlers.serialize(e.dt,handlers) +end + +local function verbose_text(e,handlers) + handlers.handle(e) +end + +local function verbose_document(e,handlers) + local serialize = handlers.serialize + local functions = handlers.functions + for i=1,#e do + local ei = e[i] + if type(ei) == "string" then + functions["@tx@"](ei,handlers) else - handle(e) + serialize(ei,handlers) end - else - for i=1,#e do - local ei = e[i] - if type(ei) == "string" then - if textconverter then - handle(textconverter(ei)) - else - handle(ei) - end - else - serialize(ei,handle,textconverter,attributeconverter,specialconverter,nocommands) - end + end +end + +local function serialize(e,handlers,...) + local initialize = handlers.initialize + local finalize = handlers.finalize + local functions = handlers.functions + if initialize then + local state = initialize(...) + if not state == true then + return state end end + local etg = e.tg + if etg then + (functions[etg] or functions["@el@"])(e,handlers) + -- elseif type(e) == "string" then + -- functions["@tx@"](e,handlers) + else + functions["@dc@"](e,handlers) + end + if finalize then + return finalize() + end end -xml.serialize = serialize +local function xserialize(e,handlers) + local functions = handlers.functions + local etg = e.tg + if etg then + (functions[etg] or functions["@el@"])(e,handlers) + -- elseif type(e) == "string" then + -- functions["@tx@"](e,handlers) + else + functions["@dc@"](e,handlers) + end +end -function xml.checkbom(root) -- can be made faster - if root.ri then - local dt, found = root.dt, false - for k=1,#dt do - local v = dt[k] - if type(v) == "table" and v.special and v.tg == "@pi" and find(v.dt,"xml.*version=") then - found = true - break +local handlers = { } + +local function newhandlers(settings) + local t = table.copy(handlers.verbose or { }) -- merge + if settings then + for k,v in next, settings do + if type(v) == "table" then + tk = t[k] if not tk then tk = { } t[k] = tk end + for kk,vv in next, v do + tk[kk] = vv + end + else + t[k] = v end end - if not found then - insert(dt, 1, { special=true, ns="", tg="@pi@", dt = { "xml version='1.0' standalone='yes'"} } ) - insert(dt, 2, "\n" ) + if settings.name then + handlers[settings.name] = t end end + return t end +local nofunction = function() end + +function xml.sethandlersfunction(handler,name,fnc) + handler.functions[name] = fnc or nofunction +end + +function xml.gethandlersfunction(handler,name) + return handler.functions[name] +end + +function xml.gethandlers(name) + return handlers[name] +end + +newhandlers { + name = "verbose", + initialize = false, -- faster than nil and mt lookup + finalize = false, -- faster than nil and mt lookup + serialize = xserialize, + handle = print, + functions = { + ["@dc@"] = verbose_document, + ["@dt@"] = verbose_doctype, + ["@rt@"] = verbose_root, + ["@el@"] = verbose_element, + ["@pi@"] = verbose_pi, + ["@cm@"] = verbose_comment, + ["@cd@"] = verbose_cdata, + ["@tx@"] = verbose_text, + } +} + --[[ldx-- -<p>At the cost of some 25% runtime overhead you can first convert the tree to a string -and then handle the lot.</p> +<p>How you deal with saving data depends on your preferences. For a 40 MB database +file the timing on a 2.3 Core Duo are as follows (time in seconds):</p> + +<lines> +1.3 : load data from file to string +6.1 : convert string into tree +5.3 : saving in file using xmlsave +6.8 : converting to string using xml.tostring +3.6 : saving converted string in file +</lines> + +<p>Beware, these were timing with the old routine but measurements will not be that +much different I guess.</p> --ldx]]-- -function xml.tostring(root) -- 25% overhead due to collecting +-- maybe this will move to lxml-xml + +local result + +local xmlfilehandler = newhandlers { + name = "file", + initialize = function(name) result = io.open(name,"wb") return result end, + finalize = function() result:close() return true end, + handle = function(...) result:write(...) end, +} + +-- no checking on writeability here but not faster either +-- +-- local xmlfilehandler = newhandlers { +-- initialize = function(name) io.output(name,"wb") return true end, +-- finalize = function() io.close() return true end, +-- handle = io.write, +-- } + + +function xml.save(root,name) + serialize(root,xmlfilehandler,name) +end + +local result + +local xmlstringhandler = newhandlers { + name = "string", + initialize = function() result = { } return result end, + finalize = function() return concat(result) end, + handle = function(...) result[#result+1] = concat { ... } end +} + +local function xmltostring(root) -- 25% overhead due to collecting if root then if type(root) == 'string' then return root - elseif next(root) then -- next is faster than type (and >0 test) - local result = { } - serialize(root,function(s) result[#result+1] = s end) -- brrr, slow (direct printing is faster) - return concat(result,"") + else -- if next(root) then -- next is faster than type (and >0 test) + return serialize(root,xmlstringhandler) or "" end end return "" end +local function xmltext(root) -- inline + return (root and xmltostring(root)) or "" +end + +function initialize_mt(root) + mt = { __tostring = xmltext, __index = root } +end + +xml.defaulthandlers = handlers +xml.newhandlers = newhandlers +xml.serialize = serialize +xml.tostring = xmltostring + --[[ldx-- <p>The next function operated on the content only and needs a handle function that accepts a string.</p> --ldx]]-- -function xml.string(e,handle) +local function xmlstring(e,handle) if not handle or (e.special and e.tg ~= "@rt@") then -- nothing elseif e.tg then local edt = e.dt if edt then for i=1,#edt do - xml.string(edt[i],handle) + xmlstring(edt[i],handle) end end else @@ -3548,54 +4949,52 @@ function xml.string(e,handle) end end ---[[ldx-- -<p>How you deal with saving data depends on your preferences. For a 40 MB database -file the timing on a 2.3 Core Duo are as follows (time in seconds):</p> +xml.string = xmlstring -<lines> -1.3 : load data from file to string -6.1 : convert string into tree -5.3 : saving in file using xmlsave -6.8 : converting to string using xml.tostring -3.6 : saving converted string in file -</lines> - -<p>The save function is given below.</p> +--[[ldx-- +<p>A few helpers:</p> --ldx]]-- -function xml.save(root,name) - local f = io.open(name,"w") - if f then - xml.serialize(root,function(s) f:write(s) end) - f:close() +--~ xmlsetproperty(root,"settings",settings) + +function xml.settings(e) + while e do + local s = e.settings + if s then + return s + else + e = e.__p__ + end end + return nil end ---[[ldx-- -<p>A few helpers:</p> ---ldx]]-- - -function xml.body(root) - return (root.ri and root.dt[root.ri]) or root +function xml.root(e) + local r = e + while e do + e = e.__p__ + if e then + r = e + end + end + return r end -function xml.text(root) - return (root and xml.tostring(root)) or "" +function xml.parent(root) + return root.__p__ end -function xml.content(root) -- bugged - return (root and root.dt and xml.tostring(root.dt)) or "" +function xml.body(root) + return (root.ri and root.dt[root.ri]) or root -- not ok yet end -function xml.isempty(root, pattern) - if pattern == "" or pattern == "*" then - pattern = nil - end - if pattern then - -- todo - return false +function xml.name(root) + if not root then + return "" + elseif root.ns == "" then + return root.tg else - return not root or not root.dt or #root.dt == 0 or root.dt == "" + return root.ns .. ":" .. root.tg end end @@ -3603,18 +5002,15 @@ end <p>The next helper erases an element but keeps the table as it is, and since empty strings are not serialized (effectively) it does not harm. Copying the table would take more time. Usage:</p> - -<typing> -dt[k] = xml.empty() or xml.empty(dt,k) -</typing> --ldx]]-- -function xml.empty(dt,k) - if dt and k then - dt[k] = "" - return dt[k] - else - return "" +function xml.erase(dt,k) + if dt then + if k then + dt[k] = "" + else for k=1,#dt do + dt[1] = { "" } + end end end end @@ -3635,6 +5031,42 @@ function xml.assign(dt,k,root) end end +-- the following helpers may move + +--[[ldx-- +<p>The next helper assigns a tree (or string). Usage:</p> +<typing> +xml.tocdata(e) +xml.tocdata(e,"error") +</typing> +--ldx]]-- + +function xml.tocdata(e,wrapper) + local whatever = xmltostring(e.dt) + if wrapper then + whatever = format("<%s>%s</%s>",wrapper,whatever,wrapper) + end + local t = { special = true, ns = "", tg = "@cd@", at = {}, rn = "", dt = { whatever }, __p__ = e } + setmetatable(t,getmetatable(e)) + e.dt = { t } +end + +function xml.makestandalone(root) + if root.ri then + local dt = root.dt + for k=1,#dt do + local v = dt[k] + if type(v) == "table" and v.special and v.tg == "@pi@" then + local txt = v.dt[1] + if find(txt,"xml.*version=") then + v.dt[1] = txt .. " standalone='yes'" + break + end + end + end + end +end + end -- of closure @@ -3648,1420 +5080,1285 @@ if not modules then modules = { } end modules ['lxml-pth'] = { license = "see context related readme files" } +-- e.ni is only valid after a filter run + local concat, remove, insert = table.concat, table.remove, table.insert local type, next, tonumber, tostring, setmetatable, loadstring = type, next, tonumber, tostring, setmetatable, loadstring -local format, lower, gmatch, gsub, find = string.format, string.lower, string.gmatch, string.gsub, string.find +local format, upper, lower, gmatch, gsub, find, rep = string.format, string.upper, string.lower, string.gmatch, string.gsub, string.find, string.rep +local lpegmatch = lpeg.match + +-- beware, this is not xpath ... e.g. position is different (currently) and +-- we have reverse-sibling as reversed preceding sibling --[[ldx-- <p>This module can be used stand alone but also inside <l n='mkiv'/> in which case it hooks into the tracker code. Therefore we provide a few functions that set the tracers. Here we overload a previously defined function.</p> +<p>If I can get in the mood I will make a variant that is XSLT compliant +but I wonder if it makes sense.</P> --ldx]]-- -local trace_lpath = false - -if trackers then - trackers.register("xml.lpath", function(v) trace_lpath = v end) -end +--[[ldx-- +<p>Expecially the lpath code is experimental, we will support some of xpath, but +only things that make sense for us; as compensation it is possible to hook in your +own functions. Apart from preprocessing content for <l n='context'/> we also need +this module for process management, like handling <l n='ctx'/> and <l n='rlx'/> +files.</p> -local settrace = xml.settrace -- lxml-tab +<typing> +a/b/c /*/c +a/b/c/first() a/b/c/last() a/b/c/index(n) a/b/c/index(-n) +a/b/c/text() a/b/c/text(1) a/b/c/text(-1) a/b/c/text(n) +</typing> +--ldx]]-- -function xml.settrace(str,value) - if str == "lpath" then - trace_lpath = value or false - else - settrace(str,value) -- lxml-tab - end -end +local trace_lpath = false if trackers then trackers.register("xml.path", function(v) trace_lpath = v end) end +local trace_lparse = false if trackers then trackers.register("xml.parse", function(v) trace_lparse = v end) end +local trace_lprofile = false if trackers then trackers.register("xml.profile", function(v) trace_lpath = v trace_lparse = v trace_lprofile = v end) end --[[ldx-- -<p>We've now arrived at an intersting part: accessing the tree using a subset +<p>We've now arrived at an interesting part: accessing the tree using a subset of <l n='xpath'/> and since we're not compatible we call it <l n='lpath'/>. We will explain more about its usage in other documents.</p> --ldx]]-- -local lpathcalls = 0 -- statistics -local lpathcached = 0 -- statistics +local lpathcalls = 0 function xml.lpathcalls () return lpathcalls end +local lpathcached = 0 function xml.lpathcached() return lpathcached end -xml.functions = xml.functions or { } -xml.expressions = xml.expressions or { } +xml.functions = xml.functions or { } -- internal +xml.expressions = xml.expressions or { } -- in expressions +xml.finalizers = xml.finalizers or { } -- fast do-with ... (with return value other than collection) +xml.specialhandler = xml.specialhandler or { } local functions = xml.functions local expressions = xml.expressions +local finalizers = xml.finalizers -local actions = { - [10] = "stay", - [11] = "parent", - [12] = "subtree root", - [13] = "document root", - [14] = "any", - [15] = "many", - [16] = "initial", - [20] = "match", - [21] = "match one of", - [22] = "match and attribute eq", - [23] = "match and attribute ne", - [24] = "match one of and attribute eq", - [25] = "match one of and attribute ne", - [27] = "has attribute", - [28] = "has value", - [29] = "fast match", - [30] = "select", - [31] = "expression", - [40] = "processing instruction", -} - --- a rather dumb lpeg +finalizers.xml = finalizers.xml or { } +finalizers.tex = finalizers.tex or { } -local P, S, R, C, V, Cc = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc +local function fallback (t, name) + local fn = finalizers[name] + if fn then + t[name] = fn + else + logs.report("xml","unknown sub finalizer '%s'",tostring(name)) + fn = function() end + end + return fn +end --- instead of using functions we just parse a few names which saves a call --- later on +setmetatable(finalizers.xml, { __index = fallback }) +setmetatable(finalizers.tex, { __index = fallback }) -local lp_position = P("position()") / "ps" -local lp_index = P("index()") / "id" -local lp_text = P("text()") / "tx" -local lp_name = P("name()") / "(ns~='' and ns..':'..tg)" -- "((rt.ns~='' and rt.ns..':'..rt.tg) or '')" -local lp_tag = P("tag()") / "tg" -- (rt.tg or '') -local lp_ns = P("ns()") / "ns" -- (rt.ns or '') -local lp_noequal = P("!=") / "~=" + P("<=") + P(">=") + P("==") -local lp_doequal = P("=") / "==" -local lp_attribute = P("@") / "" * Cc("(at['") * R("az","AZ","--","__")^1 * Cc("'] or '')") +xml.defaultprotocol = "xml" -local lp_lua_function = C(R("az","AZ","--","__")^1 * (P(".") * R("az","AZ","--","__")^1)^1) * P("(") / function(t) -- todo: better . handling - return t .. "(" -end +-- as xsl does not follow xpath completely here we will also +-- be more liberal especially with regards to the use of | and +-- the rootpath: +-- +-- test : all 'test' under current +-- /test : 'test' relative to current +-- a|b|c : set of names +-- (a|b|c) : idem +-- ! : not +-- +-- after all, we're not doing transformations but filtering. in +-- addition we provide filter functions (last bit) +-- +-- todo: optimizer +-- +-- .. : parent +-- * : all kids +-- / : anchor here +-- // : /**/ +-- ** : all in between +-- +-- so far we had (more practical as we don't transform) +-- +-- {/test} : kids 'test' under current node +-- {test} : any kid with tag 'test' +-- {//test} : same as above -local lp_function = C(R("az","AZ","--","__")^1) * P("(") / function(t) -- todo: better . handling - if expressions[t] then - return "expressions." .. t .. "(" - else - return "expressions.error(" - end -end +-- evaluator (needs to be redone, for the moment copied) -local lparent = lpeg.P("(") -local rparent = lpeg.P(")") -local noparent = 1 - (lparent+rparent) -local nested = lpeg.P{lparent * (noparent + lpeg.V(1))^0 * rparent} -local value = lpeg.P(lparent * lpeg.C((noparent + nested)^0) * rparent) -- lpeg.P{"("*C(((1-S("()"))+V(1))^0)*")"} +-- todo: apply_axis(list,notable) and collection vs single --- if we use a dedicated namespace then we don't need to pass rt and k +local apply_axis = { } -local lp_special = (C(P("name")+P("text")+P("tag"))) * value / function(t,s) - if expressions[t] then - if s then - return "expressions." .. t .. "(r,k," .. s ..")" - else - return "expressions." .. t .. "(r,k)" +apply_axis['root'] = function(list) + local collected = { } + for l=1,#list do + local ll = list[l] + local rt = ll + while ll do + ll = ll.__p__ + if ll then + rt = ll + end end - else - return "expressions.error(" .. t .. ")" + collected[#collected+1] = rt end + return collected end -local converter = lpeg.Cs ( ( - lp_position + - lp_index + - lp_text + lp_name + -- fast one - lp_special + - lp_noequal + lp_doequal + - lp_attribute + - lp_lua_function + - lp_function + -1 )^1 ) - --- expressions,root,rootdt,k,e,edt,ns,tg,idx,hsh[tg] or 1 +apply_axis['self'] = function(list) +--~ local collected = { } +--~ for l=1,#list do +--~ collected[#collected+1] = list[l] +--~ end +--~ return collected + return list +end -local template = [[ - return function(expressions,r,d,k,e,dt,ns,tg,id,ps) - local at, tx = e.at or { }, dt[1] or "" - return %s +apply_axis['child'] = function(list) + local collected = { } + for l=1,#list do + local ll = list[l] + local dt = ll.dt + local en = 0 + for k=1,#dt do + local dk = dt[k] + if dk.tg then + collected[#collected+1] = dk + dk.ni = k -- refresh + en = en + 1 + dk.ei = en + end + end + ll.en = en end -]] - -local function make_expression(str) - str = converter:match(str) - return str, loadstring(format(template,str))() -end - -local map = { } - -local space = S(' \r\n\t') -local squote = S("'") -local dquote = S('"') -local lparent = P('(') -local rparent = P(')') -local atsign = P('@') -local lbracket = P('[') -local rbracket = P(']') -local exclam = P('!') -local period = P('.') -local eq = P('==') + P('=') -local ne = P('<>') + P('!=') -local star = P('*') -local slash = P('/') -local colon = P(':') -local bar = P('|') -local hat = P('^') -local valid = R('az', 'AZ', '09') + S('_-') -local name_yes = C(valid^1 + star) * colon * C(valid^1 + star) -- permits ns:* *:tg *:* -local name_nop = Cc("*") * C(valid^1) -local name = name_yes + name_nop -local number = C((S('+-')^0 * R('09')^1)) / tonumber -local names = (bar^0 * name)^1 -local morenames = name * (bar^0 * name)^1 -local instructiontag = P('pi::') -local spacing = C(space^0) -local somespace = space^1 -local optionalspace = space^0 -local text = C(valid^0) -local value = (squote * C((1 - squote)^0) * squote) + (dquote * C((1 - dquote)^0) * dquote) -local empty = 1-slash - -local is_eq = lbracket * atsign * name * eq * value * rbracket -local is_ne = lbracket * atsign * name * ne * value * rbracket -local is_attribute = lbracket * atsign * name * rbracket -local is_value = lbracket * value * rbracket -local is_number = lbracket * number * rbracket - -local nobracket = 1-(lbracket+rbracket) -- must be improved -local is_expression = lbracket * C(((C(nobracket^1))/make_expression)) * rbracket - -local is_expression = lbracket * (C(nobracket^1))/make_expression * rbracket - -local is_one = name -local is_none = exclam * name -local is_one_of = ((lparent * names * rparent) + morenames) -local is_none_of = exclam * ((lparent * names * rparent) + morenames) - -local stay = (period ) -local parent = (period * period ) / function( ) map[#map+1] = { 11 } end -local subtreeroot = (slash + hat ) / function( ) map[#map+1] = { 12 } end -local documentroot = (hat * hat ) / function( ) map[#map+1] = { 13 } end -local any = (star ) / function( ) map[#map+1] = { 14 } end -local many = (star * star ) / function( ) map[#map+1] = { 15 } end -local initial = (hat * hat * hat ) / function( ) map[#map+1] = { 16 } end - -local match = (is_one ) / function(...) map[#map+1] = { 20, true , ... } end -local match_one_of = (is_one_of ) / function(...) map[#map+1] = { 21, true , ... } end -local dont_match = (is_none ) / function(...) map[#map+1] = { 20, false, ... } end -local dont_match_one_of = (is_none_of ) / function(...) map[#map+1] = { 21, false, ... } end - -local match_and_eq = (is_one * is_eq ) / function(...) map[#map+1] = { 22, true , ... } end -local match_and_ne = (is_one * is_ne ) / function(...) map[#map+1] = { 23, true , ... } end -local dont_match_and_eq = (is_none * is_eq ) / function(...) map[#map+1] = { 22, false, ... } end -local dont_match_and_ne = (is_none * is_ne ) / function(...) map[#map+1] = { 23, false, ... } end - -local match_one_of_and_eq = (is_one_of * is_eq ) / function(...) map[#map+1] = { 24, true , ... } end -local match_one_of_and_ne = (is_one_of * is_ne ) / function(...) map[#map+1] = { 25, true , ... } end -local dont_match_one_of_and_eq = (is_none_of * is_eq ) / function(...) map[#map+1] = { 24, false, ... } end -local dont_match_one_of_and_ne = (is_none_of * is_ne ) / function(...) map[#map+1] = { 25, false, ... } end - -local has_attribute = (is_one * is_attribute) / function(...) map[#map+1] = { 27, true , ... } end -local has_value = (is_one * is_value ) / function(...) map[#map+1] = { 28, true , ... } end -local dont_has_attribute = (is_none * is_attribute) / function(...) map[#map+1] = { 27, false, ... } end -local dont_has_value = (is_none * is_value ) / function(...) map[#map+1] = { 28, false, ... } end -local position = (is_one * is_number ) / function(...) map[#map+1] = { 30, true, ... } end -local dont_position = (is_none * is_number ) / function(...) map[#map+1] = { 30, false, ... } end - -local expression = (is_one * is_expression)/ function(...) map[#map+1] = { 31, true, ... } end -local dont_expression = (is_none * is_expression)/ function(...) map[#map+1] = { 31, false, ... } end - -local self_expression = ( is_expression) / function(...) if #map == 0 then map[#map+1] = { 11 } end - map[#map+1] = { 31, true, "*", "*", ... } end -local dont_self_expression = (exclam * is_expression) / function(...) if #map == 0 then map[#map+1] = { 11 } end - map[#map+1] = { 31, false, "*", "*", ... } end - -local instruction = (instructiontag * text ) / function(...) map[#map+1] = { 40, ... } end -local nothing = (empty ) / function( ) map[#map+1] = { 15 } end -- 15 ? -local crap = (1-slash)^1 - --- a few ugly goodies: - -local docroottag = P('^^') / function( ) map[#map+1] = { 12 } end -local subroottag = P('^') / function( ) map[#map+1] = { 13 } end -local roottag = P('root::') / function( ) map[#map+1] = { 12 } end -local parenttag = P('parent::') / function( ) map[#map+1] = { 11 } end -local childtag = P('child::') -local selftag = P('self::') - --- there will be more and order will be optimized - -local selector = ( - instruction + --- many + any + -- brrr, not here ! - parent + stay + - dont_position + position + - dont_match_one_of_and_eq + dont_match_one_of_and_ne + - match_one_of_and_eq + match_one_of_and_ne + - dont_match_and_eq + dont_match_and_ne + - match_and_eq + match_and_ne + - dont_expression + expression + - dont_self_expression + self_expression + - has_attribute + has_value + - dont_match_one_of + match_one_of + - dont_match + match + - many + any + - crap + empty -) - -local grammar = P { "startup", - startup = (initial + documentroot + subtreeroot + roottag + docroottag + subroottag)^0 * V("followup"), - followup = ((slash + parenttag + childtag + selftag)^0 * selector)^1, -} + return collected +end -local function compose(str) - if not str or str == "" then - -- wildcard - return true - elseif str == '/' then - -- root - return false - else - map = { } - grammar:match(str) - if #map == 0 then - return true - else - local m = map[1][1] - if #map == 1 then - if m == 14 or m == 15 then - -- wildcard - return true - elseif m == 12 then - -- root - return false - end - elseif #map == 2 and m == 12 and map[2][1] == 20 then - -- return { { 29, map[2][2], map[2][3], map[2][4], map[2][5] } } - map[2][1] = 29 - return { map[2] } - end - if m ~= 11 and m ~= 12 and m ~= 13 and m ~= 14 and m ~= 15 and m ~= 16 then - insert(map, 1, { 16 }) +local function collect(list,collected) + local dt = list.dt + if dt then + local en = 0 + for k=1,#dt do + local dk = dt[k] + if dk.tg then + collected[#collected+1] = dk + dk.ni = k -- refresh + en = en + 1 + dk.ei = en + collect(dk,collected) end - -- print(gsub(table.serialize(map),"[ \n]+"," ")) - return map end + list.en = en end end +apply_axis['descendant'] = function(list) + local collected = { } + for l=1,#list do + collect(list[l],collected) + end + return collected +end -local cache = { } - -function xml.lpath(pattern,trace) - lpathcalls = lpathcalls + 1 - if type(pattern) == "string" then - local result = cache[pattern] - if result == nil then -- can be false which is valid -) - result = compose(pattern) - cache[pattern] = result - lpathcached = lpathcached + 1 +local function collect(list,collected) + local dt = list.dt + if dt then + local en = 0 + for k=1,#dt do + local dk = dt[k] + if dk.tg then + collected[#collected+1] = dk + dk.ni = k -- refresh + en = en + 1 + dk.ei = en + collect(dk,collected) + end end - if trace or trace_lpath then - xml.lshow(result) + list.en = en + end +end +apply_axis['descendant-or-self'] = function(list) + local collected = { } + for l=1,#list do + local ll = list[l] + if ll.special ~= true then -- catch double root + collected[#collected+1] = ll end - return result - else - return pattern + collect(ll,collected) end + return collected end -function xml.cached_patterns() - return cache +apply_axis['ancestor'] = function(list) + local collected = { } + for l=1,#list do + local ll = list[l] + while ll do + ll = ll.__p__ + if ll then + collected[#collected+1] = ll + end + end + end + return collected end --- we run out of locals (limited to 200) --- --- local fallbackreport = (texio and texio.write) or io.write - -function xml.lshow(pattern,report) --- report = report or fallbackreport - report = report or (texio and texio.write) or io.write - local lp = xml.lpath(pattern) - if lp == false then - report(" -: root\n") - elseif lp == true then - report(" -: wildcard\n") - else - if type(pattern) == "string" then - report(format("pattern: %s\n",pattern)) - end - for k=1,#lp do - local v = lp[k] - if #v > 1 then - local t = { } - for i=2,#v do - local vv = v[i] - if type(vv) == "string" then - t[#t+1] = (vv ~= "" and vv) or "#" - elseif type(vv) == "boolean" then - t[#t+1] = (vv and "==") or "<>" - end - end - report(format("%2i: %s %s -> %s\n", k,v[1],actions[v[1]],concat(t," "))) - else - report(format("%2i: %s %s\n", k,v[1],actions[v[1]])) +apply_axis['ancestor-or-self'] = function(list) + local collected = { } + for l=1,#list do + local ll = list[l] + collected[#collected+1] = ll + while ll do + ll = ll.__p__ + if ll then + collected[#collected+1] = ll end end end + return collected end -function xml.xshow(e,...) -- also handy when report is given, use () to isolate first e - local t = { ... } --- local report = (type(t[#t]) == "function" and t[#t]) or fallbackreport - local report = (type(t[#t]) == "function" and t[#t]) or (texio and texio.write) or io.write - if e == nil then - report("<!-- no element -->\n") - elseif type(e) ~= "table" then - report(tostring(e)) - elseif e.tg then - report(tostring(e) .. "\n") - else - for i=1,#e do - report(tostring(e[i]) .. "\n") +apply_axis['parent'] = function(list) + local collected = { } + for l=1,#list do + local pl = list[l].__p__ + if pl then + collected[#collected+1] = pl end end + return collected end ---[[ldx-- -<p>An <l n='lpath'/> is converted to a table with instructions for traversing the -tree. Hoever, simple cases are signaled by booleans. Because we don't know in -advance what we want to do with the found element the handle gets three arguments:</p> - -<lines> -<t>r</t> : the root element of the data table -<t>d</t> : the data table of the result -<t>t</t> : the index in the data table of the result -</lines> - -<p> Access to the root and data table makes it possible to construct insert and delete -functions.</p> ---ldx]]-- - -local functions = xml.functions -local expressions = xml.expressions - -expressions.contains = string.find -expressions.find = string.find -expressions.upper = string.upper -expressions.lower = string.lower -expressions.number = tonumber -expressions.boolean = toboolean - -expressions.oneof = function(s,...) -- slow - local t = {...} for i=1,#t do if s == t[i] then return true end end return false +apply_axis['attribute'] = function(list) + return { } end -expressions.error = function(str) - xml.error_handler("unknown function in lpath expression",str or "?") - return false +apply_axis['namespace'] = function(list) + return { } end -functions.text = function(root,k,n) -- unchecked, maybe one deeper - local t = type(t) - if t == "string" then - return t - else -- todo n - local rdt = root.dt - return (rdt and rdt[k]) or root[k] or "" - end +apply_axis['following'] = function(list) -- incomplete +--~ local collected = { } +--~ for l=1,#list do +--~ local ll = list[l] +--~ local p = ll.__p__ +--~ local d = p.dt +--~ for i=ll.ni+1,#d do +--~ local di = d[i] +--~ if type(di) == "table" then +--~ collected[#collected+1] = di +--~ break +--~ end +--~ end +--~ end +--~ return collected + return { } +end + +apply_axis['preceding'] = function(list) -- incomplete +--~ local collected = { } +--~ for l=1,#list do +--~ local ll = list[l] +--~ local p = ll.__p__ +--~ local d = p.dt +--~ for i=ll.ni-1,1,-1 do +--~ local di = d[i] +--~ if type(di) == "table" then +--~ collected[#collected+1] = di +--~ break +--~ end +--~ end +--~ end +--~ return collected + return { } end -functions.name = function(d,k,n) -- ns + tg - local found = false - n = n or 0 - if not k then - -- not found - elseif n == 0 then - local dk = d[k] - found = dk and (type(dk) == "table") and dk - elseif n < 0 then - for i=k-1,1,-1 do - local di = d[i] - if type(di) == "table" then - if n == -1 then - found = di - break - else - n = n + 1 - end - end - end - else - for i=k+1,#d,1 do +apply_axis['following-sibling'] = function(list) + local collected = { } + for l=1,#list do + local ll = list[l] + local p = ll.__p__ + local d = p.dt + for i=ll.ni+1,#d do local di = d[i] if type(di) == "table" then - if n == 1 then - found = di - break - else - n = n - 1 - end + collected[#collected+1] = di end end end - if found then - local ns, tg = found.rn or found.ns or "", found.tg - if ns ~= "" then - return ns .. ":" .. tg - else - return tg - end - else - return "" - end + return collected end -functions.tag = function(d,k,n) -- only tg - local found = false - n = n or 0 - if not k then - -- not found - elseif n == 0 then - local dk = d[k] - found = dk and (type(dk) == "table") and dk - elseif n < 0 then - for i=k-1,1,-1 do +apply_axis['preceding-sibling'] = function(list) + local collected = { } + for l=1,#list do + local ll = list[l] + local p = ll.__p__ + local d = p.dt + for i=1,ll.ni-1 do local di = d[i] if type(di) == "table" then - if n == -1 then - found = di - break - else - n = n + 1 - end + collected[#collected+1] = di end end - else - for i=k+1,#d,1 do + end + return collected +end + +apply_axis['reverse-sibling'] = function(list) -- reverse preceding + local collected = { } + for l=1,#list do + local ll = list[l] + local p = ll.__p__ + local d = p.dt + for i=ll.ni-1,1,-1 do local di = d[i] if type(di) == "table" then - if n == 1 then - found = di - break - else - n = n - 1 - end + collected[#collected+1] = di end end end - return (found and found.tg) or "" + return collected end -expressions.text = functions.text -expressions.name = functions.name -expressions.tag = functions.tag +apply_axis['auto-descendant-or-self'] = apply_axis['descendant-or-self'] +apply_axis['auto-descendant'] = apply_axis['descendant'] +apply_axis['auto-child'] = apply_axis['child'] +apply_axis['auto-self'] = apply_axis['self'] +apply_axis['initial-child'] = apply_axis['child'] -local function traverse(root,pattern,handle,reverse,index,parent,wildcard) -- multiple only for tags, not for namespaces - if not root then -- error - return false - elseif pattern == false then -- root - handle(root,root.dt,root.ri) - return false - elseif pattern == true then -- wildcard - local rootdt = root.dt - if rootdt then - local start, stop, step = 1, #rootdt, 1 - if reverse then - start, stop, step = stop, start, -1 - end - for k=start,stop,step do - if handle(root,rootdt,root.ri or k) then return false end - if not traverse(rootdt[k],true,handle,reverse) then return false end - end - end - return false - elseif root.dt then - index = index or 1 - local action = pattern[index] - local command = action[1] - if command == 29 then -- fast case /oeps - local rootdt = root.dt - for k=1,#rootdt do - local e = rootdt[k] - local tg = e.tg - if e.tg then - local ns = e.rn or e.ns - local ns_a, tg_a = action[3], action[4] - local matched = (ns_a == "*" or ns == ns_a) and (tg_a == "*" or tg == tg_a) - if not action[2] then matched = not matched end - if matched then - if handle(root,rootdt,k) then return false end - end - end - end - elseif command == 11 then -- parent - local ep = root.__p__ or parent - if index < #pattern then - if not traverse(ep,pattern,handle,reverse,index+1,root) then return false end - elseif handle(root,rootdt,k) then - return false +local function apply_nodes(list,directive,nodes) + -- todo: nodes[1] etc ... negated node name in set ... when needed + -- ... currently ignored + local maxn = #nodes + if maxn == 3 then --optimized loop + local nns, ntg = nodes[2], nodes[3] + if not nns and not ntg then -- wildcard + if directive then + return list + else + return { } end else - if (command == 16 or command == 12) and index == 1 then -- initial - -- wildcard = true - wildcard = command == 16 -- ok? - index = index + 1 - action = pattern[index] - command = action and action[1] or 0 -- something is wrong - end - if command == 11 then -- parent - local ep = root.__p__ or parent - if index < #pattern then - if not traverse(ep,pattern,handle,reverse,index+1,root) then return false end - elseif handle(root,rootdt,k) then - return false - end - else - local rootdt = root.dt - local start, stop, step, n, dn = 1, #rootdt, 1, 0, 1 - if command == 30 then - if action[5] < 0 then - start, stop, step = stop, start, -1 - dn = -1 - end - elseif reverse and index == #pattern then - start, stop, step = stop, start, -1 - end - local idx = 0 - local hsh = { } -- this will slooow down the lot - for k=start,stop,step do -- we used to have functions for all but a case is faster - local e = rootdt[k] - local ns, tg = e.rn or e.ns, e.tg - if tg then - -- we can optimize this for simple searches, but it probably does not pay off - hsh[tg] = (hsh[tg] or 0) + 1 - idx = idx + 1 - if command == 30 then - local ns_a, tg_a = action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - if not action[2] then matched = not matched end - if matched then - n = n + dn - if n == action[5] then - if index == #pattern then - if handle(root,rootdt,root.ri or k) then return false end - else - if not traverse(e,pattern,handle,reverse,index+1,root) then return false end - end - break - end - elseif wildcard then - if not traverse(e,pattern,handle,reverse,index,root,true) then return false end + local collected, m, p = { }, 0, nil + if not nns then -- only check tag + for l=1,#list do + local ll = list[l] + local ltg = ll.tg + if ltg then + if directive then + if ntg == ltg then + local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end + collected[#collected+1], ll.mi = ll, m end - else - local matched, multiple = false, false - if command == 20 then -- match - local ns_a, tg_a = action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - if not action[2] then matched = not matched end - elseif command == 21 then -- match one of - multiple = true - for i=3,#action,2 do - local ns_a, tg_a = action[i], action[i+1] - if (ns_a == "*" or ns == ns_a) and (tg == "*" or tg == tg_a) then - matched = true - break - end - end - if not action[2] then matched = not matched end - elseif command == 22 then -- eq - local ns_a, tg_a = action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - matched = matched and e.at[action[6]] == action[7] - elseif command == 23 then -- ne - local ns_a, tg_a = action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - if not action[2] then matched = not matched end - matched = mached and e.at[action[6]] ~= action[7] - elseif command == 24 then -- one of eq - multiple = true - for i=3,#action-2,2 do - local ns_a, tg_a = action[i], action[i+1] - if (ns_a == "*" or ns == ns_a) and (tg == "*" or tg == tg_a) then - matched = true - break - end - end - if not action[2] then matched = not matched end - matched = matched and e.at[action[#action-1]] == action[#action] - elseif command == 25 then -- one of ne - multiple = true - for i=3,#action-2,2 do - local ns_a, tg_a = action[i], action[i+1] - if (ns_a == "*" or ns == ns_a) and (tg == "*" or tg == tg_a) then - matched = true - break - end - end - if not action[2] then matched = not matched end - matched = matched and e.at[action[#action-1]] ~= action[#action] - elseif command == 27 then -- has attribute - local ns_a, tg_a = action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - if not action[2] then matched = not matched end - matched = matched and e.at[action[5]] - elseif command == 28 then -- has value - local edt, ns_a, tg_a = e.dt, action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - if not action[2] then matched = not matched end - matched = matched and edt and edt[1] == action[5] - elseif command == 31 then - local edt, ns_a, tg_a = e.dt, action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - if not action[2] then matched = not matched end - if matched then - matched = action[6](expressions,root,rootdt,k,e,edt,ns,tg,idx,hsh[tg] or 1) - end - end - if matched then -- combine tg test and at test - if index == #pattern then - if handle(root,rootdt,root.ri or k) then return false end - if wildcard then - if multiple then - if not traverse(e,pattern,handle,reverse,index,root,true) then return false end - else - -- maybe or multiple; anyhow, check on (section|title) vs just section and title in example in lxml - if not traverse(e,pattern,handle,reverse,index,root) then return false end - end - end - else - if not traverse(e,pattern,handle,reverse,index+1,root) then return false end - end - elseif command == 14 then -- any - if index == #pattern then - if handle(root,rootdt,root.ri or k) then return false end - else - if not traverse(e,pattern,handle,reverse,index+1,root) then return false end - end - elseif command == 15 then -- many - if index == #pattern then - if handle(root,rootdt,root.ri or k) then return false end - else - if not traverse(e,pattern,handle,reverse,index+1,root,true) then return false end - end - -- not here : 11 - elseif command == 11 then -- parent - local ep = e.__p__ or parent - if index < #pattern then - if not traverse(ep,pattern,handle,reverse,root,index+1) then return false end - elseif handle(root,rootdt,k) then - return false - end - elseif command == 40 and e.special and tg == "@pi@" then -- pi - local pi = action[2] - if pi ~= "" then - local pt = e.dt[1] - if pt and pt:find(pi) then - if handle(root,rootdt,k) then - return false - end - end - elseif handle(root,rootdt,k) then - return false - end - elseif wildcard then - if not traverse(e,pattern,handle,reverse,index,root,true) then return false end + elseif ntg ~= ltg then + local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end + collected[#collected+1], ll.mi = ll, m + end + end + end + elseif not ntg then -- only check namespace + for l=1,#list do + local ll = list[l] + local lns = ll.rn or ll.ns + if lns then + if directive then + if lns == nns then + local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end + collected[#collected+1], ll.mi = ll, m end + elseif lns ~= nns then + local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end + collected[#collected+1], ll.mi = ll, m end - else - -- not here : 11 - if command == 11 then -- parent - local ep = e.__p__ or parent - if index < #pattern then - if not traverse(ep,pattern,handle,reverse,index+1,root) then return false end - elseif handle(root,rootdt,k) then - return false + end + end + else -- check both + for l=1,#list do + local ll = list[l] + local ltg = ll.tg + if ltg then + local lns = ll.rn or ll.ns + local ok = ltg == ntg and lns == nns + if directive then + if ok then + local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end + collected[#collected+1], ll.mi = ll, m end - break -- else loop + elseif not ok then + local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end + collected[#collected+1], ll.mi = ll, m end end end end + return collected end + else + local collected, m, p = { }, 0, nil + for l=1,#list do + local ll = list[l] + local ltg = ll.tg + if ltg then + local lns = ll.rn or ll.ns + local ok = false + for n=1,maxn,3 do + local nns, ntg = nodes[n+1], nodes[n+2] + ok = (not ntg or ltg == ntg) and (not nns or lns == nns) + if ok then + break + end + end + if directive then + if ok then + local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end + collected[#collected+1], ll.mi = ll, m + end + elseif not ok then + local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end + collected[#collected+1], ll.mi = ll, m + end + end + end + return collected end - return true end -xml.traverse = traverse +local quit_expression = false ---[[ldx-- -<p>Next come all kind of locators and manipulators. The most generic function here -is <t>xml.filter(root,pattern)</t>. All registers functions in the filters namespace -can be path of a search path, as in:</p> +local function apply_expression(list,expression,order) + local collected = { } + quit_expression = false + for l=1,#list do + local ll = list[l] + if expression(list,ll,l,order) then -- nasty, order alleen valid als n=1 + collected[#collected+1] = ll + end + if quit_expression then + break + end + end + return collected +end + +local P, V, C, Cs, Cc, Ct, R, S, Cg, Cb = lpeg.P, lpeg.V, lpeg.C, lpeg.Cs, lpeg.Cc, lpeg.Ct, lpeg.R, lpeg.S, lpeg.Cg, lpeg.Cb + +local spaces = S(" \n\r\t\f")^0 +local lp_space = S(" \n\r\t\f") +local lp_any = P(1) +local lp_noequal = P("!=") / "~=" + P("<=") + P(">=") + P("==") +local lp_doequal = P("=") / "==" +local lp_or = P("|") / " or " +local lp_and = P("&") / " and " + +local lp_builtin = P ( + P("firstindex") / "1" + + P("lastindex") / "(#ll.__p__.dt or 1)" + + P("firstelement") / "1" + + P("lastelement") / "(ll.__p__.en or 1)" + + P("first") / "1" + + P("last") / "#list" + + P("rootposition") / "order" + + P("position") / "l" + -- is element in finalizer + P("order") / "order" + + P("element") / "(ll.ei or 1)" + + P("index") / "(ll.ni or 1)" + + P("match") / "(ll.mi or 1)" + + P("text") / "(ll.dt[1] or '')" + + -- P("name") / "(ll.ns~='' and ll.ns..':'..ll.tg)" + + P("name") / "((ll.ns~='' and ll.ns..':'..ll.tg) or ll.tg)" + + P("tag") / "ll.tg" + + P("ns") / "ll.ns" + ) * ((spaces * P("(") * spaces * P(")"))/"") + +local lp_attribute = (P("@") + P("attribute::")) / "" * Cc("(ll.at and ll.at['") * R("az","AZ","--","__")^1 * Cc("'])") +local lp_fastpos_p = ((P("+")^0 * R("09")^1 * P(-1)) / function(s) return "l==" .. s end) +local lp_fastpos_n = ((P("-") * R("09")^1 * P(-1)) / function(s) return "(" .. s .. "<0 and (#list+".. s .. "==l))" end) +local lp_fastpos = lp_fastpos_n + lp_fastpos_p +local lp_reserved = C("and") + C("or") + C("not") + C("div") + C("mod") + C("true") + C("false") + +local lp_lua_function = C(R("az","AZ","__")^1 * (P(".") * R("az","AZ","__")^1)^1) * ("(") / function(t) -- todo: better . handling + return t .. "(" +end -<typing> -local r, d, k = xml.filter(root,"/a/b/c/position(4)" -</typing> ---ldx]]-- +local lp_function = C(R("az","AZ","__")^1) * P("(") / function(t) -- todo: better . handling + if expressions[t] then + return "expr." .. t .. "(" + else + return "expr.error(" + end +end -local traverse, lpath, convert = xml.traverse, xml.lpath, xml.convert +local lparent = lpeg.P("(") +local rparent = lpeg.P(")") +local noparent = 1 - (lparent+rparent) +local nested = lpeg.P{lparent * (noparent + lpeg.V(1))^0 * rparent} +local value = lpeg.P(lparent * lpeg.C((noparent + nested)^0) * rparent) -- lpeg.P{"("*C(((1-S("()"))+V(1))^0)*")"} -xml.filters = { } +local lp_child = Cc("expr.child(ll,'") * R("az","AZ","--","__")^1 * Cc("')") +local lp_number = S("+-") * R("09")^1 +local lp_string = Cc("'") * R("az","AZ","--","__")^1 * Cc("'") +local lp_content = (P("'") * (1-P("'"))^0 * P("'") + P('"') * (1-P('"'))^0 * P('"')) -function xml.filters.default(root,pattern) - local rt, dt, dk - traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end) - return dt and dt[dk], rt, dt, dk -end +local cleaner -function xml.filters.attributes(root,pattern,arguments) - local rt, dt, dk - traverse(root, lpath(pattern), function(r,d,k) rt, dt, dk = r, d, k return true end) - local ekat = (dt and dt[dk] and dt[dk].at) or (rt and rt.at) - if ekat then - if arguments then - return ekat[arguments] or "", rt, dt, dk +local lp_special = (C(P("name")+P("text")+P("tag")+P("count")+P("child"))) * value / function(t,s) + if expressions[t] then + s = s and s ~= "" and lpegmatch(cleaner,s) + if s and s ~= "" then + return "expr." .. t .. "(ll," .. s ..")" else - return ekat, rt, dt, dk + return "expr." .. t .. "(ll)" end else - return { }, rt, dt, dk + return "expr.error(" .. t .. ")" end end -function xml.filters.reverse(root,pattern) - local rt, dt, dk - traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end, 'reverse') - return dt and dt[dk], rt, dt, dk -end +local content = + lp_builtin + + lp_attribute + + lp_special + + lp_noequal + lp_doequal + + lp_or + lp_and + + lp_reserved + + lp_lua_function + lp_function + + lp_content + -- too fragile + lp_child + + lp_any + +local converter = Cs ( + lp_fastpos + (P { lparent * (V(1))^0 * rparent + content } )^0 +) -function xml.filters.count(root,pattern,everything) - local n = 0 - traverse(root, lpath(pattern), function(r,d,t) - if everything or type(d[t]) == "table" then - n = n + 1 - end - end) - return n -end +cleaner = Cs ( ( +--~ lp_fastpos + + lp_reserved + + lp_number + + lp_string + +1 )^1 ) -function xml.filters.elements(root, pattern) -- == all - local t = { } - traverse(root, lpath(pattern), function(r,d,k) - local e = d[k] - if e then - t[#t+1] = e - end - end) - return t -end -function xml.filters.texts(root, pattern) - local t = { } - traverse(root, lpath(pattern), function(r,d,k) - local e = d[k] - if e and e.dt then - t[#t+1] = e.dt - end - end) - return t -end +--~ expr -function xml.filters.first(root,pattern) - local rt, dt, dk - traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end) - return dt and dt[dk], rt, dt, dk -end +local template_e = [[ + local expr = xml.expressions + return function(list,ll,l,order) + return %s + end +]] -function xml.filters.last(root,pattern) - local rt, dt, dk - traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end, 'reverse') - return dt and dt[dk], rt, dt, dk -end +local template_f_y = [[ + local finalizer = xml.finalizers['%s']['%s'] + return function(collection) + return finalizer(collection,%s) + end +]] -function xml.filters.index(root,pattern,arguments) - local rt, dt, dk, reverse, i = nil, nil, nil, false, tonumber(arguments or '1') or 1 - if i and i ~= 0 then - if i < 0 then - reverse, i = true, -i - end - traverse(root, lpath(pattern), function(r,d,k) rt, dt, dk, i = r, d, k, i-1 return i == 0 end, reverse) - if i == 0 then - return dt and dt[dk], rt, dt, dk - end +local template_f_n = [[ + return xml.finalizers['%s']['%s'] +]] + +-- + +local register_self = { kind = "axis", axis = "self" } -- , apply = apply_axis["self"] } +local register_parent = { kind = "axis", axis = "parent" } -- , apply = apply_axis["parent"] } +local register_descendant = { kind = "axis", axis = "descendant" } -- , apply = apply_axis["descendant"] } +local register_child = { kind = "axis", axis = "child" } -- , apply = apply_axis["child"] } +local register_descendant_or_self = { kind = "axis", axis = "descendant-or-self" } -- , apply = apply_axis["descendant-or-self"] } +local register_root = { kind = "axis", axis = "root" } -- , apply = apply_axis["root"] } +local register_ancestor = { kind = "axis", axis = "ancestor" } -- , apply = apply_axis["ancestor"] } +local register_ancestor_or_self = { kind = "axis", axis = "ancestor-or-self" } -- , apply = apply_axis["ancestor-or-self"] } +local register_attribute = { kind = "axis", axis = "attribute" } -- , apply = apply_axis["attribute"] } +local register_namespace = { kind = "axis", axis = "namespace" } -- , apply = apply_axis["namespace"] } +local register_following = { kind = "axis", axis = "following" } -- , apply = apply_axis["following"] } +local register_following_sibling = { kind = "axis", axis = "following-sibling" } -- , apply = apply_axis["following-sibling"] } +local register_preceding = { kind = "axis", axis = "preceding" } -- , apply = apply_axis["preceding"] } +local register_preceding_sibling = { kind = "axis", axis = "preceding-sibling" } -- , apply = apply_axis["preceding-sibling"] } +local register_reverse_sibling = { kind = "axis", axis = "reverse-sibling" } -- , apply = apply_axis["reverse-sibling"] } + +local register_auto_descendant_or_self = { kind = "axis", axis = "auto-descendant-or-self" } -- , apply = apply_axis["auto-descendant-or-self"] } +local register_auto_descendant = { kind = "axis", axis = "auto-descendant" } -- , apply = apply_axis["auto-descendant"] } +local register_auto_self = { kind = "axis", axis = "auto-self" } -- , apply = apply_axis["auto-self"] } +local register_auto_child = { kind = "axis", axis = "auto-child" } -- , apply = apply_axis["auto-child"] } + +local register_initial_child = { kind = "axis", axis = "initial-child" } -- , apply = apply_axis["initial-child"] } + +local register_all_nodes = { kind = "nodes", nodetest = true, nodes = { true, false, false } } + +local skip = { } + +local function errorrunner_e(str,cnv) + if not skip[str] then + logs.report("lpath","error in expression: %s => %s",str,cnv) + skip[str] = cnv or str end - return nil, nil, nil, nil + return false +end +local function errorrunner_f(str,arg) + logs.report("lpath","error in finalizer: %s(%s)",str,arg or "") + return false end -function xml.filters.attribute(root,pattern,arguments) - local rt, dt, dk - traverse(root, lpath(pattern), function(r,d,k) rt, dt, dk = r, d, k return true end) - local ekat = (dt and dt[dk] and dt[dk].at) or (rt and rt.at) - return (ekat and (ekat[arguments] or ekat[gsub(arguments,"^([\"\'])(.*)%1$","%2")])) or "" +local function register_nodes(nodetest,nodes) + return { kind = "nodes", nodetest = nodetest, nodes = nodes } end -function xml.filters.text(root,pattern,arguments) -- ?? why index, tostring slow - local dtk, rt, dt, dk = xml.filters.index(root,pattern,arguments) - if dtk then -- n - local dtkdt = dtk.dt - if not dtkdt then - return "", rt, dt, dk - elseif #dtkdt == 1 and type(dtkdt[1]) == "string" then - return dtkdt[1], rt, dt, dk - else - return xml.tostring(dtkdt), rt, dt, dk - end +local function register_expression(expression) + local converted = lpegmatch(converter,expression) + local runner = loadstring(format(template_e,converted)) + runner = (runner and runner()) or function() errorrunner_e(expression,converted) end + return { kind = "expression", expression = expression, converted = converted, evaluator = runner } +end + +local function register_finalizer(protocol,name,arguments) + local runner + if arguments and arguments ~= "" then + runner = loadstring(format(template_f_y,protocol or xml.defaultprotocol,name,arguments)) else - return "", rt, dt, dk + runner = loadstring(format(template_f_n,protocol or xml.defaultprotocol,name)) end + runner = (runner and runner()) or function() errorrunner_f(name,arguments) end + return { kind = "finalizer", name = name, arguments = arguments, finalizer = runner } end -function xml.filters.tag(root,pattern,n) - local tag = "" - traverse(root, lpath(pattern), function(r,d,k) - tag = xml.functions.tag(d,k,n and tonumber(n)) - return true - end) - return tag -end +local expression = P { "ex", + ex = "[" * C((V("sq") + V("dq") + (1 - S("[]")) + V("ex"))^0) * "]", + sq = "'" * (1 - S("'"))^0 * "'", + dq = '"' * (1 - S('"'))^0 * '"', +} -function xml.filters.name(root,pattern,n) - local tag = "" - traverse(root, lpath(pattern), function(r,d,k) - tag = xml.functions.name(d,k,n and tonumber(n)) - return true - end) - return tag +local arguments = P { "ar", + ar = "(" * Cs((V("sq") + V("dq") + V("nq") + P(1-P(")")))^0) * ")", + nq = ((1 - S("),'\""))^1) / function(s) return format("%q",s) end, + sq = P("'") * (1 - P("'"))^0 * P("'"), + dq = P('"') * (1 - P('"'))^0 * P('"'), +} + +-- todo: better arg parser + +local function register_error(str) + return { kind = "error", error = format("unparsed: %s",str) } end ---[[ldx-- -<p>For splitting the filter function from the path specification, we can -use string matching or lpeg matching. Here the difference in speed is -neglectable but the lpeg variant is more robust.</p> ---ldx]]-- +-- there is a difference in * and /*/ and so we need to catch a few special cases + +local special_1 = P("*") * Cc(register_auto_descendant) * Cc(register_all_nodes) -- last one not needed +local special_2 = P("/") * Cc(register_auto_self) +local special_3 = P("") * Cc(register_auto_self) + +local parser = Ct { "patterns", -- can be made a bit faster by moving pattern outside + + patterns = spaces * V("protocol") * spaces * ( + ( V("special") * spaces * P(-1) ) + + ( V("initial") * spaces * V("step") * spaces * (P("/") * spaces * V("step") * spaces)^0 ) + ), + + protocol = Cg(V("letters"),"protocol") * P("://") + Cg(Cc(nil),"protocol"), + + -- the / is needed for // as descendant or self is somewhat special + -- step = (V("shortcuts") + V("axis") * spaces * V("nodes")^0 + V("error")) * spaces * V("expressions")^0 * spaces * V("finalizer")^0, + step = ((V("shortcuts") + P("/") + V("axis")) * spaces * V("nodes")^0 + V("error")) * spaces * V("expressions")^0 * spaces * V("finalizer")^0, + + axis = V("descendant") + V("child") + V("parent") + V("self") + V("root") + V("ancestor") + + V("descendant_or_self") + V("following_sibling") + V("following") + + V("reverse_sibling") + V("preceding_sibling") + V("preceding") + V("ancestor_or_self") + + #(1-P(-1)) * Cc(register_auto_child), + + special = special_1 + special_2 + special_3, --- not faster but hipper ... although ... i can't get rid of the trailing / in the path + initial = (P("/") * spaces * Cc(register_initial_child))^-1, -local P, S, R, C, V, Cc = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc + error = (P(1)^1) / register_error, -local slash = P('/') -local name = (R("az","AZ","--","__"))^1 -local path = C(((1-slash)^0 * slash)^1) -local argument = P { "(" * C(((1 - S("()")) + V(1))^0) * ")" } -local action = Cc(1) * path * C(name) * argument -local attribute = Cc(2) * path * P('@') * C(name) -local direct = Cc(3) * Cc("../*") * slash^0 * C(name) * argument + shortcuts_a = V("s_descendant_or_self") + V("s_descendant") + V("s_child") + V("s_parent") + V("s_self") + V("s_root") + V("s_ancestor"), -local parser = direct + action + attribute + shortcuts = V("shortcuts_a") * (spaces * "/" * spaces * V("shortcuts_a"))^0, -local filters = xml.filters -local attribute_filter = xml.filters.attributes -local default_filter = xml.filters.default + s_descendant_or_self = (P("***/") + P("/")) * Cc(register_descendant_or_self), --- *** is a bonus + -- s_descendant_or_self = P("/") * Cc(register_descendant_or_self), + s_descendant = P("**") * Cc(register_descendant), + s_child = P("*") * #(1-P(":")) * Cc(register_child ), +-- s_child = P("*") * #(P("/")+P(-1)) * Cc(register_child ), + s_parent = P("..") * Cc(register_parent ), + s_self = P("." ) * Cc(register_self ), + s_root = P("^^") * Cc(register_root ), + s_ancestor = P("^") * Cc(register_ancestor ), --- todo: also hash, could be gc'd + descendant = P("descendant::") * Cc(register_descendant ), + child = P("child::") * Cc(register_child ), + parent = P("parent::") * Cc(register_parent ), + self = P("self::") * Cc(register_self ), + root = P('root::') * Cc(register_root ), + ancestor = P('ancestor::') * Cc(register_ancestor ), + descendant_or_self = P('descendant-or-self::') * Cc(register_descendant_or_self ), + ancestor_or_self = P('ancestor-or-self::') * Cc(register_ancestor_or_self ), + -- attribute = P('attribute::') * Cc(register_attribute ), + -- namespace = P('namespace::') * Cc(register_namespace ), + following = P('following::') * Cc(register_following ), + following_sibling = P('following-sibling::') * Cc(register_following_sibling ), + preceding = P('preceding::') * Cc(register_preceding ), + preceding_sibling = P('preceding-sibling::') * Cc(register_preceding_sibling ), + reverse_sibling = P('reverse-sibling::') * Cc(register_reverse_sibling ), + + nodes = (V("nodefunction") * spaces * P("(") * V("nodeset") * P(")") + V("nodetest") * V("nodeset")) / register_nodes, + + expressions = expression / register_expression, + + letters = R("az")^1, + name = (1-lpeg.S("/[]()|:*!"))^1, + negate = P("!") * Cc(false), + + nodefunction = V("negate") + P("not") * Cc(false) + Cc(true), + nodetest = V("negate") + Cc(true), + nodename = (V("negate") + Cc(true)) * spaces * ((V("wildnodename") * P(":") * V("wildnodename")) + (Cc(false) * V("wildnodename"))), + wildnodename = (C(V("name")) + P("*") * Cc(false)) * #(1-P("(")), + nodeset = spaces * Ct(V("nodename") * (spaces * P("|") * spaces * V("nodename"))^0) * spaces, + + finalizer = (Cb("protocol") * P("/")^-1 * C(V("name")) * arguments * P(-1)) / register_finalizer, + +} + +local cache = { } -function xml.filter(root,pattern) - local kind, a, b, c = parser:match(pattern) - if kind == 1 or kind == 3 then - return (filters[b] or default_filter)(root,a,c) - elseif kind == 2 then - return attribute_filter(root,a,b) +local function nodesettostring(set,nodetest) + local t = { } + for i=1,#set,3 do + local directive, ns, tg = set[i], set[i+1], set[i+2] + if not ns or ns == "" then ns = "*" end + if not tg or tg == "" then tg = "*" end + tg = (tg == "@rt@" and "[root]") or format("%s:%s",ns,tg) + t[#t+1] = (directive and tg) or format("not(%s)",tg) + end + if nodetest == false then + return format("not(%s)",concat(t,"|")) else - return default_filter(root,pattern) + return concat(t,"|") end end ---~ slightly faster, but first we need a proper test file ---~ ---~ local hash = { } ---~ ---~ function xml.filter(root,pattern) ---~ local h = hash[pattern] ---~ if not h then ---~ local kind, a, b, c = parser:match(pattern) ---~ if kind == 1 then ---~ h = { kind, filters[b] or default_filter, a, b, c } ---~ elseif kind == 2 then ---~ h = { kind, attribute_filter, a, b, c } ---~ else ---~ h = { kind, default_filter, a, b, c } ---~ end ---~ hash[pattern] = h ---~ end ---~ local kind = h[1] ---~ if kind == 1 then ---~ return h[2](root,h[2],h[4]) ---~ elseif kind == 2 then ---~ return h[2](root,h[2],h[3]) ---~ else ---~ return h[2](root,pattern) ---~ end ---~ end +local function tagstostring(list) + if #list == 0 then + return "no elements" + else + local t = { } + for i=1, #list do + local li = list[i] + local ns, tg = li.ns, li.tg + if not ns or ns == "" then ns = "*" end + if not tg or tg == "" then tg = "*" end + t[#t+1] = (tg == "@rt@" and "[root]") or format("%s:%s",ns,tg) + end + return concat(t," ") + end +end ---[[ldx-- -<p>The following functions collect elements and texts.</p> ---ldx]]-- +xml.nodesettostring = nodesettostring --- still somewhat bugged +local parse_pattern -- we have a harmless kind of circular reference -function xml.collect_elements(root, pattern, ignorespaces) - local rr, dd = { }, { } - traverse(root, lpath(pattern), function(r,d,k) - local dk = d and d[k] - if dk then - if ignorespaces and type(dk) == "string" and dk:find("[^%S]") then - -- ignore - else - local n = #rr+1 - rr[n], dd[n] = r, dk - end - end - end) - return dd, rr +local function lshow(parsed) + if type(parsed) == "string" then + parsed = parse_pattern(parsed) + end + local s = table.serialize_functions -- ugly + table.serialize_functions = false -- ugly + logs.report("lpath","%s://%s => %s",parsed.protocol or xml.defaultprotocol,parsed.pattern,table.serialize(parsed,false)) + table.serialize_functions = s -- ugly end -function xml.collect_texts(root, pattern, flatten) - local t = { } -- no r collector - traverse(root, lpath(pattern), function(r,d,k) - if d then - local ek = d[k] - local tx = ek and ek.dt - if flatten then - if tx then - t[#t+1] = xml.tostring(tx) or "" +xml.lshow = lshow + +local function add_comment(p,str) + local pc = p.comment + if not pc then + p.comment = { str } + else + pc[#pc+1] = str + end +end + +parse_pattern = function (pattern) -- the gain of caching is rather minimal + lpathcalls = lpathcalls + 1 + if type(pattern) == "table" then + return pattern + else + local parsed = cache[pattern] + if parsed then + lpathcached = lpathcached + 1 + else + parsed = lpegmatch(parser,pattern) + if parsed then + parsed.pattern = pattern + local np = #parsed + if np == 0 then + parsed = { pattern = pattern, register_self, state = "parsing error" } + logs.report("lpath","parsing error in '%s'",pattern) + lshow(parsed) else - t[#t+1] = "" + -- we could have done this with a more complex parser but this + -- is cleaner + local pi = parsed[1] + if pi.axis == "auto-child" then + if false then + add_comment(parsed, "auto-child replaced by auto-descendant-or-self") + parsed[1] = register_auto_descendant_or_self + else + add_comment(parsed, "auto-child replaced by auto-descendant") + parsed[1] = register_auto_descendant + end + elseif pi.axis == "initial-child" and np > 1 and parsed[2].axis then + add_comment(parsed, "initial-child removed") -- we could also make it a auto-self + remove(parsed,1) + end + local np = #parsed -- can have changed + if np > 1 then + local pnp = parsed[np] + if pnp.kind == "nodes" and pnp.nodetest == true then + local nodes = pnp.nodes + if nodes[1] == true and nodes[2] == false and nodes[3] == false then + add_comment(parsed, "redundant final wildcard filter removed") + remove(parsed,np) + end + end + end end else - t[#t+1] = tx or "" + parsed = { pattern = pattern } + end + cache[pattern] = parsed + if trace_lparse and not trace_lprofile then + lshow(parsed) end - else - t[#t+1] = "" end - end) - return t + return parsed + end end -function xml.collect_tags(root, pattern, nonamespace) - local t = { } - xml.traverse(root, xml.lpath(pattern), function(r,d,k) - local dk = d and d[k] - if dk and type(dk) == "table" then - local ns, tg = e.ns, e.tg - if nonamespace then - t[#t+1] = tg -- if needed we can return an extra table - elseif ns == "" then - t[#t+1] = tg - else - t[#t+1] = ns .. ":" .. tg +-- we can move all calls inline and then merge the trace back +-- technically we can combine axis and the next nodes which is +-- what we did before but this a bit cleaner (but slower too) +-- but interesting is that it's not that much faster when we +-- go inline +-- +-- beware: we need to return a collection even when we filter +-- else the (simple) cache gets messed up + +-- caching found lookups saves not that much (max .1 sec on a 8 sec run) +-- and it also messes up finalizers + +-- watch out: when there is a finalizer, it's always called as there +-- can be cases that a finalizer returns (or does) something in case +-- there is no match; an example of this is count() + +local profiled = { } xml.profiled = profiled + +local function profiled_apply(list,parsed,nofparsed,order) + local p = profiled[parsed.pattern] + if p then + p.tested = p.tested + 1 + else + p = { tested = 1, matched = 0, finalized = 0 } + profiled[parsed.pattern] = p + end + local collected = list + for i=1,nofparsed do + local pi = parsed[i] + local kind = pi.kind + if kind == "axis" then + collected = apply_axis[pi.axis](collected) + elseif kind == "nodes" then + collected = apply_nodes(collected,pi.nodetest,pi.nodes) + elseif kind == "expression" then + collected = apply_expression(collected,pi.evaluator,order) + elseif kind == "finalizer" then + collected = pi.finalizer(collected) + p.matched = p.matched + 1 + p.finalized = p.finalized + 1 + return collected + end + if not collected or #collected == 0 then + local pn = i < nofparsed and parsed[nofparsed] + if pn and pn.kind == "finalizer" then + collected = pn.finalizer(collected) + p.finalized = p.finalized + 1 + return collected end + return nil end - end) - return #t > 0 and {} + end + if collected then + p.matched = p.matched + 1 + end + return collected +end + +local function traced_apply(list,parsed,nofparsed,order) + if trace_lparse then + lshow(parsed) + end + logs.report("lpath", "collecting : %s",parsed.pattern) + logs.report("lpath", " root tags : %s",tagstostring(list)) + logs.report("lpath", " order : %s",order or "unset") + local collected = list + for i=1,nofparsed do + local pi = parsed[i] + local kind = pi.kind + if kind == "axis" then + collected = apply_axis[pi.axis](collected) + logs.report("lpath", "% 10i : ax : %s",(collected and #collected) or 0,pi.axis) + elseif kind == "nodes" then + collected = apply_nodes(collected,pi.nodetest,pi.nodes) + logs.report("lpath", "% 10i : ns : %s",(collected and #collected) or 0,nodesettostring(pi.nodes,pi.nodetest)) + elseif kind == "expression" then + collected = apply_expression(collected,pi.evaluator,order) + logs.report("lpath", "% 10i : ex : %s -> %s",(collected and #collected) or 0,pi.expression,pi.converted) + elseif kind == "finalizer" then + collected = pi.finalizer(collected) + logs.report("lpath", "% 10i : fi : %s : %s(%s)",(type(collected) == "table" and #collected) or 0,parsed.protocol or xml.defaultprotocol,pi.name,pi.arguments or "") + return collected + end + if not collected or #collected == 0 then + local pn = i < nofparsed and parsed[nofparsed] + if pn and pn.kind == "finalizer" then + collected = pn.finalizer(collected) + logs.report("lpath", "% 10i : fi : %s : %s(%s)",(type(collected) == "table" and #collected) or 0,parsed.protocol or xml.defaultprotocol,pn.name,pn.arguments or "") + return collected + end + return nil + end + end + return collected end ---[[ldx-- -<p>Often using an iterators looks nicer in the code than passing handler -functions. The <l n='lua'/> book describes how to use coroutines for that -purpose (<url href='http://www.lua.org/pil/9.3.html'/>). This permits -code like:</p> - -<typing> -for r, d, k in xml.elements(xml.load('text.xml'),"title") do - print(d[k]) +local function normal_apply(list,parsed,nofparsed,order) + local collected = list + for i=1,nofparsed do + local pi = parsed[i] + local kind = pi.kind + if kind == "axis" then + local axis = pi.axis + if axis ~= "self" then + collected = apply_axis[axis](collected) + end + elseif kind == "nodes" then + collected = apply_nodes(collected,pi.nodetest,pi.nodes) + elseif kind == "expression" then + collected = apply_expression(collected,pi.evaluator,order) + elseif kind == "finalizer" then + return pi.finalizer(collected) + end + if not collected or #collected == 0 then + local pf = i < nofparsed and parsed[nofparsed].finalizer + if pf then + return pf(collected) -- can be anything + end + return nil + end + end + return collected end -</typing> - -<p>Which will print all the titles in the document. The iterator variant takes -1.5 times the runtime of the function variant which is due to the overhead in -creating the wrapper. So, instead of:</p> -<typing> -function xml.filters.first(root,pattern) - for rt,dt,dk in xml.elements(root,pattern) - return dt and dt[dk], rt, dt, dk +local function parse_apply(list,pattern) + -- we avoid an extra call + local parsed = cache[pattern] + if parsed then + lpathcalls = lpathcalls + 1 + lpathcached = lpathcached + 1 + elseif type(pattern) == "table" then + lpathcalls = lpathcalls + 1 + parsed = pattern + else + parsed = parse_pattern(pattern) or pattern + end + if not parsed then + return + end + local nofparsed = #parsed + if nofparsed == 0 then + return -- something is wrong + end + local one = list[1] + if not one then + return -- something is wrong + elseif not trace_lpath then + return normal_apply(list,parsed,nofparsed,one.mi) + elseif trace_lprofile then + return profiled_apply(list,parsed,nofparsed,one.mi) + else + return traced_apply(list,parsed,nofparsed,one.mi) end - return nil, nil, nil, nil end -</typing> -<p>We use the function variants in the filters.</p> ---ldx]]-- - -local wrap, yield = coroutine.wrap, coroutine.yield +-- internal (parsed) -function xml.elements(root,pattern,reverse) - return wrap(function() traverse(root, lpath(pattern), yield, reverse) end) +expressions.child = function(e,pattern) + return parse_apply({ e },pattern) -- todo: cache end - -function xml.elements_only(root,pattern,reverse) - return wrap(function() traverse(root, lpath(pattern), function(r,d,k) yield(d[k]) end, reverse) end) +expressions.count = function(e,pattern) + local collected = parse_apply({ e },pattern) -- todo: cache + return (collected and #collected) or 0 end -function xml.each_element(root, pattern, handle, reverse) - local ok - traverse(root, lpath(pattern), function(r,d,k) ok = true handle(r,d,k) end, reverse) - return ok +-- external + +expressions.oneof = function(s,...) -- slow + local t = {...} for i=1,#t do if s == t[i] then return true end end return false +end +expressions.error = function(str) + xml.error_handler("unknown function in lpath expression",tostring(str or "?")) + return false +end +expressions.undefined = function(s) + return s == nil end -function xml.process_elements(root, pattern, handle) - traverse(root, lpath(pattern), function(r,d,k) - local dkdt = d[k].dt - if dkdt then - for i=1,#dkdt do - local v = dkdt[i] - if v.tg then handle(v) end - end - end - end) +expressions.quit = function(s) + if s or s == nil then + quit_expression = true + end + return true end -function xml.process_attributes(root, pattern, handle) - traverse(root, lpath(pattern), function(r,d,k) - local ek = d[k] - local a = ek.at or { } - handle(a) - if next(a) then -- next is faster than type (and >0 test) - ek.at = a - else - ek.at = nil - end - end) +expressions.print = function(...) + print(...) + return true end ---[[ldx-- -<p>We've now arrives at the functions that manipulate the tree.</p> ---ldx]]-- +expressions.contains = find +expressions.find = find +expressions.upper = upper +expressions.lower = lower +expressions.number = tonumber +expressions.boolean = toboolean -function xml.inject_element(root, pattern, element, prepend) - if root and element then - local matches, collect = { }, nil - if type(element) == "string" then - element = convert(element,true) - end - if element then - collect = function(r,d,k) matches[#matches+1] = { r, d, k, element } end - traverse(root, lpath(pattern), collect) - for i=1,#matches do - local m = matches[i] - local r, d, k, element, edt = m[1], m[2], m[3], m[4], nil - if element.ri then - element = element.dt[element.ri].dt - else - element = element.dt - end - if r.ri then - edt = r.dt[r.ri].dt - else - edt = d and d[k] and d[k].dt - end - if edt then - local be, af - if prepend then - be, af = xml.copy(element), edt - else - be, af = edt, xml.copy(element) - end - for i=1,#af do - be[#be+1] = af[i] - end - if r.ri then - r.dt[r.ri].dt = be - else - d[k].dt = be - end - else - -- r.dt = element.dt -- todo - end - end +-- user interface + +local function traverse(root,pattern,handle) + logs.report("xml","use 'xml.selection' instead for '%s'",pattern) + local collected = parse_apply({ root },pattern) + if collected then + for c=1,#collected do + local e = collected[c] + local r = e.__p__ + handle(r,r.dt,e.ni) end end end --- todo: copy ! - -function xml.insert_element(root, pattern, element, before) -- todo: element als functie - if root and element then - if pattern == "/" then - xml.inject_element(root, pattern, element, before) - else - local matches, collect = { }, nil - if type(element) == "string" then - element = convert(element,true) - end - if element and element.ri then - element = element.dt[element.ri] - end - if element then - collect = function(r,d,k) matches[#matches+1] = { r, d, k, element } end - traverse(root, lpath(pattern), collect) - for i=#matches,1,-1 do - local m = matches[i] - local r, d, k, element = m[1], m[2], m[3], m[4] - if not before then k = k + 1 end - if element.tg then - insert(d,k,element) -- untested ---~ elseif element.dt then ---~ for _,v in ipairs(element.dt) do -- i added ---~ insert(d,k,v) ---~ k = k + 1 ---~ end ---~ end - else - local edt = element.dt - if edt then - for i=1,#edt do - insert(d,k,edt[i]) - k = k + 1 - end - end - end - end +local function selection(root,pattern,handle) + local collected = parse_apply({ root },pattern) + if collected then + if handle then + for c=1,#collected do + handle(collected[c]) end + else + return collected end end end -xml.insert_element_after = xml.insert_element -xml.insert_element_before = function(r,p,e) xml.insert_element(r,p,e,true) end -xml.inject_element_after = xml.inject_element -xml.inject_element_before = function(r,p,e) xml.inject_element(r,p,e,true) end +xml.parse_parser = parser +xml.parse_pattern = parse_pattern +xml.parse_apply = parse_apply +xml.traverse = traverse -- old method, r, d, k +xml.selection = selection -- new method, simple handle -function xml.delete_element(root, pattern) - local matches, deleted = { }, { } - local collect = function(r,d,k) matches[#matches+1] = { r, d, k } end - traverse(root, lpath(pattern), collect) - for i=#matches,1,-1 do - local m = matches[i] - deleted[#deleted+1] = remove(m[2],m[3]) - end - return deleted +local lpath = parse_pattern + +xml.lpath = lpath + +function xml.cached_patterns() + return cache end -function xml.replace_element(root, pattern, element) - if type(element) == "string" then - element = convert(element,true) - end - if element and element.ri then - element = element.dt[element.ri] - end - if element then - traverse(root, lpath(pattern), function(rm, d, k) - d[k] = element.dt -- maybe not clever enough - end) +-- generic function finalizer (independant namespace) + +local function dofunction(collected,fnc) + if collected then + local f = functions[fnc] + if f then + for c=1,#collected do + f(collected[c]) + end + else + logs.report("xml","unknown function '%s'",fnc) + end end end -local function load_data(name) -- == io.loaddata - local f, data = io.open(name), "" - if f then - data = f:read("*all",'b') -- 'b' ? - f:close() - end - return data +xml.finalizers.xml["function"] = dofunction +xml.finalizers.tex["function"] = dofunction + +-- functions + +expressions.text = function(e,n) + local rdt = e.__p__.dt + return (rdt and rdt[n]) or "" end -function xml.include(xmldata,pattern,attribute,recursive,loaddata) - -- parse="text" (default: xml), encoding="" (todo) - -- attribute = attribute or 'href' - pattern = pattern or 'include' - loaddata = loaddata or load_data - local function include(r,d,k) - local ek, name = d[k], nil - if not attribute or attribute == "" then - local ekdt = ek.dt - name = (type(ekdt) == "table" and ekdt[1]) or ekdt - end - if not name then - if ek.at then - for a in gmatch(attribute or "href","([^|]+)") do - name = ek.at[a] - if name then break end +expressions.name = function(e,n) -- ns + tg + local found = false + n = tonumber(n) or 0 + if n == 0 then + found = type(e) == "table" and e + elseif n < 0 then + local d, k = e.__p__.dt, e.ni + for i=k-1,1,-1 do + local di = d[i] + if type(di) == "table" then + if n == -1 then + found = di + break + else + n = n + 1 end end end - local data = (name and name ~= "" and loaddata(name)) or "" - if data == "" then - xml.empty(d,k) - elseif ek.at["parse"] == "text" then -- for the moment hard coded - d[k] = xml.escaped(data) - else - local xi = xml.convert(data) - if not xi then - xml.empty(d,k) - else - if recursive then - xml.include(xi,pattern,attribute,recursive,loaddata) + else + local d, k = e.__p__.dt, e.ni + for i=k+1,#d,1 do + local di = d[i] + if type(di) == "table" then + if n == 1 then + found = di + break + else + n = n - 1 end - xml.assign(d,k,xi) end end end - xml.each_element(xmldata, pattern, include) + if found then + local ns, tg = found.rn or found.ns or "", found.tg + if ns ~= "" then + return ns .. ":" .. tg + else + return tg + end + else + return "" + end end -function xml.strip_whitespace(root, pattern, nolines) -- strips all leading and trailing space ! - traverse(root, lpath(pattern), function(r,d,k) - local dkdt = d[k].dt - if dkdt then -- can be optimized - local t = { } - for i=1,#dkdt do - local str = dkdt[i] - if type(str) == "string" then - - if str == "" then - -- stripped +expressions.tag = function(e,n) -- only tg + if not e then + return "" + else + local found = false + n = tonumber(n) or 0 + if n == 0 then + found = (type(e) == "table") and e -- seems to fail + elseif n < 0 then + local d, k = e.__p__.dt, e.ni + for i=k-1,1,-1 do + local di = d[i] + if type(di) == "table" then + if n == -1 then + found = di + break else - if nolines then - str = gsub(str,"[ \n\r\t]+"," ") - end - if str == "" then - -- stripped - else - t[#t+1] = str - end + n = n + 1 end - else - t[#t+1] = str end end - d[k].dt = t - end - end) -end - -local function rename_space(root, oldspace, newspace) -- fast variant - local ndt = #root.dt - for i=1,ndt or 0 do - local e = root[i] - if type(e) == "table" then - if e.ns == oldspace then - e.ns = newspace - if e.rn then - e.rn = newspace + else + local d, k = e.__p__.dt, e.ni + for i=k+1,#d,1 do + local di = d[i] + if type(di) == "table" then + if n == 1 then + found = di + break + else + n = n - 1 + end end end - local edt = e.dt - if edt then - rename_space(edt, oldspace, newspace) - end end + return (found and found.tg) or "" end end -xml.rename_space = rename_space +--[[ldx-- +<p>This is the main filter function. It returns whatever is asked for.</p> +--ldx]]-- -function xml.remap_tag(root, pattern, newtg) - traverse(root, lpath(pattern), function(r,d,k) - d[k].tg = newtg - end) +function xml.filter(root,pattern) -- no longer funny attribute handling here + return parse_apply({ root },pattern) end -function xml.remap_namespace(root, pattern, newns) - traverse(root, lpath(pattern), function(r,d,k) - d[k].ns = newns - end) + +--[[ldx-- +<p>Often using an iterators looks nicer in the code than passing handler +functions. The <l n='lua'/> book describes how to use coroutines for that +purpose (<url href='http://www.lua.org/pil/9.3.html'/>). This permits +code like:</p> + +<typing> +for r, d, k in xml.elements(xml.load('text.xml'),"title") do + print(d[k]) -- old method end -function xml.check_namespace(root, pattern, newns) - traverse(root, lpath(pattern), function(r,d,k) - local dk = d[k] - if (not dk.rn or dk.rn == "") and dk.ns == "" then - dk.rn = newns - end - end) +for e in xml.collected(xml.load('text.xml'),"title") do + print(e) -- new one end -function xml.remap_name(root, pattern, newtg, newns, newrn) - traverse(root, lpath(pattern), function(r,d,k) - local dk = d[k] - dk.tg = newtg - dk.ns = newns - dk.rn = newrn - end) +</typing> +--ldx]]-- + +local wrap, yield = coroutine.wrap, coroutine.yield + +function xml.elements(root,pattern,reverse) -- r, d, k + local collected = parse_apply({ root },pattern) + if collected then + if reverse then + return wrap(function() for c=#collected,1,-1 do + local e = collected[c] local r = e.__p__ yield(r,r.dt,e.ni) + end end) + else + return wrap(function() for c=1,#collected do + local e = collected[c] local r = e.__p__ yield(r,r.dt,e.ni) + end end) + end + end + return wrap(function() end) end -function xml.filters.found(root,pattern,check_content) - local found = false - traverse(root, lpath(pattern), function(r,d,k) - if check_content then - local dk = d and d[k] - found = dk and dk.dt and next(dk.dt) and true +function xml.collected(root,pattern,reverse) -- e + local collected = parse_apply({ root },pattern) + if collected then + if reverse then + return wrap(function() for c=#collected,1,-1 do yield(collected[c]) end end) else - found = true + return wrap(function() for c=1,#collected do yield(collected[c]) end end) end - return true - end) - return found + end + return wrap(function() end) end ---[[ldx-- -<p>Here are a few synonyms.</p> ---ldx]]-- -xml.filters.position = xml.filters.index +end -- of closure -xml.count = xml.filters.count -xml.index = xml.filters.index -xml.position = xml.filters.index -xml.first = xml.filters.first -xml.last = xml.filters.last -xml.found = xml.filters.found +do -- create closure to overcome 200 locals limit -xml.each = xml.each_element -xml.process = xml.process_element -xml.strip = xml.strip_whitespace -xml.collect = xml.collect_elements -xml.all = xml.collect_elements +if not modules then modules = { } end modules ['lxml-mis'] = { + version = 1.001, + comment = "this module is the basis for the lxml-* ones", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} -xml.insert = xml.insert_element_after -xml.inject = xml.inject_element_after -xml.after = xml.insert_element_after -xml.before = xml.insert_element_before -xml.delete = xml.delete_element -xml.replace = xml.replace_element +local concat = table.concat +local type, next, tonumber, tostring, setmetatable, loadstring = type, next, tonumber, tostring, setmetatable, loadstring +local format, gsub, match = string.format, string.gsub, string.match +local lpegmatch = lpeg.match --[[ldx-- -<p>The following helper functions best belong to the <t>lmxl-ini</t> +<p>The following helper functions best belong to the <t>lxml-ini</t> module. Some are here because we need then in the <t>mk</t> document and other manuals, others came up when playing with this module. Since this module is also used in <l n='mtxrun'/> we've put them here instead of loading mode modules there then needed.</p> --ldx]]-- -function xml.gsub(t,old,new) +local function xmlgsub(t,old,new) -- will be replaced local dt = t.dt if dt then for k=1,#dt do @@ -5069,28 +6366,26 @@ function xml.gsub(t,old,new) if type(v) == "string" then dt[k] = gsub(v,old,new) else - xml.gsub(v,old,new) + xmlgsub(v,old,new) end end end end +--~ xml.gsub = xmlgsub + function xml.strip_leading_spaces(dk,d,k) -- cosmetic, for manual - if d and k and d[k-1] and type(d[k-1]) == "string" then - local s = d[k-1]:match("\n(%s+)") - xml.gsub(dk,"\n"..string.rep(" ",#s),"\n") + if d and k then + local dkm = d[k-1] + if dkm and type(dkm) == "string" then + local s = match(dkm,"\n(%s+)") + xmlgsub(dk,"\n"..rep(" ",#s),"\n") + end end end -function xml.serialize_path(root,lpath,handle) - local dk, r, d, k = xml.first(root,lpath) - dk = xml.copy(dk) - xml.strip_leading_spaces(dk,d,k) - xml.serialize(dk,handle) -end - --~ xml.escapes = { ['&'] = '&', ['<'] = '<', ['>'] = '>', ['"'] = '"' } ---~ xml.unescapes = { } for k,v in pairs(xml.escapes) do xml.unescapes[v] = k end +--~ xml.unescapes = { } for k,v in next, xml.escapes do xml.unescapes[v] = k end --~ function xml.escaped (str) return (gsub(str,"(.)" , xml.escapes )) end --~ function xml.unescaped(str) return (gsub(str,"(&.-;)", xml.unescapes)) end @@ -5114,8 +6409,6 @@ local escaped = Cs(normal * (special * normal)^0) -- 100 * 1000 * "oeps< oeps> oeps&" : gsub:lpeg == 0153:0280:0151:0080 (last one by roberto) --- unescaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0) --- unescaped = Cs((((P("&")/"") * (P("lt")/"<" + P("gt")/">" + P("amp")/"&") * (P(";")/"")) + 1)^0) local normal = (1 - S"&")^0 local special = P("<")/"<" + P(">")/">" + P("&")/"&" local unescaped = Cs(normal * (special * normal)^0) @@ -5124,84 +6417,32 @@ local unescaped = Cs(normal * (special * normal)^0) local cleansed = Cs(((P("<") * (1-P(">"))^0 * P(">"))/"" + 1)^0) -function xml.escaped (str) return escaped :match(str) end -function xml.unescaped(str) return unescaped:match(str) end -function xml.cleansed (str) return cleansed :match(str) end +xml.escaped_pattern = escaped +xml.unescaped_pattern = unescaped +xml.cleansed_pattern = cleansed -function xml.join(t,separator,lastseparator) - if #t > 0 then - local result = { } - for k,v in pairs(t) do - result[k] = xml.tostring(v) - end - if lastseparator then - return concat(result,separator or "",1,#result-1) .. (lastseparator or "") .. result[#result] - else - return concat(result,separator) +function xml.escaped (str) return lpegmatch(escaped,str) end +function xml.unescaped(str) return lpegmatch(unescaped,str) end +function xml.cleansed (str) return lpegmatch(cleansed,str) end + +-- this might move + +function xml.fillin(root,pattern,str,check) + local e = xml.first(root,pattern) + if e then + local n = #e.dt + if not check or n == 0 or (n == 1 and e.dt[1] == "") then + e.dt = { str } end - else - return "" end end -function xml.statistics() - return { - lpathcalls = lpathcalls, - lpathcached = lpathcached, - } -end - --- xml.set_text_cleanup(xml.show_text_entities) --- xml.set_text_cleanup(xml.resolve_text_entities) - ---~ xml.lshow("/../../../a/(b|c)[@d='e']/f") ---~ xml.lshow("/../../../a/!(b|c)[@d='e']/f") ---~ xml.lshow("/../../../a/!b[@d!='e']/f") - ---~ x = xml.convert([[ ---~ <a> ---~ <b n='01'>01</b> ---~ <b n='02'>02</b> ---~ <b n='03'>03</b> ---~ <b n='04'>OK</b> ---~ <b n='05'>05</b> ---~ <b n='06'>06</b> ---~ <b n='07'>ALSO OK</b> ---~ </a> ---~ ]]) - ---~ xml.settrace("lpath",true) - ---~ xml.xshow(xml.first(x,"b[position() > 2 and position() < 5 and text() == 'ok']")) ---~ xml.xshow(xml.first(x,"b[position() > 2 and position() < 5 and text() == upper('ok')]")) ---~ xml.xshow(xml.first(x,"b[@n=='03' or @n=='08']")) ---~ xml.xshow(xml.all (x,"b[number(@n)>2 and number(@n)<6]")) ---~ xml.xshow(xml.first(x,"b[find(text(),'ALSO')]")) - ---~ str = [[ ---~ <?xml version="1.0" encoding="utf-8"?> ---~ <story line='mojca'> ---~ <windows>my secret</mouse> ---~ </story> ---~ ]] - ---~ x = xml.convert([[ ---~ <a><b n='01'>01</b><b n='02'>02</b><x>xx</x><b n='03'>03</b><b n='04'>OK</b></a> ---~ ]]) ---~ xml.xshow(xml.first(x,"b[tag(2) == 'x']")) ---~ xml.xshow(xml.first(x,"b[tag(1) == 'x']")) ---~ xml.xshow(xml.first(x,"b[tag(-1) == 'x']")) ---~ xml.xshow(xml.first(x,"b[tag(-2) == 'x']")) - ---~ print(xml.filter(x,"b/tag(2)")) ---~ print(xml.filter(x,"b/tag(1)")) - end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['lxml-ent'] = { +if not modules then modules = { } end modules ['lxml-aux'] = { version = 1.001, comment = "this module is the basis for the lxml-* ones", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", @@ -5209,457 +6450,836 @@ if not modules then modules = { } end modules ['lxml-ent'] = { license = "see context related readme files" } -local type, next, tonumber, tostring, setmetatable, loadstring = type, next, tonumber, tostring, setmetatable, loadstring -local format, gsub, find = string.format, string.gsub, string.find -local utfchar = unicode.utf8.char +-- not all functions here make sense anymore vbut we keep them for +-- compatibility reasons ---[[ldx-- -<p>We provide (at least here) two entity handlers. The more extensive -resolver consults a hash first, tries to convert to <l n='utf'/> next, -and finaly calls a handler when defines. When this all fails, the -original entity is returned.</p> ---ldx]]-- +local trace_manipulations = false trackers.register("lxml.manipulations", function(v) trace_manipulations = v end) -xml.entities = xml.entities or { } -- xml.entity_handler == function +local xmlparseapply, xmlconvert, xmlcopy, xmlname = xml.parse_apply, xml.convert, xml.copy, xml.name +local xmlinheritedconvert = xml.inheritedconvert -function xml.entity_handler(e) - return format("[%s]",e) -end +local type = type +local insert, remove = table.insert, table.remove +local gmatch, gsub = string.gmatch, string.gsub -local function toutf(s) - return utfchar(tonumber(s,16)) +local function report(what,pattern,c,e) + logs.report("xml","%s element '%s' (root: '%s', position: %s, index: %s, pattern: %s)",what,xmlname(e),xmlname(e.__p__),c,e.ni,pattern) end -local function utfize(root) - local d = root.dt - for k=1,#d do - local dk = d[k] - if type(dk) == "string" then - -- test prevents copying if no match - if find(dk,"&#x.-;") then - d[k] = gsub(dk,"&#x(.-);",toutf) +local function withelements(e,handle,depth) + if e and handle then + local edt = e.dt + if edt then + depth = depth or 0 + for i=1,#edt do + local e = edt[i] + if type(e) == "table" then + handle(e,depth) + withelements(e,handle,depth+1) + end end - else - utfize(dk) end end end -xml.utfize = utfize +xml.withelements = withelements -local function resolve(e) -- hex encoded always first, just to avoid mkii fallbacks - if find(e,"^#x") then - return utfchar(tonumber(e:sub(3),16)) - elseif find(e,"^#") then - return utfchar(tonumber(e:sub(2))) - else - local ee = xml.entities[e] -- we cannot shortcut this one (is reloaded) - if ee then - return ee - else - local h = xml.entity_handler - return (h and h(e)) or "&" .. e .. ";" +function xml.withelement(e,n,handle) -- slow + if e and n ~= 0 and handle then + local edt = e.dt + if edt then + if n > 0 then + for i=1,#edt do + local ei = edt[i] + if type(ei) == "table" then + if n == 1 then + handle(ei) + return + else + n = n - 1 + end + end + end + elseif n < 0 then + for i=#edt,1,-1 do + local ei = edt[i] + if type(ei) == "table" then + if n == -1 then + handle(ei) + return + else + n = n + 1 + end + end + end + end end end end -local function resolve_entities(root) - if not root.special or root.tg == "@rt@" then - local d = root.dt - for k=1,#d do - local dk = d[k] - if type(dk) == "string" then - if find(dk,"&.-;") then - d[k] = gsub(dk,"&(.-);",resolve) - end - else - resolve_entities(dk) +xml.elements_only = xml.collected + +function xml.each_element(root,pattern,handle,reverse) + local collected = xmlparseapply({ root },pattern) + if collected then + if reverse then + for c=#collected,1,-1 do + handle(collected[c]) + end + else + for c=1,#collected do + handle(collected[c]) end end + return collected end end -xml.resolve_entities = resolve_entities +xml.process_elements = xml.each_element -function xml.utfize_text(str) - if find(str,"&#") then - return (gsub(str,"&#x(.-);",toutf)) - else - return str +function xml.process_attributes(root,pattern,handle) + local collected = xmlparseapply({ root },pattern) + if collected and handle then + for c=1,#collected do + handle(collected[c].at) + end end + return collected end -function xml.resolve_text_entities(str) -- maybe an lpeg. maybe resolve inline - if find(str,"&") then - return (gsub(str,"&(.-);",resolve)) - else - return str - end +--[[ldx-- +<p>The following functions collect elements and texts.</p> +--ldx]]-- + +-- are these still needed -> lxml-cmp.lua + +function xml.collect_elements(root, pattern) + return xmlparseapply({ root },pattern) end -function xml.show_text_entities(str) - if find(str,"&") then - return (gsub(str,"&(.-);","[%1]")) - else - return str +function xml.collect_texts(root, pattern, flatten) -- todo: variant with handle + local collected = xmlparseapply({ root },pattern) + if collected and flatten then + local xmltostring = xml.tostring + for c=1,#collected do + collected[c] = xmltostring(collected[c].dt) + end end + return collected or { } end --- experimental, this will be done differently - -function xml.merge_entities(root) - local documententities = root.entities - local allentities = xml.entities - if documententities then - for k, v in next, documententities do - allentities[k] = v +function xml.collect_tags(root, pattern, nonamespace) + local collected = xmlparseapply({ root },pattern) + if collected then + local t = { } + for c=1,#collected do + local e = collected[c] + local ns, tg = e.ns, e.tg + if nonamespace then + t[#t+1] = tg + elseif ns == "" then + t[#t+1] = tg + else + t[#t+1] = ns .. ":" .. tg + end end + return t end end +--[[ldx-- +<p>We've now arrived at the functions that manipulate the tree.</p> +--ldx]]-- -end -- of closure +local no_root = { no_root = true } -do -- create closure to overcome 200 locals limit +function xml.redo_ni(d) + for k=1,#d do + local dk = d[k] + if type(dk) == "table" then + dk.ni = k + end + end +end -if not modules then modules = { } end modules ['lxml-mis'] = { - version = 1.001, - comment = "this module is the basis for the lxml-* ones", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} +local function xmltoelement(whatever,root) + if not whatever then + return nil + end + local element + if type(whatever) == "string" then + element = xmlinheritedconvert(whatever,root) + else + element = whatever -- we assume a table + end + if element.error then + return whatever -- string + end + if element then + --~ if element.ri then + --~ element = element.dt[element.ri].dt + --~ else + --~ element = element.dt + --~ end + end + return element +end -local concat = table.concat -local type, next, tonumber, tostring, setmetatable, loadstring = type, next, tonumber, tostring, setmetatable, loadstring -local format, gsub = string.format, string.gsub +xml.toelement = xmltoelement ---[[ldx-- -<p>The following helper functions best belong to the <t>lmxl-ini</t> -module. Some are here because we need then in the <t>mk</t> -document and other manuals, others came up when playing with -this module. Since this module is also used in <l n='mtxrun'/> we've -put them here instead of loading mode modules there then needed.</p> ---ldx]]-- +local function copiedelement(element,newparent) + if type(element) == "string" then + return element + else + element = xmlcopy(element).dt + if newparent and type(element) == "table" then + element.__p__ = newparent + end + return element + end +end -function xml.gsub(t,old,new) - local dt = t.dt - if dt then - for k=1,#dt do - local v = dt[k] - if type(v) == "string" then - dt[k] = gsub(v,old,new) - else - xml.gsub(v,old,new) +function xml.delete_element(root,pattern) + local collected = xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + local e = collected[c] + local p = e.__p__ + if p then + if trace_manipulations then + report('deleting',pattern,c,e) + end + local d = p.dt + remove(d,e.ni) + xml.redo_ni(d) -- can be made faster and inlined end end end end -function xml.strip_leading_spaces(dk,d,k) -- cosmetic, for manual - if d and k and d[k-1] and type(d[k-1]) == "string" then - local s = d[k-1]:match("\n(%s+)") - xml.gsub(dk,"\n"..string.rep(" ",#s),"\n") +function xml.replace_element(root,pattern,whatever) + local element = root and xmltoelement(whatever,root) + local collected = element and xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + local e = collected[c] + local p = e.__p__ + if p then + if trace_manipulations then + report('replacing',pattern,c,e) + end + local d = p.dt + d[e.ni] = copiedelement(element,p) + xml.redo_ni(d) -- probably not needed + end + end end end -function xml.serialize_path(root,lpath,handle) - local dk, r, d, k = xml.first(root,lpath) - dk = xml.copy(dk) - xml.strip_leading_spaces(dk,d,k) - xml.serialize(dk,handle) +local function inject_element(root,pattern,whatever,prepend) + local element = root and xmltoelement(whatever,root) + local collected = element and xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + local e = collected[c] + local r = e.__p__ + local d, k, rri = r.dt, e.ni, r.ri + local edt = (rri and d[rri].dt) or (d and d[k] and d[k].dt) + if edt then + local be, af + local cp = copiedelement(element,e) + if prepend then + be, af = cp, edt + else + be, af = edt, cp + end + for i=1,#af do + be[#be+1] = af[i] + end + if rri then + r.dt[rri].dt = be + else + d[k].dt = be + end + xml.redo_ni(d) + end + end + end end ---~ xml.escapes = { ['&'] = '&', ['<'] = '<', ['>'] = '>', ['"'] = '"' } ---~ xml.unescapes = { } for k,v in pairs(xml.escapes) do xml.unescapes[v] = k end +local function insert_element(root,pattern,whatever,before) -- todo: element als functie + local element = root and xmltoelement(whatever,root) + local collected = element and xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + local e = collected[c] + local r = e.__p__ + local d, k = r.dt, e.ni + if not before then + k = k + 1 + end + insert(d,k,copiedelement(element,r)) + xml.redo_ni(d) + end + end +end ---~ function xml.escaped (str) return (gsub(str,"(.)" , xml.escapes )) end ---~ function xml.unescaped(str) return (gsub(str,"(&.-;)", xml.unescapes)) end ---~ function xml.cleansed (str) return (gsub(str,"<.->" , '' )) end -- "%b<>" +xml.insert_element = insert_element +xml.insert_element_after = insert_element +xml.insert_element_before = function(r,p,e) insert_element(r,p,e,true) end +xml.inject_element = inject_element +xml.inject_element_after = inject_element +xml.inject_element_before = function(r,p,e) inject_element(r,p,e,true) end -local P, S, R, C, V, Cc, Cs = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc, lpeg.Cs +local function include(xmldata,pattern,attribute,recursive,loaddata) + -- parse="text" (default: xml), encoding="" (todo) + -- attribute = attribute or 'href' + pattern = pattern or 'include' + loaddata = loaddata or io.loaddata + local collected = xmlparseapply({ xmldata },pattern) + if collected then + for c=1,#collected do + local ek = collected[c] + local name = nil + local ekdt = ek.dt + local ekat = ek.at + local epdt = ek.__p__.dt + if not attribute or attribute == "" then + name = (type(ekdt) == "table" and ekdt[1]) or ekdt -- ckeck, probably always tab or str + end + if not name then + for a in gmatch(attribute or "href","([^|]+)") do + name = ekat[a] + if name then break end + end + end + local data = (name and name ~= "" and loaddata(name)) or "" + if data == "" then + epdt[ek.ni] = "" -- xml.empty(d,k) + elseif ekat["parse"] == "text" then + -- for the moment hard coded + epdt[ek.ni] = xml.escaped(data) -- d[k] = xml.escaped(data) + else +--~ local settings = xmldata.settings +--~ settings.parent_root = xmldata -- to be tested +--~ local xi = xmlconvert(data,settings) + local xi = xmlinheritedconvert(data,xmldata) + if not xi then + epdt[ek.ni] = "" -- xml.empty(d,k) + else + if recursive then + include(xi,pattern,attribute,recursive,loaddata) + end + epdt[ek.ni] = xml.body(xi) -- xml.assign(d,k,xi) + end + end + end + end +end --- 100 * 2500 * "oeps< oeps> oeps&" : gsub:lpeg|lpeg|lpeg --- --- 1021:0335:0287:0247 +xml.include = include --- 10 * 1000 * "oeps< oeps> oeps& asfjhalskfjh alskfjh alskfjh alskfjh ;al J;LSFDJ" --- --- 1559:0257:0288:0190 (last one suggested by roberto) +--~ local function manipulate(xmldata,pattern,manipulator) -- untested and might go away +--~ local collected = xmlparseapply({ xmldata },pattern) +--~ if collected then +--~ local xmltostring = xml.tostring +--~ for c=1,#collected do +--~ local e = collected[c] +--~ local data = manipulator(xmltostring(e)) +--~ if data == "" then +--~ epdt[e.ni] = "" +--~ else +--~ local xi = xmlinheritedconvert(data,xmldata) +--~ if not xi then +--~ epdt[e.ni] = "" +--~ else +--~ epdt[e.ni] = xml.body(xi) -- xml.assign(d,k,xi) +--~ end +--~ end +--~ end +--~ end +--~ end --- escaped = Cs((S("<&>") / xml.escapes + 1)^0) --- escaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0) -local normal = (1 - S("<&>"))^0 -local special = P("<")/"<" + P(">")/">" + P("&")/"&" -local escaped = Cs(normal * (special * normal)^0) +--~ xml.manipulate = manipulate --- 100 * 1000 * "oeps< oeps> oeps&" : gsub:lpeg == 0153:0280:0151:0080 (last one by roberto) +function xml.strip_whitespace(root, pattern, nolines) -- strips all leading and trailing space ! + local collected = xmlparseapply({ root },pattern) + if collected then + for i=1,#collected do + local e = collected[i] + local edt = e.dt + if edt then + local t = { } + for i=1,#edt do + local str = edt[i] + if type(str) == "string" then + if str == "" then + -- stripped + else + if nolines then + str = gsub(str,"[ \n\r\t]+"," ") + end + if str == "" then + -- stripped + else + t[#t+1] = str + end + end + else + --~ str.ni = i + t[#t+1] = str + end + end + e.dt = t + end + end + end +end --- unescaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0) --- unescaped = Cs((((P("&")/"") * (P("lt")/"<" + P("gt")/">" + P("amp")/"&") * (P(";")/"")) + 1)^0) -local normal = (1 - S"&")^0 -local special = P("<")/"<" + P(">")/">" + P("&")/"&" -local unescaped = Cs(normal * (special * normal)^0) +function xml.strip_whitespace(root, pattern, nolines, anywhere) -- strips all leading and trailing spacing + local collected = xmlparseapply({ root },pattern) -- beware, indices no longer are valid now + if collected then + for i=1,#collected do + local e = collected[i] + local edt = e.dt + if edt then + if anywhere then + local t = { } + for e=1,#edt do + local str = edt[e] + if type(str) ~= "string" then + t[#t+1] = str + elseif str ~= "" then + -- todo: lpeg for each case + if nolines then + str = gsub(str,"%s+"," ") + end + str = gsub(str,"^%s*(.-)%s*$","%1") + if str ~= "" then + t[#t+1] = str + end + end + end + e.dt = t + else + -- we can assume a regular sparse xml table with no successive strings + -- otherwise we should use a while loop + if #edt > 0 then + -- strip front + local str = edt[1] + if type(str) ~= "string" then + -- nothing + elseif str == "" then + remove(edt,1) + else + if nolines then + str = gsub(str,"%s+"," ") + end + str = gsub(str,"^%s+","") + if str == "" then + remove(edt,1) + else + edt[1] = str + end + end + end + if #edt > 1 then + -- strip end + local str = edt[#edt] + if type(str) ~= "string" then + -- nothing + elseif str == "" then + remove(edt) + else + if nolines then + str = gsub(str,"%s+"," ") + end + str = gsub(str,"%s+$","") + if str == "" then + remove(edt) + else + edt[#edt] = str + end + end + end + end + end + end + end +end --- 100 * 5000 * "oeps <oeps bla='oeps' foo='bar'> oeps </oeps> oeps " : gsub:lpeg == 623:501 msec (short tags, less difference) +local function rename_space(root, oldspace, newspace) -- fast variant + local ndt = #root.dt + for i=1,ndt or 0 do + local e = root[i] + if type(e) == "table" then + if e.ns == oldspace then + e.ns = newspace + if e.rn then + e.rn = newspace + end + end + local edt = e.dt + if edt then + rename_space(edt, oldspace, newspace) + end + end + end +end -local cleansed = Cs(((P("<") * (1-P(">"))^0 * P(">"))/"" + 1)^0) +xml.rename_space = rename_space -xml.escaped_pattern = escaped -xml.unescaped_pattern = unescaped -xml.cleansed_pattern = cleansed +function xml.remap_tag(root, pattern, newtg) + local collected = xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + collected[c].tg = newtg + end + end +end -function xml.escaped (str) return escaped :match(str) end -function xml.unescaped(str) return unescaped:match(str) end -function xml.cleansed (str) return cleansed :match(str) end +function xml.remap_namespace(root, pattern, newns) + local collected = xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + collected[c].ns = newns + end + end +end -function xml.join(t,separator,lastseparator) - if #t > 0 then - local result = { } - for k,v in pairs(t) do - result[k] = xml.tostring(v) +function xml.check_namespace(root, pattern, newns) + local collected = xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + local e = collected[c] + if (not e.rn or e.rn == "") and e.ns == "" then + e.rn = newns + end end - if lastseparator then - return concat(result,separator or "",1,#result-1) .. (lastseparator or "") .. result[#result] - else - return concat(result,separator) + end +end + +function xml.remap_name(root, pattern, newtg, newns, newrn) + local collected = xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + local e = collected[c] + e.tg, e.ns, e.rn = newtg, newns, newrn end - else - return "" end end +--[[ldx-- +<p>Here are a few synonyms.</p> +--ldx]]-- + +xml.each = xml.each_element +xml.process = xml.process_element +xml.strip = xml.strip_whitespace +xml.collect = xml.collect_elements +xml.all = xml.collect_elements + +xml.insert = xml.insert_element_after +xml.inject = xml.inject_element_after +xml.after = xml.insert_element_after +xml.before = xml.insert_element_before +xml.delete = xml.delete_element +xml.replace = xml.replace_element + end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['trac-tra'] = { +if not modules then modules = { } end modules ['lxml-xml'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "this module is the basis for the lxml-* ones", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } --- the <anonymous> tag is kind of generic and used for functions that are not --- bound to a variable, like node.new, node.copy etc (contrary to for instance --- node.has_attribute which is bound to a has_attribute local variable in mkiv) +local finalizers = xml.finalizers.xml +local xmlfilter = xml.filter -- we could inline this one for speed +local xmltostring = xml.tostring +local xmlserialize = xml.serialize -debugger = debugger or { } +local function first(collected) -- wrong ? + return collected and collected[1] +end -local counters = { } -local names = { } -local getinfo = debug.getinfo -local format, find, lower, gmatch = string.format, string.find, string.lower, string.gmatch +local function last(collected) + return collected and collected[#collected] +end --- one +local function all(collected) + return collected +end -local function hook() - local f = getinfo(2,"f").func - local n = getinfo(2,"Sn") --- if n.what == "C" and n.name then print (n.namewhat .. ': ' .. n.name) end - if f then - local cf = counters[f] - if cf == nil then - counters[f] = 1 - names[f] = n - else - counters[f] = cf + 1 +local function reverse(collected) + if collected then + local reversed = { } + for c=#collected,1,-1 do + reversed[#reversed+1] = collected[c] end + return reversed end end -local function getname(func) - local n = names[func] - if n then - if n.what == "C" then - return n.name or '<anonymous>' - else - -- source short_src linedefined what name namewhat nups func - local name = n.name or n.namewhat or n.what - if not name or name == "" then name = "?" end - return format("%s : %s : %s", n.short_src or "unknown source", n.linedefined or "--", name) - end - else - return "unknown" + +local function attribute(collected,name) + if collected and #collected > 0 then + local at = collected[1].at + return at and at[name] end end -function debugger.showstats(printer,threshold) - printer = printer or texio.write or print - threshold = threshold or 0 - local total, grandtotal, functions = 0, 0, 0 - printer("\n") -- ugly but ok - -- table.sort(counters) - for func, count in pairs(counters) do - if count > threshold then - local name = getname(func) - if not name:find("for generator") then - printer(format("%8i %s", count, name)) - total = total + count - end - end - grandtotal = grandtotal + count - functions = functions + 1 - end - printer(format("functions: %s, total: %s, grand total: %s, threshold: %s\n", functions, total, grandtotal, threshold)) + +local function att(id,name) + local at = id.at + return at and at[name] end --- two +local function count(collected) + return (collected and #collected) or 0 +end ---~ local function hook() ---~ local n = getinfo(2) ---~ if n.what=="C" and not n.name then ---~ local f = tostring(debug.traceback()) ---~ local cf = counters[f] ---~ if cf == nil then ---~ counters[f] = 1 ---~ names[f] = n ---~ else ---~ counters[f] = cf + 1 ---~ end ---~ end ---~ end ---~ function debugger.showstats(printer,threshold) ---~ printer = printer or texio.write or print ---~ threshold = threshold or 0 ---~ local total, grandtotal, functions = 0, 0, 0 ---~ printer("\n") -- ugly but ok ---~ -- table.sort(counters) ---~ for func, count in pairs(counters) do ---~ if count > threshold then ---~ printer(format("%8i %s", count, func)) ---~ total = total + count ---~ end ---~ grandtotal = grandtotal + count ---~ functions = functions + 1 ---~ end ---~ printer(format("functions: %s, total: %s, grand total: %s, threshold: %s\n", functions, total, grandtotal, threshold)) ---~ end +local function position(collected,n) + if collected then + n = tonumber(n) or 0 + if n < 0 then + return collected[#collected + n + 1] + elseif n > 0 then + return collected[n] + else + return collected[1].mi or 0 + end + end +end --- rest +local function match(collected) + return (collected and collected[1].mi) or 0 -- match +end -function debugger.savestats(filename,threshold) - local f = io.open(filename,'w') - if f then - debugger.showstats(function(str) f:write(str) end,threshold) - f:close() +local function index(collected) + if collected then + return collected[1].ni end end -function debugger.enable() - debug.sethook(hook,"c") +local function attributes(collected,arguments) + if collected then + local at = collected[1].at + if arguments then + return at[arguments] + elseif next(at) then + return at -- all of them + end + end end -function debugger.disable() - debug.sethook() ---~ counters[debug.getinfo(2,"f").func] = nil +local function chainattribute(collected,arguments) -- todo: optional levels + if collected then + local e = collected[1] + while e do + local at = e.at + if at then + local a = at[arguments] + if a then + return a + end + else + break -- error + end + e = e.__p__ + end + end + return "" end -function debugger.tracing() - local n = tonumber(os.env['MTX.TRACE.CALLS']) or tonumber(os.env['MTX_TRACE_CALLS']) or 0 - if n > 0 then - function debugger.tracing() return true end ; return true +local function raw(collected) -- hybrid + if collected then + local e = collected[1] or collected + return (e and xmlserialize(e)) or "" -- only first as we cannot concat function else - function debugger.tracing() return false end ; return false + return "" end end ---~ debugger.enable() - ---~ print(math.sin(1*.5)) ---~ print(math.sin(1*.5)) ---~ print(math.sin(1*.5)) ---~ print(math.sin(1*.5)) ---~ print(math.sin(1*.5)) - ---~ debugger.disable() - ---~ print("") ---~ debugger.showstats() ---~ print("") ---~ debugger.showstats(print,3) - -trackers = trackers or { } +local function text(collected) -- hybrid + if collected then + local e = collected[1] or collected + return (e and xmltostring(e.dt)) or "" + else + return "" + end +end -local data, done = { }, { } +local function texts(collected) + if collected then + local t = { } + for c=1,#collected do + local e = collection[c] + if e and e.dt then + t[#t+1] = e.dt + end + end + return t + end +end -local function set(what,value) - if type(what) == "string" then - what = aux.settings_to_array(what) +local function tag(collected,n) + if collected then + local c + if n == 0 or not n then + c = collected[1] + elseif n > 1 then + c = collected[n] + else + c = collected[#collected-n+1] + end + return c and c.tg end - for i=1,#what do - local w = what[i] - for d, f in next, data do - if done[d] then - -- prevent recursion due to wildcards - elseif find(d,w) then - done[d] = true - for i=1,#f do - f[i](value) - end +end + +local function name(collected,n) + if collected then + local c + if n == 0 or not n then + c = collected[1] + elseif n > 1 then + c = collected[n] + else + c = collected[#collected-n+1] + end + if c then + if c.ns == "" then + return c.tg + else + return c.ns .. ":" .. c.tg end end end end -local function reset() - for d, f in next, data do - for i=1,#f do - f[i](false) +local function tags(collected,nonamespace) + if collected then + local t = { } + for c=1,#collected do + local e = collected[c] + local ns, tg = e.ns, e.tg + if nonamespace or ns == "" then + t[#t+1] = tg + else + t[#t+1] = ns .. ":" .. tg + end end + return t end end -function trackers.register(what,...) - what = lower(what) - local w = data[what] - if not w then - w = { } - data[what] = w - end - for _, fnc in next, { ... } do - local typ = type(fnc) - if typ == "function" then - w[#w+1] = fnc - elseif typ == "string" then - w[#w+1] = function(value) set(fnc,value,nesting) end +local function empty(collected) + if collected then + for c=1,#collected do + local e = collected[c] + if e then + local edt = e.dt + if edt then + local n = #edt + if n == 1 then + local edk = edt[1] + local typ = type(edk) + if typ == "table" then + return false + elseif edk ~= "" then -- maybe an extra tester for spacing only + return false + end + elseif n > 1 then + return false + end + end + end end end + return true end -function trackers.enable(what) - done = { } - set(what,true) +finalizers.first = first +finalizers.last = last +finalizers.all = all +finalizers.reverse = reverse +finalizers.elements = all +finalizers.default = all +finalizers.attribute = attribute +finalizers.att = att +finalizers.count = count +finalizers.position = position +finalizers.match = match +finalizers.index = index +finalizers.attributes = attributes +finalizers.chainattribute = chainattribute +finalizers.text = text +finalizers.texts = texts +finalizers.tag = tag +finalizers.name = name +finalizers.tags = tags +finalizers.empty = empty + +-- shortcuts -- we could support xmlfilter(id,pattern,first) + +function xml.first(id,pattern) + return first(xmlfilter(id,pattern)) end -function trackers.disable(what) - done = { } - if not what or what == "" then - trackers.reset(what) +function xml.last(id,pattern) + return last(xmlfilter(id,pattern)) +end + +function xml.count(id,pattern) + return count(xmlfilter(id,pattern)) +end + +function xml.attribute(id,pattern,a,default) + return attribute(xmlfilter(id,pattern),a,default) +end + +function xml.raw(id,pattern) + if pattern then + return raw(xmlfilter(id,pattern)) else - set(what,false) + return raw(id) end end -function trackers.reset(what) - done = { } - reset() +function xml.text(id,pattern) + if pattern then + -- return text(xmlfilter(id,pattern)) + local collected = xmlfilter(id,pattern) + return (collected and xmltostring(collected[1].dt)) or "" + elseif id then + -- return text(id) + return xmltostring(id.dt) or "" + else + return "" + end end -function trackers.list() -- pattern - local list = table.sortedkeys(data) - local user, system = { }, { } - for l=1,#list do - local what = list[l] - if find(what,"^%*") then - system[#system+1] = what - else - user[#user+1] = what - end - end - return user, system +xml.content = text + +function xml.position(id,pattern,n) -- element + return position(xmlfilter(id,pattern),n) +end + +function xml.match(id,pattern) -- number + return match(xmlfilter(id,pattern)) +end + +function xml.empty(id,pattern) + return empty(xmlfilter(id,pattern)) end +xml.all = xml.filter +xml.index = xml.position +xml.found = xml.filter + end -- of closure @@ -5667,7 +7287,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['luat-env'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -5679,10 +7299,10 @@ if not modules then modules = { } end modules ['luat-env'] = { -- evolved before bytecode arrays were available and so a lot of -- code has disappeared already. -local trace_verbose = false trackers.register("resolvers.verbose", function(v) trace_verbose = v end) -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v trackers.enable("resolvers.verbose") end) +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) -local format = string.format +local format, sub, match, gsub, find = string.format, string.sub, string.match, string.gsub, string.find +local unquote, quote = string.unquote, string.quote -- precautions @@ -5716,13 +7336,14 @@ if not environment.jobname then environ function environment.initialize_arguments(arg) local arguments, files = { }, { } environment.arguments, environment.files, environment.sortedflags = arguments, files, nil - for index, argument in pairs(arg) do + for index=1,#arg do + local argument = arg[index] if index > 0 then - local flag, value = argument:match("^%-+(.+)=(.-)$") + local flag, value = match(argument,"^%-+(.-)=(.-)$") if flag then - arguments[flag] = string.unquote(value or "") + arguments[flag] = unquote(value or "") else - flag = argument:match("^%-+(.+)") + flag = match(argument,"^%-+(.+)") if flag then arguments[flag] = true else @@ -5749,25 +7370,30 @@ function environment.argument(name,partial) return arguments[name] elseif partial then if not sortedflags then - sortedflags = { } - for _,v in pairs(table.sortedkeys(arguments)) do - sortedflags[#sortedflags+1] = "^" .. v + sortedflags = table.sortedkeys(arguments) + for k=1,#sortedflags do + sortedflags[k] = "^" .. sortedflags[k] end environment.sortedflags = sortedflags end -- example of potential clash: ^mode ^modefile - for _,v in ipairs(sortedflags) do - if name:find(v) then - return arguments[v:sub(2,#v)] + for k=1,#sortedflags do + local v = sortedflags[k] + if find(name,v) then + return arguments[sub(v,2,#v)] end end end return nil end +environment.argument("x",true) + function environment.split_arguments(separator) -- rather special, cut-off before separator local done, before, after = false, { }, { } - for _,v in ipairs(environment.original_arguments) do + local original_arguments = environment.original_arguments + for k=1,#original_arguments do + local v = original_arguments[k] if not done and v == separator then done = true elseif done then @@ -5784,16 +7410,17 @@ function environment.reconstruct_commandline(arg,noquote) if noquote and #arg == 1 then local a = arg[1] a = resolvers.resolve(a) - a = a:unquote() + a = unquote(a) return a - elseif next(arg) then + elseif #arg > 0 then local result = { } - for _,a in ipairs(arg) do -- ipairs 1 .. #n + for i=1,#arg do + local a = arg[i] a = resolvers.resolve(a) - a = a:unquote() - a = a:gsub('"','\\"') -- tricky - if a:find(" ") then - result[#result+1] = a:quote() + a = unquote(a) + a = gsub(a,'"','\\"') -- tricky + if find(a," ") then + result[#result+1] = quote(a) else result[#result+1] = a end @@ -5806,17 +7433,18 @@ end if arg then - -- new, reconstruct quoted snippets (maybe better just remnove the " then and add them later) + -- new, reconstruct quoted snippets (maybe better just remove the " then and add them later) local newarg, instring = { }, false - for index, argument in ipairs(arg) do - if argument:find("^\"") then - newarg[#newarg+1] = argument:gsub("^\"","") - if not argument:find("\"$") then + for index=1,#arg do + local argument = arg[index] + if find(argument,"^\"") then + newarg[#newarg+1] = gsub(argument,"^\"","") + if not find(argument,"\"$") then instring = true end - elseif argument:find("\"$") then - newarg[#newarg] = newarg[#newarg] .. " " .. argument:gsub("\"$","") + elseif find(argument,"\"$") then + newarg[#newarg] = newarg[#newarg] .. " " .. gsub(argument,"\"$","") instring = false elseif instring then newarg[#newarg] = newarg[#newarg] .. " " .. argument @@ -5871,12 +7499,12 @@ function environment.luafilechunk(filename) -- used for loading lua bytecode in filename = file.replacesuffix(filename, "lua") local fullname = environment.luafile(filename) if fullname and fullname ~= "" then - if trace_verbose then + if trace_locating then logs.report("fileio","loading file %s", fullname) end return environment.loadedluacode(fullname) else - if trace_verbose then + if trace_locating then logs.report("fileio","unknown file %s", filename) end return nil @@ -5896,7 +7524,7 @@ function environment.loadluafile(filename, version) -- when not overloaded by explicit suffix we look for a luc file first local fullname = (lucname and environment.luafile(lucname)) or "" if fullname ~= "" then - if trace_verbose then + if trace_locating then logs.report("fileio","loading %s", fullname) end chunk = loadfile(fullname) -- this way we don't need a file exists check @@ -5914,7 +7542,7 @@ function environment.loadluafile(filename, version) if v == version then return true else - if trace_verbose then + if trace_locating then logs.report("fileio","version mismatch for %s: lua=%s, luc=%s", filename, v, version) end environment.loadluafile(filename) @@ -5925,12 +7553,12 @@ function environment.loadluafile(filename, version) end fullname = (luaname and environment.luafile(luaname)) or "" if fullname ~= "" then - if trace_verbose then + if trace_locating then logs.report("fileio","loading %s", fullname) end chunk = loadfile(fullname) -- this way we don't need a file exists check if not chunk then - if verbose then + if trace_locating then logs.report("fileio","unknown file %s", filename) end else @@ -5948,7 +7576,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['trac-inf'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to trac-inf.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -5973,6 +7601,14 @@ function statistics.hastimer(instance) return instance and instance.starttime end +function statistics.resettiming(instance) + if not instance then + notimer = { timing = 0, loadtime = 0 } + else + instance.timing, instance.loadtime = 0, 0 + end +end + function statistics.starttiming(instance) if not instance then notimer = { } @@ -5987,6 +7623,8 @@ function statistics.starttiming(instance) if not instance.loadtime then instance.loadtime = 0 end + else +--~ logs.report("system","nested timing (%s)",tostring(instance)) end instance.timing = it + 1 end @@ -6032,6 +7670,12 @@ function statistics.elapsedindeed(instance) return t > statistics.threshold end +function statistics.elapsedseconds(instance,rest) -- returns nil if 0 seconds + if statistics.elapsedindeed(instance) then + return format("%s seconds %s", statistics.elapsedtime(instance),rest or "") + end +end + -- general function function statistics.register(tag,fnc) @@ -6110,14 +7754,32 @@ function statistics.timed(action,report) report("total runtime: %s",statistics.elapsedtime(timer)) end +-- where, not really the best spot for this: + +commands = commands or { } + +local timer + +function commands.resettimer() + statistics.resettiming(timer) + statistics.starttiming(timer) +end + +function commands.elapsedtime() + statistics.stoptiming(timer) + tex.sprint(statistics.elapsedtime(timer)) +end + +commands.resettimer() + end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['luat-log'] = { +if not modules then modules = { } end modules ['trac-log'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to trac-log.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -6125,7 +7787,11 @@ if not modules then modules = { } end modules ['luat-log'] = { -- this is old code that needs an overhaul -local write_nl, write, format = texio.write_nl or print, texio.write or io.write, string.format +--~ io.stdout:setvbuf("no") +--~ io.stderr:setvbuf("no") + +local write_nl, write = texio.write_nl or print, texio.write or io.write +local format, gmatch = string.format, string.gmatch local texcount = tex and tex.count if texlua then @@ -6206,25 +7872,48 @@ function logs.tex.line(fmt,...) -- new end end +--~ function logs.tex.start_page_number() +--~ local real, user, sub = texcount.realpageno, texcount.userpageno, texcount.subpageno +--~ if real > 0 then +--~ if user > 0 then +--~ if sub > 0 then +--~ write(format("[%s.%s.%s",real,user,sub)) +--~ else +--~ write(format("[%s.%s",real,user)) +--~ end +--~ else +--~ write(format("[%s",real)) +--~ end +--~ else +--~ write("[-") +--~ end +--~ end + +--~ function logs.tex.stop_page_number() +--~ write("]") +--~ end + +local real, user, sub + function logs.tex.start_page_number() - local real, user, sub = texcount.realpageno, texcount.userpageno, texcount.subpageno + real, user, sub = texcount.realpageno, texcount.userpageno, texcount.subpageno +end + +function logs.tex.stop_page_number() if real > 0 then if user > 0 then if sub > 0 then - write(format("[%s.%s.%s",real,user,sub)) + logs.report("pages", "flushing realpage %s, userpage %s, subpage %s",real,user,sub) else - write(format("[%s.%s",real,user)) + logs.report("pages", "flushing realpage %s, userpage %s",real,user) end else - write(format("[%s",real)) + logs.report("pages", "flushing realpage %s",real) end else - write("[-") + logs.report("pages", "flushing page") end -end - -function logs.tex.stop_page_number() - write("]") + io.flush() end logs.tex.report_job_stat = statistics.show_job_stat @@ -6324,7 +8013,7 @@ end function logs.setprogram(_name_,_banner_,_verbose_) name, banner = _name_, _banner_ if _verbose_ then - trackers.enable("resolvers.verbose") + trackers.enable("resolvers.locating") end logs.set_method("tex") logs.report = report -- also used in libraries @@ -6337,9 +8026,9 @@ end function logs.setverbose(what) if what then - trackers.enable("resolvers.verbose") + trackers.enable("resolvers.locating") else - trackers.disable("resolvers.verbose") + trackers.disable("resolvers.locating") end logs.verbose = what or false end @@ -6356,7 +8045,7 @@ logs.report = logs.tex.report logs.simple = logs.tex.report function logs.reportlines(str) -- todo: <lines></lines> - for line in str:gmatch("(.-)[\n\r]") do + for line in gmatch(str,"(.-)[\n\r]") do logs.report(line) end end @@ -6367,8 +8056,12 @@ end logs.simpleline = logs.reportline -function logs.help(message,option) +function logs.reportbanner() -- for scripts too logs.report(banner) +end + +function logs.help(message,option) + logs.reportbanner() logs.reportline() logs.reportlines(message) local moreinfo = logs.moreinfo or "" @@ -6400,6 +8093,11 @@ end --~ logs.system(syslogname,"context","test","fonts","font %s recached due to newer version (%s)","blabla","123") --~ end +function logs.fatal(where,...) + logs.report(where,"fatal error: %s, aborting now",format(...)) + os.exit() +end + end -- of closure @@ -6407,10 +8105,10 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-inp'] = { version = 1.001, + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files", - comment = "companion to luat-lib.tex", } -- After a few years using the code the large luat-inp.lua file @@ -6422,7 +8120,7 @@ if not modules then modules = { } end modules ['data-inp'] = { -- * some public auxiliary functions were made private -- -- TODO: os.getenv -> os.env[] --- TODO: instances.[hashes,cnffiles,configurations,522] -> ipairs (alles check, sneller) +-- TODO: instances.[hashes,cnffiles,configurations,522] -- TODO: check escaping in find etc, too much, too slow -- This lib is multi-purpose and can be loaded again later on so that @@ -6443,12 +8141,13 @@ if not modules then modules = { } end modules ['data-inp'] = { local format, gsub, find, lower, upper, match, gmatch = string.format, string.gsub, string.find, string.lower, string.upper, string.match, string.gmatch local concat, insert, sortedkeys = table.concat, table.insert, table.sortedkeys local next, type = next, type +local lpegmatch = lpeg.match -local trace_locating, trace_detail, trace_verbose = false, false, false +local trace_locating, trace_detail, trace_expansions = false, false, false -trackers.register("resolvers.verbose", function(v) trace_verbose = v end) -trackers.register("resolvers.locating", function(v) trace_locating = v trackers.enable("resolvers.verbose") end) -trackers.register("resolvers.detail", function(v) trace_detail = v trackers.enable("resolvers.verbose,resolvers.detail") end) +trackers.register("resolvers.locating", function(v) trace_locating = v end) +trackers.register("resolvers.details", function(v) trace_detail = v end) +trackers.register("resolvers.expansions", function(v) trace_expansions = v end) -- todo if not resolvers then resolvers = { @@ -6472,7 +8171,7 @@ resolvers.generators.notfound = { nil } resolvers.cacheversion = '1.0.1' resolvers.cnfname = 'texmf.cnf' resolvers.luaname = 'texmfcnf.lua' -resolvers.homedir = os.env[os.platform == "windows" and 'USERPROFILE'] or os.env['HOME'] or '~' +resolvers.homedir = os.env[os.type == "windows" and 'USERPROFILE'] or os.env['HOME'] or '~' resolvers.cnfdefault = '{$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c}' local dummy_path_expr = "^!*unset/*$" @@ -6514,8 +8213,8 @@ suffixes['lua'] = { 'lua', 'luc', 'tma', 'tmc' } alternatives['map files'] = 'map' alternatives['enc files'] = 'enc' -alternatives['cid files'] = 'cid' -alternatives['fea files'] = 'fea' +alternatives['cid maps'] = 'cid' -- great, why no cid files +alternatives['font feature files'] = 'fea' -- and fea files here alternatives['opentype fonts'] = 'otf' alternatives['truetype fonts'] = 'ttf' alternatives['truetype collections'] = 'ttc' @@ -6531,6 +8230,11 @@ formats ['sfd'] = 'SFDFONTS' suffixes ['sfd'] = { 'sfd' } alternatives['subfont definition files'] = 'sfd' +-- lib paths + +formats ['lib'] = 'CLUAINPUTS' -- new (needs checking) +suffixes['lib'] = (os.libsuffix and { os.libsuffix }) or { 'dll', 'so' } + -- In practice we will work within one tds tree, but i want to keep -- the option open to build tools that look at multiple trees, which is -- why we keep the tree specific data in a table. We used to pass the @@ -6653,8 +8357,10 @@ local function check_configuration() -- not yet ok, no time for debugging now -- bad luck end fix("LUAINPUTS" , ".;$TEXINPUTS;$TEXMFSCRIPTS") -- no progname, hm - fix("FONTFEATURES", ".;$TEXMF/fonts/fea//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") - fix("FONTCIDMAPS" , ".;$TEXMF/fonts/cid//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") + -- this will go away some day + fix("FONTFEATURES", ".;$TEXMF/fonts/{data,fea}//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") + fix("FONTCIDMAPS" , ".;$TEXMF/fonts/{data,cid}//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") + -- fix("LUATEXLIBS" , ".;$TEXMF/luatex/lua//") end @@ -6669,7 +8375,7 @@ function resolvers.settrace(n) -- no longer number but: 'locating' or 'detail' end end -resolvers.settrace(os.getenv("MTX.resolvers.TRACE") or os.getenv("MTX_INPUT_TRACE")) +resolvers.settrace(os.getenv("MTX_INPUT_TRACE")) function resolvers.osenv(key) local ie = instance.environment @@ -6757,37 +8463,43 @@ end -- work that well; the parsing is ok, but dealing with the resulting -- table is a pain because we need to work inside-out recursively +local function do_first(a,b) + local t = { } + for s in gmatch(b,"[^,]+") do t[#t+1] = a .. s end + return "{" .. concat(t,",") .. "}" +end + +local function do_second(a,b) + local t = { } + for s in gmatch(a,"[^,]+") do t[#t+1] = s .. b end + return "{" .. concat(t,",") .. "}" +end + +local function do_both(a,b) + local t = { } + for sa in gmatch(a,"[^,]+") do + for sb in gmatch(b,"[^,]+") do + t[#t+1] = sa .. sb + end + end + return "{" .. concat(t,",") .. "}" +end + +local function do_three(a,b,c) + return a .. b.. c +end + local function splitpathexpr(str, t, validate) -- no need for further optimization as it is only called a - -- few times, we can use lpeg for the sub; we could move - -- the local functions outside the body + -- few times, we can use lpeg for the sub + if trace_expansions then + logs.report("fileio","expanding variable '%s'",str) + end t = t or { } str = gsub(str,",}",",@}") str = gsub(str,"{,","{@,") -- str = "@" .. str .. "@" local ok, done - local function do_first(a,b) - local t = { } - for s in gmatch(b,"[^,]+") do t[#t+1] = a .. s end - return "{" .. concat(t,",") .. "}" - end - local function do_second(a,b) - local t = { } - for s in gmatch(a,"[^,]+") do t[#t+1] = s .. b end - return "{" .. concat(t,",") .. "}" - end - local function do_both(a,b) - local t = { } - for sa in gmatch(a,"[^,]+") do - for sb in gmatch(b,"[^,]+") do - t[#t+1] = sa .. sb - end - end - return "{" .. concat(t,",") .. "}" - end - local function do_three(a,b,c) - return a .. b.. c - end while true do done = false while true do @@ -6818,6 +8530,11 @@ local function splitpathexpr(str, t, validate) t[#t+1] = s end end + if trace_expansions then + for k=1,#t do + logs.report("fileio","% 4i: %s",k,t[k]) + end + end return t end @@ -6857,18 +8574,27 @@ end -- also we now follow the stupid route: if not set then just assume *one* -- cnf file under texmf (i.e. distribution) -resolvers.ownpath = resolvers.ownpath or nil -resolvers.ownbin = resolvers.ownbin or arg[-2] or arg[-1] or arg[0] or "luatex" -resolvers.autoselfdir = true -- false may be handy for debugging +local args = environment and environment.original_arguments or arg -- this needs a cleanup + +resolvers.ownbin = resolvers.ownbin or args[-2] or arg[-2] or args[-1] or arg[-1] or arg[0] or "luatex" +resolvers.ownbin = gsub(resolvers.ownbin,"\\","/") function resolvers.getownpath() - if not resolvers.ownpath then - if resolvers.autoselfdir and os.selfdir then - resolvers.ownpath = os.selfdir - else - local binary = resolvers.ownbin - if os.platform == "windows" then - binary = file.replacesuffix(binary,"exe") + local ownpath = resolvers.ownpath or os.selfdir + if not ownpath or ownpath == "" or ownpath == "unset" then + ownpath = args[-1] or arg[-1] + ownpath = ownpath and file.dirname(gsub(ownpath,"\\","/")) + if not ownpath or ownpath == "" then + ownpath = args[-0] or arg[-0] + ownpath = ownpath and file.dirname(gsub(ownpath,"\\","/")) + end + local binary = resolvers.ownbin + if not ownpath or ownpath == "" then + ownpath = ownpath and file.dirname(binary) + end + if not ownpath or ownpath == "" then + if os.binsuffix ~= "" then + binary = file.replacesuffix(binary,os.binsuffix) end for p in gmatch(os.getenv("PATH"),"[^"..io.pathseparator.."]+") do local b = file.join(p,binary) @@ -6880,30 +8606,39 @@ function resolvers.getownpath() local olddir = lfs.currentdir() if lfs.chdir(p) then local pp = lfs.currentdir() - if trace_verbose and p ~= pp then - logs.report("fileio","following symlink %s to %s",p,pp) + if trace_locating and p ~= pp then + logs.report("fileio","following symlink '%s' to '%s'",p,pp) end - resolvers.ownpath = pp + ownpath = pp lfs.chdir(olddir) else - if trace_verbose then - logs.report("fileio","unable to check path %s",p) + if trace_locating then + logs.report("fileio","unable to check path '%s'",p) end - resolvers.ownpath = p + ownpath = p end break end end end - if not resolvers.ownpath then resolvers.ownpath = '.' end + if not ownpath or ownpath == "" then + ownpath = "." + logs.report("fileio","forcing fallback ownpath .") + elseif trace_locating then + logs.report("fileio","using ownpath '%s'",ownpath) + end + end + resolvers.ownpath = ownpath + function resolvers.getownpath() + return resolvers.ownpath end - return resolvers.ownpath + return ownpath end local own_places = { "SELFAUTOLOC", "SELFAUTODIR", "SELFAUTOPARENT", "TEXMFCNF" } local function identify_own() - local ownpath = resolvers.getownpath() or lfs.currentdir() + local ownpath = resolvers.getownpath() or dir.current() local ie = instance.environment if ownpath then if resolvers.env('SELFAUTOLOC') == "" then os.env['SELFAUTOLOC'] = file.collapse_path(ownpath) end @@ -6916,10 +8651,10 @@ local function identify_own() if resolvers.env('TEXMFCNF') == "" then os.env['TEXMFCNF'] = resolvers.cnfdefault end if resolvers.env('TEXOS') == "" then os.env['TEXOS'] = resolvers.env('SELFAUTODIR') end if resolvers.env('TEXROOT') == "" then os.env['TEXROOT'] = resolvers.env('SELFAUTOPARENT') end - if trace_verbose then + if trace_locating then for i=1,#own_places do local v = own_places[i] - logs.report("fileio","variable %s set to %s",v,resolvers.env(v) or "unknown") + logs.report("fileio","variable '%s' set to '%s'",v,resolvers.env(v) or "unknown") end end identify_own = function() end @@ -6951,10 +8686,8 @@ end local function load_cnf_file(fname) fname = resolvers.clean_path(fname) local lname = file.replacesuffix(fname,'lua') - local f = io.open(lname) - if f then -- this will go - f:close() - local dname = file.dirname(fname) + if lfs.isfile(lname) then + local dname = file.dirname(fname) -- fname ? if not instance.configuration[dname] then resolvers.load_data(dname,'configuration',lname and file.basename(lname)) instance.order[#instance.order+1] = instance.configuration[dname] @@ -6962,8 +8695,8 @@ local function load_cnf_file(fname) else f = io.open(fname) if f then - if trace_verbose then - logs.report("fileio","loading %s", fname) + if trace_locating then + logs.report("fileio","loading configuration file %s", fname) end local line, data, n, k, v local dname = file.dirname(fname) @@ -6997,14 +8730,16 @@ local function load_cnf_file(fname) end end f:close() - elseif trace_verbose then - logs.report("fileio","skipping %s", fname) + elseif trace_locating then + logs.report("fileio","skipping configuration file '%s'", fname) end end end local function collapse_cnf_data() -- potential optimization: pass start index (setup and configuration are shared) - for _,c in ipairs(instance.order) do + local order = instance.order + for i=1,#order do + local c = order[i] for k,v in next, c do if not instance.variables[k] then if instance.environment[k] then @@ -7020,19 +8755,24 @@ end function resolvers.load_cnf() local function loadoldconfigdata() - for _, fname in ipairs(instance.cnffiles) do - load_cnf_file(fname) + local cnffiles = instance.cnffiles + for i=1,#cnffiles do + load_cnf_file(cnffiles[i]) end end -- instance.cnffiles contain complete names now ! + -- we still use a funny mix of cnf and new but soon + -- we will switch to lua exclusively as we only use + -- the file to collect the tree roots if #instance.cnffiles == 0 then - if trace_verbose then + if trace_locating then logs.report("fileio","no cnf files found (TEXMFCNF may not be set/known)") end else - instance.rootpath = instance.cnffiles[1] - for k,fname in ipairs(instance.cnffiles) do - instance.cnffiles[k] = file.collapse_path(gsub(fname,"\\",'/')) + local cnffiles = instance.cnffiles + instance.rootpath = cnffiles[1] + for k=1,#cnffiles do + instance.cnffiles[k] = file.collapse_path(cnffiles[k]) end for i=1,3 do instance.rootpath = file.dirname(instance.rootpath) @@ -7060,8 +8800,9 @@ function resolvers.load_lua() -- yet harmless else instance.rootpath = instance.luafiles[1] - for k,fname in ipairs(instance.luafiles) do - instance.luafiles[k] = file.collapse_path(gsub(fname,"\\",'/')) + local luafiles = instance.luafiles + for k=1,#luafiles do + instance.luafiles[k] = file.collapse_path(luafiles[k]) end for i=1,3 do instance.rootpath = file.dirname(instance.rootpath) @@ -7093,14 +8834,14 @@ end function resolvers.append_hash(type,tag,name) if trace_locating then - logs.report("fileio","= hash append: %s",tag) + logs.report("fileio","hash '%s' appended",tag) end insert(instance.hashes, { ['type']=type, ['tag']=tag, ['name']=name } ) end function resolvers.prepend_hash(type,tag,name) if trace_locating then - logs.report("fileio","= hash prepend: %s",tag) + logs.report("fileio","hash '%s' prepended",tag) end insert(instance.hashes, 1, { ['type']=type, ['tag']=tag, ['name']=name } ) end @@ -7124,9 +8865,11 @@ end -- locators function resolvers.locatelists() - for _, path in ipairs(resolvers.clean_path_list('TEXMF')) do - if trace_verbose then - logs.report("fileio","locating list of %s",path) + local texmfpaths = resolvers.clean_path_list('TEXMF') + for i=1,#texmfpaths do + local path = texmfpaths[i] + if trace_locating then + logs.report("fileio","locating list of '%s'",path) end resolvers.locatedatabase(file.collapse_path(path)) end @@ -7139,11 +8882,11 @@ end function resolvers.locators.tex(specification) if specification and specification ~= '' and lfs.isdir(specification) then if trace_locating then - logs.report("fileio",'! tex locator found: %s',specification) + logs.report("fileio","tex locator '%s' found",specification) end resolvers.append_hash('file',specification,filename) elseif trace_locating then - logs.report("fileio",'? tex locator not found: %s',specification) + logs.report("fileio","tex locator '%s' not found",specification) end end @@ -7157,7 +8900,9 @@ function resolvers.loadfiles() instance.loaderror = false instance.files = { } if not instance.renewcache then - for _, hash in ipairs(instance.hashes) do + local hashes = instance.hashes + for k=1,#hashes do + local hash = hashes[k] resolvers.hashdatabase(hash.tag,hash.name) if instance.loaderror then break end end @@ -7171,8 +8916,9 @@ end -- generators: function resolvers.loadlists() - for _, hash in ipairs(instance.hashes) do - resolvers.generatedatabase(hash.tag) + local hashes = instance.hashes + for i=1,#hashes do + resolvers.generatedatabase(hashes[i].tag) end end @@ -7184,10 +8930,27 @@ end local weird = lpeg.P(".")^1 + lpeg.anywhere(lpeg.S("~`!#$%^&*()={}[]:;\"\'||<>,?\n\r\t")) +--~ local l_forbidden = lpeg.S("~`!#$%^&*()={}[]:;\"\'||\\/<>,?\n\r\t") +--~ local l_confusing = lpeg.P(" ") +--~ local l_character = lpeg.patterns.utf8 +--~ local l_dangerous = lpeg.P(".") + +--~ local l_normal = (l_character - l_forbidden - l_confusing - l_dangerous) * (l_character - l_forbidden - l_confusing^2)^0 * lpeg.P(-1) +--~ ----- l_normal = l_normal * lpeg.Cc(true) + lpeg.Cc(false) + +--~ local function test(str) +--~ print(str,lpeg.match(l_normal,str)) +--~ end +--~ test("ヒラギノ明朝 Pro W3") +--~ test("..ヒラギノ明朝 Pro W3") +--~ test(":ヒラギノ明朝 Pro W3;") +--~ test("ヒラギノ明朝 /Pro W3;") +--~ test("ヒラギノ明朝 Pro W3") + function resolvers.generators.tex(specification) local tag = specification - if trace_verbose then - logs.report("fileio","scanning path %s",specification) + if trace_locating then + logs.report("fileio","scanning path '%s'",specification) end instance.files[tag] = { } local files = instance.files[tag] @@ -7203,7 +8966,8 @@ function resolvers.generators.tex(specification) full = spec end for name in directory(full) do - if not weird:match(name) then + if not lpegmatch(weird,name) then + -- if lpegmatch(l_normal,name) then local mode = attributes(full..name,'mode') if mode == 'file' then if path then @@ -7236,7 +9000,7 @@ function resolvers.generators.tex(specification) end end action() - if trace_verbose then + if trace_locating then logs.report("fileio","%s files found on %s directories with %s uppercase remappings",n,m,r) end end @@ -7251,11 +9015,48 @@ end -- we join them and split them after the expansion has taken place. This -- is more convenient. +--~ local checkedsplit = string.checkedsplit + +local cache = { } + +local splitter = lpeg.Ct(lpeg.splitat(lpeg.S(os.type == "windows" and ";" or ":;"))) + +local function split_kpse_path(str) -- beware, this can be either a path or a {specification} + local found = cache[str] + if not found then + if str == "" then + found = { } + else + str = gsub(str,"\\","/") +--~ local split = (find(str,";") and checkedsplit(str,";")) or checkedsplit(str,io.pathseparator) +local split = lpegmatch(splitter,str) + found = { } + for i=1,#split do + local s = split[i] + if not find(s,"^{*unset}*") then + found[#found+1] = s + end + end + if trace_expansions then + logs.report("fileio","splitting path specification '%s'",str) + for k=1,#found do + logs.report("fileio","% 4i: %s",k,found[k]) + end + end + cache[str] = found + end + end + return found +end + +resolvers.split_kpse_path = split_kpse_path + function resolvers.splitconfig() - for i,c in ipairs(instance) do - for k,v in pairs(c) do + for i=1,#instance do + local c = instance[i] + for k,v in next, c do if type(v) == 'string' then - local t = file.split_path(v) + local t = split_kpse_path(v) if #t > 1 then c[k] = t end @@ -7265,21 +9066,25 @@ function resolvers.splitconfig() end function resolvers.joinconfig() - for i,c in ipairs(instance.order) do - for k,v in pairs(c) do -- ipairs? + local order = instance.order + for i=1,#order do + local c = order[i] + for k,v in next, c do -- indexed? if type(v) == 'table' then c[k] = file.join_path(v) end end end end + function resolvers.split_path(str) if type(str) == 'table' then return str else - return file.split_path(str) + return split_kpse_path(str) end end + function resolvers.join_path(str) if type(str) == 'table' then return file.join_path(str) @@ -7291,8 +9096,9 @@ end function resolvers.splitexpansions() local ie = instance.expansions for k,v in next, ie do - local t, h = { }, { } - for _,vv in ipairs(file.split_path(v)) do + local t, h, p = { }, { }, split_kpse_path(v) + for kk=1,#p do + local vv = p[kk] if vv ~= "" and not h[vv] then t[#t+1] = vv h[vv] = true @@ -7339,11 +9145,15 @@ function resolvers.serialize(files) end t[#t+1] = "return {" if instance.sortdata then - for _, k in pairs(sortedkeys(files)) do -- ipairs + local sortedfiles = sortedkeys(files) + for i=1,#sortedfiles do + local k = sortedfiles[i] local fk = files[k] if type(fk) == 'table' then t[#t+1] = "\t['" .. k .. "']={" - for _, kk in pairs(sortedkeys(fk)) do -- ipairs + local sortedfk = sortedkeys(fk) + for j=1,#sortedfk do + local kk = sortedfk[j] t[#t+1] = dump(kk,fk[kk],"\t\t") end t[#t+1] = "\t}," @@ -7368,12 +9178,18 @@ function resolvers.serialize(files) return concat(t,"\n") end +local data_state = { } + +function resolvers.data_state() + return data_state or { } +end + function resolvers.save_data(dataname, makename) -- untested without cache overload for cachename, files in next, instance[dataname] do local name = (makename or file.join)(cachename,dataname) local luaname, lucname = name .. ".lua", name .. ".luc" - if trace_verbose then - logs.report("fileio","preparing %s for %s",dataname,cachename) + if trace_locating then + logs.report("fileio","preparing '%s' for '%s'",dataname,cachename) end for k, v in next, files do if type(v) == "table" and #v == 1 then @@ -7387,24 +9203,25 @@ function resolvers.save_data(dataname, makename) -- untested without cache overl date = os.date("%Y-%m-%d"), time = os.date("%H:%M:%S"), content = files, + uuid = os.uuid(), } local ok = io.savedata(luaname,resolvers.serialize(data)) if ok then - if trace_verbose then - logs.report("fileio","%s saved in %s",dataname,luaname) + if trace_locating then + logs.report("fileio","'%s' saved in '%s'",dataname,luaname) end if utils.lua.compile(luaname,lucname,false,true) then -- no cleanup but strip - if trace_verbose then - logs.report("fileio","%s compiled to %s",dataname,lucname) + if trace_locating then + logs.report("fileio","'%s' compiled to '%s'",dataname,lucname) end else - if trace_verbose then - logs.report("fileio","compiling failed for %s, deleting file %s",dataname,lucname) + if trace_locating then + logs.report("fileio","compiling failed for '%s', deleting file '%s'",dataname,lucname) end os.remove(lucname) end - elseif trace_verbose then - logs.report("fileio","unable to save %s in %s (access error)",dataname,luaname) + elseif trace_locating then + logs.report("fileio","unable to save '%s' in '%s' (access error)",dataname,luaname) end end end @@ -7416,19 +9233,20 @@ function resolvers.load_data(pathname,dataname,filename,makename) -- untested wi if blob then local data = blob() if data and data.content and data.type == dataname and data.version == resolvers.cacheversion then - if trace_verbose then - logs.report("fileio","loading %s for %s from %s",dataname,pathname,filename) + data_state[#data_state+1] = data.uuid + if trace_locating then + logs.report("fileio","loading '%s' for '%s' from '%s'",dataname,pathname,filename) end instance[dataname][pathname] = data.content else - if trace_verbose then - logs.report("fileio","skipping %s for %s from %s",dataname,pathname,filename) + if trace_locating then + logs.report("fileio","skipping '%s' for '%s' from '%s'",dataname,pathname,filename) end instance[dataname][pathname] = { } instance.loaderror = true end - elseif trace_verbose then - logs.report("fileio","skipping %s for %s from %s",dataname,pathname,filename) + elseif trace_locating then + logs.report("fileio","skipping '%s' for '%s' from '%s'",dataname,pathname,filename) end end @@ -7447,15 +9265,17 @@ function resolvers.resetconfig() end function resolvers.loadnewconfig() - for _, cnf in ipairs(instance.luafiles) do + local luafiles = instance.luafiles + for i=1,#luafiles do + local cnf = luafiles[i] local pathname = file.dirname(cnf) local filename = file.join(pathname,resolvers.luaname) local blob = loadfile(filename) if blob then local data = blob() if data then - if trace_verbose then - logs.report("fileio","loading configuration file %s",filename) + if trace_locating then + logs.report("fileio","loading configuration file '%s'",filename) end if true then -- flatten to variable.progname @@ -7476,14 +9296,14 @@ function resolvers.loadnewconfig() instance['setup'][pathname] = data end else - if trace_verbose then - logs.report("fileio","skipping configuration file %s",filename) + if trace_locating then + logs.report("fileio","skipping configuration file '%s'",filename) end instance['setup'][pathname] = { } instance.loaderror = true end - elseif trace_verbose then - logs.report("fileio","skipping configuration file %s",filename) + elseif trace_locating then + logs.report("fileio","skipping configuration file '%s'",filename) end instance.order[#instance.order+1] = instance.setup[pathname] if instance.loaderror then break end @@ -7492,7 +9312,9 @@ end function resolvers.loadoldconfig() if not instance.renewcache then - for _, cnf in ipairs(instance.cnffiles) do + local cnffiles = instance.cnffiles + for i=1,#cnffiles do + local cnf = cnffiles[i] local dname = file.dirname(cnf) resolvers.load_data(dname,'configuration') instance.order[#instance.order+1] = instance.configuration[dname] @@ -7682,7 +9504,7 @@ end function resolvers.expanded_path_list(str) if not str then - return ep or { } + return ep or { } -- ep ? elseif instance.savelists then -- engine+progname hash str = gsub(str,"%$","") @@ -7700,9 +9522,9 @@ end function resolvers.expanded_path_list_from_var(str) -- brrr local tmp = resolvers.var_of_format_or_suffix(gsub(str,"%$","")) if tmp ~= "" then - return resolvers.expanded_path_list(str) - else return resolvers.expanded_path_list(tmp) + else + return resolvers.expanded_path_list(str) end end @@ -7749,9 +9571,9 @@ function resolvers.isreadable.file(name) local readable = lfs.isfile(name) -- brrr if trace_detail then if readable then - logs.report("fileio","+ readable: %s",name) + logs.report("fileio","file '%s' is readable",name) else - logs.report("fileio","- readable: %s", name) + logs.report("fileio","file '%s' is not readable", name) end end return readable @@ -7767,7 +9589,7 @@ local function collect_files(names) for k=1,#names do local fname = names[k] if trace_detail then - logs.report("fileio","? blobpath asked: %s",fname) + logs.report("fileio","checking name '%s'",fname) end local bname = file.basename(fname) local dname = file.dirname(fname) @@ -7783,7 +9605,7 @@ local function collect_files(names) local files = blobpath and instance.files[blobpath] if files then if trace_detail then - logs.report("fileio",'? blobpath do: %s (%s)',blobpath,bname) + logs.report("fileio","deep checking '%s' (%s)",blobpath,bname) end local blobfile = files[bname] if not blobfile then @@ -7817,7 +9639,7 @@ local function collect_files(names) end end elseif trace_locating then - logs.report("fileio",'! blobpath no: %s (%s)',blobpath,bname) + logs.report("fileio","no match in '%s' (%s)",blobpath,bname) end end end @@ -7867,14 +9689,13 @@ end local function collect_instance_files(filename,collected) -- todo : plugin (scanners, checkers etc) local result = collected or { } local stamp = nil - filename = file.collapse_path(filename) -- elsewhere - filename = file.collapse_path(gsub(filename,"\\","/")) -- elsewhere + filename = file.collapse_path(filename) -- speed up / beware: format problem if instance.remember then stamp = filename .. "--" .. instance.engine .. "--" .. instance.progname .. "--" .. instance.format if instance.found[stamp] then if trace_locating then - logs.report("fileio",'! remembered: %s',filename) + logs.report("fileio","remembering file '%s'",filename) end return instance.found[stamp] end @@ -7882,7 +9703,7 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan if not dangerous[instance.format or "?"] then if resolvers.isreadable.file(filename) then if trace_detail then - logs.report("fileio",'= found directly: %s',filename) + logs.report("fileio","file '%s' found directly",filename) end instance.found[stamp] = { filename } return { filename } @@ -7890,13 +9711,13 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan end if find(filename,'%*') then if trace_locating then - logs.report("fileio",'! wildcard: %s', filename) + logs.report("fileio","checking wildcard '%s'", filename) end result = resolvers.find_wildcard_files(filename) elseif file.is_qualified_path(filename) then if resolvers.isreadable.file(filename) then if trace_locating then - logs.report("fileio",'! qualified: %s', filename) + logs.report("fileio","qualified name '%s'", filename) end result = { filename } else @@ -7906,7 +9727,7 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan forcedname = filename .. ".tex" if resolvers.isreadable.file(forcedname) then if trace_locating then - logs.report("fileio",'! no suffix, forcing standard filetype: tex') + logs.report("fileio","no suffix, forcing standard filetype 'tex'") end result, ok = { forcedname }, true end @@ -7916,7 +9737,7 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan forcedname = filename .. "." .. s if resolvers.isreadable.file(forcedname) then if trace_locating then - logs.report("fileio",'! no suffix, forcing format filetype: %s', s) + logs.report("fileio","no suffix, forcing format filetype '%s'", s) end result, ok = { forcedname }, true break @@ -7928,7 +9749,7 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan -- try to find in tree (no suffix manipulation), here we search for the -- matching last part of the name local basename = file.basename(filename) - local pattern = (filename .. "$"):gsub("([%.%-])","%%%1") + local pattern = gsub(filename .. "$","([%.%-])","%%%1") local savedformat = instance.format local format = savedformat or "" if format == "" then @@ -7938,19 +9759,21 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan instance.format = "othertextfiles" -- kind of everything, maybe texinput is better end -- - local resolved = collect_instance_files(basename) - if #result == 0 then - local lowered = lower(basename) - if filename ~= lowered then - resolved = collect_instance_files(lowered) + if basename ~= filename then + local resolved = collect_instance_files(basename) + if #result == 0 then + local lowered = lower(basename) + if filename ~= lowered then + resolved = collect_instance_files(lowered) + end end - end - resolvers.format = savedformat - -- - for r=1,#resolved do - local rr = resolved[r] - if rr:find(pattern) then - result[#result+1], ok = rr, true + resolvers.format = savedformat + -- + for r=1,#resolved do + local rr = resolved[r] + if find(rr,pattern) then + result[#result+1], ok = rr, true + end end end -- a real wildcard: @@ -7959,14 +9782,14 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan -- local filelist = collect_files({basename}) -- for f=1,#filelist do -- local ff = filelist[f][3] or "" - -- if ff:find(pattern) then + -- if find(ff,pattern) then -- result[#result+1], ok = ff, true -- end -- end -- end end if not ok and trace_locating then - logs.report("fileio",'? qualified: %s', filename) + logs.report("fileio","qualified name '%s'", filename) end end else @@ -7985,12 +9808,12 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan wantedfiles[#wantedfiles+1] = forcedname filetype = resolvers.format_of_suffix(forcedname) if trace_locating then - logs.report("fileio",'! forcing filetype: %s',filetype) + logs.report("fileio","forcing filetype '%s'",filetype) end else filetype = resolvers.format_of_suffix(filename) if trace_locating then - logs.report("fileio",'! using suffix based filetype: %s',filetype) + logs.report("fileio","using suffix based filetype '%s'",filetype) end end else @@ -8002,7 +9825,7 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan end filetype = instance.format if trace_locating then - logs.report("fileio",'! using given filetype: %s',filetype) + logs.report("fileio","using given filetype '%s'",filetype) end end local typespec = resolvers.variable_of_format(filetype) @@ -8010,9 +9833,7 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan if not pathlist or #pathlist == 0 then -- no pathlist, access check only / todo == wildcard if trace_detail then - logs.report("fileio",'? filename: %s',filename) - logs.report("fileio",'? filetype: %s',filetype or '?') - logs.report("fileio",'? wanted files: %s',concat(wantedfiles," | ")) + logs.report("fileio","checking filename '%s', filetype '%s', wanted files '%s'",filename, filetype or '?',concat(wantedfiles," | ")) end for k=1,#wantedfiles do local fname = wantedfiles[k] @@ -8033,36 +9854,59 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan else -- list search local filelist = collect_files(wantedfiles) - local doscan, recurse + local dirlist = { } + if filelist then + for i=1,#filelist do + dirlist[i] = file.dirname(filelist[i][2]) .. "/" + end + end if trace_detail then - logs.report("fileio",'? filename: %s',filename) + logs.report("fileio","checking filename '%s'",filename) end -- a bit messy ... esp the doscan setting here + local doscan for k=1,#pathlist do local path = pathlist[k] if find(path,"^!!") then doscan = false else doscan = true end - if find(path,"//$") then recurse = true else recurse = false end local pathname = gsub(path,"^!+", '') done = false -- using file list - if filelist and not (done and not instance.allresults) and recurse then - -- compare list entries with permitted pattern - pathname = gsub(pathname,"([%-%.])","%%%1") -- this also influences - pathname = gsub(pathname,"/+$", '/.*') -- later usage of pathname - pathname = gsub(pathname,"//", '/.-/') -- not ok for /// but harmless - local expr = "^" .. pathname + if filelist then + local expression + -- compare list entries with permitted pattern -- /xx /xx// + if not find(pathname,"/$") then + expression = pathname .. "/" + else + expression = pathname + end + expression = gsub(expression,"([%-%.])","%%%1") -- this also influences + expression = gsub(expression,"//+$", '/.*') -- later usage of pathname + expression = gsub(expression,"//", '/.-/') -- not ok for /// but harmless + expression = "^" .. expression .. "$" + if trace_detail then + logs.report("fileio","using pattern '%s' for path '%s'",expression,pathname) + end for k=1,#filelist do local fl = filelist[k] local f = fl[2] - if find(f,expr) then - if trace_detail then - logs.report("fileio",'= found in hash: %s',f) - end + local d = dirlist[k] + if find(d,expression) then --- todo, test for readable result[#result+1] = fl[3] resolvers.register_in_trees(f) -- for tracing used files done = true - if not instance.allresults then break end + if instance.allresults then + if trace_detail then + logs.report("fileio","match in hash for file '%s' on path '%s', continue scanning",f,d) + end + else + if trace_detail then + logs.report("fileio","match in hash for file '%s' on path '%s', quit scanning",f,d) + end + break + end + elseif trace_detail then + logs.report("fileio","no match in hash for file '%s' on path '%s'",f,d) end end end @@ -8078,7 +9922,7 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan local fname = file.join(ppname,w) if resolvers.isreadable.file(fname) then if trace_detail then - logs.report("fileio",'= found by scanning: %s',fname) + logs.report("fileio","found '%s' by scanning",fname) end result[#result+1] = fname done = true @@ -8141,7 +9985,7 @@ function resolvers.find_given_files(filename) local hashes = instance.hashes for k=1,#hashes do local hash = hashes[k] - local files = instance.files[hash.tag] + local files = instance.files[hash.tag] or { } local blist = files[bname] if not blist then local rname = "remap:"..bname @@ -8251,9 +10095,9 @@ function resolvers.load(option) statistics.starttiming(instance) resolvers.resetconfig() resolvers.identify_cnf() - resolvers.load_lua() + resolvers.load_lua() -- will become the new method resolvers.expand_variables() - resolvers.load_cnf() + resolvers.load_cnf() -- will be skipped when we have a lua file resolvers.expand_variables() if option ~= "nofiles" then resolvers.load_hash() @@ -8265,22 +10109,23 @@ end function resolvers.for_files(command, files, filetype, mustexist) if files and #files > 0 then local function report(str) - if trace_verbose then + if trace_locating then logs.report("fileio",str) -- has already verbose else print(str) end end - if trace_verbose then - report('') + if trace_locating then + report('') -- ? end - for _, file in ipairs(files) do + for f=1,#files do + local file = files[f] local result = command(file,filetype,mustexist) if type(result) == 'string' then report(result) else - for _,v in ipairs(result) do - report(v) + for i=1,#result do + report(result[i]) -- could be unpack end end end @@ -8327,18 +10172,19 @@ end function table.sequenced(t,sep) -- temp here local s = { } - for k, v in pairs(t) do -- pairs? - s[#s+1] = k .. "=" .. v + for k, v in next, t do -- indexed? + s[#s+1] = k .. "=" .. tostring(v) end return concat(s, sep or " | ") end function resolvers.methodhandler(what, filename, filetype) -- ... + filename = file.collapse_path(filename) local specification = (type(filename) == "string" and resolvers.splitmethod(filename)) or filename -- no or { }, let it bomb local scheme = specification.scheme if resolvers[what][scheme] then if trace_locating then - logs.report("fileio",'= handler: %s -> %s -> %s',specification.original,what,table.sequenced(specification)) + logs.report("fileio","handler '%s' -> '%s' -> '%s'",specification.original,what,table.sequenced(specification)) end return resolvers[what][scheme](filename,filetype) -- todo: specification else @@ -8358,8 +10204,9 @@ function resolvers.clean_path(str) end function resolvers.do_with_path(name,func) - for _, v in pairs(resolvers.expanded_path_list(name)) do -- pairs? - func("^"..resolvers.clean_path(v)) + local pathlist = resolvers.expanded_path_list(name) + for i=1,#pathlist do + func("^"..resolvers.clean_path(pathlist[i])) end end @@ -8368,7 +10215,9 @@ function resolvers.do_with_var(name,func) end function resolvers.with_files(pattern,handle) - for _, hash in ipairs(instance.hashes) do + local hashes = instance.hashes + for i=1,#hashes do + local hash = hashes[i] local blobpath = hash.tag local blobtype = hash.type if blobpath then @@ -8383,7 +10232,7 @@ function resolvers.with_files(pattern,handle) if type(v) == "string" then handle(blobtype,blobpath,v,k) else - for _,vv in pairs(v) do -- ipairs? + for _,vv in next, v do -- indexed handle(blobtype,blobpath,vv,k) end end @@ -8395,7 +10244,7 @@ function resolvers.with_files(pattern,handle) end function resolvers.locate_format(name) - local barename, fmtname = name:gsub("%.%a+$",""), "" + local barename, fmtname = gsub(name,"%.%a+$",""), "" if resolvers.usecache then local path = file.join(caches.setpath("formats")) -- maybe platform fmtname = file.join(path,barename..".fmt") or "" @@ -8443,7 +10292,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-tmp'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -8467,7 +10316,7 @@ luatools with a recache feature.</p> local format, lower, gsub = string.format, string.lower, string.gsub -local trace_cache = false trackers.register("resolvers.cache", function(v) trace_cache = v end) +local trace_cache = false trackers.register("resolvers.cache", function(v) trace_cache = v end) -- not used yet caches = caches or { } @@ -8554,7 +10403,8 @@ function caches.setpath(...) caches.path = '.' end caches.path = resolvers.clean_path(caches.path) - if not table.is_empty({...}) then + local dirs = { ... } + if #dirs > 0 then local pth = dir.mkdirs(caches.path,...) return pth end @@ -8600,6 +10450,7 @@ function caches.savedata(filepath,filename,data,raw) if raw then reduce, simplify = false, false end + data.cache_uuid = os.uuid() if caches.direct then file.savedata(tmaname, table.serialize(data,'return',false,true,false)) -- no hex else @@ -8625,7 +10476,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-res'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -8660,6 +10511,14 @@ prefixes.relative = function(str,n) return resolvers.clean_path(str) end +prefixes.auto = function(str) + local fullname = prefixes.relative(str) + if not lfs.isfile(fullname) then + fullname = prefixes.locate(str) + end + return fullname +end + prefixes.locate = function(str) local fullname = resolvers.find_given_file(str) or "" return resolvers.clean_path((fullname ~= "" and fullname) or str) @@ -8683,6 +10542,16 @@ prefixes.full = prefixes.locate prefixes.file = prefixes.filename prefixes.path = prefixes.pathname +function resolvers.allprefixes(separator) + local all = table.sortedkeys(prefixes) + if separator then + for i=1,#all do + all[i] = all[i] .. ":" + end + end + return all +end + local function _resolve_(method,target) if prefixes[method] then return prefixes[method](target) @@ -8693,7 +10562,8 @@ end local function resolve(str) if type(str) == "table" then - for k, v in pairs(str) do -- ipairs + for k=1,#str do + local v = str[k] str[k] = resolve(v) or v end elseif str and str ~= "" then @@ -8706,7 +10576,7 @@ resolvers.resolve = resolve if os.uname then - for k, v in pairs(os.uname()) do + for k, v in next, os.uname() do if not prefixes[k] then prefixes[k] = function() return v end end @@ -8721,7 +10591,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-inp'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -8742,7 +10612,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-out'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -8758,7 +10628,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-con'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -8769,8 +10639,6 @@ local format, lower, gsub = string.format, string.lower, string.gsub local trace_cache = false trackers.register("resolvers.cache", function(v) trace_cache = v end) local trace_containers = false trackers.register("resolvers.containers", function(v) trace_containers = v end) local trace_storage = false trackers.register("resolvers.storage", function(v) trace_storage = v end) -local trace_verbose = false trackers.register("resolvers.verbose", function(v) trace_verbose = v end) -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v trackers.enable("resolvers.verbose") end) --[[ldx-- <p>Once we found ourselves defining similar cache constructs @@ -8834,7 +10702,7 @@ end function containers.is_valid(container, name) if name and name ~= "" then local storage = container.storage[name] - return storage and not table.is_empty(storage) and storage.cache_version == container.version + return storage and storage.cache_version == container.version else return false end @@ -8886,16 +10754,15 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-use'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -local format, lower, gsub = string.format, string.lower, string.gsub +local format, lower, gsub, find = string.format, string.lower, string.gsub, string.find -local trace_verbose = false trackers.register("resolvers.verbose", function(v) trace_verbose = v end) -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v trackers.enable("resolvers.verbose") end) +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) -- since we want to use the cache instead of the tree, we will now -- reimplement the saver. @@ -8939,19 +10806,20 @@ resolvers.automounted = resolvers.automounted or { } function resolvers.automount(usecache) local mountpaths = resolvers.clean_path_list(resolvers.expansion('TEXMFMOUNT')) - if table.is_empty(mountpaths) and usecache then + if (not mountpaths or #mountpaths == 0) and usecache then mountpaths = { caches.setpath("mount") } end - if not table.is_empty(mountpaths) then + if mountpaths and #mountpaths > 0 then statistics.starttiming(resolvers.instance) - for k, root in pairs(mountpaths) do + for k=1,#mountpaths do + local root = mountpaths[k] local f = io.open(root.."/url.tmi") if f then for line in f:lines() do if line then - if line:find("^[%%#%-]") then -- or %W + if find(line,"^[%%#%-]") then -- or %W -- skip - elseif line:find("^zip://") then + elseif find(line,"^zip://") then if trace_locating then logs.report("fileio","mounting %s",line) end @@ -8996,11 +10864,13 @@ function statistics.check_fmt_status(texname) local luv = dofile(luvname) if luv and luv.sourcefile then local sourcehash = md5.hex(io.loaddata(resolvers.find_file(luv.sourcefile)) or "unknown") - if luv.enginebanner and luv.enginebanner ~= enginebanner then - return "engine mismatch" + local luvbanner = luv.enginebanner or "?" + if luvbanner ~= enginebanner then + return string.format("engine mismatch (luv:%s <> bin:%s)",luvbanner,enginebanner) end - if luv.sourcehash and luv.sourcehash ~= sourcehash then - return "source mismatch" + local luvhash = luv.sourcehash or "?" + if luvhash ~= sourcehash then + return string.format("source mismatch (luv:%s <> bin:%s)",luvhash,sourcehash) end else return "invalid status file" @@ -9019,18 +10889,22 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-zip'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -local format, find = string.format, string.find +local format, find, match = string.format, string.find, string.match +local unpack = unpack or table.unpack -local trace_locating, trace_verbose = false, false +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) -trackers.register("resolvers.verbose", function(v) trace_verbose = v end) -trackers.register("resolvers.locating", function(v) trace_locating = v trace_verbose = v end) +-- zip:///oeps.zip?name=bla/bla.tex +-- zip:///oeps.zip?tree=tex/texmf-local +-- zip:///texmf.zip?tree=/tex/texmf +-- zip:///texmf.zip?tree=/tex/texmf-local +-- zip:///texmf-mine.zip?tree=/tex/texmf-projects zip = zip or { } zip.archives = zip.archives or { } @@ -9041,9 +10915,6 @@ local locators, hashers, concatinators = resolvers.locators, resolvers.hashers, local archives = zip.archives --- zip:///oeps.zip?name=bla/bla.tex --- zip:///oeps.zip?tree=tex/texmf-local - local function validzip(str) -- todo: use url splitter if not find(str,"^zip://") then return "zip:///" .. str @@ -9073,26 +10944,22 @@ function zip.closearchive(name) end end --- zip:///texmf.zip?tree=/tex/texmf --- zip:///texmf.zip?tree=/tex/texmf-local --- zip:///texmf-mine.zip?tree=/tex/texmf-projects - function locators.zip(specification) -- where is this used? startup zips (untested) specification = resolvers.splitmethod(specification) local zipfile = specification.path local zfile = zip.openarchive(name) -- tricky, could be in to be initialized tree if trace_locating then if zfile then - logs.report("fileio",'! zip locator, found: %s',specification.original) + logs.report("fileio","zip locator, archive '%s' found",specification.original) else - logs.report("fileio",'? zip locator, not found: %s',specification.original) + logs.report("fileio","zip locator, archive '%s' not found",specification.original) end end end function hashers.zip(tag,name) - if trace_verbose then - logs.report("fileio","loading zip file %s as %s",name,tag) + if trace_locating then + logs.report("fileio","loading zip file '%s' as '%s'",name,tag) end resolvers.usezipfile(format("%s?tree=%s",tag,name)) end @@ -9117,23 +10984,25 @@ function finders.zip(specification,filetype) local zfile = zip.openarchive(specification.path) if zfile then if trace_locating then - logs.report("fileio",'! zip finder, path: %s',specification.path) + logs.report("fileio","zip finder, archive '%s' found",specification.path) end local dfile = zfile:open(q.name) if dfile then dfile = zfile:close() if trace_locating then - logs.report("fileio",'+ zip finder, name: %s',q.name) + logs.report("fileio","zip finder, file '%s' found",q.name) end return specification.original + elseif trace_locating then + logs.report("fileio","zip finder, file '%s' not found",q.name) end elseif trace_locating then - logs.report("fileio",'? zip finder, path %s',specification.path) + logs.report("fileio","zip finder, unknown archive '%s'",specification.path) end end end if trace_locating then - logs.report("fileio",'- zip finder, name: %s',filename) + logs.report("fileio","zip finder, '%s' not found",filename) end return unpack(finders.notfound) end @@ -9146,20 +11015,25 @@ function openers.zip(specification) local zfile = zip.openarchive(zipspecification.path) if zfile then if trace_locating then - logs.report("fileio",'+ zip starter, path: %s',zipspecification.path) + logs.report("fileio","zip opener, archive '%s' opened",zipspecification.path) end local dfile = zfile:open(q.name) if dfile then logs.show_open(specification) + if trace_locating then + logs.report("fileio","zip opener, file '%s' found",q.name) + end return openers.text_opener(specification,dfile,'zip') + elseif trace_locating then + logs.report("fileio","zip opener, file '%s' not found",q.name) end elseif trace_locating then - logs.report("fileio",'- zip starter, path %s',zipspecification.path) + logs.report("fileio","zip opener, unknown archive '%s'",zipspecification.path) end end end if trace_locating then - logs.report("fileio",'- zip opener, name: %s',filename) + logs.report("fileio","zip opener, '%s' not found",filename) end return unpack(openers.notfound) end @@ -9172,25 +11046,27 @@ function loaders.zip(specification) local zfile = zip.openarchive(specification.path) if zfile then if trace_locating then - logs.report("fileio",'+ zip starter, path: %s',specification.path) + logs.report("fileio","zip loader, archive '%s' opened",specification.path) end local dfile = zfile:open(q.name) if dfile then logs.show_load(filename) if trace_locating then - logs.report("fileio",'+ zip loader, name: %s',filename) + logs.report("fileio","zip loader, file '%s' loaded",filename) end local s = dfile:read("*all") dfile:close() return true, s, #s + elseif trace_locating then + logs.report("fileio","zip loader, file '%s' not found",q.name) end elseif trace_locating then - logs.report("fileio",'- zip starter, path: %s',specification.path) + logs.report("fileio","zip loader, unknown archive '%s'",specification.path) end end end if trace_locating then - logs.report("fileio",'- zip loader, name: %s',filename) + logs.report("fileio","zip loader, '%s' not found",filename) end return unpack(openers.notfound) end @@ -9200,21 +11076,15 @@ end function resolvers.usezipfile(zipname) zipname = validzip(zipname) - if trace_locating then - logs.report("fileio",'! zip use, file: %s',zipname) - end local specification = resolvers.splitmethod(zipname) local zipfile = specification.path if zipfile and not zip.registeredfiles[zipname] then local tree = url.query(specification.query).tree or "" - if trace_locating then - logs.report("fileio",'! zip register, file: %s',zipname) - end local z = zip.openarchive(zipfile) if z then local instance = resolvers.instance if trace_locating then - logs.report("fileio","= zipfile, registering: %s",zipname) + logs.report("fileio","zip registering, registering archive '%s'",zipname) end statistics.starttiming(instance) resolvers.prepend_hash('zip',zipname,zipfile) @@ -9223,10 +11093,10 @@ function resolvers.usezipfile(zipname) instance.files[zipname] = resolvers.register_zip_file(z,tree or "") statistics.stoptiming(instance) elseif trace_locating then - logs.report("fileio","? zipfile, unknown: %s",zipname) + logs.report("fileio","zip registering, unknown archive '%s'",zipname) end elseif trace_locating then - logs.report("fileio",'! zip register, no file: %s',zipname) + logs.report("fileio","zip registering, '%s' not found",zipname) end end @@ -9238,11 +11108,11 @@ function resolvers.register_zip_file(z,tree) filter = format("^%s/(.+)/(.-)$",tree) end if trace_locating then - logs.report("fileio",'= zip filter: %s',filter) + logs.report("fileio","zip registering, using filter '%s'",filter) end local register, n = resolvers.register_file, 0 for i in z:files() do - local path, name = i.filename:match(filter) + local path, name = match(i.filename,filter) if path then if name and name ~= '' then register(files, name, path) @@ -9255,7 +11125,7 @@ function resolvers.register_zip_file(z,tree) n = n + 1 end end - logs.report("fileio",'= zip entries: %s',n) + logs.report("fileio","zip registering, %s files registered",n) return files end @@ -9266,12 +11136,14 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-crl'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } +local gsub = string.gsub + curl = curl or { } curl.cached = { } @@ -9280,9 +11152,9 @@ curl.cachepath = caches.definepath("curl") local finders, openers, loaders = resolvers.finders, resolvers.openers, resolvers.loaders function curl.fetch(protocol, name) - local cachename = curl.cachepath() .. "/" .. name:gsub("[^%a%d%.]+","-") --- cachename = cachename:gsub("[\\/]", io.fileseparator) - cachename = cachename:gsub("[\\]", "/") -- cleanup + local cachename = curl.cachepath() .. "/" .. gsub(name,"[^%a%d%.]+","-") +-- cachename = gsub(cachename,"[\\/]", io.fileseparator) + cachename = gsub(cachename,"[\\]", "/") -- cleanup if not curl.cached[name] then if not io.exists(cachename) then curl.cached[name] = cachename @@ -9328,6 +11200,164 @@ end -- of closure do -- create closure to overcome 200 locals limit +if not modules then modules = { } end modules ['data-lua'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- some loading stuff ... we might move this one to slot 2 depending +-- on the developments (the loaders must not trigger kpse); we could +-- of course use a more extensive lib path spec + +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) + +local gsub, insert = string.gsub, table.insert +local unpack = unpack or table.unpack + +local libformats = { 'luatexlibs', 'tex', 'texmfscripts', 'othertextfiles' } -- 'luainputs' +local clibformats = { 'lib' } + +local _path_, libpaths, _cpath_, clibpaths + +function package.libpaths() + if not _path_ or package.path ~= _path_ then + _path_ = package.path + libpaths = file.split_path(_path_,";") + end + return libpaths +end + +function package.clibpaths() + if not _cpath_ or package.cpath ~= _cpath_ then + _cpath_ = package.cpath + clibpaths = file.split_path(_cpath_,";") + end + return clibpaths +end + +local function thepath(...) + local t = { ... } t[#t+1] = "?.lua" + local path = file.join(unpack(t)) + if trace_locating then + logs.report("fileio","! appending '%s' to 'package.path'",path) + end + return path +end + +local p_libpaths, a_libpaths = { }, { } + +function package.append_libpath(...) + insert(a_libpath,thepath(...)) +end + +function package.prepend_libpath(...) + insert(p_libpaths,1,thepath(...)) +end + +-- beware, we need to return a loadfile result ! + +local function loaded(libpaths,name,simple) + for i=1,#libpaths do -- package.path, might become option + local libpath = libpaths[i] + local resolved = gsub(libpath,"%?",simple) + if trace_locating then -- more detail + logs.report("fileio","! checking for '%s' on 'package.path': '%s' => '%s'",simple,libpath,resolved) + end + if resolvers.isreadable.file(resolved) then + if trace_locating then + logs.report("fileio","! lib '%s' located via 'package.path': '%s'",name,resolved) + end + return loadfile(resolved) + end + end +end + + +package.loaders[2] = function(name) -- was [#package.loaders+1] + if trace_locating then -- mode detail + logs.report("fileio","! locating '%s'",name) + end + for i=1,#libformats do + local format = libformats[i] + local resolved = resolvers.find_file(name,format) or "" + if trace_locating then -- mode detail + logs.report("fileio","! checking for '%s' using 'libformat path': '%s'",name,format) + end + if resolved ~= "" then + if trace_locating then + logs.report("fileio","! lib '%s' located via environment: '%s'",name,resolved) + end + return loadfile(resolved) + end + end + -- libpaths + local libpaths, clibpaths = package.libpaths(), package.clibpaths() + local simple = gsub(name,"%.lua$","") + local simple = gsub(simple,"%.","/") + local resolved = loaded(p_libpaths,name,simple) or loaded(libpaths,name,simple) or loaded(a_libpaths,name,simple) + if resolved then + return resolved + end + -- + local libname = file.addsuffix(simple,os.libsuffix) + for i=1,#clibformats do + -- better have a dedicated loop + local format = clibformats[i] + local paths = resolvers.expanded_path_list_from_var(format) + for p=1,#paths do + local path = paths[p] + local resolved = file.join(path,libname) + if trace_locating then -- mode detail + logs.report("fileio","! checking for '%s' using 'clibformat path': '%s'",libname,path) + end + if resolvers.isreadable.file(resolved) then + if trace_locating then + logs.report("fileio","! lib '%s' located via 'clibformat': '%s'",libname,resolved) + end + return package.loadlib(resolved,name) + end + end + end + for i=1,#clibpaths do -- package.path, might become option + local libpath = clibpaths[i] + local resolved = gsub(libpath,"?",simple) + if trace_locating then -- more detail + logs.report("fileio","! checking for '%s' on 'package.cpath': '%s'",simple,libpath) + end + if resolvers.isreadable.file(resolved) then + if trace_locating then + logs.report("fileio","! lib '%s' located via 'package.cpath': '%s'",name,resolved) + end + return package.loadlib(resolved,name) + end + end + -- just in case the distribution is messed up + if trace_loading then -- more detail + logs.report("fileio","! checking for '%s' using 'luatexlibs': '%s'",name) + end + local resolved = resolvers.find_file(file.basename(name),'luatexlibs') or "" + if resolved ~= "" then + if trace_locating then + logs.report("fileio","! lib '%s' located by basename via environment: '%s'",name,resolved) + end + return loadfile(resolved) + end + if trace_locating then + logs.report("fileio",'? unable to locate lib: %s',name) + end +-- return "unable to locate " .. name +end + +resolvers.loadlualib = require + + +end -- of closure + +do -- create closure to overcome 200 locals limit + if not modules then modules = { } end modules ['luat-kps'] = { version = 1.001, comment = "companion to luatools.lua", @@ -9437,7 +11467,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-aux'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -9445,47 +11475,47 @@ if not modules then modules = { } end modules ['data-aux'] = { local find = string.find -local trace_verbose = false trackers.register("resolvers.verbose", function(v) trace_verbose = v end) +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) function resolvers.update_script(oldname,newname) -- oldname -> own.name, not per se a suffix local scriptpath = "scripts/context/lua" newname = file.addsuffix(newname,"lua") local oldscript = resolvers.clean_path(oldname) - if trace_verbose then + if trace_locating then logs.report("fileio","to be replaced old script %s", oldscript) end local newscripts = resolvers.find_files(newname) or { } if #newscripts == 0 then - if trace_verbose then + if trace_locating then logs.report("fileio","unable to locate new script") end else for i=1,#newscripts do local newscript = resolvers.clean_path(newscripts[i]) - if trace_verbose then + if trace_locating then logs.report("fileio","checking new script %s", newscript) end if oldscript == newscript then - if trace_verbose then + if trace_locating then logs.report("fileio","old and new script are the same") end elseif not find(newscript,scriptpath) then - if trace_verbose then + if trace_locating then logs.report("fileio","new script should come from %s",scriptpath) end elseif not (find(oldscript,file.removesuffix(newname).."$") or find(oldscript,newname.."$")) then - if trace_verbose then + if trace_locating then logs.report("fileio","invalid new script name") end else local newdata = io.loaddata(newscript) if newdata then - if trace_verbose then + if trace_locating then logs.report("fileio","old script content replaced by new content") end io.savedata(oldscript,newdata) break - elseif trace_verbose then + elseif trace_locating then logs.report("fileio","unable to load new script") end end @@ -9500,25 +11530,28 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-tmf'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } +local find, gsub, match = string.find, string.gsub, string.match +local getenv, setenv = os.getenv, os.setenv + -- loads *.tmf files in minimal tree roots (to be optimized and documented) function resolvers.check_environment(tree) logs.simpleline() - os.setenv('TMP', os.getenv('TMP') or os.getenv('TEMP') or os.getenv('TMPDIR') or os.getenv('HOME')) - os.setenv('TEXOS', os.getenv('TEXOS') or ("texmf-" .. os.currentplatform())) - os.setenv('TEXPATH', (tree or "tex"):gsub("\/+$",'')) - os.setenv('TEXMFOS', os.getenv('TEXPATH') .. "/" .. os.getenv('TEXOS')) + setenv('TMP', getenv('TMP') or getenv('TEMP') or getenv('TMPDIR') or getenv('HOME')) + setenv('TEXOS', getenv('TEXOS') or ("texmf-" .. os.platform)) + setenv('TEXPATH', gsub(tree or "tex","\/+$",'')) + setenv('TEXMFOS', getenv('TEXPATH') .. "/" .. getenv('TEXOS')) logs.simpleline() - logs.simple("preset : TEXPATH => %s", os.getenv('TEXPATH')) - logs.simple("preset : TEXOS => %s", os.getenv('TEXOS')) - logs.simple("preset : TEXMFOS => %s", os.getenv('TEXMFOS')) - logs.simple("preset : TMP => %s", os.getenv('TMP')) + logs.simple("preset : TEXPATH => %s", getenv('TEXPATH')) + logs.simple("preset : TEXOS => %s", getenv('TEXOS')) + logs.simple("preset : TEXMFOS => %s", getenv('TEXMFOS')) + logs.simple("preset : TMP => %s", getenv('TMP')) logs.simple('') end @@ -9526,27 +11559,27 @@ function resolvers.load_environment(name) -- todo: key=value as well as lua local f = io.open(name) if f then for line in f:lines() do - if line:find("^[%%%#]") then + if find(line,"^[%%%#]") then -- skip comment else - local key, how, value = line:match("^(.-)%s*([<=>%?]+)%s*(.*)%s*$") + local key, how, value = match(line,"^(.-)%s*([<=>%?]+)%s*(.*)%s*$") if how then - value = value:gsub("%%(.-)%%", function(v) return os.getenv(v) or "" end) + value = gsub(value,"%%(.-)%%", function(v) return getenv(v) or "" end) if how == "=" or how == "<<" then - os.setenv(key,value) + setenv(key,value) elseif how == "?" or how == "??" then - os.setenv(key,os.getenv(key) or value) + setenv(key,getenv(key) or value) elseif how == "<" or how == "+=" then - if os.getenv(key) then - os.setenv(key,os.getenv(key) .. io.fileseparator .. value) + if getenv(key) then + setenv(key,getenv(key) .. io.fileseparator .. value) else - os.setenv(key,value) + setenv(key,value) end elseif how == ">" or how == "=+" then - if os.getenv(key) then - os.setenv(key,value .. io.pathseparator .. os.getenv(key)) + if getenv(key) then + setenv(key,value .. io.pathseparator .. getenv(key)) else - os.setenv(key,value) + setenv(key,value) end end end @@ -9585,6 +11618,9 @@ if not modules then modules = { } end modules ['luat-sta'] = { -- this code is used in the updater +local gmatch, match = string.gmatch, string.match +local type = type + states = states or { } states.data = states.data or { } states.hash = states.hash or { } @@ -9613,13 +11649,17 @@ function states.set_by_tag(tag,key,value,default,persistent) if d then if type(d) == "table" then local dkey, hkey = key, key - local pre, post = key:match("(.+)%.([^%.]+)$") + local pre, post = match(key,"(.+)%.([^%.]+)$") if pre and post then - for k in pre:gmatch("[^%.]+") do + for k in gmatch(pre,"[^%.]+") do local dk = d[k] if not dk then dk = { } d[k] = dk + elseif type(dk) == "string" then + -- invalid table, unable to upgrade structure + -- hope for the best or delete the state file + break end d = dk end @@ -9647,7 +11687,7 @@ function states.get_by_tag(tag,key,default) else local d = states.data[tag] if d then - for k in key:gmatch("[^%.]+") do + for k in gmatch(key,"[^%.]+") do local dk = d[k] if dk then d = dk @@ -9782,6 +11822,7 @@ own.libs = { -- todo: check which ones are really needed 'l-os.lua', 'l-file.lua', 'l-md5.lua', + 'l-url.lua', 'l-dir.lua', 'l-boolean.lua', 'l-math.lua', @@ -9790,11 +11831,13 @@ own.libs = { -- todo: check which ones are really needed 'l-utils.lua', 'l-aux.lua', -- 'l-xml.lua', + 'trac-tra.lua', 'lxml-tab.lua', - 'lxml-pth.lua', - 'lxml-ent.lua', + 'lxml-lpt.lua', +-- 'lxml-ent.lua', 'lxml-mis.lua', - 'trac-tra.lua', + 'lxml-aux.lua', + 'lxml-xml.lua', 'luat-env.lua', 'trac-inf.lua', 'trac-log.lua', @@ -9809,7 +11852,7 @@ own.libs = { -- todo: check which ones are really needed -- 'data-bin.lua', 'data-zip.lua', 'data-crl.lua', --- 'data-lua.lua', + 'data-lua.lua', 'data-kps.lua', -- so that we can replace kpsewhich 'data-aux.lua', -- updater 'data-tmf.lua', -- tree files @@ -9827,7 +11870,8 @@ end -- End of hack. -own.name = (environment and environment.ownname) or arg[0] or 'luatools.lua' +own.name = (environment and environment.ownname) or arg[0] or 'luatools.lua' + own.path = string.match(own.name,"^(.+)[\\/].-$") or "." own.list = { '.' } @@ -9865,18 +11909,25 @@ if not resolvers then os.exit() end -logs.setprogram('MTXrun',"TDS Runner Tool 1.22",environment.arguments["verbose"] or false) +logs.setprogram('MTXrun',"TDS Runner Tool 1.24",environment.arguments["verbose"] or false) local instance = resolvers.reset() +local trackspec = environment.argument("trackers") or environment.argument("track") + +if trackspec then + trackers.enable(trackspec) +end + runners = runners or { } -- global messages = messages or { } messages.help = [[ ---script run an mtx script (--noquotes) ---execute run a script or program (--noquotes) +--script run an mtx script (lua prefered method) (--noquotes), no script gives list +--execute run a script or program (texmfstart method) (--noquotes) --resolve resolve prefixed arguments --ctxlua run internally (using preloaded libs) +--internal run script using built in libraries (same as --ctxlua) --locate locate given filename --autotree use texmf tree cf. env 'texmfstart_tree' or 'texmfstarttree' @@ -9893,16 +11944,20 @@ messages.help = [[ --unix create unix (linux) stubs --verbose give a bit more info +--trackers=list enable given trackers --engine=str target engine --progname=str format or backend --edit launch editor with found file --launch (--all) launch files like manuals, assumes os support ---intern run script using built in libraries +--timedrun run a script an time its run +--autogenerate regenerate databases if needed (handy when used to run context in an editor) + +--usekpse use kpse as fallback (when no mkiv and cache installed, often slower) +--forcekpse force using kpse (handy when no mkiv and cache installed but less functionality) ---usekpse use kpse as fallback (when no mkiv and cache installed, often slower) ---forcekpse force using kpse (handy when no mkiv and cache installed but less functionality) +--prefixes show supported prefixes ]] runners.applications = { @@ -9918,20 +11973,17 @@ runners.suffixes = { } runners.registered = { - texexec = { 'texexec.rb', true }, -- context mkii runner (only tool not to be luafied) + texexec = { 'texexec.rb', false }, -- context mkii runner (only tool not to be luafied) texutil = { 'texutil.rb', true }, -- old perl based index sorter for mkii (old versions need it) texfont = { 'texfont.pl', true }, -- perl script that makes mkii font metric files texfind = { 'texfind.pl', false }, -- perltk based tex searching tool, mostly used at pragma texshow = { 'texshow.pl', false }, -- perltk based context help system, will be luafied - -- texwork = { \texwork.pl', false }, -- perltk based editing environment, only used at pragma - + -- texwork = { 'texwork.pl', false }, -- perltk based editing environment, only used at pragma makempy = { 'makempy.pl', true }, mptopdf = { 'mptopdf.pl', true }, pstopdf = { 'pstopdf.rb', true }, -- converts ps (and some more) images, does some cleaning (replaced) - -- examplex = { 'examplex.rb', false }, concheck = { 'concheck.rb', false }, - runtools = { 'runtools.rb', true }, textools = { 'textools.rb', true }, tmftools = { 'tmftools.rb', true }, @@ -9943,7 +11995,6 @@ runners.registered = { xmltools = { 'xmltools.rb', true }, -- luatools = { 'luatools.lua', true }, mtxtools = { 'mtxtools.rb', true }, - pdftrimwhite = { 'pdftrimwhite.pl', false } } @@ -9952,6 +12003,13 @@ runners.launchers = { unix = { } } +-- like runners.libpath("framework"): looks on script's subpath + +function runners.libpath(...) + package.prepend_libpath(file.dirname(environment.ownscript),...) + package.prepend_libpath(file.dirname(environment.ownname) ,...) +end + function runners.prepare() local checkname = environment.argument("ifchanged") if checkname and checkname ~= "" then @@ -9996,7 +12054,7 @@ function runners.prepare() return "run" end -function runners.execute_script(fullname,internal) +function runners.execute_script(fullname,internal,nosplit) local noquote = environment.argument("noquotes") if fullname and fullname ~= "" then local state = runners.prepare() @@ -10036,17 +12094,20 @@ function runners.execute_script(fullname,internal) end end if result and result ~= "" then - local before, after = environment.split_arguments(fullname) -- already done - environment.arguments_before, environment.arguments_after = before, after + if not no_split then + local before, after = environment.split_arguments(fullname) -- already done + environment.arguments_before, environment.arguments_after = before, after + end if internal then - arg = { } for _,v in pairs(after) do arg[#arg+1] = v end + arg = { } for _,v in pairs(environment.arguments_after) do arg[#arg+1] = v end + environment.ownscript = result dofile(result) else local binary = runners.applications[file.extname(result)] if binary and binary ~= "" then result = binary .. " " .. result end - local command = result .. " " .. environment.reconstruct_commandline(after,noquote) + local command = result .. " " .. environment.reconstruct_commandline(environment.arguments_after,noquote) if logs.verbose then logs.simpleline() logs.simple("executing: %s",command) @@ -10054,8 +12115,24 @@ function runners.execute_script(fullname,internal) logs.simpleline() io.flush() end - local code = os.exec(command) -- maybe spawn - return code == 0 + -- no os.exec because otherwise we get the wrong return value + local code = os.execute(command) -- maybe spawn + if code == 0 then + return true + else + if binary then + binary = file.addsuffix(binary,os.binsuffix) + for p in string.gmatch(os.getenv("PATH"),"[^"..io.pathseparator.."]+") do + if lfs.isfile(file.join(p,binary)) then + return false + end + end + logs.simpleline() + logs.simple("This script needs '%s' which seems not to be installed.",binary) + logs.simpleline() + end + return false + end end end end @@ -10088,7 +12165,7 @@ function runners.execute_program(fullname) return false end --- the --usekpse flag will fallback on kpse +-- the --usekpse flag will fallback on kpse (hm, we can better update mtx-stubs) local windows_stub = '@echo off\013\010setlocal\013\010set ownpath=%%~dp0%%\013\010texlua "%%ownpath%%mtxrun.lua" --usekpse --execute %s %%*\013\010endlocal\013\010' local unix_stub = '#!/bin/sh\010mtxrun --usekpse --execute %s \"$@\"\010' @@ -10143,7 +12220,7 @@ function runners.locate_file(filename) end function runners.locate_platform() - runners.report_location(os.currentplatform()) + runners.report_location(os.platform) end function runners.report_location(result) @@ -10176,7 +12253,8 @@ end function runners.save_script_session(filename, list) local t = { } - for _, key in ipairs(list) do + for i=1,#list do + local key = list[i] t[key] = environment.arguments[key] end io.savedata(filename,table.serialize(t,true)) @@ -10265,20 +12343,22 @@ function runners.find_mtx_script(filename) if fullname and fullname ~= "" then return fullname end + -- mtx- prefix checking + local mtxprefix = (filename:find("^mtx%-") and "") or "mtx-" -- context namespace, mtx-<filename> - fullname = "mtx-" .. filename + fullname = mtxprefix .. filename fullname = found(fullname) or resolvers.find_file(fullname) if fullname and fullname ~= "" then return fullname end -- context namespace, mtx-<filename>s - fullname = "mtx-" .. basename .. "s" .. "." .. suffix + fullname = mtxprefix .. basename .. "s" .. "." .. suffix fullname = found(fullname) or resolvers.find_file(fullname) if fullname and fullname ~= "" then return fullname end -- context namespace, mtx-<filename minus trailing s> - fullname = "mtx-" .. basename:gsub("s$","") .. "." .. suffix + fullname = mtxprefix .. basename:gsub("s$","") .. "." .. suffix fullname = found(fullname) or resolvers.find_file(fullname) if fullname and fullname ~= "" then return fullname @@ -10288,9 +12368,17 @@ function runners.find_mtx_script(filename) return fullname end -function runners.execute_ctx_script(filename,arguments) +function runners.execute_ctx_script(filename) + local arguments = environment.arguments_after local fullname = runners.find_mtx_script(filename) or "" - -- retyr after generate but only if --autogenerate + if file.extname(fullname) == "cld" then + -- handy in editors where we force --autopdf + logs.simple("running cld script: %s",filename) + table.insert(arguments,1,fullname) + table.insert(arguments,"--autopdf") + fullname = runners.find_mtx_script("context") or "" + end + -- retry after generate but only if --autogenerate if fullname == "" and environment.argument("autogenerate") then -- might become the default instance.renewcache = true logs.setverbose(true) @@ -10319,32 +12407,51 @@ function runners.execute_ctx_script(filename,arguments) if logs.verbose then logs.simple("using script: %s\n",fullname) end + environment.ownscript = fullname dofile(fullname) local savename = environment.arguments['save'] - if savename and runners.save_list and not table.is_empty(runners.save_list or { }) then - if type(savename) ~= "string" then savename = file.basename(fullname) end - savename = file.replacesuffix(savename,"cfg") - runners.save_script_session(savename, runners.save_list) + if savename then + local save_list = runners.save_list + if save_list and next(save_list) then + if type(savename) ~= "string" then savename = file.basename(fullname) end + savename = file.replacesuffix(savename,"cfg") + runners.save_script_session(savename,save_list) + end end return true end else - logs.setverbose(true) - if filename == "" then - logs.simple("unknown script, no name given") + -- logs.setverbose(true) + if filename == "" or filename == "help" then local context = resolvers.find_file("mtx-context.lua") + logs.setverbose(true) if context ~= "" then local result = dir.glob((string.gsub(context,"mtx%-context","mtx-*"))) -- () needed local valid = { } - for _, scriptname in ipairs(result) do - scriptname = string.match(scriptname,".*mtx%-([^%-]-)%.lua") - if scriptname then - valid[#valid+1] = scriptname + table.sort(result) + for i=1,#result do + local scriptname = result[i] + local scriptbase = string.match(scriptname,".*mtx%-([^%-]-)%.lua") + if scriptbase then + local data = io.loaddata(scriptname) + local banner, version = string.match(data,"[\n\r]logs%.extendbanner%s*%(%s*[\"\']([^\n\r]+)%s*(%d+%.%d+)") + if banner then + valid[#valid+1] = { scriptbase, version, banner } + end end end if #valid > 0 then - logs.simple("known scripts: %s",table.concat(valid,", ")) + logs.reportbanner() + logs.reportline() + logs.simple("no script name given, known scripts:") + logs.simple() + for k=1,#valid do + local v = valid[k] + logs.simple("%-12s %4s %s",v[1],v[2],v[3]) + end end + else + logs.simple("no script name given") end else filename = file.addsuffix(filename,"lua") @@ -10358,6 +12465,12 @@ function runners.execute_ctx_script(filename,arguments) end end +function runners.prefixes() + logs.reportbanner() + logs.reportline() + logs.simple(table.concat(resolvers.allprefixes(true)," ")) +end + function runners.timedrun(filename) -- just for me if filename and filename ~= "" then runners.timed(function() os.execute(filename) end) @@ -10385,7 +12498,9 @@ instance.lsrmode = environment.argument("lsr") or false -- maybe the unset has to go to this level -if environment.argument("usekpse") or environment.argument("forcekpse") then +local is_mkii_stub = runners.registered[file.removesuffix(file.basename(filename))] + +if environment.argument("usekpse") or environment.argument("forcekpse") or is_mkii_stub then os.setenv("engine","") os.setenv("progname","") @@ -10420,7 +12535,7 @@ if environment.argument("usekpse") or environment.argument("forcekpse") then return (kpse_initialized():show_path(name)) or "" end - elseif environment.argument("usekpse") then + elseif environment.argument("usekpse") or is_mkii_stub then resolvers.load() @@ -10449,7 +12564,6 @@ else end - if environment.argument("selfmerge") then -- embed used libraries utils.merger.selfmerge(own.name,own.libs,own.list) @@ -10462,9 +12576,14 @@ elseif environment.argument("selfupdate") then elseif environment.argument("ctxlua") or environment.argument("internal") then -- run a script by loading it (using libs) ok = runners.execute_script(filename,true) -elseif environment.argument("script") or environment.argument("s") then +elseif environment.argument("script") or environment.argument("scripts") then -- run a script by loading it (using libs), pass args - ok = runners.execute_ctx_script(filename,after) + if is_mkii_stub then + -- execute mkii script + ok = runners.execute_script(filename,false,true) + else + ok = runners.execute_ctx_script(filename) + end elseif environment.argument("execute") then -- execute script ok = runners.execute_script(filename) @@ -10491,6 +12610,8 @@ elseif environment.argument("locate") then elseif environment.argument("platform")then -- locate platform runners.locate_platform() +elseif environment.argument("prefixes") then + runners.prefixes() elseif environment.argument("timedrun") then -- locate platform runners.timedrun(filename) @@ -10499,8 +12620,14 @@ elseif environment.argument("help") or filename=='help' or filename == "" then -- execute script elseif filename:find("^bin:") then ok = runners.execute_program(filename) +elseif is_mkii_stub then + -- execute mkii script + ok = runners.execute_script(filename,false,true) else - ok = runners.execute_script(filename) + ok = runners.execute_ctx_script(filename) + if not ok then + ok = runners.execute_script(filename) + end end if os.platform == "unix" then diff --git a/Master/texmf-dist/scripts/context/stubs/unix/mtxtools b/Master/texmf-dist/scripts/context/stubs/unix/mtxtools deleted file mode 100755 index 3803c1c6f4e..00000000000 --- a/Master/texmf-dist/scripts/context/stubs/unix/mtxtools +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -mtxrun --usekpse --execute mtxtools.rb "$@" diff --git a/Master/texmf-dist/scripts/context/stubs/unix/mtxworks b/Master/texmf-dist/scripts/context/stubs/unix/mtxworks deleted file mode 100644 index ef8f230c3a3..00000000000 --- a/Master/texmf-dist/scripts/context/stubs/unix/mtxworks +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -mtxrun --script texworks --start diff --git a/Master/texmf-dist/scripts/context/stubs/unix/pdftools b/Master/texmf-dist/scripts/context/stubs/unix/pdftools deleted file mode 100755 index da7bd64cf2a..00000000000 --- a/Master/texmf-dist/scripts/context/stubs/unix/pdftools +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -mtxrun --usekpse --execute pdftools.rb "$@" diff --git a/Master/texmf-dist/scripts/context/stubs/unix/pstopdf b/Master/texmf-dist/scripts/context/stubs/unix/pstopdf deleted file mode 100755 index 059812cceac..00000000000 --- a/Master/texmf-dist/scripts/context/stubs/unix/pstopdf +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -mtxrun --usekpse --execute pstopdf.rb "$@" diff --git a/Master/texmf-dist/scripts/context/stubs/unix/rlxtools b/Master/texmf-dist/scripts/context/stubs/unix/rlxtools deleted file mode 100755 index d01987b3c1f..00000000000 --- a/Master/texmf-dist/scripts/context/stubs/unix/rlxtools +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -mtxrun --usekpse --execute rlxtools.rb "$@" diff --git a/Master/texmf-dist/scripts/context/stubs/unix/runtools b/Master/texmf-dist/scripts/context/stubs/unix/runtools deleted file mode 100755 index e21c1a24405..00000000000 --- a/Master/texmf-dist/scripts/context/stubs/unix/runtools +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -mtxrun --usekpse --execute runtools.rb "$@" diff --git a/Master/texmf-dist/scripts/context/stubs/unix/texexec b/Master/texmf-dist/scripts/context/stubs/unix/texexec index 083e500c69c..cd5900ff84c 100755 --- a/Master/texmf-dist/scripts/context/stubs/unix/texexec +++ b/Master/texmf-dist/scripts/context/stubs/unix/texexec @@ -1,2 +1,2 @@ #!/bin/sh -mtxrun --usekpse --execute texexec.rb "$@" +mtxrun --usekpse --execute texexec "$@" diff --git a/Master/texmf-dist/scripts/context/stubs/unix/texfont b/Master/texmf-dist/scripts/context/stubs/unix/texfont deleted file mode 100755 index bc811a640af..00000000000 --- a/Master/texmf-dist/scripts/context/stubs/unix/texfont +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -mtxrun --usekpse --execute texfont.pl "$@" diff --git a/Master/texmf-dist/scripts/context/stubs/unix/textools b/Master/texmf-dist/scripts/context/stubs/unix/textools deleted file mode 100755 index 76087ca5729..00000000000 --- a/Master/texmf-dist/scripts/context/stubs/unix/textools +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -mtxrun --usekpse --execute textools.rb "$@" diff --git a/Master/texmf-dist/scripts/context/stubs/unix/texutil b/Master/texmf-dist/scripts/context/stubs/unix/texutil deleted file mode 100755 index f5d9b6f1d81..00000000000 --- a/Master/texmf-dist/scripts/context/stubs/unix/texutil +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -mtxrun --usekpse --execute texutil.rb "$@" diff --git a/Master/texmf-dist/scripts/context/stubs/unix/tmftools b/Master/texmf-dist/scripts/context/stubs/unix/tmftools deleted file mode 100755 index 48d32f0fd3a..00000000000 --- a/Master/texmf-dist/scripts/context/stubs/unix/tmftools +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -mtxrun --usekpse --execute tmftools.rb "$@" diff --git a/Master/texmf-dist/scripts/context/stubs/unix/xmltools b/Master/texmf-dist/scripts/context/stubs/unix/xmltools deleted file mode 100755 index a673d1e7a03..00000000000 --- a/Master/texmf-dist/scripts/context/stubs/unix/xmltools +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -mtxrun --usekpse --execute xmltools.rb "$@" |