diff options
Diffstat (limited to 'Master/texmf-dist/scripts/context/stubs/unix/luatools')
-rw-r--r-- | Master/texmf-dist/scripts/context/stubs/unix/luatools | 2222 |
1 files changed, 1570 insertions, 652 deletions
diff --git a/Master/texmf-dist/scripts/context/stubs/unix/luatools b/Master/texmf-dist/scripts/context/stubs/unix/luatools index 433d1b8dc0a..1d87322c108 100644 --- a/Master/texmf-dist/scripts/context/stubs/unix/luatools +++ b/Master/texmf-dist/scripts/context/stubs/unix/luatools @@ -39,13 +39,16 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-string'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -local sub, gsub, find, match, gmatch, format, char, byte, rep = string.sub, string.gsub, string.find, string.match, string.gmatch, string.format, string.char, string.byte, string.rep +local sub, gsub, find, match, gmatch, format, char, byte, rep, lower = string.sub, string.gsub, string.find, string.match, string.gmatch, string.format, string.char, string.byte, string.rep, string.lower +local lpegmatch = lpeg.match + +-- some functions may disappear as they are not used anywhere if not string.split then @@ -85,8 +88,16 @@ function string:unquote() return (gsub(self,"^([\"\'])(.*)%1$","%2")) end +--~ function string:unquote() +--~ if find(self,"^[\'\"]") then +--~ return sub(self,2,-2) +--~ else +--~ return self +--~ end +--~ end + function string:quote() -- we could use format("%q") - return '"' .. self:unquote() .. '"' + return format("%q",self) end function string:count(pattern) -- variant 3 @@ -106,12 +117,23 @@ function string:limit(n,sentinel) end end -function string:strip() - return (gsub(self,"^%s*(.-)%s*$", "%1")) +--~ function string:strip() -- the .- is quite efficient +--~ -- return match(self,"^%s*(.-)%s*$") or "" +--~ -- return match(self,'^%s*(.*%S)') or '' -- posted on lua list +--~ return find(s,'^%s*$') and '' or match(s,'^%s*(.*%S)') +--~ end + +do -- roberto's variant: + local space = lpeg.S(" \t\v\n") + local nospace = 1 - space + local stripper = space^0 * lpeg.C((space^0 * nospace^1)^0) + function string.strip(str) + return lpegmatch(stripper,str) or "" + end end function string:is_empty() - return not find(find,"%S") + return not find(self,"%S") end function string:enhance(pattern,action) @@ -145,14 +167,14 @@ if not string.characters then local function nextchar(str, index) index = index + 1 - return (index <= #str) and index or nil, str:sub(index,index) + return (index <= #str) and index or nil, sub(str,index,index) end function string:characters() return nextchar, self, 0 end local function nextbyte(str, index) index = index + 1 - return (index <= #str) and index or nil, byte(str:sub(index,index)) + return (index <= #str) and index or nil, byte(sub(str,index,index)) end function string:bytes() return nextbyte, self, 0 @@ -165,7 +187,7 @@ end function string:rpadd(n,chr) local m = n-#self if m > 0 then - return self .. self.rep(chr or " ",m) + return self .. rep(chr or " ",m) else return self end @@ -174,7 +196,7 @@ end function string:lpadd(n,chr) local m = n-#self if m > 0 then - return self.rep(chr or " ",m) .. self + return rep(chr or " ",m) .. self else return self end @@ -222,6 +244,17 @@ function string:pattesc() return (gsub(self,".",patterns_escapes)) end +local simple_escapes = { + ["-"] = "%-", + ["."] = "%.", + ["?"] = ".", + ["*"] = ".*", +} + +function string:simpleesc() + return (gsub(self,".",simple_escapes)) +end + function string:tohash() local t = { } for s in gmatch(self,"([^, ]+)") do -- lpeg @@ -233,10 +266,10 @@ end local pattern = lpeg.Ct(lpeg.C(1)^0) function string:totable() - return pattern:match(self) + return lpegmatch(pattern,self) end ---~ for _, str in ipairs { +--~ local t = { --~ "1234567123456712345671234567", --~ "a\tb\tc", --~ "aa\tbb\tcc", @@ -244,7 +277,10 @@ end --~ "aaaa\tbbbb\tcccc", --~ "aaaaa\tbbbbb\tccccc", --~ "aaaaaa\tbbbbbb\tcccccc", ---~ } do print(string.tabtospace(str)) end +--~ } +--~ for k,v do +--~ print(string.tabtospace(t[k])) +--~ end function string.tabtospace(str,tab) -- we don't handle embedded newlines @@ -252,7 +288,7 @@ function string.tabtospace(str,tab) local s = find(str,"\t") if s then if not tab then tab = 7 end -- only when found - local d = tab-(s-1)%tab + local d = tab-(s-1) % tab if d > 0 then str = gsub(str,"\t",rep(" ",d),1) else @@ -271,6 +307,25 @@ function string:compactlong() -- strips newlines and leading spaces return self end +function string:striplong() -- strips newlines and leading spaces + self = gsub(self,"^%s*","") + self = gsub(self,"[\n\r]+ *","\n") + return self +end + +function string:topattern(lowercase,strict) + if lowercase then + self = lower(self) + end + self = gsub(self,".",simple_escapes) + if self == "" then + self = ".*" + elseif strict then + self = "^" .. self .. "$" + end + return self +end + end -- of closure @@ -278,58 +333,64 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-lpeg'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -local P, S, Ct, C, Cs, Cc = lpeg.P, lpeg.S, lpeg.Ct, lpeg.C, lpeg.Cs, lpeg.Cc - ---~ l-lpeg.lua : - ---~ lpeg.digit = lpeg.R('09')^1 ---~ lpeg.sign = lpeg.S('+-')^1 ---~ lpeg.cardinal = lpeg.P(lpeg.sign^0 * lpeg.digit^1) ---~ lpeg.integer = lpeg.P(lpeg.sign^0 * lpeg.digit^1) ---~ lpeg.float = lpeg.P(lpeg.sign^0 * lpeg.digit^0 * lpeg.P('.') * lpeg.digit^1) ---~ lpeg.number = lpeg.float + lpeg.integer ---~ lpeg.oct = lpeg.P("0") * lpeg.R('07')^1 ---~ lpeg.hex = lpeg.P("0x") * (lpeg.R('09') + lpeg.R('AF'))^1 ---~ lpeg.uppercase = lpeg.P("AZ") ---~ lpeg.lowercase = lpeg.P("az") - ---~ lpeg.eol = lpeg.S('\r\n\f')^1 -- includes formfeed ---~ lpeg.space = lpeg.S(' ')^1 ---~ lpeg.nonspace = lpeg.P(1-lpeg.space)^1 ---~ lpeg.whitespace = lpeg.S(' \r\n\f\t')^1 ---~ lpeg.nonwhitespace = lpeg.P(1-lpeg.whitespace)^1 - -local hash = { } +local lpeg = require("lpeg") + +lpeg.patterns = lpeg.patterns or { } -- so that we can share +local patterns = lpeg.patterns + +local P, R, S, Ct, C, Cs, Cc, V = lpeg.P, lpeg.R, lpeg.S, lpeg.Ct, lpeg.C, lpeg.Cs, lpeg.Cc, lpeg.V +local match = lpeg.match + +local digit, sign = R('09'), S('+-') +local cr, lf, crlf = P("\r"), P("\n"), P("\r\n") +local utf8byte = R("\128\191") + +patterns.utf8byte = utf8byte +patterns.utf8one = R("\000\127") +patterns.utf8two = R("\194\223") * utf8byte +patterns.utf8three = R("\224\239") * utf8byte * utf8byte +patterns.utf8four = R("\240\244") * utf8byte * utf8byte * utf8byte + +patterns.digit = digit +patterns.sign = sign +patterns.cardinal = sign^0 * digit^1 +patterns.integer = sign^0 * digit^1 +patterns.float = sign^0 * digit^0 * P('.') * digit^1 +patterns.number = patterns.float + patterns.integer +patterns.oct = P("0") * R("07")^1 +patterns.octal = patterns.oct +patterns.HEX = P("0x") * R("09","AF")^1 +patterns.hex = P("0x") * R("09","af")^1 +patterns.hexadecimal = P("0x") * R("09","AF","af")^1 +patterns.lowercase = R("az") +patterns.uppercase = R("AZ") +patterns.letter = patterns.lowercase + patterns.uppercase +patterns.space = S(" ") +patterns.eol = S("\n\r") +patterns.spacer = S(" \t\f\v") -- + string.char(0xc2, 0xa0) if we want utf (cf mail roberto) +patterns.newline = crlf + cr + lf +patterns.nonspace = 1 - patterns.space +patterns.nonspacer = 1 - patterns.spacer +patterns.whitespace = patterns.eol + patterns.spacer +patterns.nonwhitespace = 1 - patterns.whitespace +patterns.utf8 = patterns.utf8one + patterns.utf8two + patterns.utf8three + patterns.utf8four +patterns.utfbom = P('\000\000\254\255') + P('\255\254\000\000') + P('\255\254') + P('\254\255') + P('\239\187\191') function lpeg.anywhere(pattern) --slightly adapted from website - return P { P(pattern) + 1 * lpeg.V(1) } -end - -function lpeg.startswith(pattern) --slightly adapted - return P(pattern) + return P { P(pattern) + 1 * V(1) } -- why so complex? end function lpeg.splitter(pattern, action) return (((1-P(pattern))^1)/action+1)^0 end --- variant: - ---~ local parser = lpeg.Ct(lpeg.splitat(newline)) - -local crlf = P("\r\n") -local cr = P("\r") -local lf = P("\n") -local space = S(" \t\f\v") -- + string.char(0xc2, 0xa0) if we want utf (cf mail roberto) -local newline = crlf + cr + lf -local spacing = space^0 * newline - +local spacing = patterns.spacer^0 * patterns.newline -- sort of strip local empty = spacing * Cc("") local nonempty = Cs((1-spacing)^1) * spacing^-1 local content = (empty + nonempty)^1 @@ -337,15 +398,15 @@ local content = (empty + nonempty)^1 local capture = Ct(content^0) function string:splitlines() - return capture:match(self) + return match(capture,self) end -lpeg.linebyline = content -- better make a sublibrary +patterns.textline = content ---~ local p = lpeg.splitat("->",false) print(p:match("oeps->what->more")) -- oeps what more ---~ local p = lpeg.splitat("->",true) print(p:match("oeps->what->more")) -- oeps what->more ---~ local p = lpeg.splitat("->",false) print(p:match("oeps")) -- oeps ---~ local p = lpeg.splitat("->",true) print(p:match("oeps")) -- oeps +--~ local p = lpeg.splitat("->",false) print(match(p,"oeps->what->more")) -- oeps what more +--~ local p = lpeg.splitat("->",true) print(match(p,"oeps->what->more")) -- oeps what->more +--~ local p = lpeg.splitat("->",false) print(match(p,"oeps")) -- oeps +--~ local p = lpeg.splitat("->",true) print(match(p,"oeps")) -- oeps local splitters_s, splitters_m = { }, { } @@ -355,7 +416,7 @@ local function splitat(separator,single) separator = P(separator) if single then local other, any = C((1 - separator)^0), P(1) - splitter = other * (separator * C(any^0) + "") + splitter = other * (separator * C(any^0) + "") -- ? splitters_s[separator] = splitter else local other = C((1 - separator)^0) @@ -370,15 +431,72 @@ lpeg.splitat = splitat local cache = { } +function lpeg.split(separator,str) + local c = cache[separator] + if not c then + c = Ct(splitat(separator)) + cache[separator] = c + end + return match(c,str) +end + function string:split(separator) local c = cache[separator] if not c then c = Ct(splitat(separator)) cache[separator] = c end - return c:match(self) + return match(c,self) +end + +lpeg.splitters = cache + +local cache = { } + +function lpeg.checkedsplit(separator,str) + local c = cache[separator] + if not c then + separator = P(separator) + local other = C((1 - separator)^0) + c = Ct(separator^0 * other * (separator^1 * other)^0) + cache[separator] = c + end + return match(c,str) +end + +function string:checkedsplit(separator) + local c = cache[separator] + if not c then + separator = P(separator) + local other = C((1 - separator)^0) + c = Ct(separator^0 * other * (separator^1 * other)^0) + cache[separator] = c + end + return match(c,self) end +--~ function lpeg.append(list,pp) +--~ local p = pp +--~ for l=1,#list do +--~ if p then +--~ p = p + P(list[l]) +--~ else +--~ p = P(list[l]) +--~ end +--~ end +--~ return p +--~ end + +--~ from roberto's site: + +local f1 = string.byte + +local function f2(s) local c1, c2 = f1(s,1,2) return c1 * 64 + c2 - 12416 end +local function f3(s) local c1, c2, c3 = f1(s,1,3) return (c1 * 64 + c2) * 64 + c3 - 925824 end +local function f4(s) local c1, c2, c3, c4 = f1(s,1,4) return ((c1 * 64 + c2) * 64 + c3) * 64 + c4 - 63447168 end + +patterns.utf8byte = patterns.utf8one/f1 + patterns.utf8two/f2 + patterns.utf8three/f3 + patterns.utf8four/f4 + end -- of closure @@ -386,7 +504,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-table'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -395,9 +513,10 @@ if not modules then modules = { } end modules ['l-table'] = { table.join = table.concat local concat, sort, insert, remove = table.concat, table.sort, table.insert, table.remove -local format, find, gsub, lower, dump = string.format, string.find, string.gsub, string.lower, string.dump +local format, find, gsub, lower, dump, match = string.format, string.find, string.gsub, string.lower, string.dump, string.match local getmetatable, setmetatable = getmetatable, setmetatable -local type, next, tostring, ipairs = type, next, tostring, ipairs +local type, next, tostring, tonumber, ipairs = type, next, tostring, tonumber, ipairs +local unpack = unpack or table.unpack function table.strip(tab) local lst = { } @@ -412,6 +531,14 @@ function table.strip(tab) return lst end +function table.keys(t) + local k = { } + for key, _ in next, t do + k[#k+1] = key + end + return k +end + local function compare(a,b) return (tostring(a) < tostring(b)) end @@ -455,7 +582,7 @@ end table.sortedkeys = sortedkeys table.sortedhashkeys = sortedhashkeys -function table.sortedpairs(t) +function table.sortedhash(t) local s = sortedhashkeys(t) -- maybe just sortedkeys local n = 0 local function kv(s) @@ -466,6 +593,8 @@ function table.sortedpairs(t) return kv, s end +table.sortedpairs = table.sortedhash + function table.append(t, list) for _,v in next, list do insert(t,v) @@ -588,18 +717,18 @@ end -- slower than #t on indexed tables (#t only returns the size of the numerically indexed slice) -function table.is_empty(t) +function table.is_empty(t) -- obolete, use inline code instead return not t or not next(t) end -function table.one_entry(t) +function table.one_entry(t) -- obolete, use inline code instead local n = next(t) return n and not next(t,n) end -function table.starts_at(t) - return ipairs(t,1)(t,0) -end +--~ function table.starts_at(t) -- obsolete, not nice +--~ return ipairs(t,1)(t,0) +--~ end function table.tohash(t,value) local h = { } @@ -677,6 +806,8 @@ end -- -- local propername = lpeg.P(lpeg.R("AZ","az","__") * lpeg.R("09","AZ","az", "__")^0 * lpeg.P(-1) ) +-- problem: there no good number_to_string converter with the best resolution + local function do_serialize(root,name,depth,level,indexed) if level > 0 then depth = depth .. " " @@ -699,8 +830,9 @@ local function do_serialize(root,name,depth,level,indexed) handle(format("%s{",depth)) end end + -- we could check for k (index) being number (cardinal) if root and next(root) then - local first, last = nil, 0 -- #root cannot be trusted here + local first, last = nil, 0 -- #root cannot be trusted here (will be ok in 5.2 when ipairs is gone) if compact then -- NOT: for k=1,#root do (we need to quit at nil) for k,v in ipairs(root) do -- can we use next? @@ -721,10 +853,10 @@ local function do_serialize(root,name,depth,level,indexed) if hexify then handle(format("%s 0x%04X,",depth,v)) else - handle(format("%s %s,",depth,v)) + handle(format("%s %s,",depth,v)) -- %.99g end elseif t == "string" then - if reduce and (find(v,"^[%-%+]?[%d]-%.?[%d+]$") == 1) then + if reduce and tonumber(v) then handle(format("%s %s,",depth,v)) else handle(format("%s %q,",depth,v)) @@ -761,29 +893,29 @@ local function do_serialize(root,name,depth,level,indexed) --~ if hexify then --~ handle(format("%s %s=0x%04X,",depth,key(k),v)) --~ else - --~ handle(format("%s %s=%s,",depth,key(k),v)) + --~ handle(format("%s %s=%s,",depth,key(k),v)) -- %.99g --~ end if type(k) == "number" then -- or find(k,"^%d+$") then if hexify then handle(format("%s [0x%04X]=0x%04X,",depth,k,v)) else - handle(format("%s [%s]=%s,",depth,k,v)) + handle(format("%s [%s]=%s,",depth,k,v)) -- %.99g end elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then if hexify then handle(format("%s %s=0x%04X,",depth,k,v)) else - handle(format("%s %s=%s,",depth,k,v)) + handle(format("%s %s=%s,",depth,k,v)) -- %.99g end else if hexify then handle(format("%s [%q]=0x%04X,",depth,k,v)) else - handle(format("%s [%q]=%s,",depth,k,v)) + handle(format("%s [%q]=%s,",depth,k,v)) -- %.99g end end elseif t == "string" then - if reduce and (find(v,"^[%-%+]?[%d]-%.?[%d+]$") == 1) then + if reduce and tonumber(v) then --~ handle(format("%s %s=%s,",depth,key(k),v)) if type(k) == "number" then -- or find(k,"^%d+$") then if hexify then @@ -992,7 +1124,7 @@ function table.tofile(filename,root,name,reduce,noquotes,hexify) end end -local function flatten(t,f,complete) +local function flatten(t,f,complete) -- is this used? meybe a variant with next, ... for i=1,#t do local v = t[i] if type(v) == "table" then @@ -1021,6 +1153,24 @@ end table.flatten_one_level = table.unnest +-- a better one: + +local function flattened(t,f) + if not f then + f = { } + end + for k, v in next, t do + if type(v) == "table" then + flattened(v,f) + else + f[k] = v + end + end + return f +end + +table.flattened = flattened + -- the next three may disappear function table.remove_value(t,value) -- todo: n @@ -1156,7 +1306,7 @@ function table.clone(t,p) -- t is optional or nil or table elseif not t then t = { } end - setmetatable(t, { __index = function(_,key) return p[key] end }) + setmetatable(t, { __index = function(_,key) return p[key] end }) -- why not __index = p ? return t end @@ -1184,21 +1334,35 @@ function table.reverse(t) return tt end ---~ function table.keys(t) ---~ local k = { } ---~ for k,_ in next, t do ---~ k[#k+1] = k ---~ end ---~ return k ---~ end +function table.insert_before_value(t,value,extra) + for i=1,#t do + if t[i] == extra then + remove(t,i) + end + end + for i=1,#t do + if t[i] == value then + insert(t,i,extra) + return + end + end + insert(t,1,extra) +end ---~ function table.keys_as_string(t) ---~ local k = { } ---~ for k,_ in next, t do ---~ k[#k+1] = k ---~ end ---~ return concat(k,"") ---~ end +function table.insert_after_value(t,value,extra) + for i=1,#t do + if t[i] == extra then + remove(t,i) + end + end + for i=1,#t do + if t[i] == value then + insert(t,i+1,extra) + return + end + end + insert(t,#t+1,extra) +end end -- of closure @@ -1207,13 +1371,13 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-io'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -local byte = string.byte +local byte, find, gsub = string.byte, string.find, string.gsub if string.find(os.getenv("PATH"),";") then io.fileseparator, io.pathseparator = "\\", ";" @@ -1242,7 +1406,7 @@ function io.savedata(filename,data,joiner) elseif type(data) == "function" then data(f) else - f:write(data) + f:write(data or "") end f:close() return true @@ -1371,20 +1535,21 @@ function io.ask(question,default,options) end io.write(string.format(" ")) local answer = io.read() - answer = answer:gsub("^%s*(.*)%s*$","%1") + answer = gsub(answer,"^%s*(.*)%s*$","%1") if answer == "" and default then return default elseif not options then return answer else - for _,v in pairs(options) do - if v == answer then + for k=1,#options do + if options[k] == answer then return answer end end local pattern = "^" .. answer - for _,v in pairs(options) do - if v:find(pattern) then + for k=1,#options do + local v = options[k] + if find(v,pattern) then return v end end @@ -1399,20 +1564,22 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-number'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -local format = string.format +local tostring = tostring +local format, floor, insert, match = string.format, math.floor, table.insert, string.match +local lpegmatch = lpeg.match number = number or { } -- a,b,c,d,e,f = number.toset(100101) function number.toset(n) - return (tostring(n)):match("(.?)(.?)(.?)(.?)(.?)(.?)(.?)(.?)") + return match(tostring(n),"(.?)(.?)(.?)(.?)(.?)(.?)(.?)(.?)") end function number.toevenhex(n) @@ -1438,10 +1605,21 @@ end local one = lpeg.C(1-lpeg.S(''))^1 function number.toset(n) - return one:match(tostring(n)) + return lpegmatch(one,tostring(n)) end - +function number.bits(n,zero) + local t, i = { }, (zero and 0) or 1 + while n > 0 do + local m = n % 2 + if m > 0 then + insert(t,1,i) + end + n = floor(n/2) + i = i + 1 + end + return t +end end -- of closure @@ -1450,7 +1628,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-set'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -1540,46 +1718,63 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-os'] = { version = 1.001, - comment = "companion to luat-lub.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -local find = string.find +-- maybe build io.flush in os.execute + +local find, format, gsub = string.find, string.format, string.gsub +local random, ceil = math.random, math.ceil + +local execute, spawn, exec, ioflush = os.execute, os.spawn or os.execute, os.exec or os.execute, io.flush + +function os.execute(...) ioflush() return execute(...) end +function os.spawn (...) ioflush() return spawn (...) end +function os.exec (...) ioflush() return exec (...) end function os.resultof(command) - return io.popen(command,"r"):read("*all") + ioflush() -- else messed up logging + local handle = io.popen(command,"r") + if not handle then + -- print("unknown command '".. command .. "' in os.resultof") + return "" + else + return handle:read("*all") or "" + end end -if not os.exec then os.exec = os.execute end -if not os.spawn then os.spawn = os.execute end - ---~ os.type : windows | unix (new, we already guessed os.platform) ---~ os.name : windows | msdos | linux | macosx | solaris | .. | generic (new) +--~ os.type : windows | unix (new, we already guessed os.platform) +--~ os.name : windows | msdos | linux | macosx | solaris | .. | generic (new) +--~ os.platform : extended os.name with architecture if not io.fileseparator then if find(os.getenv("PATH"),";") then - io.fileseparator, io.pathseparator, os.platform = "\\", ";", os.type or "windows" + io.fileseparator, io.pathseparator, os.type = "\\", ";", os.type or "mswin" else - io.fileseparator, io.pathseparator, os.platform = "/" , ":", os.type or "unix" + io.fileseparator, io.pathseparator, os.type = "/" , ":", os.type or "unix" end end -os.platform = os.platform or os.type or (io.pathseparator == ";" and "windows") or "unix" +os.type = os.type or (io.pathseparator == ";" and "windows") or "unix" +os.name = os.name or (os.type == "windows" and "mswin" ) or "linux" + +if os.type == "windows" then + os.libsuffix, os.binsuffix = 'dll', 'exe' +else + os.libsuffix, os.binsuffix = 'so', '' +end function os.launch(str) - if os.platform == "windows" then + if os.type == "windows" then os.execute("start " .. str) -- os.spawn ? else os.execute(str .. " &") -- os.spawn ? end end -if not os.setenv then - function os.setenv() return false end -end - if not os.times then -- utime = user time -- stime = system time @@ -1609,64 +1804,218 @@ end --~ print(os.date("%H:%M:%S",os.gettimeofday())) --~ print(os.date("%H:%M:%S",os.time())) -os.arch = os.arch or function() - local a = os.resultof("uname -m") or "linux" - os.arch = function() - return a +-- no need for function anymore as we have more clever code and helpers now +-- this metatable trickery might as well disappear + +os.resolvers = os.resolvers or { } + +local resolvers = os.resolvers + +local osmt = getmetatable(os) or { __index = function(t,k) t[k] = "unset" return "unset" end } -- maybe nil +local osix = osmt.__index + +osmt.__index = function(t,k) + return (resolvers[k] or osix)(t,k) +end + +setmetatable(os,osmt) + +if not os.setenv then + + -- we still store them but they won't be seen in + -- child processes although we might pass them some day + -- using command concatination + + local env, getenv = { }, os.getenv + + function os.setenv(k,v) + env[k] = v end - return a + + function os.getenv(k) + return env[k] or getenv(k) + end + end -local platform +-- we can use HOSTTYPE on some platforms -function os.currentplatform(name,default) - if not platform then - local name = os.name or os.platform or name -- os.name is built in, os.platform is mine - if not name then - platform = default or "linux" - elseif name == "windows" or name == "mswin" or name == "win32" or name == "msdos" then - if os.getenv("PROCESSOR_ARCHITECTURE") == "AMD64" then - platform = "mswin-64" - else - platform = "mswin" - end +local name, platform = os.name or "linux", os.getenv("MTX_PLATFORM") or "" + +local function guess() + local architecture = os.resultof("uname -m") or "" + if architecture ~= "" then + return architecture + end + architecture = os.getenv("HOSTTYPE") or "" + if architecture ~= "" then + return architecture + end + return os.resultof("echo $HOSTTYPE") or "" +end + +if platform ~= "" then + + os.platform = platform + +elseif os.type == "windows" then + + -- we could set the variable directly, no function needed here + + function os.resolvers.platform(t,k) + local platform, architecture = "", os.getenv("PROCESSOR_ARCHITECTURE") or "" + if find(architecture,"AMD64") then + platform = "mswin-64" else - local architecture = os.arch() - if name == "linux" then - if find(architecture,"x86_64") then - platform = "linux-64" - elseif find(architecture,"ppc") then - platform = "linux-ppc" - else - platform = "linux" - end - elseif name == "macosx" then - if find(architecture,"i386") then - platform = "osx-intel" - else - platform = "osx-ppc" - end - elseif name == "sunos" then - if find(architecture,"sparc") then - platform = "solaris-sparc" - else -- if architecture == 'i86pc' - platform = "solaris-intel" - end - elseif name == "freebsd" then - if find(architecture,"amd64") then - platform = "freebsd-amd64" - else - platform = "freebsd" - end - else - platform = default or name - end + platform = "mswin" end - function os.currentplatform() - return platform + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +elseif name == "linux" then + + function os.resolvers.platform(t,k) + -- we sometims have HOSTTYPE set so let's check that first + local platform, architecture = "", os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" + if find(architecture,"x86_64") then + platform = "linux-64" + elseif find(architecture,"ppc") then + platform = "linux-ppc" + else + platform = "linux" + end + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +elseif name == "macosx" then + + --[[ + Identifying the architecture of OSX is quite a mess and this + is the best we can come up with. For some reason $HOSTTYPE is + a kind of pseudo environment variable, not known to the current + environment. And yes, uname cannot be trusted either, so there + is a change that you end up with a 32 bit run on a 64 bit system. + Also, some proper 64 bit intel macs are too cheap (low-end) and + therefore not permitted to run the 64 bit kernel. + ]]-- + + function os.resolvers.platform(t,k) + -- local platform, architecture = "", os.getenv("HOSTTYPE") or "" + -- if architecture == "" then + -- architecture = os.resultof("echo $HOSTTYPE") or "" + -- end + local platform, architecture = "", os.resultof("echo $HOSTTYPE") or "" + if architecture == "" then + -- print("\nI have no clue what kind of OSX you're running so let's assume an 32 bit intel.\n") + platform = "osx-intel" + elseif find(architecture,"i386") then + platform = "osx-intel" + elseif find(architecture,"x86_64") then + platform = "osx-64" + else + platform = "osx-ppc" + end + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +elseif name == "sunos" then + + function os.resolvers.platform(t,k) + local platform, architecture = "", os.resultof("uname -m") or "" + if find(architecture,"sparc") then + platform = "solaris-sparc" + else -- if architecture == 'i86pc' + platform = "solaris-intel" + end + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +elseif name == "freebsd" then + + function os.resolvers.platform(t,k) + local platform, architecture = "", os.resultof("uname -m") or "" + if find(architecture,"amd64") then + platform = "freebsd-amd64" + else + platform = "freebsd" + end + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +elseif name == "kfreebsd" then + + function os.resolvers.platform(t,k) + -- we sometims have HOSTTYPE set so let's check that first + local platform, architecture = "", os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" + if find(architecture,"x86_64") then + platform = "kfreebsd-64" + else + platform = "kfreebsd-i386" + end + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +else + + -- platform = "linux" + -- os.setenv("MTX_PLATFORM",platform) + -- os.platform = platform + + function os.resolvers.platform(t,k) + local platform = "linux" + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +end + +-- beware, we set the randomseed + +-- from wikipedia: Version 4 UUIDs use a scheme relying only on random numbers. This algorithm sets the +-- version number as well as two reserved bits. All other bits are set using a random or pseudorandom +-- data source. Version 4 UUIDs have the form xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx with hexadecimal +-- digits x and hexadecimal digits 8, 9, A, or B for y. e.g. f47ac10b-58cc-4372-a567-0e02b2c3d479. +-- +-- as we don't call this function too often there is not so much risk on repetition + +local t = { 8, 9, "a", "b" } + +function os.uuid() + return format("%04x%04x-4%03x-%s%03x-%04x-%04x%04x%04x", + random(0xFFFF),random(0xFFFF), + random(0x0FFF), + t[ceil(random(4))] or 8,random(0x0FFF), + random(0xFFFF), + random(0xFFFF),random(0xFFFF),random(0xFFFF) + ) +end + +local d + +function os.timezone(delta) + d = d or tonumber(tonumber(os.date("%H")-os.date("!%H"))) + if delta then + if d > 0 then + return format("+%02i:00",d) + else + return format("-%02i:00",-d) end + else + return 1 end - return platform end @@ -1676,7 +2025,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-file'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -1687,14 +2036,17 @@ if not modules then modules = { } end modules ['l-file'] = { file = file or { } local concat = table.concat -local find, gmatch, match, gsub = string.find, string.gmatch, string.match, string.gsub +local find, gmatch, match, gsub, sub, char = string.find, string.gmatch, string.match, string.gsub, string.sub, string.char +local lpegmatch = lpeg.match function file.removesuffix(filename) return (gsub(filename,"%.[%a%d]+$","")) end function file.addsuffix(filename, suffix) - if not find(filename,"%.[%a%d]+$") then + if not suffix or suffix == "" then + return filename + elseif not find(filename,"%.[%a%d]+$") then return filename .. "." .. suffix else return filename @@ -1717,20 +2069,39 @@ function file.nameonly(name) return (gsub(match(name,"^.+[/\\](.-)$") or name,"%..*$","")) end -function file.extname(name) - return match(name,"^.+%.([^/\\]-)$") or "" +function file.extname(name,default) + return match(name,"^.+%.([^/\\]-)$") or default or "" end file.suffix = file.extname ---~ print(file.join("x/","/y")) ---~ print(file.join("http://","/y")) ---~ print(file.join("http://a","/y")) ---~ print(file.join("http:///a","/y")) ---~ print(file.join("//nas-1","/y")) +--~ function file.join(...) +--~ local pth = concat({...},"/") +--~ pth = gsub(pth,"\\","/") +--~ local a, b = match(pth,"^(.*://)(.*)$") +--~ if a and b then +--~ return a .. gsub(b,"//+","/") +--~ end +--~ a, b = match(pth,"^(//)(.*)$") +--~ if a and b then +--~ return a .. gsub(b,"//+","/") +--~ end +--~ return (gsub(pth,"//+","/")) +--~ end + +local trick_1 = char(1) +local trick_2 = "^" .. trick_1 .. "/+" function file.join(...) - local pth = concat({...},"/") + local lst = { ... } + local a, b = lst[1], lst[2] + if a == "" then + lst[1] = trick_1 + elseif b and find(a,"^/+$") and find(b,"^/") then + lst[1] = "" + lst[2] = gsub(b,"^/+","") + end + local pth = concat(lst,"/") pth = gsub(pth,"\\","/") local a, b = match(pth,"^(.*://)(.*)$") if a and b then @@ -1740,17 +2111,28 @@ function file.join(...) if a and b then return a .. gsub(b,"//+","/") end + pth = gsub(pth,trick_2,"") return (gsub(pth,"//+","/")) end +--~ print(file.join("//","/y")) +--~ print(file.join("/","/y")) +--~ print(file.join("","/y")) +--~ print(file.join("/x/","/y")) +--~ print(file.join("x/","/y")) +--~ print(file.join("http://","/y")) +--~ print(file.join("http://a","/y")) +--~ print(file.join("http:///a","/y")) +--~ print(file.join("//nas-1","/y")) + function file.iswritable(name) local a = lfs.attributes(name) or lfs.attributes(file.dirname(name,".")) - return a and a.permissions:sub(2,2) == "w" + return a and sub(a.permissions,2,2) == "w" end function file.isreadable(name) local a = lfs.attributes(name) - return a and a.permissions:sub(1,1) == "r" + return a and sub(a.permissions,1,1) == "r" end file.is_readable = file.isreadable @@ -1758,36 +2140,50 @@ file.is_writable = file.iswritable -- todo: lpeg -function file.split_path(str) - local t = { } - str = gsub(str,"\\", "/") - str = gsub(str,"(%a):([;/])", "%1\001%2") - for name in gmatch(str,"([^;:]+)") do - if name ~= "" then - t[#t+1] = gsub(name,"\001",":") - end - end - return t +--~ function file.split_path(str) +--~ local t = { } +--~ str = gsub(str,"\\", "/") +--~ str = gsub(str,"(%a):([;/])", "%1\001%2") +--~ for name in gmatch(str,"([^;:]+)") do +--~ if name ~= "" then +--~ t[#t+1] = gsub(name,"\001",":") +--~ end +--~ end +--~ return t +--~ end + +local checkedsplit = string.checkedsplit + +function file.split_path(str,separator) + str = gsub(str,"\\","/") + return checkedsplit(str,separator or io.pathseparator) end function file.join_path(tab) return concat(tab,io.pathseparator) -- can have trailing // end +-- we can hash them weakly + function file.collapse_path(str) - str = gsub(str,"/%./","/") - local n, m = 1, 1 - while n > 0 or m > 0 do - str, n = gsub(str,"[^/%.]+/%.%.$","") - str, m = gsub(str,"[^/%.]+/%.%./","") - end - str = gsub(str,"([^/])/$","%1") - str = gsub(str,"^%./","") - str = gsub(str,"/%.$","") + str = gsub(str,"\\","/") + if find(str,"/") then + str = gsub(str,"^%./",(gsub(lfs.currentdir(),"\\","/")) .. "/") -- ./xx in qualified + str = gsub(str,"/%./","/") + local n, m = 1, 1 + while n > 0 or m > 0 do + str, n = gsub(str,"[^/%.]+/%.%.$","") + str, m = gsub(str,"[^/%.]+/%.%./","") + end + str = gsub(str,"([^/])/$","%1") + -- str = gsub(str,"^%./","") -- ./xx in qualified + str = gsub(str,"/%.$","") + end if str == "" then str = "." end return str end +--~ print(file.collapse_path("/a")) --~ print(file.collapse_path("a/./b/..")) --~ print(file.collapse_path("a/aa/../b/bb")) --~ print(file.collapse_path("a/../..")) @@ -1817,27 +2213,27 @@ end --~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * lpeg.C(noperiod^1) * -1 --~ function file.extname(name) ---~ return pattern:match(name) or "" +--~ return lpegmatch(pattern,name) or "" --~ end --~ local pattern = lpeg.Cs(((period * noperiod^1 * -1)/"" + 1)^1) --~ function file.removesuffix(name) ---~ return pattern:match(name) +--~ return lpegmatch(pattern,name) --~ end --~ local pattern = (noslashes^0 * slashes)^1 * lpeg.C(noslashes^1) * -1 --~ function file.basename(name) ---~ return pattern:match(name) or name +--~ return lpegmatch(pattern,name) or name --~ end --~ local pattern = (noslashes^0 * slashes)^1 * lpeg.Cp() * noslashes^1 * -1 --~ function file.dirname(name) ---~ local p = pattern:match(name) +--~ local p = lpegmatch(pattern,name) --~ if p then ---~ return name:sub(1,p-2) +--~ return sub(name,1,p-2) --~ else --~ return "" --~ end @@ -1846,7 +2242,7 @@ end --~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * lpeg.Cp() * noperiod^1 * -1 --~ function file.addsuffix(name, suffix) ---~ local p = pattern:match(name) +--~ local p = lpegmatch(pattern,name) --~ if p then --~ return name --~ else @@ -1857,9 +2253,9 @@ end --~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * lpeg.Cp() * noperiod^1 * -1 --~ function file.replacesuffix(name,suffix) ---~ local p = pattern:match(name) +--~ local p = lpegmatch(pattern,name) --~ if p then ---~ return name:sub(1,p-2) .. "." .. suffix +--~ return sub(name,1,p-2) .. "." .. suffix --~ else --~ return name .. "." .. suffix --~ end @@ -1868,11 +2264,11 @@ end --~ local pattern = (noslashes^0 * slashes)^0 * lpeg.Cp() * ((noperiod^1 * period)^1 * lpeg.Cp() + lpeg.P(true)) * noperiod^1 * -1 --~ function file.nameonly(name) ---~ local a, b = pattern:match(name) +--~ local a, b = lpegmatch(pattern,name) --~ if b then ---~ return name:sub(a,b-2) +--~ return sub(name,a,b-2) --~ elseif a then ---~ return name:sub(a) +--~ return sub(name,a) --~ else --~ return name --~ end @@ -1906,11 +2302,11 @@ local rootbased = lpeg.P("/") + letter*lpeg.P(":") -- ./name ../name /name c: :// name/name function file.is_qualified_path(filename) - return qualified:match(filename) + return lpegmatch(qualified,filename) ~= nil end function file.is_rootbased_path(filename) - return rootbased:match(filename) + return lpegmatch(rootbased,filename) ~= nil end local slash = lpeg.S("\\/") @@ -1923,16 +2319,25 @@ local base = lpeg.C((1-suffix)^0) local pattern = (drive + lpeg.Cc("")) * (path + lpeg.Cc("")) * (base + lpeg.Cc("")) * (suffix + lpeg.Cc("")) function file.splitname(str) -- returns drive, path, base, suffix - return pattern:match(str) + return lpegmatch(pattern,str) end --- function test(t) for k, v in pairs(t) do print(v, "=>", file.splitname(v)) end end +-- function test(t) for k, v in next, t do print(v, "=>", file.splitname(v)) end end -- -- test { "c:", "c:/aa", "c:/aa/bb", "c:/aa/bb/cc", "c:/aa/bb/cc.dd", "c:/aa/bb/cc.dd.ee" } -- test { "c:", "c:aa", "c:aa/bb", "c:aa/bb/cc", "c:aa/bb/cc.dd", "c:aa/bb/cc.dd.ee" } -- test { "/aa", "/aa/bb", "/aa/bb/cc", "/aa/bb/cc.dd", "/aa/bb/cc.dd.ee" } -- test { "aa", "aa/bb", "aa/bb/cc", "aa/bb/cc.dd", "aa/bb/cc.dd.ee" } +--~ -- todo: +--~ +--~ if os.type == "windows" then +--~ local currentdir = lfs.currentdir +--~ function lfs.currentdir() +--~ return (gsub(currentdir(),"\\","/")) +--~ end +--~ end + end -- of closure @@ -1997,7 +2402,7 @@ end function file.loadchecksum(name) if md5 then local data = io.loaddata(name .. ".md5") - return data and data:gsub("%s","") + return data and (gsub(data,"%s","")) end return nil end @@ -2018,14 +2423,15 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-url'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -local char, gmatch = string.char, string.gmatch +local char, gmatch, gsub = string.char, string.gmatch, string.gsub local tonumber, type = tonumber, type +local lpegmatch = lpeg.match -- from the spec (on the web): -- @@ -2049,7 +2455,9 @@ local hexdigit = lpeg.R("09","AF","af") local plus = lpeg.P("+") local escaped = (plus / " ") + (percent * lpeg.C(hexdigit * hexdigit) / tochar) -local scheme = lpeg.Cs((escaped+(1-colon-slash-qmark-hash))^0) * colon + lpeg.Cc("") +-- we assume schemes with more than 1 character (in order to avoid problems with windows disks) + +local scheme = lpeg.Cs((escaped+(1-colon-slash-qmark-hash))^2) * colon + lpeg.Cc("") local authority = slash * slash * lpeg.Cs((escaped+(1- slash-qmark-hash))^0) + lpeg.Cc("") local path = slash * lpeg.Cs((escaped+(1- qmark-hash))^0) + lpeg.Cc("") local query = qmark * lpeg.Cs((escaped+(1- hash))^0) + lpeg.Cc("") @@ -2057,25 +2465,51 @@ local fragment = hash * lpeg.Cs((escaped+(1- endofstring))^0 local parser = lpeg.Ct(scheme * authority * path * query * fragment) +-- todo: reconsider Ct as we can as well have five return values (saves a table) +-- so we can have two parsers, one with and one without + function url.split(str) - return (type(str) == "string" and parser:match(str)) or str + return (type(str) == "string" and lpegmatch(parser,str)) or str end +-- todo: cache them + function url.hashed(str) local s = url.split(str) + local somescheme = s[1] ~= "" return { - scheme = (s[1] ~= "" and s[1]) or "file", + scheme = (somescheme and s[1]) or "file", authority = s[2], - path = s[3], - query = s[4], - fragment = s[5], - original = str + path = s[3], + query = s[4], + fragment = s[5], + original = str, + noscheme = not somescheme, } end +function url.hasscheme(str) + return url.split(str)[1] ~= "" +end + +function url.addscheme(str,scheme) + return (url.hasscheme(str) and str) or ((scheme or "file:///") .. str) +end + +function url.construct(hash) + local fullurl = hash.sheme .. "://".. hash.authority .. hash.path + if hash.query then + fullurl = fullurl .. "?".. hash.query + end + if hash.fragment then + fullurl = fullurl .. "?".. hash.fragment + end + return fullurl +end + function url.filename(filename) local t = url.hashed(filename) - return (t.scheme == "file" and t.path:gsub("^/([a-zA-Z])([:|])/)","%1:")) or filename + return (t.scheme == "file" and (gsub(t.path,"^/([a-zA-Z])([:|])/)","%1:"))) or filename end function url.query(str) @@ -2129,17 +2563,26 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-dir'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } +-- dir.expand_name will be merged with cleanpath and collapsepath + local type = type -local find, gmatch = string.find, string.gmatch +local find, gmatch, match, gsub = string.find, string.gmatch, string.match, string.gsub +local lpegmatch = lpeg.match dir = dir or { } +-- handy + +function dir.current() + return (gsub(lfs.currentdir(),"\\","/")) +end + -- optimizing for no string.find (*) does not save time local attributes = lfs.attributes @@ -2170,6 +2613,35 @@ end dir.glob_pattern = glob_pattern +local function collect_pattern(path,patt,recurse,result) + local ok, scanner + result = result or { } + if path == "/" then + ok, scanner = xpcall(function() return walkdir(path..".") end, function() end) -- kepler safe + else + ok, scanner = xpcall(function() return walkdir(path) end, function() end) -- kepler safe + end + if ok and type(scanner) == "function" then + if not find(path,"/$") then path = path .. '/' end + for name in scanner do + local full = path .. name + local attr = attributes(full) + local mode = attr.mode + if mode == 'file' then + if find(full,patt) then + result[name] = attr + end + elseif recurse and (mode == "directory") and (name ~= '.') and (name ~= "..") then + attr.list = collect_pattern(full,patt,recurse) + result[name] = attr + end + end + end + return result +end + +dir.collect_pattern = collect_pattern + local P, S, R, C, Cc, Cs, Ct, Cv, V = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.Cc, lpeg.Cs, lpeg.Ct, lpeg.Cv, lpeg.V local pattern = Ct { @@ -2189,29 +2661,48 @@ local filter = Cs ( ( )^0 ) local function glob(str,t) - if type(str) == "table" then - local t = t or { } - for s=1,#str do - glob(str[s],t) + if type(t) == "function" then + if type(str) == "table" then + for s=1,#str do + glob(str[s],t) + end + elseif lfs.isfile(str) then + t(str) + else + local split = lpegmatch(pattern,str) + if split then + local root, path, base = split[1], split[2], split[3] + local recurse = find(base,"%*%*") + local start = root .. path + local result = lpegmatch(filter,start .. base) + glob_pattern(start,result,recurse,t) + end end - return t - elseif lfs.isfile(str) then - local t = t or { } - t[#t+1] = str - return t else - local split = pattern:match(str) - if split then + if type(str) == "table" then local t = t or { } - local action = action or function(name) t[#t+1] = name end - local root, path, base = split[1], split[2], split[3] - local recurse = find(base,"%*%*") - local start = root .. path - local result = filter:match(start .. base) - glob_pattern(start,result,recurse,action) + for s=1,#str do + glob(str[s],t) + end + return t + elseif lfs.isfile(str) then + local t = t or { } + t[#t+1] = str return t else - return { } + local split = lpegmatch(pattern,str) + if split then + local t = t or { } + local action = action or function(name) t[#t+1] = name end + local root, path, base = split[1], split[2], split[3] + local recurse = find(base,"%*%*") + local start = root .. path + local result = lpegmatch(filter,start .. base) + glob_pattern(start,result,recurse,action) + return t + else + return { } + end end end end @@ -2273,11 +2764,12 @@ end local make_indeed = true -- false -if string.find(os.getenv("PATH"),";") then +if string.find(os.getenv("PATH"),";") then -- os.type == "windows" function dir.mkdirs(...) - local str, pth = "", "" - for _, s in ipairs({...}) do + local str, pth, t = "", "", { ... } + for i=1,#t do + local s = t[i] if s ~= "" then if str ~= "" then str = str .. "/" .. s @@ -2288,13 +2780,13 @@ if string.find(os.getenv("PATH"),";") then end local first, middle, last local drive = false - first, middle, last = str:match("^(//)(//*)(.*)$") + first, middle, last = match(str,"^(//)(//*)(.*)$") if first then -- empty network path == local path else - first, last = str:match("^(//)/*(.-)$") + first, last = match(str,"^(//)/*(.-)$") if first then - middle, last = str:match("([^/]+)/+(.-)$") + middle, last = match(str,"([^/]+)/+(.-)$") if middle then pth = "//" .. middle else @@ -2302,11 +2794,11 @@ if string.find(os.getenv("PATH"),";") then last = "" end else - first, middle, last = str:match("^([a-zA-Z]:)(/*)(.-)$") + first, middle, last = match(str,"^([a-zA-Z]:)(/*)(.-)$") if first then pth, drive = first .. middle, true else - middle, last = str:match("^(/*)(.-)$") + middle, last = match(str,"^(/*)(.-)$") if not middle then last = str end @@ -2340,34 +2832,31 @@ if string.find(os.getenv("PATH"),";") then --~ print(dir.mkdirs("///a/b/c")) --~ print(dir.mkdirs("a/bbb//ccc/")) - function dir.expand_name(str) - local first, nothing, last = str:match("^(//)(//*)(.*)$") + function dir.expand_name(str) -- will be merged with cleanpath and collapsepath + local first, nothing, last = match(str,"^(//)(//*)(.*)$") if first then - first = lfs.currentdir() .. "/" - first = first:gsub("\\","/") + first = dir.current() .. "/" end if not first then - first, last = str:match("^(//)/*(.*)$") + first, last = match(str,"^(//)/*(.*)$") end if not first then - first, last = str:match("^([a-zA-Z]:)(.*)$") + first, last = match(str,"^([a-zA-Z]:)(.*)$") if first and not find(last,"^/") then local d = lfs.currentdir() if lfs.chdir(first) then - first = lfs.currentdir() - first = first:gsub("\\","/") + first = dir.current() end lfs.chdir(d) end end if not first then - first, last = lfs.currentdir(), str - first = first:gsub("\\","/") + first, last = dir.current(), str end - last = last:gsub("//","/") - last = last:gsub("/%./","/") - last = last:gsub("^/*","") - first = first:gsub("/*$","") + last = gsub(last,"//","/") + last = gsub(last,"/%./","/") + last = gsub(last,"^/*","") + first = gsub(first,"/*$","") if last == "" then return first else @@ -2378,8 +2867,9 @@ if string.find(os.getenv("PATH"),";") then else function dir.mkdirs(...) - local str, pth = "", "" - for _, s in ipairs({...}) do + local str, pth, t = "", "", { ... } + for i=1,#t do + local s = t[i] if s ~= "" then if str ~= "" then str = str .. "/" .. s @@ -2388,7 +2878,7 @@ else end end end - str = str:gsub("/+","/") + str = gsub(str,"/+","/") if find(str,"^/") then pth = "/" for s in gmatch(str,"[^/]+") do @@ -2422,12 +2912,12 @@ else --~ print(dir.mkdirs("///a/b/c")) --~ print(dir.mkdirs("a/bbb//ccc/")) - function dir.expand_name(str) + function dir.expand_name(str) -- will be merged with cleanpath and collapsepath if not find(str,"^/") then str = lfs.currentdir() .. "/" .. str end - str = str:gsub("//","/") - str = str:gsub("/%./","/") + str = gsub(str,"//","/") + str = gsub(str,"/%./","/") return str end @@ -2442,7 +2932,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-boolean'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -2503,19 +2993,40 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-unicode'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } +if not unicode then + + unicode = { utf8 = { } } + + local floor, char = math.floor, string.char + + function unicode.utf8.utfchar(n) + if n < 0x80 then + return char(n) + elseif n < 0x800 then + return char(0xC0 + floor(n/0x40)) .. char(0x80 + (n % 0x40)) + elseif n < 0x10000 then + return char(0xE0 + floor(n/0x1000)) .. char(0x80 + (floor(n/0x40) % 0x40)) .. char(0x80 + (n % 0x40)) + elseif n < 0x40000 then + return char(0xF0 + floor(n/0x40000)) .. char(0x80 + floor(n/0x1000)) .. char(0x80 + (floor(n/0x40) % 0x40)) .. char(0x80 + (n % 0x40)) + else -- wrong: + -- return char(0xF1 + floor(n/0x1000000)) .. char(0x80 + floor(n/0x40000)) .. char(0x80 + floor(n/0x1000)) .. char(0x80 + (floor(n/0x40) % 0x40)) .. char(0x80 + (n % 0x40)) + return "?" + end + end + +end + utf = utf or unicode.utf8 local concat, utfchar, utfgsub = table.concat, utf.char, utf.gsub local char, byte, find, bytepairs = string.char, string.byte, string.find, string.bytepairs -unicode = unicode or { } - -- 0 EF BB BF UTF-8 -- 1 FF FE UTF-16-little-endian -- 2 FE FF UTF-16-big-endian @@ -2530,14 +3041,20 @@ unicode.utfname = { [4] = 'utf-32-be' } -function unicode.utftype(f) -- \000 fails ! +-- \000 fails in <= 5.0 but is valid in >=5.1 where %z is depricated + +function unicode.utftype(f) local str = f:read(4) if not str then f:seek('set') return 0 - elseif find(str,"^%z%z\254\255") then + -- elseif find(str,"^%z%z\254\255") then -- depricated + -- elseif find(str,"^\000\000\254\255") then -- not permitted and bugged + elseif find(str,"\000\000\254\255",1,true) then -- seems to work okay (TH) return 4 - elseif find(str,"^\255\254%z%z") then + -- elseif find(str,"^\255\254%z%z") then -- depricated + -- elseif find(str,"^\255\254\000\000") then -- not permitted and bugged + elseif find(str,"\255\254\000\000",1,true) then -- seems to work okay (TH) return 3 elseif find(str,"^\254\255") then f:seek('set',2) @@ -2681,7 +3198,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-math'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -2728,7 +3245,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-utils'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -2736,6 +3253,10 @@ if not modules then modules = { } end modules ['l-utils'] = { -- hm, quite unreadable +local gsub = string.gsub +local concat = table.concat +local type, next = type, next + if not utils then utils = { } end if not utils.merger then utils.merger = { } end if not utils.lua then utils.lua = { } end @@ -2773,7 +3294,7 @@ function utils.merger._self_load_(name) end if data and utils.merger.strip_comment then -- saves some 20K - data = data:gsub("%-%-~[^\n\r]*[\r\n]", "") + data = gsub(data,"%-%-~[^\n\r]*[\r\n]", "") end return data or "" end @@ -2791,7 +3312,7 @@ end function utils.merger._self_swap_(data,code) if data ~= "" then - return (data:gsub(utils.merger.pattern, function(s) + return (gsub(data,utils.merger.pattern, function(s) return "\n\n" .. "-- "..utils.merger.m_begin .. "\n" .. code .. "\n" .. "-- "..utils.merger.m_end .. "\n\n" end, 1)) else @@ -2801,8 +3322,8 @@ end --~ stripper: --~ ---~ data = string.gsub(data,"%-%-~[^\n]*\n","") ---~ data = string.gsub(data,"\n\n+","\n") +--~ data = gsub(data,"%-%-~[^\n]*\n","") +--~ data = gsub(data,"\n\n+","\n") function utils.merger._self_libs_(libs,list) local result, f, frozen = { }, nil, false @@ -2810,9 +3331,10 @@ function utils.merger._self_libs_(libs,list) if type(libs) == 'string' then libs = { libs } end if type(list) == 'string' then list = { list } end local foundpath = nil - for _, lib in ipairs(libs) do - for _, pth in ipairs(list) do - pth = string.gsub(pth,"\\","/") -- file.clean_path + for i=1,#libs do + local lib = libs[i] + for j=1,#list do + local pth = gsub(list[j],"\\","/") -- file.clean_path utils.report("checking library path %s",pth) local name = pth .. "/" .. lib if lfs.isfile(name) then @@ -2824,7 +3346,8 @@ function utils.merger._self_libs_(libs,list) if foundpath then utils.report("using library path %s",foundpath) local right, wrong = { }, { } - for _, lib in ipairs(libs) do + for i=1,#libs do + local lib = libs[i] local fullname = foundpath .. "/" .. lib if lfs.isfile(fullname) then -- right[#right+1] = lib @@ -2838,15 +3361,15 @@ function utils.merger._self_libs_(libs,list) end end if #right > 0 then - utils.report("merged libraries: %s",table.concat(right," ")) + utils.report("merged libraries: %s",concat(right," ")) end if #wrong > 0 then - utils.report("skipped libraries: %s",table.concat(wrong," ")) + utils.report("skipped libraries: %s",concat(wrong," ")) end else utils.report("no valid library path found") end - return table.concat(result, "\n\n") + return concat(result, "\n\n") end function utils.merger.selfcreate(libs,list,target) @@ -2904,16 +3427,28 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-aux'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } +-- for inline, no store split : for s in string.gmatch(str,",* *([^,]+)") do .. end + aux = aux or { } local concat, format, gmatch = table.concat, string.format, string.gmatch local tostring, type = tostring, type +local lpegmatch = lpeg.match + +local P, R, V = lpeg.P, lpeg.R, lpeg.V + +local escape, left, right = P("\\"), P('{'), P('}') + +lpeg.patterns.balanced = P { + [1] = ((escape * (left+right)) + (1 - (left+right)) + V(2))^0, + [2] = left * V(1) * right +} local space = lpeg.P(' ') local equal = lpeg.P("=") @@ -2921,7 +3456,7 @@ local comma = lpeg.P(",") local lbrace = lpeg.P("{") local rbrace = lpeg.P("}") local nobrace = 1 - (lbrace+rbrace) -local nested = lpeg.P{ lbrace * (nobrace + lpeg.V(1))^0 * rbrace } +local nested = lpeg.P { lbrace * (nobrace + lpeg.V(1))^0 * rbrace } local spaces = space^0 local value = lpeg.P(lbrace * lpeg.C((nobrace + nested)^0) * rbrace) + lpeg.C((nested + (1-comma))^0) @@ -2959,13 +3494,13 @@ function aux.make_settings_to_hash_pattern(set,how) end end -function aux.settings_to_hash(str) +function aux.settings_to_hash(str,existing) if str and str ~= "" then - hash = { } + hash = existing or { } if moretolerant then - pattern_b_s:match(str) + lpegmatch(pattern_b_s,str) else - pattern_a_s:match(str) + lpegmatch(pattern_a_s,str) end return hash else @@ -2973,39 +3508,41 @@ function aux.settings_to_hash(str) end end -function aux.settings_to_hash_tolerant(str) +function aux.settings_to_hash_tolerant(str,existing) if str and str ~= "" then - hash = { } - pattern_b_s:match(str) + hash = existing or { } + lpegmatch(pattern_b_s,str) return hash else return { } end end -function aux.settings_to_hash_strict(str) +function aux.settings_to_hash_strict(str,existing) if str and str ~= "" then - hash = { } - pattern_c_s:match(str) + hash = existing or { } + lpegmatch(pattern_c_s,str) return next(hash) and hash else return nil end end -local seperator = comma * space^0 +local separator = comma * space^0 local value = lpeg.P(lbrace * lpeg.C((nobrace + nested)^0) * rbrace) + lpeg.C((nested + (1-comma))^0) -local pattern = lpeg.Ct(value*(seperator*value)^0) +local pattern = lpeg.Ct(value*(separator*value)^0) -- "aap, {noot}, mies" : outer {} removes, leading spaces ignored aux.settings_to_array_pattern = pattern +-- we could use a weak table as cache + function aux.settings_to_array(str) if not str or str == "" then return { } else - return pattern:match(str) + return lpegmatch(pattern,str) end end @@ -3014,10 +3551,10 @@ local function set(t,v) end local value = lpeg.P(lpeg.Carg(1)*value) / set -local pattern = value*(seperator*value)^0 * lpeg.Carg(1) +local pattern = value*(separator*value)^0 * lpeg.Carg(1) function aux.add_settings_to_array(t,str) - return pattern:match(str, nil, t) + return lpegmatch(pattern,str,nil,t) end function aux.hash_to_string(h,separator,yes,no,strict,omit) @@ -3065,6 +3602,13 @@ function aux.settings_to_set(str,t) return t end +local value = lbrace * lpeg.C((nobrace + nested)^0) * rbrace +local pattern = lpeg.Ct((space + value)^0) + +function aux.arguments_to_table(str) + return lpegmatch(pattern,str) +end + -- temporary here function aux.getparameters(self,class,parentclass,settings) @@ -3073,36 +3617,31 @@ function aux.getparameters(self,class,parentclass,settings) sc = table.clone(self[parent]) self[class] = sc end - aux.add_settings_to_array(sc, settings) + aux.settings_to_hash(settings,sc) end -- temporary here -local digit = lpeg.R("09") -local period = lpeg.P(".") -local zero = lpeg.P("0") - ---~ local finish = lpeg.P(-1) ---~ local nodigit = (1-digit) + finish ---~ local case_1 = (period * zero^1 * #nodigit)/"" -- .000 ---~ local case_2 = (period * (1-(zero^0/"") * #nodigit)^1 * (zero^0/"") * nodigit) -- .010 .10 .100100 - +local digit = lpeg.R("09") +local period = lpeg.P(".") +local zero = lpeg.P("0") local trailingzeros = zero^0 * -digit -- suggested by Roberto R -local case_1 = period * trailingzeros / "" -local case_2 = period * (digit - trailingzeros)^1 * (trailingzeros / "") - -local number = digit^1 * (case_1 + case_2) -local stripper = lpeg.Cs((number + 1)^0) +local case_1 = period * trailingzeros / "" +local case_2 = period * (digit - trailingzeros)^1 * (trailingzeros / "") +local number = digit^1 * (case_1 + case_2) +local stripper = lpeg.Cs((number + 1)^0) --~ local sample = "bla 11.00 bla 11 bla 0.1100 bla 1.00100 bla 0.00 bla 0.001 bla 1.1100 bla 0.100100100 bla 0.00100100100" --~ collectgarbage("collect") --~ str = string.rep(sample,10000) --~ local ts = os.clock() ---~ stripper:match(str) ---~ print(#str, os.clock()-ts, stripper:match(sample)) +--~ lpegmatch(stripper,str) +--~ print(#str, os.clock()-ts, lpegmatch(stripper,sample)) + +lpeg.patterns.strip_zeros = stripper function aux.strip_zeros(str) - return stripper:match(str) + return lpegmatch(stripper,str) end function aux.definetable(target) -- defines undefined tables @@ -3126,6 +3665,24 @@ function aux.accesstable(target) return t end +-- as we use this a lot ... + +--~ function aux.cachefunction(action,weak) +--~ local cache = { } +--~ if weak then +--~ setmetatable(cache, { __mode = "kv" } ) +--~ end +--~ local function reminder(str) +--~ local found = cache[str] +--~ if not found then +--~ found = action(str) +--~ cache[str] = found +--~ end +--~ return found +--~ end +--~ return reminder, cache +--~ end + end -- of closure @@ -3133,7 +3690,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['trac-tra'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to trac-tra.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -3143,12 +3700,17 @@ if not modules then modules = { } end modules ['trac-tra'] = { -- bound to a variable, like node.new, node.copy etc (contrary to for instance -- node.has_attribute which is bound to a has_attribute local variable in mkiv) +local debug = require "debug" + +local getinfo = debug.getinfo +local type, next = type, next +local concat = table.concat +local format, find, lower, gmatch, gsub = string.format, string.find, string.lower, string.gmatch, string.gsub + debugger = debugger or { } local counters = { } local names = { } -local getinfo = debug.getinfo -local format, find, lower, gmatch = string.format, string.find, string.lower, string.gmatch -- one @@ -3187,10 +3749,10 @@ function debugger.showstats(printer,threshold) local total, grandtotal, functions = 0, 0, 0 printer("\n") -- ugly but ok -- table.sort(counters) - for func, count in pairs(counters) do + for func, count in next, counters do if count > threshold then local name = getname(func) - if not name:find("for generator") then + if not find(name,"for generator") then printer(format("%8i %s", count, name)) total = total + count end @@ -3222,7 +3784,7 @@ end --~ local total, grandtotal, functions = 0, 0, 0 --~ printer("\n") -- ugly but ok --~ -- table.sort(counters) ---~ for func, count in pairs(counters) do +--~ for func, count in next, counters do --~ if count > threshold then --~ printer(format("%8i %s", count, func)) --~ total = total + count @@ -3276,38 +3838,77 @@ end --~ print("") --~ debugger.showstats(print,3) -trackers = trackers or { } +setters = setters or { } +setters.data = setters.data or { } -local data, done = { }, { } +--~ local function set(t,what,value) +--~ local data, done = t.data, t.done +--~ if type(what) == "string" then +--~ what = aux.settings_to_array(what) -- inefficient but ok +--~ end +--~ for i=1,#what do +--~ local w = what[i] +--~ for d, f in next, data do +--~ if done[d] then +--~ -- prevent recursion due to wildcards +--~ elseif find(d,w) then +--~ done[d] = true +--~ for i=1,#f do +--~ f[i](value) +--~ end +--~ end +--~ end +--~ end +--~ end -local function set(what,value) +local function set(t,what,value) + local data, done = t.data, t.done if type(what) == "string" then - what = aux.settings_to_array(what) + what = aux.settings_to_hash(what) -- inefficient but ok end - for i=1,#what do - local w = what[i] + for w, v in next, what do + if v == "" then + v = value + else + v = toboolean(v) + end for d, f in next, data do if done[d] then -- prevent recursion due to wildcards elseif find(d,w) then done[d] = true for i=1,#f do - f[i](value) + f[i](v) end end end end end -local function reset() - for d, f in next, data do +local function reset(t) + for d, f in next, t.data do for i=1,#f do f[i](false) end end end -function trackers.register(what,...) +local function enable(t,what) + set(t,what,true) +end + +local function disable(t,what) + local data = t.data + if not what or what == "" then + t.done = { } + reset(t) + else + set(t,what,false) + end +end + +function setters.register(t,what,...) + local data = t.data what = lower(what) local w = data[what] if not w then @@ -3319,32 +3920,32 @@ function trackers.register(what,...) if typ == "function" then w[#w+1] = fnc elseif typ == "string" then - w[#w+1] = function(value) set(fnc,value,nesting) end + w[#w+1] = function(value) set(t,fnc,value,nesting) end end end end -function trackers.enable(what) - done = { } - set(what,true) +function setters.enable(t,what) + local e = t.enable + t.enable, t.done = enable, { } + enable(t,string.simpleesc(tostring(what))) + t.enable, t.done = e, { } end -function trackers.disable(what) - done = { } - if not what or what == "" then - trackers.reset(what) - else - set(what,false) - end +function setters.disable(t,what) + local e = t.disable + t.disable, t.done = disable, { } + disable(t,string.simpleesc(tostring(what))) + t.disable, t.done = e, { } end -function trackers.reset(what) - done = { } - reset() +function setters.reset(t) + t.done = { } + reset(t) end -function trackers.list() -- pattern - local list = table.sortedkeys(data) +function setters.list(t) -- pattern + local list = table.sortedkeys(t.data) local user, system = { }, { } for l=1,#list do local what = list[l] @@ -3357,6 +3958,78 @@ function trackers.list() -- pattern return user, system end +function setters.show(t) + commands.writestatus("","") + local list = setters.list(t) + for k=1,#list do + commands.writestatus(t.name,list[k]) + end + commands.writestatus("","") +end + +-- we could have used a bit of oo and the trackers:enable syntax but +-- there is already a lot of code around using the singular tracker + +-- we could make this into a module + +function setters.new(name) + local t + t = { + data = { }, + name = name, + enable = function(...) setters.enable (t,...) end, + disable = function(...) setters.disable (t,...) end, + register = function(...) setters.register(t,...) end, + list = function(...) setters.list (t,...) end, + show = function(...) setters.show (t,...) end, + } + setters.data[name] = t + return t +end + +trackers = setters.new("trackers") +directives = setters.new("directives") +experiments = setters.new("experiments") + +-- nice trick: we overload two of the directives related functions with variants that +-- do tracing (itself using a tracker) .. proof of concept + +local trace_directives = false local trace_directives = false trackers.register("system.directives", function(v) trace_directives = v end) +local trace_experiments = false local trace_experiments = false trackers.register("system.experiments", function(v) trace_experiments = v end) + +local e = directives.enable +local d = directives.disable + +function directives.enable(...) + commands.writestatus("directives","enabling: %s",concat({...}," ")) + e(...) +end + +function directives.disable(...) + commands.writestatus("directives","disabling: %s",concat({...}," ")) + d(...) +end + +local e = experiments.enable +local d = experiments.disable + +function experiments.enable(...) + commands.writestatus("experiments","enabling: %s",concat({...}," ")) + e(...) +end + +function experiments.disable(...) + commands.writestatus("experiments","disabling: %s",concat({...}," ")) + d(...) +end + +-- a useful example + +directives.register("system.nostatistics", function(v) + statistics.enable = not v +end) + + end -- of closure @@ -3364,7 +4037,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['luat-env'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -3376,10 +4049,10 @@ if not modules then modules = { } end modules ['luat-env'] = { -- evolved before bytecode arrays were available and so a lot of -- code has disappeared already. -local trace_verbose = false trackers.register("resolvers.verbose", function(v) trace_verbose = v end) -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v trackers.enable("resolvers.verbose") end) +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) -local format = string.format +local format, sub, match, gsub, find = string.format, string.sub, string.match, string.gsub, string.find +local unquote, quote = string.unquote, string.quote -- precautions @@ -3413,13 +4086,14 @@ if not environment.jobname then environ function environment.initialize_arguments(arg) local arguments, files = { }, { } environment.arguments, environment.files, environment.sortedflags = arguments, files, nil - for index, argument in pairs(arg) do + for index=1,#arg do + local argument = arg[index] if index > 0 then - local flag, value = argument:match("^%-+(.+)=(.-)$") + local flag, value = match(argument,"^%-+(.-)=(.-)$") if flag then - arguments[flag] = string.unquote(value or "") + arguments[flag] = unquote(value or "") else - flag = argument:match("^%-+(.+)") + flag = match(argument,"^%-+(.+)") if flag then arguments[flag] = true else @@ -3446,25 +4120,30 @@ function environment.argument(name,partial) return arguments[name] elseif partial then if not sortedflags then - sortedflags = { } - for _,v in pairs(table.sortedkeys(arguments)) do - sortedflags[#sortedflags+1] = "^" .. v + sortedflags = table.sortedkeys(arguments) + for k=1,#sortedflags do + sortedflags[k] = "^" .. sortedflags[k] end environment.sortedflags = sortedflags end -- example of potential clash: ^mode ^modefile - for _,v in ipairs(sortedflags) do - if name:find(v) then - return arguments[v:sub(2,#v)] + for k=1,#sortedflags do + local v = sortedflags[k] + if find(name,v) then + return arguments[sub(v,2,#v)] end end end return nil end +environment.argument("x",true) + function environment.split_arguments(separator) -- rather special, cut-off before separator local done, before, after = false, { }, { } - for _,v in ipairs(environment.original_arguments) do + local original_arguments = environment.original_arguments + for k=1,#original_arguments do + local v = original_arguments[k] if not done and v == separator then done = true elseif done then @@ -3481,16 +4160,17 @@ function environment.reconstruct_commandline(arg,noquote) if noquote and #arg == 1 then local a = arg[1] a = resolvers.resolve(a) - a = a:unquote() + a = unquote(a) return a - elseif next(arg) then + elseif #arg > 0 then local result = { } - for _,a in ipairs(arg) do -- ipairs 1 .. #n + for i=1,#arg do + local a = arg[i] a = resolvers.resolve(a) - a = a:unquote() - a = a:gsub('"','\\"') -- tricky - if a:find(" ") then - result[#result+1] = a:quote() + a = unquote(a) + a = gsub(a,'"','\\"') -- tricky + if find(a," ") then + result[#result+1] = quote(a) else result[#result+1] = a end @@ -3503,17 +4183,18 @@ end if arg then - -- new, reconstruct quoted snippets (maybe better just remnove the " then and add them later) + -- new, reconstruct quoted snippets (maybe better just remove the " then and add them later) local newarg, instring = { }, false - for index, argument in ipairs(arg) do - if argument:find("^\"") then - newarg[#newarg+1] = argument:gsub("^\"","") - if not argument:find("\"$") then + for index=1,#arg do + local argument = arg[index] + if find(argument,"^\"") then + newarg[#newarg+1] = gsub(argument,"^\"","") + if not find(argument,"\"$") then instring = true end - elseif argument:find("\"$") then - newarg[#newarg] = newarg[#newarg] .. " " .. argument:gsub("\"$","") + elseif find(argument,"\"$") then + newarg[#newarg] = newarg[#newarg] .. " " .. gsub(argument,"\"$","") instring = false elseif instring then newarg[#newarg] = newarg[#newarg] .. " " .. argument @@ -3568,12 +4249,12 @@ function environment.luafilechunk(filename) -- used for loading lua bytecode in filename = file.replacesuffix(filename, "lua") local fullname = environment.luafile(filename) if fullname and fullname ~= "" then - if trace_verbose then + if trace_locating then logs.report("fileio","loading file %s", fullname) end return environment.loadedluacode(fullname) else - if trace_verbose then + if trace_locating then logs.report("fileio","unknown file %s", filename) end return nil @@ -3593,7 +4274,7 @@ function environment.loadluafile(filename, version) -- when not overloaded by explicit suffix we look for a luc file first local fullname = (lucname and environment.luafile(lucname)) or "" if fullname ~= "" then - if trace_verbose then + if trace_locating then logs.report("fileio","loading %s", fullname) end chunk = loadfile(fullname) -- this way we don't need a file exists check @@ -3611,7 +4292,7 @@ function environment.loadluafile(filename, version) if v == version then return true else - if trace_verbose then + if trace_locating then logs.report("fileio","version mismatch for %s: lua=%s, luc=%s", filename, v, version) end environment.loadluafile(filename) @@ -3622,12 +4303,12 @@ function environment.loadluafile(filename, version) end fullname = (luaname and environment.luafile(luaname)) or "" if fullname ~= "" then - if trace_verbose then + if trace_locating then logs.report("fileio","loading %s", fullname) end chunk = loadfile(fullname) -- this way we don't need a file exists check if not chunk then - if verbose then + if trace_locating then logs.report("fileio","unknown file %s", filename) end else @@ -3645,7 +4326,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['trac-inf'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to trac-inf.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -3670,6 +4351,14 @@ function statistics.hastimer(instance) return instance and instance.starttime end +function statistics.resettiming(instance) + if not instance then + notimer = { timing = 0, loadtime = 0 } + else + instance.timing, instance.loadtime = 0, 0 + end +end + function statistics.starttiming(instance) if not instance then notimer = { } @@ -3684,6 +4373,8 @@ function statistics.starttiming(instance) if not instance.loadtime then instance.loadtime = 0 end + else +--~ logs.report("system","nested timing (%s)",tostring(instance)) end instance.timing = it + 1 end @@ -3729,6 +4420,12 @@ function statistics.elapsedindeed(instance) return t > statistics.threshold end +function statistics.elapsedseconds(instance,rest) -- returns nil if 0 seconds + if statistics.elapsedindeed(instance) then + return format("%s seconds %s", statistics.elapsedtime(instance),rest or "") + end +end + -- general function function statistics.register(tag,fnc) @@ -3807,14 +4504,32 @@ function statistics.timed(action,report) report("total runtime: %s",statistics.elapsedtime(timer)) end +-- where, not really the best spot for this: + +commands = commands or { } + +local timer + +function commands.resettimer() + statistics.resettiming(timer) + statistics.starttiming(timer) +end + +function commands.elapsedtime() + statistics.stoptiming(timer) + tex.sprint(statistics.elapsedtime(timer)) +end + +commands.resettimer() + end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['luat-log'] = { +if not modules then modules = { } end modules ['trac-log'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to trac-log.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -3822,7 +4537,11 @@ if not modules then modules = { } end modules ['luat-log'] = { -- this is old code that needs an overhaul -local write_nl, write, format = texio.write_nl or print, texio.write or io.write, string.format +--~ io.stdout:setvbuf("no") +--~ io.stderr:setvbuf("no") + +local write_nl, write = texio.write_nl or print, texio.write or io.write +local format, gmatch = string.format, string.gmatch local texcount = tex and tex.count if texlua then @@ -3903,25 +4622,48 @@ function logs.tex.line(fmt,...) -- new end end +--~ function logs.tex.start_page_number() +--~ local real, user, sub = texcount.realpageno, texcount.userpageno, texcount.subpageno +--~ if real > 0 then +--~ if user > 0 then +--~ if sub > 0 then +--~ write(format("[%s.%s.%s",real,user,sub)) +--~ else +--~ write(format("[%s.%s",real,user)) +--~ end +--~ else +--~ write(format("[%s",real)) +--~ end +--~ else +--~ write("[-") +--~ end +--~ end + +--~ function logs.tex.stop_page_number() +--~ write("]") +--~ end + +local real, user, sub + function logs.tex.start_page_number() - local real, user, sub = texcount.realpageno, texcount.userpageno, texcount.subpageno + real, user, sub = texcount.realpageno, texcount.userpageno, texcount.subpageno +end + +function logs.tex.stop_page_number() if real > 0 then if user > 0 then if sub > 0 then - write(format("[%s.%s.%s",real,user,sub)) + logs.report("pages", "flushing realpage %s, userpage %s, subpage %s",real,user,sub) else - write(format("[%s.%s",real,user)) + logs.report("pages", "flushing realpage %s, userpage %s",real,user) end else - write(format("[%s",real)) + logs.report("pages", "flushing realpage %s",real) end else - write("[-") + logs.report("pages", "flushing page") end -end - -function logs.tex.stop_page_number() - write("]") + io.flush() end logs.tex.report_job_stat = statistics.show_job_stat @@ -4021,7 +4763,7 @@ end function logs.setprogram(_name_,_banner_,_verbose_) name, banner = _name_, _banner_ if _verbose_ then - trackers.enable("resolvers.verbose") + trackers.enable("resolvers.locating") end logs.set_method("tex") logs.report = report -- also used in libraries @@ -4034,9 +4776,9 @@ end function logs.setverbose(what) if what then - trackers.enable("resolvers.verbose") + trackers.enable("resolvers.locating") else - trackers.disable("resolvers.verbose") + trackers.disable("resolvers.locating") end logs.verbose = what or false end @@ -4053,7 +4795,7 @@ logs.report = logs.tex.report logs.simple = logs.tex.report function logs.reportlines(str) -- todo: <lines></lines> - for line in str:gmatch("(.-)[\n\r]") do + for line in gmatch(str,"(.-)[\n\r]") do logs.report(line) end end @@ -4064,8 +4806,12 @@ end logs.simpleline = logs.reportline -function logs.help(message,option) +function logs.reportbanner() -- for scripts too logs.report(banner) +end + +function logs.help(message,option) + logs.reportbanner() logs.reportline() logs.reportlines(message) local moreinfo = logs.moreinfo or "" @@ -4097,6 +4843,11 @@ end --~ logs.system(syslogname,"context","test","fonts","font %s recached due to newer version (%s)","blabla","123") --~ end +function logs.fatal(where,...) + logs.report(where,"fatal error: %s, aborting now",format(...)) + os.exit() +end + end -- of closure @@ -4104,10 +4855,10 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-inp'] = { version = 1.001, + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files", - comment = "companion to luat-lib.tex", } -- After a few years using the code the large luat-inp.lua file @@ -4119,7 +4870,7 @@ if not modules then modules = { } end modules ['data-inp'] = { -- * some public auxiliary functions were made private -- -- TODO: os.getenv -> os.env[] --- TODO: instances.[hashes,cnffiles,configurations,522] -> ipairs (alles check, sneller) +-- TODO: instances.[hashes,cnffiles,configurations,522] -- TODO: check escaping in find etc, too much, too slow -- This lib is multi-purpose and can be loaded again later on so that @@ -4140,12 +4891,13 @@ if not modules then modules = { } end modules ['data-inp'] = { local format, gsub, find, lower, upper, match, gmatch = string.format, string.gsub, string.find, string.lower, string.upper, string.match, string.gmatch local concat, insert, sortedkeys = table.concat, table.insert, table.sortedkeys local next, type = next, type +local lpegmatch = lpeg.match -local trace_locating, trace_detail, trace_verbose = false, false, false +local trace_locating, trace_detail, trace_expansions = false, false, false -trackers.register("resolvers.verbose", function(v) trace_verbose = v end) -trackers.register("resolvers.locating", function(v) trace_locating = v trackers.enable("resolvers.verbose") end) -trackers.register("resolvers.detail", function(v) trace_detail = v trackers.enable("resolvers.verbose,resolvers.detail") end) +trackers.register("resolvers.locating", function(v) trace_locating = v end) +trackers.register("resolvers.details", function(v) trace_detail = v end) +trackers.register("resolvers.expansions", function(v) trace_expansions = v end) -- todo if not resolvers then resolvers = { @@ -4169,7 +4921,7 @@ resolvers.generators.notfound = { nil } resolvers.cacheversion = '1.0.1' resolvers.cnfname = 'texmf.cnf' resolvers.luaname = 'texmfcnf.lua' -resolvers.homedir = os.env[os.platform == "windows" and 'USERPROFILE'] or os.env['HOME'] or '~' +resolvers.homedir = os.env[os.type == "windows" and 'USERPROFILE'] or os.env['HOME'] or '~' resolvers.cnfdefault = '{$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c}' local dummy_path_expr = "^!*unset/*$" @@ -4211,8 +4963,8 @@ suffixes['lua'] = { 'lua', 'luc', 'tma', 'tmc' } alternatives['map files'] = 'map' alternatives['enc files'] = 'enc' -alternatives['cid files'] = 'cid' -alternatives['fea files'] = 'fea' +alternatives['cid maps'] = 'cid' -- great, why no cid files +alternatives['font feature files'] = 'fea' -- and fea files here alternatives['opentype fonts'] = 'otf' alternatives['truetype fonts'] = 'ttf' alternatives['truetype collections'] = 'ttc' @@ -4228,6 +4980,11 @@ formats ['sfd'] = 'SFDFONTS' suffixes ['sfd'] = { 'sfd' } alternatives['subfont definition files'] = 'sfd' +-- lib paths + +formats ['lib'] = 'CLUAINPUTS' -- new (needs checking) +suffixes['lib'] = (os.libsuffix and { os.libsuffix }) or { 'dll', 'so' } + -- In practice we will work within one tds tree, but i want to keep -- the option open to build tools that look at multiple trees, which is -- why we keep the tree specific data in a table. We used to pass the @@ -4350,8 +5107,10 @@ local function check_configuration() -- not yet ok, no time for debugging now -- bad luck end fix("LUAINPUTS" , ".;$TEXINPUTS;$TEXMFSCRIPTS") -- no progname, hm - fix("FONTFEATURES", ".;$TEXMF/fonts/fea//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") - fix("FONTCIDMAPS" , ".;$TEXMF/fonts/cid//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") + -- this will go away some day + fix("FONTFEATURES", ".;$TEXMF/fonts/{data,fea}//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") + fix("FONTCIDMAPS" , ".;$TEXMF/fonts/{data,cid}//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") + -- fix("LUATEXLIBS" , ".;$TEXMF/luatex/lua//") end @@ -4366,7 +5125,7 @@ function resolvers.settrace(n) -- no longer number but: 'locating' or 'detail' end end -resolvers.settrace(os.getenv("MTX.resolvers.TRACE") or os.getenv("MTX_INPUT_TRACE")) +resolvers.settrace(os.getenv("MTX_INPUT_TRACE")) function resolvers.osenv(key) local ie = instance.environment @@ -4454,37 +5213,43 @@ end -- work that well; the parsing is ok, but dealing with the resulting -- table is a pain because we need to work inside-out recursively +local function do_first(a,b) + local t = { } + for s in gmatch(b,"[^,]+") do t[#t+1] = a .. s end + return "{" .. concat(t,",") .. "}" +end + +local function do_second(a,b) + local t = { } + for s in gmatch(a,"[^,]+") do t[#t+1] = s .. b end + return "{" .. concat(t,",") .. "}" +end + +local function do_both(a,b) + local t = { } + for sa in gmatch(a,"[^,]+") do + for sb in gmatch(b,"[^,]+") do + t[#t+1] = sa .. sb + end + end + return "{" .. concat(t,",") .. "}" +end + +local function do_three(a,b,c) + return a .. b.. c +end + local function splitpathexpr(str, t, validate) -- no need for further optimization as it is only called a - -- few times, we can use lpeg for the sub; we could move - -- the local functions outside the body + -- few times, we can use lpeg for the sub + if trace_expansions then + logs.report("fileio","expanding variable '%s'",str) + end t = t or { } str = gsub(str,",}",",@}") str = gsub(str,"{,","{@,") -- str = "@" .. str .. "@" local ok, done - local function do_first(a,b) - local t = { } - for s in gmatch(b,"[^,]+") do t[#t+1] = a .. s end - return "{" .. concat(t,",") .. "}" - end - local function do_second(a,b) - local t = { } - for s in gmatch(a,"[^,]+") do t[#t+1] = s .. b end - return "{" .. concat(t,",") .. "}" - end - local function do_both(a,b) - local t = { } - for sa in gmatch(a,"[^,]+") do - for sb in gmatch(b,"[^,]+") do - t[#t+1] = sa .. sb - end - end - return "{" .. concat(t,",") .. "}" - end - local function do_three(a,b,c) - return a .. b.. c - end while true do done = false while true do @@ -4515,6 +5280,11 @@ local function splitpathexpr(str, t, validate) t[#t+1] = s end end + if trace_expansions then + for k=1,#t do + logs.report("fileio","% 4i: %s",k,t[k]) + end + end return t end @@ -4554,18 +5324,27 @@ end -- also we now follow the stupid route: if not set then just assume *one* -- cnf file under texmf (i.e. distribution) -resolvers.ownpath = resolvers.ownpath or nil -resolvers.ownbin = resolvers.ownbin or arg[-2] or arg[-1] or arg[0] or "luatex" -resolvers.autoselfdir = true -- false may be handy for debugging +local args = environment and environment.original_arguments or arg -- this needs a cleanup + +resolvers.ownbin = resolvers.ownbin or args[-2] or arg[-2] or args[-1] or arg[-1] or arg[0] or "luatex" +resolvers.ownbin = gsub(resolvers.ownbin,"\\","/") function resolvers.getownpath() - if not resolvers.ownpath then - if resolvers.autoselfdir and os.selfdir then - resolvers.ownpath = os.selfdir - else - local binary = resolvers.ownbin - if os.platform == "windows" then - binary = file.replacesuffix(binary,"exe") + local ownpath = resolvers.ownpath or os.selfdir + if not ownpath or ownpath == "" or ownpath == "unset" then + ownpath = args[-1] or arg[-1] + ownpath = ownpath and file.dirname(gsub(ownpath,"\\","/")) + if not ownpath or ownpath == "" then + ownpath = args[-0] or arg[-0] + ownpath = ownpath and file.dirname(gsub(ownpath,"\\","/")) + end + local binary = resolvers.ownbin + if not ownpath or ownpath == "" then + ownpath = ownpath and file.dirname(binary) + end + if not ownpath or ownpath == "" then + if os.binsuffix ~= "" then + binary = file.replacesuffix(binary,os.binsuffix) end for p in gmatch(os.getenv("PATH"),"[^"..io.pathseparator.."]+") do local b = file.join(p,binary) @@ -4577,30 +5356,39 @@ function resolvers.getownpath() local olddir = lfs.currentdir() if lfs.chdir(p) then local pp = lfs.currentdir() - if trace_verbose and p ~= pp then - logs.report("fileio","following symlink %s to %s",p,pp) + if trace_locating and p ~= pp then + logs.report("fileio","following symlink '%s' to '%s'",p,pp) end - resolvers.ownpath = pp + ownpath = pp lfs.chdir(olddir) else - if trace_verbose then - logs.report("fileio","unable to check path %s",p) + if trace_locating then + logs.report("fileio","unable to check path '%s'",p) end - resolvers.ownpath = p + ownpath = p end break end end end - if not resolvers.ownpath then resolvers.ownpath = '.' end + if not ownpath or ownpath == "" then + ownpath = "." + logs.report("fileio","forcing fallback ownpath .") + elseif trace_locating then + logs.report("fileio","using ownpath '%s'",ownpath) + end end - return resolvers.ownpath + resolvers.ownpath = ownpath + function resolvers.getownpath() + return resolvers.ownpath + end + return ownpath end local own_places = { "SELFAUTOLOC", "SELFAUTODIR", "SELFAUTOPARENT", "TEXMFCNF" } local function identify_own() - local ownpath = resolvers.getownpath() or lfs.currentdir() + local ownpath = resolvers.getownpath() or dir.current() local ie = instance.environment if ownpath then if resolvers.env('SELFAUTOLOC') == "" then os.env['SELFAUTOLOC'] = file.collapse_path(ownpath) end @@ -4613,10 +5401,10 @@ local function identify_own() if resolvers.env('TEXMFCNF') == "" then os.env['TEXMFCNF'] = resolvers.cnfdefault end if resolvers.env('TEXOS') == "" then os.env['TEXOS'] = resolvers.env('SELFAUTODIR') end if resolvers.env('TEXROOT') == "" then os.env['TEXROOT'] = resolvers.env('SELFAUTOPARENT') end - if trace_verbose then + if trace_locating then for i=1,#own_places do local v = own_places[i] - logs.report("fileio","variable %s set to %s",v,resolvers.env(v) or "unknown") + logs.report("fileio","variable '%s' set to '%s'",v,resolvers.env(v) or "unknown") end end identify_own = function() end @@ -4648,10 +5436,8 @@ end local function load_cnf_file(fname) fname = resolvers.clean_path(fname) local lname = file.replacesuffix(fname,'lua') - local f = io.open(lname) - if f then -- this will go - f:close() - local dname = file.dirname(fname) + if lfs.isfile(lname) then + local dname = file.dirname(fname) -- fname ? if not instance.configuration[dname] then resolvers.load_data(dname,'configuration',lname and file.basename(lname)) instance.order[#instance.order+1] = instance.configuration[dname] @@ -4659,8 +5445,8 @@ local function load_cnf_file(fname) else f = io.open(fname) if f then - if trace_verbose then - logs.report("fileio","loading %s", fname) + if trace_locating then + logs.report("fileio","loading configuration file %s", fname) end local line, data, n, k, v local dname = file.dirname(fname) @@ -4694,14 +5480,16 @@ local function load_cnf_file(fname) end end f:close() - elseif trace_verbose then - logs.report("fileio","skipping %s", fname) + elseif trace_locating then + logs.report("fileio","skipping configuration file '%s'", fname) end end end local function collapse_cnf_data() -- potential optimization: pass start index (setup and configuration are shared) - for _,c in ipairs(instance.order) do + local order = instance.order + for i=1,#order do + local c = order[i] for k,v in next, c do if not instance.variables[k] then if instance.environment[k] then @@ -4717,19 +5505,24 @@ end function resolvers.load_cnf() local function loadoldconfigdata() - for _, fname in ipairs(instance.cnffiles) do - load_cnf_file(fname) + local cnffiles = instance.cnffiles + for i=1,#cnffiles do + load_cnf_file(cnffiles[i]) end end -- instance.cnffiles contain complete names now ! + -- we still use a funny mix of cnf and new but soon + -- we will switch to lua exclusively as we only use + -- the file to collect the tree roots if #instance.cnffiles == 0 then - if trace_verbose then + if trace_locating then logs.report("fileio","no cnf files found (TEXMFCNF may not be set/known)") end else - instance.rootpath = instance.cnffiles[1] - for k,fname in ipairs(instance.cnffiles) do - instance.cnffiles[k] = file.collapse_path(gsub(fname,"\\",'/')) + local cnffiles = instance.cnffiles + instance.rootpath = cnffiles[1] + for k=1,#cnffiles do + instance.cnffiles[k] = file.collapse_path(cnffiles[k]) end for i=1,3 do instance.rootpath = file.dirname(instance.rootpath) @@ -4757,8 +5550,9 @@ function resolvers.load_lua() -- yet harmless else instance.rootpath = instance.luafiles[1] - for k,fname in ipairs(instance.luafiles) do - instance.luafiles[k] = file.collapse_path(gsub(fname,"\\",'/')) + local luafiles = instance.luafiles + for k=1,#luafiles do + instance.luafiles[k] = file.collapse_path(luafiles[k]) end for i=1,3 do instance.rootpath = file.dirname(instance.rootpath) @@ -4790,14 +5584,14 @@ end function resolvers.append_hash(type,tag,name) if trace_locating then - logs.report("fileio","= hash append: %s",tag) + logs.report("fileio","hash '%s' appended",tag) end insert(instance.hashes, { ['type']=type, ['tag']=tag, ['name']=name } ) end function resolvers.prepend_hash(type,tag,name) if trace_locating then - logs.report("fileio","= hash prepend: %s",tag) + logs.report("fileio","hash '%s' prepended",tag) end insert(instance.hashes, 1, { ['type']=type, ['tag']=tag, ['name']=name } ) end @@ -4821,9 +5615,11 @@ end -- locators function resolvers.locatelists() - for _, path in ipairs(resolvers.clean_path_list('TEXMF')) do - if trace_verbose then - logs.report("fileio","locating list of %s",path) + local texmfpaths = resolvers.clean_path_list('TEXMF') + for i=1,#texmfpaths do + local path = texmfpaths[i] + if trace_locating then + logs.report("fileio","locating list of '%s'",path) end resolvers.locatedatabase(file.collapse_path(path)) end @@ -4836,11 +5632,11 @@ end function resolvers.locators.tex(specification) if specification and specification ~= '' and lfs.isdir(specification) then if trace_locating then - logs.report("fileio",'! tex locator found: %s',specification) + logs.report("fileio","tex locator '%s' found",specification) end resolvers.append_hash('file',specification,filename) elseif trace_locating then - logs.report("fileio",'? tex locator not found: %s',specification) + logs.report("fileio","tex locator '%s' not found",specification) end end @@ -4854,7 +5650,9 @@ function resolvers.loadfiles() instance.loaderror = false instance.files = { } if not instance.renewcache then - for _, hash in ipairs(instance.hashes) do + local hashes = instance.hashes + for k=1,#hashes do + local hash = hashes[k] resolvers.hashdatabase(hash.tag,hash.name) if instance.loaderror then break end end @@ -4868,8 +5666,9 @@ end -- generators: function resolvers.loadlists() - for _, hash in ipairs(instance.hashes) do - resolvers.generatedatabase(hash.tag) + local hashes = instance.hashes + for i=1,#hashes do + resolvers.generatedatabase(hashes[i].tag) end end @@ -4881,10 +5680,27 @@ end local weird = lpeg.P(".")^1 + lpeg.anywhere(lpeg.S("~`!#$%^&*()={}[]:;\"\'||<>,?\n\r\t")) +--~ local l_forbidden = lpeg.S("~`!#$%^&*()={}[]:;\"\'||\\/<>,?\n\r\t") +--~ local l_confusing = lpeg.P(" ") +--~ local l_character = lpeg.patterns.utf8 +--~ local l_dangerous = lpeg.P(".") + +--~ local l_normal = (l_character - l_forbidden - l_confusing - l_dangerous) * (l_character - l_forbidden - l_confusing^2)^0 * lpeg.P(-1) +--~ ----- l_normal = l_normal * lpeg.Cc(true) + lpeg.Cc(false) + +--~ local function test(str) +--~ print(str,lpeg.match(l_normal,str)) +--~ end +--~ test("ヒラギノ明朝 Pro W3") +--~ test("..ヒラギノ明朝 Pro W3") +--~ test(":ヒラギノ明朝 Pro W3;") +--~ test("ヒラギノ明朝 /Pro W3;") +--~ test("ヒラギノ明朝 Pro W3") + function resolvers.generators.tex(specification) local tag = specification - if trace_verbose then - logs.report("fileio","scanning path %s",specification) + if trace_locating then + logs.report("fileio","scanning path '%s'",specification) end instance.files[tag] = { } local files = instance.files[tag] @@ -4900,7 +5716,8 @@ function resolvers.generators.tex(specification) full = spec end for name in directory(full) do - if not weird:match(name) then + if not lpegmatch(weird,name) then + -- if lpegmatch(l_normal,name) then local mode = attributes(full..name,'mode') if mode == 'file' then if path then @@ -4933,7 +5750,7 @@ function resolvers.generators.tex(specification) end end action() - if trace_verbose then + if trace_locating then logs.report("fileio","%s files found on %s directories with %s uppercase remappings",n,m,r) end end @@ -4948,11 +5765,48 @@ end -- we join them and split them after the expansion has taken place. This -- is more convenient. +--~ local checkedsplit = string.checkedsplit + +local cache = { } + +local splitter = lpeg.Ct(lpeg.splitat(lpeg.S(os.type == "windows" and ";" or ":;"))) + +local function split_kpse_path(str) -- beware, this can be either a path or a {specification} + local found = cache[str] + if not found then + if str == "" then + found = { } + else + str = gsub(str,"\\","/") +--~ local split = (find(str,";") and checkedsplit(str,";")) or checkedsplit(str,io.pathseparator) +local split = lpegmatch(splitter,str) + found = { } + for i=1,#split do + local s = split[i] + if not find(s,"^{*unset}*") then + found[#found+1] = s + end + end + if trace_expansions then + logs.report("fileio","splitting path specification '%s'",str) + for k=1,#found do + logs.report("fileio","% 4i: %s",k,found[k]) + end + end + cache[str] = found + end + end + return found +end + +resolvers.split_kpse_path = split_kpse_path + function resolvers.splitconfig() - for i,c in ipairs(instance) do - for k,v in pairs(c) do + for i=1,#instance do + local c = instance[i] + for k,v in next, c do if type(v) == 'string' then - local t = file.split_path(v) + local t = split_kpse_path(v) if #t > 1 then c[k] = t end @@ -4962,21 +5816,25 @@ function resolvers.splitconfig() end function resolvers.joinconfig() - for i,c in ipairs(instance.order) do - for k,v in pairs(c) do -- ipairs? + local order = instance.order + for i=1,#order do + local c = order[i] + for k,v in next, c do -- indexed? if type(v) == 'table' then c[k] = file.join_path(v) end end end end + function resolvers.split_path(str) if type(str) == 'table' then return str else - return file.split_path(str) + return split_kpse_path(str) end end + function resolvers.join_path(str) if type(str) == 'table' then return file.join_path(str) @@ -4988,8 +5846,9 @@ end function resolvers.splitexpansions() local ie = instance.expansions for k,v in next, ie do - local t, h = { }, { } - for _,vv in ipairs(file.split_path(v)) do + local t, h, p = { }, { }, split_kpse_path(v) + for kk=1,#p do + local vv = p[kk] if vv ~= "" and not h[vv] then t[#t+1] = vv h[vv] = true @@ -5036,11 +5895,15 @@ function resolvers.serialize(files) end t[#t+1] = "return {" if instance.sortdata then - for _, k in pairs(sortedkeys(files)) do -- ipairs + local sortedfiles = sortedkeys(files) + for i=1,#sortedfiles do + local k = sortedfiles[i] local fk = files[k] if type(fk) == 'table' then t[#t+1] = "\t['" .. k .. "']={" - for _, kk in pairs(sortedkeys(fk)) do -- ipairs + local sortedfk = sortedkeys(fk) + for j=1,#sortedfk do + local kk = sortedfk[j] t[#t+1] = dump(kk,fk[kk],"\t\t") end t[#t+1] = "\t}," @@ -5065,12 +5928,18 @@ function resolvers.serialize(files) return concat(t,"\n") end +local data_state = { } + +function resolvers.data_state() + return data_state or { } +end + function resolvers.save_data(dataname, makename) -- untested without cache overload for cachename, files in next, instance[dataname] do local name = (makename or file.join)(cachename,dataname) local luaname, lucname = name .. ".lua", name .. ".luc" - if trace_verbose then - logs.report("fileio","preparing %s for %s",dataname,cachename) + if trace_locating then + logs.report("fileio","preparing '%s' for '%s'",dataname,cachename) end for k, v in next, files do if type(v) == "table" and #v == 1 then @@ -5084,24 +5953,25 @@ function resolvers.save_data(dataname, makename) -- untested without cache overl date = os.date("%Y-%m-%d"), time = os.date("%H:%M:%S"), content = files, + uuid = os.uuid(), } local ok = io.savedata(luaname,resolvers.serialize(data)) if ok then - if trace_verbose then - logs.report("fileio","%s saved in %s",dataname,luaname) + if trace_locating then + logs.report("fileio","'%s' saved in '%s'",dataname,luaname) end if utils.lua.compile(luaname,lucname,false,true) then -- no cleanup but strip - if trace_verbose then - logs.report("fileio","%s compiled to %s",dataname,lucname) + if trace_locating then + logs.report("fileio","'%s' compiled to '%s'",dataname,lucname) end else - if trace_verbose then - logs.report("fileio","compiling failed for %s, deleting file %s",dataname,lucname) + if trace_locating then + logs.report("fileio","compiling failed for '%s', deleting file '%s'",dataname,lucname) end os.remove(lucname) end - elseif trace_verbose then - logs.report("fileio","unable to save %s in %s (access error)",dataname,luaname) + elseif trace_locating then + logs.report("fileio","unable to save '%s' in '%s' (access error)",dataname,luaname) end end end @@ -5113,19 +5983,20 @@ function resolvers.load_data(pathname,dataname,filename,makename) -- untested wi if blob then local data = blob() if data and data.content and data.type == dataname and data.version == resolvers.cacheversion then - if trace_verbose then - logs.report("fileio","loading %s for %s from %s",dataname,pathname,filename) + data_state[#data_state+1] = data.uuid + if trace_locating then + logs.report("fileio","loading '%s' for '%s' from '%s'",dataname,pathname,filename) end instance[dataname][pathname] = data.content else - if trace_verbose then - logs.report("fileio","skipping %s for %s from %s",dataname,pathname,filename) + if trace_locating then + logs.report("fileio","skipping '%s' for '%s' from '%s'",dataname,pathname,filename) end instance[dataname][pathname] = { } instance.loaderror = true end - elseif trace_verbose then - logs.report("fileio","skipping %s for %s from %s",dataname,pathname,filename) + elseif trace_locating then + logs.report("fileio","skipping '%s' for '%s' from '%s'",dataname,pathname,filename) end end @@ -5144,15 +6015,17 @@ function resolvers.resetconfig() end function resolvers.loadnewconfig() - for _, cnf in ipairs(instance.luafiles) do + local luafiles = instance.luafiles + for i=1,#luafiles do + local cnf = luafiles[i] local pathname = file.dirname(cnf) local filename = file.join(pathname,resolvers.luaname) local blob = loadfile(filename) if blob then local data = blob() if data then - if trace_verbose then - logs.report("fileio","loading configuration file %s",filename) + if trace_locating then + logs.report("fileio","loading configuration file '%s'",filename) end if true then -- flatten to variable.progname @@ -5173,14 +6046,14 @@ function resolvers.loadnewconfig() instance['setup'][pathname] = data end else - if trace_verbose then - logs.report("fileio","skipping configuration file %s",filename) + if trace_locating then + logs.report("fileio","skipping configuration file '%s'",filename) end instance['setup'][pathname] = { } instance.loaderror = true end - elseif trace_verbose then - logs.report("fileio","skipping configuration file %s",filename) + elseif trace_locating then + logs.report("fileio","skipping configuration file '%s'",filename) end instance.order[#instance.order+1] = instance.setup[pathname] if instance.loaderror then break end @@ -5189,7 +6062,9 @@ end function resolvers.loadoldconfig() if not instance.renewcache then - for _, cnf in ipairs(instance.cnffiles) do + local cnffiles = instance.cnffiles + for i=1,#cnffiles do + local cnf = cnffiles[i] local dname = file.dirname(cnf) resolvers.load_data(dname,'configuration') instance.order[#instance.order+1] = instance.configuration[dname] @@ -5379,7 +6254,7 @@ end function resolvers.expanded_path_list(str) if not str then - return ep or { } + return ep or { } -- ep ? elseif instance.savelists then -- engine+progname hash str = gsub(str,"%$","") @@ -5397,9 +6272,9 @@ end function resolvers.expanded_path_list_from_var(str) -- brrr local tmp = resolvers.var_of_format_or_suffix(gsub(str,"%$","")) if tmp ~= "" then - return resolvers.expanded_path_list(str) - else return resolvers.expanded_path_list(tmp) + else + return resolvers.expanded_path_list(str) end end @@ -5446,9 +6321,9 @@ function resolvers.isreadable.file(name) local readable = lfs.isfile(name) -- brrr if trace_detail then if readable then - logs.report("fileio","+ readable: %s",name) + logs.report("fileio","file '%s' is readable",name) else - logs.report("fileio","- readable: %s", name) + logs.report("fileio","file '%s' is not readable", name) end end return readable @@ -5464,7 +6339,7 @@ local function collect_files(names) for k=1,#names do local fname = names[k] if trace_detail then - logs.report("fileio","? blobpath asked: %s",fname) + logs.report("fileio","checking name '%s'",fname) end local bname = file.basename(fname) local dname = file.dirname(fname) @@ -5480,7 +6355,7 @@ local function collect_files(names) local files = blobpath and instance.files[blobpath] if files then if trace_detail then - logs.report("fileio",'? blobpath do: %s (%s)',blobpath,bname) + logs.report("fileio","deep checking '%s' (%s)",blobpath,bname) end local blobfile = files[bname] if not blobfile then @@ -5514,7 +6389,7 @@ local function collect_files(names) end end elseif trace_locating then - logs.report("fileio",'! blobpath no: %s (%s)',blobpath,bname) + logs.report("fileio","no match in '%s' (%s)",blobpath,bname) end end end @@ -5564,14 +6439,13 @@ end local function collect_instance_files(filename,collected) -- todo : plugin (scanners, checkers etc) local result = collected or { } local stamp = nil - filename = file.collapse_path(filename) -- elsewhere - filename = file.collapse_path(gsub(filename,"\\","/")) -- elsewhere + filename = file.collapse_path(filename) -- speed up / beware: format problem if instance.remember then stamp = filename .. "--" .. instance.engine .. "--" .. instance.progname .. "--" .. instance.format if instance.found[stamp] then if trace_locating then - logs.report("fileio",'! remembered: %s',filename) + logs.report("fileio","remembering file '%s'",filename) end return instance.found[stamp] end @@ -5579,7 +6453,7 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan if not dangerous[instance.format or "?"] then if resolvers.isreadable.file(filename) then if trace_detail then - logs.report("fileio",'= found directly: %s',filename) + logs.report("fileio","file '%s' found directly",filename) end instance.found[stamp] = { filename } return { filename } @@ -5587,13 +6461,13 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan end if find(filename,'%*') then if trace_locating then - logs.report("fileio",'! wildcard: %s', filename) + logs.report("fileio","checking wildcard '%s'", filename) end result = resolvers.find_wildcard_files(filename) elseif file.is_qualified_path(filename) then if resolvers.isreadable.file(filename) then if trace_locating then - logs.report("fileio",'! qualified: %s', filename) + logs.report("fileio","qualified name '%s'", filename) end result = { filename } else @@ -5603,7 +6477,7 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan forcedname = filename .. ".tex" if resolvers.isreadable.file(forcedname) then if trace_locating then - logs.report("fileio",'! no suffix, forcing standard filetype: tex') + logs.report("fileio","no suffix, forcing standard filetype 'tex'") end result, ok = { forcedname }, true end @@ -5613,7 +6487,7 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan forcedname = filename .. "." .. s if resolvers.isreadable.file(forcedname) then if trace_locating then - logs.report("fileio",'! no suffix, forcing format filetype: %s', s) + logs.report("fileio","no suffix, forcing format filetype '%s'", s) end result, ok = { forcedname }, true break @@ -5625,7 +6499,7 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan -- try to find in tree (no suffix manipulation), here we search for the -- matching last part of the name local basename = file.basename(filename) - local pattern = (filename .. "$"):gsub("([%.%-])","%%%1") + local pattern = gsub(filename .. "$","([%.%-])","%%%1") local savedformat = instance.format local format = savedformat or "" if format == "" then @@ -5635,19 +6509,21 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan instance.format = "othertextfiles" -- kind of everything, maybe texinput is better end -- - local resolved = collect_instance_files(basename) - if #result == 0 then - local lowered = lower(basename) - if filename ~= lowered then - resolved = collect_instance_files(lowered) + if basename ~= filename then + local resolved = collect_instance_files(basename) + if #result == 0 then + local lowered = lower(basename) + if filename ~= lowered then + resolved = collect_instance_files(lowered) + end end - end - resolvers.format = savedformat - -- - for r=1,#resolved do - local rr = resolved[r] - if rr:find(pattern) then - result[#result+1], ok = rr, true + resolvers.format = savedformat + -- + for r=1,#resolved do + local rr = resolved[r] + if find(rr,pattern) then + result[#result+1], ok = rr, true + end end end -- a real wildcard: @@ -5656,14 +6532,14 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan -- local filelist = collect_files({basename}) -- for f=1,#filelist do -- local ff = filelist[f][3] or "" - -- if ff:find(pattern) then + -- if find(ff,pattern) then -- result[#result+1], ok = ff, true -- end -- end -- end end if not ok and trace_locating then - logs.report("fileio",'? qualified: %s', filename) + logs.report("fileio","qualified name '%s'", filename) end end else @@ -5682,12 +6558,12 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan wantedfiles[#wantedfiles+1] = forcedname filetype = resolvers.format_of_suffix(forcedname) if trace_locating then - logs.report("fileio",'! forcing filetype: %s',filetype) + logs.report("fileio","forcing filetype '%s'",filetype) end else filetype = resolvers.format_of_suffix(filename) if trace_locating then - logs.report("fileio",'! using suffix based filetype: %s',filetype) + logs.report("fileio","using suffix based filetype '%s'",filetype) end end else @@ -5699,7 +6575,7 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan end filetype = instance.format if trace_locating then - logs.report("fileio",'! using given filetype: %s',filetype) + logs.report("fileio","using given filetype '%s'",filetype) end end local typespec = resolvers.variable_of_format(filetype) @@ -5707,9 +6583,7 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan if not pathlist or #pathlist == 0 then -- no pathlist, access check only / todo == wildcard if trace_detail then - logs.report("fileio",'? filename: %s',filename) - logs.report("fileio",'? filetype: %s',filetype or '?') - logs.report("fileio",'? wanted files: %s',concat(wantedfiles," | ")) + logs.report("fileio","checking filename '%s', filetype '%s', wanted files '%s'",filename, filetype or '?',concat(wantedfiles," | ")) end for k=1,#wantedfiles do local fname = wantedfiles[k] @@ -5730,36 +6604,59 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan else -- list search local filelist = collect_files(wantedfiles) - local doscan, recurse + local dirlist = { } + if filelist then + for i=1,#filelist do + dirlist[i] = file.dirname(filelist[i][2]) .. "/" + end + end if trace_detail then - logs.report("fileio",'? filename: %s',filename) + logs.report("fileio","checking filename '%s'",filename) end -- a bit messy ... esp the doscan setting here + local doscan for k=1,#pathlist do local path = pathlist[k] if find(path,"^!!") then doscan = false else doscan = true end - if find(path,"//$") then recurse = true else recurse = false end local pathname = gsub(path,"^!+", '') done = false -- using file list - if filelist and not (done and not instance.allresults) and recurse then - -- compare list entries with permitted pattern - pathname = gsub(pathname,"([%-%.])","%%%1") -- this also influences - pathname = gsub(pathname,"/+$", '/.*') -- later usage of pathname - pathname = gsub(pathname,"//", '/.-/') -- not ok for /// but harmless - local expr = "^" .. pathname + if filelist then + local expression + -- compare list entries with permitted pattern -- /xx /xx// + if not find(pathname,"/$") then + expression = pathname .. "/" + else + expression = pathname + end + expression = gsub(expression,"([%-%.])","%%%1") -- this also influences + expression = gsub(expression,"//+$", '/.*') -- later usage of pathname + expression = gsub(expression,"//", '/.-/') -- not ok for /// but harmless + expression = "^" .. expression .. "$" + if trace_detail then + logs.report("fileio","using pattern '%s' for path '%s'",expression,pathname) + end for k=1,#filelist do local fl = filelist[k] local f = fl[2] - if find(f,expr) then - if trace_detail then - logs.report("fileio",'= found in hash: %s',f) - end + local d = dirlist[k] + if find(d,expression) then --- todo, test for readable result[#result+1] = fl[3] resolvers.register_in_trees(f) -- for tracing used files done = true - if not instance.allresults then break end + if instance.allresults then + if trace_detail then + logs.report("fileio","match in hash for file '%s' on path '%s', continue scanning",f,d) + end + else + if trace_detail then + logs.report("fileio","match in hash for file '%s' on path '%s', quit scanning",f,d) + end + break + end + elseif trace_detail then + logs.report("fileio","no match in hash for file '%s' on path '%s'",f,d) end end end @@ -5775,7 +6672,7 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan local fname = file.join(ppname,w) if resolvers.isreadable.file(fname) then if trace_detail then - logs.report("fileio",'= found by scanning: %s',fname) + logs.report("fileio","found '%s' by scanning",fname) end result[#result+1] = fname done = true @@ -5838,7 +6735,7 @@ function resolvers.find_given_files(filename) local hashes = instance.hashes for k=1,#hashes do local hash = hashes[k] - local files = instance.files[hash.tag] + local files = instance.files[hash.tag] or { } local blist = files[bname] if not blist then local rname = "remap:"..bname @@ -5948,9 +6845,9 @@ function resolvers.load(option) statistics.starttiming(instance) resolvers.resetconfig() resolvers.identify_cnf() - resolvers.load_lua() + resolvers.load_lua() -- will become the new method resolvers.expand_variables() - resolvers.load_cnf() + resolvers.load_cnf() -- will be skipped when we have a lua file resolvers.expand_variables() if option ~= "nofiles" then resolvers.load_hash() @@ -5962,22 +6859,23 @@ end function resolvers.for_files(command, files, filetype, mustexist) if files and #files > 0 then local function report(str) - if trace_verbose then + if trace_locating then logs.report("fileio",str) -- has already verbose else print(str) end end - if trace_verbose then - report('') + if trace_locating then + report('') -- ? end - for _, file in ipairs(files) do + for f=1,#files do + local file = files[f] local result = command(file,filetype,mustexist) if type(result) == 'string' then report(result) else - for _,v in ipairs(result) do - report(v) + for i=1,#result do + report(result[i]) -- could be unpack end end end @@ -6024,18 +6922,19 @@ end function table.sequenced(t,sep) -- temp here local s = { } - for k, v in pairs(t) do -- pairs? - s[#s+1] = k .. "=" .. v + for k, v in next, t do -- indexed? + s[#s+1] = k .. "=" .. tostring(v) end return concat(s, sep or " | ") end function resolvers.methodhandler(what, filename, filetype) -- ... + filename = file.collapse_path(filename) local specification = (type(filename) == "string" and resolvers.splitmethod(filename)) or filename -- no or { }, let it bomb local scheme = specification.scheme if resolvers[what][scheme] then if trace_locating then - logs.report("fileio",'= handler: %s -> %s -> %s',specification.original,what,table.sequenced(specification)) + logs.report("fileio","handler '%s' -> '%s' -> '%s'",specification.original,what,table.sequenced(specification)) end return resolvers[what][scheme](filename,filetype) -- todo: specification else @@ -6055,8 +6954,9 @@ function resolvers.clean_path(str) end function resolvers.do_with_path(name,func) - for _, v in pairs(resolvers.expanded_path_list(name)) do -- pairs? - func("^"..resolvers.clean_path(v)) + local pathlist = resolvers.expanded_path_list(name) + for i=1,#pathlist do + func("^"..resolvers.clean_path(pathlist[i])) end end @@ -6065,7 +6965,9 @@ function resolvers.do_with_var(name,func) end function resolvers.with_files(pattern,handle) - for _, hash in ipairs(instance.hashes) do + local hashes = instance.hashes + for i=1,#hashes do + local hash = hashes[i] local blobpath = hash.tag local blobtype = hash.type if blobpath then @@ -6080,7 +6982,7 @@ function resolvers.with_files(pattern,handle) if type(v) == "string" then handle(blobtype,blobpath,v,k) else - for _,vv in pairs(v) do -- ipairs? + for _,vv in next, v do -- indexed handle(blobtype,blobpath,vv,k) end end @@ -6092,7 +6994,7 @@ function resolvers.with_files(pattern,handle) end function resolvers.locate_format(name) - local barename, fmtname = name:gsub("%.%a+$",""), "" + local barename, fmtname = gsub(name,"%.%a+$",""), "" if resolvers.usecache then local path = file.join(caches.setpath("formats")) -- maybe platform fmtname = file.join(path,barename..".fmt") or "" @@ -6140,7 +7042,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-tmp'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -6164,7 +7066,7 @@ luatools with a recache feature.</p> local format, lower, gsub = string.format, string.lower, string.gsub -local trace_cache = false trackers.register("resolvers.cache", function(v) trace_cache = v end) +local trace_cache = false trackers.register("resolvers.cache", function(v) trace_cache = v end) -- not used yet caches = caches or { } @@ -6251,7 +7153,8 @@ function caches.setpath(...) caches.path = '.' end caches.path = resolvers.clean_path(caches.path) - if not table.is_empty({...}) then + local dirs = { ... } + if #dirs > 0 then local pth = dir.mkdirs(caches.path,...) return pth end @@ -6297,6 +7200,7 @@ function caches.savedata(filepath,filename,data,raw) if raw then reduce, simplify = false, false end + data.cache_uuid = os.uuid() if caches.direct then file.savedata(tmaname, table.serialize(data,'return',false,true,false)) -- no hex else @@ -6322,7 +7226,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-inp'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -6343,7 +7247,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-out'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -6359,7 +7263,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-con'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -6370,8 +7274,6 @@ local format, lower, gsub = string.format, string.lower, string.gsub local trace_cache = false trackers.register("resolvers.cache", function(v) trace_cache = v end) local trace_containers = false trackers.register("resolvers.containers", function(v) trace_containers = v end) local trace_storage = false trackers.register("resolvers.storage", function(v) trace_storage = v end) -local trace_verbose = false trackers.register("resolvers.verbose", function(v) trace_verbose = v end) -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v trackers.enable("resolvers.verbose") end) --[[ldx-- <p>Once we found ourselves defining similar cache constructs @@ -6435,7 +7337,7 @@ end function containers.is_valid(container, name) if name and name ~= "" then local storage = container.storage[name] - return storage and not table.is_empty(storage) and storage.cache_version == container.version + return storage and storage.cache_version == container.version else return false end @@ -6487,16 +7389,15 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-use'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -local format, lower, gsub = string.format, string.lower, string.gsub +local format, lower, gsub, find = string.format, string.lower, string.gsub, string.find -local trace_verbose = false trackers.register("resolvers.verbose", function(v) trace_verbose = v end) -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v trackers.enable("resolvers.verbose") end) +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) -- since we want to use the cache instead of the tree, we will now -- reimplement the saver. @@ -6540,19 +7441,20 @@ resolvers.automounted = resolvers.automounted or { } function resolvers.automount(usecache) local mountpaths = resolvers.clean_path_list(resolvers.expansion('TEXMFMOUNT')) - if table.is_empty(mountpaths) and usecache then + if (not mountpaths or #mountpaths == 0) and usecache then mountpaths = { caches.setpath("mount") } end - if not table.is_empty(mountpaths) then + if mountpaths and #mountpaths > 0 then statistics.starttiming(resolvers.instance) - for k, root in pairs(mountpaths) do + for k=1,#mountpaths do + local root = mountpaths[k] local f = io.open(root.."/url.tmi") if f then for line in f:lines() do if line then - if line:find("^[%%#%-]") then -- or %W + if find(line,"^[%%#%-]") then -- or %W -- skip - elseif line:find("^zip://") then + elseif find(line,"^zip://") then if trace_locating then logs.report("fileio","mounting %s",line) end @@ -6597,11 +7499,13 @@ function statistics.check_fmt_status(texname) local luv = dofile(luvname) if luv and luv.sourcefile then local sourcehash = md5.hex(io.loaddata(resolvers.find_file(luv.sourcefile)) or "unknown") - if luv.enginebanner and luv.enginebanner ~= enginebanner then - return "engine mismatch" + local luvbanner = luv.enginebanner or "?" + if luvbanner ~= enginebanner then + return string.format("engine mismatch (luv:%s <> bin:%s)",luvbanner,enginebanner) end - if luv.sourcehash and luv.sourcehash ~= sourcehash then - return "source mismatch" + local luvhash = luv.sourcehash or "?" + if luvhash ~= sourcehash then + return string.format("source mismatch (luv:%s <> bin:%s)",luvhash,sourcehash) end else return "invalid status file" @@ -6727,7 +7631,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-aux'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -6735,47 +7639,47 @@ if not modules then modules = { } end modules ['data-aux'] = { local find = string.find -local trace_verbose = false trackers.register("resolvers.verbose", function(v) trace_verbose = v end) +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) function resolvers.update_script(oldname,newname) -- oldname -> own.name, not per se a suffix local scriptpath = "scripts/context/lua" newname = file.addsuffix(newname,"lua") local oldscript = resolvers.clean_path(oldname) - if trace_verbose then + if trace_locating then logs.report("fileio","to be replaced old script %s", oldscript) end local newscripts = resolvers.find_files(newname) or { } if #newscripts == 0 then - if trace_verbose then + if trace_locating then logs.report("fileio","unable to locate new script") end else for i=1,#newscripts do local newscript = resolvers.clean_path(newscripts[i]) - if trace_verbose then + if trace_locating then logs.report("fileio","checking new script %s", newscript) end if oldscript == newscript then - if trace_verbose then + if trace_locating then logs.report("fileio","old and new script are the same") end elseif not find(newscript,scriptpath) then - if trace_verbose then + if trace_locating then logs.report("fileio","new script should come from %s",scriptpath) end elseif not (find(oldscript,file.removesuffix(newname).."$") or find(oldscript,newname.."$")) then - if trace_verbose then + if trace_locating then logs.report("fileio","invalid new script name") end else local newdata = io.loaddata(newscript) if newdata then - if trace_verbose then + if trace_locating then logs.report("fileio","old script content replaced by new content") end io.savedata(oldscript,newdata) break - elseif trace_verbose then + elseif trace_locating then logs.report("fileio","unable to load new script") end end @@ -6790,7 +7694,7 @@ do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-lst'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -6814,7 +7718,9 @@ local function list(list,report) local instance = resolvers.instance local pat = upper(pattern or "","") local report = report or texio.write_nl - for _,key in pairs(table.sortedkeys(list)) do + local sorted = table.sortedkeys(list) + for i=1,#sorted do + local key = sorted[i] if instance.pattern == "" or find(upper(key),pat) then if instance.kpseonly then if instance.kpsevars[key] then @@ -6833,11 +7739,14 @@ function resolvers.listers.expansions() list(resolvers.instance.expansions) end function resolvers.listers.configurations(report) local report = report or texio.write_nl local instance = resolvers.instance - for _,key in ipairs(table.sortedkeys(instance.kpsevars)) do + local sorted = table.sortedkeys(instance.kpsevars) + for i=1,#sorted do + local key = sorted[i] if not instance.pattern or (instance.pattern=="") or find(key,instance.pattern) then report(format("%s\n",key)) - for i,c in ipairs(instance.order) do - local str = c[key] + local order = instance.order + for i=1,#order do + local str = order[i][key] if str then report(format("\t%s\t%s",i,str)) end @@ -6943,7 +7852,7 @@ if not resolvers then os.exit() end -logs.setprogram('LuaTools',"TDS Management Tool 1.31",environment.arguments["verbose"] or false) +logs.setprogram('LuaTools',"TDS Management Tool 1.32",environment.arguments["verbose"] or false) local instance = resolvers.reset() @@ -7000,6 +7909,12 @@ end if environment.arguments["trace"] then resolvers.settrace(environment.arguments["trace"]) end +local trackspec = environment.argument("trackers") or environment.argument("track") + +if trackspec then + trackers.enable(trackspec) +end + runners = runners or { } messages = messages or { } @@ -7033,6 +7948,7 @@ messages.help = [[ --engine=str target engine --progname=str format or backend --pattern=str filter variables +--trackers=list enable given trackers ]] function runners.make_format(texname) @@ -7091,8 +8007,9 @@ function runners.make_format(texname) logs.simple("using uncompiled initialization file: %s",luaname) end else - for _, v in pairs({instance.luaname, instance.progname, barename}) do - v = string.gsub(v..".lua","%.lua%.lua$",".lua") + local what = { instance.luaname, instance.progname, barename } + for k=1,#what do + local v = string.gsub(what[k]..".lua","%.lua%.lua$",".lua") if v and (v ~= "") then luaname = resolvers.find_files(v)[1] or "" if luaname ~= "" then @@ -7116,7 +8033,8 @@ function runners.make_format(texname) logs.simple("using lua initialization file: %s",luaname) local mp = dir.glob(file.removesuffix(file.basename(luaname)).."-*.mem") if mp and #mp > 0 then - for _, name in ipairs(mp) do + for i=1,#mp do + local name = mp[i] logs.simple("removing related mplib format %s", file.basename(name)) os.remove(name) end |