diff options
Diffstat (limited to 'Master/bin/i386-linux/mtxrun')
| -rwxr-xr-x | Master/bin/i386-linux/mtxrun | 12366 |
1 files changed, 7479 insertions, 4887 deletions
diff --git a/Master/bin/i386-linux/mtxrun b/Master/bin/i386-linux/mtxrun index 2bdd1b8f020..cc5ebb478ac 100755 --- a/Master/bin/i386-linux/mtxrun +++ b/Master/bin/i386-linux/mtxrun @@ -8,7 +8,6 @@ if not modules then modules = { } end modules ['mtxrun'] = { license = "see context related readme files" } - -- one can make a stub: -- -- #!/bin/sh @@ -38,8 +37,6 @@ if not modules then modules = { } end modules ['mtxrun'] = { -- remember for subruns: _CTX_K_S_#{original}_ -- remember for subruns: TEXMFSTART.#{original} [tex.rb texmfstart.rb] -texlua = true - -- begin library merge @@ -54,8 +51,9 @@ if not modules then modules = { } end modules ['l-string'] = { license = "see context related readme files" } +local string = string local sub, gsub, find, match, gmatch, format, char, byte, rep, lower = string.sub, string.gsub, string.find, string.match, string.gmatch, string.format, string.char, string.byte, string.rep, string.lower -local lpegmatch = lpeg.match +local lpegmatch, S, C, Ct = lpeg.match, lpeg.S, lpeg.C, lpeg.Ct -- some functions may disappear as they are not used anywhere @@ -63,196 +61,68 @@ if not string.split then -- this will be overloaded by a faster lpeg variant - function string:split(pattern) - if #self > 0 then - local t = { } - for s in gmatch(self..pattern,"(.-)"..pattern) do - t[#t+1] = s + function string.split(str,pattern) + local t = { } + if #str > 0 then + local n = 1 + for s in gmatch(str..pattern,"(.-)"..pattern) do + t[n] = s + n = n + 1 end - return t - else - return { } end + return t end end -local chr_to_esc = { - ["%"] = "%%", - ["."] = "%.", - ["+"] = "%+", ["-"] = "%-", ["*"] = "%*", - ["^"] = "%^", ["$"] = "%$", - ["["] = "%[", ["]"] = "%]", - ["("] = "%(", [")"] = "%)", - ["{"] = "%{", ["}"] = "%}" -} - -string.chr_to_esc = chr_to_esc - -function string:esc() -- variant 2 - return (gsub(self,"(.)",chr_to_esc)) -end - -function string:unquote() - return (gsub(self,"^([\"\'])(.*)%1$","%2")) +function string.unquoted(str) + return (gsub(str,"^([\"\'])(.*)%1$","%2")) end ---~ function string:unquote() ---~ if find(self,"^[\'\"]") then ---~ return sub(self,2,-2) ---~ else ---~ return self ---~ end ---~ end -function string:quote() -- we could use format("%q") - return format("%q",self) +function string.quoted(str) + return format("%q",str) -- always " end -function string:count(pattern) -- variant 3 +function string.count(str,pattern) -- variant 3 local n = 0 - for _ in gmatch(self,pattern) do + for _ in gmatch(str,pattern) do -- not for utf n = n + 1 end return n end -function string:limit(n,sentinel) - if #self > n then - sentinel = sentinel or " ..." - return sub(self,1,(n-#sentinel)) .. sentinel - else - return self - end -end - ---~ function string:strip() -- the .- is quite efficient ---~ -- return match(self,"^%s*(.-)%s*$") or "" ---~ -- return match(self,'^%s*(.*%S)') or '' -- posted on lua list ---~ return find(s,'^%s*$') and '' or match(s,'^%s*(.*%S)') ---~ end - -do -- roberto's variant: - local space = lpeg.S(" \t\v\n") - local nospace = 1 - space - local stripper = space^0 * lpeg.C((space^0 * nospace^1)^0) - function string.strip(str) - return lpegmatch(stripper,str) or "" - end -end - -function string:is_empty() - return not find(self,"%S") -end - -function string:enhance(pattern,action) - local ok, n = true, 0 - while ok do - ok = false - self = gsub(self,pattern, function(...) - ok, n = true, n + 1 - return action(...) - end) - end - return self, n -end - -local chr_to_hex, hex_to_chr = { }, { } - -for i=0,255 do - local c, h = char(i), format("%02X",i) - chr_to_hex[c], hex_to_chr[h] = h, c -end - -function string:to_hex() - return (gsub(self or "","(.)",chr_to_hex)) -end - -function string:from_hex() - return (gsub(self or "","(..)",hex_to_chr)) -end - -if not string.characters then - - local function nextchar(str, index) - index = index + 1 - return (index <= #str) and index or nil, sub(str,index,index) - end - function string:characters() - return nextchar, self, 0 - end - local function nextbyte(str, index) - index = index + 1 - return (index <= #str) and index or nil, byte(sub(str,index,index)) - end - function string:bytes() - return nextbyte, self, 0 - end - -end - --- we can use format for this (neg n) - -function string:rpadd(n,chr) - local m = n-#self - if m > 0 then - return self .. rep(chr or " ",m) +function string.limit(str,n,sentinel) -- not utf proof + if #str > n then + sentinel = sentinel or "..." + return sub(str,1,(n-#sentinel)) .. sentinel else - return self - end -end - -function string:lpadd(n,chr) - local m = n-#self - if m > 0 then - return rep(chr or " ",m) .. self - else - return self + return str end end -string.padd = string.rpadd +local space = S(" \t\v\n") +local nospace = 1 - space +local stripper = space^0 * C((space^0 * nospace^1)^0) -- roberto's code -function is_number(str) -- tonumber - return find(str,"^[%-%+]?[%d]-%.?[%d+]$") == 1 +function string.strip(str) + return lpegmatch(stripper,str) or "" end ---~ print(is_number("1")) ---~ print(is_number("1.1")) ---~ print(is_number(".1")) ---~ print(is_number("-0.1")) ---~ print(is_number("+0.1")) ---~ print(is_number("-.1")) ---~ print(is_number("+.1")) - -function string:split_settings() -- no {} handling, see l-aux for lpeg variant - if find(self,"=") then - local t = { } - for k,v in gmatch(self,"(%a+)=([^%,]*)") do - t[k] = v - end - return t - else - return nil - end +function string.is_empty(str) + return not find(str,"%S") end local patterns_escapes = { - ["-"] = "%-", - ["."] = "%.", - ["+"] = "%+", - ["*"] = "%*", ["%"] = "%%", - ["("] = "%)", - [")"] = "%)", - ["["] = "%[", - ["]"] = "%]", + ["."] = "%.", + ["+"] = "%+", ["-"] = "%-", ["*"] = "%*", + ["["] = "%[", ["]"] = "%]", + ["("] = "%(", [")"] = "%)", + -- ["{"] = "%{", ["}"] = "%}" + -- ["^"] = "%^", ["$"] = "%$", } -function string:pattesc() - return (gsub(self,".",patterns_escapes)) -end - local simple_escapes = { ["-"] = "%-", ["."] = "%.", @@ -260,80 +130,30 @@ local simple_escapes = { ["*"] = ".*", } -function string:simpleesc() - return (gsub(self,".",simple_escapes)) -end - -function string:tohash() - local t = { } - for s in gmatch(self,"([^, ]+)") do -- lpeg - t[s] = true - end - return t -end - -local pattern = lpeg.Ct(lpeg.C(1)^0) - -function string:totable() - return lpegmatch(pattern,self) +function string.escapedpattern(str,simple) + return (gsub(str,".",simple and simple_escapes or patterns_escapes)) end ---~ local t = { ---~ "1234567123456712345671234567", ---~ "a\tb\tc", ---~ "aa\tbb\tcc", ---~ "aaa\tbbb\tccc", ---~ "aaaa\tbbbb\tcccc", ---~ "aaaaa\tbbbbb\tccccc", ---~ "aaaaaa\tbbbbbb\tcccccc", ---~ } ---~ for k,v do ---~ print(string.tabtospace(t[k])) ---~ end - -function string.tabtospace(str,tab) - -- we don't handle embedded newlines - while true do - local s = find(str,"\t") - if s then - if not tab then tab = 7 end -- only when found - local d = tab-(s-1) % tab - if d > 0 then - str = gsub(str,"\t",rep(" ",d),1) - else - str = gsub(str,"\t","",1) - end +function string.topattern(str,lowercase,strict) + if str == "" then + return ".*" + else + str = gsub(str,".",simple_escapes) + if lowercase then + str = lower(str) + end + if strict then + return "^" .. str .. "$" else - break + return str end end - return str end -function string:compactlong() -- strips newlines and leading spaces - self = gsub(self,"[\n\r]+ *","") - self = gsub(self,"^ *","") - return self -end - -function string:striplong() -- strips newlines and leading spaces - self = gsub(self,"^%s*","") - self = gsub(self,"[\n\r]+ *","\n") - return self -end +-- obsolete names: -function string:topattern(lowercase,strict) - if lowercase then - self = lower(self) - end - self = gsub(self,".",simple_escapes) - if self == "" then - self = ".*" - elseif strict then - self = "^" .. self .. "$" - end - return self -end +string.quote = string.quoted +string.unquote = string.unquoted end -- of closure @@ -350,28 +170,75 @@ if not modules then modules = { } end modules ['l-lpeg'] = { local lpeg = require("lpeg") +local type = type + +-- Beware, we predefine a bunch of patterns here and one reason for doing so +-- is that we get consistent behaviour in some of the visualizers. + lpeg.patterns = lpeg.patterns or { } -- so that we can share local patterns = lpeg.patterns -local P, R, S, Ct, C, Cs, Cc, V = lpeg.P, lpeg.R, lpeg.S, lpeg.Ct, lpeg.C, lpeg.Cs, lpeg.Cc, lpeg.V -local match = lpeg.match +local P, R, S, V, match = lpeg.P, lpeg.R, lpeg.S, lpeg.V, lpeg.match +local Ct, C, Cs, Cc = lpeg.Ct, lpeg.C, lpeg.Cs, lpeg.Cc +local lpegtype = lpeg.type + +local utfcharacters = string.utfcharacters +local utfgmatch = unicode and unicode.utf8.gmatch + +local anything = P(1) +local endofstring = P(-1) +local alwaysmatched = P(true) + +patterns.anything = anything +patterns.endofstring = endofstring +patterns.beginofstring = alwaysmatched +patterns.alwaysmatched = alwaysmatched local digit, sign = R('09'), S('+-') local cr, lf, crlf = P("\r"), P("\n"), P("\r\n") -local utf8byte = R("\128\191") +local newline = crlf + cr + lf +local escaped = P("\\") * anything +local squote = P("'") +local dquote = P('"') +local space = P(" ") + +local utfbom_32_be = P('\000\000\254\255') +local utfbom_32_le = P('\255\254\000\000') +local utfbom_16_be = P('\255\254') +local utfbom_16_le = P('\254\255') +local utfbom_8 = P('\239\187\191') +local utfbom = utfbom_32_be + utfbom_32_le + + utfbom_16_be + utfbom_16_le + + utfbom_8 +local utftype = utfbom_32_be / "utf-32-be" + utfbom_32_le / "utf-32-le" + + utfbom_16_be / "utf-16-be" + utfbom_16_le / "utf-16-le" + + utfbom_8 / "utf-8" + alwaysmatched / "unknown" + +local utf8next = R("\128\191") -patterns.utf8byte = utf8byte patterns.utf8one = R("\000\127") -patterns.utf8two = R("\194\223") * utf8byte -patterns.utf8three = R("\224\239") * utf8byte * utf8byte -patterns.utf8four = R("\240\244") * utf8byte * utf8byte * utf8byte +patterns.utf8two = R("\194\223") * utf8next +patterns.utf8three = R("\224\239") * utf8next * utf8next +patterns.utf8four = R("\240\244") * utf8next * utf8next * utf8next +patterns.utfbom = utfbom +patterns.utftype = utftype + +local utf8char = patterns.utf8one + patterns.utf8two + patterns.utf8three + patterns.utf8four +local validutf8char = utf8char^0 * endofstring * Cc(true) + Cc(false) + +patterns.utf8 = utf8char +patterns.utf8char = utf8char +patterns.validutf8 = validutf8char +patterns.validutf8char = validutf8char patterns.digit = digit patterns.sign = sign patterns.cardinal = sign^0 * digit^1 patterns.integer = sign^0 * digit^1 patterns.float = sign^0 * digit^0 * P('.') * digit^1 +patterns.cfloat = sign^0 * digit^0 * P(',') * digit^1 patterns.number = patterns.float + patterns.integer +patterns.cnumber = patterns.cfloat + patterns.integer patterns.oct = P("0") * R("07")^1 patterns.octal = patterns.oct patterns.HEX = P("0x") * R("09","AF")^1 @@ -380,16 +247,42 @@ patterns.hexadecimal = P("0x") * R("09","AF","af")^1 patterns.lowercase = R("az") patterns.uppercase = R("AZ") patterns.letter = patterns.lowercase + patterns.uppercase -patterns.space = S(" ") +patterns.space = space +patterns.tab = P("\t") +patterns.spaceortab = patterns.space + patterns.tab patterns.eol = S("\n\r") patterns.spacer = S(" \t\f\v") -- + string.char(0xc2, 0xa0) if we want utf (cf mail roberto) -patterns.newline = crlf + cr + lf -patterns.nonspace = 1 - patterns.space +patterns.newline = newline +patterns.emptyline = newline^1 patterns.nonspacer = 1 - patterns.spacer patterns.whitespace = patterns.eol + patterns.spacer patterns.nonwhitespace = 1 - patterns.whitespace -patterns.utf8 = patterns.utf8one + patterns.utf8two + patterns.utf8three + patterns.utf8four -patterns.utfbom = P('\000\000\254\255') + P('\255\254\000\000') + P('\255\254') + P('\254\255') + P('\239\187\191') +patterns.equal = P("=") +patterns.comma = P(",") +patterns.commaspacer = P(",") * patterns.spacer^0 +patterns.period = P(".") +patterns.colon = P(":") +patterns.semicolon = P(";") +patterns.underscore = P("_") +patterns.escaped = escaped +patterns.squote = squote +patterns.dquote = dquote +patterns.nosquote = (escaped + (1-squote))^0 +patterns.nodquote = (escaped + (1-dquote))^0 +patterns.unsingle = (squote/"") * patterns.nosquote * (squote/"") +patterns.undouble = (dquote/"") * patterns.nodquote * (dquote/"") +patterns.unquoted = patterns.undouble + patterns.unsingle -- more often undouble +patterns.unspacer = ((patterns.spacer^1)/"")^0 + +patterns.somecontent = (anything - newline - space)^1 -- (utf8char - newline - space)^1 +patterns.beginline = #(1-newline) + +local unquoted = Cs(patterns.unquoted * endofstring) -- not C + +function string.unquoted(str) + return match(unquoted,str) or str +end + function lpeg.anywhere(pattern) --slightly adapted from website return P { P(pattern) + 1 * V(1) } -- why so complex? @@ -399,36 +292,18 @@ function lpeg.splitter(pattern, action) return (((1-P(pattern))^1)/action+1)^0 end -local spacing = patterns.spacer^0 * patterns.newline -- sort of strip -local empty = spacing * Cc("") -local nonempty = Cs((1-spacing)^1) * spacing^-1 -local content = (empty + nonempty)^1 - -local capture = Ct(content^0) - -function string:splitlines() - return match(capture,self) -end - -patterns.textline = content - ---~ local p = lpeg.splitat("->",false) print(match(p,"oeps->what->more")) -- oeps what more ---~ local p = lpeg.splitat("->",true) print(match(p,"oeps->what->more")) -- oeps what->more ---~ local p = lpeg.splitat("->",false) print(match(p,"oeps")) -- oeps ---~ local p = lpeg.splitat("->",true) print(match(p,"oeps")) -- oeps - local splitters_s, splitters_m = { }, { } local function splitat(separator,single) local splitter = (single and splitters_s[separator]) or splitters_m[separator] if not splitter then separator = P(separator) + local other = C((1 - separator)^0) if single then - local other, any = C((1 - separator)^0), P(1) + local any = anything splitter = other * (separator * C(any^0) + "") -- ? splitters_s[separator] = splitter else - local other = C((1 - separator)^0) splitter = other * (separator * other)^0 splitters_m[separator] = splitter end @@ -438,6 +313,7 @@ end lpeg.splitat = splitat + local cache = { } function lpeg.split(separator,str) @@ -449,16 +325,39 @@ function lpeg.split(separator,str) return match(c,str) end -function string:split(separator) +function string.split(str,separator) local c = cache[separator] if not c then c = Ct(splitat(separator)) cache[separator] = c end - return match(c,self) + return match(c,str) +end + +local spacing = patterns.spacer^0 * newline -- sort of strip +local empty = spacing * Cc("") +local nonempty = Cs((1-spacing)^1) * spacing^-1 +local content = (empty + nonempty)^1 + +patterns.textline = content + + +local linesplitter = Ct(splitat(newline)) + +patterns.linesplitter = linesplitter + +function string.splitlines(str) + return match(linesplitter,str) +end + +local utflinesplitter = utfbom^-1 * Ct(splitat(newline)) + +patterns.utflinesplitter = utflinesplitter + +function string.utfsplitlines(str) + return match(utflinesplitter,str) end -lpeg.splitters = cache local cache = { } @@ -466,37 +365,24 @@ function lpeg.checkedsplit(separator,str) local c = cache[separator] if not c then separator = P(separator) - local other = C((1 - separator)^0) + local other = C((1 - separator)^1) c = Ct(separator^0 * other * (separator^1 * other)^0) cache[separator] = c end return match(c,str) end -function string:checkedsplit(separator) +function string.checkedsplit(str,separator) local c = cache[separator] if not c then separator = P(separator) - local other = C((1 - separator)^0) + local other = C((1 - separator)^1) c = Ct(separator^0 * other * (separator^1 * other)^0) cache[separator] = c end - return match(c,self) + return match(c,str) end ---~ function lpeg.append(list,pp) ---~ local p = pp ---~ for l=1,#list do ---~ if p then ---~ p = p + P(list[l]) ---~ else ---~ p = P(list[l]) ---~ end ---~ end ---~ return p ---~ end - ---~ from roberto's site: local f1 = string.byte @@ -504,7 +390,273 @@ local function f2(s) local c1, c2 = f1(s,1,2) return c1 * 64 + c2 local function f3(s) local c1, c2, c3 = f1(s,1,3) return (c1 * 64 + c2) * 64 + c3 - 925824 end local function f4(s) local c1, c2, c3, c4 = f1(s,1,4) return ((c1 * 64 + c2) * 64 + c3) * 64 + c4 - 63447168 end -patterns.utf8byte = patterns.utf8one/f1 + patterns.utf8two/f2 + patterns.utf8three/f3 + patterns.utf8four/f4 +local utf8byte = patterns.utf8one/f1 + patterns.utf8two/f2 + patterns.utf8three/f3 + patterns.utf8four/f4 + +patterns.utf8byte = utf8byte + + + +local cache = { } + +function lpeg.stripper(str) + if type(str) == "string" then + local s = cache[str] + if not s then + s = Cs(((S(str)^1)/"" + 1)^0) + cache[str] = s + end + return s + else + return Cs(((str^1)/"" + 1)^0) + end +end + +local cache = { } + +function lpeg.keeper(str) + if type(str) == "string" then + local s = cache[str] + if not s then + s = Cs((((1-S(str))^1)/"" + 1)^0) + cache[str] = s + end + return s + else + return Cs((((1-str)^1)/"" + 1)^0) + end +end + +function lpeg.frontstripper(str) -- or pattern (yet undocumented) + return (P(str) + P(true)) * Cs(P(1)^0) +end + +function lpeg.endstripper(str) -- or pattern (yet undocumented) + return Cs((1 - P(str) * P(-1))^0) +end + +-- Just for fun I looked at the used bytecode and +-- p = (p and p + pp) or pp gets one more (testset). + +function lpeg.replacer(one,two) + if type(one) == "table" then + local no = #one + if no > 0 then + local p + for i=1,no do + local o = one[i] + local pp = P(o[1]) / o[2] + if p then + p = p + pp + else + p = pp + end + end + return Cs((p + 1)^0) + end + else + two = two or "" + return Cs((P(one)/two + 1)^0) + end +end + +local splitters_f, splitters_s = { }, { } + +function lpeg.firstofsplit(separator) -- always return value + local splitter = splitters_f[separator] + if not splitter then + separator = P(separator) + splitter = C((1 - separator)^0) + splitters_f[separator] = splitter + end + return splitter +end + +function lpeg.secondofsplit(separator) -- nil if not split + local splitter = splitters_s[separator] + if not splitter then + separator = P(separator) + splitter = (1 - separator)^0 * separator * C(anything^0) + splitters_s[separator] = splitter + end + return splitter +end + +function lpeg.balancer(left,right) + left, right = P(left), P(right) + return P { left * ((1 - left - right) + V(1))^0 * right } +end + + + +local nany = utf8char/"" + +function lpeg.counter(pattern) + pattern = Cs((P(pattern)/" " + nany)^0) + return function(str) + return #match(pattern,str) + end +end + +if utfgmatch then + + function lpeg.count(str,what) -- replaces string.count + if type(what) == "string" then + local n = 0 + for _ in utfgmatch(str,what) do + n = n + 1 + end + return n + else -- 4 times slower but still faster than / function + return #match(Cs((P(what)/" " + nany)^0),str) + end + end + +else + + local cache = { } + + function lpeg.count(str,what) -- replaces string.count + if type(what) == "string" then + local p = cache[what] + if not p then + p = Cs((P(what)/" " + nany)^0) + cache[p] = p + end + return #match(p,str) + else -- 4 times slower but still faster than / function + return #match(Cs((P(what)/" " + nany)^0),str) + end + end + +end + +local patterns_escapes = { -- also defines in l-string + ["%"] = "%%", + ["."] = "%.", + ["+"] = "%+", ["-"] = "%-", ["*"] = "%*", + ["["] = "%[", ["]"] = "%]", + ["("] = "%)", [")"] = "%)", + -- ["{"] = "%{", ["}"] = "%}" + -- ["^"] = "%^", ["$"] = "%$", +} + +local simple_escapes = { -- also defines in l-string + ["-"] = "%-", + ["."] = "%.", + ["?"] = ".", + ["*"] = ".*", +} + +local p = Cs((S("-.+*%()[]") / patterns_escapes + anything)^0) +local s = Cs((S("-.+*%()[]") / simple_escapes + anything)^0) + +function string.escapedpattern(str,simple) + return match(simple and s or p,str) +end + +-- utf extensies + +lpeg.UP = lpeg.P + +if utfcharacters then + + function lpeg.US(str) + local p + for uc in utfcharacters(str) do + if p then + p = p + P(uc) + else + p = P(uc) + end + end + return p + end + + +elseif utfgmatch then + + function lpeg.US(str) + local p + for uc in utfgmatch(str,".") do + if p then + p = p + P(uc) + else + p = P(uc) + end + end + return p + end + +else + + function lpeg.US(str) + local p + local f = function(uc) + if p then + p = p + P(uc) + else + p = P(uc) + end + end + match((utf8char/f)^0,str) + return p + end + +end + +local range = Cs(utf8byte) * (Cs(utf8byte) + Cc(false)) + +local utfchar = unicode and unicode.utf8 and unicode.utf8.char + +function lpeg.UR(str,more) + local first, last + if type(str) == "number" then + first = str + last = more or first + else + first, last = match(range,str) + if not last then + return P(str) + end + end + if first == last then + return P(str) + elseif utfchar and last - first < 8 then -- a somewhat arbitrary criterium + local p + for i=first,last do + if p then + p = p + P(utfchar(i)) + else + p = P(utfchar(i)) + end + end + return p -- nil when invalid range + else + local f = function(b) + return b >= first and b <= last + end + return utf8byte / f -- nil when invalid range + end +end + + + +function lpeg.oneof(list,...) -- lpeg.oneof("elseif","else","if","then") + if type(list) ~= "table" then + list = { list, ... } + end + -- sort(list) -- longest match first + local p = P(list[1]) + for l=2,#list do + p = p + P(list[l]) + end + return p +end + +function lpeg.is_lpeg(p) + return p and lpegtype(p) == "pattern" +end + end -- of closure @@ -519,12 +671,10 @@ if not modules then modules = { } end modules ['l-table'] = { license = "see context related readme files" } -table.join = table.concat - +local type, next, tostring, tonumber, ipairs, table, string = type, next, tostring, tonumber, ipairs, table, string local concat, sort, insert, remove = table.concat, table.sort, table.insert, table.remove local format, find, gsub, lower, dump, match = string.format, string.find, string.gsub, string.lower, string.dump, string.match local getmetatable, setmetatable = getmetatable, setmetatable -local type, next, tostring, tonumber, ipairs = type, next, tostring, tonumber, ipairs -- Starting with version 5.2 Lua no longer provide ipairs, which makes -- sense. As we already used the for loop and # in most places the @@ -576,50 +726,56 @@ end -- extra functions, some might go (when not used) function table.strip(tab) - local lst = { } + local lst, l = { }, 0 for i=1,#tab do local s = gsub(tab[i],"^%s*(.-)%s*$","%1") if s == "" then -- skip this one else - lst[#lst+1] = s + l = l + 1 + lst[l] = s end end return lst end function table.keys(t) - local k = { } + local keys, k = { }, 0 for key, _ in next, t do - k[#k+1] = key + k = k + 1 + keys[k] = key end - return k + return keys end local function compare(a,b) - return (tostring(a) < tostring(b)) + local ta, tb = type(a), type(b) -- needed, else 11 < 2 + if ta == tb then + return a < b + else + return tostring(a) < tostring(b) + end end local function sortedkeys(tab) - local srt, kind = { }, 0 -- 0=unknown 1=string, 2=number 3=mixed + local srt, category, s = { }, 0, 0 -- 0=unknown 1=string, 2=number 3=mixed for key,_ in next, tab do - srt[#srt+1] = key - if kind == 3 then + s = s + 1 + srt[s] = key + if category == 3 then -- no further check else local tkey = type(key) if tkey == "string" then - -- if kind == 2 then kind = 3 else kind = 1 end - kind = (kind == 2 and 3) or 1 + category = (category == 2 and 3) or 1 elseif tkey == "number" then - -- if kind == 1 then kind = 3 else kind = 2 end - kind = (kind == 1 and 3) or 2 + category = (category == 1 and 3) or 2 else - kind = 3 + category = 3 end end end - if kind == 0 or kind == 3 then + if category == 0 or category == 3 then sort(srt,compare) else sort(srt) @@ -628,9 +784,12 @@ local function sortedkeys(tab) end local function sortedhashkeys(tab) -- fast one - local srt = { } + local srt, s = { }, 0 for key,_ in next, tab do - srt[#srt+1] = key + if key then + s= s + 1 + srt[s] = key + end end sort(srt) return srt @@ -639,34 +798,50 @@ end table.sortedkeys = sortedkeys table.sortedhashkeys = sortedhashkeys -function table.sortedhash(t) - local s = sortedhashkeys(t) -- maybe just sortedkeys - local n = 0 - local function kv(s) - n = n + 1 - local k = s[n] - return k, t[k] +local function nothing() end + +local function sortedhash(t) + if t then + local n, s = 0, sortedkeys(t) -- the robust one + local function kv(s) + n = n + 1 + local k = s[n] + return k, t[k] + end + return kv, s + else + return nothing end - return kv, s end -table.sortedpairs = table.sortedhash +table.sortedhash = sortedhash +table.sortedpairs = sortedhash function table.append(t, list) - for _,v in next, list do - insert(t,v) + local n = #t + for i=1,#list do + n = n + 1 + t[n] = list[i] end + return t end function table.prepend(t, list) - for k,v in next, list do - insert(t,k,v) + local nl = #list + local nt = nl + #t + for i=#t,1,-1 do + t[nt] = t[i] + nt = nt - 1 + end + for i=1,#list do + t[i] = list[i] end + return t end function table.merge(t, ...) -- first one is target - t = t or {} - local lst = {...} + t = t or { } + local lst = { ... } for i=1,#lst do for k, v in next, lst[i] do t[k] = v @@ -676,7 +851,7 @@ function table.merge(t, ...) -- first one is target end function table.merged(...) - local tmp, lst = { }, {...} + local tmp, lst = { }, { ... } for i=1,#lst do for k, v in next, lst[i] do tmp[k] = v @@ -686,41 +861,45 @@ function table.merged(...) end function table.imerge(t, ...) - local lst = {...} + local lst, nt = { ... }, #t for i=1,#lst do local nst = lst[i] for j=1,#nst do - t[#t+1] = nst[j] + nt = nt + 1 + t[nt] = nst[j] end end return t end function table.imerged(...) - local tmp, lst = { }, {...} + local tmp, ntmp, lst = { }, 0, {...} for i=1,#lst do local nst = lst[i] for j=1,#nst do - tmp[#tmp+1] = nst[j] + ntmp = ntmp + 1 + tmp[ntmp] = nst[j] end end return tmp end -local function fastcopy(old) -- fast one +local function fastcopy(old,metatabletoo) -- fast one if old then local new = { } for k,v in next, old do if type(v) == "table" then - new[k] = fastcopy(v) -- was just table.copy + new[k] = fastcopy(v,metatabletoo) -- was just table.copy else new[k] = v end end - -- optional second arg - local mt = getmetatable(old) - if mt then - setmetatable(new,mt) + if metatabletoo then + -- optional second arg + local mt = getmetatable(old) + if mt then + setmetatable(new,mt) + end end return new else @@ -728,6 +907,8 @@ local function fastcopy(old) -- fast one end end +-- todo : copy without metatable + local function copy(t, tables) -- taken from lua wiki, slightly adapted tables = tables or { } local tcopy = {} @@ -760,33 +941,14 @@ end table.fastcopy = fastcopy table.copy = copy --- roughly: copy-loop : unpack : sub == 0.9 : 0.4 : 0.45 (so in critical apps, use unpack) - -function table.sub(t,i,j) - return { unpack(t,i,j) } -end - -function table.replace(a,b) - for k,v in next, b do - a[k] = v +function table.derive(parent) + local child = { } + if parent then + setmetatable(child,{ __index = parent }) end + return child end --- slower than #t on indexed tables (#t only returns the size of the numerically indexed slice) - -function table.is_empty(t) -- obolete, use inline code instead - return not t or not next(t) -end - -function table.one_entry(t) -- obolete, use inline code instead - local n = next(t) - return n and not next(t,n) -end - ---~ function table.starts_at(t) -- obsolete, not nice anyway ---~ return ipairs(t,1)(t,0) ---~ end - function table.tohash(t,value) local h = { } if t then @@ -799,27 +961,19 @@ function table.tohash(t,value) end function table.fromhash(t) - local h = { } + local hsh, h = { }, 0 for k, v in next, t do -- no ipairs here - if v then h[#h+1] = k end + if v then + h = h + 1 + hsh[h] = k + end end - return h + return hsh end ---~ print(table.serialize(t), "\n") ---~ print(table.serialize(t,"name"), "\n") ---~ print(table.serialize(t,false), "\n") ---~ print(table.serialize(t,true), "\n") ---~ print(table.serialize(t,"name",true), "\n") ---~ print(table.serialize(t,"name",true,true), "\n") - -table.serialize_functions = true -table.serialize_compact = true -table.serialize_inline = true - local noquotes, hexify, handle, reduce, compact, inline, functions -local reserved = table.tohash { -- intercept a language flaw, no reserved words as key +local reserved = table.tohash { -- intercept a language inconvenience: no reserved words as key 'and', 'break', 'do', 'else', 'elseif', 'end', 'false', 'for', 'function', 'if', 'in', 'local', 'nil', 'not', 'or', 'repeat', 'return', 'then', 'true', 'until', 'while', } @@ -831,20 +985,23 @@ local function simple_table(t) n = n + 1 end if n == #t then - local tt = { } + local tt, nt = { }, 0 for i=1,#t do local v = t[i] local tv = type(v) if tv == "number" then + nt = nt + 1 if hexify then - tt[#tt+1] = format("0x%04X",v) + tt[nt] = format("0x%04X",v) else - tt[#tt+1] = tostring(v) -- tostring not needed + tt[nt] = tostring(v) -- tostring not needed end elseif tv == "boolean" then - tt[#tt+1] = tostring(v) + nt = nt + 1 + tt[nt] = tostring(v) elseif tv == "string" then - tt[#tt+1] = format("%q",v) + nt = nt + 1 + tt[nt] = format("%q",v) else tt = nil break @@ -870,21 +1027,25 @@ local function do_serialize(root,name,depth,level,indexed) depth = depth .. " " if indexed then handle(format("%s{",depth)) - elseif name then - --~ handle(format("%s%s={",depth,key(name))) - if type(name) == "number" then -- or find(k,"^%d+$") then + else + local tn = type(name) + if tn == "number" then -- or find(k,"^%d+$") then if hexify then handle(format("%s[0x%04X]={",depth,name)) else handle(format("%s[%s]={",depth,name)) end - elseif noquotes and not reserved[name] and find(name,"^%a[%w%_]*$") then - handle(format("%s%s={",depth,name)) + elseif tn == "string" then + if noquotes and not reserved[name] and find(name,"^%a[%w%_]*$") then + handle(format("%s%s={",depth,name)) + else + handle(format("%s[%q]={",depth,name)) + end + elseif tn == "boolean" then + handle(format("%s[%s]={",depth,tostring(name))) else - handle(format("%s[%q]={",depth,name)) + handle(format("%s{",depth)) end - else - handle(format("%s{",depth)) end end -- we could check for k (index) being number (cardinal) @@ -901,11 +1062,9 @@ local function do_serialize(root,name,depth,level,indexed) for i=1,#sk do local k = sk[i] local v = root[k] - --~ if v == root then - -- circular - --~ else - local t = type(v) - if compact and first and type(k) == "number" and k >= first and k <= last then + -- circular + local t, tk = type(v), type(k) + if compact and first and tk == "number" and k >= first and k <= last then if t == "number" then if hexify then handle(format("%s 0x%04X,",depth,v)) @@ -947,17 +1106,18 @@ local function do_serialize(root,name,depth,level,indexed) handle(format("%s __p__=nil,",depth)) end elseif t == "number" then - --~ if hexify then - --~ handle(format("%s %s=0x%04X,",depth,key(k),v)) - --~ else - --~ handle(format("%s %s=%s,",depth,key(k),v)) -- %.99g - --~ end - if type(k) == "number" then -- or find(k,"^%d+$") then + if tk == "number" then -- or find(k,"^%d+$") then if hexify then handle(format("%s [0x%04X]=0x%04X,",depth,k,v)) else handle(format("%s [%s]=%s,",depth,k,v)) -- %.99g end + elseif tk == "boolean" then + if hexify then + handle(format("%s [%s]=0x%04X,",depth,tostring(k),v)) + else + handle(format("%s [%s]=%s,",depth,tostring(k),v)) -- %.99g + end elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then if hexify then handle(format("%s %s=0x%04X,",depth,k,v)) @@ -973,26 +1133,28 @@ local function do_serialize(root,name,depth,level,indexed) end elseif t == "string" then if reduce and tonumber(v) then - --~ handle(format("%s %s=%s,",depth,key(k),v)) - if type(k) == "number" then -- or find(k,"^%d+$") then + if tk == "number" then -- or find(k,"^%d+$") then if hexify then handle(format("%s [0x%04X]=%s,",depth,k,v)) else handle(format("%s [%s]=%s,",depth,k,v)) end + elseif tk == "boolean" then + handle(format("%s [%s]=%s,",depth,tostring(k),v)) elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then handle(format("%s %s=%s,",depth,k,v)) else handle(format("%s [%q]=%s,",depth,k,v)) end else - --~ handle(format("%s %s=%q,",depth,key(k),v)) - if type(k) == "number" then -- or find(k,"^%d+$") then + if tk == "number" then -- or find(k,"^%d+$") then if hexify then handle(format("%s [0x%04X]=%q,",depth,k,v)) else handle(format("%s [%s]=%q,",depth,k,v)) end + elseif tk == "boolean" then + handle(format("%s [%s]=%q,",depth,tostring(k),v)) elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then handle(format("%s %s=%q,",depth,k,v)) else @@ -1001,13 +1163,14 @@ local function do_serialize(root,name,depth,level,indexed) end elseif t == "table" then if not next(v) then - --~ handle(format("%s %s={},",depth,key(k))) - if type(k) == "number" then -- or find(k,"^%d+$") then + if tk == "number" then -- or find(k,"^%d+$") then if hexify then handle(format("%s [0x%04X]={},",depth,k)) else handle(format("%s [%s]={},",depth,k)) end + elseif tk == "boolean" then + handle(format("%s [%s]={},",depth,tostring(k))) elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then handle(format("%s %s={},",depth,k)) else @@ -1016,13 +1179,14 @@ local function do_serialize(root,name,depth,level,indexed) elseif inline then local st = simple_table(v) if st then - --~ handle(format("%s %s={ %s },",depth,key(k),concat(st,", "))) - if type(k) == "number" then -- or find(k,"^%d+$") then + if tk == "number" then -- or find(k,"^%d+$") then if hexify then handle(format("%s [0x%04X]={ %s },",depth,k,concat(st,", "))) else handle(format("%s [%s]={ %s },",depth,k,concat(st,", "))) end + elseif tk == "boolean" then -- or find(k,"^%d+$") then + handle(format("%s [%s]={ %s },",depth,tostring(k),concat(st,", "))) elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then handle(format("%s %s={ %s },",depth,k,concat(st,", "))) else @@ -1035,13 +1199,14 @@ local function do_serialize(root,name,depth,level,indexed) do_serialize(v,k,depth,level+1) end elseif t == "boolean" then - --~ handle(format("%s %s=%s,",depth,key(k),tostring(v))) - if type(k) == "number" then -- or find(k,"^%d+$") then + if tk == "number" then -- or find(k,"^%d+$") then if hexify then handle(format("%s [0x%04X]=%s,",depth,k,tostring(v))) else handle(format("%s [%s]=%s,",depth,k,tostring(v))) end + elseif tk == "boolean" then -- or find(k,"^%d+$") then + handle(format("%s [%s]=%s,",depth,tostring(k),tostring(v))) elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then handle(format("%s %s=%s,",depth,k,tostring(v))) else @@ -1049,35 +1214,37 @@ local function do_serialize(root,name,depth,level,indexed) end elseif t == "function" then if functions then - --~ handle(format('%s %s=loadstring(%q),',depth,key(k),dump(v))) - if type(k) == "number" then -- or find(k,"^%d+$") then + if tk == "number" then -- or find(k,"^%d+$") then if hexify then handle(format("%s [0x%04X]=loadstring(%q),",depth,k,dump(v))) else handle(format("%s [%s]=loadstring(%q),",depth,k,dump(v))) end + elseif tk == "boolean" then + handle(format("%s [%s]=loadstring(%q),",depth,tostring(k),dump(v))) elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then handle(format("%s %s=loadstring(%q),",depth,k,dump(v))) else - handle(format("%s [%q]=loadstring(%q),",depth,k,dump(v))) + -- handle(format("%s [%q]=loadstring(%q),",depth,k,dump(v))) + handle(format("%s [%q]=loadstring(%q),",depth,k,debug.getinfo(v).what == "C" and "C code" or dump(v))) end end else - --~ handle(format("%s %s=%q,",depth,key(k),tostring(v))) - if type(k) == "number" then -- or find(k,"^%d+$") then + if tk == "number" then -- or find(k,"^%d+$") then if hexify then handle(format("%s [0x%04X]=%q,",depth,k,tostring(v))) else handle(format("%s [%s]=%q,",depth,k,tostring(v))) end + elseif tk == "boolean" then -- or find(k,"^%d+$") then + handle(format("%s [%s]=%q,",depth,tostring(k),tostring(v))) elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then handle(format("%s %s=%q,",depth,k,tostring(v))) else handle(format("%s [%q]=%q,",depth,k,tostring(v))) end end - --~ end - end + end end if level > 0 then handle(format("%s},",depth)) @@ -1087,15 +1254,34 @@ end -- replacing handle by a direct t[#t+1] = ... (plus test) is not much -- faster (0.03 on 1.00 for zapfino.tma) -local function serialize(root,name,_handle,_reduce,_noquotes,_hexify) - noquotes = _noquotes - hexify = _hexify - handle = _handle or print - reduce = _reduce or false - compact = table.serialize_compact - inline = compact and table.serialize_inline - functions = table.serialize_functions +local function serialize(_handle,root,name,specification) -- handle wins local tname = type(name) + if type(specification) == "table" then + noquotes = specification.noquotes + hexify = specification.hexify + handle = _handle or specification.handle or print + reduce = specification.reduce or false + functions = specification.functions + compact = specification.compact + inline = specification.inline and compact + if functions == nil then + functions = true + end + if compact == nil then + compact = true + end + if inline == nil then + inline = compact + end + else + noquotes = false + hexify = false + handle = _handle or print + reduce = false + compact = true + inline = true + functions = true + end if tname == "string" then if name == "return" then handle("return {") @@ -1117,33 +1303,33 @@ local function serialize(root,name,_handle,_reduce,_noquotes,_hexify) else handle("t={") end - if root and next(root) then - do_serialize(root,name,"",0,indexed) + if root then + -- The dummy access will initialize a table that has a delayed initialization + -- using a metatable. (maybe explicitly test for metatable) + if getmetatable(root) then -- todo: make this an option, maybe even per subtable + local dummy = root._w_h_a_t_e_v_e_r_ + root._w_h_a_t_e_v_e_r_ = nil + end + -- Let's forget about empty tables. + if next(root) then + do_serialize(root,name,"",0) + end end handle("}") end ---~ name: ---~ ---~ true : return { } ---~ false : { } ---~ nil : t = { } ---~ string : string = { } ---~ 'return' : return { } ---~ number : [number] = { } -function table.serialize(root,name,reduce,noquotes,hexify) - local t = { } +function table.serialize(root,name,specification) + local t, n = { }, 0 local function flush(s) - t[#t+1] = s + n = n + 1 + t[n] = s end - serialize(root,name,flush,reduce,noquotes,hexify) + serialize(flush,root,name,specification) return concat(t,"\n") end -function table.tohandle(handle,root,name,reduce,noquotes,hexify) - serialize(root,name,handle,reduce,noquotes,hexify) -end +table.tohandle = serialize -- sometimes tables are real use (zapfino extra pro is some 85M) in which -- case a stepwise serialization is nice; actually, we could consider: @@ -1154,73 +1340,63 @@ end -- -- so this is on the todo list -table.tofile_maxtab = 2*1024 +local maxtab = 2*1024 -function table.tofile(filename,root,name,reduce,noquotes,hexify) +function table.tofile(filename,root,name,specification) local f = io.open(filename,'w') if f then - local maxtab = table.tofile_maxtab if maxtab > 1 then - local t = { } + local t, n = { }, 0 local function flush(s) - t[#t+1] = s - if #t > maxtab then + n = n + 1 + t[n] = s + if n > maxtab then f:write(concat(t,"\n"),"\n") -- hm, write(sometable) should be nice - t = { } + t, n = { }, 0 -- we could recycle t if needed end end - serialize(root,name,flush,reduce,noquotes,hexify) + serialize(flush,root,name,specification) f:write(concat(t,"\n"),"\n") else local function flush(s) f:write(s,"\n") end - serialize(root,name,flush,reduce,noquotes,hexify) + serialize(flush,root,name,specification) end f:close() + io.flush() end end -local function flatten(t,f,complete) -- is this used? meybe a variant with next, ... - for i=1,#t do - local v = t[i] - if type(v) == "table" then - if complete or type(v[1]) == "table" then - flatten(v,f,complete) +local function flattened(t,f,depth) + if f == nil then + f = { } + depth = 0xFFFF + elseif tonumber(f) then + -- assume then only two arguments are given + depth = f + f = { } + elseif not depth then + depth = 0xFFFF + end + for k, v in next, t do + if type(k) ~= "number" then + if depth > 0 and type(v) == "table" then + flattened(v,f,depth-1) else - f[#f+1] = v + f[k] = v end - else - f[#f+1] = v end end -end - -function table.flatten(t) - local f = { } - flatten(t,f,true) - return f -end - -function table.unnest(t) -- bad name - local f = { } - flatten(t,f,false) - return f -end - -table.flatten_one_level = table.unnest - --- a better one: - -local function flattened(t,f) - if not f then - f = { } - end - for k, v in next, t do - if type(v) == "table" then - flattened(v,f) + local n = #f + for k=1,#t do + local v = t[k] + if depth > 0 and type(v) == "table" then + flattened(v,f,depth-1) + n = #f else - f[k] = v + n = n + 1 + f[n] = v end end return f @@ -1228,49 +1404,27 @@ end table.flattened = flattened --- the next three may disappear - -function table.remove_value(t,value) -- todo: n - if value then - for i=1,#t do - if t[i] == value then - remove(t,i) - -- remove all, so no: return - end - end +local function unnest(t,f) -- only used in mk, for old times sake + if not f then -- and only relevant for token lists + f = { } end -end - -function table.insert_before_value(t,value,str) - if str then - if value then - for i=1,#t do - if t[i] == value then - insert(t,i,str) - return - end + for i=1,#t do + local v = t[i] + if type(v) == "table" then + if type(v[1]) == "table" then + unnest(v,f) + else + f[#f+1] = v end + else + f[#f+1] = v end - insert(t,1,str) - elseif value then - insert(t,1,value) end + return f end -function table.insert_after_value(t,value,str) - if str then - if value then - for i=1,#t do - if t[i] == value then - insert(t,i+1,str) - return - end - end - end - t[#t+1] = str - elseif value then - t[#t+1] = value - end +function table.unnest(t) -- bad name + return unnest(t) end local function are_equal(a,b,n,m) -- indexed @@ -1297,7 +1451,7 @@ end local function identical(a,b) -- assumes same structure for ka, va in next, a do - local vb = b[k] + local vb = b[ka] if va == vb then -- same elseif type(va) == "table" and type(vb) == "table" then @@ -1311,8 +1465,8 @@ local function identical(a,b) -- assumes same structure return true end -table.are_equal = are_equal table.identical = identical +table.are_equal = are_equal -- maybe also make a combined one @@ -1338,89 +1492,84 @@ function table.contains(t, v) end function table.count(t) - local n, e = 0, next(t) - while e do - n, e = n + 1, next(t,e) + local n = 0 + for k, v in next, t do + n = n + 1 end return n end -function table.swapped(t) - local s = { } +function table.swapped(t,s) -- hash + local n = { } + if s then + for k, v in next, s do + n[k] = v + end + end for k, v in next, t do - s[v] = k + n[v] = k end - return s + return n end ---~ function table.are_equal(a,b) ---~ return table.serialize(a) == table.serialize(b) ---~ end - -function table.clone(t,p) -- t is optional or nil or table - if not p then - t, p = { }, t or { } - elseif not t then - t = { } +function table.reversed(t) + if t then + local tt, tn = { }, #t + if tn > 0 then + local ttn = 0 + for i=tn,1,-1 do + ttn = ttn + 1 + tt[ttn] = t[i] + end + end + return tt end - setmetatable(t, { __index = function(_,key) return p[key] end }) -- why not __index = p ? - return t -end - -function table.hexed(t,seperator) - local tt = { } - for i=1,#t do tt[i] = format("0x%04X",t[i]) end - return concat(tt,seperator or " ") end -function table.reverse_hash(h) - local r = { } - for k,v in next, h do - r[v] = lower(gsub(k," ","")) +function table.sequenced(t,sep,simple) -- hash only + local s, n = { }, 0 + for k, v in sortedhash(t) do + if simple then + if v == true then + n = n + 1 + s[n] = k + elseif v and v~= "" then + n = n + 1 + s[n] = k .. "=" .. tostring(v) + end + else + n = n + 1 + s[n] = k .. "=" .. tostring(v) + end end - return r + return concat(s, sep or " | ") end -function table.reverse(t) - local tt = { } - if #t > 0 then - for i=#t,1,-1 do - tt[#tt+1] = t[i] - end +function table.print(t,...) + if type(t) ~= "table" then + print(tostring(t)) + else + table.tohandle(print,t,...) end - return tt end -function table.insert_before_value(t,value,extra) - for i=1,#t do - if t[i] == extra then - remove(t,i) - end - end - for i=1,#t do - if t[i] == value then - insert(t,i,extra) - return - end - end - insert(t,1,extra) +-- -- -- obsolete but we keep them for a while and might comment them later -- -- -- + +-- roughly: copy-loop : unpack : sub == 0.9 : 0.4 : 0.45 (so in critical apps, use unpack) + +function table.sub(t,i,j) + return { unpack(t,i,j) } end -function table.insert_after_value(t,value,extra) - for i=1,#t do - if t[i] == extra then - remove(t,i) - end - end - for i=1,#t do - if t[i] == value then - insert(t,i+1,extra) - return - end - end - insert(t,#t+1,extra) +-- slower than #t on indexed tables (#t only returns the size of the numerically indexed slice) + +function table.is_empty(t) + return not t or not next(t) end +function table.has_one_entry(t) + return t and not next(t,next(t)) +end end -- of closure @@ -1435,7 +1584,10 @@ if not modules then modules = { } end modules ['l-io'] = { license = "see context related readme files" } -local byte, find, gsub = string.byte, string.find, string.gsub +local io = io +local byte, find, gsub, format = string.byte, string.find, string.gsub, string.format +local concat = table.concat +local type = type if string.find(os.getenv("PATH"),";") then io.fileseparator, io.pathseparator = "\\", ";" @@ -1446,9 +1598,7 @@ end function io.loaddata(filename,textmode) local f = io.open(filename,(textmode and 'r') or 'rb') if f then - -- collectgarbage("step") -- sometimes makes a big difference in mem consumption local data = f:read('*all') - -- garbagecollector.check(data) f:close() return data else @@ -1460,13 +1610,14 @@ function io.savedata(filename,data,joiner) local f = io.open(filename,"wb") if f then if type(data) == "table" then - f:write(table.join(data,joiner or "")) + f:write(concat(data,joiner or "")) elseif type(data) == "function" then data(f) else f:write(data or "") end f:close() + io.flush() return true else return false @@ -1495,12 +1646,19 @@ function io.size(filename) end function io.noflines(f) - local n = 0 - for _ in f:lines() do - n = n + 1 + if type(f) == "string" then + local f = io.open(filename) + local n = f and io.noflines(f) or 0 + assert(f:close()) + return n + else + local n = 0 + for _ in f:lines() do + n = n + 1 + end + f:seek('set',0) + return n end - f:seek('set',0) - return n end local nextchar = { @@ -1526,8 +1684,6 @@ local nextchar = { function io.characters(f,n) if f then return nextchar[n or 1], f - else - return nil, nil end end @@ -1536,40 +1692,42 @@ local nextbyte = { local a, b, c, d = f:read(1,1,1,1) if d then return byte(a), byte(b), byte(c), byte(d) - else - return nil, nil, nil, nil + end + end, + [3] = function(f) + local a, b, c = f:read(1,1,1) + if b then + return byte(a), byte(b), byte(c) end end, [2] = function(f) local a, b = f:read(1,1) if b then return byte(a), byte(b) - else - return nil, nil end end, [1] = function (f) local a = f:read(1) if a then return byte(a) - else - return nil end end, [-2] = function (f) local a, b = f:read(1,1) if b then return byte(b), byte(a) - else - return nil, nil + end + end, + [-3] = function(f) + local a, b, c = f:read(1,1,1) + if b then + return byte(c), byte(b), byte(a) end end, [-4] = function(f) local a, b, c, d = f:read(1,1,1,1) if d then return byte(d), byte(c), byte(b), byte(a) - else - return nil, nil, nil, nil end end } @@ -1586,12 +1744,13 @@ function io.ask(question,default,options) while true do io.write(question) if options then - io.write(string.format(" [%s]",table.concat(options,"|"))) + io.write(format(" [%s]",concat(options,"|"))) end if default then - io.write(string.format(" [%s]",default)) + io.write(format(" [%s]",default)) end - io.write(string.format(" ")) + io.write(format(" ")) + io.flush() local answer = io.read() answer = gsub(answer,"^%s*(.*)%s*$","%1") if answer == "" and default then @@ -1615,6 +1774,63 @@ function io.ask(question,default,options) end end +local function readnumber(f,n,m) + if m then + f:seek("set",n) + n = m + end + if n == 1 then + return byte(f:read(1)) + elseif n == 2 then + local a, b = byte(f:read(2),1,2) + return 256 * a + b + elseif n == 3 then + local a, b, c = byte(f:read(3),1,3) + return 256*256 * a + 256 * b + c + elseif n == 4 then + local a, b, c, d = byte(f:read(4),1,4) + return 256*256*256 * a + 256*256 * b + 256 * c + d + elseif n == 8 then + local a, b = readnumber(f,4), readnumber(f,4) + return 256 * a + b + elseif n == 12 then + local a, b, c = readnumber(f,4), readnumber(f,4), readnumber(f,4) + return 256*256 * a + 256 * b + c + elseif n == -2 then + local b, a = byte(f:read(2),1,2) + return 256*a + b + elseif n == -3 then + local c, b, a = byte(f:read(3),1,3) + return 256*256 * a + 256 * b + c + elseif n == -4 then + local d, c, b, a = byte(f:read(4),1,4) + return 256*256*256 * a + 256*256 * b + 256*c + d + elseif n == -8 then + local h, g, f, e, d, c, b, a = byte(f:read(8),1,8) + return 256*256*256*256*256*256*256 * a + + 256*256*256*256*256*256 * b + + 256*256*256*256*256 * c + + 256*256*256*256 * d + + 256*256*256 * e + + 256*256 * f + + 256 * g + + h + else + return 0 + end +end + +io.readnumber = readnumber + +function io.readstring(f,n,m) + if m then + f:seek("set",n) + n = m + end + local str = gsub(f:read(n),"%z","") + return str +end + end -- of closure @@ -1628,11 +1844,15 @@ if not modules then modules = { } end modules ['l-number'] = { license = "see context related readme files" } +-- this module will be replaced when we have the bit library + local tostring = tostring -local format, floor, insert, match = string.format, math.floor, table.insert, string.match +local format, floor, match, rep = string.format, math.floor, string.match, string.rep +local concat, insert = table.concat, table.insert local lpegmatch = lpeg.match -number = number or { } +number = number or { } +local number = number -- a,b,c,d,e,f = number.toset(100101) @@ -1680,6 +1900,54 @@ function number.bits(n,zero) end +function number.bit(p) + return 2 ^ (p - 1) -- 1-based indexing +end + +function number.hasbit(x, p) -- typical call: if hasbit(x, bit(3)) then ... + return x % (p + p) >= p +end + +function number.setbit(x, p) + return hasbit(x, p) and x or x + p +end + +function number.clearbit(x, p) + return hasbit(x, p) and x - p or x +end + + +function number.tobitstring(n,m) + if n == 0 then + if m then + rep("00000000",m) + else + return "00000000" + end + else + local t = { } + while n > 0 do + insert(t,1,n % 2 > 0 and 1 or 0) + n = floor(n/2) + end + local nn = 8 - #t % 8 + if nn > 0 and nn < 8 then + for i=1,nn do + insert(t,1,0) + end + end + if m then + m = m * 8 - #t + if m > 0 then + insert(t,1,rep("0",m)) + end + end + return concat(t) + end +end + + + end -- of closure do -- create closure to overcome 200 locals limit @@ -1692,6 +1960,8 @@ if not modules then modules = { } end modules ['l-set'] = { license = "see context related readme files" } +-- This will become obsolete when we have the bitset library embedded. + set = set or { } local nums = { } @@ -1735,10 +2005,11 @@ function set.tolist(n) if n == 0 or not tabs[n] then return "" else - local t = { } + local t, n = { }, 0 for k, v in next, tabs[n] do if v then - t[#t+1] = k + n = n + 1 + t[n] = k end end return concat(t," ") @@ -1756,17 +2027,6 @@ function set.contains(n,s) end end ---~ local c = set.create{'aap','noot','mies'} ---~ local s = set.tonumber(c) ---~ local t = set.totable(s) ---~ print(t['aap']) ---~ local c = set.create{'zus','wim','jet'} ---~ local s = set.tonumber(c) ---~ local t = set.totable(s) ---~ print(t['aap']) ---~ print(t['jet']) ---~ print(set.contains(t,'jet')) ---~ print(set.contains(t,'aap')) @@ -1782,32 +2042,119 @@ if not modules then modules = { } end modules ['l-os'] = { license = "see context related readme files" } +-- This file deals with some operating system issues. Please don't bother me +-- with the pros and cons of operating systems as they all have their flaws +-- and benefits. Bashing one of them won't help solving problems and fixing +-- bugs faster and is a waste of time and energy. +-- +-- path separators: / or \ ... we can use / everywhere +-- suffixes : dll so exe <none> ... no big deal +-- quotes : we can use "" in most cases +-- expansion : unless "" are used * might give side effects +-- piping/threads : somewhat different for each os +-- locations : specific user file locations and settings can change over time +-- +-- os.type : windows | unix (new, we already guessed os.platform) +-- os.name : windows | msdos | linux | macosx | solaris | .. | generic (new) +-- os.platform : extended os.name with architecture + -- maybe build io.flush in os.execute -local find, format, gsub = string.find, string.format, string.gsub +local os = os +local find, format, gsub, upper = string.find, string.format, string.gsub, string.upper +local concat = table.concat local random, ceil = math.random, math.ceil +local rawget, rawset, type, getmetatable, setmetatable, tonumber = rawget, rawset, type, getmetatable, setmetatable, tonumber + +-- The following code permits traversing the environment table, at least +-- in luatex. Internally all environment names are uppercase. + +if not os.__getenv__ then + + os.__getenv__ = os.getenv + os.__setenv__ = os.setenv + + if os.env then + + local osgetenv = os.getenv + local ossetenv = os.setenv + local osenv = os.env local _ = osenv.PATH -- initialize the table + + function os.setenv(k,v) + if v == nil then + v = "" + end + local K = upper(k) + osenv[K] = v + if type(v) == "table" then + v = concat(v,";") -- path + end + ossetenv(K,v) + end + + function os.getenv(k) + local K = upper(k) + local v = osenv[K] or osenv[k] or osgetenv(K) or osgetenv(k) + if v == "" then + return nil + else + return v + end + end + + else + + local ossetenv = os.setenv + local osgetenv = os.getenv + local osenv = { } + + function os.setenv(k,v) + if v == nil then + v = "" + end + local K = upper(k) + osenv[K] = v + end -local execute, spawn, exec, ioflush = os.execute, os.spawn or os.execute, os.exec or os.execute, io.flush + function os.getenv(k) + local K = upper(k) + local v = osenv[K] or osgetenv(K) or osgetenv(k) + if v == "" then + return nil + else + return v + end + end + + local function __index(t,k) + return os.getenv(k) + end + local function __newindex(t,k,v) + os.setenv(k,v) + end + + os.env = { } + + setmetatable(os.env, { __index = __index, __newindex = __newindex } ) + + end + +end + +-- end of environment hack + +local execute, spawn, exec, iopopen, ioflush = os.execute, os.spawn or os.execute, os.exec or os.execute, io.popen, io.flush function os.execute(...) ioflush() return execute(...) end function os.spawn (...) ioflush() return spawn (...) end function os.exec (...) ioflush() return exec (...) end +function io.popen (...) ioflush() return iopopen(...) end function os.resultof(command) - ioflush() -- else messed up logging local handle = io.popen(command,"r") - if not handle then - -- print("unknown command '".. command .. "' in os.resultof") - return "" - else - return handle:read("*all") or "" - end + return handle and handle:read("*all") or "" end ---~ os.type : windows | unix (new, we already guessed os.platform) ---~ os.name : windows | msdos | linux | macosx | solaris | .. | generic (new) ---~ os.platform : extended os.name with architecture - if not io.fileseparator then if find(os.getenv("PATH"),";") then io.fileseparator, io.pathseparator, os.type = "\\", ";", os.type or "mswin" @@ -1856,16 +2203,11 @@ function os.runtime() return os.gettimeofday() - startuptime end ---~ print(os.gettimeofday()-os.time()) ---~ os.sleep(1.234) ---~ print (">>",os.runtime()) ---~ print(os.date("%H:%M:%S",os.gettimeofday())) ---~ print(os.date("%H:%M:%S",os.time())) -- no need for function anymore as we have more clever code and helpers now -- this metatable trickery might as well disappear -os.resolvers = os.resolvers or { } +os.resolvers = os.resolvers or { } -- will become private local resolvers = os.resolvers @@ -1878,24 +2220,6 @@ end setmetatable(os,osmt) -if not os.setenv then - - -- we still store them but they won't be seen in - -- child processes although we might pass them some day - -- using command concatination - - local env, getenv = { }, os.getenv - - function os.setenv(k,v) - env[k] = v - end - - function os.getenv(k) - return env[k] or getenv(k) - end - -end - -- we can use HOSTTYPE on some platforms local name, platform = os.name or "linux", os.getenv("MTX_PLATFORM") or "" @@ -1935,7 +2259,7 @@ elseif os.type == "windows" then elseif name == "linux" then function os.resolvers.platform(t,k) - -- we sometims have HOSTTYPE set so let's check that first + -- we sometimes have HOSTTYPE set so let's check that first local platform, architecture = "", os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" if find(architecture,"x86_64") then platform = "linux-64" @@ -2013,10 +2337,10 @@ elseif name == "freebsd" then elseif name == "kfreebsd" then function os.resolvers.platform(t,k) - -- we sometims have HOSTTYPE set so let's check that first + -- we sometimes have HOSTTYPE set so let's check that first local platform, architecture = "", os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" if find(architecture,"x86_64") then - platform = "kfreebsd-64" + platform = "kfreebsd-amd64" else platform = "kfreebsd-i386" end @@ -2091,61 +2415,86 @@ if not modules then modules = { } end modules ['l-file'] = { -- needs a cleanup -file = file or { } +file = file or { } +local file = file -local concat = table.concat -local find, gmatch, match, gsub, sub, char = string.find, string.gmatch, string.match, string.gsub, string.sub, string.char +local insert, concat = table.insert, table.concat +local find, gmatch, match, gsub, sub, char, lower = string.find, string.gmatch, string.match, string.gsub, string.sub, string.char, string.lower local lpegmatch = lpeg.match +local getcurrentdir, attributes = lfs.currentdir, lfs.attributes -function file.removesuffix(filename) - return (gsub(filename,"%.[%a%d]+$","")) +local P, R, S, C, Cs, Cp, Cc = lpeg.P, lpeg.R, lpeg.S, lpeg.C, lpeg.Cs, lpeg.Cp, lpeg.Cc + +local function dirname(name,default) + return match(name,"^(.+)[/\\].-$") or (default or "") end -function file.addsuffix(filename, suffix) - if not suffix or suffix == "" then - return filename - elseif not find(filename,"%.[%a%d]+$") then - return filename .. "." .. suffix - else - return filename - end +local function basename(name) + return match(name,"^.+[/\\](.-)$") or name end -function file.replacesuffix(filename, suffix) - return (gsub(filename,"%.[%a%d]+$","")) .. "." .. suffix +local function nameonly(name) + return (gsub(match(name,"^.+[/\\](.-)$") or name,"%..*$","")) end -function file.dirname(name,default) - return match(name,"^(.+)[/\\].-$") or (default or "") +local function extname(name,default) + return match(name,"^.+%.([^/\\]-)$") or default or "" end -function file.basename(name) - return match(name,"^.+[/\\](.-)$") or name +local function splitname(name) + local n, s = match(name,"^(.+)%.([^/\\]-)$") + return n or name, s or "" end -function file.nameonly(name) - return (gsub(match(name,"^.+[/\\](.-)$") or name,"%..*$","")) +file.basename = basename +file.dirname = dirname +file.nameonly = nameonly +file.extname = extname +file.suffix = extname + +function file.removesuffix(filename) + return (gsub(filename,"%.[%a%d]+$","")) end -function file.extname(name,default) - return match(name,"^.+%.([^/\\]-)$") or default or "" +function file.addsuffix(filename, suffix, criterium) + if not suffix or suffix == "" then + return filename + elseif criterium == true then + return filename .. "." .. suffix + elseif not criterium then + local n, s = splitname(filename) + if not s or s == "" then + return filename .. "." .. suffix + else + return filename + end + else + local n, s = splitname(filename) + if s and s ~= "" then + local t = type(criterium) + if t == "table" then + -- keep if in criterium + for i=1,#criterium do + if s == criterium[i] then + return filename + end + end + elseif t == "string" then + -- keep if criterium + if s == criterium then + return filename + end + end + end + return n .. "." .. suffix + end end -file.suffix = file.extname ---~ function file.join(...) ---~ local pth = concat({...},"/") ---~ pth = gsub(pth,"\\","/") ---~ local a, b = match(pth,"^(.*://)(.*)$") ---~ if a and b then ---~ return a .. gsub(b,"//+","/") ---~ end ---~ a, b = match(pth,"^(//)(.*)$") ---~ if a and b then ---~ return a .. gsub(b,"//+","/") ---~ end ---~ return (gsub(pth,"//+","/")) ---~ end +function file.replacesuffix(filename, suffix) + return (gsub(filename,"%.[%a%d]+$","")) .. "." .. suffix +end + local trick_1 = char(1) local trick_2 = "^" .. trick_1 .. "/+" @@ -2173,60 +2522,37 @@ function file.join(...) return (gsub(pth,"//+","/")) end ---~ print(file.join("//","/y")) ---~ print(file.join("/","/y")) ---~ print(file.join("","/y")) ---~ print(file.join("/x/","/y")) ---~ print(file.join("x/","/y")) ---~ print(file.join("http://","/y")) ---~ print(file.join("http://a","/y")) ---~ print(file.join("http:///a","/y")) ---~ print(file.join("//nas-1","/y")) -function file.iswritable(name) - local a = lfs.attributes(name) or lfs.attributes(file.dirname(name,".")) +function file.is_writable(name) + local a = attributes(name) or attributes(dirname(name,".")) return a and sub(a.permissions,2,2) == "w" end -function file.isreadable(name) - local a = lfs.attributes(name) +function file.is_readable(name) + local a = attributes(name) return a and sub(a.permissions,1,1) == "r" end -file.is_readable = file.isreadable -file.is_writable = file.iswritable +file.isreadable = file.is_readable -- depricated +file.iswritable = file.is_writable -- depricated --- todo: lpeg - ---~ function file.split_path(str) ---~ local t = { } ---~ str = gsub(str,"\\", "/") ---~ str = gsub(str,"(%a):([;/])", "%1\001%2") ---~ for name in gmatch(str,"([^;:]+)") do ---~ if name ~= "" then ---~ t[#t+1] = gsub(name,"\001",":") ---~ end ---~ end ---~ return t ---~ end +-- todo: lpeg \\ / .. does not save much local checkedsplit = string.checkedsplit -function file.split_path(str,separator) +function file.splitpath(str,separator) -- string str = gsub(str,"\\","/") return checkedsplit(str,separator or io.pathseparator) end -function file.join_path(tab) - return concat(tab,io.pathseparator) -- can have trailing // +function file.joinpath(tab,separator) -- table + return concat(tab,separator or io.pathseparator) -- can have trailing // end -- we can hash them weakly -local insert = table.insert - -function file.collapse_path(str,anchor) +function file.collapsepath(str,anchor) if anchor and not find(str,"^/") and not find(str,"^%a:") then str = getcurrentdir() .. "/" .. str end @@ -2280,62 +2606,15 @@ function file.collapse_path(str,anchor) return concat(newelements, '/') end end -function file.xcollapse_path(str) - print('collapse', str) - if str == "" or str =="." then - return "." - elseif find(str,"^%.%.") then - return str - elseif not find(str,"%.") then - return str - end - str = gsub(str,"^%./",lfs.currentdir() .. "/") -- ./xx in qualified - str = gsub(str,"\\","/") - local oldelements = checkedsplit(str,"/") - local newelements = { } - local i = #oldelements - while i > 0 do - local element = oldelements[i] - if element == '.' then - -- do nothing - elseif element == '..' then - local n = i -1 - while n > 0 do - local element = oldelements[n] - if element ~= '..' and element ~= '.' then - oldelements[n] = '.' - break - else - n = n - 1 - end - end - if n<1 then - insert(newelements,1,'..') - end - elseif element ~= "" then - insert(newelements,1,element) - end - i = i - 1 - end - if #newelements == 0 then - return "." - elseif find(str,"^/") then - return "/" .. concat(newelements,'/') - else - return concat(newelements, '/') - end -end ---~ print(file.collapse_path("/a")) ---~ print(file.collapse_path("a/./b/..")) ---~ print(file.collapse_path("a/aa/../b/bb")) ---~ print(file.collapse_path("a/../..")) ---~ print(file.collapse_path("a/.././././b/..")) ---~ print(file.collapse_path("a/./././b/..")) ---~ print(file.collapse_path("a/b/c/../..")) -function file.robustname(str) - return (gsub(str,"[^%a%d%/%-%.\\]+","-")) +function file.robustname(str,strict) + str = gsub(str,"[^%a%d%/%-%.\\]+","-") + if strict then + return lower(gsub(str,"^%-*(.-)%-*$","%1")) + else + return str + end end file.readdata = io.loaddata @@ -2347,100 +2626,34 @@ end -- lpeg variants, slightly faster, not always ---~ local period = lpeg.P(".") ---~ local slashes = lpeg.S("\\/") ---~ local noperiod = 1-period ---~ local noslashes = 1-slashes ---~ local name = noperiod^1 - ---~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * lpeg.C(noperiod^1) * -1 - ---~ function file.extname(name) ---~ return lpegmatch(pattern,name) or "" ---~ end - ---~ local pattern = lpeg.Cs(((period * noperiod^1 * -1)/"" + 1)^1) - ---~ function file.removesuffix(name) ---~ return lpegmatch(pattern,name) ---~ end - ---~ local pattern = (noslashes^0 * slashes)^1 * lpeg.C(noslashes^1) * -1 - ---~ function file.basename(name) ---~ return lpegmatch(pattern,name) or name ---~ end - ---~ local pattern = (noslashes^0 * slashes)^1 * lpeg.Cp() * noslashes^1 * -1 - ---~ function file.dirname(name) ---~ local p = lpegmatch(pattern,name) ---~ if p then ---~ return sub(name,1,p-2) ---~ else ---~ return "" ---~ end ---~ end - ---~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * lpeg.Cp() * noperiod^1 * -1 - ---~ function file.addsuffix(name, suffix) ---~ local p = lpegmatch(pattern,name) ---~ if p then ---~ return name ---~ else ---~ return name .. "." .. suffix ---~ end ---~ end - ---~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * lpeg.Cp() * noperiod^1 * -1 - ---~ function file.replacesuffix(name,suffix) ---~ local p = lpegmatch(pattern,name) ---~ if p then ---~ return sub(name,1,p-2) .. "." .. suffix ---~ else ---~ return name .. "." .. suffix ---~ end ---~ end - ---~ local pattern = (noslashes^0 * slashes)^0 * lpeg.Cp() * ((noperiod^1 * period)^1 * lpeg.Cp() + lpeg.P(true)) * noperiod^1 * -1 - ---~ function file.nameonly(name) ---~ local a, b = lpegmatch(pattern,name) ---~ if b then ---~ return sub(name,a,b-2) ---~ elseif a then ---~ return sub(name,a) ---~ else ---~ return name ---~ end ---~ end - ---~ local test = file.extname ---~ local test = file.basename ---~ local test = file.dirname ---~ local test = file.addsuffix ---~ local test = file.replacesuffix ---~ local test = file.nameonly - ---~ print(1,test("./a/b/c/abd.def.xxx","!!!")) ---~ print(2,test("./../b/c/abd.def.xxx","!!!")) ---~ print(3,test("a/b/c/abd.def.xxx","!!!")) ---~ print(4,test("a/b/c/def.xxx","!!!")) ---~ print(5,test("a/b/c/def","!!!")) ---~ print(6,test("def","!!!")) ---~ print(7,test("def.xxx","!!!")) - ---~ local tim = os.clock() for i=1,250000 do local ext = test("abd.def.xxx","!!!") end print(os.clock()-tim) + + + + + + + + + + + + + + + + + -- also rewrite previous -local letter = lpeg.R("az","AZ") + lpeg.S("_-+") -local separator = lpeg.P("://") +local letter = R("az","AZ") + S("_-+") +local separator = P("://") + +local qualified = P(".")^0 * P("/") + letter*P(":") + letter^1*separator + letter^1 * P("/") +local rootbased = P("/") + letter*P(":") -local qualified = lpeg.P(".")^0 * lpeg.P("/") + letter*lpeg.P(":") + letter^1*separator + letter^1 * lpeg.P("/") -local rootbased = lpeg.P("/") + letter*lpeg.P(":") +lpeg.patterns.qualified = qualified +lpeg.patterns.rootbased = rootbased -- ./name ../name /name c: :// name/name @@ -2452,14 +2665,16 @@ function file.is_rootbased_path(filename) return lpegmatch(rootbased,filename) ~= nil end -local slash = lpeg.S("\\/") -local period = lpeg.P(".") -local drive = lpeg.C(lpeg.R("az","AZ")) * lpeg.P(":") -local path = lpeg.C(((1-slash)^0 * slash)^0) -local suffix = period * lpeg.C(lpeg.P(1-period)^0 * lpeg.P(-1)) -local base = lpeg.C((1-suffix)^0) +-- actually these are schemes -local pattern = (drive + lpeg.Cc("")) * (path + lpeg.Cc("")) * (base + lpeg.Cc("")) * (suffix + lpeg.Cc("")) +local slash = S("\\/") +local period = P(".") +local drive = C(R("az","AZ")) * P(":") +local path = C(((1-slash)^0 * slash)^0) +local suffix = period * C(P(1-period)^0 * P(-1)) +local base = C((1-suffix)^0) + +local pattern = (drive + Cc("")) * (path + Cc("")) * (base + Cc("")) * (suffix + Cc("")) function file.splitname(str) -- returns drive, path, base, suffix return lpegmatch(pattern,str) @@ -2472,14 +2687,13 @@ end -- test { "/aa", "/aa/bb", "/aa/bb/cc", "/aa/bb/cc.dd", "/aa/bb/cc.dd.ee" } -- test { "aa", "aa/bb", "aa/bb/cc", "aa/bb/cc.dd", "aa/bb/cc.dd.ee" } ---~ -- todo: ---~ ---~ if os.type == "windows" then ---~ local currentdir = lfs.currentdir ---~ function lfs.currentdir() ---~ return (gsub(currentdir(),"\\","/")) ---~ end ---~ end + +-- for myself: + +function file.strip(name,dir) + local b, a = match(name,"^(.-)" .. dir .. "(.*)$") + return a ~= "" and a or name +end end -- of closure @@ -2495,6 +2709,7 @@ if not modules then modules = { } end modules ['l-md5'] = { -- This also provides file checksums and checkers. +local md5, file = md5, file local gsub, format, byte = string.gsub, string.format, string.byte local function convert(str,fmt) @@ -2505,27 +2720,13 @@ if not md5.HEX then function md5.HEX(str) return convert(str,"%02X") end end if not md5.hex then function md5.hex(str) return convert(str,"%02x") end end if not md5.dec then function md5.dec(str) return convert(str,"%03i") end end ---~ if not md5.HEX then ---~ local function remap(chr) return format("%02X",byte(chr)) end ---~ function md5.HEX(str) return (gsub(md5.sum(str),".",remap)) end ---~ end ---~ if not md5.hex then ---~ local function remap(chr) return format("%02x",byte(chr)) end ---~ function md5.hex(str) return (gsub(md5.sum(str),".",remap)) end ---~ end ---~ if not md5.dec then ---~ local function remap(chr) return format("%03i",byte(chr)) end ---~ function md5.dec(str) return (gsub(md5.sum(str),".",remap)) end ---~ end - -file.needs_updating_threshold = 1 - -function file.needs_updating(oldname,newname) -- size modification access change + +function file.needs_updating(oldname,newname,threshold) -- size modification access change local oldtime = lfs.attributes(oldname, modification) local newtime = lfs.attributes(newname, modification) if newtime >= oldtime then return false - elseif oldtime - newtime < file.needs_updating_threshold then + elseif oldtime - newtime < (threshold or 1) then return false else return true @@ -2572,86 +2773,171 @@ if not modules then modules = { } end modules ['l-url'] = { license = "see context related readme files" } -local char, gmatch, gsub = string.char, string.gmatch, string.gsub +local char, gmatch, gsub, format, byte, find = string.char, string.gmatch, string.gsub, string.format, string.byte, string.find +local concat = table.concat local tonumber, type = tonumber, type -local lpegmatch = lpeg.match +local P, C, R, S, Cs, Cc, Ct = lpeg.P, lpeg.C, lpeg.R, lpeg.S, lpeg.Cs, lpeg.Cc, lpeg.Ct +local lpegmatch, lpegpatterns, replacer = lpeg.match, lpeg.patterns, lpeg.replacer --- from the spec (on the web): +-- from wikipedia: -- --- foo://example.com:8042/over/there?name=ferret#nose --- \_/ \______________/\_________/ \_________/ \__/ --- | | | | | --- scheme authority path query fragment --- | _____________________|__ --- / \ / \ --- urn:example:animal:ferret:nose +-- foo://username:password@example.com:8042/over/there/index.dtb?type=animal;name=narwhal#nose +-- \_/ \_______________/ \_________/ \__/ \___/ \_/ \______________________/ \__/ +-- | | | | | | | | +-- | userinfo hostname port | | query fragment +-- | \________________________________/\_____________|____|/ +-- scheme | | | | +-- | authority path | | +-- | | | +-- | path interpretable as filename +-- | ___________|____________ | +-- / \ / \ | +-- urn:example:animal:ferret:nose interpretable as extension + +url = url or { } +local url = url + +local tochar = function(s) return char(tonumber(s,16)) end + +local colon = P(":") +local qmark = P("?") +local hash = P("#") +local slash = P("/") +local percent = P("%") +local endofstring = P(-1) + +local hexdigit = R("09","AF","af") +local plus = P("+") +local nothing = Cc("") +local escaped = (plus / " ") + (percent * C(hexdigit * hexdigit) / tochar) -url = url or { } +-- we assume schemes with more than 1 character (in order to avoid problems with windows disks) -local function tochar(s) - return char(tonumber(s,16)) -end +local scheme = Cs((escaped+(1-colon-slash-qmark-hash))^2) * colon + nothing +local authority = slash * slash * Cs((escaped+(1- slash-qmark-hash))^0) + nothing +local path = slash * Cs((escaped+(1- qmark-hash))^0) + nothing +local query = qmark * Cs((escaped+(1- hash))^0) + nothing +local fragment = hash * Cs((escaped+(1- endofstring))^0) + nothing -local colon, qmark, hash, slash, percent, endofstring = lpeg.P(":"), lpeg.P("?"), lpeg.P("#"), lpeg.P("/"), lpeg.P("%"), lpeg.P(-1) +local validurl = scheme * authority * path * query * fragment +local parser = Ct(validurl) -local hexdigit = lpeg.R("09","AF","af") -local plus = lpeg.P("+") -local escaped = (plus / " ") + (percent * lpeg.C(hexdigit * hexdigit) / tochar) +lpegpatterns.url = validurl +lpegpatterns.urlsplitter = parser --- we assume schemes with more than 1 character (in order to avoid problems with windows disks) +local escapes = { } ; for i=0,255 do escapes[i] = format("%%%02X",i) end -local scheme = lpeg.Cs((escaped+(1-colon-slash-qmark-hash))^2) * colon + lpeg.Cc("") -local authority = slash * slash * lpeg.Cs((escaped+(1- slash-qmark-hash))^0) + lpeg.Cc("") -local path = slash * lpeg.Cs((escaped+(1- qmark-hash))^0) + lpeg.Cc("") -local query = qmark * lpeg.Cs((escaped+(1- hash))^0) + lpeg.Cc("") -local fragment = hash * lpeg.Cs((escaped+(1- endofstring))^0) + lpeg.Cc("") +local escaper = Cs((R("09","AZ","az") + S("-./_") + P(1) / escapes)^0) -local parser = lpeg.Ct(scheme * authority * path * query * fragment) +lpegpatterns.urlescaper = escaper -- todo: reconsider Ct as we can as well have five return values (saves a table) -- so we can have two parsers, one with and one without -function url.split(str) +local function split(str) return (type(str) == "string" and lpegmatch(parser,str)) or str end +local function hasscheme(str) + local scheme = lpegmatch(scheme,str) -- at least one character + return scheme and scheme ~= "" +end + -- todo: cache them -function url.hashed(str) - local s = url.split(str) +local rootletter = R("az","AZ") + + S("_-+") +local separator = P("://") +local qualified = P(".")^0 * P("/") + + rootletter * P(":") + + rootletter^1 * separator + + rootletter^1 * P("/") +local rootbased = P("/") + + rootletter * P(":") + +local barswapper = replacer("|",":") +local backslashswapper = replacer("\\","/") + +local function hashed(str) -- not yet ok (/test?test) + local s = split(str) local somescheme = s[1] ~= "" - return { - scheme = (somescheme and s[1]) or "file", - authority = s[2], - path = s[3], - query = s[4], - fragment = s[5], - original = str, - noscheme = not somescheme, - } + local somequery = s[4] ~= "" + if not somescheme and not somequery then + s = { + scheme = "file", + authority = "", + path = str, + query = "", + fragment = "", + original = str, + noscheme = true, + filename = str, + } + else -- not always a filename but handy anyway + local authority, path, filename = s[2], s[3] + if authority == "" then + filename = path + else + filename = authority .. "/" .. path + end + s = { + scheme = s[1], + authority = authority, + path = path, + query = s[4], + fragment = s[5], + original = str, + noscheme = false, + filename = filename, + } + end + return s end -function url.hasscheme(str) - return url.split(str)[1] ~= "" -end +-- Here we assume: +-- +-- files: /// = relative +-- files: //// = absolute (!) + + + +url.split = split +url.hasscheme = hasscheme +url.hashed = hashed -function url.addscheme(str,scheme) - return (url.hasscheme(str) and str) or ((scheme or "file:///") .. str) +function url.addscheme(str,scheme) -- no authority + if hasscheme(str) then + return str + elseif not scheme then + return "file:///" .. str + else + return scheme .. ":///" .. str + end end -function url.construct(hash) - local fullurl = hash.sheme .. "://".. hash.authority .. hash.path - if hash.query then - fullurl = fullurl .. "?".. hash.query +function url.construct(hash) -- dodo: we need to escape ! + local fullurl, f = { }, 0 + local scheme, authority, path, query, fragment = hash.scheme, hash.authority, hash.path, hash.query, hash.fragment + if scheme and scheme ~= "" then + f = f + 1 ; fullurl[f] = scheme .. "://" + end + if authority and authority ~= "" then + f = f + 1 ; fullurl[f] = authority end - if hash.fragment then - fullurl = fullurl .. "?".. hash.fragment + if path and path ~= "" then + f = f + 1 ; fullurl[f] = "/" .. path end - return fullurl + if query and query ~= "" then + f = f + 1 ; fullurl[f] = "?".. query + end + if fragment and fragment ~= "" then + f = f + 1 ; fullurl[f] = "#".. fragment + end + return lpegmatch(escaper,concat(fullurl)) end function url.filename(filename) - local t = url.hashed(filename) + local t = hashed(filename) return (t.scheme == "file" and (gsub(t.path,"^/([a-zA-Z])([:|])/)","%1:"))) or filename end @@ -2667,37 +2953,13 @@ function url.query(str) end end ---~ print(url.filename("file:///c:/oeps.txt")) ---~ print(url.filename("c:/oeps.txt")) ---~ print(url.filename("file:///oeps.txt")) ---~ print(url.filename("file:///etc/test.txt")) ---~ print(url.filename("/oeps.txt")) - ---~ from the spec on the web (sort of): ---~ ---~ function test(str) ---~ print(table.serialize(url.hashed(str))) ---~ end ---~ ---~ test("%56pass%20words") ---~ test("file:///c:/oeps.txt") ---~ test("file:///c|/oeps.txt") ---~ test("file:///etc/oeps.txt") ---~ test("file://./etc/oeps.txt") ---~ test("file:////etc/oeps.txt") ---~ test("ftp://ftp.is.co.za/rfc/rfc1808.txt") ---~ test("http://www.ietf.org/rfc/rfc2396.txt") ---~ test("ldap://[2001:db8::7]/c=GB?objectClass?one#what") ---~ test("mailto:John.Doe@example.com") ---~ test("news:comp.infosystems.www.servers.unix") ---~ test("tel:+1-816-555-1212") ---~ test("telnet://192.0.2.16:80/") ---~ test("urn:oasis:names:specification:docbook:dtd:xml:4.1.2") ---~ test("/etc/passwords") ---~ test("http://www.pragma-ade.com/spaced%20name") - ---~ test("zip:///oeps/oeps.zip#bla/bla.tex") ---~ test("zip:///oeps/oeps.zip?bla/bla.tex") + + + + + + + end -- of closure @@ -2712,35 +2974,53 @@ if not modules then modules = { } end modules ['l-dir'] = { license = "see context related readme files" } --- dir.expand_name will be merged with cleanpath and collapsepath +-- dir.expandname will be merged with cleanpath and collapsepath local type = type local find, gmatch, match, gsub = string.find, string.gmatch, string.match, string.gsub +local concat, insert, remove = table.concat, table.insert, table.remove local lpegmatch = lpeg.match +local P, S, R, C, Cc, Cs, Ct, Cv, V = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.Cc, lpeg.Cs, lpeg.Ct, lpeg.Cv, lpeg.V + dir = dir or { } +local dir = dir +local lfs = lfs + +local attributes = lfs.attributes +local walkdir = lfs.dir +local isdir = lfs.isdir +local isfile = lfs.isfile +local mkdir = lfs.mkdir +local chdir = lfs.chdir +local currentdir = lfs.currentdir -- handy function dir.current() - return (gsub(lfs.currentdir(),"\\","/")) + return (gsub(currentdir(),"\\","/")) end --- optimizing for no string.find (*) does not save time +-- optimizing for no find (*) does not save time -local attributes = lfs.attributes -local walkdir = lfs.dir -local function glob_pattern(path,patt,recurse,action) - local ok, scanner +local lfsisdir = isdir + +local function isdir(path) + path = gsub(path,"[/\\]+$","") + return lfsisdir(path) +end + +lfs.isdir = isdir + +local function globpattern(path,patt,recurse,action) if path == "/" then - ok, scanner = xpcall(function() return walkdir(path..".") end, function() end) -- kepler safe - else - ok, scanner = xpcall(function() return walkdir(path) end, function() end) -- kepler safe + path = path .. "." + elseif not find(path,"/$") then + path = path .. '/' end - if ok and type(scanner) == "function" then - if not find(path,"/$") then path = path .. '/' end - for name in scanner do + if isdir(path) then -- lfs.isdir does not like trailing / + for name in walkdir(path) do -- lfs.dir accepts trailing / local full = path .. name local mode = attributes(full,'mode') if mode == 'file' then @@ -2748,15 +3028,15 @@ local function glob_pattern(path,patt,recurse,action) action(full) end elseif recurse and (mode == "directory") and (name ~= '.') and (name ~= "..") then - glob_pattern(full,patt,recurse,action) + globpattern(full,patt,recurse,action) end end end end -dir.glob_pattern = glob_pattern +dir.globpattern = globpattern -local function collect_pattern(path,patt,recurse,result) +local function collectpattern(path,patt,recurse,result) local ok, scanner result = result or { } if path == "/" then @@ -2775,7 +3055,7 @@ local function collect_pattern(path,patt,recurse,result) result[name] = attr end elseif recurse and (mode == "directory") and (name ~= '.') and (name ~= "..") then - attr.list = collect_pattern(full,patt,recurse) + attr.list = collectpattern(full,patt,recurse) result[name] = attr end end @@ -2783,9 +3063,7 @@ local function collect_pattern(path,patt,recurse,result) return result end -dir.collect_pattern = collect_pattern - -local P, S, R, C, Cc, Cs, Ct, Cv, V = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.Cc, lpeg.Cs, lpeg.Ct, lpeg.Cv, lpeg.V +dir.collectpattern = collectpattern local pattern = Ct { [1] = (C(P(".") + P("/")^1) + C(R("az","AZ") * P(":") * P("/")^0) + Cc("./")) * V(2) * V(3), @@ -2809,16 +3087,16 @@ local function glob(str,t) for s=1,#str do glob(str[s],t) end - elseif lfs.isfile(str) then + elseif isfile(str) then t(str) else - local split = lpegmatch(pattern,str) + local split = lpegmatch(pattern,str) -- we could use the file splitter if split then local root, path, base = split[1], split[2], split[3] local recurse = find(base,"%*%*") local start = root .. path local result = lpegmatch(filter,start .. base) - glob_pattern(start,result,recurse,t) + globpattern(start,result,recurse,t) end end else @@ -2828,12 +3106,15 @@ local function glob(str,t) glob(str[s],t) end return t - elseif lfs.isfile(str) then - local t = t or { } - t[#t+1] = str - return t + elseif isfile(str) then + if t then + t[#t+1] = str + return t + else + return { str } + end else - local split = lpegmatch(pattern,str) + local split = lpegmatch(pattern,str) -- we could use the file splitter if split then local t = t or { } local action = action or function(name) t[#t+1] = name end @@ -2841,7 +3122,7 @@ local function glob(str,t) local recurse = find(base,"%*%*") local start = root .. path local result = lpegmatch(filter,start .. base) - glob_pattern(start,result,recurse,action) + globpattern(start,result,recurse,action) return t else return { } @@ -2852,18 +3133,14 @@ end dir.glob = glob ---~ list = dir.glob("**/*.tif") ---~ list = dir.glob("/**/*.tif") ---~ list = dir.glob("./**/*.tif") ---~ list = dir.glob("oeps/**/*.tif") ---~ list = dir.glob("/oeps/**/*.tif") local function globfiles(path,recurse,func,files) -- func == pattern or function if type(func) == "string" then - local s = func -- alas, we need this indirect way + local s = func func = function(name) return find(name,s) end end files = files or { } + local noffiles = #files for name in walkdir(path) do if find(name,"^%.") then --- skip @@ -2874,12 +3151,9 @@ local function globfiles(path,recurse,func,files) -- func == pattern or function globfiles(path .. "/" .. name,recurse,func,files) end elseif mode == "file" then - if func then - if func(name) then - files[#files+1] = path .. "/" .. name - end - else - files[#files+1] = path .. "/" .. name + if not func or func(name) then + noffiles = noffiles + 1 + files[noffiles] = path .. "/" .. name end end end @@ -2897,17 +3171,15 @@ dir.globfiles = globfiles -- print(dir.ls("*.tex")) function dir.ls(pattern) - return table.concat(glob(pattern),"\n") + return concat(glob(pattern),"\n") end ---~ mkdirs("temp") ---~ mkdirs("a/b/c") ---~ mkdirs(".","/a/b/c") ---~ mkdirs("a","b","c") local make_indeed = true -- false -if string.find(os.getenv("PATH"),";") then -- os.type == "windows" +local onwindows = os.type == "windows" or find(os.getenv("PATH"),";") + +if onwindows then function dir.mkdirs(...) local str, pth, t = "", "", { ... } @@ -2956,64 +3228,21 @@ if string.find(os.getenv("PATH"),";") then -- os.type == "windows" else pth = pth .. "/" .. s end - if make_indeed and not lfs.isdir(pth) then - lfs.mkdir(pth) + if make_indeed and not isdir(pth) then + mkdir(pth) end end - return pth, (lfs.isdir(pth) == true) - end - ---~ print(dir.mkdirs("","","a","c")) ---~ print(dir.mkdirs("a")) ---~ print(dir.mkdirs("a:")) ---~ print(dir.mkdirs("a:/b/c")) ---~ print(dir.mkdirs("a:b/c")) ---~ print(dir.mkdirs("a:/bbb/c")) ---~ print(dir.mkdirs("/a/b/c")) ---~ print(dir.mkdirs("/aaa/b/c")) ---~ print(dir.mkdirs("//a/b/c")) ---~ print(dir.mkdirs("///a/b/c")) ---~ print(dir.mkdirs("a/bbb//ccc/")) - - function dir.expand_name(str) -- will be merged with cleanpath and collapsepath - local first, nothing, last = match(str,"^(//)(//*)(.*)$") - if first then - first = dir.current() .. "/" - end - if not first then - first, last = match(str,"^(//)/*(.*)$") - end - if not first then - first, last = match(str,"^([a-zA-Z]:)(.*)$") - if first and not find(last,"^/") then - local d = lfs.currentdir() - if lfs.chdir(first) then - first = dir.current() - end - lfs.chdir(d) - end - end - if not first then - first, last = dir.current(), str - end - last = gsub(last,"//","/") - last = gsub(last,"/%./","/") - last = gsub(last,"^/*","") - first = gsub(first,"/*$","") - if last == "" then - return first - else - return first .. "/" .. last - end + return pth, (isdir(pth) == true) end + else function dir.mkdirs(...) local str, pth, t = "", "", { ... } for i=1,#t do local s = t[i] - if s ~= "" then + if s and s ~= "" then -- we catch nil and false if str ~= "" then str = str .. "/" .. s else @@ -3031,42 +3260,92 @@ else else pth = pth .. "/" .. s end - if make_indeed and not first and not lfs.isdir(pth) then - lfs.mkdir(pth) + if make_indeed and not first and not isdir(pth) then + mkdir(pth) end end else pth = "." for s in gmatch(str,"[^/]+") do pth = pth .. "/" .. s - if make_indeed and not lfs.isdir(pth) then - lfs.mkdir(pth) + if make_indeed and not isdir(pth) then + mkdir(pth) + end + end + end + return pth, (isdir(pth) == true) + end + + +end + +dir.makedirs = dir.mkdirs + +-- we can only define it here as it uses dir.current + +if onwindows then + + function dir.expandname(str) -- will be merged with cleanpath and collapsepath + local first, nothing, last = match(str,"^(//)(//*)(.*)$") + if first then + first = dir.current() .. "/" + end + if not first then + first, last = match(str,"^(//)/*(.*)$") + end + if not first then + first, last = match(str,"^([a-zA-Z]:)(.*)$") + if first and not find(last,"^/") then + local d = currentdir() + if chdir(first) then + first = dir.current() end + chdir(d) end end - return pth, (lfs.isdir(pth) == true) + if not first then + first, last = dir.current(), str + end + last = gsub(last,"//","/") + last = gsub(last,"/%./","/") + last = gsub(last,"^/*","") + first = gsub(first,"/*$","") + if last == "" or last == "." then + return first + else + return first .. "/" .. last + end end ---~ print(dir.mkdirs("","","a","c")) ---~ print(dir.mkdirs("a")) ---~ print(dir.mkdirs("/a/b/c")) ---~ print(dir.mkdirs("/aaa/b/c")) ---~ print(dir.mkdirs("//a/b/c")) ---~ print(dir.mkdirs("///a/b/c")) ---~ print(dir.mkdirs("a/bbb//ccc/")) +else - function dir.expand_name(str) -- will be merged with cleanpath and collapsepath + function dir.expandname(str) -- will be merged with cleanpath and collapsepath if not find(str,"^/") then - str = lfs.currentdir() .. "/" .. str + str = currentdir() .. "/" .. str end str = gsub(str,"//","/") str = gsub(str,"/%./","/") + str = gsub(str,"(.)/%.$","%1") return str end end -dir.makedirs = dir.mkdirs +file.expandname = dir.expandname -- for convenience + +local stack = { } + +function dir.push(newdir) + insert(stack,lfs.currentdir()) +end + +function dir.pop() + local d = remove(stack) + if d then + lfs.chdir(d) + end + return d +end end -- of closure @@ -3081,12 +3360,17 @@ if not modules then modules = { } end modules ['l-boolean'] = { license = "see context related readme files" } +local type, tonumber = type, tonumber + boolean = boolean or { } +local boolean = boolean -local type, tonumber = type, tonumber +-- function boolean.tonumber(b) +-- return b and 1 or 0 -- test and test and return or return +-- end function boolean.tonumber(b) - if b then return 1 else return 0 end + if b then return 1 else return 0 end -- test and return or return end function toboolean(str,tolerant) @@ -3110,7 +3394,9 @@ function toboolean(str,tolerant) end end -function string.is_boolean(str) +string.toboolean = toboolean + +function string.is_boolean(str,default) if type(str) == "string" then if str == "true" or str == "yes" or str == "on" or str == "t" then return true @@ -3118,18 +3404,301 @@ function string.is_boolean(str) return false end end - return nil + return default end -function boolean.alwaystrue() - return true + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['l-unicode'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +if not unicode then + + unicode = { utf8 = { } } + + local floor, char = math.floor, string.char + + function unicode.utf8.utfchar(n) + if n < 0x80 then + return char(n) + elseif n < 0x800 then + return char( + 0xC0 + floor(n/0x40), + 0x80 + (n % 0x40) + ) + elseif n < 0x10000 then + return char( + 0xE0 + floor(n/0x1000), + 0x80 + (floor(n/0x40) % 0x40), + 0x80 + (n % 0x40) + ) + elseif n < 0x40000 then + return char( + 0xF0 + floor(n/0x40000), + 0x80 + floor(n/0x1000), + 0x80 + (floor(n/0x40) % 0x40), + 0x80 + (n % 0x40) + ) + else + -- return char( + -- 0xF1 + floor(n/0x1000000), + -- 0x80 + floor(n/0x40000), + -- 0x80 + floor(n/0x1000), + -- 0x80 + (floor(n/0x40) % 0x40), + -- 0x80 + (n % 0x40) + -- ) + return "?" + end + end + end -function boolean.falsetrue() - return false +local unicode = unicode + +utf = utf or unicode.utf8 + +local concat = table.concat +local utfchar, utfbyte, utfgsub = utf.char, utf.byte, utf.gsub +local char, byte, find, bytepairs, utfvalues, format = string.char, string.byte, string.find, string.bytepairs, string.utfvalues, string.format +local type = type + +local utfsplitlines = string.utfsplitlines + +-- 0 EF BB BF UTF-8 +-- 1 FF FE UTF-16-little-endian +-- 2 FE FF UTF-16-big-endian +-- 3 FF FE 00 00 UTF-32-little-endian +-- 4 00 00 FE FF UTF-32-big-endian + +unicode.utfname = { + [0] = 'utf-8', + [1] = 'utf-16-le', + [2] = 'utf-16-be', + [3] = 'utf-32-le', + [4] = 'utf-32-be' +} + +-- \000 fails in <= 5.0 but is valid in >=5.1 where %z is depricated + +function unicode.utftype(f) + local str = f:read(4) + if not str then + f:seek('set') + return 0 + -- elseif find(str,"^%z%z\254\255") then -- depricated + -- elseif find(str,"^\000\000\254\255") then -- not permitted and bugged + elseif find(str,"\000\000\254\255",1,true) then -- seems to work okay (TH) + return 4 + -- elseif find(str,"^\255\254%z%z") then -- depricated + -- elseif find(str,"^\255\254\000\000") then -- not permitted and bugged + elseif find(str,"\255\254\000\000",1,true) then -- seems to work okay (TH) + return 3 + elseif find(str,"^\254\255") then + f:seek('set',2) + return 2 + elseif find(str,"^\255\254") then + f:seek('set',2) + return 1 + elseif find(str,"^\239\187\191") then + f:seek('set',3) + return 0 + else + f:seek('set') + return 0 + end +end + + + +local function utf16_to_utf8_be(t) + if type(t) == "string" then + t = utfsplitlines(str) + end + local result = { } -- we reuse result + for i=1,#t do + local r, more = 0, 0 + for left, right in bytepairs(t[i]) do + if right then + local now = 256*left + right + if more > 0 then + now = (more-0xD800)*0x400 + (now-0xDC00) + 0x10000 + more = 0 + r = r + 1 + result[r] = utfchar(now) + elseif now >= 0xD800 and now <= 0xDBFF then + more = now + else + r = r + 1 + result[r] = utfchar(now) + end + end + end + t[i] = concat(result,"",1,r) -- we reused tmp, hence t + end + return t +end + +local function utf16_to_utf8_le(t) + if type(t) == "string" then + t = utfsplitlines(str) + end + local result = { } -- we reuse result + for i=1,#t do + local r, more = 0, 0 + for left, right in bytepairs(t[i]) do + if right then + local now = 256*right + left + if more > 0 then + now = (more-0xD800)*0x400 + (now-0xDC00) + 0x10000 + more = 0 + r = r + 1 + result[r] = utfchar(now) + elseif now >= 0xD800 and now <= 0xDBFF then + more = now + else + r = r + 1 + result[r] = utfchar(now) + end + end + end + t[i] = concat(result,"",1,r) -- we reused tmp, hence t + end + return t +end + +local function utf32_to_utf8_be(str) + if type(t) == "string" then + t = utfsplitlines(str) + end + local result = { } -- we reuse result + for i=1,#t do + local r, more = 0, -1 + for a,b in bytepairs(str) do + if a and b then + if more < 0 then + more = 256*256*256*a + 256*256*b + else + r = r + 1 + result[t] = utfchar(more + 256*a + b) + more = -1 + end + else + break + end + end + t[i] = concat(result,"",1,r) + end + return result +end + +local function utf32_to_utf8_le(str) + if type(t) == "string" then + t = utfsplitlines(str) + end + local result = { } -- we reuse result + for i=1,#t do + local r, more = 0, -1 + for a,b in bytepairs(str) do + if a and b then + if more < 0 then + more = 256*b + a + else + r = r + 1 + result[t] = utfchar(more + 256*256*256*b + 256*256*a) + more = -1 + end + else + break + end + end + t[i] = concat(result,"",1,r) + end + return result +end + +unicode.utf32_to_utf8_be = utf32_to_utf8_be +unicode.utf32_to_utf8_le = utf32_to_utf8_le +unicode.utf16_to_utf8_be = utf16_to_utf8_be +unicode.utf16_to_utf8_le = utf16_to_utf8_le + +function unicode.utf8_to_utf8(t) + return type(t) == "string" and utfsplitlines(t) or t +end + +function unicode.utf16_to_utf8(t,endian) + return endian and utf16_to_utf8_be(t) or utf16_to_utf8_le(t) or t +end + +function unicode.utf32_to_utf8(t,endian) + return endian and utf32_to_utf8_be(t) or utf32_to_utf8_le(t) or t +end + +local function little(c) + local b = byte(c) + if b < 0x10000 then + return char(b%256,b/256) + else + b = b - 0x10000 + local b1, b2 = b/1024 + 0xD800, b%1024 + 0xDC00 + return char(b1%256,b1/256,b2%256,b2/256) + end +end + +local function big(c) + local b = byte(c) + if b < 0x10000 then + return char(b/256,b%256) + else + b = b - 0x10000 + local b1, b2 = b/1024 + 0xD800, b%1024 + 0xDC00 + return char(b1/256,b1%256,b2/256,b2%256) + end +end + +function unicode.utf8_to_utf16(str,littleendian) + if littleendian then + return char(255,254) .. utfgsub(str,".",little) + else + return char(254,255) .. utfgsub(str,".",big) + end +end + +function unicode.utfcodes(str) + local t, n = { }, 0 + for u in utfvalues(str) do + n = n + 1 + t[n] = format("0x%04X",u) + end + return concat(t,separator or " ") +end + +function unicode.ustring(s) + return format("U+%05X",type(s) == "number" and s or utfbyte(s)) +end + +function unicode.xstring(s) + return format("0x%05X",type(s) == "number" and s or utfbyte(s)) +end + + +local lpegmatch = lpeg.match +local utftype = lpeg.patterns.utftype + +function unicode.filetype(data) + return data and lpegmatch(utftype,data) or "unknown" end + + end -- of closure do -- create closure to overcome 200 locals limit @@ -3145,35 +3714,289 @@ if not modules then modules = { } end modules ['l-math'] = { local floor, sin, cos, tan = math.floor, math.sin, math.cos, math.tan if not math.round then - function math.round(x) - return floor(x + 0.5) - end + function math.round(x) return floor(x + 0.5) end end if not math.div then - function math.div(n,m) - return floor(n/m) - end + function math.div(n,m) return floor(n/m) end end if not math.mod then - function math.mod(n,m) - return n % m - end + function math.mod(n,m) return n % m end end local pipi = 2*math.pi/360 -function math.sind(d) - return sin(d*pipi) +if not math.sind then + function math.sind(d) return sin(d*pipi) end + function math.cosd(d) return cos(d*pipi) end + function math.tand(d) return tan(d*pipi) end end -function math.cosd(d) - return cos(d*pipi) +if not math.odd then + function math.odd (n) return n % 2 == 0 end + function math.even(n) return n % 2 ~= 0 end end -function math.tand(d) - return tan(d*pipi) + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['util-tab'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +utilities = utilities or {} +utilities.tables = utilities.tables or { } +local tables = utilities.tables + +local format, gmatch = string.format, string.gmatch +local concat, insert, remove = table.concat, table.insert, table.remove +local setmetatable, getmetatable, tonumber, tostring = setmetatable, getmetatable, tonumber, tostring + +function tables.definetable(target) -- defines undefined tables + local composed, t, n = nil, { }, 0 + for name in gmatch(target,"([^%.]+)") do + n = n + 1 + if composed then + composed = composed .. "." .. name + else + composed = name + end + t[n] = format("%s = %s or { }",composed,composed) + end + return concat(t,"\n") +end + +function tables.accesstable(target) + local t = _G + for name in gmatch(target,"([^%.]+)") do + t = t[name] + end + return t +end + +function tables.removevalue(t,value) -- todo: n + if value then + for i=1,#t do + if t[i] == value then + remove(t,i) + -- remove all, so no: return + end + end + end +end + +function tables.insertbeforevalue(t,value,extra) + for i=1,#t do + if t[i] == extra then + remove(t,i) + end + end + for i=1,#t do + if t[i] == value then + insert(t,i,extra) + return + end + end + insert(t,1,extra) +end + +function tables.insertaftervalue(t,value,extra) + for i=1,#t do + if t[i] == extra then + remove(t,i) + end + end + for i=1,#t do + if t[i] == value then + insert(t,i+1,extra) + return + end + end + insert(t,#t+1,extra) +end + +-- experimental + +local function toxml(t,d,result) + for k, v in table.sortedpairs(t) do + if type(v) == "table" then + result[#result+1] = format("%s<%s>",d,k) + toxml(v,d.." ",result) + result[#result+1] = format("%s</%s>",d,k) + elseif tonumber(k) then + result[#result+1] = format("%s<entry n='%s'>%s</entry>",d,k,v,k) + else + result[#result+1] = format("%s<%s>%s</%s>",d,k,tostring(v),k) + end + end +end + +function table.toxml(t,name,nobanner) + local noroot = name == false + local result = (nobanner or noroot) and { } or { "<?xml version='1.0' standalone='yes' ?>" } + if noroot then + toxml( t, "", result) + else + toxml( { [name or "root"] = t }, "", result) + end + return concat(result,"\n") +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['util-sto'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local setmetatable, getmetatable = setmetatable, getmetatable + +utilities = utilities or { } +utilities.storage = utilities.storage or { } +local storage = utilities.storage + +function storage.mark(t) + if not t then + texio.write_nl("fatal error: storage '%s' cannot be marked",t) + os.exit() + end + local m = getmetatable(t) + if not m then + m = { } + setmetatable(t,m) + end + m.__storage__ = true + return t +end + +function storage.allocate(t) + t = t or { } + local m = getmetatable(t) + if not m then + m = { } + setmetatable(t,m) + end + m.__storage__ = true + return t +end + +function storage.marked(t) + local m = getmetatable(t) + return m and m.__storage__ +end + +function storage.checked(t) + if not t then + texio.write_nl("fatal error: storage '%s' has not been allocated",t) + os.exit() + end + return t +end + + +function storage.setinitializer(data,initialize) + local m = getmetatable(data) or { } + m.__index = function(data,k) + m.__index = nil -- so that we can access the entries during initializing + initialize() + return data[k] + end + setmetatable(data, m) +end + +local keyisvalue = { __index = function(t,k) + t[k] = k + return k +end } + +function storage.sparse(t) + t = t or { } + setmetatable(t,keyisvalue) + return t +end + +-- table namespace ? + +local function f_empty () return "" end -- t,k +local function f_self (t,k) t[k] = k return k end +local function f_ignore() end -- t,k,v + +local t_empty = { __index = empty } +local t_self = { __index = self } +local t_ignore = { __newindex = ignore } + +function table.setmetatableindex(t,f) + local m = getmetatable(t) + if m then + if f == "empty" then + m.__index = f_empty + elseif f == "key" then + m.__index = f_self + else + m.__index = f + end + else + if f == "empty" then + setmetatable(t, t_empty) + elseif f == "key" then + setmetatable(t, t_self) + else + setmetatable(t,{ __index = f }) + end + end +end + +function table.setmetatablenewindex(t,f) + local m = getmetatable(t) + if m then + if f == "ignore" then + m.__newindex = f_ignore + else + m.__newindex = f + end + else + if f == "ignore" then + setmetatable(t, t_ignore) + else + setmetatable(t,{ __newindex = f }) + end + end +end + +function table.setmetatablecall(t,f) + local m = getmetatable(t) + if m then + m.__call = f + else + setmetatable(t,{ __call = f }) + end +end + +function table.setmetatablekey(t,key,value) + local m = getmetatable(t) + if not m then + m = { } + setmetatable(t,m) + end + m[key] = value +end + +function table.getmetatablekey(t,key,value) + local m = getmetatable(t) + return m and m[key] end @@ -3181,7 +4004,7 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['l-utils'] = { +if not modules then modules = { } end modules ['util-mrg'] = { version = 1.001, comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", @@ -3191,89 +4014,85 @@ if not modules then modules = { } end modules ['l-utils'] = { -- hm, quite unreadable -local gsub = string.gsub +local gsub, format = string.gsub, string.format local concat = table.concat local type, next = type, next -if not utils then utils = { } end -if not utils.merger then utils.merger = { } end -if not utils.lua then utils.lua = { } end +utilities = utilities or {} +utilities.merger = utilities.merger or { } -- maybe mergers +utilities.report = logs and logs.reporter("system") or print + +local merger = utilities.merger + +merger.strip_comment = true + +local m_begin_merge = "begin library merge" +local m_end_merge = "end library merge" +local m_begin_closure = "do -- create closure to overcome 200 locals limit" +local m_end_closure = "end -- of closure" -utils.merger.m_begin = "begin library merge" -utils.merger.m_end = "end library merge" -utils.merger.pattern = +local m_pattern = "%c+" .. - "%-%-%s+" .. utils.merger.m_begin .. + "%-%-%s+" .. m_begin_merge .. "%c+(.-)%c+" .. - "%-%-%s+" .. utils.merger.m_end .. + "%-%-%s+" .. m_end_merge .. "%c+" -function utils.merger._self_fake_() - return - "-- " .. "created merged file" .. "\n\n" .. - "-- " .. utils.merger.m_begin .. "\n\n" .. - "-- " .. utils.merger.m_end .. "\n\n" -end +local m_format = + "\n\n-- " .. m_begin_merge .. + "\n%s\n" .. + "-- " .. m_end_merge .. "\n\n" -function utils.report(...) - print(...) +local m_faked = + "-- " .. "created merged file" .. "\n\n" .. + "-- " .. m_begin_merge .. "\n\n" .. + "-- " .. m_end_merge .. "\n\n" + +local function self_fake() + return m_faked end -utils.merger.strip_comment = true +local function self_nothing() + return "" +end -function utils.merger._self_load_(name) - local f, data = io.open(name), "" - if f then - utils.report("reading merge from %s",name) - data = f:read("*all") - f:close() +local function self_load(name) + local data = io.loaddata(name) or "" + if data == "" then + utilities.report("merge: unknown file %s",name) else - utils.report("unknown file to merge %s",name) - end - if data and utils.merger.strip_comment then - -- saves some 20K - data = gsub(data,"%-%-~[^\n\r]*[\r\n]", "") + utilities.report("merge: inserting %s",name) end return data or "" end -function utils.merger._self_save_(name, data) +local function self_save(name, data) if data ~= "" then - local f = io.open(name,'w') - if f then - utils.report("saving merge from %s",name) - f:write(data) - f:close() + if merger.strip_comment then + -- saves some 20K + local n = #data + data = gsub(data,"%-%-~[^\n\r]*[\r\n]","") + utilities.report("merge: %s bytes of comment stripped, %s bytes of code left",n-#data,#data) end + io.savedata(name,data) + utilities.report("merge: saving %s",name) end end -function utils.merger._self_swap_(data,code) - if data ~= "" then - return (gsub(data,utils.merger.pattern, function(s) - return "\n\n" .. "-- "..utils.merger.m_begin .. "\n" .. code .. "\n" .. "-- "..utils.merger.m_end .. "\n\n" - end, 1)) - else - return "" - end +local function self_swap(data,code) + return data ~= "" and (gsub(data,m_pattern, function() return format(m_format,code) end, 1)) or "" end ---~ stripper: ---~ ---~ data = gsub(data,"%-%-~[^\n]*\n","") ---~ data = gsub(data,"\n\n+","\n") - -function utils.merger._self_libs_(libs,list) - local result, f, frozen = { }, nil, false +local function self_libs(libs,list) + local result, f, frozen, foundpath = { }, nil, false, nil result[#result+1] = "\n" if type(libs) == 'string' then libs = { libs } end if type(list) == 'string' then list = { list } end - local foundpath = nil for i=1,#libs do local lib = libs[i] for j=1,#list do local pth = gsub(list[j],"\\","/") -- file.clean_path - utils.report("checking library path %s",pth) + utilities.report("merge: checking library path %s",pth) local name = pth .. "/" .. lib if lfs.isfile(name) then foundpath = pth @@ -3282,76 +4101,75 @@ function utils.merger._self_libs_(libs,list) if foundpath then break end end if foundpath then - utils.report("using library path %s",foundpath) + utilities.report("merge: using library path %s",foundpath) local right, wrong = { }, { } for i=1,#libs do local lib = libs[i] local fullname = foundpath .. "/" .. lib if lfs.isfile(fullname) then - -- right[#right+1] = lib - utils.report("merging library %s",fullname) - result[#result+1] = "do -- create closure to overcome 200 locals limit" + utilities.report("merge: using library %s",fullname) + right[#right+1] = lib + result[#result+1] = m_begin_closure result[#result+1] = io.loaddata(fullname,true) - result[#result+1] = "end -- of closure" + result[#result+1] = m_end_closure else - -- wrong[#wrong+1] = lib - utils.report("no library %s",fullname) + utilities.report("merge: skipping library %s",fullname) + wrong[#wrong+1] = lib end end if #right > 0 then - utils.report("merged libraries: %s",concat(right," ")) + utilities.report("merge: used libraries: %s",concat(right," ")) end if #wrong > 0 then - utils.report("skipped libraries: %s",concat(wrong," ")) + utilities.report("merge: skipped libraries: %s",concat(wrong," ")) end else - utils.report("no valid library path found") + utilities.report("merge: no valid library path found") end return concat(result, "\n\n") end -function utils.merger.selfcreate(libs,list,target) +function merger.selfcreate(libs,list,target) if target then - utils.merger._self_save_( - target, - utils.merger._self_swap_( - utils.merger._self_fake_(), - utils.merger._self_libs_(libs,list) - ) - ) + self_save(target,self_swap(self_fake(),self_libs(libs,list))) end end -function utils.merger.selfmerge(name,libs,list,target) - utils.merger._self_save_( - target or name, - utils.merger._self_swap_( - utils.merger._self_load_(name), - utils.merger._self_libs_(libs,list) - ) - ) +function merger.selfmerge(name,libs,list,target) + self_save(target or name,self_swap(self_load(name),self_libs(libs,list))) end -function utils.merger.selfclean(name) - utils.merger._self_save_( - name, - utils.merger._self_swap_( - utils.merger._self_load_(name), - "" - ) - ) +function merger.selfclean(name) + self_save(name,self_swap(self_load(name),self_nothing())) end -function utils.lua.compile(luafile, lucfile, cleanup, strip) -- defaults: cleanup=false strip=true - -- utils.report("compiling",luafile,"into",lucfile) + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['util-lua'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +utilities = utilities or {} +utilities.lua = utilities.lua or { } +utilities.report = logs and logs.reporter("system") or print + +function utilities.lua.compile(luafile,lucfile,cleanup,strip) -- defaults: cleanup=false strip=true + utilities.report("lua: compiling %s into %s",luafile,lucfile) os.remove(lucfile) - local command = "-o " .. string.quote(lucfile) .. " " .. string.quote(luafile) + local command = "-o " .. string.quoted(lucfile) .. " " .. string.quoted(luafile) if strip ~= false then command = "-s " .. command end - local done = (os.spawn("texluac " .. command) == 0) or (os.spawn("luac " .. command) == 0) + local done = os.spawn("texluac " .. command) == 0 or os.spawn("luac " .. command) == 0 if done and cleanup == true and lfs.isfile(lucfile) and lfs.isfile(luafile) then - -- utils.report("removing",luafile) + utilities.report("lua: removing %s",luafile) os.remove(luafile) end return done @@ -3359,11 +4177,16 @@ end + + + + + end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['l-aux'] = { +if not modules then modules = { } end modules ['util-prs'] = { version = 1.001, comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", @@ -3371,15 +4194,20 @@ if not modules then modules = { } end modules ['l-aux'] = { license = "see context related readme files" } --- for inline, no store split : for s in string.gmatch(str,",* *([^,]+)") do .. end +local P, R, V, C, Ct, Carg = lpeg.P, lpeg.R, lpeg.V, lpeg.C, lpeg.Ct, lpeg.Carg +local lpegmatch = lpeg.match +local concat, format, gmatch = table.concat, string.format, string.gmatch +local tostring, type, next = tostring, type, next -aux = aux or { } +utilities = utilities or {} +utilities.parsers = utilities.parsers or { } +local parsers = utilities.parsers +parsers.patterns = parsers.patterns or { } -local concat, format, gmatch = table.concat, string.format, string.gmatch -local tostring, type = tostring, type -local lpegmatch = lpeg.match +local setmetatableindex = table.setmetatableindex +local sortedhash = table.sortedhash -local P, R, V = lpeg.P, lpeg.R, lpeg.V +-- we could use a Cf Cg construct local escape, left, right = P("\\"), P('{'), P('}') @@ -3388,29 +4216,35 @@ lpeg.patterns.balanced = P { [2] = left * V(1) * right } -local space = lpeg.P(' ') -local equal = lpeg.P("=") -local comma = lpeg.P(",") -local lbrace = lpeg.P("{") -local rbrace = lpeg.P("}") +local space = P(' ') +local equal = P("=") +local comma = P(",") +local lbrace = P("{") +local rbrace = P("}") local nobrace = 1 - (lbrace+rbrace) -local nested = lpeg.P { lbrace * (nobrace + lpeg.V(1))^0 * rbrace } +local nested = P { lbrace * (nobrace + V(1))^0 * rbrace } local spaces = space^0 -local value = lpeg.P(lbrace * lpeg.C((nobrace + nested)^0) * rbrace) + lpeg.C((nested + (1-comma))^0) +lpeg.patterns.nested = nested + +local value = P(lbrace * C((nobrace + nested)^0) * rbrace) + C((nested + (1-comma))^0) -local key = lpeg.C((1-equal-comma)^1) -local pattern_a = (space+comma)^0 * (key * equal * value + key * lpeg.C("")) +local key = C((1-equal-comma)^1) +local pattern_a = (space+comma)^0 * (key * equal * value + key * C("")) local pattern_c = (space+comma)^0 * (key * equal * value) -local key = lpeg.C((1-space-equal-comma)^1) -local pattern_b = spaces * comma^0 * spaces * (key * ((spaces * equal * spaces * value) + lpeg.C(""))) +local key = C((1-space-equal-comma)^1) +local pattern_b = spaces * comma^0 * spaces * (key * ((spaces * equal * spaces * value) + C(""))) -- "a=1, b=2, c=3, d={a{b,c}d}, e=12345, f=xx{a{b,c}d}xx, g={}" : outer {} removes, leading spaces ignored local hash = { } -local function set(key,value) -- using Carg is slower here +local function set(key,value) + hash[key] = value +end + +local function set(key,value) hash[key] = value end @@ -3418,11 +4252,11 @@ local pattern_a_s = (pattern_a/set)^1 local pattern_b_s = (pattern_b/set)^1 local pattern_c_s = (pattern_c/set)^1 -aux.settings_to_hash_pattern_a = pattern_a_s -aux.settings_to_hash_pattern_b = pattern_b_s -aux.settings_to_hash_pattern_c = pattern_c_s +parsers.patterns.settings_to_hash_a = pattern_a_s +parsers.patterns.settings_to_hash_b = pattern_b_s +parsers.patterns.settings_to_hash_c = pattern_c_s -function aux.make_settings_to_hash_pattern(set,how) +function parsers.make_settings_to_hash_pattern(set,how) if how == "strict" then return (pattern_c/set)^1 elseif how == "tolerant" then @@ -3432,21 +4266,17 @@ function aux.make_settings_to_hash_pattern(set,how) end end -function aux.settings_to_hash(str,existing) +function parsers.settings_to_hash(str,existing) if str and str ~= "" then hash = existing or { } - if moretolerant then - lpegmatch(pattern_b_s,str) - else - lpegmatch(pattern_a_s,str) - end + lpegmatch(pattern_a_s,str) return hash else return { } end end -function aux.settings_to_hash_tolerant(str,existing) +function parsers.settings_to_hash_tolerant(str,existing) if str and str ~= "" then hash = existing or { } lpegmatch(pattern_b_s,str) @@ -3456,7 +4286,7 @@ function aux.settings_to_hash_tolerant(str,existing) end end -function aux.settings_to_hash_strict(str,existing) +function parsers.settings_to_hash_strict(str,existing) if str and str ~= "" then hash = existing or { } lpegmatch(pattern_c_s,str) @@ -3467,16 +4297,16 @@ function aux.settings_to_hash_strict(str,existing) end local separator = comma * space^0 -local value = lpeg.P(lbrace * lpeg.C((nobrace + nested)^0) * rbrace) + lpeg.C((nested + (1-comma))^0) -local pattern = lpeg.Ct(value*(separator*value)^0) +local value = P(lbrace * C((nobrace + nested)^0) * rbrace) + C((nested + (1-comma))^0) +local pattern = Ct(value*(separator*value)^0) -- "aap, {noot}, mies" : outer {} removes, leading spaces ignored -aux.settings_to_array_pattern = pattern +parsers.patterns.settings_to_array = pattern -- we could use a weak table as cache -function aux.settings_to_array(str) +function parsers.settings_to_array(str) if not str or str == "" then return { } else @@ -3488,16 +4318,16 @@ local function set(t,v) t[#t+1] = v end -local value = lpeg.P(lpeg.Carg(1)*value) / set -local pattern = value*(separator*value)^0 * lpeg.Carg(1) +local value = P(Carg(1)*value) / set +local pattern = value*(separator*value)^0 * Carg(1) -function aux.add_settings_to_array(t,str) +function parsers.add_settings_to_array(t,str) return lpegmatch(pattern,str,nil,t) end -function aux.hash_to_string(h,separator,yes,no,strict,omit) +function parsers.hash_to_string(h,separator,yes,no,strict,omit) if h then - local t, s = { }, table.sortedkeys(h) + local t, tn, s = { }, 0, table.sortedkeys(h) omit = omit and table.tohash(omit) for i=1,#s do local key = s[i] @@ -3506,15 +4336,19 @@ function aux.hash_to_string(h,separator,yes,no,strict,omit) if type(value) == "boolean" then if yes and no then if value then - t[#t+1] = key .. '=' .. yes + tn = tn + 1 + t[tn] = key .. '=' .. yes elseif not strict then - t[#t+1] = key .. '=' .. no + tn = tn + 1 + t[tn] = key .. '=' .. no end elseif value or not strict then - t[#t+1] = key .. '=' .. tostring(value) + tn = tn + 1 + t[tn] = key .. '=' .. tostring(value) end else - t[#t+1] = key .. '=' .. value + tn = tn + 1 + t[tn] = key .. '=' .. value end end end @@ -3524,7 +4358,7 @@ function aux.hash_to_string(h,separator,yes,no,strict,omit) end end -function aux.array_to_string(a,separator) +function parsers.array_to_string(a,separator) if a then return concat(a,separator or ",") else @@ -3532,107 +4366,160 @@ function aux.array_to_string(a,separator) end end -function aux.settings_to_set(str,t) +function parsers.settings_to_set(str,t) -- tohash? -- todo: lpeg -- duplicate anyway t = t or { } - for s in gmatch(str,"%s*([^,]+)") do +-- for s in gmatch(str,"%s*([^, ]+)") do -- space added + for s in gmatch(str,"[^, ]+") do -- space added t[s] = true end return t end -local value = lbrace * lpeg.C((nobrace + nested)^0) * rbrace -local pattern = lpeg.Ct((space + value)^0) +function parsers.simple_hash_to_string(h, separator) + local t, tn = { }, 0 + for k, v in sortedhash(h) do + if v then + tn = tn + 1 + t[tn] = k + end + end + return concat(t,separator or ",") +end + +local value = lbrace * C((nobrace + nested)^0) * rbrace +local pattern = Ct((space + value)^0) -function aux.arguments_to_table(str) +function parsers.arguments_to_table(str) return lpegmatch(pattern,str) end --- temporary here +-- temporary here (unoptimized) -function aux.getparameters(self,class,parentclass,settings) +function parsers.getparameters(self,class,parentclass,settings) local sc = self[class] if not sc then - sc = table.clone(self[parent]) + sc = { } self[class] = sc + if parentclass then + local sp = self[parentclass] + if not sp then + sp = { } + self[parentclass] = sp + end + setmetatableindex(sc,sp) + end end - aux.settings_to_hash(settings,sc) + parsers.settings_to_hash(settings,sc) +end + +function parsers.listitem(str) + return gmatch(str,"[^, ]+") end + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['util-fmt'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +utilities = utilities or { } +utilities.formatters = utilities.formatters or { } +local formatters = utilities.formatters + +local concat, format = table.concat, string.format +local tostring, type = tostring, type +local strip = string.strip + +local P, R, Cs = lpeg.P, lpeg.R, lpeg.Cs +local lpegmatch = lpeg.match + -- temporary here -local digit = lpeg.R("09") -local period = lpeg.P(".") -local zero = lpeg.P("0") +local digit = R("09") +local period = P(".") +local zero = P("0") local trailingzeros = zero^0 * -digit -- suggested by Roberto R local case_1 = period * trailingzeros / "" local case_2 = period * (digit - trailingzeros)^1 * (trailingzeros / "") local number = digit^1 * (case_1 + case_2) -local stripper = lpeg.Cs((number + 1)^0) +local stripper = Cs((number + 1)^0) ---~ local sample = "bla 11.00 bla 11 bla 0.1100 bla 1.00100 bla 0.00 bla 0.001 bla 1.1100 bla 0.100100100 bla 0.00100100100" ---~ collectgarbage("collect") ---~ str = string.rep(sample,10000) ---~ local ts = os.clock() ---~ lpegmatch(stripper,str) ---~ print(#str, os.clock()-ts, lpegmatch(stripper,sample)) -lpeg.patterns.strip_zeros = stripper +lpeg.patterns.stripzeros = stripper -function aux.strip_zeros(str) +function formatters.stripzeros(str) return lpegmatch(stripper,str) end -function aux.definetable(target) -- defines undefined tables - local composed, t = nil, { } - for name in gmatch(target,"([^%.]+)") do - if composed then - composed = composed .. "." .. name - else - composed = name +function formatters.formatcolumns(result,between) + if result and #result > 0 then + between = between or " " + local widths, numbers = { }, { } + local first = result[1] + local n = #first + for i=1,n do + widths[i] = 0 + end + for i=1,#result do + local r = result[i] + for j=1,n do + local rj = r[j] + local tj = type(rj) + if tj == "number" then + numbers[j] = true + end + if tj ~= "string" then + rj = tostring(rj) + r[j] = rj + end + local w = #rj + if w > widths[j] then + widths[j] = w + end + end + end + for i=1,n do + local w = widths[i] + if numbers[i] then + if w > 80 then + widths[i] = "%s" .. between + else + widths[i] = "%0" .. w .. "i" .. between + end + else + if w > 80 then + widths[i] = "%s" .. between + elseif w > 0 then + widths[i] = "%-" .. w .. "s" .. between + else + widths[i] = "%s" + end + end + end + local template = strip(concat(widths)) + for i=1,#result do + local str = format(template,unpack(result[i])) + result[i] = strip(str) end - t[#t+1] = format("%s = %s or { }",composed,composed) - end - return concat(t,"\n") -end - -function aux.accesstable(target) - local t = _G - for name in gmatch(target,"([^%.]+)") do - t = t[name] end - return t + return result end ---~ function string.commaseparated(str) ---~ return gmatch(str,"([^,%s]+)") ---~ end - --- as we use this a lot ... - ---~ function aux.cachefunction(action,weak) ---~ local cache = { } ---~ if weak then ---~ setmetatable(cache, { __mode = "kv" } ) ---~ end ---~ local function reminder(str) ---~ local found = cache[str] ---~ if not found then ---~ found = action(str) ---~ cache[str] = found ---~ end ---~ return found ---~ end ---~ return reminder, cache ---~ end - end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['trac-tra'] = { +if not modules then modules = { } end modules ['util.deb'] = { version = 1.001, - comment = "companion to trac-tra.mkiv", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -3645,14 +4532,16 @@ if not modules then modules = { } end modules ['trac-tra'] = { local debug = require "debug" local getinfo = debug.getinfo -local type, next = type, next -local concat = table.concat -local format, find, lower, gmatch, gsub = string.format, string.find, string.lower, string.gmatch, string.gsub +local type, next, tostring = type, next, tostring +local format, find = string.format, string.find +local is_boolean = string.is_boolean -debugger = debugger or { } +utilities = utilities or { } +utilities.debugger = utilities.debugger or { } +local debugger = utilities.debugger local counters = { } -local names = { } +local names = { } -- one @@ -3670,6 +4559,7 @@ local function hook() end end end + local function getname(func) local n = names[func] if n then @@ -3685,6 +4575,7 @@ local function getname(func) return "unknown" end end + function debugger.showstats(printer,threshold) printer = printer or texio.write or print threshold = threshold or 0 @@ -3707,35 +4598,6 @@ end -- two ---~ local function hook() ---~ local n = getinfo(2) ---~ if n.what=="C" and not n.name then ---~ local f = tostring(debug.traceback()) ---~ local cf = counters[f] ---~ if cf == nil then ---~ counters[f] = 1 ---~ names[f] = n ---~ else ---~ counters[f] = cf + 1 ---~ end ---~ end ---~ end ---~ function debugger.showstats(printer,threshold) ---~ printer = printer or texio.write or print ---~ threshold = threshold or 0 ---~ local total, grandtotal, functions = 0, 0, 0 ---~ printer("\n") -- ugly but ok ---~ -- table.sort(counters) ---~ for func, count in next, counters do ---~ if count > threshold then ---~ printer(format("%8i %s", count, func)) ---~ total = total + count ---~ end ---~ grandtotal = grandtotal + count ---~ functions = functions + 1 ---~ end ---~ printer(format("functions: %s, total: %s, grand total: %s, threshold: %s\n", functions, total, grandtotal, threshold)) ---~ end -- rest @@ -3753,85 +4615,335 @@ end function debugger.disable() debug.sethook() ---~ counters[debug.getinfo(2,"f").func] = nil end -function debugger.tracing() - local n = tonumber(os.env['MTX.TRACE.CALLS']) or tonumber(os.env['MTX_TRACE_CALLS']) or 0 - if n > 0 then - function debugger.tracing() return true end ; return true + + + + +local is_node = node and node.is_node + +function inspect(i) -- global function + local ti = type(i) + if ti == "table" then + table.print(i,"table") + elseif is_node and is_node(i) then + table.print(nodes.astable(i),tostring(i)) else - function debugger.tracing() return false end ; return false + print(tostring(i)) end end ---~ debugger.enable() ---~ print(math.sin(1*.5)) ---~ print(math.sin(1*.5)) ---~ print(math.sin(1*.5)) ---~ print(math.sin(1*.5)) ---~ print(math.sin(1*.5)) +end -- of closure ---~ debugger.disable() +do -- create closure to overcome 200 locals limit ---~ print("") ---~ debugger.showstats() ---~ print("") ---~ debugger.showstats(print,3) +if not modules then modules = { } end modules ['trac-inf'] = { + version = 1.001, + comment = "companion to trac-inf.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- As we want to protect the global tables, we no longer store the timing +-- in the tables themselves but in a hidden timers table so that we don't +-- get warnings about assignments. This is more efficient than using rawset +-- and rawget. + +local format = string.format +local clock = os.gettimeofday or os.clock -- should go in environment +local write_nl = texio.write_nl + +statistics = statistics or { } +local statistics = statistics + +statistics.enable = true +statistics.threshold = 0.05 + +local statusinfo, n, registered, timers = { }, 0, { }, { } + +local function hastiming(instance) + return instance and timers[instance] +end + +local function resettiming(instance) + timers[instance or "notimer"] = { timing = 0, loadtime = 0 } +end -setters = setters or { } -setters.data = setters.data or { } +local function starttiming(instance) + local timer = timers[instance or "notimer"] + if not timer then + timer = { } + timers[instance or "notimer"] = timer + end + local it = timer.timing + if not it then + it = 0 + end + if it == 0 then + timer.starttime = clock() + if not timer.loadtime then + timer.loadtime = 0 + end + end + timer.timing = it + 1 +end ---~ local function set(t,what,value) ---~ local data, done = t.data, t.done ---~ if type(what) == "string" then ---~ what = aux.settings_to_array(what) -- inefficient but ok ---~ end ---~ for i=1,#what do ---~ local w = what[i] ---~ for d, f in next, data do ---~ if done[d] then ---~ -- prevent recursion due to wildcards ---~ elseif find(d,w) then ---~ done[d] = true ---~ for i=1,#f do ---~ f[i](value) ---~ end ---~ end ---~ end ---~ end ---~ end +local function stoptiming(instance, report) + local timer = timers[instance or "notimer"] + local it = timer.timing + if it > 1 then + timer.timing = it - 1 + else + local starttime = timer.starttime + if starttime then + local stoptime = clock() + local loadtime = stoptime - starttime + timer.stoptime = stoptime + timer.loadtime = timer.loadtime + loadtime + if report then + statistics.report("load time %0.3f",loadtime) + end + timer.timing = 0 + return loadtime + end + end + return 0 +end -local function set(t,what,value) +local function elapsedtime(instance) + local timer = timers[instance or "notimer"] + return format("%0.3f",timer and timer.loadtime or 0) +end + +local function elapsedindeed(instance) + local timer = timers[instance or "notimer"] + return (timer and timer.loadtime or 0) > statistics.threshold +end + +local function elapsedseconds(instance,rest) -- returns nil if 0 seconds + if elapsedindeed(instance) then + return format("%s seconds %s", elapsedtime(instance),rest or "") + end +end + +statistics.hastiming = hastiming +statistics.resettiming = resettiming +statistics.starttiming = starttiming +statistics.stoptiming = stoptiming +statistics.elapsedtime = elapsedtime +statistics.elapsedindeed = elapsedindeed +statistics.elapsedseconds = elapsedseconds + +-- general function + +function statistics.register(tag,fnc) + if statistics.enable and type(fnc) == "function" then + local rt = registered[tag] or (#statusinfo + 1) + statusinfo[rt] = { tag, fnc } + registered[tag] = rt + if #tag > n then n = #tag end + end +end + +function statistics.show(reporter) + if statistics.enable then + if not reporter then reporter = function(tag,data,n) write_nl(tag .. " " .. data) end end + -- this code will move + local register = statistics.register + register("luatex banner", function() + return string.lower(status.banner) + end) + register("control sequences", function() + return format("%s of %s", status.cs_count, status.hash_size+status.hash_extra) + end) + register("callbacks", function() + local total, indirect = status.callbacks or 0, status.indirect_callbacks or 0 + return format("%s direct, %s indirect, %s total", total-indirect, indirect, total) + end) + collectgarbage("collect") + register("current memory usage", statistics.memused) + register("runtime",statistics.runtime) + for i=1,#statusinfo do + local s = statusinfo[i] + local r = s[2]() + if r then + reporter(s[1],r,n) + end + end + write_nl("") -- final newline + statistics.enable = false + end +end + +local template, report_statistics, nn = nil, nil, 0 -- we only calcute it once + +function statistics.showjobstat(tag,data,n) + if not logs then + -- sorry + elseif type(data) == "table" then + for i=1,#data do + statistics.showjobstat(tag,data[i],n) + end + else + if not template or n > nn then + template, n = format("%%-%ss - %%s",n), nn + report_statistics = logs.reporter("mkiv lua stats") + end + report_statistics(format(template,tag,data)) + end +end + +function statistics.memused() -- no math.round yet -) + local round = math.round or math.floor + return format("%s MB (ctx: %s MB)",round(collectgarbage("count")/1000), round(status.luastate_bytes/1000000)) +end + +starttiming(statistics) + +function statistics.formatruntime(runtime) -- indirect so it can be overloaded and + return format("%s seconds", runtime) -- indeed that happens in cure-uti.lua +end + +function statistics.runtime() + stoptiming(statistics) + return statistics.formatruntime(elapsedtime(statistics)) +end + +function statistics.timed(action,report) + report = report or logs.reporter("system") + starttiming("run") + action() + stoptiming("run") + report("total runtime: %s",elapsedtime("run")) +end + +-- where, not really the best spot for this: + +commands = commands or { } + +function commands.resettimer(name) + resettiming(name or "whatever") + starttiming(name or "whatever") +end + +function commands.elapsedtime(name) + stoptiming(name or "whatever") + tex.sprint(elapsedtime(name or "whatever")) +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['trac-set'] = { -- might become util-set.lua + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local type, next, tostring = type, next, tostring +local concat = table.concat +local format, find, lower, gsub, escapedpattern = string.format, string.find, string.lower, string.gsub, string.escapedpattern +local is_boolean = string.is_boolean +local settings_to_hash = utilities.parsers.settings_to_hash +local allocate = utilities.storage.allocate + +utilities = utilities or { } +local utilities = utilities +utilities.setters = utilities.setters or { } +local setters = utilities.setters + +local data = { } -- maybe just local + +-- We can initialize from the cnf file. This is sort of tricky as +-- later defined setters also need to be initialized then. If set +-- this way, we need to ensure that they are not reset later on. + +local trace_initialize = false -- only for testing during development + +function setters.initialize(filename,name,values) -- filename only for diagnostics + local setter = data[name] + if setter then + local data = setter.data + if data then + for key, value in next, values do + -- key = gsub(key,"_",".") + value = is_boolean(value,value) + local functions = data[key] + if functions then + if #functions > 0 and not functions.value then + if trace_initialize then + setter.report("executing %s (%s -> %s)",key,filename,tostring(value)) + end + for i=1,#functions do + functions[i](value) + end + functions.value = value + else + if trace_initialize then + setter.report("skipping %s (%s -> %s)",key,filename,tostring(value)) + end + end + else + -- we do a simple preregistration i.e. not in the + -- list as it might be an obsolete entry + functions = { default = value } + data[key] = functions + if trace_initialize then + setter.report("storing %s (%s -> %s)",key,filename,tostring(value)) + end + end + end + return true + end + end +end + +-- user interface code + +local function set(t,what,newvalue) local data, done = t.data, t.done if type(what) == "string" then - what = aux.settings_to_hash(what) -- inefficient but ok + what = settings_to_hash(what) -- inefficient but ok + end + if type(what) ~= "table" then + return end - for w, v in next, what do - if v == "" then - v = value + for w, value in next, what do + if value == "" then + value = newvalue + elseif not value then + value = false -- catch nil else - v = toboolean(v) + value = is_boolean(value,value) end - for d, f in next, data do - if done[d] then + w = "^" .. escapedpattern(w,true) .. "$" -- new: anchored + for name, functions in next, data do + if done[name] then -- prevent recursion due to wildcards - elseif find(d,w) then - done[d] = true - for i=1,#f do - f[i](v) + elseif find(name,w) then + done[name] = true + for i=1,#functions do + functions[i](value) end + functions.value = value end end end end local function reset(t) - for d, f in next, t.data do - for i=1,#f do - f[i](false) + for name, functions in next, t.data do + for i=1,#functions do + functions[i](false) end + functions.value = false end end @@ -3852,32 +4964,51 @@ end function setters.register(t,what,...) local data = t.data what = lower(what) - local w = data[what] - if not w then - w = { } - data[what] = w + local functions = data[what] + if not functions then + functions = { } + data[what] = functions + if trace_initialize then + t.report("defining %s",what) + end end + local default = functions.default -- can be set from cnf file for _, fnc in next, { ... } do local typ = type(fnc) - if typ == "function" then - w[#w+1] = fnc - elseif typ == "string" then - w[#w+1] = function(value) set(t,fnc,value,nesting) end + if typ == "string" then + if trace_initialize then + t.report("coupling %s to %s",what,fnc) + end + local s = fnc -- else wrong reference + fnc = function(value) set(t,s,value) end + elseif typ ~= "function" then + fnc = nil + end + if fnc then + functions[#functions+1] = fnc + -- default: set at command line or in cnf file + -- value : set in tex run (needed when loading runtime) + local value = functions.value or default + if value ~= nil then + fnc(value) + functions.value = value + end end end + return false -- so we can use it in an assignment end function setters.enable(t,what) local e = t.enable t.enable, t.done = enable, { } - enable(t,string.simpleesc(tostring(what))) + enable(t,what) t.enable, t.done = e, { } end function setters.disable(t,what) local e = t.disable t.disable, t.done = disable, { } - disable(t,string.simpleesc(tostring(what))) + disable(t,what) t.disable, t.done = e, { } end @@ -3901,68 +5032,94 @@ function setters.list(t) -- pattern end function setters.show(t) - commands.writestatus("","") + local category = t.name local list = setters.list(t) + t.report() for k=1,#list do - commands.writestatus(t.name,list[k]) + local name = list[k] + local functions = t.data[name] + if functions then + local value, default, modules = functions.value, functions.default, #functions + value = value == nil and "unset" or tostring(value) + default = default == nil and "unset" or tostring(default) + t.report("%-30s modules: %2i default: %6s value: %6s",name,modules,default,value) + end end - commands.writestatus("","") + t.report() end -- we could have used a bit of oo and the trackers:enable syntax but -- there is already a lot of code around using the singular tracker --- we could make this into a module +-- we could make this into a module but we also want the rest avaliable + +local enable, disable, register, list, show = setters.enable, setters.disable, setters.register, setters.list, setters.show + +local function report(setter,...) + local report = logs and logs.report + if report then + report(setter.name,...) + else -- fallback, as this module is loaded before the logger + write_nl(format("%-15s : %s\n",setter.name,format(...))) + end +end function setters.new(name) - local t - t = { - data = { }, + local setter -- we need to access it in setter itself + setter = { + data = allocate(), -- indexed, but also default and value fields name = name, - enable = function(...) setters.enable (t,...) end, - disable = function(...) setters.disable (t,...) end, - register = function(...) setters.register(t,...) end, - list = function(...) setters.list (t,...) end, - show = function(...) setters.show (t,...) end, + report = function(...) report (setter,...) end, + enable = function(...) enable (setter,...) end, + disable = function(...) disable (setter,...) end, + register = function(...) register(setter,...) end, + list = function(...) list (setter,...) end, + show = function(...) show (setter,...) end, } - setters.data[name] = t - return t + data[name] = setter + return setter end trackers = setters.new("trackers") directives = setters.new("directives") experiments = setters.new("experiments") +local t_enable, t_disable, t_report = trackers .enable, trackers .disable, trackers .report +local d_enable, d_disable, d_report = directives .enable, directives .disable, directives .report +local e_enable, e_disable, e_report = experiments.enable, experiments.disable, experiments.report + -- nice trick: we overload two of the directives related functions with variants that -- do tracing (itself using a tracker) .. proof of concept local trace_directives = false local trace_directives = false trackers.register("system.directives", function(v) trace_directives = v end) local trace_experiments = false local trace_experiments = false trackers.register("system.experiments", function(v) trace_experiments = v end) -local e = directives.enable -local d = directives.disable - function directives.enable(...) - commands.writestatus("directives","enabling: %s",concat({...}," ")) - e(...) + if trace_directives then + d_report("enabling: %s",concat({...}," ")) + end + d_enable(...) end function directives.disable(...) - commands.writestatus("directives","disabling: %s",concat({...}," ")) - d(...) + if trace_directives then + d_report("disabling: %s",concat({...}," ")) + end + d_disable(...) end -local e = experiments.enable -local d = experiments.disable - function experiments.enable(...) - commands.writestatus("experiments","enabling: %s",concat({...}," ")) - e(...) + if trace_experiments then + e_report("enabling: %s",concat({...}," ")) + end + e_enable(...) end function experiments.disable(...) - commands.writestatus("experiments","disabling: %s",concat({...}," ")) - d(...) + if trace_experiments then + e_report("disabling: %s",concat({...}," ")) + end + e_disable(...) end -- a useful example @@ -3971,6 +5128,1121 @@ directives.register("system.nostatistics", function(v) statistics.enable = not v end) +directives.register("system.nolibraries", function(v) + libraries = nil -- we discard this tracing for security +end) + +-- experiment + +local flags = environment and environment.engineflags + +if flags then + if trackers and flags.trackers then + setters.initialize("flags","trackers", settings_to_hash(flags.trackers)) + -- t_enable(flags.trackers) + end + if directives and flags.directives then + setters.initialize("flags","directives", settings_to_hash(flags.directives)) + -- d_enable(flags.directives) + end +end + +-- here + +if texconfig then + + local function set(k,v) + v = tonumber(v) + if v then + texconfig[k] = v + end + end + + directives.register("luatex.expanddepth", function(v) set("expand_depth",v) end) + directives.register("luatex.hashextra", function(v) set("hash_extra",v) end) + directives.register("luatex.nestsize", function(v) set("nest_size",v) end) + directives.register("luatex.maxinopen", function(v) set("max_in_open",v) end) + directives.register("luatex.maxprintline", function(v) set("max_print_line",v) end) + directives.register("luatex.maxstrings", function(v) set("max_strings",v) end) + directives.register("luatex.paramsize", function(v) set("param_size",v) end) + directives.register("luatex.savesize", function(v) set("save_size",v) end) + directives.register("luatex.stacksize", function(v) set("stack_size",v) end) + +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['trac-log'] = { + version = 1.001, + comment = "companion to trac-log.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- todo: less categories, more subcategories (e.g. nodes) + + +local write_nl, write = texio and texio.write_nl or print, texio and texio.write or io.write +local format, gmatch, find = string.format, string.gmatch, string.find +local concat, insert, remove = table.concat, table.insert, table.remove +local escapedpattern = string.escapedpattern +local texcount = tex and tex.count +local next, type = next, type + +local setmetatableindex = table.setmetatableindex + +--[[ldx-- +<p>This is a prelude to a more extensive logging module. We no longer +provide <l n='xml'/> based logging a sparsing is relatively easy anyway.</p> +--ldx]]-- + +logs = logs or { } +local logs = logs + +local moreinfo = [[ +More information about ConTeXt and the tools that come with it can be found at: + +maillist : ntg-context@ntg.nl / http://www.ntg.nl/mailman/listinfo/ntg-context +webpage : http://www.pragma-ade.nl / http://tex.aanhet.net +wiki : http://contextgarden.net +]] + +-- basic loggers + +local function ignore() end + +setmetatableindex(logs, function(t,k) t[k] = ignore ; return ignore end) + +local report, subreport, status, settarget, setformats, settranslations + +local direct, subdirect, writer, pushtarget, poptarget + +if tex and tex.jobname or tex.formatname then + + local valueiskey = { __index = function(t,k) t[k] = k return k end } -- will be helper + + local target = "term and log" + + logs.flush = io.flush + + local formats = { } setmetatable(formats, valueiskey) + local translations = { } setmetatable(translations,valueiskey) + + writer = function(...) + write_nl(target,...) + end + + report = function(a,b,c,...) + if c then + write_nl(target,format("%-15s > %s\n",translations[a],format(formats[b],c,...))) + elseif b then + write_nl(target,format("%-15s > %s\n",translations[a],formats[b])) + elseif a then + write_nl(target,format("%-15s >\n", translations[a])) + else + write_nl(target,"\n") + end + end + + direct = function(a,b,c,...) + if c then + return format("%-15s > %s",translations[a],format(formats[b],c,...)) + elseif b then + return format("%-15s > %s",translations[a],formats[b]) + elseif a then + return format("%-15s >", translations[a]) + else + return "" + end + end + + subreport = function(a,s,b,c,...) + if c then + write_nl(target,format("%-15s > %s > %s\n",translations[a],translations[s],format(formats[b],c,...))) + elseif b then + write_nl(target,format("%-15s > %s > %s\n",translations[a],translations[s],formats[b])) + elseif a then + write_nl(target,format("%-15s > %s >\n", translations[a],translations[s])) + else + write_nl(target,"\n") + end + end + + subdirect = function(a,s,b,c,...) + if c then + return format("%-15s > %s > %s",translations[a],translations[s],format(formats[b],c,...)) + elseif b then + return format("%-15s > %s > %s",translations[a],translations[s],formats[b]) + elseif a then + return format("%-15s > %s >", translations[a],translations[s]) + else + return "" + end + end + + status = function(a,b,c,...) + if c then + write_nl(target,format("%-15s : %s\n",translations[a],format(formats[b],c,...))) + elseif b then + write_nl(target,format("%-15s : %s\n",translations[a],formats[b])) + elseif a then + write_nl(target,format("%-15s :\n", translations[a])) + else + write_nl(target,"\n") + end + end + + local targets = { + logfile = "log", + log = "log", + file = "log", + console = "term", + terminal = "term", + both = "term and log", + } + + settarget = function(whereto) + target = targets[whereto or "both"] or targets.both + if target == "term" or target == "term and log" then + logs.flush = io.flush + else + logs.flush = ignore + end + end + + local stack = { } + + pushtarget = function(newtarget) + insert(stack,target) + settarget(newtarget) + end + + poptarget = function() + if #stack > 0 then + settarget(remove(stack)) + end + end + + setformats = function(f) + formats = f + end + + settranslations = function(t) + translations = t + end + +else + + logs.flush = ignore + + writer = write_nl + + report = function(a,b,c,...) + if c then + write_nl(format("%-15s | %s",a,format(b,c,...))) + elseif b then + write_nl(format("%-15s | %s",a,b)) + elseif a then + write_nl(format("%-15s |", a)) + else + write_nl("") + end + end + + subreport = function(a,sub,b,c,...) + if c then + write_nl(format("%-15s | %s | %s",a,sub,format(b,c,...))) + elseif b then + write_nl(format("%-15s | %s | %s",a,sub,b)) + elseif a then + write_nl(format("%-15s | %s |", a,sub)) + else + write_nl("") + end + end + + status = function(a,b,c,...) -- not to be used in lua anyway + if c then + write_nl(format("%-15s : %s\n",a,format(b,c,...))) + elseif b then + write_nl(format("%-15s : %s\n",a,b)) -- b can have %'s + elseif a then + write_nl(format("%-15s :\n", a)) + else + write_nl("\n") + end + end + + direct = ignore + subdirect = ignore + + settarget = ignore + pushtarget = ignore + poptarget = ignore + setformats = ignore + settranslations = ignore + +end + +logs.report = report +logs.subreport = subreport +logs.status = status +logs.settarget = settarget +logs.pushtarget = pushtarget +logs.poptarget = poptarget +logs.setformats = setformats +logs.settranslations = settranslations + +logs.direct = direct +logs.subdirect = subdirect +logs.writer = writer + +-- installer + +-- todo: renew (un) locks when a new one is added and wildcard + +local data, states = { }, nil + +function logs.reporter(category,subcategory) + local logger = data[category] + if not logger then + local state = false + if states == true then + state = true + elseif type(states) == "table" then + for c, _ in next, states do + if find(category,c) then + state = true + break + end + end + end + logger = { + reporters = { }, + state = state, + } + data[category] = logger + end + local reporter = logger.reporters[subcategory or "default"] + if not reporter then + if subcategory then + reporter = function(...) + if not logger.state then + subreport(category,subcategory,...) + end + end + logger.reporters[subcategory] = reporter + else + local tag = category + reporter = function(...) + if not logger.state then + report(category,...) + end + end + logger.reporters.default = reporter + end + end + return reporter +end + +logs.new = logs.reporter -- for old times sake + +-- context specicific: this ends up in the macro stream + +local ctxreport = logs.writer + +function logs.setmessenger(m) + ctxreport = m +end + +function logs.messenger(category,subcategory) + -- we need to avoid catcode mess (todo: fast context) + if subcategory then + return function(...) + ctxreport(subdirect(category,subcategory,...)) + end + else + return function(...) + ctxreport(direct(category,...)) + end + end +end + +-- so far + +local function setblocked(category,value) + if category == true then + -- lock all + category, value = "*", true + elseif category == false then + -- unlock all + category, value = "*", false + elseif value == nil then + -- lock selective + value = true + end + if category == "*" then + states = value + for k, v in next, data do + v.state = value + end + else + states = utilities.parsers.settings_to_hash(category) + for c, _ in next, states do + if data[c] then + v.state = value + else + c = escapedpattern(c,true) + for k, v in next, data do + if find(k,c) then + v.state = value + end + end + end + end + end +end + +function logs.disable(category,value) + setblocked(category,value == nil and true or value) +end + +function logs.enable(category) + setblocked(category,false) +end + +function logs.categories() + return table.sortedkeys(data) +end + +function logs.show() + local n, c, s, max = 0, 0, 0, 0 + for category, v in table.sortedpairs(data) do + n = n + 1 + local state = v.state + local reporters = v.reporters + local nc = #category + if nc > c then + c = nc + end + for subcategory, _ in next, reporters do + local ns = #subcategory + if ns > c then + s = ns + end + local m = nc + ns + if m > max then + max = m + end + end + local subcategories = concat(table.sortedkeys(reporters),", ") + if state == true then + state = "disabled" + elseif state == false then + state = "enabled" + else + state = "unknown" + end + -- no new here + report("logging","category: '%s', subcategories: '%s', state: '%s'",category,subcategories,state) + end + report("logging","categories: %s, max category: %s, max subcategory: %s, max combined: %s",n,c,s,max) +end + +directives.register("logs.blocked", function(v) + setblocked(v,true) +end) + +directives.register("logs.target", function(v) + settarget(v) +end) + +-- tex specific loggers (might move elsewhere) + +local report_pages = logs.reporter("pages") -- not needed but saves checking when we grep for it + +local real, user, sub + +function logs.start_page_number() + real, user, sub = texcount.realpageno, texcount.userpageno, texcount.subpageno +end + +function logs.stop_page_number() + if real > 0 then + if user > 0 then + if sub > 0 then + report_pages("flushing realpage %s, userpage %s, subpage %s",real,user,sub) + else + report_pages("flushing realpage %s, userpage %s",real,user) + end + else + report_pages("flushing realpage %s",real) + end + else + report_pages("flushing page") + end + logs.flush() +end + +logs.report_job_stat = statistics and statistics.showjobstat + +local report_files = logs.reporter("files") + +local nesting = 0 +local verbose = false +local hasscheme = url.hasscheme + +-- we don't have show_open and show_close callbacks yet + +function logs.show_open(name) + -- if hasscheme(name) ~= "virtual" then + -- if verbose then + -- nesting = nesting + 1 + -- report_files("level %s, opening %s",nesting,name) + -- else + -- write(format("(%s",name)) -- tex adds a space + -- end + -- end +end + +function logs.show_close(name) + -- if hasscheme(name) ~= "virtual" then + -- if verbose then + -- report_files("level %s, closing %s",nesting,name) + -- nesting = nesting - 1 + -- else + -- write(")") -- tex adds a space + -- end + -- end +end + +function logs.show_load(name) + -- if hasscheme(name) ~= "virtual" then + -- if verbose then + -- report_files("level %s, loading %s",nesting+1,name) + -- else + -- write(format("(%s)",name)) + -- end + -- end +end + +-- there may be scripts out there using this: + +local simple = logs.reporter("comment") + +logs.simple = simple +logs.simpleline = simple + +-- obsolete + +function logs.setprogram () end -- obsolete +function logs.extendbanner() end -- obsolete +function logs.reportlines () end -- obsolete +function logs.reportbanner() end -- obsolete +function logs.reportline () end -- obsolete +function logs.simplelines () end -- obsolete +function logs.help () end -- obsolete + +-- applications + +local function reportlines(t,str) + if str then + for line in gmatch(str,"(.-)[\n\r]") do + t.report(line) + end + end +end + +local function reportbanner(t) + local banner = t.banner + if banner then + t.report(banner) + t.report() + end +end + +local function reporthelp(t,...) + local helpinfo = t.helpinfo + if type(helpinfo) == "string" then + reportlines(t,helpinfo) + elseif type(helpinfo) == "table" then + local tags = { ... } + for i=1,#tags do + reportlines(t,t.helpinfo[tags[i]]) + if i < #tags then + t.report() + end + end + end +end + +local function reportinfo(t) + t.report() + reportlines(t,moreinfo) +end + +function logs.application(t) + t.name = t.name or "unknown" + t.banner = t.banner + t.report = logs.reporter(t.name) + t.help = function(...) reportbanner(t) ; reporthelp(t,...) ; reportinfo(t) end + t.identify = function() reportbanner(t) end + return t +end + +-- somewhat special + +-- logging to a file + + +function logs.system(whereto,process,jobname,category,...) + local message = format("%s %s => %s => %s => %s\r",os.date("%d/%m/%y %H:%m:%S"),process,jobname,category,format(...)) + for i=1,10 do + local f = io.open(whereto,"a") -- we can consider keepint the file open + if f then + f:write(message) + f:close() + break + else + sleep(0.1) + end + end +end + +local report_system = logs.reporter("system","logs") + +function logs.obsolete(old,new) + local o = loadstring("return " .. new)() + if type(o) == "function" then + return function(...) + report_system("function %s is obsolete, use %s",old,new) + loadstring(old .. "=" .. new .. " return ".. old)()(...) + end + elseif type(o) == "table" then + local t, m = { }, { } + m.__index = function(t,k) + report_system("table %s is obsolete, use %s",old,new) + m.__index, m.__newindex = o, o + return o[k] + end + m.__newindex = function(t,k,v) + report_system("table %s is obsolete, use %s",old,new) + m.__index, m.__newindex = o, o + o[k] = v + end + if libraries then + libraries.obsolete[old] = t -- true + end + setmetatable(t,m) + return t + end +end + +if utilities then + utilities.report = report_system +end + +if tex and tex.error then + function logs.texerrormessage(...) -- for the moment we put this function here + tex.error(format(...), { }) + end +else + function logs.texerrormessage(...) + print(format(...)) + end +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['trac-pro'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local getmetatable, setmetatable, rawset, type = getmetatable, setmetatable, rawset, type + +-- The protection implemented here is probably not that tight but good enough to catch +-- problems due to naive usage. +-- +-- There's a more extensive version (trac-xxx.lua) that supports nesting. +-- +-- This will change when we have _ENV in lua 5.2+ + +local trace_namespaces = false trackers.register("system.namespaces", function(v) trace_namespaces = v end) + +local report_system = logs.reporter("system","protection") + +namespaces = namespaces or { } +local namespaces = namespaces + +local registered = { } + +local function report_index(k,name) + if trace_namespaces then + report_system("reference to '%s' in protected namespace '%s', %s",k,name,debug.traceback()) + else + report_system("reference to '%s' in protected namespace '%s'",k,name) + end +end + +local function report_newindex(k,name) + if trace_namespaces then + report_system("assignment to '%s' in protected namespace '%s', %s",k,name,debug.traceback()) + else + report_system("assignment to '%s' in protected namespace '%s'",k,name) + end +end + +local function register(name) + local data = name == "global" and _G or _G[name] + if not data then + return -- error + end + registered[name] = data + local m = getmetatable(data) + if not m then + m = { } + setmetatable(data,m) + end + local index, newindex = { }, { } + m.__saved__index = m.__index + m.__no__index = function(t,k) + if not index[k] then + index[k] = true + report_index(k,name) + end + return nil + end + m.__saved__newindex = m.__newindex + m.__no__newindex = function(t,k,v) + if not newindex[k] then + newindex[k] = true + report_newindex(k,name) + end + rawset(t,k,v) + end + m.__protection__depth = 0 +end + +local function private(name) -- maybe save name + local data = registered[name] + if not data then + data = _G[name] + if not data then + data = { } + _G[name] = data + end + register(name) + end + return data +end + +local function protect(name) + local data = registered[name] + if not data then + return + end + local m = getmetatable(data) + local pd = m.__protection__depth + if pd > 0 then + m.__protection__depth = pd + 1 + else + m.__save_d_index, m.__saved__newindex = m.__index, m.__newindex + m.__index, m.__newindex = m.__no__index, m.__no__newindex + m.__protection__depth = 1 + end +end + +local function unprotect(name) + local data = registered[name] + if not data then + return + end + local m = getmetatable(data) + local pd = m.__protection__depth + if pd > 1 then + m.__protection__depth = pd - 1 + else + m.__index, m.__newindex = m.__saved__index, m.__saved__newindex + m.__protection__depth = 0 + end +end + +local function protectall() + for name, _ in next, registered do + if name ~= "global" then + protect(name) + end + end +end + +local function unprotectall() + for name, _ in next, registered do + if name ~= "global" then + unprotect(name) + end + end +end + +namespaces.register = register -- register when defined +namespaces.private = private -- allocate and register if needed +namespaces.protect = protect +namespaces.unprotect = unprotect +namespaces.protectall = protectall +namespaces.unprotectall = unprotectall + +namespaces.private("namespaces") registered = { } register("global") -- unreachable + +directives.register("system.protect", function(v) + if v then + protectall() + else + unprotectall() + end +end) + +directives.register("system.checkglobals", function(v) + if v then + report_system("enabling global namespace guard") + protect("global") + else + report_system("disabling global namespace guard") + unprotect("global") + end +end) + +-- dummy section (will go to luat-dum.lua) + + + + + + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['luat-env'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- A former version provided functionality for non embeded core +-- scripts i.e. runtime library loading. Given the amount of +-- Lua code we use now, this no longer makes sense. Much of this +-- evolved before bytecode arrays were available and so a lot of +-- code has disappeared already. + +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) + +local report_lua = logs.reporter("resolvers","lua") + +local allocate, mark = utilities.storage.allocate, utilities.storage.mark + +local format, sub, match, gsub, find = string.format, string.sub, string.match, string.gsub, string.find +local unquoted, quoted = string.unquoted, string.quoted +local concat = table.concat + +-- precautions + +os.setlocale(nil,nil) -- useless feature and even dangerous in luatex + +function os.setlocale() + -- no way you can mess with it +end + +-- dirty tricks + +if arg and (arg[0] == 'luatex' or arg[0] == 'luatex.exe') and arg[1] == "--luaonly" then + arg[-1] = arg[0] + arg[ 0] = arg[2] + for k=3,#arg do + arg[k-2] = arg[k] + end + arg[#arg] = nil -- last + arg[#arg] = nil -- pre-last +end + +-- environment + +environment = environment or { } +local environment = environment + +environment.arguments = allocate() +environment.files = allocate() +environment.sortedflags = nil + +local mt = { + __index = function(_,k) + if k == "version" then + local version = tex.toks and tex.toks.contextversiontoks + if version and version ~= "" then + rawset(environment,"version",version) + return version + else + return "unknown" + end + elseif k == "jobname" or k == "formatname" then + local name = tex and tex[k] + if name or name== "" then + rawset(environment,k,name) + return name + else + return "unknown" + end + elseif k == "outputfilename" then + local name = environment.jobname + rawset(environment,k,name) + return name + end + end +} + +setmetatable(environment,mt) + +function environment.initializearguments(arg) + local arguments, files = { }, { } + environment.arguments, environment.files, environment.sortedflags = arguments, files, nil + for index=1,#arg do + local argument = arg[index] + if index > 0 then + local flag, value = match(argument,"^%-+(.-)=(.-)$") + if flag then + arguments[flag] = unquoted(value or "") + else + flag = match(argument,"^%-+(.+)") + if flag then + arguments[flag] = true + else + files[#files+1] = argument + end + end + end + end + environment.ownname = environment.ownname or arg[0] or 'unknown.lua' +end + +function environment.setargument(name,value) + environment.arguments[name] = value +end + +-- todo: defaults, better checks e.g on type (boolean versus string) +-- +-- tricky: too many hits when we support partials unless we add +-- a registration of arguments so from now on we have 'partial' + +function environment.argument(name,partial) + local arguments, sortedflags = environment.arguments, environment.sortedflags + if arguments[name] then + return arguments[name] + elseif partial then + if not sortedflags then + sortedflags = allocate(table.sortedkeys(arguments)) + for k=1,#sortedflags do + sortedflags[k] = "^" .. sortedflags[k] + end + environment.sortedflags = sortedflags + end + -- example of potential clash: ^mode ^modefile + for k=1,#sortedflags do + local v = sortedflags[k] + if find(name,v) then + return arguments[sub(v,2,#v)] + end + end + end + return nil +end + +function environment.splitarguments(separator) -- rather special, cut-off before separator + local done, before, after = false, { }, { } + local originalarguments = environment.originalarguments + for k=1,#originalarguments do + local v = originalarguments[k] + if not done and v == separator then + done = true + elseif done then + after[#after+1] = v + else + before[#before+1] = v + end + end + return before, after +end + +function environment.reconstructcommandline(arg,noquote) + arg = arg or environment.originalarguments + if noquote and #arg == 1 then + -- we could just do: return unquoted(resolvers.resolve(arg[i])) + local a = arg[1] + a = resolvers.resolve(a) + a = unquoted(a) + return a + elseif #arg > 0 then + local result = { } + for i=1,#arg do + -- we could just do: result[#result+1] = format("%q",unquoted(resolvers.resolve(arg[i]))) + local a = arg[i] + a = resolvers.resolve(a) + a = unquoted(a) + a = gsub(a,'"','\\"') -- tricky + if find(a," ") then + result[#result+1] = quoted(a) + else + result[#result+1] = a + end + end + return concat(result," ") + else + return "" + end +end + + +if arg then + + -- new, reconstruct quoted snippets (maybe better just remove the " then and add them later) + local newarg, instring = { }, false + + for index=1,#arg do + local argument = arg[index] + if find(argument,"^\"") then + newarg[#newarg+1] = gsub(argument,"^\"","") + if not find(argument,"\"$") then + instring = true + end + elseif find(argument,"\"$") then + newarg[#newarg] = newarg[#newarg] .. " " .. gsub(argument,"\"$","") + instring = false + elseif instring then + newarg[#newarg] = newarg[#newarg] .. " " .. argument + else + newarg[#newarg+1] = argument + end + end + for i=1,-5,-1 do + newarg[i] = arg[i] + end + + environment.initializearguments(newarg) + + environment.originalarguments = mark(newarg) + environment.rawarguments = mark(arg) + + arg = { } -- prevent duplicate handling + +end + +-- weird place ... depends on a not yet loaded module + +function environment.texfile(filename) + return resolvers.findfile(filename,'tex') +end + +function environment.luafile(filename) + local resolved = resolvers.findfile(filename,'tex') or "" + if resolved ~= "" then + return resolved + end + resolved = resolvers.findfile(filename,'texmfscripts') or "" + if resolved ~= "" then + return resolved + end + return resolvers.findfile(filename,'luatexlibs') or "" +end + +environment.loadedluacode = loadfile -- can be overloaded + +function environment.luafilechunk(filename,silent) -- used for loading lua bytecode in the format + filename = file.replacesuffix(filename, "lua") + local fullname = environment.luafile(filename) + if fullname and fullname ~= "" then + local data = environment.loadedluacode(fullname) + if trace_locating then + report_lua("loading file %s%s", fullname, not data and " failed" or "") + elseif not silent then + texio.write("<",data and "+ " or "- ",fullname,">") + end + return data + else + if trace_locating then + report_lua("unknown file %s", filename) + end + return nil + end +end + +-- the next ones can use the previous ones / combine + +function environment.loadluafile(filename, version) + local lucname, luaname, chunk + local basename = file.removesuffix(filename) + if basename == filename then + lucname, luaname = basename .. ".luc", basename .. ".lua" + else + lucname, luaname = nil, basename -- forced suffix + end + -- when not overloaded by explicit suffix we look for a luc file first + local fullname = (lucname and environment.luafile(lucname)) or "" + if fullname ~= "" then + if trace_locating then + report_lua("loading %s", fullname) + end + chunk = loadfile(fullname) -- this way we don't need a file exists check + end + if chunk then + assert(chunk)() + if version then + -- we check of the version number of this chunk matches + local v = version -- can be nil + if modules and modules[filename] then + v = modules[filename].version -- new method + elseif versions and versions[filename] then + v = versions[filename] -- old method + end + if v == version then + return true + else + if trace_locating then + report_lua("version mismatch for %s: lua=%s, luc=%s", filename, v, version) + end + environment.loadluafile(filename) + end + else + return true + end + end + fullname = (luaname and environment.luafile(luaname)) or "" + if fullname ~= "" then + if trace_locating then + report_lua("loading %s", fullname) + end + chunk = loadfile(fullname) -- this way we don't need a file exists check + if not chunk then + if trace_locating then + report_lua("unknown file %s", filename) + end + else + assert(chunk)() + return true + end + end + return false +end end -- of closure @@ -3991,6 +6263,8 @@ if not modules then modules = { } end modules ['lxml-tab'] = { local trace_entities = false trackers.register("xml.entities", function(v) trace_entities = v end) +local report_xml = logs and logs.reporter("xml","core") or function(...) print(format(...)) end + --[[ldx-- <p>The parser used here is inspired by the variant discussed in the lua book, but handles comment and processing instructions, has a different structure, provides @@ -4001,11 +6275,15 @@ The find based parser can be found in l-xml-edu.lua along with other older code. <p>Beware, the interface may change. For instance at, ns, tg, dt may get more verbose names. Once the code is stable we will also remove some tracing and optimize the code.</p> + +<p>I might even decide to reimplement the parser using the latest <l n='lpeg'/> trickery +as the current variant was written when <l n='lpeg'/> showed up and it's easier now to +build tables in one go.</p> --ldx]]-- xml = xml or { } +local xml = xml ---~ local xml = xml local concat, remove, insert = table.concat, table.remove, table.insert local type, next, setmetatable, getmetatable, tonumber = type, next, setmetatable, getmetatable, tonumber @@ -4129,7 +6407,7 @@ local dcache, hcache, acache = { }, { }, { } local mt = { } -function initialize_mt(root) +local function initialize_mt(root) mt = { __index = root } -- will be redefined later end @@ -4137,8 +6415,8 @@ function xml.setproperty(root,k,v) getmetatable(root).__index[k] = v end -function xml.check_error(top,toclose) - return "" +function xml.checkerror(top,toclose) + return "" -- can be set end local function add_attribute(namespace,tag,value) @@ -4194,9 +6472,9 @@ local function add_end(spacing, namespace, tag) local toclose = remove(stack) top = stack[#stack] if #stack < 1 then - errorstr = format("nothing to close with %s %s", tag, xml.check_error(top,toclose) or "") + errorstr = format("nothing to close with %s %s", tag, xml.checkerror(top,toclose) or "") elseif toclose.tg ~= tag then -- no namespace check - errorstr = format("unable to close %s with %s %s", toclose.tg, tag, xml.check_error(top,toclose) or "") + errorstr = format("unable to close %s with %s %s", toclose.tg, tag, xml.checkerror(top,toclose) or "") end dt = top.dt dt[#dt+1] = toclose @@ -4233,24 +6511,29 @@ local reported_attribute_errors = { } local function attribute_value_error(str) if not reported_attribute_errors[str] then - logs.report("xml","invalid attribute value: %q",str) + report_xml("invalid attribute value: %q",str) reported_attribute_errors[str] = true at._error_ = str end return str end + local function attribute_specification_error(str) if not reported_attribute_errors[str] then - logs.report("xml","invalid attribute specification: %q",str) + report_xml("invalid attribute specification: %q",str) reported_attribute_errors[str] = true at._error_ = str end return str end -function xml.unknown_dec_entity_format(str) return (str == "" and "&error;") or format("&%s;",str) end -function xml.unknown_hex_entity_format(str) return format("&#x%s;",str) end -function xml.unknown_any_entity_format(str) return format("&#x%s;",str) end +xml.placeholders = { + unknown_dec_entity = function(str) return (str == "" and "&error;") or format("&%s;",str) end, + unknown_hex_entity = function(str) return format("&#x%s;",str) end, + unknown_any_entity = function(str) return format("&#x%s;",str) end, +} + +local placeholders = xml.placeholders local function fromhex(s) local n = tonumber(s,16) @@ -4304,18 +6587,18 @@ local function handle_hex_entity(str) h = unify_predefined and predefined_unified[n] if h then if trace_entities then - logs.report("xml","utfize, converting hex entity &#x%s; into %s",str,h) + report_xml("utfize, converting hex entity &#x%s; into %s",str,h) end elseif utfize then - h = (n and utfchar(n)) or xml.unknown_hex_entity_format(str) or "" + h = (n and utfchar(n)) or xml.unknown_hex_entity(str) or "" if not n then - logs.report("xml","utfize, ignoring hex entity &#x%s;",str) + report_xml("utfize, ignoring hex entity &#x%s;",str) elseif trace_entities then - logs.report("xml","utfize, converting hex entity &#x%s; into %s",str,h) + report_xml("utfize, converting hex entity &#x%s; into %s",str,h) end else if trace_entities then - logs.report("xml","found entity &#x%s;",str) + report_xml("found entity &#x%s;",str) end h = "&#x" .. str .. ";" end @@ -4331,18 +6614,18 @@ local function handle_dec_entity(str) d = unify_predefined and predefined_unified[n] if d then if trace_entities then - logs.report("xml","utfize, converting dec entity &#%s; into %s",str,d) + report_xml("utfize, converting dec entity &#%s; into %s",str,d) end elseif utfize then - d = (n and utfchar(n)) or xml.unknown_dec_entity_format(str) or "" + d = (n and utfchar(n)) or placeholders.unknown_dec_entity(str) or "" if not n then - logs.report("xml","utfize, ignoring dec entity &#%s;",str) + report_xml("utfize, ignoring dec entity &#%s;",str) elseif trace_entities then - logs.report("xml","utfize, converting dec entity &#%s; into %s",str,h) + report_xml("utfize, converting dec entity &#%s; into %s",str,h) end else if trace_entities then - logs.report("xml","found entity &#%s;",str) + report_xml("found entity &#%s;",str) end d = "&#" .. str .. ";" end @@ -4367,20 +6650,21 @@ local function handle_any_entity(str) end if a then if trace_entities then - logs.report("xml","resolved entity &%s; -> %s (internal)",str,a) + report_xml("resolved entity &%s; -> %s (internal)",str,a) end a = lpegmatch(parsedentity,a) or a else - if xml.unknown_any_entity_format then - a = xml.unknown_any_entity_format(str) or "" + local unknown_any_entity = placeholders.unknown_any_entity + if unknown_any_entity then + a = unknown_any_entity(str) or "" end if a then if trace_entities then - logs.report("xml","resolved entity &%s; -> %s (external)",str,a) + report_xml("resolved entity &%s; -> %s (external)",str,a) end else if trace_entities then - logs.report("xml","keeping entity &%s;",str) + report_xml("keeping entity &%s;",str) end if str == "" then a = "&error;" @@ -4392,7 +6676,7 @@ local function handle_any_entity(str) acache[str] = a elseif trace_entities then if not acache[str] then - logs.report("xml","converting entity &%s; into %s",str,a) + report_xml("converting entity &%s; into %s",str,a) acache[str] = a end end @@ -4401,7 +6685,7 @@ local function handle_any_entity(str) local a = acache[str] if not a then if trace_entities then - logs.report("xml","found entity &%s;",str) + report_xml("found entity &%s;",str) end a = resolve_predefined and predefined_simplified[str] if a then @@ -4420,7 +6704,7 @@ local function handle_any_entity(str) end local function handle_end_entity(chr) - logs.report("xml","error in entity, %q found instead of ';'",chr) + report_xml("error in entity, %q found instead of ';'",chr) end local space = S(' \r\n\t') @@ -4555,7 +6839,7 @@ local function xmlconvert(data, settings) resolve_predefined = settings.resolve_predefined_entities -- in case we have escaped entities unify_predefined = settings.unify_predefined_entities -- & -> & cleanup = settings.text_cleanup - stack, top, at, xmlns, errorstr, result, entities = { }, { }, { }, { }, nil, nil, settings.entities or { } + stack, top, at, xmlns, errorstr, entities = { }, { }, { }, { }, nil, settings.entities or { } acache, hcache, dcache = { }, { }, { } -- not stored reported_attribute_errors = { } if settings.parent_root then @@ -4583,16 +6867,17 @@ local function xmlconvert(data, settings) else errorstr = "invalid xml file - no text at all" end + local result if errorstr and errorstr ~= "" then result = { dt = { { ns = "", tg = "error", dt = { errorstr }, at={ }, er = true } } } setmetatable(stack, mt) - local error_handler = settings.error_handler - if error_handler == false then + local errorhandler = settings.error_handler + if errorhandler == false then -- no error message else - error_handler = error_handler or xml.error_handler - if error_handler then - xml.error_handler("load",errorstr) + errorhandler = errorhandler or xml.errorhandler + if errorhandler then + xml.errorhandler("load",errorstr) end end else @@ -4652,7 +6937,7 @@ function xml.is_valid(root) return root and not root.error end -xml.error_handler = (logs and logs.report) or (input and logs.report) or print +xml.errorhandler = report --[[ldx-- <p>We cannot load an <l n='lpeg'/> from a filehandle so we need to load @@ -4763,7 +7048,7 @@ local function verbose_element(e,handlers) ats[#ats+1] = format('%s=%q',k,v) end end - if ern and trace_remap and ern ~= ens then + if ern and trace_entities and ern ~= ens then ens = ern end if ens ~= "" then @@ -4868,7 +7153,7 @@ local function serialize(e,handlers,...) -- elseif type(e) == "string" then -- functions["@tx@"](e,handlers) else - functions["@dc@"](e,handlers) + functions["@dc@"](e,handlers) -- dc ? end if finalize then return finalize() @@ -4894,7 +7179,7 @@ local function newhandlers(settings) if settings then for k,v in next, settings do if type(v) == "table" then - tk = t[k] if not tk then tk = { } t[k] = tk end + local tk = t[k] if not tk then tk = { } t[k] = tk end for kk,vv in next, v do tk[kk] = vv end @@ -4906,6 +7191,7 @@ local function newhandlers(settings) handlers[settings.name] = t end end + utilities.storage.mark(t) return t end @@ -5005,7 +7291,7 @@ local function xmltext(root) -- inline return (root and xmltostring(root)) or "" end -function initialize_mt(root) +initialize_mt = function(root) -- redefinition mt = { __tostring = xmltext, __index = root } end @@ -5040,7 +7326,6 @@ xml.string = xmlstring <p>A few helpers:</p> --ldx]]-- ---~ xmlsetproperty(root,"settings",settings) function xml.settings(e) while e do @@ -5150,6 +7435,7 @@ function xml.makestandalone(root) end end end + return root end @@ -5166,11 +7452,14 @@ if not modules then modules = { } end modules ['lxml-pth'] = { } -- e.ni is only valid after a filter run +-- todo: B/C/[get first match] local concat, remove, insert = table.concat, table.remove, table.insert local type, next, tonumber, tostring, setmetatable, loadstring = type, next, tonumber, tostring, setmetatable, loadstring local format, upper, lower, gmatch, gsub, find, rep = string.format, string.upper, string.lower, string.gmatch, string.gsub, string.find, string.rep -local lpegmatch = lpeg.match +local lpegmatch, lpegpatterns = lpeg.match, lpeg.patterns + +local setmetatableindex = table.setmetatableindex -- beware, this is not xpath ... e.g. position is different (currently) and -- we have reverse-sibling as reversed preceding sibling @@ -5202,23 +7491,33 @@ local trace_lpath = false if trackers then trackers.register("xml.path", local trace_lparse = false if trackers then trackers.register("xml.parse", function(v) trace_lparse = v end) end local trace_lprofile = false if trackers then trackers.register("xml.profile", function(v) trace_lpath = v trace_lparse = v trace_lprofile = v end) end +local report_lpath = logs.reporter("xml","lpath") + --[[ldx-- <p>We've now arrived at an interesting part: accessing the tree using a subset of <l n='xpath'/> and since we're not compatible we call it <l n='lpath'/>. We will explain more about its usage in other documents.</p> --ldx]]-- +local xml = xml + local lpathcalls = 0 function xml.lpathcalls () return lpathcalls end local lpathcached = 0 function xml.lpathcached() return lpathcached end -xml.functions = xml.functions or { } -- internal -xml.expressions = xml.expressions or { } -- in expressions -xml.finalizers = xml.finalizers or { } -- fast do-with ... (with return value other than collection) -xml.specialhandler = xml.specialhandler or { } +xml.functions = xml.functions or { } -- internal +local functions = xml.functions + +xml.expressions = xml.expressions or { } -- in expressions +local expressions = xml.expressions + +xml.finalizers = xml.finalizers or { } -- fast do-with ... (with return value other than collection) +local finalizers = xml.finalizers -local functions = xml.functions -local expressions = xml.expressions -local finalizers = xml.finalizers +xml.specialhandler = xml.specialhandler or { } +local specialhandler = xml.specialhandler + +lpegpatterns.xml = lpegpatterns.xml or { } +local xmlpatterns = lpegpatterns.xml finalizers.xml = finalizers.xml or { } finalizers.tex = finalizers.tex or { } @@ -5228,14 +7527,14 @@ local function fallback (t, name) if fn then t[name] = fn else - logs.report("xml","unknown sub finalizer '%s'",tostring(name)) + report_lpath("unknown sub finalizer '%s'",tostring(name)) fn = function() end end return fn end -setmetatable(finalizers.xml, { __index = fallback }) -setmetatable(finalizers.tex, { __index = fallback }) +setmetatableindex(finalizers.xml, fallback) +setmetatableindex(finalizers.tex, fallback) xml.defaultprotocol = "xml" @@ -5283,102 +7582,106 @@ apply_axis['root'] = function(list) rt = ll end end - collected[#collected+1] = rt + collected[l] = rt end return collected end apply_axis['self'] = function(list) ---~ local collected = { } ---~ for l=1,#list do ---~ collected[#collected+1] = list[l] ---~ end ---~ return collected return list end apply_axis['child'] = function(list) - local collected = { } + local collected, c = { }, 0 for l=1,#list do local ll = list[l] local dt = ll.dt - local en = 0 - for k=1,#dt do - local dk = dt[k] - if dk.tg then - collected[#collected+1] = dk - dk.ni = k -- refresh - en = en + 1 - dk.ei = en + if dt then -- weird that this is needed + local en = 0 + for k=1,#dt do + local dk = dt[k] + if dk.tg then + c = c + 1 + collected[c] = dk + dk.ni = k -- refresh + en = en + 1 + dk.ei = en + end end + ll.en = en end - ll.en = en end return collected end -local function collect(list,collected) +local function collect(list,collected,c) local dt = list.dt if dt then local en = 0 for k=1,#dt do local dk = dt[k] if dk.tg then - collected[#collected+1] = dk + c = c + 1 + collected[c] = dk dk.ni = k -- refresh en = en + 1 dk.ei = en - collect(dk,collected) + c = collect(dk,collected,c) end end list.en = en end + return c end apply_axis['descendant'] = function(list) - local collected = { } + local collected, c = { }, 0 for l=1,#list do - collect(list[l],collected) + c = collect(list[l],collected,c) end return collected end -local function collect(list,collected) +local function collect(list,collected,c) local dt = list.dt if dt then local en = 0 for k=1,#dt do local dk = dt[k] if dk.tg then - collected[#collected+1] = dk + c = c + 1 + collected[c] = dk dk.ni = k -- refresh en = en + 1 dk.ei = en - collect(dk,collected) + c = collect(dk,collected,c) end end list.en = en end + return c end apply_axis['descendant-or-self'] = function(list) - local collected = { } + local collected, c = { }, 0 for l=1,#list do local ll = list[l] if ll.special ~= true then -- catch double root - collected[#collected+1] = ll + c = c + 1 + collected[c] = ll end - collect(ll,collected) + c = collect(ll,collected,c) end return collected end apply_axis['ancestor'] = function(list) - local collected = { } + local collected, c = { }, 0 for l=1,#list do local ll = list[l] while ll do ll = ll.__p__ if ll then - collected[#collected+1] = ll + c = c + 1 + collected[c] = ll end end end @@ -5386,14 +7689,16 @@ apply_axis['ancestor'] = function(list) end apply_axis['ancestor-or-self'] = function(list) - local collected = { } + local collected, c = { }, 0 for l=1,#list do local ll = list[l] - collected[#collected+1] = ll + c = c + 1 + collected[c] = ll while ll do ll = ll.__p__ if ll then - collected[#collected+1] = ll + c = c + 1 + collected[c] = ll end end end @@ -5401,11 +7706,12 @@ apply_axis['ancestor-or-self'] = function(list) end apply_axis['parent'] = function(list) - local collected = { } + local collected, c = { }, 0 for l=1,#list do local pl = list[l].__p__ if pl then - collected[#collected+1] = pl + c = c + 1 + collected[c] = pl end end return collected @@ -5420,43 +7726,15 @@ apply_axis['namespace'] = function(list) end apply_axis['following'] = function(list) -- incomplete ---~ local collected = { } ---~ for l=1,#list do ---~ local ll = list[l] ---~ local p = ll.__p__ ---~ local d = p.dt ---~ for i=ll.ni+1,#d do ---~ local di = d[i] ---~ if type(di) == "table" then ---~ collected[#collected+1] = di ---~ break ---~ end ---~ end ---~ end ---~ return collected return { } end apply_axis['preceding'] = function(list) -- incomplete ---~ local collected = { } ---~ for l=1,#list do ---~ local ll = list[l] ---~ local p = ll.__p__ ---~ local d = p.dt ---~ for i=ll.ni-1,1,-1 do ---~ local di = d[i] ---~ if type(di) == "table" then ---~ collected[#collected+1] = di ---~ break ---~ end ---~ end ---~ end ---~ return collected return { } end apply_axis['following-sibling'] = function(list) - local collected = { } + local collected, c = { }, 0 for l=1,#list do local ll = list[l] local p = ll.__p__ @@ -5464,7 +7742,8 @@ apply_axis['following-sibling'] = function(list) for i=ll.ni+1,#d do local di = d[i] if type(di) == "table" then - collected[#collected+1] = di + c = c + 1 + collected[c] = di end end end @@ -5472,7 +7751,7 @@ apply_axis['following-sibling'] = function(list) end apply_axis['preceding-sibling'] = function(list) - local collected = { } + local collected, c = { }, 0 for l=1,#list do local ll = list[l] local p = ll.__p__ @@ -5480,7 +7759,8 @@ apply_axis['preceding-sibling'] = function(list) for i=1,ll.ni-1 do local di = d[i] if type(di) == "table" then - collected[#collected+1] = di + c = c + 1 + collected[c] = di end end end @@ -5488,7 +7768,7 @@ apply_axis['preceding-sibling'] = function(list) end apply_axis['reverse-sibling'] = function(list) -- reverse preceding - local collected = { } + local collected, c = { }, 0 for l=1,#list do local ll = list[l] local p = ll.__p__ @@ -5496,7 +7776,8 @@ apply_axis['reverse-sibling'] = function(list) -- reverse preceding for i=ll.ni-1,1,-1 do local di = d[i] if type(di) == "table" then - collected[#collected+1] = di + c = c + 1 + collected[c] = di end end end @@ -5522,7 +7803,7 @@ local function apply_nodes(list,directive,nodes) return { } end else - local collected, m, p = { }, 0, nil + local collected, c, m, p = { }, 0, 0, nil if not nns then -- only check tag for l=1,#list do local ll = list[l] @@ -5531,11 +7812,13 @@ local function apply_nodes(list,directive,nodes) if directive then if ntg == ltg then local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end - collected[#collected+1], ll.mi = ll, m + c = c + 1 + collected[c], ll.mi = ll, m end elseif ntg ~= ltg then local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end - collected[#collected+1], ll.mi = ll, m + c = c + 1 + collected[c], ll.mi = ll, m end end end @@ -5547,11 +7830,13 @@ local function apply_nodes(list,directive,nodes) if directive then if lns == nns then local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end - collected[#collected+1], ll.mi = ll, m + c = c + 1 + collected[c], ll.mi = ll, m end elseif lns ~= nns then local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end - collected[#collected+1], ll.mi = ll, m + c = c + 1 + collected[c], ll.mi = ll, m end end end @@ -5565,11 +7850,13 @@ local function apply_nodes(list,directive,nodes) if directive then if ok then local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end - collected[#collected+1], ll.mi = ll, m + c = c + 1 + collected[c], ll.mi = ll, m end elseif not ok then local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end - collected[#collected+1], ll.mi = ll, m + c = c + 1 + collected[c], ll.mi = ll, m end end end @@ -5577,7 +7864,7 @@ local function apply_nodes(list,directive,nodes) return collected end else - local collected, m, p = { }, 0, nil + local collected, c, m, p = { }, 0, 0, nil for l=1,#list do local ll = list[l] local ltg = ll.tg @@ -5594,11 +7881,13 @@ local function apply_nodes(list,directive,nodes) if directive then if ok then local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end - collected[#collected+1], ll.mi = ll, m + c = c + 1 + collected[c], ll.mi = ll, m end elseif not ok then local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end - collected[#collected+1], ll.mi = ll, m + c = c + 1 + collected[c], ll.mi = ll, m end end end @@ -5609,12 +7898,13 @@ end local quit_expression = false local function apply_expression(list,expression,order) - local collected = { } + local collected, c = { }, 0 quit_expression = false for l=1,#list do local ll = list[l] if expression(list,ll,l,order) then -- nasty, order alleen valid als n=1 - collected[#collected+1] = ll + c = c + 1 + collected[c] = ll end if quit_expression then break @@ -5653,7 +7943,9 @@ local lp_builtin = P ( P("ns") / "ll.ns" ) * ((spaces * P("(") * spaces * P(")"))/"") -local lp_attribute = (P("@") + P("attribute::")) / "" * Cc("(ll.at and ll.at['") * R("az","AZ","--","__")^1 * Cc("'])") +-- for the moment we keep namespaces with attributes + +local lp_attribute = (P("@") + P("attribute::")) / "" * Cc("(ll.at and ll.at['") * ((R("az","AZ") + S("-_:"))^1) * Cc("'])") local lp_fastpos_p = ((P("+")^0 * R("09")^1 * P(-1)) / function(s) return "l==" .. s end) local lp_fastpos_n = ((P("-") * R("09")^1 * P(-1)) / function(s) return "(" .. s .. "<0 and (#list+".. s .. "==l))" end) local lp_fastpos = lp_fastpos_n + lp_fastpos_p @@ -5714,14 +8006,12 @@ local converter = Cs ( ) cleaner = Cs ( ( ---~ lp_fastpos + lp_reserved + lp_number + lp_string + 1 )^1 ) ---~ expr local template_e = [[ local expr = xml.expressions @@ -5772,13 +8062,13 @@ local skip = { } local function errorrunner_e(str,cnv) if not skip[str] then - logs.report("lpath","error in expression: %s => %s",str,cnv) + report_lpath("error in expression: %s => %s",str,cnv) skip[str] = cnv or str end return false end local function errorrunner_f(str,arg) - logs.report("lpath","error in finalizer: %s(%s)",str,arg or "") + report_lpath("error in finalizer: %s(%s)",str,arg or "") return false end @@ -5829,7 +8119,7 @@ local special_1 = P("*") * Cc(register_auto_descendant) * Cc(register_all_nodes local special_2 = P("/") * Cc(register_auto_self) local special_3 = P("") * Cc(register_auto_self) -local parser = Ct { "patterns", -- can be made a bit faster by moving pattern outside +local pathparser = Ct { "patterns", -- can be made a bit faster by moving pattern outside patterns = spaces * V("protocol") * spaces * ( ( V("special") * spaces * P(-1) ) + @@ -5901,6 +8191,8 @@ local parser = Ct { "patterns", -- can be made a bit faster by moving pattern ou } +xmlpatterns.pathparser = pathparser + local cache = { } local function nodesettostring(set,nodetest) @@ -5910,7 +8202,7 @@ local function nodesettostring(set,nodetest) if not ns or ns == "" then ns = "*" end if not tg or tg == "" then tg = "*" end tg = (tg == "@rt@" and "[root]") or format("%s:%s",ns,tg) - t[#t+1] = (directive and tg) or format("not(%s)",tg) + t[i] = (directive and tg) or format("not(%s)",tg) end if nodetest == false then return format("not(%s)",concat(t,"|")) @@ -5929,7 +8221,7 @@ local function tagstostring(list) local ns, tg = li.ns, li.tg if not ns or ns == "" then ns = "*" end if not tg or tg == "" then tg = "*" end - t[#t+1] = (tg == "@rt@" and "[root]") or format("%s:%s",ns,tg) + t[i] = (tg == "@rt@" and "[root]") or format("%s:%s",ns,tg) end return concat(t," ") end @@ -5937,16 +8229,16 @@ end xml.nodesettostring = nodesettostring -local parse_pattern -- we have a harmless kind of circular reference +local lpath -- we have a harmless kind of circular reference + +local lshowoptions = { functions = false } local function lshow(parsed) if type(parsed) == "string" then - parsed = parse_pattern(parsed) + parsed = lpath(parsed) end - local s = table.serialize_functions -- ugly - table.serialize_functions = false -- ugly - logs.report("lpath","%s://%s => %s",parsed.protocol or xml.defaultprotocol,parsed.pattern,table.serialize(parsed,false)) - table.serialize_functions = s -- ugly + report_lpath("%s://%s => %s",parsed.protocol or xml.defaultprotocol,parsed.pattern, + table.serialize(parsed,false,lshowoptions)) end xml.lshow = lshow @@ -5960,7 +8252,7 @@ local function add_comment(p,str) end end -parse_pattern = function (pattern) -- the gain of caching is rather minimal +lpath = function (pattern) -- the gain of caching is rather minimal lpathcalls = lpathcalls + 1 if type(pattern) == "table" then return pattern @@ -5969,13 +8261,13 @@ parse_pattern = function (pattern) -- the gain of caching is rather minimal if parsed then lpathcached = lpathcached + 1 else - parsed = lpegmatch(parser,pattern) + parsed = lpegmatch(pathparser,pattern) if parsed then parsed.pattern = pattern local np = #parsed if np == 0 then parsed = { pattern = pattern, register_self, state = "parsing error" } - logs.report("lpath","parsing error in '%s'",pattern) + report_lpath("parsing error in '%s'",pattern) lshow(parsed) else -- we could have done this with a more complex parser but this @@ -6017,6 +8309,8 @@ parse_pattern = function (pattern) -- the gain of caching is rather minimal end end +xml.lpath = lpath + -- we can move all calls inline and then merge the trace back -- technically we can combine axis and the next nodes which is -- what we did before but this a bit cleaner (but slower too) @@ -6079,32 +8373,32 @@ local function traced_apply(list,parsed,nofparsed,order) if trace_lparse then lshow(parsed) end - logs.report("lpath", "collecting : %s",parsed.pattern) - logs.report("lpath", " root tags : %s",tagstostring(list)) - logs.report("lpath", " order : %s",order or "unset") + report_lpath("collecting: %s",parsed.pattern) + report_lpath("root tags : %s",tagstostring(list)) + report_lpath("order : %s",order or "unset") local collected = list for i=1,nofparsed do local pi = parsed[i] local kind = pi.kind if kind == "axis" then collected = apply_axis[pi.axis](collected) - logs.report("lpath", "% 10i : ax : %s",(collected and #collected) or 0,pi.axis) + report_lpath("% 10i : ax : %s",(collected and #collected) or 0,pi.axis) elseif kind == "nodes" then collected = apply_nodes(collected,pi.nodetest,pi.nodes) - logs.report("lpath", "% 10i : ns : %s",(collected and #collected) or 0,nodesettostring(pi.nodes,pi.nodetest)) + report_lpath("% 10i : ns : %s",(collected and #collected) or 0,nodesettostring(pi.nodes,pi.nodetest)) elseif kind == "expression" then collected = apply_expression(collected,pi.evaluator,order) - logs.report("lpath", "% 10i : ex : %s -> %s",(collected and #collected) or 0,pi.expression,pi.converted) + report_lpath("% 10i : ex : %s -> %s",(collected and #collected) or 0,pi.expression,pi.converted) elseif kind == "finalizer" then collected = pi.finalizer(collected) - logs.report("lpath", "% 10i : fi : %s : %s(%s)",(type(collected) == "table" and #collected) or 0,parsed.protocol or xml.defaultprotocol,pi.name,pi.arguments or "") + report_lpath("% 10i : fi : %s : %s(%s)",(type(collected) == "table" and #collected) or 0,parsed.protocol or xml.defaultprotocol,pi.name,pi.arguments or "") return collected end if not collected or #collected == 0 then local pn = i < nofparsed and parsed[nofparsed] if pn and pn.kind == "finalizer" then collected = pn.finalizer(collected) - logs.report("lpath", "% 10i : fi : %s : %s(%s)",(type(collected) == "table" and #collected) or 0,parsed.protocol or xml.defaultprotocol,pn.name,pn.arguments or "") + report_lpath("% 10i : fi : %s : %s(%s)",(type(collected) == "table" and #collected) or 0,parsed.protocol or xml.defaultprotocol,pn.name,pn.arguments or "") return collected end return nil @@ -6141,8 +8435,11 @@ local function normal_apply(list,parsed,nofparsed,order) return collected end -local function parse_apply(list,pattern) - -- we avoid an extra call + +local function applylpath(list,pattern) + if not list then + return + end local parsed = cache[pattern] if parsed then lpathcalls = lpathcalls + 1 @@ -6151,7 +8448,7 @@ local function parse_apply(list,pattern) lpathcalls = lpathcalls + 1 parsed = pattern else - parsed = parse_pattern(pattern) or pattern + parsed = lpath(pattern) or pattern end if not parsed then return @@ -6160,25 +8457,32 @@ local function parse_apply(list,pattern) if nofparsed == 0 then return -- something is wrong end - local one = list[1] - if not one then - return -- something is wrong - elseif not trace_lpath then - return normal_apply(list,parsed,nofparsed,one.mi) + if not trace_lpath then + return normal_apply ({ list },parsed,nofparsed,list.mi) elseif trace_lprofile then - return profiled_apply(list,parsed,nofparsed,one.mi) + return profiled_apply({ list },parsed,nofparsed,list.mi) else - return traced_apply(list,parsed,nofparsed,one.mi) + return traced_apply ({ list },parsed,nofparsed,list.mi) end end +xml.applylpath = applylpath -- takes a table as first argment, which is what xml.filter will do + +--[[ldx-- +<p>This is the main filter function. It returns whatever is asked for.</p> +--ldx]]-- + +function xml.filter(root,pattern) -- no longer funny attribute handling here + return applylpath(root,pattern) +end + -- internal (parsed) expressions.child = function(e,pattern) - return parse_apply({ e },pattern) -- todo: cache + return applylpath(e,pattern) -- todo: cache end expressions.count = function(e,pattern) - local collected = parse_apply({ e },pattern) -- todo: cache + local collected = applylpath(e,pattern) -- todo: cache return (collected and #collected) or 0 end @@ -6188,7 +8492,7 @@ expressions.oneof = function(s,...) -- slow local t = {...} for i=1,#t do if s == t[i] then return true end end return false end expressions.error = function(str) - xml.error_handler("unknown function in lpath expression",tostring(str or "?")) + xml.errorhandler("unknown function in lpath expression",tostring(str or "?")) return false end expressions.undefined = function(s) @@ -6217,8 +8521,8 @@ expressions.boolean = toboolean -- user interface local function traverse(root,pattern,handle) - logs.report("xml","use 'xml.selection' instead for '%s'",pattern) - local collected = parse_apply({ root },pattern) + report_lpath("use 'xml.selection' instead for '%s'",pattern) + local collected = applylpath(root,pattern) if collected then for c=1,#collected do local e = collected[c] @@ -6229,7 +8533,7 @@ local function traverse(root,pattern,handle) end local function selection(root,pattern,handle) - local collected = parse_apply({ root },pattern) + local collected = applylpath(root,pattern) if collected then if handle then for c=1,#collected do @@ -6241,19 +8545,9 @@ local function selection(root,pattern,handle) end end -xml.parse_parser = parser -xml.parse_pattern = parse_pattern -xml.parse_apply = parse_apply xml.traverse = traverse -- old method, r, d, k xml.selection = selection -- new method, simple handle -local lpath = parse_pattern - -xml.lpath = lpath - -function xml.cached_patterns() - return cache -end -- generic function finalizer (independant namespace) @@ -6265,13 +8559,13 @@ local function dofunction(collected,fnc) f(collected[c]) end else - logs.report("xml","unknown function '%s'",fnc) + report_lpath("unknown function '%s'",fnc) end end end -xml.finalizers.xml["function"] = dofunction -xml.finalizers.tex["function"] = dofunction +finalizers.xml["function"] = dofunction +finalizers.tex["function"] = dofunction -- functions @@ -6364,14 +8658,6 @@ expressions.tag = function(e,n) -- only tg end --[[ldx-- -<p>This is the main filter function. It returns whatever is asked for.</p> ---ldx]]-- - -function xml.filter(root,pattern) -- no longer funny attribute handling here - return parse_apply({ root },pattern) -end - ---[[ldx-- <p>Often using an iterators looks nicer in the code than passing handler functions. The <l n='lua'/> book describes how to use coroutines for that purpose (<url href='http://www.lua.org/pil/9.3.html'/>). This permits @@ -6390,7 +8676,7 @@ end local wrap, yield = coroutine.wrap, coroutine.yield function xml.elements(root,pattern,reverse) -- r, d, k - local collected = parse_apply({ root },pattern) + local collected = applylpath(root,pattern) if collected then if reverse then return wrap(function() for c=#collected,1,-1 do @@ -6406,7 +8692,7 @@ function xml.elements(root,pattern,reverse) -- r, d, k end function xml.collected(root,pattern,reverse) -- e - local collected = parse_apply({ root },pattern) + local collected = applylpath(root,pattern) if collected then if reverse then return wrap(function() for c=#collected,1,-1 do yield(collected[c]) end end) @@ -6417,6 +8703,15 @@ function xml.collected(root,pattern,reverse) -- e return wrap(function() end) end +-- handy + +function xml.inspect(collection,pattern) + pattern = pattern or "." + for e in xml.collected(collection,pattern or ".") do + report_lpath("pattern %q\n\n%s\n",pattern,xml.tostring(e)) + end +end + end -- of closure @@ -6430,10 +8725,16 @@ if not modules then modules = { } end modules ['lxml-mis'] = { license = "see context related readme files" } +local xml, lpeg, string = xml, lpeg, string + local concat = table.concat local type, next, tonumber, tostring, setmetatable, loadstring = type, next, tonumber, tostring, setmetatable, loadstring local format, gsub, match = string.format, string.gsub, string.match -local lpegmatch = lpeg.match +local lpegmatch, lpegpatterns = lpeg.match, lpeg.patterns +local P, S, R, C, V, Cc, Cs = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc, lpeg.Cs + +lpegpatterns.xml = lpegpatterns.xml or { } +local xmlpatterns = lpegpatterns.xml --[[ldx-- <p>The following helper functions best belong to the <t>lxml-ini</t> @@ -6457,9 +8758,8 @@ local function xmlgsub(t,old,new) -- will be replaced end end ---~ xml.gsub = xmlgsub -function xml.strip_leading_spaces(dk,d,k) -- cosmetic, for manual +function xml.stripleadingspaces(dk,d,k) -- cosmetic, for manual if d and k then local dkm = d[k-1] if dkm and type(dkm) == "string" then @@ -6469,14 +8769,7 @@ function xml.strip_leading_spaces(dk,d,k) -- cosmetic, for manual end end ---~ xml.escapes = { ['&'] = '&', ['<'] = '<', ['>'] = '>', ['"'] = '"' } ---~ xml.unescapes = { } for k,v in next, xml.escapes do xml.unescapes[v] = k end ---~ function xml.escaped (str) return (gsub(str,"(.)" , xml.escapes )) end ---~ function xml.unescaped(str) return (gsub(str,"(&.-;)", xml.unescapes)) end ---~ function xml.cleansed (str) return (gsub(str,"<.->" , '' )) end -- "%b<>" - -local P, S, R, C, V, Cc, Cs = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc, lpeg.Cs -- 100 * 2500 * "oeps< oeps> oeps&" : gsub:lpeg|lpeg|lpeg -- @@ -6502,9 +8795,9 @@ local unescaped = Cs(normal * (special * normal)^0) local cleansed = Cs(((P("<") * (1-P(">"))^0 * P(">"))/"" + 1)^0) -xml.escaped_pattern = escaped -xml.unescaped_pattern = unescaped -xml.cleansed_pattern = cleansed +xmlpatterns.escaped = escaped +xmlpatterns.unescaped = unescaped +xmlpatterns.cleansed = cleansed function xml.escaped (str) return lpegmatch(escaped,str) end function xml.unescaped(str) return lpegmatch(unescaped,str) end @@ -6540,15 +8833,20 @@ if not modules then modules = { } end modules ['lxml-aux'] = { local trace_manipulations = false trackers.register("lxml.manipulations", function(v) trace_manipulations = v end) -local xmlparseapply, xmlconvert, xmlcopy, xmlname = xml.parse_apply, xml.convert, xml.copy, xml.name +local report_xml = logs.reporter("xml") + +local xml = xml + +local xmlconvert, xmlcopy, xmlname = xml.convert, xml.copy, xml.name local xmlinheritedconvert = xml.inheritedconvert +local xmlapplylpath = xml.applylpath -local type = type -local insert, remove = table.insert, table.remove +local type, setmetatable, getmetatable = type, setmetatable, getmetatable +local insert, remove, fastcopy = table.insert, table.remove, table.fastcopy local gmatch, gsub = string.gmatch, string.gsub local function report(what,pattern,c,e) - logs.report("xml","%s element '%s' (root: '%s', position: %s, index: %s, pattern: %s)",what,xmlname(e),xmlname(e.__p__),c,e.ni,pattern) + report_xml("%s element '%s' (root: '%s', position: %s, index: %s, pattern: %s)",what,xmlname(e),xmlname(e.__p__),c,e.ni,pattern) end local function withelements(e,handle,depth) @@ -6602,10 +8900,8 @@ function xml.withelement(e,n,handle) -- slow end end -xml.elements_only = xml.collected - -function xml.each_element(root,pattern,handle,reverse) - local collected = xmlparseapply({ root },pattern) +function xml.each(root,pattern,handle,reverse) + local collected = xmlapplylpath(root,pattern) if collected then if reverse then for c=#collected,1,-1 do @@ -6620,10 +8916,8 @@ function xml.each_element(root,pattern,handle,reverse) end end -xml.process_elements = xml.each_element - -function xml.process_attributes(root,pattern,handle) - local collected = xmlparseapply({ root },pattern) +function xml.processattributes(root,pattern,handle) + local collected = xmlapplylpath(root,pattern) if collected and handle then for c=1,#collected do handle(collected[c].at) @@ -6638,12 +8932,12 @@ end -- are these still needed -> lxml-cmp.lua -function xml.collect_elements(root, pattern) - return xmlparseapply({ root },pattern) +function xml.collect(root, pattern) + return xmlapplylpath(root,pattern) end -function xml.collect_texts(root, pattern, flatten) -- todo: variant with handle - local collected = xmlparseapply({ root },pattern) +function xml.collecttexts(root, pattern, flatten) -- todo: variant with handle + local collected = xmlapplylpath(root,pattern) if collected and flatten then local xmltostring = xml.tostring for c=1,#collected do @@ -6654,18 +8948,19 @@ function xml.collect_texts(root, pattern, flatten) -- todo: variant with handle end function xml.collect_tags(root, pattern, nonamespace) - local collected = xmlparseapply({ root },pattern) + local collected = xmlapplylpath(root,pattern) if collected then - local t = { } + local t, n = { }, 0 for c=1,#collected do local e = collected[c] local ns, tg = e.ns, e.tg + n = n + 1 if nonamespace then - t[#t+1] = tg + t[n] = tg elseif ns == "" then - t[#t+1] = tg + t[n] = tg else - t[#t+1] = ns .. ":" .. tg + t[n] = ns .. ":" .. tg end end return t @@ -6678,7 +8973,7 @@ end local no_root = { no_root = true } -function xml.redo_ni(d) +local function redo_ni(d) for k=1,#d do local dk = d[k] if type(dk) == "table" then @@ -6701,12 +8996,7 @@ local function xmltoelement(whatever,root) return whatever -- string end if element then - --~ if element.ri then - --~ element = element.dt[element.ri].dt - --~ else - --~ element = element.dt - --~ end - end + end return element end @@ -6724,8 +9014,8 @@ local function copiedelement(element,newparent) end end -function xml.delete_element(root,pattern) - local collected = xmlparseapply({ root },pattern) +function xml.delete(root,pattern) + local collected = xmlapplylpath(root,pattern) if collected then for c=1,#collected do local e = collected[c] @@ -6736,15 +9026,15 @@ function xml.delete_element(root,pattern) end local d = p.dt remove(d,e.ni) - xml.redo_ni(d) -- can be made faster and inlined + redo_ni(d) -- can be made faster and inlined end end end end -function xml.replace_element(root,pattern,whatever) +function xml.replace(root,pattern,whatever) local element = root and xmltoelement(whatever,root) - local collected = element and xmlparseapply({ root },pattern) + local collected = element and xmlapplylpath(root,pattern) if collected then for c=1,#collected do local e = collected[c] @@ -6755,15 +9045,50 @@ function xml.replace_element(root,pattern,whatever) end local d = p.dt d[e.ni] = copiedelement(element,p) - xml.redo_ni(d) -- probably not needed + redo_ni(d) -- probably not needed + end + end + end +end + +local function wrap(e,wrapper) + local t = { + rn = e.rn, + tg = e.tg, + ns = e.ns, + at = e.at, + dt = e.dt, + __p__ = e, + } + setmetatable(t,getmetatable(e)) + e.rn = wrapper.rn or e.rn or "" + e.tg = wrapper.tg or e.tg or "" + e.ns = wrapper.ns or e.ns or "" + e.at = fastcopy(wrapper.at) + e.dt = { t } +end + +function xml.wrap(root,pattern,whatever) + if whatever then + local wrapper = xmltoelement(whatever,root) + local collected = xmlapplylpath(root,pattern) + if collected then + for c=1,#collected do + local e = collected[c] + if trace_manipulations then + report('wrapping',pattern,c,e) + end + wrap(e,wrapper) end end + else + wrap(root,xmltoelement(pattern)) end end local function inject_element(root,pattern,whatever,prepend) local element = root and xmltoelement(whatever,root) - local collected = element and xmlparseapply({ root },pattern) + local collected = element and xmlapplylpath(root,pattern) if collected then for c=1,#collected do local e = collected[c] @@ -6778,15 +9103,17 @@ local function inject_element(root,pattern,whatever,prepend) else be, af = edt, cp end + local bn = #be for i=1,#af do - be[#be+1] = af[i] + bn = bn + 1 + be[bn] = af[i] end if rri then r.dt[rri].dt = be else d[k].dt = be end - xml.redo_ni(d) + redo_ni(d) end end end @@ -6794,7 +9121,7 @@ end local function insert_element(root,pattern,whatever,before) -- todo: element als functie local element = root and xmltoelement(whatever,root) - local collected = element and xmlparseapply({ root },pattern) + local collected = element and xmlapplylpath(root,pattern) if collected then for c=1,#collected do local e = collected[c] @@ -6804,24 +9131,23 @@ local function insert_element(root,pattern,whatever,before) -- todo: element als k = k + 1 end insert(d,k,copiedelement(element,r)) - xml.redo_ni(d) + redo_ni(d) end end end -xml.insert_element = insert_element -xml.insert_element_after = insert_element -xml.insert_element_before = function(r,p,e) insert_element(r,p,e,true) end -xml.inject_element = inject_element -xml.inject_element_after = inject_element -xml.inject_element_before = function(r,p,e) inject_element(r,p,e,true) end +xml.insert_element = insert_element +xml.insertafter = insert_element +xml.insertbefore = function(r,p,e) insert_element(r,p,e,true) end +xml.injectafter = inject_element +xml.injectbefore = function(r,p,e) inject_element(r,p,e,true) end local function include(xmldata,pattern,attribute,recursive,loaddata) -- parse="text" (default: xml), encoding="" (todo) -- attribute = attribute or 'href' pattern = pattern or 'include' loaddata = loaddata or io.loaddata - local collected = xmlparseapply({ xmldata },pattern) + local collected = xmlapplylpath(xmldata,pattern) if collected then for c=1,#collected do local ek = collected[c] @@ -6830,7 +9156,7 @@ local function include(xmldata,pattern,attribute,recursive,loaddata) local ekat = ek.at local epdt = ek.__p__.dt if not attribute or attribute == "" then - name = (type(ekdt) == "table" and ekdt[1]) or ekdt -- ckeck, probably always tab or str + name = (type(ekdt) == "table" and ekdt[1]) or ekdt -- check, probably always tab or str end if not name then for a in gmatch(attribute or "href","([^|]+)") do @@ -6845,9 +9171,6 @@ local function include(xmldata,pattern,attribute,recursive,loaddata) -- for the moment hard coded epdt[ek.ni] = xml.escaped(data) -- d[k] = xml.escaped(data) else ---~ local settings = xmldata.settings ---~ settings.parent_root = xmldata -- to be tested ---~ local xi = xmlconvert(data,settings) local xi = xmlinheritedconvert(data,xmldata) if not xi then epdt[ek.ni] = "" -- xml.empty(d,k) @@ -6864,136 +9187,88 @@ end xml.include = include ---~ local function manipulate(xmldata,pattern,manipulator) -- untested and might go away ---~ local collected = xmlparseapply({ xmldata },pattern) ---~ if collected then ---~ local xmltostring = xml.tostring ---~ for c=1,#collected do ---~ local e = collected[c] ---~ local data = manipulator(xmltostring(e)) ---~ if data == "" then ---~ epdt[e.ni] = "" ---~ else ---~ local xi = xmlinheritedconvert(data,xmldata) ---~ if not xi then ---~ epdt[e.ni] = "" ---~ else ---~ epdt[e.ni] = xml.body(xi) -- xml.assign(d,k,xi) ---~ end ---~ end ---~ end ---~ end ---~ end - ---~ xml.manipulate = manipulate - -function xml.strip_whitespace(root, pattern, nolines) -- strips all leading and trailing space ! - local collected = xmlparseapply({ root },pattern) - if collected then - for i=1,#collected do - local e = collected[i] - local edt = e.dt - if edt then - local t = { } - for i=1,#edt do - local str = edt[i] - if type(str) == "string" then - if str == "" then - -- stripped - else - if nolines then - str = gsub(str,"[ \n\r\t]+"," ") - end - if str == "" then - -- stripped - else - t[#t+1] = str - end - end +local function stripelement(e,nolines,anywhere) + local edt = e.dt + if edt then + if anywhere then + local t, n = { }, 0 + for e=1,#edt do + local str = edt[e] + if type(str) ~= "string" then + n = n + 1 + t[n] = str + elseif str ~= "" then + -- todo: lpeg for each case + if nolines then + str = gsub(str,"%s+"," ") + end + str = gsub(str,"^%s*(.-)%s*$","%1") + if str ~= "" then + n = n + 1 + t[n] = str + end + end + end + e.dt = t + else + -- we can assume a regular sparse xml table with no successive strings + -- otherwise we should use a while loop + if #edt > 0 then + -- strip front + local str = edt[1] + if type(str) ~= "string" then + -- nothing + elseif str == "" then + remove(edt,1) + else + if nolines then + str = gsub(str,"%s+"," ") + end + str = gsub(str,"^%s+","") + if str == "" then + remove(edt,1) + else + edt[1] = str + end + end + end + local nedt = #edt + if nedt > 0 then + -- strip end + local str = edt[nedt] + if type(str) ~= "string" then + -- nothing + elseif str == "" then + remove(edt) + else + if nolines then + str = gsub(str,"%s+"," ") + end + str = gsub(str,"%s+$","") + if str == "" then + remove(edt) else - --~ str.ni = i - t[#t+1] = str + edt[nedt] = str end end - e.dt = t end end end + return e -- convenient end -function xml.strip_whitespace(root, pattern, nolines, anywhere) -- strips all leading and trailing spacing - local collected = xmlparseapply({ root },pattern) -- beware, indices no longer are valid now +xml.stripelement = stripelement + +function xml.strip(root,pattern,nolines,anywhere) -- strips all leading and trailing spacing + local collected = xmlapplylpath(root,pattern) -- beware, indices no longer are valid now if collected then for i=1,#collected do - local e = collected[i] - local edt = e.dt - if edt then - if anywhere then - local t = { } - for e=1,#edt do - local str = edt[e] - if type(str) ~= "string" then - t[#t+1] = str - elseif str ~= "" then - -- todo: lpeg for each case - if nolines then - str = gsub(str,"%s+"," ") - end - str = gsub(str,"^%s*(.-)%s*$","%1") - if str ~= "" then - t[#t+1] = str - end - end - end - e.dt = t - else - -- we can assume a regular sparse xml table with no successive strings - -- otherwise we should use a while loop - if #edt > 0 then - -- strip front - local str = edt[1] - if type(str) ~= "string" then - -- nothing - elseif str == "" then - remove(edt,1) - else - if nolines then - str = gsub(str,"%s+"," ") - end - str = gsub(str,"^%s+","") - if str == "" then - remove(edt,1) - else - edt[1] = str - end - end - end - if #edt > 1 then - -- strip end - local str = edt[#edt] - if type(str) ~= "string" then - -- nothing - elseif str == "" then - remove(edt) - else - if nolines then - str = gsub(str,"%s+"," ") - end - str = gsub(str,"%s+$","") - if str == "" then - remove(edt) - else - edt[#edt] = str - end - end - end - end - end + stripelement(collected[i],nolines,anywhere) end end end -local function rename_space(root, oldspace, newspace) -- fast variant +local function renamespace(root, oldspace, newspace) -- fast variant local ndt = #root.dt for i=1,ndt or 0 do local e = root[i] @@ -7006,16 +9281,16 @@ local function rename_space(root, oldspace, newspace) -- fast variant end local edt = e.dt if edt then - rename_space(edt, oldspace, newspace) + renamespace(edt, oldspace, newspace) end end end end -xml.rename_space = rename_space +xml.renamespace = renamespace -function xml.remap_tag(root, pattern, newtg) - local collected = xmlparseapply({ root },pattern) +function xml.remaptag(root, pattern, newtg) + local collected = xmlapplylpath(root,pattern) if collected then for c=1,#collected do collected[c].tg = newtg @@ -7023,8 +9298,8 @@ function xml.remap_tag(root, pattern, newtg) end end -function xml.remap_namespace(root, pattern, newns) - local collected = xmlparseapply({ root },pattern) +function xml.remapnamespace(root, pattern, newns) + local collected = xmlapplylpath(root,pattern) if collected then for c=1,#collected do collected[c].ns = newns @@ -7032,8 +9307,8 @@ function xml.remap_namespace(root, pattern, newns) end end -function xml.check_namespace(root, pattern, newns) - local collected = xmlparseapply({ root },pattern) +function xml.checknamespace(root, pattern, newns) + local collected = xmlapplylpath(root,pattern) if collected then for c=1,#collected do local e = collected[c] @@ -7044,8 +9319,8 @@ function xml.check_namespace(root, pattern, newns) end end -function xml.remap_name(root, pattern, newtg, newns, newrn) - local collected = xmlparseapply({ root },pattern) +function xml.remapname(root, pattern, newtg, newns, newrn) + local collected = xmlapplylpath(root,pattern) if collected then for c=1,#collected do local e = collected[c] @@ -7058,18 +9333,34 @@ end <p>Here are a few synonyms.</p> --ldx]]-- -xml.each = xml.each_element -xml.process = xml.process_element -xml.strip = xml.strip_whitespace -xml.collect = xml.collect_elements -xml.all = xml.collect_elements - -xml.insert = xml.insert_element_after -xml.inject = xml.inject_element_after -xml.after = xml.insert_element_after -xml.before = xml.insert_element_before -xml.delete = xml.delete_element -xml.replace = xml.replace_element +xml.all = xml.each +xml.insert = xml.insertafter +xml.inject = xml.injectafter +xml.after = xml.insertafter +xml.before = xml.insertbefore +xml.process = xml.each + +-- obsolete + +xml.obsolete = xml.obsolete or { } +local obsolete = xml.obsolete + +xml.strip_whitespace = xml.strip obsolete.strip_whitespace = xml.strip +xml.collect_elements = xml.collect obsolete.collect_elements = xml.collect +xml.delete_element = xml.delete obsolete.delete_element = xml.delete +xml.replace_element = xml.replace obsolete.replace_element = xml.replacet +xml.each_element = xml.each obsolete.each_element = xml.each +xml.process_elements = xml.process obsolete.process_elements = xml.process +xml.insert_element_after = xml.insertafter obsolete.insert_element_after = xml.insertafter +xml.insert_element_before = xml.insertbefore obsolete.insert_element_before = xml.insertbefore +xml.inject_element_after = xml.injectafter obsolete.inject_element_after = xml.injectafter +xml.inject_element_before = xml.injectbefore obsolete.inject_element_before = xml.injectbefore +xml.process_attributes = xml.processattributes obsolete.process_attributes = xml.processattributes +xml.collect_texts = xml.collecttexts obsolete.collect_texts = xml.collecttexts +xml.inject_element = xml.inject obsolete.inject_element = xml.inject +xml.remap_tag = xml.remaptag obsolete.remap_tag = xml.remaptag +xml.remap_name = xml.remapname obsolete.remap_name = xml.remapname +xml.remap_namespace = xml.remapnamespace obsolete.remap_namespace = xml.remapnamespace end -- of closure @@ -7084,10 +9375,15 @@ if not modules then modules = { } end modules ['lxml-xml'] = { license = "see context related readme files" } +local concat = string.concat + +local xml = xml + local finalizers = xml.finalizers.xml local xmlfilter = xml.filter -- we could inline this one for speed local xmltostring = xml.tostring local xmlserialize = xml.serialize +local xmlcollected = xml.collected local function first(collected) -- wrong ? return collected and collected[1] @@ -7101,15 +9397,8 @@ local function all(collected) return collected end -local function reverse(collected) - if collected then - local reversed = { } - for c=#collected,1,-1 do - reversed[#reversed+1] = collected[c] - end - return reversed - end -end + +local reverse = table.reversed local function attribute(collected,name) if collected and #collected > 0 then @@ -7200,11 +9489,12 @@ end local function texts(collected) if collected then - local t = { } + local t, n = { }, 0 for c=1,#collected do - local e = collection[c] + local e = collected[c] if e and e.dt then - t[#t+1] = e.dt + n = n + 1 + t[n] = e.dt end end return t @@ -7247,14 +9537,15 @@ end local function tags(collected,nonamespace) if collected then - local t = { } + local t, n = { }, 0 for c=1,#collected do local e = collected[c] local ns, tg = e.ns, e.tg + n = n + 1 if nonamespace or ns == "" then - t[#t+1] = tg + t[n] = tg else - t[#t+1] = ns .. ":" .. tg + t[n] = ns .. ":" .. tg end end return t @@ -7365,293 +9656,273 @@ xml.all = xml.filter xml.index = xml.position xml.found = xml.filter +-- a nice one: + +local function totable(x) + local t = { } + for e in xmlcollected(x[1] or x,"/*") do + t[e.tg] = xmltostring(e.dt) or "" + end + return next(t) and t or nil +end + +xml.table = totable +finalizers.table = totable + +local function textonly(e,t) + if e then + local edt = e.dt + if edt then + for i=1,#edt do + local e = edt[i] + if type(e) == "table" then + textonly(e,t) + else + t[#t+1] = e + end + end + end + end + return t +end + +function xml.textonly(e) -- no pattern + return concat(textonly(e,{})) +end + end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['luat-env'] = { +if not modules then modules = { } end modules ['data-ini'] = { version = 1.001, comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" + license = "see context related readme files", } --- A former version provided functionality for non embeded core --- scripts i.e. runtime library loading. Given the amount of --- Lua code we use now, this no longer makes sense. Much of this --- evolved before bytecode arrays were available and so a lot of --- code has disappeared already. +local gsub, find, gmatch = string.gsub, string.find, string.gmatch +local concat = table.concat +local next, type = next, type -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) +local filedirname, filebasename, fileextname, filejoin = file.dirname, file.basename, file.extname, file.join -local format, sub, match, gsub, find = string.format, string.sub, string.match, string.gsub, string.find -local unquote, quote = string.unquote, string.quote +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) +local trace_detail = false trackers.register("resolvers.details", function(v) trace_detail = v end) +local trace_expansions = false trackers.register("resolvers.expansions", function(v) trace_expansions = v end) --- precautions +local report_initialization = logs.reporter("resolvers","initialization") -os.setlocale(nil,nil) -- useless feature and even dangerous in luatex +local ostype, osname, ossetenv, osgetenv = os.type, os.name, os.setenv, os.getenv -function os.setlocale() - -- no way you can mess with it -end +-- The code here used to be part of a data-res but for convenience +-- we now split it over multiple files. As this file is now the +-- starting point we introduce resolvers here. --- dirty tricks +resolvers = resolvers or { } +local resolvers = resolvers -if arg and (arg[0] == 'luatex' or arg[0] == 'luatex.exe') and arg[1] == "--luaonly" then - arg[-1]=arg[0] arg[0]=arg[2] for k=3,#arg do arg[k-2]=arg[k] end arg[#arg]=nil arg[#arg]=nil -end +-- We don't want the kpse library to kick in. Also, we want to be able to +-- execute programs. Control over execution is implemented later. -if profiler and os.env["MTX_PROFILE_RUN"] == "YES" then - profiler.start("luatex-profile.log") -end +texconfig.kpse_init = false +texconfig.shell_escape = 't' --- environment +kpse = { original = kpse } -environment = environment or { } -environment.arguments = { } -environment.files = { } -environment.sortedflags = nil +setmetatable(kpse, { + __index = function(kp,name) + report_initialization("fatal error: kpse library is accessed (key: %s)",name) + os.exit() + end +} ) -if not environment.jobname or environment.jobname == "" then if tex then environment.jobname = tex.jobname end end -if not environment.version or environment.version == "" then environment.version = "unknown" end -if not environment.jobname then environment.jobname = "unknown" end +-- First we check a couple of environment variables. Some might be +-- set already but we need then later on. We start with the system +-- font path. -function environment.initialize_arguments(arg) - local arguments, files = { }, { } - environment.arguments, environment.files, environment.sortedflags = arguments, files, nil - for index=1,#arg do - local argument = arg[index] - if index > 0 then - local flag, value = match(argument,"^%-+(.-)=(.-)$") - if flag then - arguments[flag] = unquote(value or "") - else - flag = match(argument,"^%-+(.+)") - if flag then - arguments[flag] = true - else - files[#files+1] = argument - end - end - end +do + + local osfontdir = osgetenv("OSFONTDIR") + + if osfontdir and osfontdir ~= "" then + -- ok + elseif osname == "windows" then + ossetenv("OSFONTDIR","c:/windows/fonts//") + elseif osname == "macosx" then + ossetenv("OSFONTDIR","$HOME/Library/Fonts//;/Library/Fonts//;/System/Library/Fonts//") end - environment.ownname = environment.ownname or arg[0] or 'unknown.lua' + end -function environment.setargument(name,value) - environment.arguments[name] = value +-- Next comes the user's home path. We need this as later on we have +-- to replace ~ with its value. + +do + + local homedir = osgetenv(ostype == "windows" and 'USERPROFILE' or 'HOME') or '' + + if not homedir or homedir == "" then + homedir = string.char(127) -- we need a value, later we wil trigger on it + end + + homedir = file.collapsepath(homedir) + + ossetenv("HOME", homedir) -- can be used in unix cnf files + ossetenv("USERPROFILE",homedir) -- can be used in windows cnf files + + environment.homedir = homedir + end --- todo: defaults, better checks e.g on type (boolean versus string) --- --- tricky: too many hits when we support partials unless we add --- a registration of arguments so from now on we have 'partial' +-- The following code sets the name of the own binary and its +-- path. This is fallback code as we have os.selfdir now. -function environment.argument(name,partial) - local arguments, sortedflags = environment.arguments, environment.sortedflags - if arguments[name] then - return arguments[name] - elseif partial then - if not sortedflags then - sortedflags = table.sortedkeys(arguments) - for k=1,#sortedflags do - sortedflags[k] = "^" .. sortedflags[k] - end - environment.sortedflags = sortedflags +do + + local args = environment.originalarguments or arg -- this needs a cleanup + + local ownbin = environment.ownbin or args[-2] or arg[-2] or args[-1] or arg[-1] or arg[0] or "luatex" + local ownpath = environment.ownpath or os.selfdir + + ownbin = file.collapsepath(ownbin) + ownpath = file.collapsepath(ownpath) + + if not ownpath or ownpath == "" or ownpath == "unset" then + ownpath = args[-1] or arg[-1] + ownpath = ownpath and filedirname(gsub(ownpath,"\\","/")) + if not ownpath or ownpath == "" then + ownpath = args[-0] or arg[-0] + ownpath = ownpath and filedirname(gsub(ownpath,"\\","/")) end - -- example of potential clash: ^mode ^modefile - for k=1,#sortedflags do - local v = sortedflags[k] - if find(name,v) then - return arguments[sub(v,2,#v)] + local binary = ownbin + if not ownpath or ownpath == "" then + ownpath = ownpath and filedirname(binary) + end + if not ownpath or ownpath == "" then + if os.binsuffix ~= "" then + binary = file.replacesuffix(binary,os.binsuffix) + end + local path = osgetenv("PATH") + if path then + for p in gmatch(path,"[^"..io.pathseparator.."]+") do + local b = filejoin(p,binary) + if lfs.isfile(b) then + -- we assume that after changing to the path the currentdir function + -- resolves to the real location and use this side effect here; this + -- trick is needed because on the mac installations use symlinks in the + -- path instead of real locations + local olddir = lfs.currentdir() + if lfs.chdir(p) then + local pp = lfs.currentdir() + if trace_locating and p ~= pp then + report_initialization("following symlink '%s' to '%s'",p,pp) + end + ownpath = pp + lfs.chdir(olddir) + else + if trace_locating then + report_initialization("unable to check path '%s'",p) + end + ownpath = p + end + break + end + end end end + if not ownpath or ownpath == "" then + ownpath = "." + report_initialization("forcing fallback ownpath .") + elseif trace_locating then + report_initialization("using ownpath '%s'",ownpath) + end end - return nil + + environment.ownbin = ownbin + environment.ownpath = ownpath + end -environment.argument("x",true) +resolvers.ownpath = environment.ownpath -function environment.split_arguments(separator) -- rather special, cut-off before separator - local done, before, after = false, { }, { } - local original_arguments = environment.original_arguments - for k=1,#original_arguments do - local v = original_arguments[k] - if not done and v == separator then - done = true - elseif done then - after[#after+1] = v - else - before[#before+1] = v - end - end - return before, after +function resolvers.getownpath() + return environment.ownpath end -function environment.reconstruct_commandline(arg,noquote) - arg = arg or environment.original_arguments - if noquote and #arg == 1 then - local a = arg[1] - a = resolvers.resolve(a) - a = unquote(a) - return a - elseif #arg > 0 then - local result = { } - for i=1,#arg do - local a = arg[i] - a = resolvers.resolve(a) - a = unquote(a) - a = gsub(a,'"','\\"') -- tricky - if find(a," ") then - result[#result+1] = quote(a) - else - result[#result+1] = a - end - end - return table.join(result," ") +-- The self variables permit us to use only a few (or even no) +-- environment variables. + +do + + local ownpath = environment.ownpath or dir.current() + + if ownpath then + ossetenv('SELFAUTOLOC', file.collapsepath(ownpath)) + ossetenv('SELFAUTODIR', file.collapsepath(ownpath .. "/..")) + ossetenv('SELFAUTOPARENT', file.collapsepath(ownpath .. "/../..")) else - return "" + report_initialization("error: unable to locate ownpath") + os.exit() end + end -if arg then +-- The running os: - -- new, reconstruct quoted snippets (maybe better just remove the " then and add them later) - local newarg, instring = { }, false +-- todo: check is context sits here os.platform is more trustworthy +-- that the bin check as mtx-update runs from another path - for index=1,#arg do - local argument = arg[index] - if find(argument,"^\"") then - newarg[#newarg+1] = gsub(argument,"^\"","") - if not find(argument,"\"$") then - instring = true - end - elseif find(argument,"\"$") then - newarg[#newarg] = newarg[#newarg] .. " " .. gsub(argument,"\"$","") - instring = false - elseif instring then - newarg[#newarg] = newarg[#newarg] .. " " .. argument - else - newarg[#newarg+1] = argument - end - end - for i=1,-5,-1 do - newarg[i] = arg[i] - end +local texos = environment.texos or osgetenv("TEXOS") +local texmfos = environment.texmfos or osgetenv('SELFAUTODIR') - environment.initialize_arguments(newarg) - environment.original_arguments = newarg - environment.raw_arguments = arg +if not texos or texos == "" then + texos = file.basename(texmfos) +end - arg = { } -- prevent duplicate handling +ossetenv('TEXMFOS', texmfos) -- full bin path +ossetenv('TEXOS', texos) -- partial bin parent +ossetenv('SELFAUTOSYSTEM',os.platform) -- bonus -end +environment.texos = texos +environment.texmfos = texmfos --- weird place ... depends on a not yet loaded module +-- The current root: -function environment.texfile(filename) - return resolvers.find_file(filename,'tex') -end +local texroot = environment.texroot or osgetenv("TEXROOT") -function environment.luafile(filename) - local resolved = resolvers.find_file(filename,'tex') or "" - if resolved ~= "" then - return resolved - end - resolved = resolvers.find_file(filename,'texmfscripts') or "" - if resolved ~= "" then - return resolved - end - return resolvers.find_file(filename,'luatexlibs') or "" +if not texroot or texroot == "" then + texroot = osgetenv('SELFAUTOPARENT') + ossetenv('TEXROOT',texroot) end -environment.loadedluacode = loadfile -- can be overloaded +environment.texroot = file.collapsepath(texroot) ---~ function environment.loadedluacode(name) ---~ if os.spawn("texluac -s -o texluac.luc " .. name) == 0 then ---~ local chunk = loadstring(io.loaddata("texluac.luc")) ---~ os.remove("texluac.luc") ---~ return chunk ---~ else ---~ environment.loadedluacode = loadfile -- can be overloaded ---~ return loadfile(name) ---~ end ---~ end - -function environment.luafilechunk(filename) -- used for loading lua bytecode in the format - filename = file.replacesuffix(filename, "lua") - local fullname = environment.luafile(filename) - if fullname and fullname ~= "" then - if trace_locating then - logs.report("fileio","loading file %s", fullname) - end - return environment.loadedluacode(fullname) - else - if trace_locating then - logs.report("fileio","unknown file %s", filename) - end - return nil +-- Tracing. Todo ... + +function resolvers.settrace(n) -- no longer number but: 'locating' or 'detail' + if n then + trackers.disable("resolvers.*") + trackers.enable("resolvers."..n) end end --- the next ones can use the previous ones / combine +resolvers.settrace(osgetenv("MTX_INPUT_TRACE")) -function environment.loadluafile(filename, version) - local lucname, luaname, chunk - local basename = file.removesuffix(filename) - if basename == filename then - lucname, luaname = basename .. ".luc", basename .. ".lua" - else - lucname, luaname = nil, basename -- forced suffix - end - -- when not overloaded by explicit suffix we look for a luc file first - local fullname = (lucname and environment.luafile(lucname)) or "" - if fullname ~= "" then - if trace_locating then - logs.report("fileio","loading %s", fullname) - end - chunk = loadfile(fullname) -- this way we don't need a file exists check - end - if chunk then - assert(chunk)() - if version then - -- we check of the version number of this chunk matches - local v = version -- can be nil - if modules and modules[filename] then - v = modules[filename].version -- new method - elseif versions and versions[filename] then - v = versions[filename] -- old method - end - if v == version then - return true - else - if trace_locating then - logs.report("fileio","version mismatch for %s: lua=%s, luc=%s", filename, v, version) - end - environment.loadluafile(filename) - end - else - return true - end - end - fullname = (luaname and environment.luafile(luaname)) or "" - if fullname ~= "" then - if trace_locating then - logs.report("fileio","loading %s", fullname) - end - chunk = loadfile(fullname) -- this way we don't need a file exists check - if not chunk then - if trace_locating then - logs.report("fileio","unknown file %s", filename) - end - else - assert(chunk)() - return true - end - end - return false +-- todo: + +-- if profiler and osgetenv("MTX_PROFILE_RUN") == "YES" then +-- profiler.start("luatex-profile.log") +-- end + +-- a forward definition + +if not resolvers.resolve then + function resolvers.resolve (s) return s end + function resolvers.unresolve(s) return s end + function resolvers.repath (s) return s end end @@ -7659,536 +9930,1152 @@ end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['trac-inf'] = { +if not modules then modules = { } end modules ['data-exp'] = { version = 1.001, - comment = "companion to trac-inf.mkiv", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" + license = "see context related readme files", } -local format = string.format +local format, find, gmatch, lower = string.format, string.find, string.gmatch, string.lower +local concat, sort = table.concat, table.sort +local lpegmatch, lpegpatterns = lpeg.match, lpeg.patterns +local Ct, Cs, Cc, P, C, S = lpeg.Ct, lpeg.Cs, lpeg.Cc, lpeg.P, lpeg.C, lpeg.S +local type, next = type, next -local statusinfo, n, registered = { }, 0, { } +local ostype = os.type +local collapsepath = file.collapsepath -statistics = statistics or { } +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) +local trace_expansions = false trackers.register("resolvers.expansions", function(v) trace_expansions = v end) -statistics.enable = true -statistics.threshold = 0.05 +local report_expansions = logs.reporter("resolvers","expansions") --- timing functions +local resolvers = resolvers -local clock = os.gettimeofday or os.clock +-- As this bit of code is somewhat special it gets its own module. After +-- all, when working on the main resolver code, I don't want to scroll +-- past this every time. See data-obs.lua for the gsub variant. -local notimer +-- {a,b,c,d} +-- a,b,c/{p,q,r},d +-- a,b,c/{p,q,r}/d/{x,y,z}// +-- a,b,c/{p,q/{x,y,z},r},d/{p,q,r} +-- a,b,c/{p,q/{x,y,z},r},d/{p,q,r} +-- a{b,c}{d,e}f +-- {a,b,c,d} +-- {a,b,c/{p,q,r},d} +-- {a,b,c/{p,q,r}/d/{x,y,z}//} +-- {a,b,c/{p,q/{x,y,z}},d/{p,q,r}} +-- {a,b,c/{p,q/{x,y,z},w}v,d/{p,q,r}} +-- {$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c} -function statistics.hastimer(instance) - return instance and instance.starttime +local function f_first(a,b) + local t, n = { }, 0 + for s in gmatch(b,"[^,]+") do + n = n + 1 ; t[n] = a .. s + end + return concat(t,",") end -function statistics.resettiming(instance) - if not instance then - notimer = { timing = 0, loadtime = 0 } - else - instance.timing, instance.loadtime = 0, 0 +local function f_second(a,b) + local t, n = { }, 0 + for s in gmatch(a,"[^,]+") do + n = n + 1 ; t[n] = s .. b end + return concat(t,",") end -function statistics.starttiming(instance) - if not instance then - notimer = { } - instance = notimer - end - local it = instance.timing - if not it then - it = 0 - end - if it == 0 then - instance.starttime = clock() - if not instance.loadtime then - instance.loadtime = 0 +local function f_both(a,b) + local t, n = { }, 0 + for sa in gmatch(a,"[^,]+") do + for sb in gmatch(b,"[^,]+") do + n = n + 1 ; t[n] = sa .. sb end - else ---~ logs.report("system","nested timing (%s)",tostring(instance)) end - instance.timing = it + 1 + return concat(t,",") end -function statistics.stoptiming(instance, report) - if not instance then - instance = notimer - end - if instance then - local it = instance.timing - if it > 1 then - instance.timing = it - 1 - else - local starttime = instance.starttime - if starttime then - local stoptime = clock() - local loadtime = stoptime - starttime - instance.stoptime = stoptime - instance.loadtime = instance.loadtime + loadtime - if report then - statistics.report("load time %0.3f",loadtime) - end - instance.timing = 0 - return loadtime +local left = P("{") +local right = P("}") +local var = P((1 - S("{}" ))^0) +local set = P((1 - S("{},"))^0) +local other = P(1) + +local l_first = Cs( ( Cc("{") * (C(set) * left * C(var) * right / f_first) * Cc("}") + other )^0 ) +local l_second = Cs( ( Cc("{") * (left * C(var) * right * C(set) / f_second) * Cc("}") + other )^0 ) +local l_both = Cs( ( Cc("{") * (left * C(var) * right * left * C(var) * right / f_both) * Cc("}") + other )^0 ) +local l_rest = Cs( ( left * var * (left/"") * var * (right/"") * var * right + other )^0 ) + +local stripper_1 = lpeg.stripper ("{}@") +local replacer_1 = lpeg.replacer { { ",}", ",@}" }, { "{,", "{@," }, } + +local function splitpathexpr(str, newlist, validate) -- I couldn't resist lpegging it (nice exercise). + if trace_expansions then + report_expansions("expanding variable '%s'",str) + end + local t, ok, done = newlist or { }, false, false + local n = #t + str = lpegmatch(replacer_1,str) + repeat local old = str + repeat local old = str ; str = lpegmatch(l_first, str) until old == str + repeat local old = str ; str = lpegmatch(l_second,str) until old == str + repeat local old = str ; str = lpegmatch(l_both, str) until old == str + repeat local old = str ; str = lpegmatch(l_rest, str) until old == str + until old == str -- or not find(str,"{") + str = lpegmatch(stripper_1,str) + if validate then + for s in gmatch(str,"[^,]+") do + s = validate(s) + if s then + n = n + 1 ; t[n] = s end end + else + for s in gmatch(str,"[^,]+") do + n = n + 1 ; t[n] = s + end end - return 0 + if trace_expansions then + for k=1,#t do + report_expansions("% 4i: %s",k,t[k]) + end + end + return t end -function statistics.elapsedtime(instance) - if not instance then - instance = notimer - end - return format("%0.3f",(instance and instance.loadtime) or 0) +-- We could make the previous one public. + +local function validate(s) + s = collapsepath(s) -- already keeps the // + return s ~= "" and not find(s,"^!*unset/*$") and s end -function statistics.elapsedindeed(instance) - if not instance then - instance = notimer +resolvers.validatedpath = validate -- keeps the trailing // + +function resolvers.expandedpathfromlist(pathlist) + local newlist = { } + for k=1,#pathlist do + splitpathexpr(pathlist[k],newlist,validate) end - local t = (instance and instance.loadtime) or 0 - return t > statistics.threshold + return newlist end -function statistics.elapsedseconds(instance,rest) -- returns nil if 0 seconds - if statistics.elapsedindeed(instance) then - return format("%s seconds %s", statistics.elapsedtime(instance),rest or "") +local cleanup = lpeg.replacer { + { "!" , "" }, + { "\\" , "/" }, +} + +local homedir + +function resolvers.cleanpath(str) + if not homedir then + homedir = lpegmatch(cleanup,environment.homedir or "") + if homedir == string.char(127) or homedir == "" or not lfs.isdir(homedir) then + if trace_expansions then + report_expansions("no home dir set, ignoring dependent paths") + end + function resolvers.cleanpath(str) + if find(str,"~") then + return "" -- special case + else + return str and lpegmatch(cleanup,str) + end + end + else + cleanup = lpeg.replacer { + { "!" , "" }, + { "\\" , "/" }, + { "~" , homedir }, + } + function resolvers.cleanpath(str) + return str and lpegmatch(cleanup,str) + end + end end + return resolvers.cleanpath(str) end --- general function +-- This one strips quotes and funny tokens. -function statistics.register(tag,fnc) - if statistics.enable and type(fnc) == "function" then - local rt = registered[tag] or (#statusinfo + 1) - statusinfo[rt] = { tag, fnc } - registered[tag] = rt - if #tag > n then n = #tag end - end +local expandhome = P("~") / "$HOME" -- environment.homedir + +local dodouble = P('"')/"" * (expandhome + (1 - P('"')))^0 * P('"')/"" +local dosingle = P("'")/"" * (expandhome + (1 - P("'")))^0 * P("'")/"" +local dostring = (expandhome + 1 )^0 + +local stripper = Cs( + lpegpatterns.unspacer * (dosingle + dodouble + dostring) * lpegpatterns.unspacer +) + +function resolvers.checkedvariable(str) -- assumes str is a string + return lpegmatch(stripper,str) or str end -function statistics.show(reporter) - if statistics.enable then - if not reporter then reporter = function(tag,data,n) texio.write_nl(tag .. " " .. data) end end - -- this code will move - local register = statistics.register - register("luatex banner", function() - return string.lower(status.banner) - end) - register("control sequences", function() - return format("%s of %s", status.cs_count, status.hash_size+status.hash_extra) - end) - register("callbacks", function() - local total, indirect = status.callbacks or 0, status.indirect_callbacks or 0 - return format("direct: %s, indirect: %s, total: %s", total-indirect, indirect, total) - end) - register("current memory usage", statistics.memused) - register("runtime",statistics.runtime) --- -- - for i=1,#statusinfo do - local s = statusinfo[i] - local r = s[2]() - if r then - reporter(s[1],r,n) +-- The path splitter: + +-- A config (optionally) has the paths split in tables. Internally +-- we join them and split them after the expansion has taken place. This +-- is more convenient. + +local cache = { } + +---- splitter = Ct(lpeg.splitat(S(ostype == "windows" and ";" or ":;"))) -- maybe add , +local splitter = Ct(lpeg.splitat(";")) -- as we move towards urls, prefixes and use tables we no longer do : + +local backslashswapper = lpeg.replacer("\\","/") + +local function splitconfigurationpath(str) -- beware, this can be either a path or a { specification } + if str then + local found = cache[str] + if not found then + if str == "" then + found = { } + else + local split = lpegmatch(splitter,lpegmatch(backslashswapper,str)) -- can be combined + found = { } + local noffound = 0 + for i=1,#split do + local s = split[i] + if not find(s,"^{*unset}*") then + noffound = noffound + 1 + found[noffound] = s + end + end + if trace_expansions then + report_expansions("splitting path specification '%s'",str) + for k=1,noffound do + report_expansions("% 4i: %s",k,found[k]) + end + end + cache[str] = found end end - texio.write_nl("") -- final newline - statistics.enable = false + return found end end -function statistics.show_job_stat(tag,data,n) - texio.write_nl(format("%-15s: %s - %s","mkiv lua stats",tag:rpadd(n," "),data)) -end +resolvers.splitconfigurationpath = splitconfigurationpath -function statistics.memused() -- no math.round yet -) - local round = math.round or math.floor - return format("%s MB (ctx: %s MB)",round(collectgarbage("count")/1000), round(status.luastate_bytes/1000000)) +function resolvers.splitpath(str) + if type(str) == 'table' then + return str + else + return splitconfigurationpath(str) + end end -if statistics.runtime then - -- already loaded and set -elseif luatex and luatex.starttime then - statistics.starttime = luatex.starttime - statistics.loadtime = 0 - statistics.timing = 0 -else - statistics.starttiming(statistics) +function resolvers.joinpath(str) + if type(str) == 'table' then + return file.joinpath(str) + else + return str + end end -function statistics.runtime() - statistics.stoptiming(statistics) - return statistics.formatruntime(statistics.elapsedtime(statistics)) -end +-- The next function scans directories and returns a hash where the +-- entries are either strings or tables. -function statistics.formatruntime(runtime) - return format("%s seconds", statistics.elapsedtime(statistics)) -end +-- starting with . or .. etc or funny char -function statistics.timed(action,report) - local timer = { } - report = report or logs.simple - statistics.starttiming(timer) - action() - statistics.stoptiming(timer) - report("total runtime: %s",statistics.elapsedtime(timer)) -end --- where, not really the best spot for this: -commands = commands or { } -local timer +local weird = P(".")^1 + lpeg.anywhere(S("~`!#$%^&*()={}[]:;\"\'||<>,?\n\r\t")) -function commands.resettimer() - statistics.resettiming(timer) - statistics.starttiming(timer) +local attributes, directory = lfs.attributes, lfs.dir + +local function scan(files,spec,path,n,m,r) + local full = (path == "" and spec) or (spec .. path .. '/') + local dirs, nofdirs = { }, 0 + for name in directory(full) do + if not lpegmatch(weird,name) then + local mode = attributes(full..name,'mode') + if mode == 'file' then + n = n + 1 + local f = files[name] + if f then + if type(f) == 'string' then + files[name] = { f, path } + else + f[#f+1] = path + end + else -- probably unique anyway + files[name] = path + local lower = lower(name) + if name ~= lower then + files["remap:"..lower] = name + r = r + 1 + end + end + elseif mode == 'directory' then + m = m + 1 + nofdirs = nofdirs + 1 + if path ~= "" then + dirs[nofdirs] = path..'/'..name + else + dirs[nofdirs] = name + end + end + end + end + if nofdirs > 0 then + sort(dirs) + for i=1,nofdirs do + files, n, m, r = scan(files,spec,dirs[i],n,m,r) + end + end + return files, n, m, r end -function commands.elapsedtime() - statistics.stoptiming(timer) - tex.sprint(statistics.elapsedtime(timer)) +function resolvers.scanfiles(path,branch) + if trace_locating then + report_expansions("scanning path '%s', branch '%s'",path, branch or path) + end + local realpath = resolvers.resolve(path) -- no shortcut + local files, n, m, r = scan({ },realpath .. '/',"",0,0,0) + files.__path__ = path -- can be selfautoparent:texmf-whatever + files.__files__ = n + files.__directories__ = m + files.__remappings__ = r + if trace_locating then + report_expansions("%s files found on %s directories with %s uppercase remappings",n,m,r) + end + return files end -commands.resettimer() end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['trac-log'] = { +if not modules then modules = { } end modules ['data-env'] = { version = 1.001, - comment = "companion to trac-log.mkiv", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" + license = "see context related readme files", } --- this is old code that needs an overhaul +local lower, gsub = string.lower, string.gsub ---~ io.stdout:setvbuf("no") ---~ io.stderr:setvbuf("no") +local resolvers = resolvers -local write_nl, write = texio.write_nl or print, texio.write or io.write -local format, gmatch = string.format, string.gmatch -local texcount = tex and tex.count +local allocate = utilities.storage.allocate +local setmetatableindex = table.setmetatableindex +local fileextname = file.extname + +local formats = allocate() +local suffixes = allocate() +local dangerous = allocate() +local suffixmap = allocate() + +resolvers.formats = formats +resolvers.suffixes = suffixes +resolvers.dangerous = dangerous +resolvers.suffixmap = suffixmap + +local relations = allocate { -- todo: handlers also here + core = { + ofm = { -- will become obsolete + names = { "ofm", "omega font metric", "omega font metrics" }, + variable = 'OFMFONTS', + suffixes = { 'ofm', 'tfm' }, + }, + ovf = { -- will become obsolete + names = { "ovf", "omega virtual font", "omega virtual fonts" }, + variable = 'OVFFONTS', + suffixes = { 'ovf', 'vf' }, + }, + tfm = { + names = { "tfm", "tex font metric", "tex font metrics" }, + variable = 'TFMFONTS', + suffixes = { 'tfm' }, + }, + vf = { + names = { "vf", "virtual font", "virtual fonts" }, + variable = 'VFFONTS', + suffixes = { 'vf' }, + }, + otf = { + names = { "otf", "opentype", "opentype font", "opentype fonts"}, + variable = 'OPENTYPEFONTS', + suffixes = { 'otf' }, + }, + ttf = { + names = { "ttf", "truetype", "truetype font", "truetype fonts", "truetype collection", "truetype collections", "truetype dictionary", "truetype dictionaries" }, + variable = 'TTFONTS', + suffixes = { 'ttf', 'ttc', 'dfont' }, + }, + afm = { + names = { "afm", "adobe font metric", "adobe font metrics" }, + variable = "AFMFONTS", + suffixes = { "afm" }, + }, + pfb = { + names = { "pfb", "type1", "type 1", "type1 font", "type 1 font", "type1 fonts", "type 1 fonts" }, + variable = 'T1FONTS', + suffixes = { 'pfb', 'pfa' }, + }, + fea = { + names = { "fea", "font feature", "font features", "font feature file", "font feature files" }, + variable = 'FONTFEATURES', + suffixes = { 'fea' }, + }, + cid = { + names = { "cid", "cid map", "cid maps", "cid file", "cid files" }, + variable = 'FONTCIDMAPS', + suffixes = { 'cid', 'cidmap' }, + }, + fmt = { + names = { "fmt", "format", "tex format" }, + variable = 'TEXFORMATS', + suffixes = { 'fmt' }, + }, + mem = { -- will become obsolete + names = { 'mem', "metapost format" }, + variable = 'MPMEMS', + suffixes = { 'mem' }, + }, + mp = { + names = { "mp" }, + variable = 'MPINPUTS', + suffixes = { 'mp' }, + }, + tex = { + names = { "tex" }, + variable = 'TEXINPUTS', + suffixes = { 'tex', "mkiv", "mkiv", "mkii" }, + }, + icc = { + names = { "icc", "icc profile", "icc profiles" }, + variable = 'ICCPROFILES', + suffixes = { 'icc' }, + }, + texmfscripts = { + names = { "texmfscript", "texmfscripts", "script", "scripts" }, + variable = 'TEXMFSCRIPTS', + suffixes = { 'rb', 'pl', 'py' }, + }, + lua = { + names = { "lua" }, + variable = 'LUAINPUTS', + suffixes = { 'lua', 'luc', 'tma', 'tmc' }, + }, + lib = { + names = { "lib" }, + variable = 'CLUAINPUTS', + suffixes = os.libsuffix and { os.libsuffix } or { 'dll', 'so' }, + }, + bib = { + names = { 'bib' }, + suffixes = { 'bib' }, + }, + bst = { + names = { 'bst' }, + suffixes = { 'bst' }, + }, + fontconfig = { + names = { 'fontconfig', 'fontconfig file', 'fontconfig files' }, + variable = 'FONTCONFIG_PATH', + }, + }, + obsolete = { + enc = { + names = { "enc", "enc files", "enc file", "encoding files", "encoding file" }, + variable = 'ENCFONTS', + suffixes = { 'enc' }, + }, + map = { + names = { "map", "map files", "map file" }, + variable = 'TEXFONTMAPS', + suffixes = { 'map' }, + }, + lig = { + names = { "lig files", "lig file", "ligature file", "ligature files" }, + variable = 'LIGFONTS', + suffixes = { 'lig' }, + }, + opl = { + names = { "opl" }, + variable = 'OPLFONTS', + suffixes = { 'opl' }, + }, + ovp = { + names = { "ovp" }, + variable = 'OVPFONTS', + suffixes = { 'ovp' }, + }, + }, + kpse = { -- subset + base = { + names = { 'base', "metafont format" }, + variable = 'MFBASES', + suffixes = { 'base', 'bas' }, + }, + cmap = { + names = { 'cmap', 'cmap files', 'cmap file' }, + variable = 'CMAPFONTS', + suffixes = { 'cmap' }, + }, + cnf = { + names = { 'cnf' }, + suffixes = { 'cnf' }, + }, + web = { + names = { 'web' }, + suffixes = { 'web', 'ch' } + }, + cweb = { + names = { 'cweb' }, + suffixes = { 'w', 'web', 'ch' }, + }, + gf = { + names = { 'gf' }, + suffixes = { '<resolution>gf' }, + }, + mf = { + names = { 'mf' }, + variable = 'MFINPUTS', + suffixes = { 'mf' }, + }, + mft = { + names = { 'mft' }, + suffixes = { 'mft' }, + }, + pk = { + names = { 'pk' }, + suffixes = { '<resolution>pk' }, + }, + }, +} -if texlua then - write_nl = print - write = io.write +resolvers.relations = relations + +-- formats: maps a format onto a variable + +function resolvers.updaterelations() + for category, categories in next, relations do + for name, relation in next, categories do + local rn = relation.names + local rv = relation.variable + local rs = relation.suffixes + if rn and rv then + for i=1,#rn do + local rni = lower(gsub(rn[i]," ","")) + formats[rni] = rv + if rs then + suffixes[rni] = rs + for i=1,#rs do + local rsi = rs[i] + suffixmap[rsi] = rni + end + end + end + end + if rs then + end + end + end end ---[[ldx-- -<p>This is a prelude to a more extensive logging module. For the sake -of parsing log files, in addition to the standard logging we will -provide an <l n='xml'/> structured file. Actually, any logging that -is hooked into callbacks will be \XML\ by default.</p> ---ldx]]-- +resolvers.updaterelations() -- push this in the metatable -> newindex + +local function simplified(t,k) + return rawget(t,lower(gsub(k," ",""))) +end -logs = logs or { } -logs.xml = logs.xml or { } -logs.tex = logs.tex or { } +setmetatableindex(formats, simplified) +setmetatableindex(suffixes, simplified) +setmetatableindex(suffixmap, simplified) ---[[ldx-- -<p>This looks pretty ugly but we need to speed things up a bit.</p> ---ldx]]-- +-- A few accessors, mostly for command line tool. -logs.moreinfo = [[ -more information about ConTeXt and the tools that come with it can be found at: +function resolvers.suffixofformat(str) + local s = suffixes[str] + return s and s[1] or "" +end -maillist : ntg-context@ntg.nl / http://www.ntg.nl/mailman/listinfo/ntg-context -webpage : http://www.pragma-ade.nl / http://tex.aanhet.net -wiki : http://contextgarden.net -]] +function resolvers.suffixofformat(str) + return suffixes[str] or { } +end -logs.levels = { - ['error'] = 1, - ['warning'] = 2, - ['info'] = 3, - ['debug'] = 4, -} +for name, format in next, formats do + dangerous[name] = true -- still needed ? +end -logs.functions = { - 'report', 'start', 'stop', 'push', 'pop', 'line', 'direct', - 'start_run', 'stop_run', - 'start_page_number', 'stop_page_number', - 'report_output_pages', 'report_output_log', - 'report_tex_stat', 'report_job_stat', - 'show_open', 'show_close', 'show_load', -} +-- because vf searching is somewhat dangerous, we want to prevent +-- too liberal searching esp because we do a lookup on the current +-- path anyway; only tex (or any) is safe -logs.tracers = { -} +dangerous.tex = nil -logs.level = 0 -logs.mode = string.lower((os.getenv("MTX.LOG.MODE") or os.getenv("MTX_LOG_MODE") or "tex")) -function logs.set_level(level) - logs.level = logs.levels[level] or level -end +-- more helpers -function logs.set_method(method) - for _, v in next, logs.functions do - logs[v] = logs[method][v] or function() end - end +function resolvers.formatofvariable(str) + return formats[str] or '' end --- tex logging +function resolvers.formatofsuffix(str) -- of file + return suffixmap[fileextname(str)] or 'tex' -- so many map onto tex (like mkiv, cld etc) +end -function logs.tex.report(category,fmt,...) -- new - if fmt then - write_nl(category .. " | " .. format(fmt,...)) - else - write_nl(category .. " |") - end +function resolvers.variableofformat(str) + return formats[str] or '' end -function logs.tex.line(fmt,...) -- new - if fmt then - write_nl(format(fmt,...)) - else - write_nl("") +function resolvers.variableofformatorsuffix(str) + local v = formats[str] + if v then + return v + end + v = suffixmap[fileextname(str)] + if v then + return formats[v] end + return '' end ---~ function logs.tex.start_page_number() ---~ local real, user, sub = texcount.realpageno, texcount.userpageno, texcount.subpageno ---~ if real > 0 then ---~ if user > 0 then ---~ if sub > 0 then ---~ write(format("[%s.%s.%s",real,user,sub)) ---~ else ---~ write(format("[%s.%s",real,user)) ---~ end ---~ else ---~ write(format("[%s",real)) ---~ end ---~ else ---~ write("[-") ---~ end ---~ end ---~ function logs.tex.stop_page_number() ---~ write("]") ---~ end -local real, user, sub +end -- of closure -function logs.tex.start_page_number() - real, user, sub = texcount.realpageno, texcount.userpageno, texcount.subpageno +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['data-tmp'] = { + version = 1.100, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +--[[ldx-- +<p>This module deals with caching data. It sets up the paths and +implements loaders and savers for tables. Best is to set the +following variable. When not set, the usual paths will be +checked. Personally I prefer the (users) temporary path.</p> + +</code> +TEXMFCACHE=$TMP;$TEMP;$TMPDIR;$TEMPDIR;$HOME;$TEXMFVAR;$VARTEXMF;. +</code> + +<p>Currently we do no locking when we write files. This is no real +problem because most caching involves fonts and the chance of them +being written at the same time is small. We also need to extend +luatools with a recache feature.</p> +--ldx]]-- + +local format, lower, gsub, concat = string.format, string.lower, string.gsub, table.concat +local mkdirs, isdir = dir.mkdirs, lfs.isdir + +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) +local trace_cache = false trackers.register("resolvers.cache", function(v) trace_cache = v end) + +local report_caches = logs.reporter("resolvers","caches") +local report_resolvers = logs.reporter("resolvers","caching") + +local resolvers = resolvers + +-- intermezzo + +local directive_cleanup = false directives.register("system.compile.cleanup", function(v) directive_cleanup = v end) +local directive_strip = true directives.register("system.compile.strip", function(v) directive_strip = v end) + +local compile = utilities.lua.compile + +function utilities.lua.compile(luafile,lucfile,cleanup,strip) + if cleanup == nil then cleanup = directive_cleanup end + if strip == nil then strip = directive_strip end + return compile(luafile,lucfile,cleanup,strip) end -function logs.tex.stop_page_number() - if real > 0 then - if user > 0 then - if sub > 0 then - logs.report("pages", "flushing realpage %s, userpage %s, subpage %s",real,user,sub) - else - logs.report("pages", "flushing realpage %s, userpage %s",real,user) +-- end of intermezzo + +caches = caches or { } +local caches = caches + +caches.base = caches.base or "luatex-cache" +caches.more = caches.more or "context" +caches.direct = false -- true is faster but may need huge amounts of memory +caches.tree = false +caches.force = true +caches.ask = false +caches.relocate = false +caches.defaults = { "TMPDIR", "TEMPDIR", "TMP", "TEMP", "HOME", "HOMEPATH" } + +local writable, readables, usedreadables = nil, { }, { } + +-- we could use a metatable for writable and readable but not yet + +local function identify() + -- Combining the loops makes it messy. First we check the format cache path + -- and when the last component is not present we try to create it. + local texmfcaches = resolvers.cleanpathlist("TEXMFCACHE") + if texmfcaches then + for k=1,#texmfcaches do + local cachepath = texmfcaches[k] + if cachepath ~= "" then + cachepath = resolvers.resolve(cachepath) + cachepath = resolvers.cleanpath(cachepath) + cachepath = file.collapsepath(cachepath) + local valid = isdir(cachepath) + if valid then + if file.is_readable(cachepath) then + readables[#readables+1] = cachepath + if not writable and file.is_writable(cachepath) then + writable = cachepath + end + end + elseif not writable and caches.force then + local cacheparent = file.dirname(cachepath) + if file.is_writable(cacheparent) then + if not caches.ask or io.ask(format("\nShould I create the cache path %s?",cachepath), "no", { "yes", "no" }) == "yes" then + mkdirs(cachepath) + if isdir(cachepath) and file.is_writable(cachepath) then + report_caches("created: %s",cachepath) + writable = cachepath + readables[#readables+1] = cachepath + end + end + end + end end - else - logs.report("pages", "flushing realpage %s",real) + end + end + -- As a last resort we check some temporary paths but this time we don't + -- create them. + local texmfcaches = caches.defaults + if texmfcaches then + for k=1,#texmfcaches do + local cachepath = texmfcaches[k] + cachepath = resolvers.expansion(cachepath) -- was getenv + if cachepath ~= "" then + cachepath = resolvers.resolve(cachepath) + cachepath = resolvers.cleanpath(cachepath) + local valid = isdir(cachepath) + if valid and file.is_readable(cachepath) then + if not writable and file.is_writable(cachepath) then + readables[#readables+1] = cachepath + writable = cachepath + break + end + end + end + end + end + -- Some extra checking. If we have no writable or readable path then we simply + -- quit. + if not writable then + report_caches("fatal error: there is no valid writable cache path defined") + os.exit() + elseif #readables == 0 then + report_caches("fatal error: there is no valid readable cache path defined") + os.exit() + end + -- why here + writable = dir.expandname(resolvers.cleanpath(writable)) -- just in case + -- moved here + local base, more, tree = caches.base, caches.more, caches.tree or caches.treehash() -- we have only one writable tree + if tree then + caches.tree = tree + writable = mkdirs(writable,base,more,tree) + for i=1,#readables do + readables[i] = file.join(readables[i],base,more,tree) end else - logs.report("pages", "flushing page") + writable = mkdirs(writable,base,more) + for i=1,#readables do + readables[i] = file.join(readables[i],base,more) + end + end + -- end + if trace_cache then + for i=1,#readables do + report_caches("using readable path '%s' (order %s)",readables[i],i) + end + report_caches("using writable path '%s'",writable) + end + identify = function() + return writable, readables end - io.flush() + return writable, readables end -logs.tex.report_job_stat = statistics.show_job_stat +function caches.usedpaths() + local writable, readables = identify() + if #readables > 1 then + local result = { } + for i=1,#readables do + local readable = readables[i] + if usedreadables[i] or readable == writable then + result[#result+1] = format("readable: '%s' (order %s)",readable,i) + end + end + result[#result+1] = format("writable: '%s'",writable) + return result + else + return writable + end +end --- xml logging +function caches.configfiles() + return table.concat(resolvers.instance.specification,";") +end -function logs.xml.report(category,fmt,...) -- new - if fmt then - write_nl(format("<r category='%s'>%s</r>",category,format(fmt,...))) - else - write_nl(format("<r category='%s'/>",category)) +function caches.hashed(tree) + tree = gsub(tree,"\\$","/") + tree = gsub(tree,"/+$","") + tree = lower(tree) + local hash = md5.hex(tree) + if trace_cache or trace_locating then + report_caches("hashing tree %s, hash %s",tree,hash) end + return hash end -function logs.xml.line(fmt,...) -- new - if fmt then - write_nl(format("<r>%s</r>",format(fmt,...))) + +function caches.treehash() + local tree = caches.configfiles() + if not tree or tree == "" then + return false else - write_nl("<r/>") + return caches.hashed(tree) end end -function logs.xml.start() if logs.level > 0 then tw("<%s>" ) end end -function logs.xml.stop () if logs.level > 0 then tw("</%s>") end end -function logs.xml.push () if logs.level > 0 then tw("<!-- ") end end -function logs.xml.pop () if logs.level > 0 then tw(" -->" ) end end +local r_cache, w_cache = { }, { } -- normally w in in r but who cares -function logs.xml.start_run() - write_nl("<?xml version='1.0' standalone='yes'?>") - write_nl("<job>") -- xmlns='www.pragma-ade.com/luatex/schemas/context-job.rng' - write_nl("") -end - -function logs.xml.stop_run() - write_nl("</job>") +local function getreadablepaths(...) -- we can optimize this as we have at most 2 tags + local tags = { ... } + local hash = concat(tags,"/") + local done = r_cache[hash] + if not done then + local writable, readables = identify() -- exit if not found + if #tags > 0 then + done = { } + for i=1,#readables do + done[i] = file.join(readables[i],...) + end + else + done = readables + end + r_cache[hash] = done + end + return done end -function logs.xml.start_page_number() - write_nl(format("<p real='%s' page='%s' sub='%s'", texcount.realpageno, texcount.userpageno, texcount.subpageno)) +local function getwritablepath(...) + local tags = { ... } + local hash = concat(tags,"/") + local done = w_cache[hash] + if not done then + local writable, readables = identify() -- exit if not found + if #tags > 0 then + done = mkdirs(writable,...) + else + done = writable + end + w_cache[hash] = done + end + return done end -function logs.xml.stop_page_number() - write("/>") - write_nl("") -end +caches.getreadablepaths = getreadablepaths +caches.getwritablepath = getwritablepath -function logs.xml.report_output_pages(p,b) - write_nl(format("<v k='pages' v='%s'/>", p)) - write_nl(format("<v k='bytes' v='%s'/>", b)) - write_nl("") +function caches.getfirstreadablefile(filename,...) + local rd = getreadablepaths(...) + for i=1,#rd do + local path = rd[i] + local fullname = file.join(path,filename) + if file.is_readable(fullname) then + usedreadables[i] = true + return fullname, path + end + end + return caches.setfirstwritablefile(filename,...) end -function logs.xml.report_output_log() +function caches.setfirstwritablefile(filename,...) + local wr = getwritablepath(...) + local fullname = file.join(wr,filename) + return fullname, wr end -function logs.xml.report_tex_stat(k,v) - texiowrite_nl("log","<v k='"..k.."'>"..tostring(v).."</v>") +function caches.define(category,subcategory) -- for old times sake + return function() + return getwritablepath(category,subcategory) + end end -local level = 0 - -function logs.xml.show_open(name) - level = level + 1 - texiowrite_nl(format("<f l='%s' n='%s'>",level,name)) +function caches.setluanames(path,name) + return path .. "/" .. name .. ".tma", path .. "/" .. name .. ".tmc" end -function logs.xml.show_close(name) - texiowrite("</f> ") - level = level - 1 +function caches.loaddata(readables,name) + if type(readables) == "string" then + readables = { readables } + end + for i=1,#readables do + local path = readables[i] + local tmaname, tmcname = caches.setluanames(path,name) + local loader = loadfile(tmcname) or loadfile(tmaname) + if loader then + loader = loader() + collectgarbage("step") + return loader + end + end + return false end -function logs.xml.show_load(name) - texiowrite_nl(format("<f l='%s' n='%s'/>",level+1,name)) +function caches.is_writable(filepath,filename) + local tmaname, tmcname = caches.setluanames(filepath,filename) + return file.is_writable(tmaname) end --- - -local name, banner = 'report', 'context' +local saveoptions = { compact = true } -local function report(category,fmt,...) - if fmt then - write_nl(format("%s | %s: %s",name,category,format(fmt,...))) - elseif category then - write_nl(format("%s | %s",name,category)) +function caches.savedata(filepath,filename,data,raw) + local tmaname, tmcname = caches.setluanames(filepath,filename) + local reduce, simplify = true, true + if raw then + reduce, simplify = false, false + end + data.cache_uuid = os.uuid() + if caches.direct then + file.savedata(tmaname,table.serialize(data,true,saveoptions)) else - write_nl(format("%s |",name)) + table.tofile(tmaname,data,true,saveoptions) end + utilities.lua.compile(tmaname,tmcname) end -local function simple(fmt,...) - if fmt then - write_nl(format("%s | %s",name,format(fmt,...))) - else - write_nl(format("%s |",name)) - end +-- moved from data-res: + +local content_state = { } + +function caches.contentstate() + return content_state or { } end -function logs.setprogram(_name_,_banner_,_verbose_) - name, banner = _name_, _banner_ - if _verbose_ then - trackers.enable("resolvers.locating") - end - logs.set_method("tex") - logs.report = report -- also used in libraries - logs.simple = simple -- only used in scripts ! - if utils then - utils.report = simple +function caches.loadcontent(cachename,dataname) + local name = caches.hashed(cachename) + local full, path = caches.getfirstreadablefile(name ..".lua","trees") + local filename = file.join(path,name) + local blob = loadfile(filename .. ".luc") or loadfile(filename .. ".lua") + if blob then + local data = blob() + if data and data.content then + if data.type == dataname then + if data.version == resolvers.cacheversion then + content_state[#content_state+1] = data.uuid + if trace_locating then + report_resolvers("loading '%s' for '%s' from '%s'",dataname,cachename,filename) + end + return data.content + else + report_resolvers("skipping '%s' for '%s' from '%s' (version mismatch)",dataname,cachename,filename) + end + else + report_resolvers("skipping '%s' for '%s' from '%s' (datatype mismatch)",dataname,cachename,filename) + end + elseif trace_locating then + report_resolvers("skipping '%s' for '%s' from '%s' (no content)",dataname,cachename,filename) + end + elseif trace_locating then + report_resolvers("skipping '%s' for '%s' from '%s' (invalid file)",dataname,cachename,filename) end - logs.verbose = _verbose_ end -function logs.setverbose(what) - if what then - trackers.enable("resolvers.locating") - else - trackers.disable("resolvers.locating") +function caches.collapsecontent(content) + for k, v in next, content do + if type(v) == "table" and #v == 1 then + content[k] = v[1] + end end - logs.verbose = what or false end -function logs.extendbanner(_banner_,_verbose_) - banner = banner .. " | ".. _banner_ - if _verbose_ ~= nil then - logs.setverbose(what) +function caches.savecontent(cachename,dataname,content) + local name = caches.hashed(cachename) + local full, path = caches.setfirstwritablefile(name ..".lua","trees") + local filename = file.join(path,name) -- is full + local luaname, lucname = filename .. ".lua", filename .. ".luc" + if trace_locating then + report_resolvers("preparing '%s' for '%s'",dataname,cachename) + end + local data = { + type = dataname, + root = cachename, + version = resolvers.cacheversion, + date = os.date("%Y-%m-%d"), + time = os.date("%H:%M:%S"), + content = content, + uuid = os.uuid(), + } + local ok = io.savedata(luaname,table.serialize(data,true)) + if ok then + if trace_locating then + report_resolvers("category '%s', cachename '%s' saved in '%s'",dataname,cachename,luaname) + end + if utilities.lua.compile(luaname,lucname) then + if trace_locating then + report_resolvers("'%s' compiled to '%s'",dataname,lucname) + end + return true + else + if trace_locating then + report_resolvers("compiling failed for '%s', deleting file '%s'",dataname,lucname) + end + os.remove(lucname) + end + elseif trace_locating then + report_resolvers("unable to save '%s' in '%s' (access error)",dataname,luaname) end end -logs.verbose = false -logs.report = logs.tex.report -logs.simple = logs.tex.report -function logs.reportlines(str) -- todo: <lines></lines> - for line in gmatch(str,"(.-)[\n\r]") do - logs.report(line) - end -end +end -- of closure -function logs.reportline() -- for scripts too - logs.report() -end +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['data-met'] = { + version = 1.100, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local find, format = string.find, string.format +local sequenced = table.sequenced +local addurlscheme, urlhashed = url.addscheme, url.hashed + +local trace_locating = false + +trackers.register("resolvers.locating", function(v) trace_methods = v end) +trackers.register("resolvers.methods", function(v) trace_methods = v end) + + +local report_methods = logs.reporter("resolvers","methods") -logs.simpleline = logs.reportline +local allocate = utilities.storage.allocate -function logs.reportbanner() -- for scripts too - logs.report(banner) +local resolvers = resolvers + +local registered = { } + +local function splitmethod(filename) -- todo: filetype in specification + if not filename then + return { scheme = "unknown", original = filename } + end + if type(filename) == "table" then + return filename -- already split + end + filename = file.collapsepath(filename) + if not find(filename,"://") then + return { scheme = "file", path = filename, original = filename, filename = filename } + end + local specification = url.hashed(filename) + if not specification.scheme or specification.scheme == "" then + return { scheme = "file", path = filename, original = filename, filename = filename } + else + return specification + end end -function logs.help(message,option) - logs.reportbanner() - logs.reportline() - logs.reportlines(message) - local moreinfo = logs.moreinfo or "" - if moreinfo ~= "" and option ~= "nomoreinfo" then - logs.reportline() - logs.reportlines(moreinfo) +resolvers.splitmethod = splitmethod -- bad name but ok + +-- the second argument is always analyzed (saves time later on) and the original +-- gets passed as original but also as argument + +local function methodhandler(what,first,...) -- filename can be nil or false + local method = registered[what] + if method then + local how, namespace = method.how, method.namespace + if how == "uri" or how == "url" then + local specification = splitmethod(first) + local scheme = specification.scheme + local resolver = namespace and namespace[scheme] + if resolver then + if trace_methods then + report_methods("resolver: method=%s, how=%s, scheme=%s, argument=%s",what,how,scheme,first) + end + return resolver(specification,...) + else + resolver = namespace.default or namespace.file + if resolver then + if trace_methods then + report_methods("resolver: method=%s, how=%s, default, argument=%s",what,how,first) + end + return resolver(specification,...) + elseif trace_methods then + report_methods("resolver: method=%s, how=%s, no handler",what,how) + end + end + elseif how == "tag" then + local resolver = namespace and namespace[first] + if resolver then + if trace_methods then + report_methods("resolver: method=%s, how=%s, tag=%s",what,how,first) + end + return resolver(...) + else + resolver = namespace.default or namespace.file + if resolver then + if trace_methods then + report_methods("resolver: method=%s, how=%s, default",what,how) + end + return resolver(...) + elseif trace_methods then + report_methods("resolver: method=%s, how=%s, unknown",what,how) + end + end + end + else + report_methods("resolver: method=%s, unknown",what) end end -logs.set_level('error') -logs.set_method('tex') +resolvers.methodhandler = methodhandler -function logs.system(whereto,process,jobname,category,...) - for i=1,10 do - local f = io.open(whereto,"a") - if f then - f:write(format("%s %s => %s => %s => %s\r",os.date("%d/%m/%y %H:%m:%S"),process,jobname,category,format(...))) - f:close() - break +function resolvers.registermethod(name,namespace,how) + registered[name] = { how = how or "tag", namespace = namespace } + namespace["byscheme"] = function(scheme,filename,...) + if scheme == "file" then + return methodhandler(name,filename,...) else - sleep(0.1) + return methodhandler(name,addurlscheme(filename,scheme),...) end end end ---~ local syslogname = "oeps.xxx" ---~ ---~ for i=1,10 do ---~ logs.system(syslogname,"context","test","fonts","font %s recached due to newer version (%s)","blabla","123") ---~ end +local concatinators = allocate { notfound = file.join } -- concatinate paths +local locators = allocate { notfound = function() end } -- locate databases +local hashers = allocate { notfound = function() end } -- load databases +local generators = allocate { notfound = function() end } -- generate databases -function logs.fatal(where,...) - logs.report(where,"fatal error: %s, aborting now",format(...)) - os.exit() -end +resolvers.concatinators = concatinators +resolvers.locators = locators +resolvers.hashers = hashers +resolvers.generators = generators + +local registermethod = resolvers.registermethod + +registermethod("concatinators",concatinators,"tag") +registermethod("locators", locators, "uri") +registermethod("hashers", hashers, "uri") +registermethod("generators", generators, "uri") end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-inp'] = { +if not modules then modules = { } end modules ['data-res'] = { version = 1.001, comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", @@ -8196,981 +11083,510 @@ if not modules then modules = { } end modules ['data-inp'] = { license = "see context related readme files", } --- After a few years using the code the large luat-inp.lua file --- has been split up a bit. In the process some functionality was --- dropped: --- --- * support for reading lsr files --- * selective scanning (subtrees) --- * some public auxiliary functions were made private --- --- TODO: os.getenv -> os.env[] --- TODO: instances.[hashes,cnffiles,configurations,522] --- TODO: check escaping in find etc, too much, too slow - --- This lib is multi-purpose and can be loaded again later on so that --- additional functionality becomes available. We will split thislogs.report("fileio", --- module in components once we're done with prototyping. This is the --- first code I wrote for LuaTeX, so it needs some cleanup. Before changing --- something in this module one can best check with Taco or Hans first; there --- is some nasty trickery going on that relates to traditional kpse support. - --- To be considered: hash key lowercase, first entry in table filename --- (any case), rest paths (so no need for optimization). Or maybe a --- separate table that matches lowercase names to mixed case when --- present. In that case the lower() cases can go away. I will do that --- only when we run into problems with names ... well ... Iwona-Regular. +-- In practice we will work within one tds tree, but i want to keep +-- the option open to build tools that look at multiple trees, which is +-- why we keep the tree specific data in a table. We used to pass the +-- instance but for practical purposes we now avoid this and use a +-- instance variable. We always have one instance active (sort of global). --- Beware, loading and saving is overloaded in luat-tmp! +-- todo: cache:/// home:/// local format, gsub, find, lower, upper, match, gmatch = string.format, string.gsub, string.find, string.lower, string.upper, string.match, string.gmatch local concat, insert, sortedkeys = table.concat, table.insert, table.sortedkeys -local next, type = next, type -local lpegmatch = lpeg.match +local next, type, rawget = next, type, rawget +local os = os -local trace_locating, trace_detail, trace_expansions = false, false, false +local P, S, R, C, Cc, Cs, Ct, Carg = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.Cc, lpeg.Cs, lpeg.Ct, lpeg.Carg +local lpegmatch, lpegpatterns = lpeg.match, lpeg.patterns -trackers.register("resolvers.locating", function(v) trace_locating = v end) -trackers.register("resolvers.details", function(v) trace_detail = v end) -trackers.register("resolvers.expansions", function(v) trace_expansions = v end) -- todo +local filedirname = file.dirname +local filebasename = file.basename +local fileextname = file.extname +local filejoin = file.join +local collapsepath = file.collapsepath +local joinpath = file.joinpath +local allocate = utilities.storage.allocate +local setmetatableindex = table.setmetatableindex -if not resolvers then - resolvers = { - suffixes = { }, - formats = { }, - dangerous = { }, - suffixmap = { }, - alternatives = { }, - locators = { }, -- locate databases - hashers = { }, -- load databases - generators = { }, -- generate databases - } -end +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) +local trace_detail = false trackers.register("resolvers.details", function(v) trace_detail = v end) +local trace_expansions = false trackers.register("resolvers.expansions", function(v) trace_expansions = v end) -local resolvers = resolvers +local report_resolving = logs.reporter("resolvers","resolving") -resolvers.locators .notfound = { nil } -resolvers.hashers .notfound = { nil } -resolvers.generators.notfound = { nil } - -resolvers.cacheversion = '1.0.1' -resolvers.cnfname = 'texmf.cnf' -resolvers.luaname = 'texmfcnf.lua' -resolvers.homedir = os.env[os.type == "windows" and 'USERPROFILE'] or os.env['HOME'] or '~' -resolvers.cnfdefault = '{$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c}' - -local dummy_path_expr = "^!*unset/*$" - -local formats = resolvers.formats -local suffixes = resolvers.suffixes -local dangerous = resolvers.dangerous -local suffixmap = resolvers.suffixmap -local alternatives = resolvers.alternatives - -formats['afm'] = 'AFMFONTS' suffixes['afm'] = { 'afm' } -formats['enc'] = 'ENCFONTS' suffixes['enc'] = { 'enc' } -formats['fmt'] = 'TEXFORMATS' suffixes['fmt'] = { 'fmt' } -formats['map'] = 'TEXFONTMAPS' suffixes['map'] = { 'map' } -formats['mp'] = 'MPINPUTS' suffixes['mp'] = { 'mp' } -formats['ocp'] = 'OCPINPUTS' suffixes['ocp'] = { 'ocp' } -formats['ofm'] = 'OFMFONTS' suffixes['ofm'] = { 'ofm', 'tfm' } -formats['otf'] = 'OPENTYPEFONTS' suffixes['otf'] = { 'otf' } -- 'ttf' -formats['opl'] = 'OPLFONTS' suffixes['opl'] = { 'opl' } -formats['otp'] = 'OTPINPUTS' suffixes['otp'] = { 'otp' } -formats['ovf'] = 'OVFFONTS' suffixes['ovf'] = { 'ovf', 'vf' } -formats['ovp'] = 'OVPFONTS' suffixes['ovp'] = { 'ovp' } -formats['tex'] = 'TEXINPUTS' suffixes['tex'] = { 'tex' } -formats['tfm'] = 'TFMFONTS' suffixes['tfm'] = { 'tfm' } -formats['ttf'] = 'TTFONTS' suffixes['ttf'] = { 'ttf', 'ttc', 'dfont' } -formats['pfb'] = 'T1FONTS' suffixes['pfb'] = { 'pfb', 'pfa' } -formats['vf'] = 'VFFONTS' suffixes['vf'] = { 'vf' } - -formats['fea'] = 'FONTFEATURES' suffixes['fea'] = { 'fea' } -formats['cid'] = 'FONTCIDMAPS' suffixes['cid'] = { 'cid', 'cidmap' } - -formats ['texmfscripts'] = 'TEXMFSCRIPTS' -- new -suffixes['texmfscripts'] = { 'rb', 'pl', 'py' } -- 'lua' - -formats ['lua'] = 'LUAINPUTS' -- new -suffixes['lua'] = { 'lua', 'luc', 'tma', 'tmc' } - --- backward compatible ones - -alternatives['map files'] = 'map' -alternatives['enc files'] = 'enc' -alternatives['cid maps'] = 'cid' -- great, why no cid files -alternatives['font feature files'] = 'fea' -- and fea files here -alternatives['opentype fonts'] = 'otf' -alternatives['truetype fonts'] = 'ttf' -alternatives['truetype collections'] = 'ttc' -alternatives['truetype dictionary'] = 'dfont' -alternatives['type1 fonts'] = 'pfb' - --- obscure ones - -formats ['misc fonts'] = '' -suffixes['misc fonts'] = { } - -formats ['sfd'] = 'SFDFONTS' -suffixes ['sfd'] = { 'sfd' } -alternatives['subfont definition files'] = 'sfd' - --- lib paths - -formats ['lib'] = 'CLUAINPUTS' -- new (needs checking) -suffixes['lib'] = (os.libsuffix and { os.libsuffix }) or { 'dll', 'so' } +local resolvers = resolvers --- In practice we will work within one tds tree, but i want to keep --- the option open to build tools that look at multiple trees, which is --- why we keep the tree specific data in a table. We used to pass the --- instance but for practical pusposes we now avoid this and use a --- instance variable. +local expandedpathfromlist = resolvers.expandedpathfromlist +local checkedvariable = resolvers.checkedvariable +local splitconfigurationpath = resolvers.splitconfigurationpath +local methodhandler = resolvers.methodhandler --- here we catch a few new thingies (todo: add these paths to context.tmf) --- --- FONTFEATURES = .;$TEXMF/fonts/fea// --- FONTCIDMAPS = .;$TEXMF/fonts/cid// +local initializesetter = utilities.setters.initialize --- we always have one instance active +local ostype, osname, osenv, ossetenv, osgetenv = os.type, os.name, os.env, os.setenv, os.getenv -resolvers.instance = resolvers.instance or nil -- the current one (slow access) -local instance = resolvers.instance or nil -- the current one (fast access) +resolvers.cacheversion = '1.0.1' +resolvers.configbanner = '' +resolvers.homedir = environment.homedir +resolvers.criticalvars = allocate { "SELFAUTOLOC", "SELFAUTODIR", "SELFAUTOPARENT", "TEXMFCNF", "TEXMF", "TEXOS" } +resolvers.luacnfname = 'texmfcnf.lua' +resolvers.luacnfstate = "unknown" -function resolvers.newinstance() +-- resolvers.luacnfspec = '{$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,}/web2c}' -- what a rubish path +resolvers.luacnfspec = 'selfautoparent:{/texmf{-local,}{,/web2c},}}' - -- store once, freeze and faster (once reset we can best use - -- instance.environment) maybe better have a register suffix - -- function - for k, v in next, suffixes do - for i=1,#v do - local vi = v[i] - if vi then - suffixmap[vi] = k - end - end - end - -- because vf searching is somewhat dangerous, we want to prevent - -- too liberal searching esp because we do a lookup on the current - -- path anyway; only tex (or any) is safe +local unset_variable = "unset" - for k, v in next, formats do - dangerous[k] = true - end - dangerous.tex = nil +local formats = resolvers.formats +local suffixes = resolvers.suffixes +local dangerous = resolvers.dangerous +local suffixmap = resolvers.suffixmap - -- the instance +resolvers.defaultsuffixes = { "tex" } -- "mkiv", "cld" -- too tricky - local newinstance = { - rootpath = '', - treepath = '', - progname = 'context', - engine = 'luatex', - format = '', - environment = { }, - variables = { }, - expansions = { }, - files = { }, - remap = { }, - configuration = { }, - setup = { }, - order = { }, - found = { }, - foundintrees = { }, - kpsevars = { }, - hashes = { }, - cnffiles = { }, - luafiles = { }, - lists = { }, - remember = true, - diskcache = true, - renewcache = false, - scandisk = true, - cachepath = nil, - loaderror = false, - sortdata = false, - savelists = true, - cleanuppaths = true, - allresults = false, - pattern = nil, -- lists - data = { }, -- only for loading - force_suffixes = true, - fakepaths = { }, - } +resolvers.instance = resolvers.instance or nil -- the current one (slow access) +local instance = resolvers.instance or nil -- the current one (fast access) - local ne = newinstance.environment +-- An instance has an environment (coming from the outside, kept raw), variables +-- (coming from the configuration file), and expansions (variables with nested +-- variables replaced). One can push something into the outer environment and +-- its internal copy, but only the later one will be the raw unprefixed variant. - for k,v in next, os.env do - ne[k] = resolvers.bare_variable(v) +function resolvers.setenv(key,value,raw) + if instance then + -- this one will be consulted first when we stay inside + -- the current environment + instance.environment[key] = value + -- we feed back into the environment, and as this is used + -- by other applications (via os.execute) we need to make + -- sure that prefixes are resolve + ossetenv(key,raw and value or resolvers.resolve(value)) end +end - return newinstance +-- Beware we don't want empty here as this one can be called early on +-- and therefore we use rawget. +local function getenv(key) + local value = rawget(instance.environment,key) + if value and value ~= "" then + return value + else + local e = osgetenv(key) + return e ~= nil and e ~= "" and checkedvariable(e) or "" + end end -function resolvers.setinstance(someinstance) - instance = someinstance - resolvers.instance = someinstance - return someinstance -end +resolvers.getenv = getenv +resolvers.env = getenv -function resolvers.reset() - return resolvers.setinstance(resolvers.newinstance()) -end +-- We are going to use some metatable trickery where we backtrack from +-- expansion to variable to environment. -local function reset_hashes() - instance.lists = { } - instance.found = { } +local function resolve(k) + return instance.expansions[k] end -local function check_configuration() -- not yet ok, no time for debugging now - if os.env["OSFONTDIR"] then - -- ok - elseif os.type == "windows" then - os.setenv("OSFONTDIR","c:/windows/fonts//") - elseif os.type == "macosx" then - os.setenv("OSFONTDIR","$HOME/Library/Fonts//;/Library/Fonts//;/System/Library/Fonts//") - end -end +local dollarstripper = lpeg.stripper("$") +local inhibitstripper = P("!")^0 * Cs(P(1)^0) +local backslashswapper = lpeg.replacer("\\","/") -function resolvers.bare_variable(str) -- assumes str is a string - return (gsub(str,"\s*([\"\']?)(.+)%1\s*", "%2")) -end +local somevariable = P("$") / "" +local somekey = C(R("az","AZ","09","__","--")^1) +local somethingelse = P(";") * ((1-S("!{}/\\"))^1 * P(";") / "") + + P(";") * (P(";") / "") + + P(1) +local variableexpander = Cs( (somevariable * (somekey/resolve) + somethingelse)^1 ) -function resolvers.settrace(n) -- no longer number but: 'locating' or 'detail' - if n then - trackers.disable("resolvers.*") - trackers.enable("resolvers."..n) - end -end +local cleaner = P("\\") / "/" + P(";") * S("!{}/\\")^0 * P(";")^1 / ";" +local variablecleaner = Cs((cleaner + P(1))^0) -resolvers.settrace(os.getenv("MTX_INPUT_TRACE")) +local somevariable = R("az","AZ","09","__","--")^1 / resolve +local variable = (P("$")/"") * (somevariable + (P("{")/"") * somevariable * (P("}")/"")) +local variableresolver = Cs((variable + P(1))^0) -function resolvers.osenv(key) - local ie = instance.environment - local value = ie[key] - if value == nil then - -- local e = os.getenv(key) - local e = os.env[key] - if e == nil then - -- value = "" -- false - else - value = resolvers.bare_variable(e) - end - ie[key] = value - end - return value or "" +local function expandedvariable(var) + return lpegmatch(variableexpander,var) or var end -function resolvers.env(key) - return instance.environment[key] or resolvers.osenv(key) -end +function resolvers.newinstance() -- todo: all vars will become lowercase and alphanum only --- + if trace_locating then + report_resolving("creating instance") + end -local function expand_vars(lst) -- simple vars - local variables, env = instance.variables, resolvers.env - local function resolve(a) - return variables[a] or env(a) - end - for k=1,#lst do - lst[k] = gsub(lst[k],"%$([%a%d%_%-]+)",resolve) - end -end + local environment, variables, expansions, order = allocate(), allocate(), allocate(), allocate() -local function expanded_var(var) -- simple vars - local function resolve(a) - return instance.variables[a] or resolvers.env(a) - end - return (gsub(var,"%$([%a%d%_%-]+)",resolve)) -end + local newinstance = { + environment = environment, + variables = variables, + expansions = expansions, + order = order, + files = allocate(), + setups = allocate(), + found = allocate(), + foundintrees = allocate(), + hashes = allocate(), + hashed = allocate(), + specification = allocate(), + lists = allocate(), + data = allocate(), -- only for loading + fakepaths = allocate(), + remember = true, + diskcache = true, + renewcache = false, + loaderror = false, + savelists = true, + pattern = nil, -- lists + force_suffixes = true, + } -local function entry(entries,name) - if name and (name ~= "") then - name = gsub(name,'%$','') - local result = entries[name..'.'..instance.progname] or entries[name] - if result then - return result - else - result = resolvers.env(name) - if result then - instance.variables[name] = result - resolvers.expand_variables() - return instance.expansions[name] or "" + setmetatableindex(variables,function(t,k) + local v + for i=1,#order do + v = order[i][k] + if v ~= nil then + t[k] = v + return v end end - end - return "" -end - -local function is_entry(entries,name) - if name and name ~= "" then - name = gsub(name,'%$','') - return (entries[name..'.'..instance.progname] or entries[name]) ~= nil - else - return false - end -end - --- {a,b,c,d} --- a,b,c/{p,q,r},d --- a,b,c/{p,q,r}/d/{x,y,z}// --- a,b,c/{p,q/{x,y,z},r},d/{p,q,r} --- a,b,c/{p,q/{x,y,z},r},d/{p,q,r} --- a{b,c}{d,e}f --- {a,b,c,d} --- {a,b,c/{p,q,r},d} --- {a,b,c/{p,q,r}/d/{x,y,z}//} --- {a,b,c/{p,q/{x,y,z}},d/{p,q,r}} --- {a,b,c/{p,q/{x,y,z},w}v,d/{p,q,r}} --- {$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c} + if v == nil then + v = "" + end + t[k] = v + return v + end) --- this one is better and faster, but it took me a while to realize --- that this kind of replacement is cleaner than messy parsing and --- fuzzy concatenating we can probably gain a bit with selectively --- applying lpeg, but experiments with lpeg parsing this proved not to --- work that well; the parsing is ok, but dealing with the resulting --- table is a pain because we need to work inside-out recursively + setmetatableindex(environment, function(t,k) + local v = osgetenv(k) + if v == nil then + v = variables[k] + end + if v ~= nil then + v = checkedvariable(v) or "" + end + v = resolvers.repath(v) -- for taco who has a : separated osfontdir + t[k] = v + return v + end) -local function do_first(a,b) - local t = { } - for s in gmatch(b,"[^,]+") do t[#t+1] = a .. s end - return "{" .. concat(t,",") .. "}" -end + setmetatableindex(expansions, function(t,k) + local v = environment[k] + if type(v) == "string" then + v = lpegmatch(variableresolver,v) + v = lpegmatch(variablecleaner,v) + end + t[k] = v + return v + end) -local function do_second(a,b) - local t = { } - for s in gmatch(a,"[^,]+") do t[#t+1] = s .. b end - return "{" .. concat(t,",") .. "}" -end + return newinstance -local function do_both(a,b) - local t = { } - for sa in gmatch(a,"[^,]+") do - for sb in gmatch(b,"[^,]+") do - t[#t+1] = sa .. sb - end - end - return "{" .. concat(t,",") .. "}" end -local function do_three(a,b,c) - return a .. b.. c +function resolvers.setinstance(someinstance) -- only one instance is active + instance = someinstance + resolvers.instance = someinstance + return someinstance end -local function splitpathexpr(str, t, validate) - -- no need for further optimization as it is only called a - -- few times, we can use lpeg for the sub - if trace_expansions then - logs.report("fileio","expanding variable '%s'",str) - end - t = t or { } - str = gsub(str,",}",",@}") - str = gsub(str,"{,","{@,") - -- str = "@" .. str .. "@" - local ok, done - while true do - done = false - while true do - str, ok = gsub(str,"([^{},]+){([^{}]+)}",do_first) - if ok > 0 then done = true else break end - end - while true do - str, ok = gsub(str,"{([^{}]+)}([^{},]+)",do_second) - if ok > 0 then done = true else break end - end - while true do - str, ok = gsub(str,"{([^{}]+)}{([^{}]+)}",do_both) - if ok > 0 then done = true else break end - end - str, ok = gsub(str,"({[^{}]*){([^{}]+)}([^{}]*})",do_three) - if ok > 0 then done = true end - if not done then break end - end - str = gsub(str,"[{}]", "") - str = gsub(str,"@","") - if validate then - for s in gmatch(str,"[^,]+") do - s = validate(s) - if s then t[#t+1] = s end - end - else - for s in gmatch(str,"[^,]+") do - t[#t+1] = s - end - end - if trace_expansions then - for k=1,#t do - logs.report("fileio","% 4i: %s",k,t[k]) - end - end - return t +function resolvers.reset() + return resolvers.setinstance(resolvers.newinstance()) end -local function expanded_path_from_list(pathlist) -- maybe not a list, just a path - -- a previous version fed back into pathlist - local newlist, ok = { }, false - for k=1,#pathlist do - if find(pathlist[k],"[{}]") then - ok = true - break - end - end - if ok then - local function validate(s) - s = file.collapse_path(s) - return s ~= "" and not find(s,dummy_path_expr) and s - end - for k=1,#pathlist do - splitpathexpr(pathlist[k],newlist,validate) - end - else - for k=1,#pathlist do - for p in gmatch(pathlist[k],"([^,]+)") do - p = file.collapse_path(p) - if p ~= "" then newlist[#newlist+1] = p end - end - end - end - return newlist +local function reset_hashes() + instance.lists = { } + instance.found = { } end --- we follow a rather traditional approach: --- --- (1) texmf.cnf given in TEXMFCNF --- (2) texmf.cnf searched in default variable --- --- also we now follow the stupid route: if not set then just assume *one* --- cnf file under texmf (i.e. distribution) +local slash = P("/") -local args = environment and environment.original_arguments or arg -- this needs a cleanup +local pathexpressionpattern = Cs ( + Cc("^") * ( + Cc("%") * S(".-") + + slash^2 * P(-1) / "/.*" + + slash^2 / "/.-/" + + (1-slash) * P(-1) * Cc("/") + + P(1) + )^1 * Cc("$") -- yes or no $ +) -resolvers.ownbin = resolvers.ownbin or args[-2] or arg[-2] or args[-1] or arg[-1] or arg[0] or "luatex" -resolvers.ownbin = gsub(resolvers.ownbin,"\\","/") +local cache = { } -function resolvers.getownpath() - local ownpath = resolvers.ownpath or os.selfdir - if not ownpath or ownpath == "" or ownpath == "unset" then - ownpath = args[-1] or arg[-1] - ownpath = ownpath and file.dirname(gsub(ownpath,"\\","/")) - if not ownpath or ownpath == "" then - ownpath = args[-0] or arg[-0] - ownpath = ownpath and file.dirname(gsub(ownpath,"\\","/")) - end - local binary = resolvers.ownbin - if not ownpath or ownpath == "" then - ownpath = ownpath and file.dirname(binary) - end - if not ownpath or ownpath == "" then - if os.binsuffix ~= "" then - binary = file.replacesuffix(binary,os.binsuffix) - end - for p in gmatch(os.getenv("PATH"),"[^"..io.pathseparator.."]+") do - local b = file.join(p,binary) - if lfs.isfile(b) then - -- we assume that after changing to the path the currentdir function - -- resolves to the real location and use this side effect here; this - -- trick is needed because on the mac installations use symlinks in the - -- path instead of real locations - local olddir = lfs.currentdir() - if lfs.chdir(p) then - local pp = lfs.currentdir() - if trace_locating and p ~= pp then - logs.report("fileio","following symlink '%s' to '%s'",p,pp) - end - ownpath = pp - lfs.chdir(olddir) - else - if trace_locating then - logs.report("fileio","unable to check path '%s'",p) - end - ownpath = p - end - break - end - end - end - if not ownpath or ownpath == "" then - ownpath = "." - logs.report("fileio","forcing fallback ownpath .") - elseif trace_locating then - logs.report("fileio","using ownpath '%s'",ownpath) +local function makepathexpression(str) + if str == "." then + return "^%./$" + else + local c = cache[str] + if not c then + c = lpegmatch(pathexpressionpattern,str) + cache[str] = c end + return c end - resolvers.ownpath = ownpath - function resolvers.getownpath() - return resolvers.ownpath - end - return ownpath end -local own_places = { "SELFAUTOLOC", "SELFAUTODIR", "SELFAUTOPARENT", "TEXMFCNF" } - -local function identify_own() - local ownpath = resolvers.getownpath() or dir.current() - local ie = instance.environment - if ownpath then - if resolvers.env('SELFAUTOLOC') == "" then os.env['SELFAUTOLOC'] = file.collapse_path(ownpath) end - if resolvers.env('SELFAUTODIR') == "" then os.env['SELFAUTODIR'] = file.collapse_path(ownpath .. "/..") end - if resolvers.env('SELFAUTOPARENT') == "" then os.env['SELFAUTOPARENT'] = file.collapse_path(ownpath .. "/../..") end - else - logs.report("fileio","error: unable to locate ownpath") - os.exit() - end - if resolvers.env('TEXMFCNF') == "" then os.env['TEXMFCNF'] = resolvers.cnfdefault end - if resolvers.env('TEXOS') == "" then os.env['TEXOS'] = resolvers.env('SELFAUTODIR') end - if resolvers.env('TEXROOT') == "" then os.env['TEXROOT'] = resolvers.env('SELFAUTOPARENT') end +local function reportcriticalvariables() if trace_locating then - for i=1,#own_places do - local v = own_places[i] - logs.report("fileio","variable '%s' set to '%s'",v,resolvers.env(v) or "unknown") + for i=1,#resolvers.criticalvars do + local k = resolvers.criticalvars[i] + local v = resolvers.getenv(k) or "unknown" -- this one will not resolve ! + report_resolving("variable '%s' set to '%s'",k,v) end + report_resolving() end - identify_own = function() end + reportcriticalvariables = function() end end -function resolvers.identify_cnf() - if #instance.cnffiles == 0 then - -- fallback - identify_own() - -- the real search - resolvers.expand_variables() - local t = resolvers.split_path(resolvers.env('TEXMFCNF')) - t = expanded_path_from_list(t) - expand_vars(t) -- redundant - local function locate(filename,list) - for i=1,#t do - local ti = t[i] - local texmfcnf = file.collapse_path(file.join(ti,filename)) - if lfs.isfile(texmfcnf) then - list[#list+1] = texmfcnf +local function identify_configuration_files() + local specification = instance.specification + if #specification == 0 then + local cnfspec = getenv('TEXMFCNF') + if cnfspec == "" then + cnfspec = resolvers.luacnfspec + resolvers.luacnfstate = "default" + else + resolvers.luacnfstate = "environment" + end + reportcriticalvariables() + local cnfpaths = expandedpathfromlist(resolvers.splitpath(cnfspec)) + local luacnfname = resolvers.luacnfname + for i=1,#cnfpaths do + local filename = collapsepath(filejoin(cnfpaths[i],luacnfname)) + local realname = resolvers.resolve(filename) + if lfs.isfile(realname) then + specification[#specification+1] = filename + if trace_locating then + report_resolving("found configuration file '%s'",realname) end + elseif trace_locating then + report_resolving("unknown configuration file '%s'",realname) end end - locate(resolvers.luaname,instance.luafiles) - locate(resolvers.cnfname,instance.cnffiles) - end -end - -local function load_cnf_file(fname) - fname = resolvers.clean_path(fname) - local lname = file.replacesuffix(fname,'lua') - if lfs.isfile(lname) then - local dname = file.dirname(fname) -- fname ? - if not instance.configuration[dname] then - resolvers.load_data(dname,'configuration',lname and file.basename(lname)) - instance.order[#instance.order+1] = instance.configuration[dname] + if trace_locating then + report_resolving() end - else - f = io.open(fname) - if f then - if trace_locating then - logs.report("fileio","loading configuration file %s", fname) - end - local line, data, n, k, v - local dname = file.dirname(fname) - if not instance.configuration[dname] then - instance.configuration[dname] = { } - instance.order[#instance.order+1] = instance.configuration[dname] - end - local data = instance.configuration[dname] - while true do - local line, n = f:read(), 0 - if line then - while true do -- join lines - line, n = gsub(line,"\\%s*$", "") - if n > 0 then - line = line .. f:read() - else - break + elseif trace_locating then + report_resolving("configuration files already identified") + end +end + +local function load_configuration_files() + local specification = instance.specification + if #specification > 0 then + local luacnfname = resolvers.luacnfname + for i=1,#specification do + local filename = specification[i] + local pathname = filedirname(filename) + local filename = filejoin(pathname,luacnfname) + local realname = resolvers.resolve(filename) -- no shortcut + local blob = loadfile(realname) + if blob then + local setups = instance.setups + local data = blob() + data = data and data.content + if data then + if trace_locating then + report_resolving("loading configuration file '%s'",filename) + report_resolving() + end + local variables = data.variables or { } + local warning = false + for k, v in next, data do + local variant = type(v) + if variant == "table" then + initializesetter(filename,k,v) + elseif variables[k] == nil then + if trace_locating and not warning then + report_resolving("variables like '%s' in configuration file '%s' should move to the 'variables' subtable", + k,resolvers.resolve(filename)) + warning = true + end + variables[k] = v end end - if not find(line,"^[%%#]") then - local l = gsub(line,"%s*%%.*$","") - local k, v = match(l,"%s*(.-)%s*=%s*(.-)%s*$") - if k and v and not data[k] then - v = gsub(v,"[%%#].*",'') - data[k] = gsub(v,"~","$HOME") - instance.kpsevars[k] = true + setups[pathname] = variables + if resolvers.luacnfstate == "default" then + -- the following code is not tested + local cnfspec = variables["TEXMFCNF"] + if cnfspec then + if trace_locating then + report_resolving("reloading configuration due to TEXMF redefinition") + end + -- we push the value into the main environment (osenv) so + -- that it takes precedence over the default one and therefore + -- also over following definitions + resolvers.setenv('TEXMFCNF',cnfspec) -- resolves prefixes + -- we now identify and load the specified configuration files + instance.specification = { } + identify_configuration_files() + load_configuration_files() + -- we prevent further overload of the configuration variable + resolvers.luacnfstate = "configuration" + -- we quit the outer loop + break end end - else - break - end - end - f:close() - elseif trace_locating then - logs.report("fileio","skipping configuration file '%s'", fname) - end - end -end -local function collapse_cnf_data() -- potential optimization: pass start index (setup and configuration are shared) - local order = instance.order - for i=1,#order do - local c = order[i] - for k,v in next, c do - if not instance.variables[k] then - if instance.environment[k] then - instance.variables[k] = instance.environment[k] else - instance.kpsevars[k] = true - instance.variables[k] = resolvers.bare_variable(v) + if trace_locating then + report_resolving("skipping configuration file '%s' (no content)",filename) + end + setups[pathname] = { } + instance.loaderror = true end + elseif trace_locating then + report_resolving("skipping configuration file '%s' (no file)",filename) end - end - end -end - -function resolvers.load_cnf() - local function loadoldconfigdata() - local cnffiles = instance.cnffiles - for i=1,#cnffiles do - load_cnf_file(cnffiles[i]) - end - end - -- instance.cnffiles contain complete names now ! - -- we still use a funny mix of cnf and new but soon - -- we will switch to lua exclusively as we only use - -- the file to collect the tree roots - if #instance.cnffiles == 0 then - if trace_locating then - logs.report("fileio","no cnf files found (TEXMFCNF may not be set/known)") - end - else - local cnffiles = instance.cnffiles - instance.rootpath = cnffiles[1] - for k=1,#cnffiles do - instance.cnffiles[k] = file.collapse_path(cnffiles[k]) - end - for i=1,3 do - instance.rootpath = file.dirname(instance.rootpath) - end - instance.rootpath = file.collapse_path(instance.rootpath) - if instance.diskcache and not instance.renewcache then - resolvers.loadoldconfig(instance.cnffiles) + instance.order[#instance.order+1] = instance.setups[pathname] if instance.loaderror then - loadoldconfigdata() - resolvers.saveoldconfig() - end - else - loadoldconfigdata() - if instance.renewcache then - resolvers.saveoldconfig() + break end end - collapse_cnf_data() - end - check_configuration() -end - -function resolvers.load_lua() - if #instance.luafiles == 0 then - -- yet harmless - else - instance.rootpath = instance.luafiles[1] - local luafiles = instance.luafiles - for k=1,#luafiles do - instance.luafiles[k] = file.collapse_path(luafiles[k]) - end - for i=1,3 do - instance.rootpath = file.dirname(instance.rootpath) - end - instance.rootpath = file.collapse_path(instance.rootpath) - resolvers.loadnewconfig() - collapse_cnf_data() - end - check_configuration() -end - --- database loading - -function resolvers.load_hash() - resolvers.locatelists() - if instance.diskcache and not instance.renewcache then - resolvers.loadfiles() - if instance.loaderror then - resolvers.loadlists() - resolvers.savefiles() - end - else - resolvers.loadlists() - if instance.renewcache then - resolvers.savefiles() - end - end -end - -function resolvers.append_hash(type,tag,name) - if trace_locating then - logs.report("fileio","hash '%s' appended",tag) - end - insert(instance.hashes, { ['type']=type, ['tag']=tag, ['name']=name } ) -end - -function resolvers.prepend_hash(type,tag,name) - if trace_locating then - logs.report("fileio","hash '%s' prepended",tag) - end - insert(instance.hashes, 1, { ['type']=type, ['tag']=tag, ['name']=name } ) -end - -function resolvers.extend_texmf_var(specification) -- crap, we could better prepend the hash --- local t = resolvers.expanded_path_list('TEXMF') -- full expansion - local t = resolvers.split_path(resolvers.env('TEXMF')) - insert(t,1,specification) - local newspec = concat(t,";") - if instance.environment["TEXMF"] then - instance.environment["TEXMF"] = newspec - elseif instance.variables["TEXMF"] then - instance.variables["TEXMF"] = newspec - else - -- weird - end - resolvers.expand_variables() - reset_hashes() -end - --- locators - -function resolvers.locatelists() - local texmfpaths = resolvers.clean_path_list('TEXMF') - for i=1,#texmfpaths do - local path = texmfpaths[i] - if trace_locating then - logs.report("fileio","locating list of '%s'",path) - end - resolvers.locatedatabase(file.collapse_path(path)) - end -end - -function resolvers.locatedatabase(specification) - return resolvers.methodhandler('locators', specification) -end - -function resolvers.locators.tex(specification) - if specification and specification ~= '' and lfs.isdir(specification) then - if trace_locating then - logs.report("fileio","tex locator '%s' found",specification) - end - resolvers.append_hash('file',specification,filename) elseif trace_locating then - logs.report("fileio","tex locator '%s' not found",specification) + report_resolving("warning: no lua configuration files found") end end --- hashers - -function resolvers.hashdatabase(tag,name) - return resolvers.methodhandler('hashers',tag,name) -end +-- scheme magic ... database loading -function resolvers.loadfiles() - instance.loaderror = false - instance.files = { } +local function load_file_databases() + instance.loaderror, instance.files = false, allocate() if not instance.renewcache then local hashes = instance.hashes for k=1,#hashes do local hash = hashes[k] - resolvers.hashdatabase(hash.tag,hash.name) + resolvers.hashers.byscheme(hash.type,hash.name) if instance.loaderror then break end end end end -function resolvers.hashers.tex(tag,name) - resolvers.load_data(tag,'files') -end - --- generators: - -function resolvers.loadlists() - local hashes = instance.hashes - for i=1,#hashes do - resolvers.generatedatabase(hashes[i].tag) - end -end - -function resolvers.generatedatabase(specification) - return resolvers.methodhandler('generators', specification) -end - --- starting with . or .. etc or funny char - -local weird = lpeg.P(".")^1 + lpeg.anywhere(lpeg.S("~`!#$%^&*()={}[]:;\"\'||<>,?\n\r\t")) - ---~ local l_forbidden = lpeg.S("~`!#$%^&*()={}[]:;\"\'||\\/<>,?\n\r\t") ---~ local l_confusing = lpeg.P(" ") ---~ local l_character = lpeg.patterns.utf8 ---~ local l_dangerous = lpeg.P(".") - ---~ local l_normal = (l_character - l_forbidden - l_confusing - l_dangerous) * (l_character - l_forbidden - l_confusing^2)^0 * lpeg.P(-1) ---~ ----- l_normal = l_normal * lpeg.Cc(true) + lpeg.Cc(false) - ---~ local function test(str) ---~ print(str,lpeg.match(l_normal,str)) ---~ end ---~ test("ヒラギノ明朝 Pro W3") ---~ test("..ヒラギノ明朝 Pro W3") ---~ test(":ヒラギノ明朝 Pro W3;") ---~ test("ヒラギノ明朝 /Pro W3;") ---~ test("ヒラギノ明朝 Pro W3") - -function resolvers.generators.tex(specification) - local tag = specification - if trace_locating then - logs.report("fileio","scanning path '%s'",specification) - end - instance.files[tag] = { } - local files = instance.files[tag] - local n, m, r = 0, 0, 0 - local spec = specification .. '/' - local attributes = lfs.attributes - local directory = lfs.dir - local function action(path) - local full - if path then - full = spec .. path .. '/' - else - full = spec - end - for name in directory(full) do - if not lpegmatch(weird,name) then - -- if lpegmatch(l_normal,name) then - local mode = attributes(full..name,'mode') - if mode == 'file' then - if path then - n = n + 1 - local f = files[name] - if f then - if type(f) == 'string' then - files[name] = { f, path } - else - f[#f+1] = path - end - else -- probably unique anyway - files[name] = path - local lower = lower(name) - if name ~= lower then - files["remap:"..lower] = name - r = r + 1 - end - end - end - elseif mode == 'directory' then - m = m + 1 - if path then - action(path..'/'..name) +local function locate_file_databases() + -- todo: cache:// and tree:// (runtime) + local texmfpaths = resolvers.expandedpathlist('TEXMF') + if #texmfpaths > 0 then + for i=1,#texmfpaths do + local path = collapsepath(texmfpaths[i]) + local stripped = lpegmatch(inhibitstripper,path) -- the !! thing + if stripped ~= "" then + local runtime = stripped == path + path = resolvers.cleanpath(path) + local spec = resolvers.splitmethod(stripped) + if spec.scheme == "cache" or spec.scheme == "file" then + stripped = spec.path + elseif runtime and (spec.noscheme or spec.scheme == "file") then + stripped = "tree:///" .. stripped + end + if trace_locating then + if runtime then + report_resolving("locating list of '%s' (runtime)",path) else - action(name) + report_resolving("locating list of '%s' (cached)",path) end end + methodhandler('locators',stripped) end end - end - action() - if trace_locating then - logs.report("fileio","%s files found on %s directories with %s uppercase remappings",n,m,r) + if trace_locating then + report_resolving() + end + elseif trace_locating then + report_resolving("no texmf paths are defined (using TEXMF)") end end --- savers, todo - -function resolvers.savefiles() - resolvers.save_data('files') +local function generate_file_databases() + local hashes = instance.hashes + for k=1,#hashes do + local hash = hashes[k] + methodhandler('generators',hash.name) + end + if trace_locating then + report_resolving() + end end --- A config (optionally) has the paths split in tables. Internally --- we join them and split them after the expansion has taken place. This --- is more convenient. - ---~ local checkedsplit = string.checkedsplit - -local cache = { } - -local splitter = lpeg.Ct(lpeg.splitat(lpeg.S(os.type == "windows" and ";" or ":;"))) - -local function split_kpse_path(str) -- beware, this can be either a path or a {specification} - local found = cache[str] - if not found then - if str == "" then - found = { } - else - str = gsub(str,"\\","/") ---~ local split = (find(str,";") and checkedsplit(str,";")) or checkedsplit(str,io.pathseparator) -local split = lpegmatch(splitter,str) - found = { } - for i=1,#split do - local s = split[i] - if not find(s,"^{*unset}*") then - found[#found+1] = s - end - end - if trace_expansions then - logs.report("fileio","splitting path specification '%s'",str) - for k=1,#found do - logs.report("fileio","% 4i: %s",k,found[k]) - end +local function save_file_databases() -- will become cachers + for i=1,#instance.hashes do + local hash = instance.hashes[i] + local cachename = hash.name + if hash.cache then + local content = instance.files[cachename] + caches.collapsecontent(content) + if trace_locating then + report_resolving("saving tree '%s'",cachename) end - cache[str] = found + caches.savecontent(cachename,"files",content) + elseif trace_locating then + report_resolving("not saving runtime tree '%s'",cachename) end end - return found end -resolvers.split_kpse_path = split_kpse_path - -function resolvers.splitconfig() - for i=1,#instance do - local c = instance[i] - for k,v in next, c do - if type(v) == 'string' then - local t = split_kpse_path(v) - if #t > 1 then - c[k] = t - end - end +local function load_databases() + locate_file_databases() + if instance.diskcache and not instance.renewcache then + load_file_databases() + if instance.loaderror then + generate_file_databases() + save_file_databases() + end + else + generate_file_databases() + if instance.renewcache then + save_file_databases() end end end -function resolvers.joinconfig() - local order = instance.order - for i=1,#order do - local c = order[i] - for k,v in next, c do -- indexed? - if type(v) == 'table' then - c[k] = file.join_path(v) - end +function resolvers.appendhash(type,name,cache) + -- safeguard ... tricky as it's actually a bug when seen twice + if not instance.hashed[name] then + if trace_locating then + report_resolving("hash '%s' appended",name) end + insert(instance.hashes, { type = type, name = name, cache = cache } ) + instance.hashed[name] = cache end end -function resolvers.split_path(str) - if type(str) == 'table' then - return str - else - return split_kpse_path(str) +function resolvers.prependhash(type,name,cache) + -- safeguard ... tricky as it's actually a bug when seen twice + if not instance.hashed[name] then + if trace_locating then + report_resolving("hash '%s' prepended",name) + end + insert(instance.hashes, 1, { type = type, name = name, cache = cache } ) + instance.hashed[name] = cache end end -function resolvers.join_path(str) - if type(str) == 'table' then - return file.join_path(str) +function resolvers.extendtexmfvariable(specification) -- crap, we could better prepend the hash + local t = resolvers.splitpath(getenv('TEXMF')) + insert(t,1,specification) + local newspec = concat(t,";") + if instance.environment["TEXMF"] then + instance.environment["TEXMF"] = newspec + elseif instance.variables["TEXMF"] then + instance.variables["TEXMF"] = newspec else - return str + -- weird end + reset_hashes() end function resolvers.splitexpansions() local ie = instance.expansions for k,v in next, ie do - local t, h, p = { }, { }, split_kpse_path(v) + local t, tn, h, p = { }, 0, { }, splitconfigurationpath(v) for kk=1,#p do local vv = p[kk] if vv ~= "" and not h[vv] then - t[#t+1] = vv + tn = tn + 1 + t[tn] = vv h[vv] = true end end @@ -9184,343 +11600,92 @@ end -- end of split/join code -function resolvers.saveoldconfig() - resolvers.splitconfig() - resolvers.save_data('configuration') - resolvers.joinconfig() -end - -resolvers.configbanner = [[ --- This is a Luatex configuration file created by 'luatools.lua' or --- 'luatex.exe' directly. For comment, suggestions and questions you can --- contact the ConTeXt Development Team. This configuration file is --- not copyrighted. [HH & TH] -]] - -function resolvers.serialize(files) - -- This version is somewhat optimized for the kind of - -- tables that we deal with, so it's much faster than - -- the generic serializer. This makes sense because - -- luatools and mtxtools are called frequently. Okay, - -- we pay a small price for properly tabbed tables. - local t = { } - local function dump(k,v,m) -- could be moved inline - if type(v) == 'string' then - return m .. "['" .. k .. "']='" .. v .. "'," - elseif #v == 1 then - return m .. "['" .. k .. "']='" .. v[1] .. "'," - else - return m .. "['" .. k .. "']={'" .. concat(v,"','").. "'}," - end - end - t[#t+1] = "return {" - if instance.sortdata then - local sortedfiles = sortedkeys(files) - for i=1,#sortedfiles do - local k = sortedfiles[i] - local fk = files[k] - if type(fk) == 'table' then - t[#t+1] = "\t['" .. k .. "']={" - local sortedfk = sortedkeys(fk) - for j=1,#sortedfk do - local kk = sortedfk[j] - t[#t+1] = dump(kk,fk[kk],"\t\t") - end - t[#t+1] = "\t}," - else - t[#t+1] = dump(k,fk,"\t") - end - end - else - for k, v in next, files do - if type(v) == 'table' then - t[#t+1] = "\t['" .. k .. "']={" - for kk,vv in next, v do - t[#t+1] = dump(kk,vv,"\t\t") - end - t[#t+1] = "\t}," - else - t[#t+1] = dump(k,v,"\t") - end - end - end - t[#t+1] = "}" - return concat(t,"\n") -end - -local data_state = { } +-- we used to have 'files' and 'configurations' so therefore the following +-- shared function -function resolvers.data_state() - return data_state or { } -end - -function resolvers.save_data(dataname, makename) -- untested without cache overload - for cachename, files in next, instance[dataname] do - local name = (makename or file.join)(cachename,dataname) - local luaname, lucname = name .. ".lua", name .. ".luc" - if trace_locating then - logs.report("fileio","preparing '%s' for '%s'",dataname,cachename) - end - for k, v in next, files do - if type(v) == "table" and #v == 1 then - files[k] = v[1] - end - end - local data = { - type = dataname, - root = cachename, - version = resolvers.cacheversion, - date = os.date("%Y-%m-%d"), - time = os.date("%H:%M:%S"), - content = files, - uuid = os.uuid(), - } - local ok = io.savedata(luaname,resolvers.serialize(data)) - if ok then - if trace_locating then - logs.report("fileio","'%s' saved in '%s'",dataname,luaname) - end - if utils.lua.compile(luaname,lucname,false,true) then -- no cleanup but strip - if trace_locating then - logs.report("fileio","'%s' compiled to '%s'",dataname,lucname) - end - else - if trace_locating then - logs.report("fileio","compiling failed for '%s', deleting file '%s'",dataname,lucname) - end - os.remove(lucname) - end - elseif trace_locating then - logs.report("fileio","unable to save '%s' in '%s' (access error)",dataname,luaname) - end - end -end - -function resolvers.load_data(pathname,dataname,filename,makename) -- untested without cache overload - filename = ((not filename or (filename == "")) and dataname) or filename - filename = (makename and makename(dataname,filename)) or file.join(pathname,filename) - local blob = loadfile(filename .. ".luc") or loadfile(filename .. ".lua") - if blob then - local data = blob() - if data and data.content and data.type == dataname and data.version == resolvers.cacheversion then - data_state[#data_state+1] = data.uuid - if trace_locating then - logs.report("fileio","loading '%s' for '%s' from '%s'",dataname,pathname,filename) - end - instance[dataname][pathname] = data.content - else - if trace_locating then - logs.report("fileio","skipping '%s' for '%s' from '%s'",dataname,pathname,filename) - end - instance[dataname][pathname] = { } - instance.loaderror = true - end - elseif trace_locating then - logs.report("fileio","skipping '%s' for '%s' from '%s'",dataname,pathname,filename) - end -end - --- some day i'll use the nested approach, but not yet (actually we even drop --- engine/progname support since we have only luatex now) --- --- first texmfcnf.lua files are located, next the cached texmf.cnf files --- --- return { --- TEXMFBOGUS = 'effe checken of dit werkt', --- } - -function resolvers.resetconfig() - identify_own() - instance.configuration, instance.setup, instance.order, instance.loaderror = { }, { }, { }, false -end - -function resolvers.loadnewconfig() - local luafiles = instance.luafiles - for i=1,#luafiles do - local cnf = luafiles[i] - local pathname = file.dirname(cnf) - local filename = file.join(pathname,resolvers.luaname) - local blob = loadfile(filename) - if blob then - local data = blob() - if data then - if trace_locating then - logs.report("fileio","loading configuration file '%s'",filename) - end - if true then - -- flatten to variable.progname - local t = { } - for k, v in next, data do -- v = progname - if type(v) == "string" then - t[k] = v - else - for kk, vv in next, v do -- vv = variable - if type(vv) == "string" then - t[vv.."."..v] = kk - end - end - end - end - instance['setup'][pathname] = t - else - instance['setup'][pathname] = data - end - else - if trace_locating then - logs.report("fileio","skipping configuration file '%s'",filename) - end - instance['setup'][pathname] = { } - instance.loaderror = true - end - elseif trace_locating then - logs.report("fileio","skipping configuration file '%s'",filename) - end - instance.order[#instance.order+1] = instance.setup[pathname] - if instance.loaderror then break end - end -end - -function resolvers.loadoldconfig() - if not instance.renewcache then - local cnffiles = instance.cnffiles - for i=1,#cnffiles do - local cnf = cnffiles[i] - local dname = file.dirname(cnf) - resolvers.load_data(dname,'configuration') - instance.order[#instance.order+1] = instance.configuration[dname] - if instance.loaderror then break end - end - end - resolvers.joinconfig() -end - -function resolvers.expand_variables() - local expansions, environment, variables = { }, instance.environment, instance.variables - local env = resolvers.env - instance.expansions = expansions - local engine, progname = instance.engine, instance.progname - if type(engine) ~= "string" then instance.engine, engine = "", "" end - if type(progname) ~= "string" then instance.progname, progname = "", "" end - if engine ~= "" then environment['engine'] = engine end - if progname ~= "" then environment['progname'] = progname end - for k,v in next, environment do - local a, b = match(k,"^(%a+)%_(.*)%s*$") - if a and b then - expansions[a..'.'..b] = v - else - expansions[k] = v - end - end - for k,v in next, environment do -- move environment to expansions - if not expansions[k] then expansions[k] = v end - end - for k,v in next, variables do -- move variables to expansions - if not expansions[k] then expansions[k] = v end - end - local busy = false - local function resolve(a) - busy = true - return expansions[a] or env(a) - end - while true do - busy = false - for k,v in next, expansions do - local s, n = gsub(v,"%$([%a%d%_%-]+)",resolve) - local s, m = gsub(s,"%$%{([%a%d%_%-]+)%}",resolve) - if n > 0 or m > 0 then - expansions[k]= s - end - end - if not busy then break end - end - for k,v in next, expansions do - expansions[k] = gsub(v,"\\", '/') - end +function resolvers.datastate() + return caches.contentstate() end function resolvers.variable(name) - return entry(instance.variables,name) + local name = name and lpegmatch(dollarstripper,name) + local result = name and instance.variables[name] + return result ~= nil and result or "" end function resolvers.expansion(name) - return entry(instance.expansions,name) -end - -function resolvers.is_variable(name) - return is_entry(instance.variables,name) -end - -function resolvers.is_expansion(name) - return is_entry(instance.expansions,name) + local name = name and lpegmatch(dollarstripper,name) + local result = name and instance.expansions[name] + return result ~= nil and result or "" end -function resolvers.unexpanded_path_list(str) +function resolvers.unexpandedpathlist(str) local pth = resolvers.variable(str) - local lst = resolvers.split_path(pth) - return expanded_path_from_list(lst) + local lst = resolvers.splitpath(pth) + return expandedpathfromlist(lst) end -function resolvers.unexpanded_path(str) - return file.join_path(resolvers.unexpanded_path_list(str)) +function resolvers.unexpandedpath(str) + return joinpath(resolvers.unexpandedpathlist(str)) end -do -- no longer needed - - local done = { } +local done = { } - function resolvers.reset_extra_path() - local ep = instance.extra_paths - if not ep then - ep, done = { }, { } - instance.extra_paths = ep - elseif #ep > 0 then - instance.lists, done = { }, { } - end +function resolvers.resetextrapath() + local ep = instance.extra_paths + if not ep then + ep, done = { }, { } + instance.extra_paths = ep + elseif #ep > 0 then + instance.lists, done = { }, { } end +end - function resolvers.register_extra_path(paths,subpaths) - local ep = instance.extra_paths or { } - local n = #ep - if paths and paths ~= "" then - if subpaths and subpaths ~= "" then - for p in gmatch(paths,"[^,]+") do - -- we gmatch each step again, not that fast, but used seldom - for s in gmatch(subpaths,"[^,]+") do - local ps = p .. "/" .. s - if not done[ps] then - ep[#ep+1] = resolvers.clean_path(ps) - done[ps] = true - end - end - end - else - for p in gmatch(paths,"[^,]+") do - if not done[p] then - ep[#ep+1] = resolvers.clean_path(p) - done[p] = true - end - end - end - elseif subpaths and subpaths ~= "" then - for i=1,n do +function resolvers.registerextrapath(paths,subpaths) + local ep = instance.extra_paths or { } + local oldn = #ep + local newn = oldn + if paths and paths ~= "" then + if subpaths and subpaths ~= "" then + for p in gmatch(paths,"[^,]+") do -- we gmatch each step again, not that fast, but used seldom for s in gmatch(subpaths,"[^,]+") do - local ps = ep[i] .. "/" .. s + local ps = p .. "/" .. s if not done[ps] then - ep[#ep+1] = resolvers.clean_path(ps) + newn = newn + 1 + ep[newn] = resolvers.cleanpath(ps) done[ps] = true end end end + else + for p in gmatch(paths,"[^,]+") do + if not done[p] then + newn = newn + 1 + ep[newn] = resolvers.cleanpath(p) + done[p] = true + end + end end - if #ep > 0 then - instance.extra_paths = ep -- register paths - end - if #ep > n then - instance.lists = { } -- erase the cache + elseif subpaths and subpaths ~= "" then + for i=1,oldn do + -- we gmatch each step again, not that fast, but used seldom + for s in gmatch(subpaths,"[^,]+") do + local ps = ep[i] .. "/" .. s + if not done[ps] then + newn = newn + 1 + ep[newn] = resolvers.cleanpath(ps) + done[ps] = true + end + end end end - + if newn > 0 then + instance.extra_paths = ep -- register paths + end + if newn > oldn then + instance.lists = { } -- erase the cache + end end local function made_list(instance,list) @@ -9528,14 +11693,15 @@ local function made_list(instance,list) if not ep or #ep == 0 then return list else - local done, new = { }, { } + local done, new, newn = { }, { }, 0 -- honour . .. ../.. but only when at the start for k=1,#list do local v = list[k] if not done[v] then if find(v,"^[%.%/]$") then done[v] = true - new[#new+1] = v + newn = newn + 1 + new[newn] = v else break end @@ -9546,7 +11712,8 @@ local function made_list(instance,list) local v = ep[k] if not done[v] then done[v] = true - new[#new+1] = v + newn = newn + 1 + new[newn] = v end end -- next the formal paths @@ -9554,118 +11721,95 @@ local function made_list(instance,list) local v = list[k] if not done[v] then done[v] = true - new[#new+1] = v + newn = newn + 1 + new[newn] = v end end return new end end -function resolvers.clean_path_list(str) - local t = resolvers.expanded_path_list(str) +function resolvers.cleanpathlist(str) + local t = resolvers.expandedpathlist(str) if t then for i=1,#t do - t[i] = file.collapse_path(resolvers.clean_path(t[i])) + t[i] = collapsepath(resolvers.cleanpath(t[i])) end end return t end -function resolvers.expand_path(str) - return file.join_path(resolvers.expanded_path_list(str)) +function resolvers.expandpath(str) + return joinpath(resolvers.expandedpathlist(str)) end -function resolvers.expanded_path_list(str) +function resolvers.expandedpathlist(str) if not str then - return ep or { } -- ep ? + return { } elseif instance.savelists then - -- engine+progname hash - str = gsub(str,"%$","") + str = lpegmatch(dollarstripper,str) if not instance.lists[str] then -- cached - local lst = made_list(instance,resolvers.split_path(resolvers.expansion(str))) - instance.lists[str] = expanded_path_from_list(lst) + local lst = made_list(instance,resolvers.splitpath(resolvers.expansion(str))) + instance.lists[str] = expandedpathfromlist(lst) end return instance.lists[str] else - local lst = resolvers.split_path(resolvers.expansion(str)) - return made_list(instance,expanded_path_from_list(lst)) - end -end - -function resolvers.expanded_path_list_from_var(str) -- brrr - local tmp = resolvers.var_of_format_or_suffix(gsub(str,"%$","")) - if tmp ~= "" then - return resolvers.expanded_path_list(tmp) - else - return resolvers.expanded_path_list(str) + local lst = resolvers.splitpath(resolvers.expansion(str)) + return made_list(instance,expandedpathfromlist(lst)) end end -function resolvers.expand_path_from_var(str) - return file.join_path(resolvers.expanded_path_list_from_var(str)) +function resolvers.expandedpathlistfromvariable(str) -- brrr + str = lpegmatch(dollarstripper,str) + local tmp = resolvers.variableofformatorsuffix(str) + return resolvers.expandedpathlist(tmp ~= "" and tmp or str) end -function resolvers.format_of_var(str) - return formats[str] or formats[alternatives[str]] or '' -end -function resolvers.format_of_suffix(str) - return suffixmap[file.extname(str)] or 'tex' +function resolvers.expandpathfromvariable(str) + return joinpath(resolvers.expandedpathlistfromvariable(str)) end -function resolvers.variable_of_format(str) - return formats[str] or formats[alternatives[str]] or '' +function resolvers.expandbraces(str) -- output variable and brace expansion of STRING + local ori = resolvers.variable(str) + local pth = expandedpathfromlist(resolvers.splitpath(ori)) + return joinpath(pth) end -function resolvers.var_of_format_or_suffix(str) - local v = formats[str] - if v then - return v - end - v = formats[alternatives[str]] - if v then - return v - end - v = suffixmap[file.extname(str)] - if v then - return formats[isf] +function resolvers.registerfilehash(name,content,someerror) + if content then + instance.files[name] = content + else + instance.files[name] = { } + if somerror == true then -- can be unset + instance.loaderror = someerror + end end - return '' -end - -function resolvers.expand_braces(str) -- output variable and brace expansion of STRING - local ori = resolvers.variable(str) - local pth = expanded_path_from_list(resolvers.split_path(ori)) - return file.join_path(pth) end -resolvers.isreadable = { } - -function resolvers.isreadable.file(name) - local readable = lfs.isfile(name) -- brrr +local function isreadable(name) + local readable = lfs.isfile(name) -- not file.is_readable(name) asit can be a dir if trace_detail then if readable then - logs.report("fileio","file '%s' is readable",name) + report_resolving("file '%s' is readable",name) else - logs.report("fileio","file '%s' is not readable", name) + report_resolving("file '%s' is not readable", name) end end return readable end -resolvers.isreadable.tex = resolvers.isreadable.file - -- name -- name/name local function collect_files(names) - local filelist = { } + local filelist, noffiles = { }, 0 for k=1,#names do local fname = names[k] if trace_detail then - logs.report("fileio","checking name '%s'",fname) + report_resolving("checking name '%s'",fname) end - local bname = file.basename(fname) - local dname = file.dirname(fname) + local bname = filebasename(fname) + local dname = filedirname(fname) if dname == "" or find(dname,"^%.") then dname = false else @@ -9674,11 +11818,11 @@ local function collect_files(names) local hashes = instance.hashes for h=1,#hashes do local hash = hashes[h] - local blobpath = hash.tag + local blobpath = hash.name local files = blobpath and instance.files[blobpath] if files then if trace_detail then - logs.report("fileio","deep checking '%s' (%s)",blobpath,bname) + report_resolving("deep checking '%s' (%s)",blobpath,bname) end local blobfile = files[bname] if not blobfile then @@ -9690,62 +11834,51 @@ local function collect_files(names) end end if blobfile then + local blobroot = files.__path__ or blobpath if type(blobfile) == 'string' then if not dname or find(blobfile,dname) then - filelist[#filelist+1] = { - hash.type, - file.join(blobpath,blobfile,bname), -- search - resolvers.concatinators[hash.type](blobpath,blobfile,bname) -- result - } + local variant = hash.type + -- local search = filejoin(blobpath,blobfile,bname) + local search = filejoin(blobroot,blobfile,bname) + local result = methodhandler('concatinators',hash.type,blobroot,blobfile,bname) + if trace_detail then + report_resolving("match: variant '%s', search '%s', result '%s'",variant,search,result) + end + noffiles = noffiles + 1 + filelist[noffiles] = { variant, search, result } end else for kk=1,#blobfile do local vv = blobfile[kk] if not dname or find(vv,dname) then - filelist[#filelist+1] = { - hash.type, - file.join(blobpath,vv,bname), -- search - resolvers.concatinators[hash.type](blobpath,vv,bname) -- result - } + local variant = hash.type + -- local search = filejoin(blobpath,vv,bname) + local search = filejoin(blobroot,vv,bname) + local result = methodhandler('concatinators',hash.type,blobroot,vv,bname) + if trace_detail then + report_resolving("match: variant '%s', search '%s', result '%s'",variant,search,result) + end + noffiles = noffiles + 1 + filelist[noffiles] = { variant, search, result } end end end end elseif trace_locating then - logs.report("fileio","no match in '%s' (%s)",blobpath,bname) + report_resolving("no match in '%s' (%s)",blobpath,bname) end end end - if #filelist > 0 then - return filelist - else - return nil - end + return noffiles > 0 and filelist or nil end -function resolvers.suffix_of_format(str) - if suffixes[str] then - return suffixes[str][1] - else - return "" - end -end - -function resolvers.suffixes_of_format(str) - if suffixes[str] then - return suffixes[str] - else - return {} - end -end - -function resolvers.register_in_trees(name) +function resolvers.registerintrees(name) if not find(name,"^%.") then instance.foundintrees[name] = (instance.foundintrees[name] or 0) + 1 -- maybe only one end end --- split the next one up for readability (bu this module needs a cleanup anyway) +-- split the next one up for readability (but this module needs a cleanup anyway) local function can_be_dir(name) -- can become local local fakepaths = instance.fakepaths @@ -9756,61 +11889,62 @@ local function can_be_dir(name) -- can become local fakepaths[name] = 2 -- no directory end end - return (fakepaths[name] == 1) + return fakepaths[name] == 1 end -local function collect_instance_files(filename,collected) -- todo : plugin (scanners, checkers etc) - local result = collected or { } +local preparetreepattern = Cs((P(".")/"%%." + P("-")/"%%-" + P(1))^0 * Cc("$")) + +-- this one is split in smaller functions but it needs testing + +local function collect_instance_files(filename,askedformat,allresults) -- todo : plugin (scanners, checkers etc) + local result = { } local stamp = nil - filename = file.collapse_path(filename) + askedformat = askedformat or "" + filename = collapsepath(filename) -- speed up / beware: format problem - if instance.remember then - stamp = filename .. "--" .. instance.engine .. "--" .. instance.progname .. "--" .. instance.format + if instance.remember and not allresults then + stamp = filename .. "--" .. askedformat if instance.found[stamp] then if trace_locating then - logs.report("fileio","remembering file '%s'",filename) + report_resolving("remembered file '%s'",filename) end + resolvers.registerintrees(filename) -- for tracing used files return instance.found[stamp] end end - if not dangerous[instance.format or "?"] then - if resolvers.isreadable.file(filename) then + if not dangerous[askedformat] then + if isreadable(filename) then if trace_detail then - logs.report("fileio","file '%s' found directly",filename) + report_resolving("file '%s' found directly",filename) + end + if stamp then + instance.found[stamp] = { filename } end - instance.found[stamp] = { filename } return { filename } end end if find(filename,'%*') then if trace_locating then - logs.report("fileio","checking wildcard '%s'", filename) + report_resolving("checking wildcard '%s'", filename) end - result = resolvers.find_wildcard_files(filename) + result = resolvers.findwildcardfiles(filename) -- we can use th elocal elseif file.is_qualified_path(filename) then - if resolvers.isreadable.file(filename) then + if isreadable(filename) then if trace_locating then - logs.report("fileio","qualified name '%s'", filename) + report_resolving("qualified name '%s'", filename) end result = { filename } else - local forcedname, ok, suffix = "", false, file.extname(filename) + local forcedname, ok, suffix = "", false, fileextname(filename) if suffix == "" then -- why - if instance.format == "" then - forcedname = filename .. ".tex" - if resolvers.isreadable.file(forcedname) then - if trace_locating then - logs.report("fileio","no suffix, forcing standard filetype 'tex'") - end - result, ok = { forcedname }, true - end - else - local suffixes = resolvers.suffixes_of_format(instance.format) - for _, s in next, suffixes do + local format_suffixes = askedformat == "" and resolvers.defaultsuffixes or suffixes[askedformat] + if format_suffixes then + for i=1,#format_suffixes do + local s = format_suffixes[i] forcedname = filename .. "." .. s - if resolvers.isreadable.file(forcedname) then + if isreadable(forcedname) then if trace_locating then - logs.report("fileio","no suffix, forcing format filetype '%s'", s) + report_resolving("no suffix, forcing format filetype '%s'", s) end result, ok = { forcedname }, true break @@ -9821,23 +11955,24 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan if not ok and suffix ~= "" then -- try to find in tree (no suffix manipulation), here we search for the -- matching last part of the name - local basename = file.basename(filename) - local pattern = gsub(filename .. "$","([%.%-])","%%%1") - local savedformat = instance.format + local basename = filebasename(filename) + local pattern = lpegmatch(preparetreepattern,filename) + -- messy .. to be sorted out + local savedformat = askedformat local format = savedformat or "" if format == "" then - instance.format = resolvers.format_of_suffix(suffix) + askedformat = resolvers.formatofsuffix(suffix) end if not format then - instance.format = "othertextfiles" -- kind of everything, maybe texinput is better + askedformat = "othertextfiles" -- kind of everything, maybe texinput is better end -- if basename ~= filename then - local resolved = collect_instance_files(basename) - if #result == 0 then + local resolved = collect_instance_files(basename,askedformat,allresults) + if #result == 0 then -- shouldn't this be resolved ? local lowered = lower(basename) if filename ~= lowered then - resolved = collect_instance_files(lowered) + resolved = collect_instance_files(lowered,askedformat,allresults) end end resolvers.format = savedformat @@ -9862,57 +11997,60 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan -- end end if not ok and trace_locating then - logs.report("fileio","qualified name '%s'", filename) + report_resolving("qualified name '%s'", filename) end end else -- search spec - local filetype, extra, done, wantedfiles, ext = '', nil, false, { }, file.extname(filename) - if ext == "" then - if not instance.force_suffixes then - wantedfiles[#wantedfiles+1] = filename - end - else - wantedfiles[#wantedfiles+1] = filename - end - if instance.format == "" then - if ext == "" then - local forcedname = filename .. '.tex' - wantedfiles[#wantedfiles+1] = forcedname - filetype = resolvers.format_of_suffix(forcedname) - if trace_locating then - logs.report("fileio","forcing filetype '%s'",filetype) + local filetype, done, wantedfiles, ext = '', false, { }, fileextname(filename) + -- -- tricky as filename can be bla.1.2.3 + -- if not suffixmap[ext] then --- probably needs to be done elsewhere too + -- wantedfiles[#wantedfiles+1] = filename + -- end + wantedfiles[#wantedfiles+1] = filename + if askedformat == "" then + if ext == "" or not suffixmap[ext] then + local defaultsuffixes = resolvers.defaultsuffixes + for i=1,#defaultsuffixes do + local forcedname = filename .. '.' .. defaultsuffixes[i] + wantedfiles[#wantedfiles+1] = forcedname + filetype = resolvers.formatofsuffix(forcedname) + if trace_locating then + report_resolving("forcing filetype '%s'",filetype) + end end else - filetype = resolvers.format_of_suffix(filename) + filetype = resolvers.formatofsuffix(filename) if trace_locating then - logs.report("fileio","using suffix based filetype '%s'",filetype) + report_resolving("using suffix based filetype '%s'",filetype) end end else - if ext == "" then - local suffixes = resolvers.suffixes_of_format(instance.format) - for _, s in next, suffixes do - wantedfiles[#wantedfiles+1] = filename .. "." .. s + if ext == "" or not suffixmap[ext] then + local format_suffixes = suffixes[askedformat] + if format_suffixes then + for i=1,#format_suffixes do + wantedfiles[#wantedfiles+1] = filename .. "." .. format_suffixes[i] + end end end - filetype = instance.format + filetype = askedformat if trace_locating then - logs.report("fileio","using given filetype '%s'",filetype) + report_resolving("using given filetype '%s'",filetype) end end - local typespec = resolvers.variable_of_format(filetype) - local pathlist = resolvers.expanded_path_list(typespec) + local typespec = resolvers.variableofformat(filetype) + local pathlist = resolvers.expandedpathlist(typespec) if not pathlist or #pathlist == 0 then -- no pathlist, access check only / todo == wildcard if trace_detail then - logs.report("fileio","checking filename '%s', filetype '%s', wanted files '%s'",filename, filetype or '?',concat(wantedfiles," | ")) + report_resolving("checking filename '%s', filetype '%s', wanted files '%s'",filename, filetype or '?',concat(wantedfiles," | ")) end for k=1,#wantedfiles do local fname = wantedfiles[k] - if fname and resolvers.isreadable.file(fname) then + if fname and isreadable(fname) then filename, done = fname, true - result[#result+1] = file.join('.',fname) + result[#result+1] = filejoin('.',fname) break end end @@ -9920,8 +12058,8 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan local filelist = collect_files(wantedfiles) local fl = filelist and filelist[1] if fl then - filename = fl[3] - result[#result+1] = filename + filename = fl[3] -- not local? + result[#result+1] = resolvers.resolve(filename) done = true end else @@ -9930,76 +12068,65 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan local dirlist = { } if filelist then for i=1,#filelist do - dirlist[i] = file.dirname(filelist[i][2]) .. "/" + dirlist[i] = filedirname(filelist[i][3]) .. "/" -- was [2] .. gamble end end if trace_detail then - logs.report("fileio","checking filename '%s'",filename) + report_resolving("checking filename '%s'",filename) end - -- a bit messy ... esp the doscan setting here - local doscan for k=1,#pathlist do local path = pathlist[k] - if find(path,"^!!") then doscan = false else doscan = true end - local pathname = gsub(path,"^!+", '') + local pathname = lpegmatch(inhibitstripper,path) + local doscan = path == pathname -- no ^!! done = false -- using file list if filelist then - local expression -- compare list entries with permitted pattern -- /xx /xx// - if not find(pathname,"/$") then - expression = pathname .. "/" - else - expression = pathname - end - expression = gsub(expression,"([%-%.])","%%%1") -- this also influences - expression = gsub(expression,"//+$", '/.*') -- later usage of pathname - expression = gsub(expression,"//", '/.-/') -- not ok for /// but harmless - expression = "^" .. expression .. "$" + local expression = makepathexpression(pathname) if trace_detail then - logs.report("fileio","using pattern '%s' for path '%s'",expression,pathname) + report_resolving("using pattern '%s' for path '%s'",expression,pathname) end for k=1,#filelist do local fl = filelist[k] local f = fl[2] local d = dirlist[k] if find(d,expression) then - --- todo, test for readable - result[#result+1] = fl[3] - resolvers.register_in_trees(f) -- for tracing used files + -- todo, test for readable + result[#result+1] = resolvers.resolve(fl[3]) -- no shortcut done = true - if instance.allresults then + if allresults then if trace_detail then - logs.report("fileio","match in hash for file '%s' on path '%s', continue scanning",f,d) + report_resolving("match to '%s' in hash for file '%s' and path '%s', continue scanning",expression,f,d) end else if trace_detail then - logs.report("fileio","match in hash for file '%s' on path '%s', quit scanning",f,d) + report_resolving("match to '%s' in hash for file '%s' and path '%s', quit scanning",expression,f,d) end break end elseif trace_detail then - logs.report("fileio","no match in hash for file '%s' on path '%s'",f,d) + report_resolving("no match to '%s' in hash for file '%s' and path '%s'",expression,f,d) end end end if not done and doscan then -- check if on disk / unchecked / does not work at all / also zips - if resolvers.splitmethod(pathname).scheme == 'file' then -- ? + local scheme = url.hasscheme(pathname) + if not scheme or scheme == "file" then local pname = gsub(pathname,"%.%*$",'') if not find(pname,"%*") then local ppname = gsub(pname,"/+$","") if can_be_dir(ppname) then for k=1,#wantedfiles do local w = wantedfiles[k] - local fname = file.join(ppname,w) - if resolvers.isreadable.file(fname) then + local fname = filejoin(ppname,w) + if isreadable(fname) then if trace_detail then - logs.report("fileio","found '%s' by scanning",fname) + report_resolving("found '%s' by scanning",fname) end result[#result+1] = fname done = true - if not instance.allresults then break end + if not allresults then break end end end else @@ -10009,56 +12136,66 @@ local function collect_instance_files(filename,collected) -- todo : plugin (scan end end if not done and doscan then - -- todo: slow path scanning + -- todo: slow path scanning ... although we now have tree:// supported in $TEXMF end - if done and not instance.allresults then break end + if done and not allresults then break end end end end for k=1,#result do - result[k] = file.collapse_path(result[k]) + local rk = collapsepath(result[k]) + result[k] = rk + resolvers.registerintrees(rk) -- for tracing used files end - if instance.remember then + if stamp then instance.found[stamp] = result end return result end -if not resolvers.concatinators then resolvers.concatinators = { } end +-- -- -- begin of main file search routing -- -- -- -resolvers.concatinators.tex = file.join -resolvers.concatinators.file = resolvers.concatinators.tex -function resolvers.find_files(filename,filetype,mustexist) - if type(mustexist) == boolean then - -- all set - elseif type(filetype) == 'boolean' then - filetype, mustexist = nil, false - elseif type(filetype) ~= 'string' then - filetype, mustexist = nil, false - end - instance.format = filetype or '' - local result = collect_instance_files(filename) + + + + + + + + +-- -- -- end of main file search routing -- -- -- + +local function findfiles(filename,filetype,allresults) + local result = collect_instance_files(filename,filetype or "",allresults) if #result == 0 then local lowered = lower(filename) if filename ~= lowered then - return collect_instance_files(lowered) + return collect_instance_files(lowered,filetype or "",allresults) end end - instance.format = '' return result end -function resolvers.find_file(filename,filetype,mustexist) - return (resolvers.find_files(filename,filetype,mustexist)[1] or "") +function resolvers.findfiles(filename,filetype) + return findfiles(filename,filetype,true) +end + +function resolvers.findfile(filename,filetype) + return findfiles(filename,filetype,false)[1] or "" end -function resolvers.find_given_files(filename) - local bname, result = file.basename(filename), { } +function resolvers.findpath(filename,filetype) + return filedirname(findfiles(filename,filetype,false)[1] or "") +end + +local function findgivenfiles(filename,allresults) + local bname, result = filebasename(filename), { } local hashes = instance.hashes + local noffound = 0 for k=1,#hashes do local hash = hashes[k] - local files = instance.files[hash.tag] or { } + local files = instance.files[hash.name] or { } local blist = files[bname] if not blist then local rname = "remap:"..bname @@ -10070,13 +12207,21 @@ function resolvers.find_given_files(filename) end if blist then if type(blist) == 'string' then - result[#result+1] = resolvers.concatinators[hash.type](hash.tag,blist,bname) or "" - if not instance.allresults then break end + local found = methodhandler('concatinators',hash.type,hash.name,blist,bname) or "" + if found ~= "" then + noffound = noffound + 1 + result[noffound] = resolvers.resolve(found) + if not allresults then break end + end else for kk=1,#blist do local vv = blist[kk] - result[#result+1] = resolvers.concatinators[hash.type](hash.tag,vv,bname) or "" - if not instance.allresults then break end + local found = methodhandler('concatinators',hash.type,hash.name,vv,bname) or "" + if found ~= "" then + noffound = noffound + 1 + result[noffound] = resolvers.resolve(found) + if not allresults then break end + end end end end @@ -10084,24 +12229,31 @@ function resolvers.find_given_files(filename) return result end -function resolvers.find_given_file(filename) - return (resolvers.find_given_files(filename)[1] or "") +function resolvers.findgivenfiles(filename) + return findgivenfiles(filename,true) +end + +function resolvers.findgivenfile(filename) + return findgivenfiles(filename,false)[1] or "" end -local function doit(path,blist,bname,tag,kind,result,allresults) +local function doit(path,blist,bname,tag,variant,result,allresults) local done = false - if blist and kind then + if blist and variant then + local resolve = resolvers.resolve -- added if type(blist) == 'string' then -- make function and share code if find(lower(blist),path) then - result[#result+1] = resolvers.concatinators[kind](tag,blist,bname) or "" + local full = methodhandler('concatinators',variant,tag,blist,bname) or "" + result[#result+1] = resolve(full) done = true end else for kk=1,#blist do local vv = blist[kk] if find(lower(vv),path) then - result[#result+1] = resolvers.concatinators[kind](tag,vv,bname) or "" + local full = methodhandler('concatinators',variant,tag,vv,bname) or "" + result[#result+1] = resolve(full) done = true if not allresults then break end end @@ -10111,30 +12263,32 @@ local function doit(path,blist,bname,tag,kind,result,allresults) return done end -function resolvers.find_wildcard_files(filename) -- todo: remap: - local result = { } - local bname, dname = file.basename(filename), file.dirname(filename) - local path = gsub(dname,"^*/","") - path = gsub(path,"*",".*") - path = gsub(path,"-","%%-") - if dname == "" then - path = ".*" - end - local name = bname - name = gsub(name,"*",".*") - name = gsub(name,"-","%%-") - path = lower(path) - name = lower(name) - local files, allresults, done = instance.files, instance.allresults, false + +local makewildcard = Cs( + (P("^")^0 * P("/") * P(-1) + P(-1)) /".*" + + (P("^")^0 * P("/") / "")^0 * (P("*")/".*" + P("-")/"%%-" + P(".")/"%%." + P("?")/"."+ P("\\")/"/" + P(1))^0 +) + +function resolvers.wildcardpattern(pattern) + return lpegmatch(makewildcard,pattern) or pattern +end + +local function findwildcardfiles(filename,allresults,result) -- todo: remap: and lpeg + result = result or { } + local base = filebasename(filename) + local dirn = filedirname(filename) + local path = lower(lpegmatch(makewildcard,dirn) or dirn) + local name = lower(lpegmatch(makewildcard,base) or base) + local files, done = instance.files, false if find(name,"%*") then local hashes = instance.hashes for k=1,#hashes do local hash = hashes[k] - local tag, kind = hash.tag, hash.type - for kk, hh in next, files[hash.tag] do + local hashname, hashtype = hash.name, hash.type + for kk, hh in next, files[hashname] do if not find(kk,"^remap:") then if find(lower(kk),name) then - if doit(path,hh,kk,tag,kind,result,allresults) then done = true end + if doit(path,hh,kk,hashname,hashtype,result,allresults) then done = true end if done and not allresults then break end end end @@ -10144,8 +12298,8 @@ function resolvers.find_wildcard_files(filename) -- todo: remap: local hashes = instance.hashes for k=1,#hashes do local hash = hashes[k] - local tag, kind = hash.tag, hash.type - if doit(path,files[tag][bname],bname,tag,kind,result,allresults) then done = true end + local hashname, hashtype = hash.name, hash.type + if doit(path,files[hashname][bname],bname,hashname,hashtype,result,allresults) then done = true end if done and not allresults then break end end end @@ -10154,8 +12308,12 @@ function resolvers.find_wildcard_files(filename) -- todo: remap: return result end -function resolvers.find_wildcard_file(filename) - return (resolvers.find_wildcard_files(filename)[1] or "") +function resolvers.findwildcardfiles(filename,result) + return findwildcardfiles(filename,true,result) +end + +function resolvers.findwildcardfile(filename) + return findwildcardfiles(filename,false)[1] or "" end -- main user functions @@ -10166,34 +12324,33 @@ end function resolvers.load(option) statistics.starttiming(instance) - resolvers.resetconfig() - resolvers.identify_cnf() - resolvers.load_lua() -- will become the new method - resolvers.expand_variables() - resolvers.load_cnf() -- will be skipped when we have a lua file - resolvers.expand_variables() + identify_configuration_files() + load_configuration_files() if option ~= "nofiles" then - resolvers.load_hash() + load_databases() resolvers.automount() end statistics.stoptiming(instance) + local files = instance.files + return files and next(files) and true +end + +local function report(str) + if trace_locating then + report_resolving(str) -- has already verbose + else + print(str) + end end -function resolvers.for_files(command, files, filetype, mustexist) +function resolvers.dowithfilesandreport(command, files, ...) -- will move if files and #files > 0 then - local function report(str) - if trace_locating then - logs.report("fileio",str) -- has already verbose - else - print(str) - end - end if trace_locating then report('') -- ? end for f=1,#files do local file = files[f] - local result = command(file,filetype,mustexist) + local result = command(file,...) if type(result) == 'string' then report(result) else @@ -10205,21 +12362,16 @@ function resolvers.for_files(command, files, filetype, mustexist) end end --- strtab +-- obsolete -resolvers.var_value = resolvers.variable -- output the value of variable $STRING. -resolvers.expand_var = resolvers.expansion -- output variable expansion of STRING. +-- resolvers.varvalue = resolvers.variable -- output the value of variable $STRING. +-- resolvers.expandvar = resolvers.expansion -- output variable expansion of STRING. -function resolvers.show_path(str) -- output search path for file type NAME - return file.join_path(resolvers.expanded_path_list(resolvers.format_of_var(str))) +function resolvers.showpath(str) -- output search path for file type NAME + return joinpath(resolvers.expandedpathlist(resolvers.formatofvariable(str))) end --- resolvers.find_file(filename) --- resolvers.find_file(filename, filetype, mustexist) --- resolvers.find_file(filename, mustexist) --- resolvers.find_file(filename, filetype) - -function resolvers.register_file(files, name, path) +function resolvers.registerfile(files, name, path) if files[name] then if type(files[name]) == 'string' then files[name] = { files[name], path } @@ -10231,101 +12383,24 @@ function resolvers.register_file(files, name, path) end end -function resolvers.splitmethod(filename) - if not filename then - return { } -- safeguard - elseif type(filename) == "table" then - return filename -- already split - elseif not find(filename,"://") then - return { scheme="file", path = filename, original=filename } -- quick hack - else - return url.hashed(filename) - end -end - -function table.sequenced(t,sep) -- temp here - local s = { } - for k, v in next, t do -- indexed? - s[#s+1] = k .. "=" .. tostring(v) - end - return concat(s, sep or " | ") -end - -function resolvers.methodhandler(what, filename, filetype) -- ... - filename = file.collapse_path(filename) - local specification = (type(filename) == "string" and resolvers.splitmethod(filename)) or filename -- no or { }, let it bomb - local scheme = specification.scheme - if resolvers[what][scheme] then - if trace_locating then - logs.report("fileio","handler '%s' -> '%s' -> '%s'",specification.original,what,table.sequenced(specification)) - end - return resolvers[what][scheme](filename,filetype) -- todo: specification - else - return resolvers[what].tex(filename,filetype) -- todo: specification - end -end - -function resolvers.clean_path(str) - if str then - str = gsub(str,"\\","/") - str = gsub(str,"^!+","") - str = gsub(str,"^~",resolvers.homedir) - return str - else - return nil - end -end - -function resolvers.do_with_path(name,func) - local pathlist = resolvers.expanded_path_list(name) +function resolvers.dowithpath(name,func) + local pathlist = resolvers.expandedpathlist(name) for i=1,#pathlist do - func("^"..resolvers.clean_path(pathlist[i])) + func("^"..resolvers.cleanpath(pathlist[i])) end end -function resolvers.do_with_var(name,func) - func(expanded_var(name)) +function resolvers.dowithvariable(name,func) + func(expandedvariable(name)) end -function resolvers.with_files(pattern,handle) - local hashes = instance.hashes - for i=1,#hashes do - local hash = hashes[i] - local blobpath = hash.tag - local blobtype = hash.type - if blobpath then - local files = instance.files[blobpath] - if files then - for k,v in next, files do - if find(k,"^remap:") then - k = files[k] - v = files[k] -- chained - end - if find(k,pattern) then - if type(v) == "string" then - handle(blobtype,blobpath,v,k) - else - for _,vv in next, v do -- indexed - handle(blobtype,blobpath,vv,k) - end - end - end - end - end - end - end -end - -function resolvers.locate_format(name) - local barename, fmtname = gsub(name,"%.%a+$",""), "" - if resolvers.usecache then - local path = file.join(caches.setpath("formats")) -- maybe platform - fmtname = file.join(path,barename..".fmt") or "" - end +function resolvers.locateformat(name) + local barename = file.removesuffix(name) -- gsub(name,"%.%a+$","") + local fmtname = caches.getfirstreadablefile(barename..".fmt","formats") or "" if fmtname == "" then - fmtname = resolvers.find_files(barename..".fmt")[1] or "" + fmtname = resolvers.findfile(barename..".fmt") + fmtname = resolvers.cleanpath(fmtname) end - fmtname = resolvers.clean_path(fmtname) if fmtname ~= "" then local barename = file.removesuffix(fmtname) local luaname, lucname, luiname = barename .. ".lua", barename .. ".luc", barename .. ".lui" @@ -10340,7 +12415,7 @@ function resolvers.locate_format(name) return nil, nil end -function resolvers.boolean_variable(str,default) +function resolvers.booleanvariable(str,default) local b = resolvers.expansion(str) if b == "" then return default @@ -10350,204 +12425,62 @@ function resolvers.boolean_variable(str,default) end end -texconfig.kpse_init = false - -kpse = { original = kpse } setmetatable(kpse, { __index = function(k,v) return resolvers[v] end } ) - --- for a while - -input = resolvers - - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['data-tmp'] = { - version = 1.001, - comment = "companion to luat-lib.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - ---[[ldx-- -<p>This module deals with caching data. It sets up the paths and -implements loaders and savers for tables. Best is to set the -following variable. When not set, the usual paths will be -checked. Personally I prefer the (users) temporary path.</p> - -</code> -TEXMFCACHE=$TMP;$TEMP;$TMPDIR;$TEMPDIR;$HOME;$TEXMFVAR;$VARTEXMF;. -</code> - -<p>Currently we do no locking when we write files. This is no real -problem because most caching involves fonts and the chance of them -being written at the same time is small. We also need to extend -luatools with a recache feature.</p> ---ldx]]-- - -local format, lower, gsub = string.format, string.lower, string.gsub - -local trace_cache = false trackers.register("resolvers.cache", function(v) trace_cache = v end) -- not used yet - -caches = caches or { } - -caches.path = caches.path or nil -caches.base = caches.base or "luatex-cache" -caches.more = caches.more or "context" -caches.direct = false -- true is faster but may need huge amounts of memory -caches.tree = false -caches.paths = caches.paths or nil -caches.force = true -- changed for TEXLIVE by TH -caches.defaults = { "TEXMFCACHE", "TMPDIR", "TEMPDIR", "TMP", "TEMP", "HOME", "HOMEPATH" } - -function caches.temp() - local cachepath = nil - local function check(list,isenv) - if not cachepath then - for k=1,#list do - local v = list[k] - cachepath = (isenv and (os.env[v] or "")) or v or "" - if cachepath == "" then - -- next - else - cachepath = resolvers.clean_path(cachepath) - if lfs.isdir(cachepath) and file.iswritable(cachepath) then -- lfs.attributes(cachepath,"mode") == "directory" - break - elseif caches.force or io.ask(format("\nShould I create the cache path %s?",cachepath), "no", { "yes", "no" }) == "yes" then - dir.mkdirs(cachepath) - if lfs.isdir(cachepath) and file.iswritable(cachepath) then - break +function resolvers.dowithfilesintree(pattern,handle,before,after) -- will move, can be a nice iterator instead + local instance = resolvers.instance + local hashes = instance.hashes + for i=1,#hashes do + local hash = hashes[i] + local blobtype = hash.type + local blobpath = hash.name + if blobpath then + if before then + before(blobtype,blobpath,pattern) + end + local files = instance.files[blobpath] + local total, checked, done = 0, 0, 0 + if files then + for k,v in next, files do + total = total + 1 + if find(k,"^remap:") then + k = files[k] + v = k -- files[k] -- chained + end + if find(k,pattern) then + if type(v) == "string" then + checked = checked + 1 + if handle(blobtype,blobpath,v,k) then + done = done + 1 + end + else + checked = checked + #v + for i=1,#v do + if handle(blobtype,blobpath,v[i],k) then + done = done + 1 + end + end end end end - cachepath = nil + end + if after then + after(blobtype,blobpath,pattern,total,checked,done) end end end - check(resolvers.clean_path_list("TEXMFCACHE") or { }) - check(caches.defaults,true) - if not cachepath then - print("\nfatal error: there is no valid (writable) cache path defined\n") - os.exit() - elseif not lfs.isdir(cachepath) then -- lfs.attributes(cachepath,"mode") ~= "directory" - print(format("\nfatal error: cache path %s is not a directory\n",cachepath)) - os.exit() - end - cachepath = file.collapse_path(cachepath) - function caches.temp() - return cachepath - end - return cachepath -end - -function caches.configpath() - return table.concat(resolvers.instance.cnffiles,";") -end - -function caches.hashed(tree) - return md5.hex(gsub(lower(tree),"[\\\/]+","/")) -end - -function caches.treehash() - local tree = caches.configpath() - if not tree or tree == "" then - return false - else - return caches.hashed(tree) - end -end - -function caches.setpath(...) - if not caches.path then - if not caches.path then - caches.path = caches.temp() - end - caches.path = resolvers.clean_path(caches.path) -- to be sure - caches.tree = caches.tree or caches.treehash() - if caches.tree then - caches.path = dir.mkdirs(caches.path,caches.base,caches.more,caches.tree) - else - caches.path = dir.mkdirs(caches.path,caches.base,caches.more) - end - end - if not caches.path then - caches.path = '.' - end - caches.path = resolvers.clean_path(caches.path) - local dirs = { ... } - if #dirs > 0 then - local pth = dir.mkdirs(caches.path,...) - return pth - end - caches.path = dir.expand_name(caches.path) - return caches.path -end - -function caches.definepath(category,subcategory) - return function() - return caches.setpath(category,subcategory) - end -end - -function caches.setluanames(path,name) - return path .. "/" .. name .. ".tma", path .. "/" .. name .. ".tmc" end -function caches.loaddata(path,name) - local tmaname, tmcname = caches.setluanames(path,name) - local loader = loadfile(tmcname) or loadfile(tmaname) - if loader then - loader = loader() - collectgarbage("step") - return loader - else - return false - end -end +resolvers.obsolete = resolvers.obsolete or { } +local obsolete = resolvers.obsolete ---~ function caches.loaddata(path,name) ---~ local tmaname, tmcname = caches.setluanames(path,name) ---~ return dofile(tmcname) or dofile(tmaname) ---~ end - -function caches.iswritable(filepath,filename) - local tmaname, tmcname = caches.setluanames(filepath,filename) - return file.iswritable(tmaname) -end - -function caches.savedata(filepath,filename,data,raw) - local tmaname, tmcname = caches.setluanames(filepath,filename) - local reduce, simplify = true, true - if raw then - reduce, simplify = false, false - end - data.cache_uuid = os.uuid() - if caches.direct then - file.savedata(tmaname, table.serialize(data,'return',false,true,false)) -- no hex - else - table.tofile(tmaname, data,'return',false,true,false) -- maybe not the last true - end - local cleanup = resolvers.boolean_variable("PURGECACHE", false) - local strip = resolvers.boolean_variable("LUACSTRIP", true) - utils.lua.compile(tmaname, tmcname, cleanup, strip) -end - --- here we use the cache for format loading (texconfig.[formatname|jobname]) - ---~ if tex and texconfig and texconfig.formatname and texconfig.formatname == "" then -if tex and texconfig and (not texconfig.formatname or texconfig.formatname == "") and input and resolvers.instance then - if not texconfig.luaname then texconfig.luaname = "cont-en.lua" end -- or luc - texconfig.formatname = caches.setpath("formats") .. "/" .. gsub(texconfig.luaname,"%.lu.$",".fmt") -end +resolvers.find_file = resolvers.findfile obsolete.find_file = resolvers.findfile +resolvers.find_files = resolvers.findfiles obsolete.find_files = resolvers.findfiles end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-res'] = { +if not modules then modules = { } end modules ['data-pre'] = { version = 1.001, comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", @@ -10555,17 +12488,28 @@ if not modules then modules = { } end modules ['data-res'] = { license = "see context related readme files" } ---~ print(resolvers.resolve("abc env:tmp file:cont-en.tex path:cont-en.tex full:cont-en.tex rel:zapf/one/p-chars.tex")) +-- It could be interesting to hook the resolver in the file +-- opener so that unresolved prefixes travel around and we +-- get more abstraction. + +-- As we use this beforehand we will move this up in the chain +-- of loading. + -local upper, lower, gsub = string.upper, string.lower, string.gsub +local resolvers = resolvers +local prefixes = utilities.storage.allocate() +resolvers.prefixes = prefixes -local prefixes = { } +local gsub = string.gsub +local cleanpath, findgivenfile, expansion = resolvers.cleanpath, resolvers.findgivenfile, resolvers.expansion +local getenv = resolvers.getenv -- we can probably also use resolvers.expansion +local P, Cs, lpegmatch = lpeg.P, lpeg.Cs, lpeg.match prefixes.environment = function(str) - return resolvers.clean_path(os.getenv(str) or os.getenv(upper(str)) or os.getenv(lower(str)) or "") + return cleanpath(expansion(str)) end -prefixes.relative = function(str,n) +prefixes.relative = function(str,n) -- lfs.isfile if io.exists(str) then -- nothing elseif io.exists("./" .. str) then @@ -10581,7 +12525,7 @@ prefixes.relative = function(str,n) end end end - return resolvers.clean_path(str) + return cleanpath(str) end prefixes.auto = function(str) @@ -10593,18 +12537,34 @@ prefixes.auto = function(str) end prefixes.locate = function(str) - local fullname = resolvers.find_given_file(str) or "" - return resolvers.clean_path((fullname ~= "" and fullname) or str) + local fullname = findgivenfile(str) or "" + return cleanpath((fullname ~= "" and fullname) or str) end prefixes.filename = function(str) - local fullname = resolvers.find_given_file(str) or "" - return resolvers.clean_path(file.basename((fullname ~= "" and fullname) or str)) + local fullname = findgivenfile(str) or "" + return cleanpath(file.basename((fullname ~= "" and fullname) or str)) end prefixes.pathname = function(str) - local fullname = resolvers.find_given_file(str) or "" - return resolvers.clean_path(file.dirname((fullname ~= "" and fullname) or str)) + local fullname = findgivenfile(str) or "" + return cleanpath(file.dirname((fullname ~= "" and fullname) or str)) +end + +prefixes.selfautoloc = function(str) + return cleanpath(file.join(getenv('SELFAUTOLOC'),str)) +end + +prefixes.selfautoparent = function(str) + return cleanpath(file.join(getenv('SELFAUTOPARENT'),str)) +end + +prefixes.selfautodir = function(str) + return cleanpath(file.join(getenv('SELFAUTODIR'),str)) +end + +prefixes.home = function(str) + return cleanpath(file.join(getenv('HOME'),str)) end prefixes.env = prefixes.environment @@ -10633,19 +12593,28 @@ local function _resolve_(method,target) end end -local function resolve(str) - if type(str) == "table" then - for k=1,#str do - local v = str[k] - str[k] = resolve(v) or v - end - elseif str and str ~= "" then - str = gsub(str,"([a-z]+):([^ \"\']*)",_resolve_) +local resolved, abstract = { }, { } + +function resolvers.resetresolve(str) + resolved, abstract = { }, { } +end + +local function resolve(str) -- use schemes, this one is then for the commandline only + local res = resolved[str] + if not res then + res = gsub(str,"([a-z][a-z]+):([^ \"\']*)",_resolve_) + resolved[str] = res + abstract[res] = str end - return str + return res +end + +local function unresolve(str) + return abstract[str] or str end -resolvers.resolve = resolve +resolvers.resolve = resolve +resolvers.unresolve = unresolve if os.uname then @@ -10657,6 +12626,42 @@ if os.uname then end +if os.type == "unix" then + + local pattern + + local function makepattern(t,k,v) + local colon = P(":") + local p + for k, v in table.sortedpairs(prefixes) do + if p then + p = P(k) + p + else + p = P(k) + end + end + pattern = Cs((p * colon + colon/";" + P(1))^0) + if t then + t[k] = v + end + end + + makepattern() + + getmetatable(prefixes).__newindex = makepattern + + function resolvers.repath(str) + return lpegmatch(pattern,str) + end + +else -- already the default: + + function resolvers.repath(str) + return str + end + +end + end -- of closure @@ -10670,13 +12675,23 @@ if not modules then modules = { } end modules ['data-inp'] = { license = "see context related readme files" } -resolvers.finders = resolvers.finders or { } -resolvers.openers = resolvers.openers or { } -resolvers.loaders = resolvers.loaders or { } +local allocate = utilities.storage.allocate +local resolvers = resolvers + +local methodhandler = resolvers.methodhandler +local registermethod = resolvers.registermethod + +local finders = allocate { helpers = { }, notfound = function() end } +local openers = allocate { helpers = { }, notfound = function() end } +local loaders = allocate { helpers = { }, notfound = function() return false, nil, 0 end } -resolvers.finders.notfound = { nil } -resolvers.openers.notfound = { nil } -resolvers.loaders.notfound = { false, nil, 0 } +registermethod("finders", finders, "uri") +registermethod("openers", openers, "uri") +registermethod("loaders", loaders, "uri") + +resolvers.finders = finders +resolvers.openers = openers +resolvers.loaders = loaders end -- of closure @@ -10691,15 +12706,23 @@ if not modules then modules = { } end modules ['data-out'] = { license = "see context related readme files" } -outputs = outputs or { } +local allocate = utilities.storage.allocate +local resolvers = resolvers + +local registermethod = resolvers.registermethod + +local savers = allocate { helpers = { } } +resolvers.savers = savers + +registermethod("savers", savers, "uri") end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-con'] = { +if not modules then modules = { } end modules ['data-fil'] = { version = 1.001, comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", @@ -10707,6 +12730,124 @@ if not modules then modules = { } end modules ['data-con'] = { license = "see context related readme files" } +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) + +local report_files = logs.reporter("resolvers","files") + +local resolvers = resolvers + +local finders, openers, loaders, savers = resolvers.finders, resolvers.openers, resolvers.loaders, resolvers.savers +local locators, hashers, generators, concatinators = resolvers.locators, resolvers.hashers, resolvers.generators, resolvers.concatinators + +local checkgarbage = utilities.garbagecollector and utilities.garbagecollector.check + +function locators.file(specification) + local name = specification.filename + local realname = resolvers.resolve(name) -- no shortcut + if realname and realname ~= '' and lfs.isdir(realname) then + if trace_locating then + report_files("file locator '%s' found as '%s'",name,realname) + end + resolvers.appendhash('file',name,true) -- cache + elseif trace_locating then + report_files("file locator '%s' not found",name) + end +end + +function hashers.file(specification) + local name = specification.filename + local content = caches.loadcontent(name,'files') + resolvers.registerfilehash(name,content,content==nil) +end + +function generators.file(specification) + local path = specification.filename + local content = resolvers.scanfiles(path) + resolvers.registerfilehash(path,content,true) +end + +concatinators.file = file.join + +function finders.file(specification,filetype) + local filename = specification.filename + local foundname = resolvers.findfile(filename,filetype) + if foundname and foundname ~= "" then + if trace_locating then + report_files("file finder: '%s' found",filename) + end + return foundname + else + if trace_locating then + report_files("file finder: %s' not found",filename) + end + return finders.notfound() + end +end + +-- The default textopener will be overloaded later on. + +function openers.helpers.textopener(tag,filename,f) + return { + reader = function() return f:read () end, + close = function() logs.show_close(filename) return f:close() end, + } +end + +function openers.file(specification,filetype) + local filename = specification.filename + if filename and filename ~= "" then + local f = io.open(filename,"r") + if f then + if trace_locating then + report_files("file opener, '%s' opened",filename) + end + return openers.helpers.textopener("file",filename,f) + end + end + if trace_locating then + report_files("file opener, '%s' not found",filename) + end + return openers.notfound() +end + +function loaders.file(specification,filetype) + local filename = specification.filename + if filename and filename ~= "" then + local f = io.open(filename,"rb") + if f then + logs.show_load(filename) + if trace_locating then + report_files("file loader, '%s' loaded",filename) + end + local s = f:read("*a") + if checkgarbage then + checkgarbage(#s) + end + f:close() + if s then + return true, s, #s + end + end + end + if trace_locating then + report_files("file loader, '%s' not found",filename) + end + return loaders.notfound() +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['data-con'] = { + version = 1.100, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + local format, lower, gsub = string.format, string.lower, string.gsub local trace_cache = false trackers.register("resolvers.cache", function(v) trace_cache = v end) @@ -10726,50 +12867,63 @@ table structures without bothering about the disk cache.</p> <p>Examples of usage can be found in the font related code.</p> --ldx]]-- -containers = containers or { } - +containers = containers or { } +local containers = containers containers.usecache = true +local report_containers = logs.reporter("resolvers","containers") + local function report(container,tag,name) if trace_cache or trace_containers then - logs.report(format("%s cache",container.subcategory),"%s: %s",tag,name or 'invalid') + report_containers("container: %s, tag: %s, name: %s",container.subcategory,tag,name or 'invalid') end end local allocated = { } --- tracing +local mt = { + __index = function(t,k) + if k == "writable" then + local writable = caches.getwritablepath(t.category,t.subcategory) or { "." } + t.writable = writable + return writable + elseif k == "readables" then + local readables = caches.getreadablepaths(t.category,t.subcategory) or { "." } + t.readables = readables + return readables + end + end, + __storage__ = true +} function containers.define(category, subcategory, version, enabled) - return function() - if category and subcategory then - local c = allocated[category] - if not c then - c = { } - allocated[category] = c - end - local s = c[subcategory] - if not s then - s = { - category = category, - subcategory = subcategory, - storage = { }, - enabled = enabled, - version = version or 1.000, - trace = false, - path = caches and caches.setpath and caches.setpath(category,subcategory), - } - c[subcategory] = s - end - return s - else - return nil + if category and subcategory then + local c = allocated[category] + if not c then + c = { } + allocated[category] = c + end + local s = c[subcategory] + if not s then + s = { + category = category, + subcategory = subcategory, + storage = { }, + enabled = enabled, + version = version or math.pi, -- after all, this is TeX + trace = false, + -- writable = caches.getwritablepath and caches.getwritablepath (category,subcategory) or { "." }, + -- readables = caches.getreadablepaths and caches.getreadablepaths(category,subcategory) or { "." }, + } + setmetatable(s,mt) + c[subcategory] = s end + return s end end function containers.is_usable(container, name) - return container.enabled and caches and caches.iswritable(container.path, name) + return container.enabled and caches and caches.is_writable(container.writable, name) end function containers.is_valid(container, name) @@ -10782,18 +12936,20 @@ function containers.is_valid(container, name) end function containers.read(container,name) - if container.enabled and caches and not container.storage[name] and containers.usecache then - container.storage[name] = caches.loaddata(container.path,name) - if containers.is_valid(container,name) then + local storage = container.storage + local stored = storage[name] + if not stored and container.enabled and caches and containers.usecache then + stored = caches.loaddata(container.readables,name) + if stored and stored.cache_version == container.version then report(container,"loaded",name) else - container.storage[name] = nil + stored = nil end - end - if container.storage[name] then + storage[name] = stored + elseif stored then report(container,"reusing",name) end - return container.storage[name] + return stored end function containers.write(container, name, data) @@ -10802,7 +12958,7 @@ function containers.write(container, name, data) if container.enabled and caches then local unique, shared = data.unique, data.shared data.unique, data.shared = nil, nil - caches.savedata(container.path, name, data) + caches.savedata(container.writable, name, data) report(container,"saved",name) data.unique, data.shared = unique, shared end @@ -10837,50 +12993,18 @@ local format, lower, gsub, find = string.format, string.lower, string.gsub, stri local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) --- since we want to use the cache instead of the tree, we will now --- reimplement the saver. - -local save_data = resolvers.save_data -local load_data = resolvers.load_data +local report_mounts = logs.reporter("resolvers","mounts") -resolvers.cachepath = nil -- public, for tracing -resolvers.usecache = true -- public, for tracing - -function resolvers.save_data(dataname) - save_data(dataname, function(cachename,dataname) - resolvers.usecache = not toboolean(resolvers.expansion("CACHEINTDS") or "false",true) - if resolvers.usecache then - resolvers.cachepath = resolvers.cachepath or caches.definepath("trees") - return file.join(resolvers.cachepath(),caches.hashed(cachename)) - else - return file.join(cachename,dataname) - end - end) -end - -function resolvers.load_data(pathname,dataname,filename) - load_data(pathname,dataname,filename,function(dataname,filename) - resolvers.usecache = not toboolean(resolvers.expansion("CACHEINTDS") or "false",true) - if resolvers.usecache then - resolvers.cachepath = resolvers.cachepath or caches.definepath("trees") - return file.join(resolvers.cachepath(),caches.hashed(pathname)) - else - if not filename or (filename == "") then - filename = dataname - end - return file.join(pathname,filename) - end - end) -end +local resolvers = resolvers -- we will make a better format, maybe something xml or just text or lua resolvers.automounted = resolvers.automounted or { } function resolvers.automount(usecache) - local mountpaths = resolvers.clean_path_list(resolvers.expansion('TEXMFMOUNT')) + local mountpaths = resolvers.cleanpathlist(resolvers.expansion('TEXMFMOUNT')) if (not mountpaths or #mountpaths == 0) and usecache then - mountpaths = { caches.setpath("mount") } + mountpaths = caches.getreadablepaths("mount") end if mountpaths and #mountpaths > 0 then statistics.starttiming(resolvers.instance) @@ -10894,7 +13018,7 @@ function resolvers.automount(usecache) -- skip elseif find(line,"^zip://") then if trace_locating then - logs.report("fileio","mounting %s",line) + report_mounts("mounting %s",line) end table.insert(resolvers.automounted,line) resolvers.usezipfile(line) @@ -10910,40 +13034,40 @@ end -- status info -statistics.register("used config path", function() return caches.configpath() end) -statistics.register("used cache path", function() return caches.temp() or "?" end) +statistics.register("used config file", function() return caches.configfiles() end) +statistics.register("used cache path", function() return caches.usedpaths() end) -- experiment (code will move) -function statistics.save_fmt_status(texname,formatbanner,sourcefile) -- texname == formatname +function statistics.savefmtstatus(texname,formatbanner,sourcefile) -- texname == formatname local enginebanner = status.list().banner if formatbanner and enginebanner and sourcefile then local luvname = file.replacesuffix(texname,"luv") local luvdata = { enginebanner = enginebanner, formatbanner = formatbanner, - sourcehash = md5.hex(io.loaddata(resolvers.find_file(sourcefile)) or "unknown"), + sourcehash = md5.hex(io.loaddata(resolvers.findfile(sourcefile)) or "unknown"), sourcefile = sourcefile, } io.savedata(luvname,table.serialize(luvdata,true)) end end -function statistics.check_fmt_status(texname) +function statistics.checkfmtstatus(texname) local enginebanner = status.list().banner if enginebanner and texname then local luvname = file.replacesuffix(texname,"luv") if lfs.isfile(luvname) then local luv = dofile(luvname) if luv and luv.sourcefile then - local sourcehash = md5.hex(io.loaddata(resolvers.find_file(luv.sourcefile)) or "unknown") + local sourcehash = md5.hex(io.loaddata(resolvers.findfile(luv.sourcefile)) or "unknown") local luvbanner = luv.enginebanner or "?" if luvbanner ~= enginebanner then - return string.format("engine mismatch (luv:%s <> bin:%s)",luvbanner,enginebanner) + return format("engine mismatch (luv: %s <> bin: %s)",luvbanner,enginebanner) end local luvhash = luv.sourcehash or "?" if luvhash ~= sourcehash then - return string.format("source mismatch (luv:%s <> bin:%s)",luvhash,sourcehash) + return format("source mismatch (luv: %s <> bin: %s)",luvhash,sourcehash) end else return "invalid status file" @@ -10968,25 +13092,30 @@ if not modules then modules = { } end modules ['data-zip'] = { license = "see context related readme files" } +-- partly redone .. needs testing + local format, find, match = string.format, string.find, string.match -local unpack = unpack or table.unpack local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) +local report_zip = logs.reporter("resolvers","zip") + -- zip:///oeps.zip?name=bla/bla.tex -- zip:///oeps.zip?tree=tex/texmf-local -- zip:///texmf.zip?tree=/tex/texmf -- zip:///texmf.zip?tree=/tex/texmf-local -- zip:///texmf-mine.zip?tree=/tex/texmf-projects -zip = zip or { } -zip.archives = zip.archives or { } -zip.registeredfiles = zip.registeredfiles or { } +local resolvers = resolvers + +zip = zip or { } +local zip = zip -local finders, openers, loaders = resolvers.finders, resolvers.openers, resolvers.loaders -local locators, hashers, concatinators = resolvers.locators, resolvers.hashers, resolvers.concatinators +zip.archives = zip.archives or { } +local archives = zip.archives -local archives = zip.archives +zip.registeredfiles = zip.registeredfiles or { } +local registeredfiles = zip.registeredfiles local function validzip(str) -- todo: use url splitter if not find(str,"^zip://") then @@ -11002,7 +13131,7 @@ function zip.openarchive(name) else local arch = archives[name] if not arch then - local full = resolvers.find_file(name) or "" + local full = resolvers.findfile(name) or "" arch = (full ~= "" and zip.open(full)) or false archives[name] = arch end @@ -11017,163 +13146,162 @@ function zip.closearchive(name) end end -function locators.zip(specification) -- where is this used? startup zips (untested) - specification = resolvers.splitmethod(specification) - local zipfile = specification.path - local zfile = zip.openarchive(name) -- tricky, could be in to be initialized tree +function resolvers.locators.zip(specification) + local archive = specification.filename + local zipfile = archive and archive ~= "" and zip.openarchive(archive) -- tricky, could be in to be initialized tree if trace_locating then - if zfile then - logs.report("fileio","zip locator, archive '%s' found",specification.original) + if zipfile then + report_zip("locator, archive '%s' found",archive) else - logs.report("fileio","zip locator, archive '%s' not found",specification.original) + report_zip("locator, archive '%s' not found",archive) end end end -function hashers.zip(tag,name) +function resolvers.hashers.zip(specification) + local archive = specification.filename if trace_locating then - logs.report("fileio","loading zip file '%s' as '%s'",name,tag) + report_zip("loading file '%s'",archive) end - resolvers.usezipfile(format("%s?tree=%s",tag,name)) + resolvers.usezipfile(specification.original) end -function concatinators.zip(tag,path,name) +function resolvers.concatinators.zip(zipfile,path,name) -- ok ? if not path or path == "" then - return format('%s?name=%s',tag,name) + return format('%s?name=%s',zipfile,name) else - return format('%s?name=%s/%s',tag,path,name) + return format('%s?name=%s/%s',zipfile,path,name) end end -function resolvers.isreadable.zip(name) - return true -end - -function finders.zip(specification,filetype) - specification = resolvers.splitmethod(specification) - if specification.path then - local q = url.query(specification.query) - if q.name then - local zfile = zip.openarchive(specification.path) +function resolvers.finders.zip(specification) + local original = specification.original + local archive = specification.filename + if archive then + local query = url.query(specification.query) + local queryname = query.name + if queryname then + local zfile = zip.openarchive(archive) if zfile then if trace_locating then - logs.report("fileio","zip finder, archive '%s' found",specification.path) + report_zip("finder, archive '%s' found",archive) end - local dfile = zfile:open(q.name) + local dfile = zfile:open(queryname) if dfile then dfile = zfile:close() if trace_locating then - logs.report("fileio","zip finder, file '%s' found",q.name) + report_zip("finder, file '%s' found",queryname) end return specification.original elseif trace_locating then - logs.report("fileio","zip finder, file '%s' not found",q.name) + report_zip("finder, file '%s' not found",queryname) end elseif trace_locating then - logs.report("fileio","zip finder, unknown archive '%s'",specification.path) + report_zip("finder, unknown archive '%s'",archive) end end end if trace_locating then - logs.report("fileio","zip finder, '%s' not found",filename) + report_zip("finder, '%s' not found",original) end - return unpack(finders.notfound) + return resolvers.finders.notfound() end -function openers.zip(specification) - local zipspecification = resolvers.splitmethod(specification) - if zipspecification.path then - local q = url.query(zipspecification.query) - if q.name then - local zfile = zip.openarchive(zipspecification.path) +function resolvers.openers.zip(specification) + local original = specification.original + local archive = specification.filename + if archive then + local query = url.query(specification.query) + local queryname = query.name + if queryname then + local zfile = zip.openarchive(archive) if zfile then if trace_locating then - logs.report("fileio","zip opener, archive '%s' opened",zipspecification.path) + report_zip("opener, archive '%s' opened",archive) end - local dfile = zfile:open(q.name) + local dfile = zfile:open(queryname) if dfile then - logs.show_open(specification) if trace_locating then - logs.report("fileio","zip opener, file '%s' found",q.name) + report_zip("opener, file '%s' found",queryname) end - return openers.text_opener(specification,dfile,'zip') + return resolvers.openers.helpers.textopener('zip',original,dfile) elseif trace_locating then - logs.report("fileio","zip opener, file '%s' not found",q.name) + report_zip("opener, file '%s' not found",queryname) end elseif trace_locating then - logs.report("fileio","zip opener, unknown archive '%s'",zipspecification.path) + report_zip("opener, unknown archive '%s'",archive) end end end if trace_locating then - logs.report("fileio","zip opener, '%s' not found",filename) + report_zip("opener, '%s' not found",original) end - return unpack(openers.notfound) + return resolvers.openers.notfound() end -function loaders.zip(specification) - specification = resolvers.splitmethod(specification) - if specification.path then - local q = url.query(specification.query) - if q.name then - local zfile = zip.openarchive(specification.path) +function resolvers.loaders.zip(specification) + local original = specification.original + local archive = specification.filename + if archive then + local query = url.query(specification.query) + local queryname = query.name + if queryname then + local zfile = zip.openarchive(archive) if zfile then if trace_locating then - logs.report("fileio","zip loader, archive '%s' opened",specification.path) + report_zip("loader, archive '%s' opened",archive) end - local dfile = zfile:open(q.name) + local dfile = zfile:open(queryname) if dfile then - logs.show_load(filename) + logs.show_load(original) if trace_locating then - logs.report("fileio","zip loader, file '%s' loaded",filename) + report_zip("loader, file '%s' loaded",original) end local s = dfile:read("*all") dfile:close() return true, s, #s elseif trace_locating then - logs.report("fileio","zip loader, file '%s' not found",q.name) + report_zip("loader, file '%s' not found",queryname) end elseif trace_locating then - logs.report("fileio","zip loader, unknown archive '%s'",specification.path) + report_zip("loader, unknown archive '%s'",archive) end end end if trace_locating then - logs.report("fileio","zip loader, '%s' not found",filename) + report_zip("loader, '%s' not found",original) end - return unpack(openers.notfound) + return resolvers.openers.notfound() end -- zip:///somefile.zip -- zip:///somefile.zip?tree=texmf-local -> mount -function resolvers.usezipfile(zipname) - zipname = validzip(zipname) - local specification = resolvers.splitmethod(zipname) - local zipfile = specification.path - if zipfile and not zip.registeredfiles[zipname] then - local tree = url.query(specification.query).tree or "" - local z = zip.openarchive(zipfile) +function resolvers.usezipfile(archive) + local specification = resolvers.splitmethod(archive) -- to be sure + local archive = specification.filename + if archive and not registeredfiles[archive] then + local z = zip.openarchive(archive) if z then - local instance = resolvers.instance + local tree = url.query(specification.query).tree or "" if trace_locating then - logs.report("fileio","zip registering, registering archive '%s'",zipname) - end - statistics.starttiming(instance) - resolvers.prepend_hash('zip',zipname,zipfile) - resolvers.extend_texmf_var(zipname) -- resets hashes too - zip.registeredfiles[zipname] = z - instance.files[zipname] = resolvers.register_zip_file(z,tree or "") - statistics.stoptiming(instance) + report_zip("registering, registering archive '%s'",archive) + end + statistics.starttiming(resolvers.instance) + resolvers.prependhash('zip',archive) + resolvers.extendtexmfvariable(archive) -- resets hashes too + registeredfiles[archive] = z + instance.files[archive] = resolvers.registerzipfile(z,tree) + statistics.stoptiming(resolvers.instance) elseif trace_locating then - logs.report("fileio","zip registering, unknown archive '%s'",zipname) + report_zip("registering, unknown archive '%s'",archive) end elseif trace_locating then - logs.report("fileio","zip registering, '%s' not found",zipname) + report_zip("registering, '%s' not found",archive) end end -function resolvers.register_zip_file(z,tree) +function resolvers.registerzipfile(z,tree) local files, filter = { }, "" if tree == "" then filter = "^(.+)/(.-)$" @@ -11181,9 +13309,9 @@ function resolvers.register_zip_file(z,tree) filter = format("^%s/(.+)/(.-)$",tree) end if trace_locating then - logs.report("fileio","zip registering, using filter '%s'",filter) + report_zip("registering, using filter '%s'",filter) end - local register, n = resolvers.register_file, 0 + local register, n = resolvers.registerfile, 0 for i in z:files() do local path, name = match(i.filename,filter) if path then @@ -11198,7 +13326,7 @@ function resolvers.register_zip_file(z,tree) n = n + 1 end end - logs.report("fileio","zip registering, %s files registered",n) + report_zip("registering, %s files registered",n) return files end @@ -11207,6 +13335,84 @@ end -- of closure do -- create closure to overcome 200 locals limit +if not modules then modules = { } end modules ['data-tre'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- \input tree://oeps1/**/oeps.tex + +local find, gsub, format = string.find, string.gsub, string.format + +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) + +local report_trees = logs.reporter("resolvers","trees") + +local resolvers = resolvers + +local done, found, notfound = { }, { }, resolvers.finders.notfound + +function resolvers.finders.tree(specification) + local spec = specification.filename + local fnd = found[spec] + if fnd == nil then + if spec ~= "" then + local path, name = file.dirname(spec), file.basename(spec) + if path == "" then path = "." end + local hash = done[path] + if not hash then + local pattern = path .. "/*" -- we will use the proper splitter + hash = dir.glob(pattern) + done[path] = hash + end + local pattern = "/" .. gsub(name,"([%.%-%+])", "%%%1") .. "$" + for k=1,#hash do + local v = hash[k] + if find(v,pattern) then + found[spec] = v + return v + end + end + end + fnd = notfound() -- false + found[spec] = fnd + end + return fnd +end + +function resolvers.locators.tree(specification) + local name = specification.filename + if name ~= '' and lfs.isdir(name) then + if trace_locating then + report_trees("locator '%s' found",name) + end + resolvers.appendhash('tree',name,false) -- don't cache + elseif trace_locating then + report_trees("locator '%s' not found",name) + end +end + +function resolvers.hashers.tree(specification) + local name = specification.filename + if trace_locating then + report_trees("analysing '%s'",name) + end + resolvers.methodhandler("hashers",name) +end + +resolvers.concatinators.tree = resolvers.concatinators.file +resolvers.generators.tree = resolvers.generators.file +resolvers.openers.tree = resolvers.openers.file +resolvers.loaders.tree = resolvers.loaders.file + + +end -- of closure + +do -- create closure to overcome 200 locals limit + if not modules then modules = { } end modules ['data-crl'] = { version = 1.001, comment = "companion to luat-lib.mkiv", @@ -11215,58 +13421,59 @@ if not modules then modules = { } end modules ['data-crl'] = { license = "see context related readme files" } -local gsub = string.gsub +-- this one is replaced by data-sch.lua -- -curl = curl or { } +local gsub = string.gsub -curl.cached = { } -curl.cachepath = caches.definepath("curl") +local resolvers = resolvers local finders, openers, loaders = resolvers.finders, resolvers.openers, resolvers.loaders -function curl.fetch(protocol, name) - local cachename = curl.cachepath() .. "/" .. gsub(name,"[^%a%d%.]+","-") --- cachename = gsub(cachename,"[\\/]", io.fileseparator) - cachename = gsub(cachename,"[\\]", "/") -- cleanup - if not curl.cached[name] then +resolvers.curl = resolvers.curl or { } +local curl = resolvers.curl + +local cached = { } + +local function runcurl(specification) + local original = specification.original + -- local scheme = specification.scheme + local cleanname = gsub(original,"[^%a%d%.]+","-") + local cachename = caches.setfirstwritablefile(cleanname,"curl") + if not cached[original] then if not io.exists(cachename) then - curl.cached[name] = cachename - local command = "curl --silent --create-dirs --output " .. cachename .. " " .. name -- no protocol .. "://" + cached[original] = cachename + local command = "curl --silent --create-dirs --output " .. cachename .. " " .. original os.spawn(command) end if io.exists(cachename) then - curl.cached[name] = cachename + cached[original] = cachename else - curl.cached[name] = "" + cached[original] = "" end end - return curl.cached[name] + return cached[original] end -function finders.curl(protocol,filename) - local foundname = curl.fetch(protocol, filename) - return finders.generic(protocol,foundname,filetype) -end - -function openers.curl(protocol,filename) - return openers.generic(protocol,filename) -end +-- old code: we could be cleaner using specification (see schemes) -function loaders.curl(protocol,filename) - return loaders.generic(protocol,filename) +local function finder(specification,filetype) + return resolvers.methodhandler("finders",runcurl(specification),filetype) end --- todo: metamethod +local opener = openers.file +local loader = loaders.file -function curl.install(protocol) - finders[protocol] = function (filename,filetype) return finders.curl(protocol,filename) end - openers[protocol] = function (filename) return openers.curl(protocol,filename) end - loaders[protocol] = function (filename) return loaders.curl(protocol,filename) end +local function install(scheme) + finders[scheme] = finder + openers[scheme] = opener + loaders[scheme] = loader end -curl.install('http') -curl.install('https') -curl.install('ftp') +resolvers.curl.install = install + +install('http') +install('https') +install('ftp') end -- of closure @@ -11287,9 +13494,13 @@ if not modules then modules = { } end modules ['data-lua'] = { local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) +local report_libraries = logs.reporter("resolvers","libraries") + local gsub, insert = string.gsub, table.insert local unpack = unpack or table.unpack +local resolvers, package = resolvers, package + local libformats = { 'luatexlibs', 'tex', 'texmfscripts', 'othertextfiles' } -- 'luainputs' local clibformats = { 'lib' } @@ -11298,7 +13509,7 @@ local _path_, libpaths, _cpath_, clibpaths function package.libpaths() if not _path_ or package.path ~= _path_ then _path_ = package.path - libpaths = file.split_path(_path_,";") + libpaths = file.splitpath(_path_,";") end return libpaths end @@ -11306,7 +13517,7 @@ end function package.clibpaths() if not _cpath_ or package.cpath ~= _cpath_ then _cpath_ = package.cpath - clibpaths = file.split_path(_cpath_,";") + clibpaths = file.splitpath(_cpath_,";") end return clibpaths end @@ -11315,18 +13526,18 @@ local function thepath(...) local t = { ... } t[#t+1] = "?.lua" local path = file.join(unpack(t)) if trace_locating then - logs.report("fileio","! appending '%s' to 'package.path'",path) + report_libraries("! appending '%s' to 'package.path'",path) end return path end local p_libpaths, a_libpaths = { }, { } -function package.append_libpath(...) +function package.appendtolibpath(...) insert(a_libpath,thepath(...)) end -function package.prepend_libpath(...) +function package.prependtolibpath(...) insert(p_libpaths,1,thepath(...)) end @@ -11337,31 +13548,30 @@ local function loaded(libpaths,name,simple) local libpath = libpaths[i] local resolved = gsub(libpath,"%?",simple) if trace_locating then -- more detail - logs.report("fileio","! checking for '%s' on 'package.path': '%s' => '%s'",simple,libpath,resolved) + report_libraries("! checking for '%s' on 'package.path': '%s' => '%s'",simple,libpath,resolved) end - if resolvers.isreadable.file(resolved) then + if file.is_readable(resolved) then if trace_locating then - logs.report("fileio","! lib '%s' located via 'package.path': '%s'",name,resolved) + report_libraries("! lib '%s' located via 'package.path': '%s'",name,resolved) end return loadfile(resolved) end end end - package.loaders[2] = function(name) -- was [#package.loaders+1] if trace_locating then -- mode detail - logs.report("fileio","! locating '%s'",name) + report_libraries("! locating '%s'",name) end for i=1,#libformats do local format = libformats[i] - local resolved = resolvers.find_file(name,format) or "" + local resolved = resolvers.findfile(name,format) or "" if trace_locating then -- mode detail - logs.report("fileio","! checking for '%s' using 'libformat path': '%s'",name,format) + report_libraries("! checking for '%s' using 'libformat path': '%s'",name,format) end if resolved ~= "" then if trace_locating then - logs.report("fileio","! lib '%s' located via environment: '%s'",name,resolved) + report_libraries("! lib '%s' located via environment: '%s'",name,resolved) end return loadfile(resolved) end @@ -11379,16 +13589,16 @@ package.loaders[2] = function(name) -- was [#package.loaders+1] for i=1,#clibformats do -- better have a dedicated loop local format = clibformats[i] - local paths = resolvers.expanded_path_list_from_var(format) + local paths = resolvers.expandedpathlistfromvariable(format) for p=1,#paths do local path = paths[p] local resolved = file.join(path,libname) if trace_locating then -- mode detail - logs.report("fileio","! checking for '%s' using 'clibformat path': '%s'",libname,path) + report_libraries("! checking for '%s' using 'clibformat path': '%s'",libname,path) end - if resolvers.isreadable.file(resolved) then + if file.is_readable(resolved) then if trace_locating then - logs.report("fileio","! lib '%s' located via 'clibformat': '%s'",libname,resolved) + report_libraries("! lib '%s' located via 'clibformat': '%s'",libname,resolved) end return package.loadlib(resolved,name) end @@ -11398,140 +13608,43 @@ package.loaders[2] = function(name) -- was [#package.loaders+1] local libpath = clibpaths[i] local resolved = gsub(libpath,"?",simple) if trace_locating then -- more detail - logs.report("fileio","! checking for '%s' on 'package.cpath': '%s'",simple,libpath) + report_libraries("! checking for '%s' on 'package.cpath': '%s'",simple,libpath) end - if resolvers.isreadable.file(resolved) then + if file.is_readable(resolved) then if trace_locating then - logs.report("fileio","! lib '%s' located via 'package.cpath': '%s'",name,resolved) + report_libraries("! lib '%s' located via 'package.cpath': '%s'",name,resolved) end return package.loadlib(resolved,name) end end -- just in case the distribution is messed up if trace_loading then -- more detail - logs.report("fileio","! checking for '%s' using 'luatexlibs': '%s'",name) + report_libraries("! checking for '%s' using 'luatexlibs': '%s'",name) end - local resolved = resolvers.find_file(file.basename(name),'luatexlibs') or "" + local resolved = resolvers.findfile(file.basename(name),'luatexlibs') or "" if resolved ~= "" then if trace_locating then - logs.report("fileio","! lib '%s' located by basename via environment: '%s'",name,resolved) + report_libraries("! lib '%s' located by basename via environment: '%s'",name,resolved) end return loadfile(resolved) end if trace_locating then - logs.report("fileio",'? unable to locate lib: %s',name) + report_libraries('? unable to locate lib: %s',name) end -- return "unable to locate " .. name end resolvers.loadlualib = require +-- -- -- -- -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['luat-kps'] = { - version = 1.001, - comment = "companion to luatools.lua", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - ---[[ldx-- -<p>This file is used when we want the input handlers to behave like -<type>kpsewhich</type>. What to do with the following:</p> - -<typing> -{$SELFAUTOLOC,$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,}/web2c} -$SELFAUTOLOC : /usr/tex/bin/platform -$SELFAUTODIR : /usr/tex/bin -$SELFAUTOPARENT : /usr/tex -</typing> +package.obsolete = package.obsolete or { } -<p>How about just forgetting about them?</p> ---ldx]]-- +package.append_libpath = appendtolibpath -- will become obsolete +package.prepend_libpath = prependtolibpath -- will become obsolete -local suffixes = resolvers.suffixes -local formats = resolvers.formats - -suffixes['gf'] = { '<resolution>gf' } -suffixes['pk'] = { '<resolution>pk' } -suffixes['base'] = { 'base' } -suffixes['bib'] = { 'bib' } -suffixes['bst'] = { 'bst' } -suffixes['cnf'] = { 'cnf' } -suffixes['mem'] = { 'mem' } -suffixes['mf'] = { 'mf' } -suffixes['mfpool'] = { 'pool' } -suffixes['mft'] = { 'mft' } -suffixes['mppool'] = { 'pool' } -suffixes['graphic/figure'] = { 'eps', 'epsi' } -suffixes['texpool'] = { 'pool' } -suffixes['PostScript header'] = { 'pro' } -suffixes['ist'] = { 'ist' } -suffixes['web'] = { 'web', 'ch' } -suffixes['cweb'] = { 'w', 'web', 'ch' } -suffixes['cmap files'] = { 'cmap' } -suffixes['lig files'] = { 'lig' } -suffixes['bitmap font'] = { } -suffixes['MetaPost support'] = { } -suffixes['TeX system documentation'] = { } -suffixes['TeX system sources'] = { } -suffixes['dvips config'] = { } -suffixes['type42 fonts'] = { } -suffixes['web2c files'] = { } -suffixes['other text files'] = { } -suffixes['other binary files'] = { } -suffixes['opentype fonts'] = { 'otf' } - -suffixes['fmt'] = { 'fmt' } -suffixes['texmfscripts'] = { 'rb','lua','py','pl' } - -suffixes['pdftex config'] = { } -suffixes['Troff fonts'] = { } - -suffixes['ls-R'] = { } - ---[[ldx-- -<p>If you wondered abou tsome of the previous mappings, how about -the next bunch:</p> ---ldx]]-- - -formats['bib'] = '' -formats['bst'] = '' -formats['mft'] = '' -formats['ist'] = '' -formats['web'] = '' -formats['cweb'] = '' -formats['MetaPost support'] = '' -formats['TeX system documentation'] = '' -formats['TeX system sources'] = '' -formats['Troff fonts'] = '' -formats['dvips config'] = '' -formats['graphic/figure'] = '' -formats['ls-R'] = '' -formats['other text files'] = '' -formats['other binary files'] = '' - -formats['gf'] = '' -formats['pk'] = '' -formats['base'] = 'MFBASES' -formats['cnf'] = '' -formats['mem'] = 'MPMEMS' -formats['mf'] = 'MFINPUTS' -formats['mfpool'] = 'MFPOOL' -formats['mppool'] = 'MPPOOL' -formats['texpool'] = 'TEXPOOL' -formats['PostScript header'] = 'TEXPSHEADERS' -formats['cmap files'] = 'CMAPFONTS' -formats['type42 fonts'] = 'T42FONTS' -formats['web2c files'] = 'WEB2C' -formats['pdftex config'] = 'PDFTEXCONFIG' -formats['texmfscripts'] = 'TEXMFSCRIPTS' -formats['bitmap font'] = '' -formats['lig files'] = 'LIGFONTS' +package.obsolete.append_libpath = appendtolibpath -- will become obsolete +package.obsolete.prepend_libpath = prependtolibpath -- will become obsolete end -- of closure @@ -11547,49 +13660,54 @@ if not modules then modules = { } end modules ['data-aux'] = { } local find = string.find +local type, next = type, next local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) -function resolvers.update_script(oldname,newname) -- oldname -> own.name, not per se a suffix +local resolvers = resolvers + +local report_scripts = logs.reporter("resolvers","scripts") + +function resolvers.updatescript(oldname,newname) -- oldname -> own.name, not per se a suffix local scriptpath = "scripts/context/lua" newname = file.addsuffix(newname,"lua") - local oldscript = resolvers.clean_path(oldname) + local oldscript = resolvers.cleanpath(oldname) if trace_locating then - logs.report("fileio","to be replaced old script %s", oldscript) + report_scripts("to be replaced old script %s", oldscript) end - local newscripts = resolvers.find_files(newname) or { } + local newscripts = resolvers.findfiles(newname) or { } if #newscripts == 0 then if trace_locating then - logs.report("fileio","unable to locate new script") + report_scripts("unable to locate new script") end else for i=1,#newscripts do - local newscript = resolvers.clean_path(newscripts[i]) + local newscript = resolvers.cleanpath(newscripts[i]) if trace_locating then - logs.report("fileio","checking new script %s", newscript) + report_scripts("checking new script %s", newscript) end if oldscript == newscript then if trace_locating then - logs.report("fileio","old and new script are the same") + report_scripts("old and new script are the same") end elseif not find(newscript,scriptpath) then if trace_locating then - logs.report("fileio","new script should come from %s",scriptpath) + report_scripts("new script should come from %s",scriptpath) end elseif not (find(oldscript,file.removesuffix(newname).."$") or find(oldscript,newname.."$")) then if trace_locating then - logs.report("fileio","invalid new script name") + report_scripts("invalid new script name") end else local newdata = io.loaddata(newscript) if newdata then if trace_locating then - logs.report("fileio","old script content replaced by new content") + report_scripts("old script content replaced by new content") end io.savedata(oldscript,newdata) break elseif trace_locating then - logs.report("fileio","unable to load new script") + report_scripts("unable to load new script") end end end @@ -11609,72 +13727,133 @@ if not modules then modules = { } end modules ['data-tmf'] = { license = "see context related readme files" } -local find, gsub, match = string.find, string.gsub, string.match -local getenv, setenv = os.getenv, os.setenv +local resolvers = resolvers --- loads *.tmf files in minimal tree roots (to be optimized and documented) +local report_tds = logs.reporter("resolvers","tds") -function resolvers.check_environment(tree) - logs.simpleline() - setenv('TMP', getenv('TMP') or getenv('TEMP') or getenv('TMPDIR') or getenv('HOME')) - setenv('TEXOS', getenv('TEXOS') or ("texmf-" .. os.platform)) - setenv('TEXPATH', gsub(tree or "tex","\/+$",'')) - setenv('TEXMFOS', getenv('TEXPATH') .. "/" .. getenv('TEXOS')) - logs.simpleline() - logs.simple("preset : TEXPATH => %s", getenv('TEXPATH')) - logs.simple("preset : TEXOS => %s", getenv('TEXOS')) - logs.simple("preset : TEXMFOS => %s", getenv('TEXMFOS')) - logs.simple("preset : TMP => %s", getenv('TMP')) - logs.simple('') -end +-- = << +-- ? ?? +-- < += +-- > =+ -function resolvers.load_environment(name) -- todo: key=value as well as lua - local f = io.open(name) - if f then - for line in f:lines() do - if find(line,"^[%%%#]") then - -- skip comment - else - local key, how, value = match(line,"^(.-)%s*([<=>%?]+)%s*(.*)%s*$") - if how then - value = gsub(value,"%%(.-)%%", function(v) return getenv(v) or "" end) - if how == "=" or how == "<<" then - setenv(key,value) - elseif how == "?" or how == "??" then - setenv(key,getenv(key) or value) - elseif how == "<" or how == "+=" then - if getenv(key) then - setenv(key,getenv(key) .. io.fileseparator .. value) - else - setenv(key,value) - end - elseif how == ">" or how == "=+" then - if getenv(key) then - setenv(key,value .. io.pathseparator .. getenv(key)) - else - setenv(key,value) - end - end - end - end +function resolvers.load_tree(tree,resolve) + if type(tree) == "string" and tree ~= "" then + + local getenv, setenv = resolvers.getenv, resolvers.setenv + + -- later might listen to the raw osenv var as well + local texos = "texmf-" .. os.platform + + local oldroot = environment.texroot + local newroot = file.collapsepath(tree) + + local newtree = file.join(newroot,texos) + local newpath = file.join(newtree,"bin") + + if not lfs.isdir(newtree) then + report_tds("no '%s' under tree %s",texos,tree) + os.exit() end - f:close() + if not lfs.isdir(newpath) then + report_tds("no '%s/bin' under tree %s",texos,tree) + os.exit() + end + + local texmfos = newtree + + environment.texroot = newroot + environment.texos = texos + environment.texmfos = texmfos + + -- Beware, we need to obey the relocatable autoparent so we + -- set TEXMFCNF to its raw value. This is somewhat tricky when + -- we run a mkii job from within. Therefore, in mtxrun, there + -- is a resolve applied when we're in mkii/kpse mode. + + setenv('SELFAUTOPARENT', newroot) + setenv('SELFAUTODIR', newtree) + setenv('SELFAUTOLOC', newpath) + setenv('TEXROOT', newroot) + setenv('TEXOS', texos) + setenv('TEXMFOS', texmfos) + setenv('TEXMFCNF', resolvers.luacnfspec, not resolve) + setenv("PATH", newpath .. io.pathseparator .. getenv("PATH")) + + report_tds("changing from root '%s' to '%s'",oldroot,newroot) + report_tds("prepending '%s' to binary path",newpath) + report_tds() end end -function resolvers.load_tree(tree) - if tree and tree ~= "" then - local setuptex = 'setuptex.tmf' - if lfs.attributes(tree, "mode") == "directory" then -- check if not nil - setuptex = tree .. "/" .. setuptex - else - setuptex = tree + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['data-lst'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- used in mtxrun, can be loaded later .. todo + +local find, concat, upper, format = string.find, table.concat, string.upper, string.format + +resolvers.listers = resolvers.listers or { } + +local resolvers = resolvers + +local report_lists = logs.reporter("resolvers","lists") + +local function tabstr(str) + if type(str) == 'table' then + return concat(str," | ") + else + return str + end +end + +function resolvers.listers.variables(pattern) + local instance = resolvers.instance + local environment = instance.environment + local variables = instance.variables + local expansions = instance.expansions + local pattern = upper(pattern or "") + local configured = { } + local order = instance.order + for i=1,#order do + for k, v in next, order[i] do + if v ~= nil and configured[k] == nil then + configured[k] = v + end end - if io.exists(setuptex) then - resolvers.check_environment(tree) - resolvers.load_environment(setuptex) + end + local env = table.fastcopy(environment) + local var = table.fastcopy(variables) + local exp = table.fastcopy(expansions) + for key, value in table.sortedpairs(configured) do + if key ~= "" and (pattern == "" or find(upper(key),pattern)) then + report_lists(key) + report_lists(" env: %s",tabstr(rawget(environment,key)) or "unset") + report_lists(" var: %s",tabstr(configured[key]) or "unset") + report_lists(" exp: %s",tabstr(expansions[key]) or "unset") + report_lists(" res: %s",resolvers.resolve(expansions[key]) or "unset") end end + instance.environment = table.fastcopy(env) + instance.variables = table.fastcopy(var) + instance.expansions = table.fastcopy(exp) +end + +function resolvers.listers.configurations(report) + local configurations = resolvers.instance.specification + local report = report or texio.write_nl + for i=1,#configurations do + report(resolvers.resolve(configurations[i])) + end end @@ -11694,9 +13873,15 @@ if not modules then modules = { } end modules ['luat-sta'] = { local gmatch, match = string.gmatch, string.match local type = type -states = states or { } -states.data = states.data or { } -states.hash = states.hash or { } +states = states or { } +local states = states + +states.data = states.data or { } +local data = states.data + +states.hash = states.hash or { } +local hash = states.hash + states.tag = states.tag or "" states.filename = states.filename or "" @@ -11706,7 +13891,7 @@ function states.save(filename,tag) io.savedata(filename, "-- generator : luat-sta.lua\n" .. "-- state tag : " .. tag .. "\n\n" .. - table.serialize(states.data[tag or states.tag] or {},true) + table.serialize(data[tag or states.tag] or {},true) ) end @@ -11714,11 +13899,11 @@ function states.load(filename,tag) states.filename = filename states.tag = tag or "whatever" states.filename = file.addsuffix(states.filename,'lus') - states.data[states.tag], states.hash[states.tag] = (io.exists(filename) and dofile(filename)) or { }, { } + data[states.tag], hash[states.tag] = (io.exists(filename) and dofile(filename)) or { }, { } end -function states.set_by_tag(tag,key,value,default,persistent) - local d, h = states.data[tag], states.hash[tag] +local function set_by_tag(tag,key,value,default,persistent) + local d, h = data[tag], hash[tag] if d then if type(d) == "table" then local dkey, hkey = key, key @@ -11738,8 +13923,10 @@ function states.set_by_tag(tag,key,value,default,persistent) end dkey, hkey = post, key end - if type(value) == nil then - value = value or default + if value == nil then + value = default + elseif value == false then + -- special case elseif persistent then value = value or d[dkey] or default else @@ -11748,144 +13935,202 @@ function states.set_by_tag(tag,key,value,default,persistent) d[dkey], h[hkey] = value, value elseif type(d) == "string" then -- weird - states.data[tag], states.hash[tag] = value, value + data[tag], hash[tag] = value, value end end end -function states.get_by_tag(tag,key,default) - local h = states.hash[tag] +local function get_by_tag(tag,key,default) + local h = hash[tag] if h and h[key] then return h[key] else - local d = states.data[tag] + local d = data[tag] if d then for k in gmatch(key,"[^%.]+") do local dk = d[k] - if dk then + if dk ~= nil then d = dk else return default end end - return d or default + if d == false then + return false + else + return d or default + end end end end +states.set_by_tag = set_by_tag +states.get_by_tag = get_by_tag + function states.set(key,value,default,persistent) - states.set_by_tag(states.tag,key,value,default,persistent) + set_by_tag(states.tag,key,value,default,persistent) end function states.get(key,default) - return states.get_by_tag(states.tag,key,default) -end - ---~ states.data.update = { ---~ ["version"] = { ---~ ["major"] = 0, ---~ ["minor"] = 1, ---~ }, ---~ ["rsync"] = { ---~ ["server"] = "contextgarden.net", ---~ ["module"] = "minimals", ---~ ["repository"] = "current", ---~ ["flags"] = "-rpztlv --stats", ---~ }, ---~ ["tasks"] = { ---~ ["update"] = true, ---~ ["make"] = true, ---~ ["delete"] = false, ---~ }, ---~ ["platform"] = { ---~ ["host"] = true, ---~ ["other"] = { ---~ ["mswin"] = false, ---~ ["linux"] = false, ---~ ["linux-64"] = false, ---~ ["osx-intel"] = false, ---~ ["osx-ppc"] = false, ---~ ["sun"] = false, ---~ }, ---~ }, ---~ ["context"] = { ---~ ["available"] = {"current", "beta", "alpha", "experimental"}, ---~ ["selected"] = "current", ---~ }, ---~ ["formats"] = { ---~ ["cont-en"] = true, ---~ ["cont-nl"] = true, ---~ ["cont-de"] = false, ---~ ["cont-cz"] = false, ---~ ["cont-fr"] = false, ---~ ["cont-ro"] = false, ---~ }, ---~ ["engine"] = { ---~ ["pdftex"] = { ---~ ["install"] = true, ---~ ["formats"] = { ---~ ["pdftex"] = true, ---~ }, ---~ }, ---~ ["luatex"] = { ---~ ["install"] = true, ---~ ["formats"] = { ---~ }, ---~ }, ---~ ["xetex"] = { ---~ ["install"] = true, ---~ ["formats"] = { ---~ ["xetex"] = false, ---~ }, ---~ }, ---~ ["metapost"] = { ---~ ["install"] = true, ---~ ["formats"] = { ---~ ["mpost"] = true, ---~ ["metafun"] = true, ---~ }, ---~ }, ---~ }, ---~ ["fonts"] = { ---~ }, ---~ ["doc"] = { ---~ }, ---~ ["modules"] = { ---~ ["f-urwgaramond"] = false, ---~ ["f-urwgothic"] = false, ---~ ["t-bnf"] = false, ---~ ["t-chromato"] = false, ---~ ["t-cmscbf"] = false, ---~ ["t-cmttbf"] = false, ---~ ["t-construction-plan"] = false, ---~ ["t-degrade"] = false, ---~ ["t-french"] = false, ---~ ["t-lettrine"] = false, ---~ ["t-lilypond"] = false, ---~ ["t-mathsets"] = false, ---~ ["t-tikz"] = false, ---~ ["t-typearea"] = false, ---~ ["t-vim"] = false, ---~ }, ---~ } - ---~ states.save("teststate", "update") ---~ states.load("teststate", "update") - ---~ print(states.get_by_tag("update","rsync.server","unknown")) ---~ states.set_by_tag("update","rsync.server","oeps") ---~ print(states.get_by_tag("update","rsync.server","unknown")) ---~ states.save("teststate", "update") ---~ states.load("teststate", "update") ---~ print(states.get_by_tag("update","rsync.server","unknown")) + return get_by_tag(states.tag,key,default) +end + + + + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['luat-fmt'] = { + version = 1.001, + comment = "companion to mtxrun", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + + +local format = string.format + +local report_format = logs.reporter("resolvers","formats") + +-- helper for mtxrun + +local quoted = string.quoted + +local function primaryflags() -- not yet ok + local trackers = environment.argument("trackers") + local directives = environment.argument("directives") + local flags = "" + if trackers and trackers ~= "" then + flags = flags .. "--trackers=" .. quoted(trackers) + end + if directives and directives ~= "" then + flags = flags .. "--directives=" .. quoted(directives) + end + return flags +end + +function environment.make_format(name) + -- change to format path (early as we need expanded paths) + local olddir = lfs.currentdir() + local path = caches.getwritablepath("formats") or "" -- maybe platform + if path ~= "" then + lfs.chdir(path) + end + report_format("format path: %s",lfs.currentdir()) + -- check source file + local texsourcename = file.addsuffix(name,"mkiv") + local fulltexsourcename = resolvers.findfile(texsourcename,"tex") or "" + if fulltexsourcename == "" then + texsourcename = file.addsuffix(name,"tex") + fulltexsourcename = resolvers.findfile(texsourcename,"tex") or "" + end + if fulltexsourcename == "" then + report_format("no tex source file with name: %s (mkiv or tex)",name) + lfs.chdir(olddir) + return + else + report_format("using tex source file: %s",fulltexsourcename) + end + local texsourcepath = dir.expandname(file.dirname(fulltexsourcename)) -- really needed + -- check specification + local specificationname = file.replacesuffix(fulltexsourcename,"lus") + local fullspecificationname = resolvers.findfile(specificationname,"tex") or "" + if fullspecificationname == "" then + specificationname = file.join(texsourcepath,"context.lus") + fullspecificationname = resolvers.findfile(specificationname,"tex") or "" + end + if fullspecificationname == "" then + report_format("unknown stub specification: %s",specificationname) + lfs.chdir(olddir) + return + end + local specificationpath = file.dirname(fullspecificationname) + -- load specification + local usedluastub = nil + local usedlualibs = dofile(fullspecificationname) + if type(usedlualibs) == "string" then + usedluastub = file.join(file.dirname(fullspecificationname),usedlualibs) + elseif type(usedlualibs) == "table" then + report_format("using stub specification: %s",fullspecificationname) + local texbasename = file.basename(name) + local luastubname = file.addsuffix(texbasename,"lua") + local lucstubname = file.addsuffix(texbasename,"luc") + -- pack libraries in stub + report_format("creating initialization file: %s",luastubname) + utilities.merger.selfcreate(usedlualibs,specificationpath,luastubname) + -- compile stub file (does not save that much as we don't use this stub at startup any more) + local strip = resolvers.booleanvariable("LUACSTRIP", true) + if utilities.lua.compile(luastubname,lucstubname) and lfs.isfile(lucstubname) then + report_format("using compiled initialization file: %s",lucstubname) + usedluastub = lucstubname + else + report_format("using uncompiled initialization file: %s",luastubname) + usedluastub = luastubname + end + else + report_format("invalid stub specification: %s",fullspecificationname) + lfs.chdir(olddir) + return + end + -- generate format + local command = format("luatex --ini %s --lua=%s %s %sdump",primaryflags(),quoted(usedluastub),quoted(fulltexsourcename),os.platform == "unix" and "\\\\" or "\\") + report_format("running command: %s\n",command) + os.spawn(command) + -- remove related mem files + local pattern = file.removesuffix(file.basename(usedluastub)).."-*.mem" + -- report_format("removing related mplib format with pattern '%s'", pattern) + local mp = dir.glob(pattern) + if mp then + for i=1,#mp do + local name = mp[i] + report_format("removing related mplib format %s", file.basename(name)) + os.remove(name) + end + end + lfs.chdir(olddir) +end + +function environment.run_format(name,data,more) + -- hm, rather old code here; we can now use the file.whatever functions + if name and name ~= "" then + local barename = file.removesuffix(name) + local fmtname = caches.getfirstreadablefile(file.addsuffix(barename,"fmt"),"formats") + if fmtname == "" then + fmtname = resolvers.findfile(file.addsuffix(barename,"fmt")) or "" + end + fmtname = resolvers.cleanpath(fmtname) + if fmtname == "" then + report_format("no format with name: %s",name) + else + local barename = file.removesuffix(name) -- expanded name + local luaname = file.addsuffix(barename,"luc") + if not lfs.isfile(luaname) then + luaname = file.addsuffix(barename,"lua") + end + if not lfs.isfile(luaname) then + report_format("using format name: %s",fmtname) + report_format("no luc/lua with name: %s",barename) + else + local command = format("luatex %s --fmt=%s --lua=%s %s %s",primaryflags(),quoted(barename),quoted(luaname),quoted(data),more ~= "" and quoted(more) or "") + report_format("running command: %s",command) + os.spawn(command) + end + end + end +end end -- of closure -- end library merge -own = { } -- not local +own = { } -- not local, might change + +own.libs = { -- order can be made better -own.libs = { -- todo: check which ones are really needed 'l-string.lua', 'l-lpeg.lua', 'l-table.lua', @@ -11898,39 +14143,55 @@ own.libs = { -- todo: check which ones are really needed 'l-url.lua', 'l-dir.lua', 'l-boolean.lua', + 'l-unicode.lua', 'l-math.lua', --- 'l-unicode.lua', --- 'l-tex.lua', - 'l-utils.lua', - 'l-aux.lua', --- 'l-xml.lua', - 'trac-tra.lua', + + 'util-tab.lua', + 'util-sto.lua', + 'util-mrg.lua', + 'util-lua.lua', + 'util-prs.lua', + 'util-fmt.lua', + 'util-deb.lua', + + 'trac-inf.lua', + 'trac-set.lua', + 'trac-log.lua', + 'trac-pro.lua', + + 'luat-env.lua', -- can come before inf (as in mkiv) + 'lxml-tab.lua', 'lxml-lpt.lua', --- 'lxml-ent.lua', + -- 'lxml-ent.lua', 'lxml-mis.lua', 'lxml-aux.lua', 'lxml-xml.lua', - 'luat-env.lua', - 'trac-inf.lua', - 'trac-log.lua', - 'data-res.lua', + + 'data-ini.lua', + 'data-exp.lua', + 'data-env.lua', 'data-tmp.lua', + 'data-met.lua', + 'data-res.lua', 'data-pre.lua', 'data-inp.lua', 'data-out.lua', + 'data-fil.lua', 'data-con.lua', 'data-use.lua', -- 'data-tex.lua', -- 'data-bin.lua', 'data-zip.lua', + 'data-tre.lua', 'data-crl.lua', 'data-lua.lua', - 'data-kps.lua', -- so that we can replace kpsewhich 'data-aux.lua', -- updater - 'data-tmf.lua', -- tree files - -- needed ? - 'luat-sta.lua', -- states + 'data-tmf.lua', + 'data-lst.lua', + + 'luat-sta.lua', + 'luat-fmt.lua', } -- We need this hack till luatex is fixed. @@ -11943,35 +14204,62 @@ end -- End of hack. -own.name = (environment and environment.ownname) or arg[0] or 'luatools.lua' +local format, gsub, gmatch, match = string.format, string.gsub, string.gmatch, string.match +local concat = table.concat +own.name = (environment and environment.ownname) or arg[0] or 'mtxrun.lua' +own.path = gsub(match(own.name,"^(.+)[\\/].-$") or ".","\\","/") + +local ownpath, owntree = own.path, environment and environment.ownpath or own.path + +own.list = { + '.', + ownpath , + ownpath .. "/../sources", -- HH's development path + owntree .. "/../../texmf-local/tex/context/base", + owntree .. "/../../texmf-context/tex/context/base", + owntree .. "/../../texmf-dist/tex/context/base", + owntree .. "/../../texmf/tex/context/base", + owntree .. "/../../../texmf-local/tex/context/base", + owntree .. "/../../../texmf-context/tex/context/base", + owntree .. "/../../../texmf-dist/tex/context/base", + owntree .. "/../../../texmf/tex/context/base", +} -own.path = string.match(own.name,"^(.+)[\\/].-$") or "." -own.list = { '.' } -if own.path ~= '.' then - table.insert(own.list,own.path) -end -table.insert(own.list,own.path.."/../../texmf-dist/tex/context/base") -table.insert(own.list,own.path.."/../../../tex/context/base") -table.insert(own.list,own.path.."/mtx") -table.insert(own.list,own.path.."/../sources") +if own.path == "." then table.remove(own.list,1) end local function locate_libs() - for _, lib in pairs(own.libs) do - for _, pth in pairs(own.list) do - local filename = string.gsub(pth .. "/" .. lib,"\\","/") + for l=1,#own.libs do + local lib = own.libs[l] + for p =1,#own.list do + local pth = own.list[p] + local filename = pth .. "/" .. lib + local found = lfs.isfile(filename) + if found then + package.path = package.path .. ";" .. pth .. "/?.lua" -- in case l-* does a require + return pth + end + end + end +end + +local function load_libs() + local found = locate_libs() + if found then + for l=1,#own.libs do + local filename = found .. "/" .. own.libs[l] local codeblob = loadfile(filename) if codeblob then codeblob() - own.list = { pth } -- speed up te search - break end end + else + resolvers = nil end end if not resolvers then - locate_libs() + load_libs() end if not resolvers then @@ -11983,20 +14271,56 @@ if not resolvers then os.exit() end -logs.setprogram('MTXrun',"TDS Runner Tool 1.24",environment.arguments["verbose"] or false) +-- verbosity -local instance = resolvers.reset() +local e_verbose = environment.arguments["verbose"] -local trackspec = environment.argument("trackers") or environment.argument("track") +if e_verbose then + trackers.enable("resolvers.locating") +end + +-- some common flags (also passed through environment) + +local e_silent = environment.argument("silent") +local e_noconsole = environment.argument("noconsole") + +local e_trackers = environment.argument("trackers") +local e_directives = environment.argument("directives") +local e_experiments = environment.argument("experiments") + +if e_silent == true then + e_silent = "*" +end + +if type(e_silent) == "string" then + if type(e_directives) == "string" then + e_directives = format("%s,logs.blocked={%s}",e_directives,e_silent) + else + e_directives = format("logs.blocked={%s}",e_silent) + end +end -if trackspec then - trackers.enable(trackspec) +if e_noconsole then + if type(e_directives) == "string" then + e_directives = format("%s,logs.target=file",e_directives) + else + e_directives = format("logs.target=file") + end end -runners = runners or { } -- global -messages = messages or { } +if e_trackers then trackers .enable(e_trackers) end +if e_directives then directives .enable(e_directives) end +if e_experiments then experiments.enable(e_experiments) end -messages.help = [[ +if not environment.trackers then environment.trackers = e_trackers end +if not environment.directives then environment.directives = e_directives end +if not environment.experiments then environment.experiments = e_experiments end + +-- + +local instance = resolvers.reset() + +local helpinfo = [[ --script run an mtx script (lua prefered method) (--noquotes), no script gives list --execute run a script or program (texmfstart method) (--noquotes) --resolve resolve prefixed arguments @@ -12011,15 +14335,14 @@ messages.help = [[ --ifchanged=filename only execute when given file has changed (md checksum) --iftouched=old,new only execute when given file has changed (time stamp) ---make create stubs for (context related) scripts ---remove remove stubs (context related) scripts +--makestubs create stubs for (context related) scripts +--removestubs remove stubs (context related) scripts --stubpath=binpath paths where stubs wil be written --windows create windows (mswin) stubs --unix create unix (linux) stubs --verbose give a bit more info --trackers=list enable given trackers ---engine=str target engine --progname=str format or backend --edit launch editor with found file @@ -12032,8 +14355,35 @@ messages.help = [[ --forcekpse force using kpse (handy when no mkiv and cache installed but less functionality) --prefixes show supported prefixes + +--generate generate file database + +--variables show configuration variables +--configurations show configuration order + +--expand-braces expand complex variable +--expand-path expand variable (resolve paths) +--expand-var expand variable (resolve references) +--show-path show path expansion of ... +--var-value report value of variable +--find-file report file location +--find-path report path of file + +--pattern=str filter variables ]] +local application = logs.application { + name = "mtxrun", + banner = "ConTeXt TDS Runner Tool 1.30", + helpinfo = helpinfo, +} + +local report = application.report + +messages = messages or { } -- for the moment + +runners = runners or { } -- global (might become local) + runners.applications = { ["lua"] = "luatex --luaonly", ["luc"] = "luatex --luaonly", @@ -12056,7 +14406,7 @@ runners.registered = { makempy = { 'makempy.pl', true }, mptopdf = { 'mptopdf.pl', true }, pstopdf = { 'pstopdf.rb', true }, -- converts ps (and some more) images, does some cleaning (replaced) --- examplex = { 'examplex.rb', false }, + -- examplex = { 'examplex.rb', false }, concheck = { 'concheck.rb', false }, runtools = { 'runtools.rb', true }, textools = { 'textools.rb', true }, @@ -12065,9 +14415,9 @@ runners.registered = { rlxtools = { 'rlxtools.rb', true }, pdftools = { 'pdftools.rb', true }, mpstools = { 'mpstools.rb', true }, --- exatools = { 'exatools.rb', true }, + -- exatools = { 'exatools.rb', true }, xmltools = { 'xmltools.rb', true }, --- luatools = { 'luatools.lua', true }, + -- luatools = { 'luatools.lua', true }, mtxtools = { 'mtxtools.rb', true }, pdftrimwhite = { 'pdftrimwhite.pl', false } } @@ -12086,45 +14436,39 @@ end function runners.prepare() local checkname = environment.argument("ifchanged") - if checkname and checkname ~= "" then + if type(checkname) == "string" and checkname ~= "" then local oldchecksum = file.loadchecksum(checkname) local newchecksum = file.checksum(checkname) if oldchecksum == newchecksum then - logs.simple("file '%s' is unchanged",checkname) + if e_verbose then + report("file '%s' is unchanged",checkname) + end return "skip" - else - logs.simple("file '%s' is changed, processing started",checkname) + elseif e_verbose then + report("file '%s' is changed, processing started",checkname) end file.savechecksum(checkname) end - local oldname, newname = string.split(environment.argument("iftouched") or "", ",") - if oldname and newname and oldname ~= "" and newname ~= "" then - if not file.needs_updating(oldname,newname) then - logs.simple("file '%s' and '%s' have same age",oldname,newname) - return "skip" - else - logs.simple("file '%s' is older than '%s'",oldname,newname) - end - end - local tree = environment.argument('tree') or "" - if environment.argument('autotree') then - tree = os.getenv('TEXMFSTART_TREE') or os.getenv('TEXMFSTARTTREE') or tree - end - if tree and tree ~= "" then - resolvers.load_tree(tree) - end - local env = environment.argument('environment') or "" - if env and env ~= "" then - for _,e in pairs(string.split(env)) do - -- maybe force suffix when not given - resolvers.load_tree(e) + local touchname = environment.argument("iftouched") + if type(touchname) == "string" and touchname ~= "" then + local oldname, newname = string.split(touchname, ",") + if oldname and newname and oldname ~= "" and newname ~= "" then + if not file.needs_updating(oldname,newname) then + if e_verbose then + report("file '%s' and '%s' have same age",oldname,newname) + end + return "skip" + elseif e_verbose then + report("file '%s' is older than '%s'",oldname,newname) + end end end local runpath = environment.argument("path") - if runpath and not lfs.chdir(runpath) then - logs.simple("unable to change to path '%s'",runpath) + if type(runpath) == "string" and not lfs.chdir(runpath) then + report("unable to change to path '%s'",runpath) return "error" end + runners.prepare = function() end return "run" end @@ -12137,8 +14481,6 @@ function runners.execute_script(fullname,internal,nosplit) elseif state == 'skip' then return true elseif state == "run" then - instance.progname = environment.argument("progname") or instance.progname - instance.format = environment.argument("format") or instance.format local path, name, suffix, result = file.dirname(fullname), file.basename(fullname), file.extname(fullname), "" if path ~= "" then result = fullname @@ -12155,21 +14497,21 @@ function runners.execute_script(fullname,internal,nosplit) if suffix == "" then -- loop over known suffixes for _,s in pairs(runners.suffixes) do - result = resolvers.find_file(name .. "." .. s, 'texmfscripts') + result = resolvers.findfile(name .. "." .. s, 'texmfscripts') if result ~= "" then break end end elseif runners.applications[suffix] then - result = resolvers.find_file(name, 'texmfscripts') + result = resolvers.findfile(name, 'texmfscripts') else -- maybe look on path - result = resolvers.find_file(name, 'other text files') + result = resolvers.findfile(name, 'other text files') end end if result and result ~= "" then if not no_split then - local before, after = environment.split_arguments(fullname) -- already done + local before, after = environment.splitarguments(fullname) -- already done environment.arguments_before, environment.arguments_after = before, after end if internal then @@ -12181,12 +14523,12 @@ function runners.execute_script(fullname,internal,nosplit) if binary and binary ~= "" then result = binary .. " " .. result end - local command = result .. " " .. environment.reconstruct_commandline(environment.arguments_after,noquote) - if logs.verbose then - logs.simpleline() - logs.simple("executing: %s",command) - logs.simpleline() - logs.simpleline() + local command = result .. " " .. environment.reconstructcommandline(environment.arguments_after,noquote) + if e_verbose then + report() + report("executing: %s",command) + report() + report() io.flush() end -- no os.exec because otherwise we get the wrong return value @@ -12196,14 +14538,14 @@ function runners.execute_script(fullname,internal,nosplit) else if binary then binary = file.addsuffix(binary,os.binsuffix) - for p in string.gmatch(os.getenv("PATH"),"[^"..io.pathseparator.."]+") do + for p in gmatch(os.getenv("PATH"),"[^"..io.pathseparator.."]+") do if lfs.isfile(file.join(p,binary)) then return false end end - logs.simpleline() - logs.simple("This script needs '%s' which seems not to be installed.",binary) - logs.simpleline() + report() + report("This script needs '%s' which seems not to be installed.",binary) + report() end return false end @@ -12223,14 +14565,15 @@ function runners.execute_program(fullname) elseif state == 'skip' then return true elseif state == "run" then - local before, after = environment.split_arguments(fullname) - environment.initialize_arguments(after) + local before, after = environment.splitarguments(fullname) + for k=1,#after do after[k] = resolvers.resolve(after[k]) end + environment.initializearguments(after) fullname = fullname:gsub("^bin:","") - local command = fullname .. " " .. (environment.reconstruct_commandline(after or "",noquote) or "") - logs.simpleline() - logs.simple("executing: %s",command) - logs.simpleline() - logs.simpleline() + local command = fullname .. " " .. (environment.reconstructcommandline(after or "",noquote) or "") + report() + report("executing: %s",command) + report() + report() io.flush() local code = os.exec(command) -- (fullname,unpack(after)) does not work / maybe spawn return code == 0 @@ -12239,7 +14582,7 @@ function runners.execute_program(fullname) return false end --- the --usekpse flag will fallback on kpse (hm, we can better update mtx-stubs) +-- the --usekpse flag will fallback (not default) on kpse (hm, we can better update mtx-stubs) local windows_stub = '@echo off\013\010setlocal\013\010set ownpath=%%~dp0%%\013\010texlua "%%ownpath%%mtxrun.lua" --usekpse --execute %s %%*\013\010endlocal\013\010' local unix_stub = '#!/bin/sh\010mtxrun --usekpse --execute %s \"$@\"\010' @@ -12258,22 +14601,22 @@ function runners.handle_stubs(create) for _,v in pairs(runners.registered) do local name, doit = v[1], v[2] if doit then - local base = string.gsub(file.basename(name), "%.(.-)$", "") + local base = gsub(file.basename(name), "%.(.-)$", "") if create then if windows then - io.savedata(file.join(stubpath,base..".bat"),string.format(windows_stub,name)) - logs.simple("windows stub for '%s' created",base) + io.savedata(file.join(stubpath,base..".bat"),format(windows_stub,name)) + report("windows stub for '%s' created",base) end if unix then - io.savedata(file.join(stubpath,base),string.format(unix_stub,name)) - logs.simple("unix stub for '%s' created",base) + io.savedata(file.join(stubpath,base),format(unix_stub,name)) + report("unix stub for '%s' created",base) end else if windows and (os.remove(file.join(stubpath,base..'.bat')) or os.remove(file.join(stubpath,base..'.cmd'))) then - logs.simple("windows stub for '%s' removed", base) + report("windows stub for '%s' removed", base) end if unix and (os.remove(file.join(stubpath,base)) or os.remove(file.join(stubpath,base..'.sh'))) then - logs.simple("unix stub for '%s' removed",base) + report("unix stub for '%s' removed",base) end end end @@ -12289,7 +14632,7 @@ end function runners.locate_file(filename) -- differs from texmfstart where locate appends .com .exe .bat ... todo if filename and filename ~= "" then - runners.report_location(resolvers.find_given_file(filename)) + runners.report_location(resolvers.findgivenfile(filename)) end end @@ -12298,28 +14641,28 @@ function runners.locate_platform() end function runners.report_location(result) - if logs.verbose then - logs.simpleline() + if e_verbose then + reportline() if result and result ~= "" then - logs.simple(result) + report(result) else - logs.simple("not found") + report("not found") end else io.write(result) end end -function runners.edit_script(filename) -- we assume that vim is present on most systems - local editor = os.getenv("MTXRUN_EDITOR") or os.getenv("TEXMFSTART_EDITOR") or os.getenv("EDITOR") or 'vim' +function runners.edit_script(filename) -- we assume that gvim is present on most systems (todo: also in cnf file) + local editor = os.getenv("MTXRUN_EDITOR") or os.getenv("TEXMFSTART_EDITOR") or os.getenv("EDITOR") or 'gvim' local rest = resolvers.resolve(filename) if rest ~= "" then local command = editor .. " " .. rest - if logs.verbose then - logs.simpleline() - logs.simple("starting editor: %s",command) - logs.simple_line() - logs.simple_line() + if e_verbose then + report() + report("starting editor: %s",command) + report() + report() end os.launch(command) end @@ -12362,33 +14705,33 @@ end function runners.launch_file(filename) instance.allresults = true - logs.setverbose(true) + trackers.enable("resolvers.locating") local pattern = environment.arguments["pattern"] if not pattern or pattern == "" then pattern = filename end if not pattern or pattern == "" then - logs.simple("provide name or --pattern=") + report("provide name or --pattern=") else - local t = resolvers.find_files(pattern) + local t = resolvers.findfiles(pattern) if not t or #t == 0 then - t = resolvers.find_files("*/" .. pattern) + t = resolvers.findfiles("*/" .. pattern) end if not t or #t == 0 then - t = resolvers.find_files("*/" .. pattern .. "*") + t = resolvers.findfiles("*/" .. pattern .. "*") end if t and #t > 0 then if environment.arguments["all"] then for _, v in pairs(t) do - logs.simple("launching %s", v) + report("launching %s", v) resolvers.launch(v) end else - logs.simple("launching %s", t[1]) + report("launching %s", t[1]) resolvers.launch(t[1]) end else - logs.simple("no match for %s", pattern) + report("no match for %s", pattern) end end end @@ -12421,33 +14764,45 @@ function runners.find_mtx_script(filename) local mtxprefix = (filename:find("^mtx%-") and "") or "mtx-" -- context namespace, mtx-<filename> fullname = mtxprefix .. filename - fullname = found(fullname) or resolvers.find_file(fullname) + fullname = found(fullname) or resolvers.findfile(fullname) if fullname and fullname ~= "" then return fullname end -- context namespace, mtx-<filename>s fullname = mtxprefix .. basename .. "s" .. "." .. suffix - fullname = found(fullname) or resolvers.find_file(fullname) + fullname = found(fullname) or resolvers.findfile(fullname) if fullname and fullname ~= "" then return fullname end -- context namespace, mtx-<filename minus trailing s> fullname = mtxprefix .. basename:gsub("s$","") .. "." .. suffix - fullname = found(fullname) or resolvers.find_file(fullname) + fullname = found(fullname) or resolvers.findfile(fullname) if fullname and fullname ~= "" then return fullname end -- context namespace, just <filename> - fullname = resolvers.find_file(filename) + fullname = resolvers.findfile(filename) return fullname end -function runners.execute_ctx_script(filename) +function runners.register_arguments(...) + local arguments = environment.arguments_after + local passedon = { ... } + for i=#passedon,1,-1 do + local pi = passedon[i] + if pi then + table.insert(arguments,1,pi) + end + end +end + +function runners.execute_ctx_script(filename,...) + runners.register_arguments(...) local arguments = environment.arguments_after local fullname = runners.find_mtx_script(filename) or "" if file.extname(fullname) == "cld" then -- handy in editors where we force --autopdf - logs.simple("running cld script: %s",filename) + report("running cld script: %s",filename) table.insert(arguments,1,fullname) table.insert(arguments,"--autopdf") fullname = runners.find_mtx_script("context") or "" @@ -12455,7 +14810,7 @@ function runners.execute_ctx_script(filename) -- retry after generate but only if --autogenerate if fullname == "" and environment.argument("autogenerate") then -- might become the default instance.renewcache = true - logs.setverbose(true) + trackers.enable("resolvers.locating") resolvers.load() -- fullname = runners.find_mtx_script(filename) or "" @@ -12470,7 +14825,7 @@ function runners.execute_ctx_script(filename) elseif state == "run" then -- load and save ... kind of undocumented arg = { } for _,v in pairs(arguments) do arg[#arg+1] = resolvers.resolve(v) end - environment.initialize_arguments(arg) + environment.initializearguments(arg) local loadname = environment.arguments['load'] if loadname then if type(loadname) ~= "string" then loadname = file.basename(fullname) end @@ -12478,8 +14833,8 @@ function runners.execute_ctx_script(filename) runners.load_script_session(loadname) end filename = environment.files[1] - if logs.verbose then - logs.simple("using script: %s\n",fullname) + if e_verbose then + report("using script: %s\n",fullname) end environment.ownscript = fullname dofile(fullname) @@ -12495,44 +14850,42 @@ function runners.execute_ctx_script(filename) return true end else - -- logs.setverbose(true) if filename == "" or filename == "help" then - local context = resolvers.find_file("mtx-context.lua") - logs.setverbose(true) + local context = resolvers.findfile("mtx-context.lua") + trackers.enable("resolvers.locating") if context ~= "" then - local result = dir.glob((string.gsub(context,"mtx%-context","mtx-*"))) -- () needed + local result = dir.glob((gsub(context,"mtx%-context","mtx-*"))) -- () needed local valid = { } table.sort(result) for i=1,#result do local scriptname = result[i] - local scriptbase = string.match(scriptname,".*mtx%-([^%-]-)%.lua") + local scriptbase = match(scriptname,".*mtx%-([^%-]-)%.lua") if scriptbase then local data = io.loaddata(scriptname) - local banner, version = string.match(data,"[\n\r]logs%.extendbanner%s*%(%s*[\"\']([^\n\r]+)%s*(%d+%.%d+)") + local banner, version = match(data,"[\n\r]logs%.extendbanner%s*%(%s*[\"\']([^\n\r]+)%s*(%d+%.%d+)") if banner then valid[#valid+1] = { scriptbase, version, banner } end end end if #valid > 0 then - logs.reportbanner() - logs.reportline() - logs.simple("no script name given, known scripts:") - logs.simple() + application.identify() + report("no script name given, known scripts:") + report() for k=1,#valid do local v = valid[k] - logs.simple("%-12s %4s %s",v[1],v[2],v[3]) + report("%-12s %4s %s",v[1],v[2],v[3]) end end else - logs.simple("no script name given") + report("no script name given") end else filename = file.addsuffix(filename,"lua") if file.is_qualified_path(filename) then - logs.simple("unknown script '%s'",filename) + report("unknown script '%s'",filename) else - logs.simple("unknown script '%s' or 'mtx-%s'",filename,filename) + report("unknown script '%s' or 'mtx-%s'",filename,filename) end end return false @@ -12540,9 +14893,9 @@ function runners.execute_ctx_script(filename) end function runners.prefixes() - logs.reportbanner() - logs.reportline() - logs.simple(table.concat(resolvers.allprefixes(true)," ")) + application.identify() + report() + report(concat(resolvers.allprefixes(true)," ")) end function runners.timedrun(filename) -- just for me @@ -12562,13 +14915,11 @@ end local filename = environment.files[1] or "" local ok = true -local before, after = environment.split_arguments(filename) +local before, after = environment.splitarguments(filename) environment.arguments_before, environment.arguments_after = before, after -environment.initialize_arguments(before) +environment.initializearguments(before) -instance.engine = environment.argument("engine") or 'luatex' -instance.progname = environment.argument("progname") or 'context' -instance.lsrmode = environment.argument("lsr") or false +instance.lsrmode = environment.argument("lsr") or false -- maybe the unset has to go to this level @@ -12576,36 +14927,40 @@ local is_mkii_stub = runners.registered[file.removesuffix(file.basename(filename if environment.argument("usekpse") or environment.argument("forcekpse") or is_mkii_stub then + resolvers.load_tree(environment.argument('tree'),true) -- force resolve of TEXMFCNF + os.setenv("engine","") os.setenv("progname","") local remapper = { - otf = "opentype fonts", - ttf = "truetype fonts", - ttc = "truetype fonts", - pfb = "type1 fonts", + otf = "opentype fonts", + ttf = "truetype fonts", + ttc = "truetype fonts", + pfb = "type1 fonts", other = "other text files", } + local progname = environment.argument("progname") or 'context' + local function kpse_initialized() texconfig.kpse_init = true local t = os.clock() - local k = kpse.original.new("luatex",instance.progname) + local k = kpse.original.new("luatex",progname) local dummy = k:find_file("mtxrun.lua") -- so that we're initialized - logs.simple("kpse fallback with progname '%s' initialized in %s seconds",instance.progname,os.clock()-t) + report("kpse fallback with progname '%s' initialized in %s seconds",progname,os.clock()-t) kpse_initialized = function() return k end return k end - local find_file = resolvers.find_file - local show_path = resolvers.show_path + local findfile = resolvers.findfile + local showpath = resolvers.showpath if environment.argument("forcekpse") then - function resolvers.find_file(name,kind) - return (kpse_initialized():find_file(resolvers.clean_path(name),(kind ~= "" and (remapper[kind] or kind)) or "tex") or "") or "" + function resolvers.findfile(name,kind) + return (kpse_initialized():find_file(resolvers.cleanpath(name),(kind ~= "" and (remapper[kind] or kind)) or "tex") or "") or "" end - function resolvers.show_path(name) + function resolvers.showpath(name) return (kpse_initialized():show_path(name)) or "" end @@ -12613,16 +14968,16 @@ if environment.argument("usekpse") or environment.argument("forcekpse") or is_mk resolvers.load() - function resolvers.find_file(name,kind) - local found = find_file(name,kind) or "" + function resolvers.findfile(name,kind) + local found = findfile(name,kind) or "" if found ~= "" then return found else - return (kpse_initialized():find_file(resolvers.clean_path(name),(kind ~= "" and (remapper[kind] or kind)) or "tex") or "") or "" + return (kpse_initialized():find_file(resolvers.cleanpath(name),(kind ~= "" and (remapper[kind] or kind)) or "tex") or "") or "" end end - function resolvers.show_path(name) - local found = show_path(name) or "" + function resolvers.showpath(name) + local found = showpath(name) or "" if found ~= "" then return found else @@ -12632,80 +14987,317 @@ if environment.argument("usekpse") or environment.argument("forcekpse") or is_mk end + function runners.loadbase() + end + else - resolvers.load() + function runners.loadbase(...) + if not resolvers.load(...) then + report("forcing cache reload") + instance.renewcache = true + trackers.enable("resolvers.locating") + if not resolvers.load(...) then + report("the resolver databases are not present or outdated") + end + end + end + + resolvers.load_tree(environment.argument('tree')) end + if environment.argument("selfmerge") then + -- embed used libraries - utils.merger.selfmerge(own.name,own.libs,own.list) + + runners.loadbase() + local found = locate_libs() + if found then + utilities.merger.selfmerge(own.name,own.libs,{ found }) + end + elseif environment.argument("selfclean") then + -- remove embedded libraries - utils.merger.selfclean(own.name) + + runners.loadbase() + utilities.merger.selfclean(own.name) + elseif environment.argument("selfupdate") then - logs.setverbose(true) - resolvers.update_script(own.name,"mtxrun") + + runners.loadbase() + trackers.enable("resolvers.locating") + resolvers.updatescript(own.name,"mtxrun") + elseif environment.argument("ctxlua") or environment.argument("internal") then + -- run a script by loading it (using libs) + + runners.loadbase() ok = runners.execute_script(filename,true) + elseif environment.argument("script") or environment.argument("scripts") then + -- run a script by loading it (using libs), pass args + + runners.loadbase() if is_mkii_stub then - -- execute mkii script ok = runners.execute_script(filename,false,true) else ok = runners.execute_ctx_script(filename) end + elseif environment.argument("execute") then + -- execute script + + runners.loadbase() ok = runners.execute_script(filename) + elseif environment.argument("direct") then + -- equals bin: + + runners.loadbase() ok = runners.execute_program(filename) + elseif environment.argument("edit") then + -- edit file + + runners.loadbase() runners.edit_script(filename) + elseif environment.argument("launch") then + + runners.loadbase() runners.launch_file(filename) -elseif environment.argument("make") then - -- make stubs + +elseif environment.argument("makestubs") then + + -- make stubs (depricated) + runners.handle_stubs(true) -elseif environment.argument("remove") then - -- remove stub + +elseif environment.argument("removestubs") then + + -- remove stub (depricated) + + runners.loadbase() runners.handle_stubs(false) + elseif environment.argument("resolve") then + -- resolve string + + runners.loadbase() runners.resolve_string(filename) + elseif environment.argument("locate") then + -- locate file + + runners.loadbase() runners.locate_file(filename) -elseif environment.argument("platform")then + +elseif environment.argument("platform") or environment.argument("show-platform") then + -- locate platform + + runners.loadbase() runners.locate_platform() + elseif environment.argument("prefixes") then + + runners.loadbase() runners.prefixes() + elseif environment.argument("timedrun") then + -- locate platform + + runners.loadbase() runners.timedrun(filename) + +elseif environment.argument("variables") or environment.argument("show-variables") or environment.argument("expansions") or environment.argument("show-expansions") then + + -- luatools: runners.execute_ctx_script("mtx-base","--expansions",filename) + + resolvers.load("nofiles") + resolvers.listers.variables(environment.argument("pattern")) + +elseif environment.argument("configurations") or environment.argument("show-configurations") then + + -- luatools: runners.execute_ctx_script("mtx-base","--configurations",filename) + + resolvers.load("nofiles") + resolvers.listers.configurations() + +elseif environment.argument("find-file") then + + -- luatools: runners.execute_ctx_script("mtx-base","--find-file",filename) + + resolvers.load() + local e_pattern = environment.argument("pattern") + local e_format = environment.arguments("format") + if not e_pattern then + runners.register_arguments(filename) + environment.initializearguments(environment.arguments_after) + resolvers.dowithfilesandreport(resolvers.findfiles,environment.files,e_format) + elseif type(e_pattern) == "string" then + instance.allresults = true -- brrrr + resolvers.dowithfilesandreport(resolvers.findfiles,{ e_pattern }, e_format) + end + +elseif environment.argument("find-path") then + + -- luatools: runners.execute_ctx_script("mtx-base","--find-path",filename) + + resolvers.load() + local path = resolvers.findpath(filename, instance.my_format) + if e_verbose then + report(path) + else + print(path) + end + +elseif environment.argument("expand-braces") then + + -- luatools: runners.execute_ctx_script("mtx-base","--expand-braces",filename) + + resolvers.load("nofiles") + runners.register_arguments(filename) + environment.initializearguments(environment.arguments_after) + resolvers.dowithfilesandreport(resolvers.expandbraces, environment.files) + +elseif environment.argument("expand-path") then + + -- luatools: runners.execute_ctx_script("mtx-base","--expand-path",filename) + + resolvers.load("nofiles") + runners.register_arguments(filename) + environment.initializearguments(environment.arguments_after) + resolvers.dowithfilesandreport(resolvers.expandpath, environment.files) + +elseif environment.argument("expand-var") or environment.argument("expand-variable") then + + -- luatools: runners.execute_ctx_script("mtx-base","--expand-var",filename) + + resolvers.load("nofiles") + runners.register_arguments(filename) + environment.initializearguments(environment.arguments_after) + resolvers.dowithfilesandreport(resolvers.expansion, environment.files) + +elseif environment.argument("show-path") or environment.argument("path-value") then + + -- luatools: runners.execute_ctx_script("mtx-base","--show-path",filename) + + resolvers.load("nofiles") + runners.register_arguments(filename) + environment.initializearguments(environment.arguments_after) + resolvers.dowithfilesandreport(resolvers.showpath, environment.files) + +elseif environment.argument("var-value") or environment.argument("show-value") then + + -- luatools: runners.execute_ctx_script("mtx-base","--show-value",filename) + + resolvers.load("nofiles") + runners.register_arguments(filename) + environment.initializearguments(environment.arguments_after) + resolvers.dowithfilesandreport(resolvers.variable,environment.files) + +elseif environment.argument("format-path") then + + -- luatools: runners.execute_ctx_script("mtx-base","--format-path",filename) + + resolvers.load() + report(caches.getwritablepath("format")) + +elseif environment.argument("pattern") then + + -- luatools + + runners.execute_ctx_script("mtx-base","--pattern='" .. environment.argument("pattern") .. "'",filename) + +elseif environment.argument("generate") then + + -- luatools + + instance.renewcache = true + trackers.enable("resolvers.locating") + resolvers.load() + +elseif environment.argument("make") or environment.argument("ini") or environment.argument("compile") then + + -- luatools: runners.execute_ctx_script("mtx-base","--make",filename) + + resolvers.load() + trackers.enable("resolvers.locating") + environment.make_format(filename) + +elseif environment.argument("run") then + + -- luatools + + runners.execute_ctx_script("mtx-base","--run",filename) + +elseif environment.argument("fmt") then + + -- luatools + + runners.execute_ctx_script("mtx-base","--fmt",filename) + +elseif environment.argument("help") and filename=='base' then + + -- luatools + + runners.execute_ctx_script("mtx-base","--help") + elseif environment.argument("help") or filename=='help' or filename == "" then - logs.help(messages.help) - -- execute script + + application.help() + elseif filename:find("^bin:") then + + runners.loadbase() ok = runners.execute_program(filename) + elseif is_mkii_stub then + -- execute mkii script + + runners.loadbase() ok = runners.execute_script(filename,false,true) -else + +elseif false then + + runners.loadbase() ok = runners.execute_ctx_script(filename) if not ok then ok = runners.execute_script(filename) end + +elseif environment.files[1] == 'texmfcnf.lua' then -- so that we don't need to load mtx-base + + resolvers.load("nofiles") + resolvers.listers.configurations() + +else + + runners.loadbase() + runners.execute_ctx_script("mtx-base",filename) + +end + +if e_verbose then + report() + report("runtime: %0.3f seconds",os.runtime()) end -if os.platform == "unix" then - io.write("\n") +if os.type ~= "windows" then + texio.write("\n") -- is this still valid? end if ok == false then ok = 1 elseif ok == true then ok = 0 end |