diff options
Diffstat (limited to 'Master/bin/win32')
33 files changed, 15191 insertions, 9596 deletions
diff --git a/Master/bin/win32/context.exe b/Master/bin/win32/context.exe Binary files differindex 5777d90a17a..2d45f27494d 100755 --- a/Master/bin/win32/context.exe +++ b/Master/bin/win32/context.exe diff --git a/Master/bin/win32/ctxtools.exe b/Master/bin/win32/ctxtools.exe Binary files differindex 5777d90a17a..745eaf22464 100755 --- a/Master/bin/win32/ctxtools.exe +++ b/Master/bin/win32/ctxtools.exe diff --git a/Master/bin/win32/luatools.exe b/Master/bin/win32/luatools.exe Binary files differindex 5777d90a17a..2d45f27494d 100755 --- a/Master/bin/win32/luatools.exe +++ b/Master/bin/win32/luatools.exe diff --git a/Master/bin/win32/luatools.lua b/Master/bin/win32/luatools.lua index 89e5e0eb4db..1d87322c108 100755 --- a/Master/bin/win32/luatools.lua +++ b/Master/bin/win32/luatools.lua @@ -1,23 +1,26 @@ #!/usr/bin/env texlua +if not modules then modules = { } end modules ['luatools'] = { + version = 1.001, + comment = "companion to context.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local format = string.format + -- one can make a stub: -- -- #!/bin/sh --- env LUATEXDIR=/....../texmf/scripts/context/lua luatex --luaonly=luatools.lua "$@" --- filename : luatools.lua --- comment : companion to context.tex --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +-- env LUATEXDIR=/....../texmf/scripts/context/lua texlua luatools.lua "$@" + -- Although this script is part of the ConTeXt distribution it is -- relatively indepent of ConTeXt. The same is true for some of -- the luat files. We may may make them even less dependent in -- the future. As long as Luatex is under development the -- interfaces and names of functions may change. -banner = "version 1.2.0 - 2006+ - PRAGMA ADE / CONTEXT" -texlua = true - -- For the sake of independence we optionally can merge the library -- code here. It's too much code, but that does not harm. Much of the -- library code is used elsewhere. We don't want dependencies on @@ -26,158 +29,46 @@ texlua = true -- needed when texmfstart is used, or when the proper stub is used or -- when (windows) suffix binding is active. --- begin library merge --- filename : l-string.lua --- comment : split off from luat-lib --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files - -if not versions then versions = { } end versions['l-string'] = 1.001 - ---~ function string.split(str, pat) -- taken from the lua wiki ---~ local t = {n = 0} -- so this table has a length field, traverse with ipairs then! ---~ local fpat = "(.-)"..pat ---~ local last_end = 1 ---~ local s, e, cap = string.find(str, fpat, 1) ---~ while s ~= nil do ---~ if s~=1 or cap~="" then ---~ table.insert(t,cap) ---~ end ---~ last_end = e+1 ---~ s, e, cap = string.find(str, fpat, last_end) ---~ end ---~ if last_end<=string.len(str) then ---~ table.insert(t,(string.sub(str,last_end))) ---~ end ---~ return t ---~ end - ---~ function string:split(pat) -- taken from the lua wiki but adapted ---~ local t = { } -- self and colon usage (faster) ---~ local fpat = "(.-)"..pat ---~ local last_end = 1 ---~ local s, e, cap = self:find(fpat, 1) ---~ while s ~= nil do ---~ if s~=1 or cap~="" then ---~ t[#t+1] = cap ---~ end ---~ last_end = e+1 ---~ s, e, cap = self:find(fpat, last_end) ---~ end ---~ if last_end <= #self then ---~ t[#t+1] = self:sub(last_end) ---~ end ---~ return t ---~ end - ---~ a piece of brilliant code by Rici Lake (posted on lua list) -- only names changed ---~ ---~ function string:splitter(pat) ---~ local st, g = 1, self:gmatch("()"..pat.."()") ---~ local function splitter(self) ---~ if st then ---~ local s, f = g() ---~ local rv = self:sub(st, (s or 0)-1) ---~ st = f ---~ return rv ---~ end ---~ end ---~ return splitter, self ---~ end - -function string:splitter(pat) - -- by Rici Lake (posted on lua list) -- only names changed - -- p 79 ref man: () returns position of match - local st, g = 1, self:gmatch("()("..pat..")") - local function strgetter(self, segs, seps, sep, cap1, ...) - st = sep and seps + #sep - return self:sub(segs, (seps or 0) - 1), cap1 or sep, ... - end - local function strsplitter(self) - if st then return strgetter(self, st, g()) end - end - return strsplitter, self -end - -function string:split(separator) - local t = {} - for k in self:splitter(separator) do t[#t+1] = k end - return t -end - --- faster than a string:split: +texlua = true -function string:splitchr(chr) - if #self > 0 then - local t = { } - for s in string.gmatch(self..chr,"(.-)"..chr) do - t[#t+1] = s - end - return t - else - return { } - end -end +-- begin library merge ---~ function string.piecewise(str, pat, fnc) -- variant of split ---~ local fpat = "(.-)"..pat ---~ local last_end = 1 ---~ local s, e, cap = string.find(str, fpat, 1) ---~ while s ~= nil do ---~ if s~=1 or cap~="" then ---~ fnc(cap) ---~ end ---~ last_end = e+1 ---~ s, e, cap = string.find(str, fpat, last_end) ---~ end ---~ if last_end <= #str then ---~ fnc((string.sub(str,last_end))) ---~ end ---~ end -function string.piecewise(str, pat, fnc) -- variant of split - for k in string.splitter(str,pat) do fnc(k) end -end ---~ function string.piecewise(str, pat, fnc) -- variant of split ---~ for k in str:splitter(pat) do fnc(k) end ---~ end +do -- create closure to overcome 200 locals limit ---~ do if lpeg then +if not modules then modules = { } end modules ['l-string'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} ---~ -- this alternative is 30% faster esp when we cache them ---~ -- problem: no expressions +local sub, gsub, find, match, gmatch, format, char, byte, rep, lower = string.sub, string.gsub, string.find, string.match, string.gmatch, string.format, string.char, string.byte, string.rep, string.lower +local lpegmatch = lpeg.match ---~ splitters = { } +-- some functions may disappear as they are not used anywhere ---~ function string:split(separator) ---~ if #self > 0 then ---~ local split = splitters[separator] ---~ if not split then ---~ -- based on code by Roberto ---~ local p = lpeg.P(separator) ---~ local c = lpeg.C((1-p)^0) ---~ split = lpeg.Ct(c*(p*c)^0) ---~ splitters[separator] = split ---~ end ---~ return split:match(self) ---~ else ---~ return { } ---~ end ---~ end +if not string.split then ---~ string.splitchr = string.split + -- this will be overloaded by a faster lpeg variant ---~ function string:piecewise(separator,fnc) ---~ for _,v in pairs(self:split(separator)) do ---~ fnc(v) ---~ end ---~ end + function string:split(pattern) + if #self > 0 then + local t = { } + for s in gmatch(self..pattern,"(.-)"..pattern) do + t[#t+1] = s + end + return t + else + return { } + end + end ---~ end end +end -string.chr_to_esc = { +local chr_to_esc = { ["%"] = "%%", ["."] = "%.", ["+"] = "%+", ["-"] = "%-", ["*"] = "%*", @@ -187,21 +78,31 @@ string.chr_to_esc = { ["{"] = "%{", ["}"] = "%}" } +string.chr_to_esc = chr_to_esc + function string:esc() -- variant 2 - return (self:gsub("(.)",string.chr_to_esc)) + return (gsub(self,"(.)",chr_to_esc)) end -function string.unquote(str) - return (str:gsub("^([\"\'])(.*)%1$","%2")) +function string:unquote() + return (gsub(self,"^([\"\'])(.*)%1$","%2")) end -function string.quote(str) - return '"' .. str:unquote() .. '"' +--~ function string:unquote() +--~ if find(self,"^[\'\"]") then +--~ return sub(self,2,-2) +--~ else +--~ return self +--~ end +--~ end + +function string:quote() -- we could use format("%q") + return format("%q",self) end function string:count(pattern) -- variant 3 local n = 0 - for _ in self:gmatch(pattern) do + for _ in gmatch(self,pattern) do n = n + 1 end return n @@ -210,29 +111,36 @@ end function string:limit(n,sentinel) if #self > n then sentinel = sentinel or " ..." - return self:sub(1,(n-#sentinel)) .. sentinel + return sub(self,1,(n-#sentinel)) .. sentinel else return self end end -function string:strip() - return (self:gsub("^%s*(.-)%s*$", "%1")) -end - ---~ function string.strip(str) -- slightly different ---~ return (string.gsub(string.gsub(str,"^%s*(.-)%s*$","%1"),"%s+"," ")) +--~ function string:strip() -- the .- is quite efficient +--~ -- return match(self,"^%s*(.-)%s*$") or "" +--~ -- return match(self,'^%s*(.*%S)') or '' -- posted on lua list +--~ return find(s,'^%s*$') and '' or match(s,'^%s*(.*%S)') --~ end +do -- roberto's variant: + local space = lpeg.S(" \t\v\n") + local nospace = 1 - space + local stripper = space^0 * lpeg.C((space^0 * nospace^1)^0) + function string.strip(str) + return lpegmatch(stripper,str) or "" + end +end + function string:is_empty() - return not self:find("%S") + return not find(self,"%S") end function string:enhance(pattern,action) local ok, n = true, 0 while ok do ok = false - self = self:gsub(pattern, function(...) + self = gsub(self,pattern, function(...) ok, n = true, n + 1 return action(...) end) @@ -240,73 +148,33 @@ function string:enhance(pattern,action) return self, n end ---~ function string:enhance(pattern,action) ---~ local ok, n = 0, 0 ---~ repeat ---~ self, ok = self:gsub(pattern, function(...) ---~ n = n + 1 ---~ return action(...) ---~ end) ---~ until ok == 0 ---~ return self, n ---~ end - ---~ function string:to_hex() ---~ if self then ---~ return (self:gsub("(.)",function(c) ---~ return string.format("%02X",c:byte()) ---~ end)) ---~ else ---~ return "" ---~ end ---~ end - ---~ function string:from_hex() ---~ if self then ---~ return (self:gsub("(..)",function(c) ---~ return string.char(tonumber(c,16)) ---~ end)) ---~ else ---~ return "" ---~ end ---~ end - -string.chr_to_hex = { } -string.hex_to_chr = { } +local chr_to_hex, hex_to_chr = { }, { } for i=0,255 do - local c, h = string.char(i), string.format("%02X",i) - string.chr_to_hex[c], string.hex_to_chr[h] = h, c + local c, h = char(i), format("%02X",i) + chr_to_hex[c], hex_to_chr[h] = h, c end ---~ function string:to_hex() ---~ if self then return (self:gsub("(.)",string.chr_to_hex)) else return "" end ---~ end - ---~ function string:from_hex() ---~ if self then return (self:gsub("(..)",string.hex_to_chr)) else return "" end ---~ end - function string:to_hex() - return ((self or ""):gsub("(.)",string.chr_to_hex)) + return (gsub(self or "","(.)",chr_to_hex)) end function string:from_hex() - return ((self or ""):gsub("(..)",string.hex_to_chr)) + return (gsub(self or "","(..)",hex_to_chr)) end if not string.characters then local function nextchar(str, index) index = index + 1 - return (index <= #str) and index or nil, str:sub(index,index) + return (index <= #str) and index or nil, sub(str,index,index) end function string:characters() return nextchar, self, 0 end local function nextbyte(str, index) index = index + 1 - return (index <= #str) and index or nil, string.byte(str:sub(index,index)) + return (index <= #str) and index or nil, byte(sub(str,index,index)) end function string:bytes() return nextbyte, self, 0 @@ -314,14 +182,12 @@ if not string.characters then end ---~ function string:padd(n,chr) ---~ return self .. self.rep(chr or " ",n-#self) ---~ end +-- we can use format for this (neg n) function string:rpadd(n,chr) local m = n-#self if m > 0 then - return self .. self.rep(chr or " ",m) + return self .. rep(chr or " ",m) else return self end @@ -330,7 +196,7 @@ end function string:lpadd(n,chr) local m = n-#self if m > 0 then - return self.rep(chr or " ",m) .. self + return rep(chr or " ",m) .. self else return self end @@ -338,8 +204,8 @@ end string.padd = string.rpadd -function is_number(str) - return str:find("^[%-%+]?[%d]-%.?[%d+]$") == 1 +function is_number(str) -- tonumber + return find(str,"^[%-%+]?[%d]-%.?[%d+]$") == 1 end --~ print(is_number("1")) @@ -351,9 +217,9 @@ end --~ print(is_number("+.1")) function string:split_settings() -- no {} handling, see l-aux for lpeg variant - if self:find("=") then + if find(self,"=") then local t = { } - for k,v in self:gmatch("(%a+)=([^%,]*)") do + for k,v in gmatch(self,"(%a+)=([^%,]*)") do t[k] = v end return t @@ -375,93 +241,287 @@ local patterns_escapes = { } function string:pattesc() - return (self:gsub(".",patterns_escapes)) + return (gsub(self,".",patterns_escapes)) +end + +local simple_escapes = { + ["-"] = "%-", + ["."] = "%.", + ["?"] = ".", + ["*"] = ".*", +} + +function string:simpleesc() + return (gsub(self,".",simple_escapes)) end function string:tohash() local t = { } - for s in self:gmatch("([^, ]+)") do -- lpeg + for s in gmatch(self,"([^, ]+)") do -- lpeg t[s] = true end return t end +local pattern = lpeg.Ct(lpeg.C(1)^0) + +function string:totable() + return lpegmatch(pattern,self) +end + +--~ local t = { +--~ "1234567123456712345671234567", +--~ "a\tb\tc", +--~ "aa\tbb\tcc", +--~ "aaa\tbbb\tccc", +--~ "aaaa\tbbbb\tcccc", +--~ "aaaaa\tbbbbb\tccccc", +--~ "aaaaaa\tbbbbbb\tcccccc", +--~ } +--~ for k,v do +--~ print(string.tabtospace(t[k])) +--~ end + +function string.tabtospace(str,tab) + -- we don't handle embedded newlines + while true do + local s = find(str,"\t") + if s then + if not tab then tab = 7 end -- only when found + local d = tab-(s-1) % tab + if d > 0 then + str = gsub(str,"\t",rep(" ",d),1) + else + str = gsub(str,"\t","",1) + end + else + break + end + end + return str +end + +function string:compactlong() -- strips newlines and leading spaces + self = gsub(self,"[\n\r]+ *","") + self = gsub(self,"^ *","") + return self +end --- filename : l-lpeg.lua --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +function string:striplong() -- strips newlines and leading spaces + self = gsub(self,"^%s*","") + self = gsub(self,"[\n\r]+ *","\n") + return self +end -if not versions then versions = { } end versions['l-lpeg'] = 1.001 +function string:topattern(lowercase,strict) + if lowercase then + self = lower(self) + end + self = gsub(self,".",simple_escapes) + if self == "" then + self = ".*" + elseif strict then + self = "^" .. self .. "$" + end + return self +end ---~ l-lpeg.lua : ---~ lpeg.digit = lpeg.R('09')^1 ---~ lpeg.sign = lpeg.S('+-')^1 ---~ lpeg.cardinal = lpeg.P(lpeg.sign^0 * lpeg.digit^1) ---~ lpeg.integer = lpeg.P(lpeg.sign^0 * lpeg.digit^1) ---~ lpeg.float = lpeg.P(lpeg.sign^0 * lpeg.digit^0 * lpeg.P('.') * lpeg.digit^1) ---~ lpeg.number = lpeg.float + lpeg.integer ---~ lpeg.oct = lpeg.P("0") * lpeg.R('07')^1 ---~ lpeg.hex = lpeg.P("0x") * (lpeg.R('09') + lpeg.R('AF'))^1 ---~ lpeg.uppercase = lpeg.P("AZ") ---~ lpeg.lowercase = lpeg.P("az") +end -- of closure ---~ lpeg.eol = lpeg.S('\r\n\f')^1 -- includes formfeed ---~ lpeg.space = lpeg.S(' ')^1 ---~ lpeg.nonspace = lpeg.P(1-lpeg.space)^1 ---~ lpeg.whitespace = lpeg.S(' \r\n\f\t')^1 ---~ lpeg.nonwhitespace = lpeg.P(1-lpeg.whitespace)^1 +do -- create closure to overcome 200 locals limit -local hash = { } +if not modules then modules = { } end modules ['l-lpeg'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local lpeg = require("lpeg") + +lpeg.patterns = lpeg.patterns or { } -- so that we can share +local patterns = lpeg.patterns + +local P, R, S, Ct, C, Cs, Cc, V = lpeg.P, lpeg.R, lpeg.S, lpeg.Ct, lpeg.C, lpeg.Cs, lpeg.Cc, lpeg.V +local match = lpeg.match + +local digit, sign = R('09'), S('+-') +local cr, lf, crlf = P("\r"), P("\n"), P("\r\n") +local utf8byte = R("\128\191") + +patterns.utf8byte = utf8byte +patterns.utf8one = R("\000\127") +patterns.utf8two = R("\194\223") * utf8byte +patterns.utf8three = R("\224\239") * utf8byte * utf8byte +patterns.utf8four = R("\240\244") * utf8byte * utf8byte * utf8byte + +patterns.digit = digit +patterns.sign = sign +patterns.cardinal = sign^0 * digit^1 +patterns.integer = sign^0 * digit^1 +patterns.float = sign^0 * digit^0 * P('.') * digit^1 +patterns.number = patterns.float + patterns.integer +patterns.oct = P("0") * R("07")^1 +patterns.octal = patterns.oct +patterns.HEX = P("0x") * R("09","AF")^1 +patterns.hex = P("0x") * R("09","af")^1 +patterns.hexadecimal = P("0x") * R("09","AF","af")^1 +patterns.lowercase = R("az") +patterns.uppercase = R("AZ") +patterns.letter = patterns.lowercase + patterns.uppercase +patterns.space = S(" ") +patterns.eol = S("\n\r") +patterns.spacer = S(" \t\f\v") -- + string.char(0xc2, 0xa0) if we want utf (cf mail roberto) +patterns.newline = crlf + cr + lf +patterns.nonspace = 1 - patterns.space +patterns.nonspacer = 1 - patterns.spacer +patterns.whitespace = patterns.eol + patterns.spacer +patterns.nonwhitespace = 1 - patterns.whitespace +patterns.utf8 = patterns.utf8one + patterns.utf8two + patterns.utf8three + patterns.utf8four +patterns.utfbom = P('\000\000\254\255') + P('\255\254\000\000') + P('\255\254') + P('\254\255') + P('\239\187\191') function lpeg.anywhere(pattern) --slightly adapted from website - return lpeg.P { lpeg.P(pattern) + 1 * lpeg.V(1) } + return P { P(pattern) + 1 * V(1) } -- why so complex? end -function lpeg.startswith(pattern) --slightly adapted - return lpeg.P(pattern) +function lpeg.splitter(pattern, action) + return (((1-P(pattern))^1)/action+1)^0 end ---~ g = lpeg.splitter(" ",function(s) ... end) -- gmatch:lpeg = 3:2 +local spacing = patterns.spacer^0 * patterns.newline -- sort of strip +local empty = spacing * Cc("") +local nonempty = Cs((1-spacing)^1) * spacing^-1 +local content = (empty + nonempty)^1 -function lpeg.splitter(pattern, action) - return (((1-lpeg.P(pattern))^1)/action+1)^0 +local capture = Ct(content^0) + +function string:splitlines() + return match(capture,self) end -local crlf = lpeg.P("\r\n") -local cr = lpeg.P("\r") -local lf = lpeg.P("\n") -local space = lpeg.S(" \t\f\v") -local newline = crlf + cr + lf -local spacing = space^0 * newline +patterns.textline = content -local empty = spacing * lpeg.Cc("") -local nonempty = lpeg.Cs((1-spacing)^1) * spacing^-1 -local content = (empty + nonempty)^1 +--~ local p = lpeg.splitat("->",false) print(match(p,"oeps->what->more")) -- oeps what more +--~ local p = lpeg.splitat("->",true) print(match(p,"oeps->what->more")) -- oeps what->more +--~ local p = lpeg.splitat("->",false) print(match(p,"oeps")) -- oeps +--~ local p = lpeg.splitat("->",true) print(match(p,"oeps")) -- oeps -local capture = lpeg.Ct(content^0) +local splitters_s, splitters_m = { }, { } -function string:splitlines() - return capture:match(self) +local function splitat(separator,single) + local splitter = (single and splitters_s[separator]) or splitters_m[separator] + if not splitter then + separator = P(separator) + if single then + local other, any = C((1 - separator)^0), P(1) + splitter = other * (separator * C(any^0) + "") -- ? + splitters_s[separator] = splitter + else + local other = C((1 - separator)^0) + splitter = other * (separator * other)^0 + splitters_m[separator] = splitter + end + end + return splitter +end + +lpeg.splitat = splitat + +local cache = { } + +function lpeg.split(separator,str) + local c = cache[separator] + if not c then + c = Ct(splitat(separator)) + cache[separator] = c + end + return match(c,str) +end + +function string:split(separator) + local c = cache[separator] + if not c then + c = Ct(splitat(separator)) + cache[separator] = c + end + return match(c,self) end +lpeg.splitters = cache --- filename : l-table.lua --- comment : split off from luat-lib --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +local cache = { } -if not versions then versions = { } end versions['l-table'] = 1.001 +function lpeg.checkedsplit(separator,str) + local c = cache[separator] + if not c then + separator = P(separator) + local other = C((1 - separator)^0) + c = Ct(separator^0 * other * (separator^1 * other)^0) + cache[separator] = c + end + return match(c,str) +end + +function string:checkedsplit(separator) + local c = cache[separator] + if not c then + separator = P(separator) + local other = C((1 - separator)^0) + c = Ct(separator^0 * other * (separator^1 * other)^0) + cache[separator] = c + end + return match(c,self) +end + +--~ function lpeg.append(list,pp) +--~ local p = pp +--~ for l=1,#list do +--~ if p then +--~ p = p + P(list[l]) +--~ else +--~ p = P(list[l]) +--~ end +--~ end +--~ return p +--~ end + +--~ from roberto's site: + +local f1 = string.byte + +local function f2(s) local c1, c2 = f1(s,1,2) return c1 * 64 + c2 - 12416 end +local function f3(s) local c1, c2, c3 = f1(s,1,3) return (c1 * 64 + c2) * 64 + c3 - 925824 end +local function f4(s) local c1, c2, c3, c4 = f1(s,1,4) return ((c1 * 64 + c2) * 64 + c3) * 64 + c4 - 63447168 end + +patterns.utf8byte = patterns.utf8one/f1 + patterns.utf8two/f2 + patterns.utf8three/f3 + patterns.utf8four/f4 + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['l-table'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} table.join = table.concat +local concat, sort, insert, remove = table.concat, table.sort, table.insert, table.remove +local format, find, gsub, lower, dump, match = string.format, string.find, string.gsub, string.lower, string.dump, string.match +local getmetatable, setmetatable = getmetatable, setmetatable +local type, next, tostring, tonumber, ipairs = type, next, tostring, tonumber, ipairs +local unpack = unpack or table.unpack + function table.strip(tab) local lst = { } - for k, v in ipairs(tab) do - -- s = string.gsub(v, "^%s*(.-)%s*$", "%1") - s = v:gsub("^%s*(.-)%s*$", "%1") + for i=1,#tab do + local s = gsub(tab[i],"^%s*(.-)%s*$","%1") if s == "" then -- skip this one else @@ -471,18 +531,21 @@ function table.strip(tab) return lst end ---~ function table.sortedkeys(tab) ---~ local srt = { } ---~ for key,_ in pairs(tab) do ---~ srt[#srt+1] = key ---~ end ---~ table.sort(srt) ---~ return srt ---~ end +function table.keys(t) + local k = { } + for key, _ in next, t do + k[#k+1] = key + end + return k +end + +local function compare(a,b) + return (tostring(a) < tostring(b)) +end -function table.sortedkeys(tab) +local function sortedkeys(tab) local srt, kind = { }, 0 -- 0=unknown 1=string, 2=number 3=mixed - for key,_ in pairs(tab) do + for key,_ in next, tab do srt[#srt+1] = key if kind == 3 then -- no further check @@ -500,22 +563,47 @@ function table.sortedkeys(tab) end end if kind == 0 or kind == 3 then - table.sort(srt,function(a,b) return (tostring(a) < tostring(b)) end) + sort(srt,compare) else - table.sort(srt) + sort(srt) end return srt end +local function sortedhashkeys(tab) -- fast one + local srt = { } + for key,_ in next, tab do + srt[#srt+1] = key + end + sort(srt) + return srt +end + +table.sortedkeys = sortedkeys +table.sortedhashkeys = sortedhashkeys + +function table.sortedhash(t) + local s = sortedhashkeys(t) -- maybe just sortedkeys + local n = 0 + local function kv(s) + n = n + 1 + local k = s[n] + return k, t[k] + end + return kv, s +end + +table.sortedpairs = table.sortedhash + function table.append(t, list) - for _,v in pairs(list) do - table.insert(t,v) + for _,v in next, list do + insert(t,v) end end function table.prepend(t, list) - for k,v in pairs(list) do - table.insert(t,k,v) + for k,v in next, list do + insert(t,k,v) end end @@ -523,7 +611,7 @@ function table.merge(t, ...) -- first one is target t = t or {} local lst = {...} for i=1,#lst do - for k, v in pairs(lst[i]) do + for k, v in next, lst[i] do t[k] = v end end @@ -533,7 +621,7 @@ end function table.merged(...) local tmp, lst = { }, {...} for i=1,#lst do - for k, v in pairs(lst[i]) do + for k, v in next, lst[i] do tmp[k] = v end end @@ -562,70 +650,58 @@ function table.imerged(...) return tmp end -if not table.fastcopy then do - - local type, pairs, getmetatable, setmetatable = type, pairs, getmetatable, setmetatable - - local function fastcopy(old) -- fast one - if old then - local new = { } - for k,v in pairs(old) do - if type(v) == "table" then - new[k] = fastcopy(v) -- was just table.copy - else - new[k] = v - end - end - local mt = getmetatable(old) - if mt then - setmetatable(new,mt) +local function fastcopy(old) -- fast one + if old then + local new = { } + for k,v in next, old do + if type(v) == "table" then + new[k] = fastcopy(v) -- was just table.copy + else + new[k] = v end - return new - else - return { } end + -- optional second arg + local mt = getmetatable(old) + if mt then + setmetatable(new,mt) + end + return new + else + return { } end +end - table.fastcopy = fastcopy - -end end - -if not table.copy then do - - local type, pairs, getmetatable, setmetatable = type, pairs, getmetatable, setmetatable - - local function copy(t, tables) -- taken from lua wiki, slightly adapted - tables = tables or { } - local tcopy = {} - if not tables[t] then - tables[t] = tcopy - end - for i,v in pairs(t) do -- brrr, what happens with sparse indexed - if type(i) == "table" then - if tables[i] then - i = tables[i] - else - i = copy(i, tables) - end - end - if type(v) ~= "table" then - tcopy[i] = v - elseif tables[v] then - tcopy[i] = tables[v] +local function copy(t, tables) -- taken from lua wiki, slightly adapted + tables = tables or { } + local tcopy = {} + if not tables[t] then + tables[t] = tcopy + end + for i,v in next, t do -- brrr, what happens with sparse indexed + if type(i) == "table" then + if tables[i] then + i = tables[i] else - tcopy[i] = copy(v, tables) + i = copy(i, tables) end end - local mt = getmetatable(t) - if mt then - setmetatable(tcopy,mt) + if type(v) ~= "table" then + tcopy[i] = v + elseif tables[v] then + tcopy[i] = tables[v] + else + tcopy[i] = copy(v, tables) end - return tcopy end + local mt = getmetatable(t) + if mt then + setmetatable(tcopy,mt) + end + return tcopy +end - table.copy = copy - -end end +table.fastcopy = fastcopy +table.copy = copy -- rougly: copy-loop : unpack : sub == 0.9 : 0.4 : 0.45 (so in critical apps, use unpack) @@ -634,364 +710,524 @@ function table.sub(t,i,j) end function table.replace(a,b) - for k,v in pairs(b) do + for k,v in next, b do a[k] = v end end -- slower than #t on indexed tables (#t only returns the size of the numerically indexed slice) -function table.is_empty(t) +function table.is_empty(t) -- obolete, use inline code instead return not t or not next(t) end -function table.one_entry(t) +function table.one_entry(t) -- obolete, use inline code instead local n = next(t) return n and not next(t,n) end -function table.starts_at(t) - return ipairs(t,1)(t,0) -end +--~ function table.starts_at(t) -- obsolete, not nice +--~ return ipairs(t,1)(t,0) +--~ end -do +function table.tohash(t,value) + local h = { } + if t then + if value == nil then value = true end + for _, v in next, t do -- no ipairs here + h[v] = value + end + end + return h +end - -- one of my first exercises in lua ... +function table.fromhash(t) + local h = { } + for k, v in next, t do -- no ipairs here + if v then h[#h+1] = k end + end + return h +end - -- 34.055.092 32.403.326 arabtype.tma - -- 1.620.614 1.513.863 lmroman10-italic.tma - -- 1.325.585 1.233.044 lmroman10-regular.tma - -- 1.248.157 1.158.903 lmsans10-regular.tma - -- 194.646 153.120 lmtypewriter10-regular.tma - -- 1.771.678 1.658.461 palatinosanscom-bold.tma - -- 1.695.251 1.584.491 palatinosanscom-regular.tma - -- 13.736.534 13.409.446 zapfinoextraltpro.tma +--~ print(table.serialize(t), "\n") +--~ print(table.serialize(t,"name"), "\n") +--~ print(table.serialize(t,false), "\n") +--~ print(table.serialize(t,true), "\n") +--~ print(table.serialize(t,"name",true), "\n") +--~ print(table.serialize(t,"name",true,true), "\n") - -- 13.679.038 11.774.106 arabtype.tmc - -- 886.248 754.944 lmroman10-italic.tmc - -- 729.828 466.864 lmroman10-regular.tmc - -- 688.482 441.962 lmsans10-regular.tmc - -- 128.685 95.853 lmtypewriter10-regular.tmc - -- 715.929 582.985 palatinosanscom-bold.tmc - -- 669.942 540.126 palatinosanscom-regular.tmc - -- 1.560.588 1.317.000 zapfinoextraltpro.tmc +table.serialize_functions = true +table.serialize_compact = true +table.serialize_inline = true - table.serialize_functions = true - table.serialize_compact = true - table.serialize_inline = true +local noquotes, hexify, handle, reduce, compact, inline, functions - local function key(k) - if type(k) == "number" then -- or k:find("^%d+$") then - return "["..k.."]" - elseif noquotes and k:find("^%a[%a%d%_]*$") then - return k - else - return '["'..k..'"]' - end - end +local reserved = table.tohash { -- intercept a language flaw, no reserved words as key + 'and', 'break', 'do', 'else', 'elseif', 'end', 'false', 'for', 'function', 'if', + 'in', 'local', 'nil', 'not', 'or', 'repeat', 'return', 'then', 'true', 'until', 'while', +} - local function simple_table(t) - if #t > 0 then - local n = 0 - for _,v in pairs(t) do - n = n + 1 - end - if n == #t then - local tt = { } - for i=1,#t do - local v = t[i] - local tv = type(v) - if tv == "number" or tv == "boolean" then - tt[#tt+1] = tostring(v) - elseif tv == "string" then - tt[#tt+1] = ("%q"):format(v) +local function simple_table(t) + if #t > 0 then + local n = 0 + for _,v in next, t do + n = n + 1 + end + if n == #t then + local tt = { } + for i=1,#t do + local v = t[i] + local tv = type(v) + if tv == "number" then + if hexify then + tt[#tt+1] = format("0x%04X",v) else - tt = nil - break + tt[#tt+1] = tostring(v) -- tostring not needed end + elseif tv == "boolean" then + tt[#tt+1] = tostring(v) + elseif tv == "string" then + tt[#tt+1] = format("%q",v) + else + tt = nil + break end - return tt end + return tt end - return nil end + return nil +end - local function serialize(root,name,handle,depth,level,reduce,noquotes,indexed) - handle = handle or print - reduce = reduce or false - if depth then - depth = depth .. " " - if indexed then - handle(("%s{"):format(depth)) +-- Because this is a core function of mkiv I moved some function calls +-- inline. +-- +-- twice as fast in a test: +-- +-- local propername = lpeg.P(lpeg.R("AZ","az","__") * lpeg.R("09","AZ","az", "__")^0 * lpeg.P(-1) ) + +-- problem: there no good number_to_string converter with the best resolution + +local function do_serialize(root,name,depth,level,indexed) + if level > 0 then + depth = depth .. " " + if indexed then + handle(format("%s{",depth)) + elseif name then + --~ handle(format("%s%s={",depth,key(name))) + if type(name) == "number" then -- or find(k,"^%d+$") then + if hexify then + handle(format("%s[0x%04X]={",depth,name)) + else + handle(format("%s[%s]={",depth,name)) + end + elseif noquotes and not reserved[name] and find(name,"^%a[%w%_]*$") then + handle(format("%s%s={",depth,name)) else - handle(("%s%s={"):format(depth,key(name))) + handle(format("%s[%q]={",depth,name)) end else - depth = "" - local tname = type(name) - if tname == "string" then - if name == "return" then - handle("return {") + handle(format("%s{",depth)) + end + end + -- we could check for k (index) being number (cardinal) + if root and next(root) then + local first, last = nil, 0 -- #root cannot be trusted here (will be ok in 5.2 when ipairs is gone) + if compact then + -- NOT: for k=1,#root do (we need to quit at nil) + for k,v in ipairs(root) do -- can we use next? + if not first then first = k end + last = last + 1 + end + end + local sk = sortedkeys(root) + for i=1,#sk do + local k = sk[i] + local v = root[k] + --~ if v == root then + -- circular + --~ else + local t = type(v) + if compact and first and type(k) == "number" and k >= first and k <= last then + if t == "number" then + if hexify then + handle(format("%s 0x%04X,",depth,v)) + else + handle(format("%s %s,",depth,v)) -- %.99g + end + elseif t == "string" then + if reduce and tonumber(v) then + handle(format("%s %s,",depth,v)) + else + handle(format("%s %q,",depth,v)) + end + elseif t == "table" then + if not next(v) then + handle(format("%s {},",depth)) + elseif inline then -- and #t > 0 + local st = simple_table(v) + if st then + handle(format("%s { %s },",depth,concat(st,", "))) + else + do_serialize(v,k,depth,level+1,true) + end + else + do_serialize(v,k,depth,level+1,true) + end + elseif t == "boolean" then + handle(format("%s %s,",depth,tostring(v))) + elseif t == "function" then + if functions then + handle(format('%s loadstring(%q),',depth,dump(v))) + else + handle(format('%s "function",',depth)) + end else - handle(name .. "={") + handle(format("%s %q,",depth,tostring(v))) end - elseif tname == "number" then - handle("[" .. name .. "]={") - elseif tname == "boolean" then - if name then - handle("return {") - else - handle("{") + elseif k == "__p__" then -- parent + if false then + handle(format("%s __p__=nil,",depth)) end - else - handle("t={") - end - end - if root and next(root) then - local compact = table.serialize_compact - local inline = compact and table.serialize_inline - local first, last = nil, 0 -- #root cannot be trusted here - if compact then - for k,v in ipairs(root) do -- NOT: for k=1,#root do (why) - if not first then first = k end - last = last + 1 + elseif t == "number" then + --~ if hexify then + --~ handle(format("%s %s=0x%04X,",depth,key(k),v)) + --~ else + --~ handle(format("%s %s=%s,",depth,key(k),v)) -- %.99g + --~ end + if type(k) == "number" then -- or find(k,"^%d+$") then + if hexify then + handle(format("%s [0x%04X]=0x%04X,",depth,k,v)) + else + handle(format("%s [%s]=%s,",depth,k,v)) -- %.99g + end + elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then + if hexify then + handle(format("%s %s=0x%04X,",depth,k,v)) + else + handle(format("%s %s=%s,",depth,k,v)) -- %.99g + end + else + if hexify then + handle(format("%s [%q]=0x%04X,",depth,k,v)) + else + handle(format("%s [%q]=%s,",depth,k,v)) -- %.99g + end end - end - for _,k in pairs(table.sortedkeys(root)) do - local v = root[k] - local t = type(v) - if compact and first and type(k) == "number" and k >= first and k <= last then - if t == "number" then - handle(("%s %s,"):format(depth,v)) - elseif t == "string" then - if reduce and (v:find("^[%-%+]?[%d]-%.?[%d+]$") == 1) then - handle(("%s %s,"):format(depth,v)) + elseif t == "string" then + if reduce and tonumber(v) then + --~ handle(format("%s %s=%s,",depth,key(k),v)) + if type(k) == "number" then -- or find(k,"^%d+$") then + if hexify then + handle(format("%s [0x%04X]=%s,",depth,k,v)) else - handle(("%s %q,"):format(depth,v)) + handle(format("%s [%s]=%s,",depth,k,v)) end - elseif t == "table" then - if not next(v) then - handle(("%s {},"):format(depth)) - elseif inline then - local st = simple_table(v) - if st then - handle(("%s { %s },"):format(depth,table.concat(st,", "))) - else - serialize(v,k,handle,depth,level+1,reduce,noquotes,true) - end + elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then + handle(format("%s %s=%s,",depth,k,v)) + else + handle(format("%s [%q]=%s,",depth,k,v)) + end + else + --~ handle(format("%s %s=%q,",depth,key(k),v)) + if type(k) == "number" then -- or find(k,"^%d+$") then + if hexify then + handle(format("%s [0x%04X]=%q,",depth,k,v)) else - serialize(v,k,handle,depth,level+1,reduce,noquotes,true) + handle(format("%s [%s]=%q,",depth,k,v)) end - elseif t == "boolean" then - handle(("%s %s,"):format(depth,tostring(v))) - elseif t == "function" then - if table.serialize_functions then - handle(('%s loadstring(%q),'):format(depth,string.dump(v))) + elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then + handle(format("%s %s=%q,",depth,k,v)) + else + handle(format("%s [%q]=%q,",depth,k,v)) + end + end + elseif t == "table" then + if not next(v) then + --~ handle(format("%s %s={},",depth,key(k))) + if type(k) == "number" then -- or find(k,"^%d+$") then + if hexify then + handle(format("%s [0x%04X]={},",depth,k)) else - handle(('%s "function",'):format(depth)) + handle(format("%s [%s]={},",depth,k)) end + elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then + handle(format("%s %s={},",depth,k)) else - handle(("%s %q,"):format(depth,tostring(v))) + handle(format("%s [%q]={},",depth,k)) end - elseif k == "__p__" then -- parent - if false then - handle(("%s __p__=nil,"):format(depth)) + elseif inline then + local st = simple_table(v) + if st then + --~ handle(format("%s %s={ %s },",depth,key(k),concat(st,", "))) + if type(k) == "number" then -- or find(k,"^%d+$") then + if hexify then + handle(format("%s [0x%04X]={ %s },",depth,k,concat(st,", "))) + else + handle(format("%s [%s]={ %s },",depth,k,concat(st,", "))) + end + elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then + handle(format("%s %s={ %s },",depth,k,concat(st,", "))) + else + handle(format("%s [%q]={ %s },",depth,k,concat(st,", "))) + end + else + do_serialize(v,k,depth,level+1) end - elseif t == "number" then - handle(("%s %s=%s,"):format(depth,key(k),v)) - elseif t == "string" then - if reduce and (v:find("^[%-%+]?[%d]-%.?[%d+]$") == 1) then - handle(("%s %s=%s,"):format(depth,key(k),v)) + else + do_serialize(v,k,depth,level+1) + end + elseif t == "boolean" then + --~ handle(format("%s %s=%s,",depth,key(k),tostring(v))) + if type(k) == "number" then -- or find(k,"^%d+$") then + if hexify then + handle(format("%s [0x%04X]=%s,",depth,k,tostring(v))) else - handle(("%s %s=%q,"):format(depth,key(k),v)) + handle(format("%s [%s]=%s,",depth,k,tostring(v))) end - elseif t == "table" then - if not next(v) then - handle(("%s %s={},"):format(depth,key(k))) - elseif inline then - local st = simple_table(v) - if st then - handle(("%s %s={ %s },"):format(depth,key(k),table.concat(st,", "))) + elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then + handle(format("%s %s=%s,",depth,k,tostring(v))) + else + handle(format("%s [%q]=%s,",depth,k,tostring(v))) + end + elseif t == "function" then + if functions then + --~ handle(format('%s %s=loadstring(%q),',depth,key(k),dump(v))) + if type(k) == "number" then -- or find(k,"^%d+$") then + if hexify then + handle(format("%s [0x%04X]=loadstring(%q),",depth,k,dump(v))) else - serialize(v,k,handle,depth,level+1,reduce,noquotes) + handle(format("%s [%s]=loadstring(%q),",depth,k,dump(v))) end + elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then + handle(format("%s %s=loadstring(%q),",depth,k,dump(v))) else - serialize(v,k,handle,depth,level+1,reduce,noquotes) + handle(format("%s [%q]=loadstring(%q),",depth,k,dump(v))) end - elseif t == "boolean" then - handle(("%s %s=%s,"):format(depth,key(k),tostring(v))) - elseif t == "function" then - if table.serialize_functions then - handle(('%s %s=loadstring(%q),'):format(depth,key(k),string.dump(v))) + end + else + --~ handle(format("%s %s=%q,",depth,key(k),tostring(v))) + if type(k) == "number" then -- or find(k,"^%d+$") then + if hexify then + handle(format("%s [0x%04X]=%q,",depth,k,tostring(v))) else - handle(('%s %s="function",'):format(depth,key(k))) + handle(format("%s [%s]=%q,",depth,k,tostring(v))) end + elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then + handle(format("%s %s=%q,",depth,k,tostring(v))) else - handle(("%s %s=%q,"):format(depth,key(k),tostring(v))) - -- handle(('%s %s=loadstring(%q),'):format(depth,key(k),string.dump(function() return v end))) + handle(format("%s [%q]=%q,",depth,k,tostring(v))) end end - if level > 0 then - handle(("%s},"):format(depth)) - else - handle(("%s}"):format(depth)) - end - else - handle(("%s}"):format(depth)) + --~ end end end + if level > 0 then + handle(format("%s},",depth)) + end +end - --~ name: - --~ - --~ true : return { } - --~ false : { } - --~ nil : t = { } - --~ string : string = { } - --~ 'return' : return { } - --~ number : [number] = { } +-- replacing handle by a direct t[#t+1] = ... (plus test) is not much +-- faster (0.03 on 1.00 for zapfino.tma) - function table.serialize(root,name,reduce,noquotes) - local t = { } - local function flush(s) - t[#t+1] = s +local function serialize(root,name,_handle,_reduce,_noquotes,_hexify) + noquotes = _noquotes + hexify = _hexify + handle = _handle or print + reduce = _reduce or false + compact = table.serialize_compact + inline = compact and table.serialize_inline + functions = table.serialize_functions + local tname = type(name) + if tname == "string" then + if name == "return" then + handle("return {") + else + handle(name .. "={") + end + elseif tname == "number" then + if hexify then + handle(format("[0x%04X]={",name)) + else + handle("[" .. name .. "]={") + end + elseif tname == "boolean" then + if name then + handle("return {") + else + handle("{") end - serialize(root, name, flush, nil, 0, reduce, noquotes) - return table.concat(t,"\n") + else + handle("t={") + end + if root and next(root) then + do_serialize(root,name,"",0,indexed) end + handle("}") +end - function table.tohandle(handle,root,name,reduce,noquotes) - serialize(root, name, handle, nil, 0, reduce, noquotes) +--~ name: +--~ +--~ true : return { } +--~ false : { } +--~ nil : t = { } +--~ string : string = { } +--~ 'return' : return { } +--~ number : [number] = { } + +function table.serialize(root,name,reduce,noquotes,hexify) + local t = { } + local function flush(s) + t[#t+1] = s end + serialize(root,name,flush,reduce,noquotes,hexify) + return concat(t,"\n") +end - -- sometimes tables are real use (zapfino extra pro is some 85M) in which - -- case a stepwise serialization is nice; actually, we could consider: - -- - -- for line in table.serializer(root,name,reduce,noquotes) do - -- ...(line) - -- end - -- - -- so this is on the todo list +function table.tohandle(handle,root,name,reduce,noquotes,hexify) + serialize(root,name,handle,reduce,noquotes,hexify) +end - table.tofile_maxtab = 2*1024 +-- sometimes tables are real use (zapfino extra pro is some 85M) in which +-- case a stepwise serialization is nice; actually, we could consider: +-- +-- for line in table.serializer(root,name,reduce,noquotes) do +-- ...(line) +-- end +-- +-- so this is on the todo list - function table.tofile(filename,root,name,reduce,noquotes) - local f = io.open(filename,'w') - if f then - local concat = table.concat - local maxtab = table.tofile_maxtab - if maxtab > 1 then - local t = { } - local function flush(s) - t[#t+1] = s - if #t > maxtab then - f:write(concat(t,"\n"),"\n") -- hm, write(sometable) should be nice - t = { } - end - end - serialize(root, name, flush, nil, 0, reduce, noquotes) - f:write(concat(t,"\n"),"\n") - else - local function flush(s) - f:write(s,"\n") +table.tofile_maxtab = 2*1024 + +function table.tofile(filename,root,name,reduce,noquotes,hexify) + local f = io.open(filename,'w') + if f then + local maxtab = table.tofile_maxtab + if maxtab > 1 then + local t = { } + local function flush(s) + t[#t+1] = s + if #t > maxtab then + f:write(concat(t,"\n"),"\n") -- hm, write(sometable) should be nice + t = { } end - serialize(root, name, flush, nil, 0, reduce, noquotes) end - f:close() + serialize(root,name,flush,reduce,noquotes,hexify) + f:write(concat(t,"\n"),"\n") + else + local function flush(s) + f:write(s,"\n") + end + serialize(root,name,flush,reduce,noquotes,hexify) end + f:close() end - end ---~ t = { ---~ b = "123", ---~ a = "x", ---~ c = 1.23, ---~ d = "1.23", ---~ e = true, ---~ f = { ---~ d = "1.23", ---~ a = "x", ---~ b = "123", ---~ c = 1.23, ---~ e = true, ---~ f = { ---~ e = true, ---~ f = { ---~ e = true ---~ }, ---~ }, ---~ }, ---~ g = function() end ---~ } - ---~ print(table.serialize(t), "\n") ---~ print(table.serialize(t,"name"), "\n") ---~ print(table.serialize(t,false), "\n") ---~ print(table.serialize(t,true), "\n") ---~ print(table.serialize(t,"name",true), "\n") ---~ print(table.serialize(t,"name",true,true), "\n") - -do - - local function flatten(t,f,complete) - for i=1,#t do - local v = t[i] - if type(v) == "table" then - if complete or type(v[1]) == "table" then - flatten(v,f,complete) - else - f[#f+1] = v - end +local function flatten(t,f,complete) -- is this used? meybe a variant with next, ... + for i=1,#t do + local v = t[i] + if type(v) == "table" then + if complete or type(v[1]) == "table" then + flatten(v,f,complete) else f[#f+1] = v end + else + f[#f+1] = v end end +end - function table.flatten(t) - local f = { } - flatten(t,f,true) - return f - end +function table.flatten(t) + local f = { } + flatten(t,f,true) + return f +end + +function table.unnest(t) -- bad name + local f = { } + flatten(t,f,false) + return f +end - function table.unnest(t) -- bad name - local f = { } - flatten(t,f,false) - return f +table.flatten_one_level = table.unnest + +-- a better one: + +local function flattened(t,f) + if not f then + f = { } + end + for k, v in next, t do + if type(v) == "table" then + flattened(v,f) + else + f[k] = v + end end + return f +end + +table.flattened = flattened - table.flatten_one_level = table.unnest +-- the next three may disappear +function table.remove_value(t,value) -- todo: n + if value then + for i=1,#t do + if t[i] == value then + remove(t,i) + -- remove all, so no: return + end + end + end end function table.insert_before_value(t,value,str) - for i=1,#t do - if t[i] == value then - table.insert(t,i,str) - return + if str then + if value then + for i=1,#t do + if t[i] == value then + insert(t,i,str) + return + end + end end + insert(t,1,str) + elseif value then + insert(t,1,value) end - table.insert(t,1,str) end function table.insert_after_value(t,value,str) - for i=1,#t do - if t[i] == value then - table.insert(t,i+1,str) - return + if str then + if value then + for i=1,#t do + if t[i] == value then + insert(t,i+1,str) + return + end + end end + t[#t+1] = str + elseif value then + t[#t+1] = value end - t[#t+1] = str end -function table.are_equal(a,b,n,m) - if #a == #b then +local function are_equal(a,b,n,m) -- indexed + if a and b and #a == #b then n = n or 1 m = m or #a for i=n,m do local ai, bi = a[i], b[i] - if (ai==bi) or (type(ai)=="table" and type(bi)=="table" and table.are_equal(ai,bi)) then - -- continue + if ai==bi then + -- same + elseif type(ai)=="table" and type(bi)=="table" then + if not are_equal(ai,bi) then + return false + end else return false end @@ -1002,37 +1238,42 @@ function table.are_equal(a,b,n,m) end end -function table.compact(t) - if t then - for k,v in pairs(t) do - if not next(v) then - t[k] = nil +local function identical(a,b) -- assumes same structure + for ka, va in next, a do + local vb = b[k] + if va == vb then + -- same + elseif type(va) == "table" and type(vb) == "table" then + if not identical(va,vb) then + return false end + else + return false end end + return true end -function table.tohash(t) - local h = { } - for _, v in pairs(t) do -- no ipairs here - h[v] = true - end - return h -end +table.are_equal = are_equal +table.identical = identical -function table.fromhash(t) - local h = { } - for k, v in pairs(t) do -- no ipairs here - if v then h[#h+1] = k end +-- maybe also make a combined one + +function table.compact(t) + if t then + for k,v in next, t do + if not next(v) then + t[k] = nil + end + end end - return h end function table.contains(t, v) if t then for i=1, #t do if t[i] == v then - return true + return i end end end @@ -1049,7 +1290,7 @@ end function table.swapped(t) local s = { } - for k, v in pairs(t) do + for k, v in next, t do s[v] = k end return s @@ -1065,21 +1306,20 @@ function table.clone(t,p) -- t is optional or nil or table elseif not t then t = { } end - setmetatable(t, { __index = function(_,key) return p[key] end }) + setmetatable(t, { __index = function(_,key) return p[key] end }) -- why not __index = p ? return t end - function table.hexed(t,seperator) local tt = { } - for i=1,#t do tt[i] = string.format("0x%04X",t[i]) end - return table.concat(tt,seperator or " ") + for i=1,#t do tt[i] = format("0x%04X",t[i]) end + return concat(tt,seperator or " ") end function table.reverse_hash(h) local r = { } - for k,v in pairs(h) do - r[v] = (k:gsub(" ","")):lower() + for k,v in next, h do + r[v] = lower(gsub(k," ","")) end return r end @@ -1094,14 +1334,50 @@ function table.reverse(t) return tt end +function table.insert_before_value(t,value,extra) + for i=1,#t do + if t[i] == extra then + remove(t,i) + end + end + for i=1,#t do + if t[i] == value then + insert(t,i,extra) + return + end + end + insert(t,1,extra) +end --- filename : l-io.lua --- comment : split off from luat-lib --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +function table.insert_after_value(t,value,extra) + for i=1,#t do + if t[i] == extra then + remove(t,i) + end + end + for i=1,#t do + if t[i] == value then + insert(t,i+1,extra) + return + end + end + insert(t,#t+1,extra) +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['l-io'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} -if not versions then versions = { } end versions['l-io'] = 1.001 +local byte, find, gsub = string.byte, string.find, string.gsub if string.find(os.getenv("PATH"),";") then io.fileseparator, io.pathseparator = "\\", ";" @@ -1109,10 +1385,12 @@ else io.fileseparator, io.pathseparator = "/" , ":" end -function io.loaddata(filename) - local f = io.open(filename,'rb') +function io.loaddata(filename,textmode) + local f = io.open(filename,(textmode and 'r') or 'rb') if f then + -- collectgarbage("step") -- sometimes makes a big difference in mem consumption local data = f:read('*all') + -- garbagecollector.check(data) f:close() return data else @@ -1121,16 +1399,19 @@ function io.loaddata(filename) end function io.savedata(filename,data,joiner) - local f = io.open(filename, "wb") + local f = io.open(filename,"wb") if f then if type(data) == "table" then f:write(table.join(data,joiner or "")) elseif type(data) == "function" then data(f) else - f:write(data) + f:write(data or "") end f:close() + return true + else + return false end end @@ -1164,146 +1445,83 @@ function io.noflines(f) return n end -do - - local sb = string.byte - - local nextchar = { - [ 4] = function(f) - return f:read(1,1,1,1) - end, - [ 2] = function(f) - return f:read(1,1) - end, - [ 1] = function(f) - return f:read(1) - end, - [-2] = function(f) - local a, b = f:read(1,1) - return b, a - end, - [-4] = function(f) - local a, b, c, d = f:read(1,1,1,1) - return d, c, b, a - end - } - - function io.characters(f,n) - if f then - return nextchar[n or 1], f - else - return nil, nil - end +local nextchar = { + [ 4] = function(f) + return f:read(1,1,1,1) + end, + [ 2] = function(f) + return f:read(1,1) + end, + [ 1] = function(f) + return f:read(1) + end, + [-2] = function(f) + local a, b = f:read(1,1) + return b, a + end, + [-4] = function(f) + local a, b, c, d = f:read(1,1,1,1) + return d, c, b, a end +} +function io.characters(f,n) + if f then + return nextchar[n or 1], f + else + return nil, nil + end end -do - - local sb = string.byte - ---~ local nextbyte = { ---~ [4] = function(f) ---~ local a = f:read(1) ---~ local b = f:read(1) ---~ local c = f:read(1) ---~ local d = f:read(1) ---~ if d then ---~ return sb(a), sb(b), sb(c), sb(d) ---~ else ---~ return nil, nil, nil, nil ---~ end ---~ end, ---~ [2] = function(f) ---~ local a = f:read(1) ---~ local b = f:read(1) ---~ if b then ---~ return sb(a), sb(b) ---~ else ---~ return nil, nil ---~ end ---~ end, ---~ [1] = function (f) ---~ local a = f:read(1) ---~ if a then ---~ return sb(a) ---~ else ---~ return nil ---~ end ---~ end, ---~ [-2] = function (f) ---~ local a = f:read(1) ---~ local b = f:read(1) ---~ if b then ---~ return sb(b), sb(a) ---~ else ---~ return nil, nil ---~ end ---~ end, ---~ [-4] = function(f) ---~ local a = f:read(1) ---~ local b = f:read(1) ---~ local c = f:read(1) ---~ local d = f:read(1) ---~ if d then ---~ return sb(d), sb(c), sb(b), sb(a) ---~ else ---~ return nil, nil, nil, nil ---~ end ---~ end ---~ } - - local nextbyte = { - [4] = function(f) - local a, b, c, d = f:read(1,1,1,1) - if d then - return sb(a), sb(b), sb(c), sb(d) - else - return nil, nil, nil, nil - end - end, - [2] = function(f) - local a, b = f:read(1,1) - if b then - return sb(a), sb(b) - else - return nil, nil - end - end, - [1] = function (f) - local a = f:read(1) - if a then - return sb(a) - else - return nil - end - end, - [-2] = function (f) - local a, b = f:read(1,1) - if b then - return sb(b), sb(a) - else - return nil, nil - end - end, - [-4] = function(f) - local a, b, c, d = f:read(1,1,1,1) - if d then - return sb(d), sb(c), sb(b), sb(a) - else - return nil, nil, nil, nil - end +local nextbyte = { + [4] = function(f) + local a, b, c, d = f:read(1,1,1,1) + if d then + return byte(a), byte(b), byte(c), byte(d) + else + return nil, nil, nil, nil end - } - - function io.bytes(f,n) - if f then - return nextbyte[n or 1], f + end, + [2] = function(f) + local a, b = f:read(1,1) + if b then + return byte(a), byte(b) + else + return nil, nil + end + end, + [1] = function (f) + local a = f:read(1) + if a then + return byte(a) + else + return nil + end + end, + [-2] = function (f) + local a, b = f:read(1,1) + if b then + return byte(b), byte(a) else return nil, nil end + end, + [-4] = function(f) + local a, b, c, d = f:read(1,1,1,1) + if d then + return byte(d), byte(c), byte(b), byte(a) + else + return nil, nil, nil, nil + end end +} +function io.bytes(f,n) + if f then + return nextbyte[n or 1], f + else + return nil, nil + end end function io.ask(question,default,options) @@ -1317,20 +1535,21 @@ function io.ask(question,default,options) end io.write(string.format(" ")) local answer = io.read() - answer = answer:gsub("^%s*(.*)%s*$","%1") + answer = gsub(answer,"^%s*(.*)%s*$","%1") if answer == "" and default then return default elseif not options then return answer else - for _,v in pairs(options) do - if v == answer then + for k=1,#options do + if options[k] == answer then return answer end end local pattern = "^" .. answer - for _,v in pairs(options) do - if v:find(pattern) then + for k=1,#options do + local v = options[k] + if find(v,pattern) then return v end end @@ -1339,24 +1558,30 @@ function io.ask(question,default,options) end --- filename : l-number.lua --- comment : split off from luat-lib --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['l-number'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} -if not versions then versions = { } end versions['l-number'] = 1.001 +local tostring = tostring +local format, floor, insert, match = string.format, math.floor, table.insert, string.match +local lpegmatch = lpeg.match -if not number then number = { } end +number = number or { } -- a,b,c,d,e,f = number.toset(100101) function number.toset(n) - return (tostring(n)):match("(.?)(.?)(.?)(.?)(.?)(.?)(.?)(.?)") + return match(tostring(n),"(.?)(.?)(.?)(.?)(.?)(.?)(.?)(.?)") end -local format = string.format - function number.toevenhex(n) local s = format("%X",n) if #s % 2 == 0 then @@ -1377,72 +1602,100 @@ end -- -- of course dedicated "(.)(.)(.)(.)" matches are even faster -do - local one = lpeg.C(1-lpeg.S(''))^1 +local one = lpeg.C(1-lpeg.S(''))^1 - function number.toset(n) - return one:match(tostring(n)) - end +function number.toset(n) + return lpegmatch(one,tostring(n)) end +function number.bits(n,zero) + local t, i = { }, (zero and 0) or 1 + while n > 0 do + local m = n % 2 + if m > 0 then + insert(t,1,i) + end + n = floor(n/2) + i = i + 1 + end + return t +end --- filename : l-set.lua --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +end -- of closure -if not versions then versions = { } end versions['l-set'] = 1.001 +do -- create closure to overcome 200 locals limit -if not set then set = { } end +if not modules then modules = { } end modules ['l-set'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} -do +set = set or { } - local nums = { } - local tabs = { } - local concat = table.concat +local nums = { } +local tabs = { } +local concat = table.concat +local next, type = next, type - set.create = table.tohash +set.create = table.tohash - function set.tonumber(t) - if next(t) then - local s = "" - -- we could save mem by sorting, but it slows down - for k, v in pairs(t) do - if v then - -- why bother about the leading space - s = s .. " " .. k - end - end - if not nums[s] then - tabs[#tabs+1] = t - nums[s] = #tabs +function set.tonumber(t) + if next(t) then + local s = "" + -- we could save mem by sorting, but it slows down + for k, v in next, t do + if v then + -- why bother about the leading space + s = s .. " " .. k end - return nums[s] - else - return 0 end + local n = nums[s] + if not n then + n = #tabs + 1 + tabs[n] = t + nums[s] = n + end + return n + else + return 0 end +end - function set.totable(n) - if n == 0 then - return { } - else - return tabs[n] or { } - end +function set.totable(n) + if n == 0 then + return { } + else + return tabs[n] or { } end +end - function set.contains(n,s) - if type(n) == "table" then - return n[s] - elseif n == 0 then - return false - else - local t = tabs[n] - return t and t[s] +function set.tolist(n) + if n == 0 or not tabs[n] then + return "" + else + local t = { } + for k, v in next, tabs[n] do + if v then + t[#t+1] = k + end end + return concat(t," ") end +end +function set.contains(n,s) + if type(n) == "table" then + return n[s] + elseif n == 0 then + return false + else + local t = tabs[n] + return t and t[s] + end end --~ local c = set.create{'aap','noot','mies'} @@ -1459,46 +1712,69 @@ end --- filename : l-os.lua --- comment : split off from luat-lib --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +end -- of closure -if not versions then versions = { } end versions['l-os'] = 1.001 +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['l-os'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- maybe build io.flush in os.execute + +local find, format, gsub = string.find, string.format, string.gsub +local random, ceil = math.random, math.ceil + +local execute, spawn, exec, ioflush = os.execute, os.spawn or os.execute, os.exec or os.execute, io.flush + +function os.execute(...) ioflush() return execute(...) end +function os.spawn (...) ioflush() return spawn (...) end +function os.exec (...) ioflush() return exec (...) end function os.resultof(command) - return io.popen(command,"r"):read("*all") + ioflush() -- else messed up logging + local handle = io.popen(command,"r") + if not handle then + -- print("unknown command '".. command .. "' in os.resultof") + return "" + else + return handle:read("*all") or "" + end end -if not os.exec then os.exec = os.execute end -if not os.spawn then os.spawn = os.execute end - ---~ os.type : windows | unix (new, we already guessed os.platform) ---~ os.name : windows | msdos | linux | macosx | solaris | .. | generic (new) +--~ os.type : windows | unix (new, we already guessed os.platform) +--~ os.name : windows | msdos | linux | macosx | solaris | .. | generic (new) +--~ os.platform : extended os.name with architecture if not io.fileseparator then - if string.find(os.getenv("PATH"),";") then - io.fileseparator, io.pathseparator, os.platform = "\\", ";", os.type or "windows" + if find(os.getenv("PATH"),";") then + io.fileseparator, io.pathseparator, os.type = "\\", ";", os.type or "mswin" else - io.fileseparator, io.pathseparator, os.platform = "/" , ":", os.type or "unix" + io.fileseparator, io.pathseparator, os.type = "/" , ":", os.type or "unix" end end -os.platform = os.platform or os.type or (io.pathseparator == ";" and "windows") or "unix" +os.type = os.type or (io.pathseparator == ";" and "windows") or "unix" +os.name = os.name or (os.type == "windows" and "mswin" ) or "linux" + +if os.type == "windows" then + os.libsuffix, os.binsuffix = 'dll', 'exe' +else + os.libsuffix, os.binsuffix = 'so', '' +end function os.launch(str) - if os.platform == "windows" then + if os.type == "windows" then os.execute("start " .. str) -- os.spawn ? else os.execute(str .. " &") -- os.spawn ? end end -if not os.setenv then - function os.setenv() return false end -end - if not os.times then -- utime = user time -- stime = system time @@ -1516,11 +1792,10 @@ end os.gettimeofday = os.gettimeofday or os.clock -do - local startuptime = os.gettimeofday() - function os.runtime() - return os.gettimeofday() - startuptime - end +local startuptime = os.gettimeofday() + +function os.runtime() + return os.gettimeofday() - startuptime end --~ print(os.gettimeofday()-os.time()) @@ -1529,43 +1804,249 @@ end --~ print(os.date("%H:%M:%S",os.gettimeofday())) --~ print(os.date("%H:%M:%S",os.time())) +-- no need for function anymore as we have more clever code and helpers now +-- this metatable trickery might as well disappear + +os.resolvers = os.resolvers or { } + +local resolvers = os.resolvers --- filename : l-md5.lua --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +local osmt = getmetatable(os) or { __index = function(t,k) t[k] = "unset" return "unset" end } -- maybe nil +local osix = osmt.__index -if not versions then versions = { } end versions['l-md5'] = 1.001 +osmt.__index = function(t,k) + return (resolvers[k] or osix)(t,k) +end + +setmetatable(os,osmt) -if md5 then do +if not os.setenv then - local function convert(str,fmt) - return (string.gsub(md5.sum(str),".",function(chr) return string.format(fmt,string.byte(chr)) end)) + -- we still store them but they won't be seen in + -- child processes although we might pass them some day + -- using command concatination + + local env, getenv = { }, os.getenv + + function os.setenv(k,v) + env[k] = v end - if not md5.HEX then function md5.HEX(str) return convert(str,"%02X") end end - if not md5.hex then function md5.hex(str) return convert(str,"%02x") end end - if not md5.dec then function md5.dec(str) return convert(str,"%03i") end end + function os.getenv(k) + return env[k] or getenv(k) + end -end end +end +-- we can use HOSTTYPE on some platforms --- filename : l-file.lua --- comment : split off from luat-lib --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +local name, platform = os.name or "linux", os.getenv("MTX_PLATFORM") or "" -if not versions then versions = { } end versions['l-file'] = 1.001 +local function guess() + local architecture = os.resultof("uname -m") or "" + if architecture ~= "" then + return architecture + end + architecture = os.getenv("HOSTTYPE") or "" + if architecture ~= "" then + return architecture + end + return os.resultof("echo $HOSTTYPE") or "" +end + +if platform ~= "" then + + os.platform = platform + +elseif os.type == "windows" then + + -- we could set the variable directly, no function needed here + + function os.resolvers.platform(t,k) + local platform, architecture = "", os.getenv("PROCESSOR_ARCHITECTURE") or "" + if find(architecture,"AMD64") then + platform = "mswin-64" + else + platform = "mswin" + end + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end -if not file then file = { } end +elseif name == "linux" then + + function os.resolvers.platform(t,k) + -- we sometims have HOSTTYPE set so let's check that first + local platform, architecture = "", os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" + if find(architecture,"x86_64") then + platform = "linux-64" + elseif find(architecture,"ppc") then + platform = "linux-ppc" + else + platform = "linux" + end + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +elseif name == "macosx" then + + --[[ + Identifying the architecture of OSX is quite a mess and this + is the best we can come up with. For some reason $HOSTTYPE is + a kind of pseudo environment variable, not known to the current + environment. And yes, uname cannot be trusted either, so there + is a change that you end up with a 32 bit run on a 64 bit system. + Also, some proper 64 bit intel macs are too cheap (low-end) and + therefore not permitted to run the 64 bit kernel. + ]]-- + + function os.resolvers.platform(t,k) + -- local platform, architecture = "", os.getenv("HOSTTYPE") or "" + -- if architecture == "" then + -- architecture = os.resultof("echo $HOSTTYPE") or "" + -- end + local platform, architecture = "", os.resultof("echo $HOSTTYPE") or "" + if architecture == "" then + -- print("\nI have no clue what kind of OSX you're running so let's assume an 32 bit intel.\n") + platform = "osx-intel" + elseif find(architecture,"i386") then + platform = "osx-intel" + elseif find(architecture,"x86_64") then + platform = "osx-64" + else + platform = "osx-ppc" + end + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +elseif name == "sunos" then + + function os.resolvers.platform(t,k) + local platform, architecture = "", os.resultof("uname -m") or "" + if find(architecture,"sparc") then + platform = "solaris-sparc" + else -- if architecture == 'i86pc' + platform = "solaris-intel" + end + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +elseif name == "freebsd" then + + function os.resolvers.platform(t,k) + local platform, architecture = "", os.resultof("uname -m") or "" + if find(architecture,"amd64") then + platform = "freebsd-amd64" + else + platform = "freebsd" + end + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +elseif name == "kfreebsd" then + + function os.resolvers.platform(t,k) + -- we sometims have HOSTTYPE set so let's check that first + local platform, architecture = "", os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" + if find(architecture,"x86_64") then + platform = "kfreebsd-64" + else + platform = "kfreebsd-i386" + end + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +else + + -- platform = "linux" + -- os.setenv("MTX_PLATFORM",platform) + -- os.platform = platform + + function os.resolvers.platform(t,k) + local platform = "linux" + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +end + +-- beware, we set the randomseed + +-- from wikipedia: Version 4 UUIDs use a scheme relying only on random numbers. This algorithm sets the +-- version number as well as two reserved bits. All other bits are set using a random or pseudorandom +-- data source. Version 4 UUIDs have the form xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx with hexadecimal +-- digits x and hexadecimal digits 8, 9, A, or B for y. e.g. f47ac10b-58cc-4372-a567-0e02b2c3d479. +-- +-- as we don't call this function too often there is not so much risk on repetition + +local t = { 8, 9, "a", "b" } + +function os.uuid() + return format("%04x%04x-4%03x-%s%03x-%04x-%04x%04x%04x", + random(0xFFFF),random(0xFFFF), + random(0x0FFF), + t[ceil(random(4))] or 8,random(0x0FFF), + random(0xFFFF), + random(0xFFFF),random(0xFFFF),random(0xFFFF) + ) +end + +local d + +function os.timezone(delta) + d = d or tonumber(tonumber(os.date("%H")-os.date("!%H"))) + if delta then + if d > 0 then + return format("+%02i:00",d) + else + return format("-%02i:00",-d) + end + else + return 1 + end +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['l-file'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- needs a cleanup + +file = file or { } + +local concat = table.concat +local find, gmatch, match, gsub, sub, char = string.find, string.gmatch, string.match, string.gsub, string.sub, string.char +local lpegmatch = lpeg.match function file.removesuffix(filename) - return filename:gsub("%.[%a%d]+$", "") + return (gsub(filename,"%.[%a%d]+$","")) end function file.addsuffix(filename, suffix) - if not filename:find("%.[%a%d]+$") then + if not suffix or suffix == "" then + return filename + elseif not find(filename,"%.[%a%d]+$") then return filename .. "." .. suffix else return filename @@ -1573,141 +2054,145 @@ function file.addsuffix(filename, suffix) end function file.replacesuffix(filename, suffix) - if not filename:find("%.[%a%d]+$") then - return filename .. "." .. suffix - else - return (filename:gsub("%.[%a%d]+$","."..suffix)) - end + return (gsub(filename,"%.[%a%d]+$","")) .. "." .. suffix end -function file.dirname(name) - return name:match("^(.+)[/\\].-$") or "" +function file.dirname(name,default) + return match(name,"^(.+)[/\\].-$") or (default or "") end function file.basename(name) - return name:match("^.+[/\\](.-)$") or name + return match(name,"^.+[/\\](.-)$") or name end function file.nameonly(name) - return ((name:match("^.+[/\\](.-)$") or name):gsub("%..*$","")) + return (gsub(match(name,"^.+[/\\](.-)$") or name,"%..*$","")) end -function file.extname(name) - return name:match("^.+%.([^/\\]-)$") or "" +function file.extname(name,default) + return match(name,"^.+%.([^/\\]-)$") or default or "" end file.suffix = file.extname -function file.stripsuffix(name) - return (name:gsub("%.[%a%d]+$","")) -end - --~ function file.join(...) ---~ local t = { ... } ---~ for i=1,#t do ---~ t[i] = (t[i]:gsub("\\","/")):gsub("/+$","") +--~ local pth = concat({...},"/") +--~ pth = gsub(pth,"\\","/") +--~ local a, b = match(pth,"^(.*://)(.*)$") +--~ if a and b then +--~ return a .. gsub(b,"//+","/") +--~ end +--~ a, b = match(pth,"^(//)(.*)$") +--~ if a and b then +--~ return a .. gsub(b,"//+","/") --~ end ---~ return table.concat(t,"/") +--~ return (gsub(pth,"//+","/")) --~ end ---~ print(file.join("x/","/y")) ---~ print(file.join("http://","/y")) ---~ print(file.join("http://a","/y")) ---~ print(file.join("http:///a","/y")) ---~ print(file.join("//nas-1","/y")) +local trick_1 = char(1) +local trick_2 = "^" .. trick_1 .. "/+" function file.join(...) - local pth = table.concat({...},"/") - pth = pth:gsub("\\","/") - local a, b = pth:match("^(.*://)(.*)$") + local lst = { ... } + local a, b = lst[1], lst[2] + if a == "" then + lst[1] = trick_1 + elseif b and find(a,"^/+$") and find(b,"^/") then + lst[1] = "" + lst[2] = gsub(b,"^/+","") + end + local pth = concat(lst,"/") + pth = gsub(pth,"\\","/") + local a, b = match(pth,"^(.*://)(.*)$") if a and b then - return a .. b:gsub("//+","/") + return a .. gsub(b,"//+","/") end - a, b = pth:match("^(//)(.*)$") + a, b = match(pth,"^(//)(.*)$") if a and b then - return a .. b:gsub("//+","/") + return a .. gsub(b,"//+","/") end - return (pth:gsub("//+","/")) + pth = gsub(pth,trick_2,"") + return (gsub(pth,"//+","/")) end -function file.is_writable(name) - local f = io.open(name, 'w') - if f then - f:close() - return true - else - return false - end +--~ print(file.join("//","/y")) +--~ print(file.join("/","/y")) +--~ print(file.join("","/y")) +--~ print(file.join("/x/","/y")) +--~ print(file.join("x/","/y")) +--~ print(file.join("http://","/y")) +--~ print(file.join("http://a","/y")) +--~ print(file.join("http:///a","/y")) +--~ print(file.join("//nas-1","/y")) + +function file.iswritable(name) + local a = lfs.attributes(name) or lfs.attributes(file.dirname(name,".")) + return a and sub(a.permissions,2,2) == "w" end -function file.is_readable(name) - local f = io.open(name,'r') - if f then - f:close() - return true - else - return false - end +function file.isreadable(name) + local a = lfs.attributes(name) + return a and sub(a.permissions,1,1) == "r" end +file.is_readable = file.isreadable +file.is_writable = file.iswritable + +-- todo: lpeg + --~ function file.split_path(str) ---~ if str:find(';') then ---~ return str:splitchr(";") ---~ else ---~ return str:splitchr(io.pathseparator) +--~ local t = { } +--~ str = gsub(str,"\\", "/") +--~ str = gsub(str,"(%a):([;/])", "%1\001%2") +--~ for name in gmatch(str,"([^;:]+)") do +--~ if name ~= "" then +--~ t[#t+1] = gsub(name,"\001",":") +--~ end --~ end +--~ return t --~ end --- todo: lpeg +local checkedsplit = string.checkedsplit -function file.split_path(str) - local t = { } - str = str:gsub("\\", "/") - str = str:gsub("(%a):([;/])", "%1\001%2") - for name in str:gmatch("([^;:]+)") do - if name ~= "" then - name = name:gsub("\001",":") - t[#t+1] = name - end - end - return t +function file.split_path(str,separator) + str = gsub(str,"\\","/") + return checkedsplit(str,separator or io.pathseparator) end function file.join_path(tab) - return table.concat(tab,io.pathseparator) -- can have trailing // -end - ---~ print('test' .. " == " .. file.collapse_path("test")) ---~ print("test/test" .. " == " .. file.collapse_path("test/test")) ---~ print("test/test/test" .. " == " .. file.collapse_path("test/test/test")) ---~ print("test/test" .. " == " .. file.collapse_path("test/../test/test")) ---~ print("test" .. " == " .. file.collapse_path("test/../test")) ---~ print("../test" .. " == " .. file.collapse_path("../test")) ---~ print("../test/" .. " == " .. file.collapse_path("../test/")) ---~ print("a/a" .. " == " .. file.collapse_path("a/b/c/../../a")) - ---~ function file.collapse_path(str) ---~ local ok, n = false, 0 ---~ while not ok do ---~ ok = true ---~ str, n = str:gsub("[^%./]+/%.%./", function(s) ---~ ok = false ---~ return "" ---~ end) ---~ end ---~ return (str:gsub("/%./","/")) ---~ end + return concat(tab,io.pathseparator) -- can have trailing // +end + +-- we can hash them weakly function file.collapse_path(str) - local n = 1 - while n > 0 do - str, n = str:gsub("([^/%.]+/%.%./)","") - end - return (str:gsub("/%./","/")) -end + str = gsub(str,"\\","/") + if find(str,"/") then + str = gsub(str,"^%./",(gsub(lfs.currentdir(),"\\","/")) .. "/") -- ./xx in qualified + str = gsub(str,"/%./","/") + local n, m = 1, 1 + while n > 0 or m > 0 do + str, n = gsub(str,"[^/%.]+/%.%.$","") + str, m = gsub(str,"[^/%.]+/%.%./","") + end + str = gsub(str,"([^/])/$","%1") + -- str = gsub(str,"^%./","") -- ./xx in qualified + str = gsub(str,"/%.$","") + end + if str == "" then str = "." end + return str +end + +--~ print(file.collapse_path("/a")) +--~ print(file.collapse_path("a/./b/..")) +--~ print(file.collapse_path("a/aa/../b/bb")) +--~ print(file.collapse_path("a/../..")) +--~ print(file.collapse_path("a/.././././b/..")) +--~ print(file.collapse_path("a/./././b/..")) +--~ print(file.collapse_path("a/b/c/../..")) function file.robustname(str) - return (str:gsub("[^%a%d%/%-%.\\]+","-")) + return (gsub(str,"[^%a%d%/%-%.\\]+","-")) end file.readdata = io.loaddata @@ -1717,14 +2202,236 @@ function file.copy(oldname,newname) file.savedata(newname,io.loaddata(oldname)) end +-- lpeg variants, slightly faster, not always + +--~ local period = lpeg.P(".") +--~ local slashes = lpeg.S("\\/") +--~ local noperiod = 1-period +--~ local noslashes = 1-slashes +--~ local name = noperiod^1 + +--~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * lpeg.C(noperiod^1) * -1 + +--~ function file.extname(name) +--~ return lpegmatch(pattern,name) or "" +--~ end + +--~ local pattern = lpeg.Cs(((period * noperiod^1 * -1)/"" + 1)^1) + +--~ function file.removesuffix(name) +--~ return lpegmatch(pattern,name) +--~ end + +--~ local pattern = (noslashes^0 * slashes)^1 * lpeg.C(noslashes^1) * -1 + +--~ function file.basename(name) +--~ return lpegmatch(pattern,name) or name +--~ end + +--~ local pattern = (noslashes^0 * slashes)^1 * lpeg.Cp() * noslashes^1 * -1 + +--~ function file.dirname(name) +--~ local p = lpegmatch(pattern,name) +--~ if p then +--~ return sub(name,1,p-2) +--~ else +--~ return "" +--~ end +--~ end + +--~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * lpeg.Cp() * noperiod^1 * -1 + +--~ function file.addsuffix(name, suffix) +--~ local p = lpegmatch(pattern,name) +--~ if p then +--~ return name +--~ else +--~ return name .. "." .. suffix +--~ end +--~ end + +--~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * lpeg.Cp() * noperiod^1 * -1 + +--~ function file.replacesuffix(name,suffix) +--~ local p = lpegmatch(pattern,name) +--~ if p then +--~ return sub(name,1,p-2) .. "." .. suffix +--~ else +--~ return name .. "." .. suffix +--~ end +--~ end + +--~ local pattern = (noslashes^0 * slashes)^0 * lpeg.Cp() * ((noperiod^1 * period)^1 * lpeg.Cp() + lpeg.P(true)) * noperiod^1 * -1 + +--~ function file.nameonly(name) +--~ local a, b = lpegmatch(pattern,name) +--~ if b then +--~ return sub(name,a,b-2) +--~ elseif a then +--~ return sub(name,a) +--~ else +--~ return name +--~ end +--~ end + +--~ local test = file.extname +--~ local test = file.basename +--~ local test = file.dirname +--~ local test = file.addsuffix +--~ local test = file.replacesuffix +--~ local test = file.nameonly + +--~ print(1,test("./a/b/c/abd.def.xxx","!!!")) +--~ print(2,test("./../b/c/abd.def.xxx","!!!")) +--~ print(3,test("a/b/c/abd.def.xxx","!!!")) +--~ print(4,test("a/b/c/def.xxx","!!!")) +--~ print(5,test("a/b/c/def","!!!")) +--~ print(6,test("def","!!!")) +--~ print(7,test("def.xxx","!!!")) + +--~ local tim = os.clock() for i=1,250000 do local ext = test("abd.def.xxx","!!!") end print(os.clock()-tim) + +-- also rewrite previous + +local letter = lpeg.R("az","AZ") + lpeg.S("_-+") +local separator = lpeg.P("://") + +local qualified = lpeg.P(".")^0 * lpeg.P("/") + letter*lpeg.P(":") + letter^1*separator + letter^1 * lpeg.P("/") +local rootbased = lpeg.P("/") + letter*lpeg.P(":") + +-- ./name ../name /name c: :// name/name + +function file.is_qualified_path(filename) + return lpegmatch(qualified,filename) ~= nil +end + +function file.is_rootbased_path(filename) + return lpegmatch(rootbased,filename) ~= nil +end + +local slash = lpeg.S("\\/") +local period = lpeg.P(".") +local drive = lpeg.C(lpeg.R("az","AZ")) * lpeg.P(":") +local path = lpeg.C(((1-slash)^0 * slash)^0) +local suffix = period * lpeg.C(lpeg.P(1-period)^0 * lpeg.P(-1)) +local base = lpeg.C((1-suffix)^0) + +local pattern = (drive + lpeg.Cc("")) * (path + lpeg.Cc("")) * (base + lpeg.Cc("")) * (suffix + lpeg.Cc("")) + +function file.splitname(str) -- returns drive, path, base, suffix + return lpegmatch(pattern,str) +end + +-- function test(t) for k, v in next, t do print(v, "=>", file.splitname(v)) end end +-- +-- test { "c:", "c:/aa", "c:/aa/bb", "c:/aa/bb/cc", "c:/aa/bb/cc.dd", "c:/aa/bb/cc.dd.ee" } +-- test { "c:", "c:aa", "c:aa/bb", "c:aa/bb/cc", "c:aa/bb/cc.dd", "c:aa/bb/cc.dd.ee" } +-- test { "/aa", "/aa/bb", "/aa/bb/cc", "/aa/bb/cc.dd", "/aa/bb/cc.dd.ee" } +-- test { "aa", "aa/bb", "aa/bb/cc", "aa/bb/cc.dd", "aa/bb/cc.dd.ee" } + +--~ -- todo: +--~ +--~ if os.type == "windows" then +--~ local currentdir = lfs.currentdir +--~ function lfs.currentdir() +--~ return (gsub(currentdir(),"\\","/")) +--~ end +--~ end --- filename : l-url.lua --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files -if not versions then versions = { } end versions['l-url'] = 1.001 -if not url then url = { } end +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['l-md5'] = { + version = 1.001, + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- This also provides file checksums and checkers. + +local gsub, format, byte = string.gsub, string.format, string.byte + +local function convert(str,fmt) + return (gsub(md5.sum(str),".",function(chr) return format(fmt,byte(chr)) end)) +end + +if not md5.HEX then function md5.HEX(str) return convert(str,"%02X") end end +if not md5.hex then function md5.hex(str) return convert(str,"%02x") end end +if not md5.dec then function md5.dec(str) return convert(str,"%03i") end end + +--~ if not md5.HEX then +--~ local function remap(chr) return format("%02X",byte(chr)) end +--~ function md5.HEX(str) return (gsub(md5.sum(str),".",remap)) end +--~ end +--~ if not md5.hex then +--~ local function remap(chr) return format("%02x",byte(chr)) end +--~ function md5.hex(str) return (gsub(md5.sum(str),".",remap)) end +--~ end +--~ if not md5.dec then +--~ local function remap(chr) return format("%03i",byte(chr)) end +--~ function md5.dec(str) return (gsub(md5.sum(str),".",remap)) end +--~ end + +file.needs_updating_threshold = 1 + +function file.needs_updating(oldname,newname) -- size modification access change + local oldtime = lfs.attributes(oldname, modification) + local newtime = lfs.attributes(newname, modification) + if newtime >= oldtime then + return false + elseif oldtime - newtime < file.needs_updating_threshold then + return false + else + return true + end +end + +function file.checksum(name) + if md5 then + local data = io.loaddata(name) + if data then + return md5.HEX(data) + end + end + return nil +end + +function file.loadchecksum(name) + if md5 then + local data = io.loaddata(name .. ".md5") + return data and (gsub(data,"%s","")) + end + return nil +end + +function file.savechecksum(name, checksum) + if not checksum then checksum = file.checksum(name) end + if checksum then + io.savedata(name .. ".md5",checksum) + return checksum + end + return nil +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['l-url'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local char, gmatch, gsub = string.char, string.gmatch, string.gsub +local tonumber, type = tonumber, type +local lpegmatch = lpeg.match -- from the spec (on the web): -- @@ -1736,52 +2443,79 @@ if not url then url = { } end -- / \ / \ -- urn:example:animal:ferret:nose -do +url = url or { } - local function tochar(s) - return string.char(tonumber(s,16)) - end +local function tochar(s) + return char(tonumber(s,16)) +end - local colon, qmark, hash, slash, percent, endofstring = lpeg.P(":"), lpeg.P("?"), lpeg.P("#"), lpeg.P("/"), lpeg.P("%"), lpeg.P(-1) +local colon, qmark, hash, slash, percent, endofstring = lpeg.P(":"), lpeg.P("?"), lpeg.P("#"), lpeg.P("/"), lpeg.P("%"), lpeg.P(-1) - local hexdigit = lpeg.R("09","AF","af") - local escaped = percent * lpeg.C(hexdigit * hexdigit) / tochar +local hexdigit = lpeg.R("09","AF","af") +local plus = lpeg.P("+") +local escaped = (plus / " ") + (percent * lpeg.C(hexdigit * hexdigit) / tochar) - local scheme = lpeg.Cs((escaped+(1-colon-slash-qmark-hash))^0) * colon + lpeg.Cc("") - local authority = slash * slash * lpeg.Cs((escaped+(1- slash-qmark-hash))^0) + lpeg.Cc("") - local path = slash * lpeg.Cs((escaped+(1- qmark-hash))^0) + lpeg.Cc("") - local query = qmark * lpeg.Cs((escaped+(1- hash))^0) + lpeg.Cc("") - local fragment = hash * lpeg.Cs((escaped+(1- endofstring))^0) + lpeg.Cc("") +-- we assume schemes with more than 1 character (in order to avoid problems with windows disks) - local parser = lpeg.Ct(scheme * authority * path * query * fragment) +local scheme = lpeg.Cs((escaped+(1-colon-slash-qmark-hash))^2) * colon + lpeg.Cc("") +local authority = slash * slash * lpeg.Cs((escaped+(1- slash-qmark-hash))^0) + lpeg.Cc("") +local path = slash * lpeg.Cs((escaped+(1- qmark-hash))^0) + lpeg.Cc("") +local query = qmark * lpeg.Cs((escaped+(1- hash))^0) + lpeg.Cc("") +local fragment = hash * lpeg.Cs((escaped+(1- endofstring))^0) + lpeg.Cc("") - function url.split(str) - return (type(str) == "string" and parser:match(str)) or str - end +local parser = lpeg.Ct(scheme * authority * path * query * fragment) + +-- todo: reconsider Ct as we can as well have five return values (saves a table) +-- so we can have two parsers, one with and one without +function url.split(str) + return (type(str) == "string" and lpegmatch(parser,str)) or str end +-- todo: cache them + function url.hashed(str) local s = url.split(str) + local somescheme = s[1] ~= "" return { - scheme = (s[1] ~= "" and s[1]) or "file", + scheme = (somescheme and s[1]) or "file", authority = s[2], - path = s[3], - query = s[4], - fragment = s[5], - original = str + path = s[3], + query = s[4], + fragment = s[5], + original = str, + noscheme = not somescheme, } end +function url.hasscheme(str) + return url.split(str)[1] ~= "" +end + +function url.addscheme(str,scheme) + return (url.hasscheme(str) and str) or ((scheme or "file:///") .. str) +end + +function url.construct(hash) + local fullurl = hash.sheme .. "://".. hash.authority .. hash.path + if hash.query then + fullurl = fullurl .. "?".. hash.query + end + if hash.fragment then + fullurl = fullurl .. "?".. hash.fragment + end + return fullurl +end + function url.filename(filename) local t = url.hashed(filename) - return (t.scheme == "file" and t.path:gsub("^/([a-zA-Z])([:|])/)","%1:")) or filename + return (t.scheme == "file" and (gsub(t.path,"^/([a-zA-Z])([:|])/)","%1:"))) or filename end function url.query(str) if type(str) == "string" then local t = { } - for k, v in str:gmatch("([^&=]*)=([^&=]*)") do + for k, v in gmatch(str,"([^&=]*)=([^&=]*)") do t[k] = v end return t @@ -1796,12 +2530,12 @@ end --~ print(url.filename("file:///etc/test.txt")) --~ print(url.filename("/oeps.txt")) --- from the spec on the web (sort of): +--~ from the spec on the web (sort of): --~ --~ function test(str) --~ print(table.serialize(url.hashed(str))) --~ end ----~ +--~ --~ test("%56pass%20words") --~ test("file:///c:/oeps.txt") --~ test("file:///c|/oeps.txt") @@ -1823,151 +2557,147 @@ end --~ test("zip:///oeps/oeps.zip?bla/bla.tex") --- filename : l-dir.lua --- comment : split off from luat-lib --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +end -- of closure -if not versions then versions = { } end versions['l-dir'] = 1.001 +do -- create closure to overcome 200 locals limit -dir = { } +if not modules then modules = { } end modules ['l-dir'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} --- optimizing for no string.find (*) does not save time +-- dir.expand_name will be merged with cleanpath and collapsepath -if lfs then do +local type = type +local find, gmatch, match, gsub = string.find, string.gmatch, string.match, string.gsub +local lpegmatch = lpeg.match ---~ local attributes = lfs.attributes ---~ local walkdir = lfs.dir ---~ ---~ local function glob_pattern(path,patt,recurse,action) ---~ local ok, scanner = xpcall(function() return walkdir(path) end, function() end) -- kepler safe ---~ if ok and type(scanner) == "function" then ---~ if not path:find("/$") then path = path .. '/' end ---~ for name in scanner do ---~ local full = path .. name ---~ local mode = attributes(full,'mode') ---~ if mode == 'file' then ---~ if name:find(patt) then ---~ action(full) ---~ end ---~ elseif recurse and (mode == "directory") and (name ~= '.') and (name ~= "..") then ---~ glob_pattern(full,patt,recurse,action) ---~ end ---~ end ---~ end ---~ end ---~ ---~ dir.glob_pattern = glob_pattern ---~ ---~ local function glob(pattern, action) ---~ local t = { } ---~ local action = action or function(name) t[#t+1] = name end ---~ local path, patt = pattern:match("^(.*)/*%*%*/*(.-)$") ---~ local recurse = path and patt ---~ if not recurse then ---~ path, patt = pattern:match("^(.*)/(.-)$") ---~ if not (path and patt) then ---~ path, patt = '.', pattern ---~ end ---~ end ---~ patt = patt:gsub("([%.%-%+])", "%%%1") ---~ patt = patt:gsub("%*", ".*") ---~ patt = patt:gsub("%?", ".") ---~ patt = "^" .. patt .. "$" ---~ -- print('path: ' .. path .. ' | pattern: ' .. patt .. ' | recurse: ' .. tostring(recurse)) ---~ glob_pattern(path,patt,recurse,action) ---~ return t ---~ end ---~ ---~ dir.glob = glob +dir = dir or { } - local attributes = lfs.attributes - local walkdir = lfs.dir +-- handy - local function glob_pattern(path,patt,recurse,action) - local ok, scanner - if path == "/" then - ok, scanner = xpcall(function() return walkdir(path..".") end, function() end) -- kepler safe - else - ok, scanner = xpcall(function() return walkdir(path) end, function() end) -- kepler safe +function dir.current() + return (gsub(lfs.currentdir(),"\\","/")) +end + +-- optimizing for no string.find (*) does not save time + +local attributes = lfs.attributes +local walkdir = lfs.dir + +local function glob_pattern(path,patt,recurse,action) + local ok, scanner + if path == "/" then + ok, scanner = xpcall(function() return walkdir(path..".") end, function() end) -- kepler safe + else + ok, scanner = xpcall(function() return walkdir(path) end, function() end) -- kepler safe + end + if ok and type(scanner) == "function" then + if not find(path,"/$") then path = path .. '/' end + for name in scanner do + local full = path .. name + local mode = attributes(full,'mode') + if mode == 'file' then + if find(full,patt) then + action(full) + end + elseif recurse and (mode == "directory") and (name ~= '.') and (name ~= "..") then + glob_pattern(full,patt,recurse,action) + end end - if ok and type(scanner) == "function" then - if not path:find("/$") then path = path .. '/' end - for name in scanner do - local full = path .. name - local mode = attributes(full,'mode') - if mode == 'file' then - if full:find(patt) then - action(full) - end - elseif recurse and (mode == "directory") and (name ~= '.') and (name ~= "..") then - glob_pattern(full,patt,recurse,action) + end +end + +dir.glob_pattern = glob_pattern + +local function collect_pattern(path,patt,recurse,result) + local ok, scanner + result = result or { } + if path == "/" then + ok, scanner = xpcall(function() return walkdir(path..".") end, function() end) -- kepler safe + else + ok, scanner = xpcall(function() return walkdir(path) end, function() end) -- kepler safe + end + if ok and type(scanner) == "function" then + if not find(path,"/$") then path = path .. '/' end + for name in scanner do + local full = path .. name + local attr = attributes(full) + local mode = attr.mode + if mode == 'file' then + if find(full,patt) then + result[name] = attr end + elseif recurse and (mode == "directory") and (name ~= '.') and (name ~= "..") then + attr.list = collect_pattern(full,patt,recurse) + result[name] = attr end end end + return result +end - dir.glob_pattern = glob_pattern - - --~ local function glob(pattern, action) - --~ local t = { } - --~ local path, rest, patt, recurse - --~ local action = action or function(name) t[#t+1] = name end - --~ local pattern = pattern:gsub("^%*%*","./**") - --~ local pattern = pattern:gsub("/%*/","/**/") - --~ path, rest = pattern:match("^(/)(.-)$") - --~ if path then - --~ path = path - --~ else - --~ path, rest = pattern:match("^([^/]*)/(.-)$") - --~ end - --~ if rest then - --~ patt = rest:gsub("([%.%-%+])", "%%%1") - --~ end - --~ patt = patt:gsub("%*", "[^/]*") - --~ patt = patt:gsub("%?", "[^/]") - --~ patt = patt:gsub("%[%^/%]%*%[%^/%]%*", ".*") - --~ if path == "" then path = "." end - --~ recurse = patt:find("%.%*/") ~= nil - --~ glob_pattern(path,patt,recurse,action) - --~ return t - --~ end - - local P, S, R, C, Cc, Cs, Ct, Cv, V = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.Cc, lpeg.Cs, lpeg.Ct, lpeg.Cv, lpeg.V - - local pattern = Ct { - [1] = (C(P(".") + P("/")^1) + C(R("az","AZ") * P(":") * P("/")^0) + Cc("./")) * V(2) * V(3), - [2] = C(((1-S("*?/"))^0 * P("/"))^0), - [3] = C(P(1)^0) - } +dir.collect_pattern = collect_pattern + +local P, S, R, C, Cc, Cs, Ct, Cv, V = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.Cc, lpeg.Cs, lpeg.Ct, lpeg.Cv, lpeg.V + +local pattern = Ct { + [1] = (C(P(".") + P("/")^1) + C(R("az","AZ") * P(":") * P("/")^0) + Cc("./")) * V(2) * V(3), + [2] = C(((1-S("*?/"))^0 * P("/"))^0), + [3] = C(P(1)^0) +} - local filter = Cs ( ( - P("**") / ".*" + - P("*") / "[^/]*" + - P("?") / "[^/]" + - P(".") / "%%." + - P("+") / "%%+" + - P("-") / "%%-" + - P(1) - )^0 ) - - local function glob(str,t) +local filter = Cs ( ( + P("**") / ".*" + + P("*") / "[^/]*" + + P("?") / "[^/]" + + P(".") / "%%." + + P("+") / "%%+" + + P("-") / "%%-" + + P(1) +)^0 ) + +local function glob(str,t) + if type(t) == "function" then + if type(str) == "table" then + for s=1,#str do + glob(str[s],t) + end + elseif lfs.isfile(str) then + t(str) + else + local split = lpegmatch(pattern,str) + if split then + local root, path, base = split[1], split[2], split[3] + local recurse = find(base,"%*%*") + local start = root .. path + local result = lpegmatch(filter,start .. base) + glob_pattern(start,result,recurse,t) + end + end + else if type(str) == "table" then local t = t or { } - for _, s in ipairs(str) do - glob(s,t) + for s=1,#str do + glob(str[s],t) end return t + elseif lfs.isfile(str) then + local t = t or { } + t[#t+1] = str + return t else - local split = pattern:match(str) + local split = lpegmatch(pattern,str) if split then local t = t or { } local action = action or function(name) t[#t+1] = name end local root, path, base = split[1], split[2], split[3] - local recurse = base:find("**") + local recurse = find(base,"%*%*") local start = root .. path - local result = filter:match(start .. base) + local result = lpegmatch(filter,start .. base) glob_pattern(start,result,recurse,action) return t else @@ -1975,113 +2705,120 @@ if lfs then do end end end +end - dir.glob = glob +dir.glob = glob - --~ list = dir.glob("**/*.tif") - --~ list = dir.glob("/**/*.tif") - --~ list = dir.glob("./**/*.tif") - --~ list = dir.glob("oeps/**/*.tif") - --~ list = dir.glob("/oeps/**/*.tif") +--~ list = dir.glob("**/*.tif") +--~ list = dir.glob("/**/*.tif") +--~ list = dir.glob("./**/*.tif") +--~ list = dir.glob("oeps/**/*.tif") +--~ list = dir.glob("/oeps/**/*.tif") - local function globfiles(path,recurse,func,files) -- func == pattern or function - if type(func) == "string" then - local s = func -- alas, we need this indirect way - func = function(name) return name:find(s) end - end - files = files or { } - for name in walkdir(path) do - if name:find("^%.") then - --- skip - elseif attributes(name,'mode') == "directory" then +local function globfiles(path,recurse,func,files) -- func == pattern or function + if type(func) == "string" then + local s = func -- alas, we need this indirect way + func = function(name) return find(name,s) end + end + files = files or { } + for name in walkdir(path) do + if find(name,"^%.") then + --- skip + else + local mode = attributes(name,'mode') + if mode == "directory" then if recurse then globfiles(path .. "/" .. name,recurse,func,files) end - elseif func then - if func(name) then + elseif mode == "file" then + if func then + if func(name) then + files[#files+1] = path .. "/" .. name + end + else files[#files+1] = path .. "/" .. name end - else - files[#files+1] = path .. "/" .. name end end - return files end + return files +end - dir.globfiles = globfiles +dir.globfiles = globfiles - -- t = dir.glob("c:/data/develop/context/sources/**/????-*.tex") - -- t = dir.glob("c:/data/develop/tex/texmf/**/*.tex") - -- t = dir.glob("c:/data/develop/context/texmf/**/*.tex") - -- t = dir.glob("f:/minimal/tex/**/*") - -- print(dir.ls("f:/minimal/tex/**/*")) - -- print(dir.ls("*.tex")) +-- t = dir.glob("c:/data/develop/context/sources/**/????-*.tex") +-- t = dir.glob("c:/data/develop/tex/texmf/**/*.tex") +-- t = dir.glob("c:/data/develop/context/texmf/**/*.tex") +-- t = dir.glob("f:/minimal/tex/**/*") +-- print(dir.ls("f:/minimal/tex/**/*")) +-- print(dir.ls("*.tex")) - function dir.ls(pattern) - return table.concat(glob(pattern),"\n") - end +function dir.ls(pattern) + return table.concat(glob(pattern),"\n") +end - --~ mkdirs("temp") - --~ mkdirs("a/b/c") - --~ mkdirs(".","/a/b/c") - --~ mkdirs("a","b","c") +--~ mkdirs("temp") +--~ mkdirs("a/b/c") +--~ mkdirs(".","/a/b/c") +--~ mkdirs("a","b","c") - local make_indeed = true -- false +local make_indeed = true -- false - if string.find(os.getenv("PATH"),";") then +if string.find(os.getenv("PATH"),";") then -- os.type == "windows" - function dir.mkdirs(...) - local str, pth = "", "" - for _, s in ipairs({...}) do - if s ~= "" then - if str ~= "" then - str = str .. "/" .. s - else - str = s - end + function dir.mkdirs(...) + local str, pth, t = "", "", { ... } + for i=1,#t do + local s = t[i] + if s ~= "" then + if str ~= "" then + str = str .. "/" .. s + else + str = s end end - local first, middle, last - local drive = false - first, middle, last = str:match("^(//)(//*)(.*)$") + end + local first, middle, last + local drive = false + first, middle, last = match(str,"^(//)(//*)(.*)$") + if first then + -- empty network path == local path + else + first, last = match(str,"^(//)/*(.-)$") if first then - -- empty network path == local path + middle, last = match(str,"([^/]+)/+(.-)$") + if middle then + pth = "//" .. middle + else + pth = "//" .. last + last = "" + end else - first, last = str:match("^(//)/*(.-)$") + first, middle, last = match(str,"^([a-zA-Z]:)(/*)(.-)$") if first then - middle, last = str:match("([^/]+)/+(.-)$") - if middle then - pth = "//" .. middle - else - pth = "//" .. last - last = "" - end + pth, drive = first .. middle, true else - first, middle, last = str:match("^([a-zA-Z]:)(/*)(.-)$") - if first then - pth, drive = first .. middle, true - else - middle, last = str:match("^(/*)(.-)$") - if not middle then - last = str - end + middle, last = match(str,"^(/*)(.-)$") + if not middle then + last = str end end end - for s in last:gmatch("[^/]+") do - if pth == "" then - pth = s - elseif drive then - pth, drive = pth .. s, false - else - pth = pth .. "/" .. s - end - if make_indeed and not lfs.isdir(pth) then - lfs.mkdir(pth) - end + end + for s in gmatch(last,"[^/]+") do + if pth == "" then + pth = s + elseif drive then + pth, drive = pth .. s, false + else + pth = pth .. "/" .. s + end + if make_indeed and not lfs.isdir(pth) then + lfs.mkdir(pth) end - return pth, (lfs.isdir(pth) == true) end + return pth, (lfs.isdir(pth) == true) + end --~ print(dir.mkdirs("","","a","c")) --~ print(dir.mkdirs("a")) @@ -2095,79 +2832,77 @@ if lfs then do --~ print(dir.mkdirs("///a/b/c")) --~ print(dir.mkdirs("a/bbb//ccc/")) - function dir.expand_name(str) - local first, nothing, last = str:match("^(//)(//*)(.*)$") - if first then - first = lfs.currentdir() .. "/" - first = first:gsub("\\","/") - end - if not first then - first, last = str:match("^(//)/*(.*)$") - end - if not first then - first, last = str:match("^([a-zA-Z]:)(.*)$") - if first and not last:find("^/") then - local d = lfs.currentdir() - if lfs.chdir(first) then - first = lfs.currentdir() - first = first:gsub("\\","/") - end - lfs.chdir(d) + function dir.expand_name(str) -- will be merged with cleanpath and collapsepath + local first, nothing, last = match(str,"^(//)(//*)(.*)$") + if first then + first = dir.current() .. "/" + end + if not first then + first, last = match(str,"^(//)/*(.*)$") + end + if not first then + first, last = match(str,"^([a-zA-Z]:)(.*)$") + if first and not find(last,"^/") then + local d = lfs.currentdir() + if lfs.chdir(first) then + first = dir.current() end + lfs.chdir(d) end - if not first then - first, last = lfs.currentdir(), str - first = first:gsub("\\","/") - end - last = last:gsub("//","/") - last = last:gsub("/%./","/") - last = last:gsub("^/*","") - first = first:gsub("/*$","") - if last == "" then - return first - else - return first .. "/" .. last - end end + if not first then + first, last = dir.current(), str + end + last = gsub(last,"//","/") + last = gsub(last,"/%./","/") + last = gsub(last,"^/*","") + first = gsub(first,"/*$","") + if last == "" then + return first + else + return first .. "/" .. last + end + end - else +else - function dir.mkdirs(...) - local str, pth = "", "" - for _, s in ipairs({...}) do - if s ~= "" then - if str ~= "" then - str = str .. "/" .. s - else - str = s - end + function dir.mkdirs(...) + local str, pth, t = "", "", { ... } + for i=1,#t do + local s = t[i] + if s ~= "" then + if str ~= "" then + str = str .. "/" .. s + else + str = s end end - str = str:gsub("/+","/") - if str:find("^/") then - pth = "/" - for s in str:gmatch("[^/]+") do - local first = (pth == "/") - if first then - pth = pth .. s - else - pth = pth .. "/" .. s - end - if make_indeed and not first and not lfs.isdir(pth) then - lfs.mkdir(pth) - end - end - else - pth = "." - for s in str:gmatch("[^/]+") do + end + str = gsub(str,"/+","/") + if find(str,"^/") then + pth = "/" + for s in gmatch(str,"[^/]+") do + local first = (pth == "/") + if first then + pth = pth .. s + else pth = pth .. "/" .. s - if make_indeed and not lfs.isdir(pth) then - lfs.mkdir(pth) - end + end + if make_indeed and not first and not lfs.isdir(pth) then + lfs.mkdir(pth) + end + end + else + pth = "." + for s in gmatch(str,"[^/]+") do + pth = pth .. "/" .. s + if make_indeed and not lfs.isdir(pth) then + lfs.mkdir(pth) end end - return pth, (lfs.isdir(pth) == true) end + return pth, (lfs.isdir(pth) == true) + end --~ print(dir.mkdirs("","","a","c")) --~ print(dir.mkdirs("a")) @@ -2177,30 +2912,35 @@ if lfs then do --~ print(dir.mkdirs("///a/b/c")) --~ print(dir.mkdirs("a/bbb//ccc/")) - function dir.expand_name(str) - if not str:find("^/") then - str = lfs.currentdir() .. "/" .. str - end - str = str:gsub("//","/") - str = str:gsub("/%./","/") - return str + function dir.expand_name(str) -- will be merged with cleanpath and collapsepath + if not find(str,"^/") then + str = lfs.currentdir() .. "/" .. str end - + str = gsub(str,"//","/") + str = gsub(str,"/%./","/") + return str end - dir.makedirs = dir.mkdirs +end -end end +dir.makedirs = dir.mkdirs --- filename : l-boolean.lua --- comment : split off from luat-lib --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +end -- of closure -if not versions then versions = { } end versions['l-boolean'] = 1.001 -if not boolean then boolean = { } end +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['l-boolean'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +boolean = boolean or { } + +local type, tonumber = type, tonumber function boolean.tonumber(b) if b then return 1 else return 0 end @@ -2210,7 +2950,7 @@ function toboolean(str,tolerant) if tolerant then local tstr = type(str) if tstr == "string" then - return str == "true" or str == "yes" or str == "on" or str == "1" + return str == "true" or str == "yes" or str == "on" or str == "1" or str == "t" elseif tstr == "number" then return tonumber(str) ~= 0 elseif tstr == "nil" then @@ -2229,9 +2969,9 @@ end function string.is_boolean(str) if type(str) == "string" then - if str == "true" or str == "yes" or str == "on" then + if str == "true" or str == "yes" or str == "on" or str == "t" then return true - elseif str == "false" or str == "no" or str == "off" then + elseif str == "false" or str == "no" or str == "off" or str == "f" then return false end end @@ -2247,22 +2987,46 @@ function boolean.falsetrue() end --- filename : l-unicode.lua --- comment : split off from luat-inp --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +end -- of closure -if not versions then versions = { } end versions['l-unicode'] = 1.001 -if not unicode then unicode = { } end +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['l-unicode'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +if not unicode then + + unicode = { utf8 = { } } + + local floor, char = math.floor, string.char + + function unicode.utf8.utfchar(n) + if n < 0x80 then + return char(n) + elseif n < 0x800 then + return char(0xC0 + floor(n/0x40)) .. char(0x80 + (n % 0x40)) + elseif n < 0x10000 then + return char(0xE0 + floor(n/0x1000)) .. char(0x80 + (floor(n/0x40) % 0x40)) .. char(0x80 + (n % 0x40)) + elseif n < 0x40000 then + return char(0xF0 + floor(n/0x40000)) .. char(0x80 + floor(n/0x1000)) .. char(0x80 + (floor(n/0x40) % 0x40)) .. char(0x80 + (n % 0x40)) + else -- wrong: + -- return char(0xF1 + floor(n/0x1000000)) .. char(0x80 + floor(n/0x40000)) .. char(0x80 + floor(n/0x1000)) .. char(0x80 + (floor(n/0x40) % 0x40)) .. char(0x80 + (n % 0x40)) + return "?" + end + end -if not garbagecollector then - garbagecollector = { - push = function() collectgarbage("stop") end, - pop = function() collectgarbage("restart") end, - } end +utf = utf or unicode.utf8 + +local concat, utfchar, utfgsub = table.concat, utf.char, utf.gsub +local char, byte, find, bytepairs = string.char, string.byte, string.find, string.bytepairs + -- 0 EF BB BF UTF-8 -- 1 FF FE UTF-16-little-endian -- 2 FE FF UTF-16-big-endian @@ -2277,22 +3041,28 @@ unicode.utfname = { [4] = 'utf-32-be' } -function unicode.utftype(f) -- \000 fails ! +-- \000 fails in <= 5.0 but is valid in >=5.1 where %z is depricated + +function unicode.utftype(f) local str = f:read(4) if not str then f:seek('set') return 0 - elseif str:find("^%z%z\254\255") then + -- elseif find(str,"^%z%z\254\255") then -- depricated + -- elseif find(str,"^\000\000\254\255") then -- not permitted and bugged + elseif find(str,"\000\000\254\255",1,true) then -- seems to work okay (TH) return 4 - elseif str:find("^\255\254%z%z") then + -- elseif find(str,"^\255\254%z%z") then -- depricated + -- elseif find(str,"^\255\254\000\000") then -- not permitted and bugged + elseif find(str,"\255\254\000\000",1,true) then -- seems to work okay (TH) return 3 - elseif str:find("^\254\255") then + elseif find(str,"^\254\255") then f:seek('set',2) return 2 - elseif str:find("^\255\254") then + elseif find(str,"^\255\254") then f:seek('set',2) return 1 - elseif str:find("^\239\187\191") then + elseif find(str,"^\239\187\191") then f:seek('set',3) return 0 else @@ -2301,74 +3071,70 @@ function unicode.utftype(f) -- \000 fails ! end end -function unicode.utf16_to_utf8(str, endian) - garbagecollector.push() - local result = { } - local tc, uc = table.concat, unicode.utf8.char - local tmp, n, m, p = { }, 0, 0, 0 +function unicode.utf16_to_utf8(str, endian) -- maybe a gsub is faster or an lpeg + local result, tmp, n, m, p = { }, { }, 0, 0, 0 -- lf | cr | crlf / (cr:13, lf:10) local function doit() if n == 10 then if p ~= 13 then - result[#result+1] = tc(tmp,"") + result[#result+1] = concat(tmp) tmp = { } p = 0 end elseif n == 13 then - result[#result+1] = tc(tmp,"") + result[#result+1] = concat(tmp) tmp = { } p = n else - tmp[#tmp+1] = uc(n) + tmp[#tmp+1] = utfchar(n) p = 0 end end - for l,r in str:bytepairs() do - if endian then - n = l*256 + r - else - n = r*256 + l - end - if m > 0 then - n = (m-0xD800)*0x400 + (n-0xDC00) + 0x10000 - m = 0 - doit() - elseif n >= 0xD800 and n <= 0xDBFF then - m = n - else - doit() + for l,r in bytepairs(str) do + if r then + if endian then + n = l*256 + r + else + n = r*256 + l + end + if m > 0 then + n = (m-0xD800)*0x400 + (n-0xDC00) + 0x10000 + m = 0 + doit() + elseif n >= 0xD800 and n <= 0xDBFF then + m = n + else + doit() + end end end if #tmp > 0 then - result[#result+1] = tc(tmp,"") + result[#result+1] = concat(tmp) end - garbagecollector.pop() return result end function unicode.utf32_to_utf8(str, endian) - garbagecollector.push() local result = { } - local tc, uc = table.concat, unicode.utf8.char local tmp, n, m, p = { }, 0, -1, 0 -- lf | cr | crlf / (cr:13, lf:10) local function doit() if n == 10 then if p ~= 13 then - result[#result+1] = tc(tmp,"") + result[#result+1] = concat(tmp) tmp = { } p = 0 end elseif n == 13 then - result[#result+1] = tc(tmp,"") + result[#result+1] = concat(tmp) tmp = { } p = n else - tmp[#tmp+1] = uc(n) + tmp[#tmp+1] = utfchar(n) p = 0 end end - for a,b in str:bytepairs() do + for a,b in bytepairs(str) do if a and b then if m < 0 then if endian then @@ -2390,20 +3156,106 @@ function unicode.utf32_to_utf8(str, endian) end end if #tmp > 0 then - result[#result+1] = tc(tmp,"") + result[#result+1] = concat(tmp) end - garbagecollector.pop() return result end +local function little(c) + local b = byte(c) -- b = c:byte() + if b < 0x10000 then + return char(b%256,b/256) + else + b = b - 0x10000 + local b1, b2 = b/1024 + 0xD800, b%1024 + 0xDC00 + return char(b1%256,b1/256,b2%256,b2/256) + end +end + +local function big(c) + local b = byte(c) + if b < 0x10000 then + return char(b/256,b%256) + else + b = b - 0x10000 + local b1, b2 = b/1024 + 0xD800, b%1024 + 0xDC00 + return char(b1/256,b1%256,b2/256,b2%256) + end +end + +function unicode.utf8_to_utf16(str,littleendian) + if littleendian then + return char(255,254) .. utfgsub(str,".",little) + else + return char(254,255) .. utfgsub(str,".",big) + end +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['l-math'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local floor, sin, cos, tan = math.floor, math.sin, math.cos, math.tan + +if not math.round then + function math.round(x) + return floor(x + 0.5) + end +end + +if not math.div then + function math.div(n,m) + return floor(n/m) + end +end + +if not math.mod then + function math.mod(n,m) + return n % m + end +end + +local pipi = 2*math.pi/360 + +function math.sind(d) + return sin(d*pipi) +end + +function math.cosd(d) + return cos(d*pipi) +end + +function math.tand(d) + return tan(d*pipi) +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['l-utils'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} --- filename : l-utils.lua --- comment : split off from luat-lib --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +-- hm, quite unreadable -if not versions then versions = { } end versions['l-utils'] = 1.001 +local gsub = string.gsub +local concat = table.concat +local type, next = type, next if not utils then utils = { } end if not utils.merger then utils.merger = { } end @@ -2429,11 +3281,20 @@ function utils.report(...) print(...) end +utils.merger.strip_comment = true + function utils.merger._self_load_(name) local f, data = io.open(name), "" if f then + utils.report("reading merge from %s",name) data = f:read("*all") f:close() + else + utils.report("unknown file to merge %s",name) + end + if data and utils.merger.strip_comment then + -- saves some 20K + data = gsub(data,"%-%-~[^\n\r]*[\r\n]", "") end return data or "" end @@ -2442,6 +3303,7 @@ function utils.merger._self_save_(name, data) if data ~= "" then local f = io.open(name,'w') if f then + utils.report("saving merge from %s",name) f:write(data) f:close() end @@ -2450,7 +3312,7 @@ end function utils.merger._self_swap_(data,code) if data ~= "" then - return (data:gsub(utils.merger.pattern, function(s) + return (gsub(data,utils.merger.pattern, function(s) return "\n\n" .. "-- "..utils.merger.m_begin .. "\n" .. code .. "\n" .. "-- "..utils.merger.m_end .. "\n\n" end, 1)) else @@ -2458,26 +3320,56 @@ function utils.merger._self_swap_(data,code) end end +--~ stripper: +--~ +--~ data = gsub(data,"%-%-~[^\n]*\n","") +--~ data = gsub(data,"\n\n+","\n") + function utils.merger._self_libs_(libs,list) - local result, f = { }, nil + local result, f, frozen = { }, nil, false + result[#result+1] = "\n" if type(libs) == 'string' then libs = { libs } end if type(list) == 'string' then list = { list } end - for _, lib in ipairs(libs) do - for _, pth in ipairs(list) do - local name = string.gsub(pth .. "/" .. lib,"\\","/") - f = io.open(name) - if f then - -- utils.report("merging library",name) - result[#result+1] = f:read("*all") - f:close() - list = { pth } -- speed up the search - break + local foundpath = nil + for i=1,#libs do + local lib = libs[i] + for j=1,#list do + local pth = gsub(list[j],"\\","/") -- file.clean_path + utils.report("checking library path %s",pth) + local name = pth .. "/" .. lib + if lfs.isfile(name) then + foundpath = pth + end + end + if foundpath then break end + end + if foundpath then + utils.report("using library path %s",foundpath) + local right, wrong = { }, { } + for i=1,#libs do + local lib = libs[i] + local fullname = foundpath .. "/" .. lib + if lfs.isfile(fullname) then + -- right[#right+1] = lib + utils.report("merging library %s",fullname) + result[#result+1] = "do -- create closure to overcome 200 locals limit" + result[#result+1] = io.loaddata(fullname,true) + result[#result+1] = "end -- of closure" else - -- utils.report("no library",name) + -- wrong[#wrong+1] = lib + utils.report("no library %s",fullname) end end + if #right > 0 then + utils.report("merged libraries: %s",concat(right," ")) + end + if #wrong > 0 then + utils.report("skipped libraries: %s",concat(wrong," ")) + end + else + utils.report("no valid library path found") end - return table.concat(result, "\n\n") + return concat(result, "\n\n") end function utils.merger.selfcreate(libs,list,target) @@ -2512,108 +3404,700 @@ function utils.merger.selfclean(name) ) end -utils.lua.compile_strip = true - -function utils.lua.compile(luafile, lucfile) +function utils.lua.compile(luafile, lucfile, cleanup, strip) -- defaults: cleanup=false strip=true -- utils.report("compiling",luafile,"into",lucfile) os.remove(lucfile) local command = "-o " .. string.quote(lucfile) .. " " .. string.quote(luafile) - if utils.lua.compile_strip then + if strip ~= false then command = "-s " .. command end - if os.spawn("texluac " .. command) == 0 then - return true - elseif os.spawn("luac " .. command) == 0 then - return true + local done = (os.spawn("texluac " .. command) == 0) or (os.spawn("luac " .. command) == 0) + if done and cleanup == true and lfs.isfile(lucfile) and lfs.isfile(luafile) then + -- utils.report("removing",luafile) + os.remove(luafile) + end + return done +end + + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['l-aux'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- for inline, no store split : for s in string.gmatch(str,",* *([^,]+)") do .. end + +aux = aux or { } + +local concat, format, gmatch = table.concat, string.format, string.gmatch +local tostring, type = tostring, type +local lpegmatch = lpeg.match + +local P, R, V = lpeg.P, lpeg.R, lpeg.V + +local escape, left, right = P("\\"), P('{'), P('}') + +lpeg.patterns.balanced = P { + [1] = ((escape * (left+right)) + (1 - (left+right)) + V(2))^0, + [2] = left * V(1) * right +} + +local space = lpeg.P(' ') +local equal = lpeg.P("=") +local comma = lpeg.P(",") +local lbrace = lpeg.P("{") +local rbrace = lpeg.P("}") +local nobrace = 1 - (lbrace+rbrace) +local nested = lpeg.P { lbrace * (nobrace + lpeg.V(1))^0 * rbrace } +local spaces = space^0 + +local value = lpeg.P(lbrace * lpeg.C((nobrace + nested)^0) * rbrace) + lpeg.C((nested + (1-comma))^0) + +local key = lpeg.C((1-equal-comma)^1) +local pattern_a = (space+comma)^0 * (key * equal * value + key * lpeg.C("")) +local pattern_c = (space+comma)^0 * (key * equal * value) + +local key = lpeg.C((1-space-equal-comma)^1) +local pattern_b = spaces * comma^0 * spaces * (key * ((spaces * equal * spaces * value) + lpeg.C(""))) + +-- "a=1, b=2, c=3, d={a{b,c}d}, e=12345, f=xx{a{b,c}d}xx, g={}" : outer {} removes, leading spaces ignored + +local hash = { } + +local function set(key,value) -- using Carg is slower here + hash[key] = value +end + +local pattern_a_s = (pattern_a/set)^1 +local pattern_b_s = (pattern_b/set)^1 +local pattern_c_s = (pattern_c/set)^1 + +aux.settings_to_hash_pattern_a = pattern_a_s +aux.settings_to_hash_pattern_b = pattern_b_s +aux.settings_to_hash_pattern_c = pattern_c_s + +function aux.make_settings_to_hash_pattern(set,how) + if how == "strict" then + return (pattern_c/set)^1 + elseif how == "tolerant" then + return (pattern_b/set)^1 else - return false + return (pattern_a/set)^1 end end +function aux.settings_to_hash(str,existing) + if str and str ~= "" then + hash = existing or { } + if moretolerant then + lpegmatch(pattern_b_s,str) + else + lpegmatch(pattern_a_s,str) + end + return hash + else + return { } + end +end +function aux.settings_to_hash_tolerant(str,existing) + if str and str ~= "" then + hash = existing or { } + lpegmatch(pattern_b_s,str) + return hash + else + return { } + end +end + +function aux.settings_to_hash_strict(str,existing) + if str and str ~= "" then + hash = existing or { } + lpegmatch(pattern_c_s,str) + return next(hash) and hash + else + return nil + end +end --- filename : luat-lib.lua --- comment : companion to luat-lib.tex --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +local separator = comma * space^0 +local value = lpeg.P(lbrace * lpeg.C((nobrace + nested)^0) * rbrace) + lpeg.C((nested + (1-comma))^0) +local pattern = lpeg.Ct(value*(separator*value)^0) -if not versions then versions = { } end versions['luat-lib'] = 1.001 +-- "aap, {noot}, mies" : outer {} removes, leading spaces ignored --- mostcode moved to the l-*.lua and other luat-*.lua files +aux.settings_to_array_pattern = pattern --- os / io +-- we could use a weak table as cache -os.setlocale(nil,nil) -- useless feature and even dangerous in luatex +function aux.settings_to_array(str) + if not str or str == "" then + return { } + else + return lpegmatch(pattern,str) + end +end --- os.platform +local function set(t,v) + t[#t+1] = v +end --- mswin|bccwin|mingw|cygwin windows --- darwin|rhapsody|nextstep macosx --- netbsd|unix unix --- linux linux +local value = lpeg.P(lpeg.Carg(1)*value) / set +local pattern = value*(separator*value)^0 * lpeg.Carg(1) -if not io.fileseparator then - if string.find(os.getenv("PATH"),";") then - io.fileseparator, io.pathseparator, os.platform = "\\", ";", os.type or "windows" +function aux.add_settings_to_array(t,str) + return lpegmatch(pattern,str,nil,t) +end + +function aux.hash_to_string(h,separator,yes,no,strict,omit) + if h then + local t, s = { }, table.sortedkeys(h) + omit = omit and table.tohash(omit) + for i=1,#s do + local key = s[i] + if not omit or not omit[key] then + local value = h[key] + if type(value) == "boolean" then + if yes and no then + if value then + t[#t+1] = key .. '=' .. yes + elseif not strict then + t[#t+1] = key .. '=' .. no + end + elseif value or not strict then + t[#t+1] = key .. '=' .. tostring(value) + end + else + t[#t+1] = key .. '=' .. value + end + end + end + return concat(t,separator or ",") else - io.fileseparator, io.pathseparator, os.platform = "/" , ":", os.type or "unix" + return "" end end -os.platform = os.platform or os.type or (io.pathseparator == ";" and "windows") or "unix" +function aux.array_to_string(a,separator) + if a then + return concat(a,separator or ",") + else + return "" + end +end --- arg normalization --- --- for k,v in pairs(arg) do print(k,v) end +function aux.settings_to_set(str,t) + t = t or { } + for s in gmatch(str,"%s*([^,]+)") do + t[s] = true + end + return t +end --- environment +local value = lbrace * lpeg.C((nobrace + nested)^0) * rbrace +local pattern = lpeg.Ct((space + value)^0) -if not environment then environment = { } end +function aux.arguments_to_table(str) + return lpegmatch(pattern,str) +end -environment.ownbin = environment.ownbin or arg[-2] or arg[-1] or arg[0] or "luatex" +-- temporary here -local ownpath = nil -- we could use a metatable here +function aux.getparameters(self,class,parentclass,settings) + local sc = self[class] + if not sc then + sc = table.clone(self[parent]) + self[class] = sc + end + aux.settings_to_hash(settings,sc) +end -function environment.ownpath() - if not ownpath then - for p in string.gmatch(os.getenv("PATH"),"[^"..io.pathseparator.."]+") do - local b = file.join(p,environment.ownbin) - if lfs.isfile(b..".exe") or lfs.isfile(b) then - ownpath = p - break +-- temporary here + +local digit = lpeg.R("09") +local period = lpeg.P(".") +local zero = lpeg.P("0") +local trailingzeros = zero^0 * -digit -- suggested by Roberto R +local case_1 = period * trailingzeros / "" +local case_2 = period * (digit - trailingzeros)^1 * (trailingzeros / "") +local number = digit^1 * (case_1 + case_2) +local stripper = lpeg.Cs((number + 1)^0) + +--~ local sample = "bla 11.00 bla 11 bla 0.1100 bla 1.00100 bla 0.00 bla 0.001 bla 1.1100 bla 0.100100100 bla 0.00100100100" +--~ collectgarbage("collect") +--~ str = string.rep(sample,10000) +--~ local ts = os.clock() +--~ lpegmatch(stripper,str) +--~ print(#str, os.clock()-ts, lpegmatch(stripper,sample)) + +lpeg.patterns.strip_zeros = stripper + +function aux.strip_zeros(str) + return lpegmatch(stripper,str) +end + +function aux.definetable(target) -- defines undefined tables + local composed, t = nil, { } + for name in gmatch(target,"([^%.]+)") do + if composed then + composed = composed .. "." .. name + else + composed = name + end + t[#t+1] = format("%s = %s or { }",composed,composed) + end + return concat(t,"\n") +end + +function aux.accesstable(target) + local t = _G + for name in gmatch(target,"([^%.]+)") do + t = t[name] + end + return t +end + +-- as we use this a lot ... + +--~ function aux.cachefunction(action,weak) +--~ local cache = { } +--~ if weak then +--~ setmetatable(cache, { __mode = "kv" } ) +--~ end +--~ local function reminder(str) +--~ local found = cache[str] +--~ if not found then +--~ found = action(str) +--~ cache[str] = found +--~ end +--~ return found +--~ end +--~ return reminder, cache +--~ end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['trac-tra'] = { + version = 1.001, + comment = "companion to trac-tra.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- the <anonymous> tag is kind of generic and used for functions that are not +-- bound to a variable, like node.new, node.copy etc (contrary to for instance +-- node.has_attribute which is bound to a has_attribute local variable in mkiv) + +local debug = require "debug" + +local getinfo = debug.getinfo +local type, next = type, next +local concat = table.concat +local format, find, lower, gmatch, gsub = string.format, string.find, string.lower, string.gmatch, string.gsub + +debugger = debugger or { } + +local counters = { } +local names = { } + +-- one + +local function hook() + local f = getinfo(2,"f").func + local n = getinfo(2,"Sn") +-- if n.what == "C" and n.name then print (n.namewhat .. ': ' .. n.name) end + if f then + local cf = counters[f] + if cf == nil then + counters[f] = 1 + names[f] = n + else + counters[f] = cf + 1 + end + end +end +local function getname(func) + local n = names[func] + if n then + if n.what == "C" then + return n.name or '<anonymous>' + else + -- source short_src linedefined what name namewhat nups func + local name = n.name or n.namewhat or n.what + if not name or name == "" then name = "?" end + return format("%s : %s : %s", n.short_src or "unknown source", n.linedefined or "--", name) + end + else + return "unknown" + end +end +function debugger.showstats(printer,threshold) + printer = printer or texio.write or print + threshold = threshold or 0 + local total, grandtotal, functions = 0, 0, 0 + printer("\n") -- ugly but ok + -- table.sort(counters) + for func, count in next, counters do + if count > threshold then + local name = getname(func) + if not find(name,"for generator") then + printer(format("%8i %s", count, name)) + total = total + count end end - if not ownpath then ownpath = '.' end + grandtotal = grandtotal + count + functions = functions + 1 end - return ownpath + printer(format("functions: %s, total: %s, grand total: %s, threshold: %s\n", functions, total, grandtotal, threshold)) +end + +-- two + +--~ local function hook() +--~ local n = getinfo(2) +--~ if n.what=="C" and not n.name then +--~ local f = tostring(debug.traceback()) +--~ local cf = counters[f] +--~ if cf == nil then +--~ counters[f] = 1 +--~ names[f] = n +--~ else +--~ counters[f] = cf + 1 +--~ end +--~ end +--~ end +--~ function debugger.showstats(printer,threshold) +--~ printer = printer or texio.write or print +--~ threshold = threshold or 0 +--~ local total, grandtotal, functions = 0, 0, 0 +--~ printer("\n") -- ugly but ok +--~ -- table.sort(counters) +--~ for func, count in next, counters do +--~ if count > threshold then +--~ printer(format("%8i %s", count, func)) +--~ total = total + count +--~ end +--~ grandtotal = grandtotal + count +--~ functions = functions + 1 +--~ end +--~ printer(format("functions: %s, total: %s, grand total: %s, threshold: %s\n", functions, total, grandtotal, threshold)) +--~ end + +-- rest + +function debugger.savestats(filename,threshold) + local f = io.open(filename,'w') + if f then + debugger.showstats(function(str) f:write(str) end,threshold) + f:close() + end +end + +function debugger.enable() + debug.sethook(hook,"c") +end + +function debugger.disable() + debug.sethook() +--~ counters[debug.getinfo(2,"f").func] = nil +end + +function debugger.tracing() + local n = tonumber(os.env['MTX.TRACE.CALLS']) or tonumber(os.env['MTX_TRACE_CALLS']) or 0 + if n > 0 then + function debugger.tracing() return true end ; return true + else + function debugger.tracing() return false end ; return false + end +end + +--~ debugger.enable() + +--~ print(math.sin(1*.5)) +--~ print(math.sin(1*.5)) +--~ print(math.sin(1*.5)) +--~ print(math.sin(1*.5)) +--~ print(math.sin(1*.5)) + +--~ debugger.disable() + +--~ print("") +--~ debugger.showstats() +--~ print("") +--~ debugger.showstats(print,3) + +setters = setters or { } +setters.data = setters.data or { } + +--~ local function set(t,what,value) +--~ local data, done = t.data, t.done +--~ if type(what) == "string" then +--~ what = aux.settings_to_array(what) -- inefficient but ok +--~ end +--~ for i=1,#what do +--~ local w = what[i] +--~ for d, f in next, data do +--~ if done[d] then +--~ -- prevent recursion due to wildcards +--~ elseif find(d,w) then +--~ done[d] = true +--~ for i=1,#f do +--~ f[i](value) +--~ end +--~ end +--~ end +--~ end +--~ end + +local function set(t,what,value) + local data, done = t.data, t.done + if type(what) == "string" then + what = aux.settings_to_hash(what) -- inefficient but ok + end + for w, v in next, what do + if v == "" then + v = value + else + v = toboolean(v) + end + for d, f in next, data do + if done[d] then + -- prevent recursion due to wildcards + elseif find(d,w) then + done[d] = true + for i=1,#f do + f[i](v) + end + end + end + end +end + +local function reset(t) + for d, f in next, t.data do + for i=1,#f do + f[i](false) + end + end +end + +local function enable(t,what) + set(t,what,true) +end + +local function disable(t,what) + local data = t.data + if not what or what == "" then + t.done = { } + reset(t) + else + set(t,what,false) + end +end + +function setters.register(t,what,...) + local data = t.data + what = lower(what) + local w = data[what] + if not w then + w = { } + data[what] = w + end + for _, fnc in next, { ... } do + local typ = type(fnc) + if typ == "function" then + w[#w+1] = fnc + elseif typ == "string" then + w[#w+1] = function(value) set(t,fnc,value,nesting) end + end + end +end + +function setters.enable(t,what) + local e = t.enable + t.enable, t.done = enable, { } + enable(t,string.simpleesc(tostring(what))) + t.enable, t.done = e, { } +end + +function setters.disable(t,what) + local e = t.disable + t.disable, t.done = disable, { } + disable(t,string.simpleesc(tostring(what))) + t.disable, t.done = e, { } +end + +function setters.reset(t) + t.done = { } + reset(t) +end + +function setters.list(t) -- pattern + local list = table.sortedkeys(t.data) + local user, system = { }, { } + for l=1,#list do + local what = list[l] + if find(what,"^%*") then + system[#system+1] = what + else + user[#user+1] = what + end + end + return user, system +end + +function setters.show(t) + commands.writestatus("","") + local list = setters.list(t) + for k=1,#list do + commands.writestatus(t.name,list[k]) + end + commands.writestatus("","") +end + +-- we could have used a bit of oo and the trackers:enable syntax but +-- there is already a lot of code around using the singular tracker + +-- we could make this into a module + +function setters.new(name) + local t + t = { + data = { }, + name = name, + enable = function(...) setters.enable (t,...) end, + disable = function(...) setters.disable (t,...) end, + register = function(...) setters.register(t,...) end, + list = function(...) setters.list (t,...) end, + show = function(...) setters.show (t,...) end, + } + setters.data[name] = t + return t +end + +trackers = setters.new("trackers") +directives = setters.new("directives") +experiments = setters.new("experiments") + +-- nice trick: we overload two of the directives related functions with variants that +-- do tracing (itself using a tracker) .. proof of concept + +local trace_directives = false local trace_directives = false trackers.register("system.directives", function(v) trace_directives = v end) +local trace_experiments = false local trace_experiments = false trackers.register("system.experiments", function(v) trace_experiments = v end) + +local e = directives.enable +local d = directives.disable + +function directives.enable(...) + commands.writestatus("directives","enabling: %s",concat({...}," ")) + e(...) +end + +function directives.disable(...) + commands.writestatus("directives","disabling: %s",concat({...}," ")) + d(...) +end + +local e = experiments.enable +local d = experiments.disable + +function experiments.enable(...) + commands.writestatus("experiments","enabling: %s",concat({...}," ")) + e(...) end +function experiments.disable(...) + commands.writestatus("experiments","disabling: %s",concat({...}," ")) + d(...) +end + +-- a useful example + +directives.register("system.nostatistics", function(v) + statistics.enable = not v +end) + + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['luat-env'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- A former version provided functionality for non embeded core +-- scripts i.e. runtime library loading. Given the amount of +-- Lua code we use now, this no longer makes sense. Much of this +-- evolved before bytecode arrays were available and so a lot of +-- code has disappeared already. + +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) + +local format, sub, match, gsub, find = string.format, string.sub, string.match, string.gsub, string.find +local unquote, quote = string.unquote, string.quote + +-- precautions + +os.setlocale(nil,nil) -- useless feature and even dangerous in luatex + +function os.setlocale() + -- no way you can mess with it +end + +-- dirty tricks + if arg and (arg[0] == 'luatex' or arg[0] == 'luatex.exe') and arg[1] == "--luaonly" then arg[-1]=arg[0] arg[0]=arg[2] for k=3,#arg do arg[k-2]=arg[k] end arg[#arg]=nil arg[#arg]=nil end -environment.arguments = { } -environment.files = { } -environment.sorted_argument_keys = nil +if profiler and os.env["MTX_PROFILE_RUN"] == "YES" then + profiler.start("luatex-profile.log") +end + +-- environment + +environment = environment or { } +environment.arguments = { } +environment.files = { } +environment.sortedflags = nil -environment.platform = os.platform +if not environment.jobname or environment.jobname == "" then if tex then environment.jobname = tex.jobname end end +if not environment.version or environment.version == "" then environment.version = "unknown" end +if not environment.jobname then environment.jobname = "unknown" end function environment.initialize_arguments(arg) - environment.arguments = { } - environment.files = { } - environment.sorted_argument_keys = nil - for index, argument in pairs(arg) do + local arguments, files = { }, { } + environment.arguments, environment.files, environment.sortedflags = arguments, files, nil + for index=1,#arg do + local argument = arg[index] if index > 0 then - local flag, value = argument:match("^%-+(.+)=(.-)$") + local flag, value = match(argument,"^%-+(.-)=(.-)$") if flag then - environment.arguments[flag] = string.unquote(value or "") + arguments[flag] = unquote(value or "") else - flag = argument:match("^%-+(.+)") + flag = match(argument,"^%-+(.+)") if flag then - environment.arguments[flag] = true + arguments[flag] = true else - environment.files[#environment.files+1] = argument + files[#files+1] = argument end end end @@ -2621,41 +4105,45 @@ function environment.initialize_arguments(arg) environment.ownname = environment.ownname or arg[0] or 'unknown.lua' end -function environment.showarguments() - for k,v in pairs(environment.arguments) do - print(k .. " : " .. tostring(v)) - end - if #environment.files > 0 then - print("files : " .. table.concat(environment.files, " ")) - end -end - function environment.setargument(name,value) environment.arguments[name] = value end -function environment.argument(name) - if environment.arguments[name] then - return environment.arguments[name] - else - if not environment.sorted_argument_keys then - environment.sorted_argument_keys = { } - for _,v in pairs(table.sortedkeys(environment.arguments)) do - table.insert(environment.sorted_argument_keys, "^" .. v) +-- todo: defaults, better checks e.g on type (boolean versus string) +-- +-- tricky: too many hits when we support partials unless we add +-- a registration of arguments so from now on we have 'partial' + +function environment.argument(name,partial) + local arguments, sortedflags = environment.arguments, environment.sortedflags + if arguments[name] then + return arguments[name] + elseif partial then + if not sortedflags then + sortedflags = table.sortedkeys(arguments) + for k=1,#sortedflags do + sortedflags[k] = "^" .. sortedflags[k] end + environment.sortedflags = sortedflags end - for _,v in pairs(environment.sorted_argument_keys) do - if name:find(v) then - return environment.arguments[v:sub(2,#v)] + -- example of potential clash: ^mode ^modefile + for k=1,#sortedflags do + local v = sortedflags[k] + if find(name,v) then + return arguments[sub(v,2,#v)] end end end return nil end +environment.argument("x",true) + function environment.split_arguments(separator) -- rather special, cut-off before separator local done, before, after = false, { }, { } - for _,v in ipairs(environment.original_arguments) do + local original_arguments = environment.original_arguments + for k=1,#original_arguments do + local v = original_arguments[k] if not done and v == separator then done = true elseif done then @@ -2667,355 +4155,979 @@ function environment.split_arguments(separator) -- rather special, cut-off befor return before, after end -function environment.reconstruct_commandline(arg) - if not arg then arg = environment.original_arguments end - local result = { } - for _,a in ipairs(arg) do -- ipairs 1 .. #n - local kk, vv = a:match("^(%-+.-)=(.+)$") - if kk and vv then - if vv:find(" ") then - result[#result+1] = kk .. "=" .. string.quote(vv) +function environment.reconstruct_commandline(arg,noquote) + arg = arg or environment.original_arguments + if noquote and #arg == 1 then + local a = arg[1] + a = resolvers.resolve(a) + a = unquote(a) + return a + elseif #arg > 0 then + local result = { } + for i=1,#arg do + local a = arg[i] + a = resolvers.resolve(a) + a = unquote(a) + a = gsub(a,'"','\\"') -- tricky + if find(a," ") then + result[#result+1] = quote(a) else result[#result+1] = a end - elseif a:find(" ") then - result[#result+1] = string.quote(a) - else - result[#result+1] = a end + return table.join(result," ") + else + return "" end - return table.join(result," ") end if arg then - environment.initialize_arguments(arg) - environment.original_arguments = arg + + -- new, reconstruct quoted snippets (maybe better just remove the " then and add them later) + local newarg, instring = { }, false + + for index=1,#arg do + local argument = arg[index] + if find(argument,"^\"") then + newarg[#newarg+1] = gsub(argument,"^\"","") + if not find(argument,"\"$") then + instring = true + end + elseif find(argument,"\"$") then + newarg[#newarg] = newarg[#newarg] .. " " .. gsub(argument,"\"$","") + instring = false + elseif instring then + newarg[#newarg] = newarg[#newarg] .. " " .. argument + else + newarg[#newarg+1] = argument + end + end + for i=1,-5,-1 do + newarg[i] = arg[i] + end + + environment.initialize_arguments(newarg) + environment.original_arguments = newarg + environment.raw_arguments = arg + arg = { } -- prevent duplicate handling + end +-- weird place ... depends on a not yet loaded module --- filename : luat-inp.lua --- comment : companion to luat-lib.tex --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +function environment.texfile(filename) + return resolvers.find_file(filename,'tex') +end --- This lib is multi-purpose and can be loaded again later on so that --- additional functionality becomes available. We will split this --- module in components when we're done with prototyping. +function environment.luafile(filename) + local resolved = resolvers.find_file(filename,'tex') or "" + if resolved ~= "" then + return resolved + end + resolved = resolvers.find_file(filename,'texmfscripts') or "" + if resolved ~= "" then + return resolved + end + return resolvers.find_file(filename,'luatexlibs') or "" +end --- TODO: os.getenv -> os.env[] --- TODO: instances.[hashes,cnffiles,configurations,522] -> ipairs (alles check, sneller) --- TODO: check escaping in find etc, too much, too slow +environment.loadedluacode = loadfile -- can be overloaded --- This is the first code I wrote for LuaTeX, so it needs some cleanup. +--~ function environment.loadedluacode(name) +--~ if os.spawn("texluac -s -o texluac.luc " .. name) == 0 then +--~ local chunk = loadstring(io.loaddata("texluac.luc")) +--~ os.remove("texluac.luc") +--~ return chunk +--~ else +--~ environment.loadedluacode = loadfile -- can be overloaded +--~ return loadfile(name) +--~ end +--~ end --- To be considered: hash key lowercase, first entry in table filename --- (any case), rest paths (so no need for optimization). Or maybe a --- separate table that matches lowercase names to mixed case when --- present. In that case the lower() cases can go away. I will do that --- only when we run into problems with names ... well ... Iwona-Regular. +function environment.luafilechunk(filename) -- used for loading lua bytecode in the format + filename = file.replacesuffix(filename, "lua") + local fullname = environment.luafile(filename) + if fullname and fullname ~= "" then + if trace_locating then + logs.report("fileio","loading file %s", fullname) + end + return environment.loadedluacode(fullname) + else + if trace_locating then + logs.report("fileio","unknown file %s", filename) + end + return nil + end +end --- Beware, loading and saving is overloaded in luat-tmp! +-- the next ones can use the previous ones / combine -if not versions then versions = { } end versions['luat-inp'] = 1.001 -if not environment then environment = { } end -if not file then file = { } end +function environment.loadluafile(filename, version) + local lucname, luaname, chunk + local basename = file.removesuffix(filename) + if basename == filename then + lucname, luaname = basename .. ".luc", basename .. ".lua" + else + lucname, luaname = nil, basename -- forced suffix + end + -- when not overloaded by explicit suffix we look for a luc file first + local fullname = (lucname and environment.luafile(lucname)) or "" + if fullname ~= "" then + if trace_locating then + logs.report("fileio","loading %s", fullname) + end + chunk = loadfile(fullname) -- this way we don't need a file exists check + end + if chunk then + assert(chunk)() + if version then + -- we check of the version number of this chunk matches + local v = version -- can be nil + if modules and modules[filename] then + v = modules[filename].version -- new method + elseif versions and versions[filename] then + v = versions[filename] -- old method + end + if v == version then + return true + else + if trace_locating then + logs.report("fileio","version mismatch for %s: lua=%s, luc=%s", filename, v, version) + end + environment.loadluafile(filename) + end + else + return true + end + end + fullname = (luaname and environment.luafile(luaname)) or "" + if fullname ~= "" then + if trace_locating then + logs.report("fileio","loading %s", fullname) + end + chunk = loadfile(fullname) -- this way we don't need a file exists check + if not chunk then + if trace_locating then + logs.report("fileio","unknown file %s", filename) + end + else + assert(chunk)() + return true + end + end + return false +end -if environment.aleph_mode == nil then environment.aleph_mode = true end -- temp hack -if not input then input = { } end -if not input.suffixes then input.suffixes = { } end -if not input.formats then input.formats = { } end -if not input.aux then input.aux = { } end +end -- of closure -if not input.suffixmap then input.suffixmap = { } end +do -- create closure to overcome 200 locals limit -if not input.locators then input.locators = { } end -- locate databases -if not input.hashers then input.hashers = { } end -- load databases -if not input.generators then input.generators = { } end -- generate databases -if not input.filters then input.filters = { } end -- conversion filters +if not modules then modules = { } end modules ['trac-inf'] = { + version = 1.001, + comment = "companion to trac-inf.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} local format = string.format -input.locators.notfound = { nil } -input.hashers.notfound = { nil } -input.generators.notfound = { nil } - -input.cacheversion = '1.0.1' -input.banner = nil -input.verbose = false -input.debug = false -input.cnfname = 'texmf.cnf' -input.luaname = 'texmfcnf.lua' -input.lsrname = 'ls-R' -input.luasuffix = '.tma' -input.lucsuffix = '.tmc' - --- we use a cleaned up list / format=any is a wildcard, as is *name - -input.formats['afm'] = 'AFMFONTS' input.suffixes['afm'] = { 'afm' } -input.formats['enc'] = 'ENCFONTS' input.suffixes['enc'] = { 'enc' } -input.formats['fmt'] = 'TEXFORMATS' input.suffixes['fmt'] = { 'fmt' } -input.formats['map'] = 'TEXFONTMAPS' input.suffixes['map'] = { 'map' } -input.formats['mp'] = 'MPINPUTS' input.suffixes['mp'] = { 'mp' } -input.formats['ocp'] = 'OCPINPUTS' input.suffixes['ocp'] = { 'ocp' } -input.formats['ofm'] = 'OFMFONTS' input.suffixes['ofm'] = { 'ofm', 'tfm' } -input.formats['otf'] = 'OPENTYPEFONTS' input.suffixes['otf'] = { 'otf' } -- 'ttf' -input.formats['opl'] = 'OPLFONTS' input.suffixes['opl'] = { 'opl' } -input.formats['otp'] = 'OTPINPUTS' input.suffixes['otp'] = { 'otp' } -input.formats['ovf'] = 'OVFFONTS' input.suffixes['ovf'] = { 'ovf', 'vf' } -input.formats['ovp'] = 'OVPFONTS' input.suffixes['ovp'] = { 'ovp' } -input.formats['tex'] = 'TEXINPUTS' input.suffixes['tex'] = { 'tex' } -input.formats['tfm'] = 'TFMFONTS' input.suffixes['tfm'] = { 'tfm' } -input.formats['ttf'] = 'TTFONTS' input.suffixes['ttf'] = { 'ttf', 'ttc' } -input.formats['pfb'] = 'T1FONTS' input.suffixes['pfb'] = { 'pfb', 'pfa' } -input.formats['vf'] = 'VFFONTS' input.suffixes['vf'] = { 'vf' } - -input.formats['fea'] = 'FONTFEATURES' input.suffixes['fea'] = { 'fea' } -input.formats['cid'] = 'FONTCIDMAPS' input.suffixes['cid'] = { 'cid', 'cidmap' } - -input.formats ['texmfscripts'] = 'TEXMFSCRIPTS' -- new -input.suffixes['texmfscripts'] = { 'rb', 'pl', 'py' } -- 'lua' - -input.formats ['lua'] = 'LUAINPUTS' -- new -input.suffixes['lua'] = { 'lua', 'luc', 'tma', 'tmc' } +local statusinfo, n, registered = { }, 0, { } --- here we catch a few new thingies (todo: add these paths to context.tmf) --- --- FONTFEATURES = .;$TEXMF/fonts/fea// --- FONTCIDMAPS = .;$TEXMF/fonts/cid// +statistics = statistics or { } -function input.checkconfigdata(instance) -- not yet ok, no time for debugging now - local function fix(varname,default) - local proname = varname .. "." .. instance.progname or "crap" - local p = instance.environment[proname] - local v = instance.environment[varname] - if not ((p and p ~= "") or (v and v ~= "")) then - instance.variables[varname] = default -- or environment? - end - end - fix("LUAINPUTS" , ".;$TEXINPUTS;$TEXMFSCRIPTS") - fix("FONTFEATURES", ".;$TEXMF/fonts/fea//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") - fix("FONTCIDMAPS" , ".;$TEXMF/fonts/cid//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") -end +statistics.enable = true +statistics.threshold = 0.05 --- backward compatible ones +-- timing functions -input.alternatives = { } +local clock = os.gettimeofday or os.clock -input.alternatives['map files'] = 'map' -input.alternatives['enc files'] = 'enc' -input.alternatives['cid files'] = 'cid' -input.alternatives['fea files'] = 'fea' -input.alternatives['opentype fonts'] = 'otf' -input.alternatives['truetype fonts'] = 'ttf' -input.alternatives['truetype collections'] = 'ttc' -input.alternatives['type1 fonts'] = 'pfb' +local notimer --- obscure ones +function statistics.hastimer(instance) + return instance and instance.starttime +end -input.formats ['misc fonts'] = '' -input.suffixes['misc fonts'] = { } - -input.formats ['sfd'] = 'SFDFONTS' -input.suffixes ['sfd'] = { 'sfd' } -input.alternatives['subfont definition files'] = 'sfd' - -function input.reset() - - local instance = { } - - instance.rootpath = '' - instance.treepath = '' - instance.progname = environment.progname or 'context' - instance.engine = environment.engine or 'luatex' - instance.format = '' - instance.environment = { } - instance.variables = { } - instance.expansions = { } - instance.files = { } - instance.remap = { } - instance.configuration = { } - instance.setup = { } - instance.order = { } - instance.found = { } - instance.foundintrees = { } - instance.kpsevars = { } - instance.hashes = { } - instance.cnffiles = { } - instance.luafiles = { } - instance.lists = { } - instance.remember = true - instance.diskcache = true - instance.renewcache = false - instance.scandisk = true - instance.cachepath = nil - instance.loaderror = false - instance.smallcache = false - instance.savelists = true - instance.cleanuppaths = true - instance.allresults = false - instance.pattern = nil -- lists - instance.kpseonly = false -- lists - instance.cachefile = 'tmftools' - instance.loadtime = 0 - instance.starttime = 0 - instance.stoptime = 0 - instance.validfile = function(path,name) return true end - instance.data = { } -- only for loading - instance.force_suffixes = true - instance.dummy_path_expr = "^!*unset/*$" - instance.fakepaths = { } - instance.lsrmode = false - - if os.env then - -- store once, freeze and faster - for k,v in pairs(os.env) do - instance.environment[k] = input.bare_variable(v) +function statistics.resettiming(instance) + if not instance then + notimer = { timing = 0, loadtime = 0 } + else + instance.timing, instance.loadtime = 0, 0 + end +end + +function statistics.starttiming(instance) + if not instance then + notimer = { } + instance = notimer + end + local it = instance.timing + if not it then + it = 0 + end + if it == 0 then + instance.starttime = clock() + if not instance.loadtime then + instance.loadtime = 0 end else - -- we will access os.env frequently - for k,v in pairs({'HOME','TEXMF','TEXMFCNF'}) do - local e = os.getenv(v) - if e then - -- input.report("setting",v,"to",input.bare_variable(e)) - instance.environment[v] = input.bare_variable(e) +--~ logs.report("system","nested timing (%s)",tostring(instance)) + end + instance.timing = it + 1 +end + +function statistics.stoptiming(instance, report) + if not instance then + instance = notimer + end + if instance then + local it = instance.timing + if it > 1 then + instance.timing = it - 1 + else + local starttime = instance.starttime + if starttime then + local stoptime = clock() + local loadtime = stoptime - starttime + instance.stoptime = stoptime + instance.loadtime = instance.loadtime + loadtime + if report then + statistics.report("load time %0.3f",loadtime) + end + instance.timing = 0 + return loadtime end end end + return 0 +end + +function statistics.elapsedtime(instance) + if not instance then + instance = notimer + end + return format("%0.3f",(instance and instance.loadtime) or 0) +end + +function statistics.elapsedindeed(instance) + if not instance then + instance = notimer + end + local t = (instance and instance.loadtime) or 0 + return t > statistics.threshold +end + +function statistics.elapsedseconds(instance,rest) -- returns nil if 0 seconds + if statistics.elapsedindeed(instance) then + return format("%s seconds %s", statistics.elapsedtime(instance),rest or "") + end +end - -- cross referencing +-- general function - for k, v in pairs(input.suffixes) do - for _, vv in pairs(v) do - if vv then - input.suffixmap[vv] = k +function statistics.register(tag,fnc) + if statistics.enable and type(fnc) == "function" then + local rt = registered[tag] or (#statusinfo + 1) + statusinfo[rt] = { tag, fnc } + registered[tag] = rt + if #tag > n then n = #tag end + end +end + +function statistics.show(reporter) + if statistics.enable then + if not reporter then reporter = function(tag,data,n) texio.write_nl(tag .. " " .. data) end end + -- this code will move + local register = statistics.register + register("luatex banner", function() + return string.lower(status.banner) + end) + register("control sequences", function() + return format("%s of %s", status.cs_count, status.hash_size+status.hash_extra) + end) + register("callbacks", function() + local total, indirect = status.callbacks or 0, status.indirect_callbacks or 0 + return format("direct: %s, indirect: %s, total: %s", total-indirect, indirect, total) + end) + register("current memory usage", statistics.memused) + register("runtime",statistics.runtime) +-- -- + for i=1,#statusinfo do + local s = statusinfo[i] + local r = s[2]() + if r then + reporter(s[1],r,n) end end + texio.write_nl("") -- final newline + statistics.enable = false end +end - return instance +function statistics.show_job_stat(tag,data,n) + texio.write_nl(format("%-15s: %s - %s","mkiv lua stats",tag:rpadd(n," "),data)) +end +function statistics.memused() -- no math.round yet -) + local round = math.round or math.floor + return format("%s MB (ctx: %s MB)",round(collectgarbage("count")/1000), round(status.luastate_bytes/1000000)) end -function input.reset_hashes(instance) - instance.lists = { } - instance.found = { } +if statistics.runtime then + -- already loaded and set +elseif luatex and luatex.starttime then + statistics.starttime = luatex.starttime + statistics.loadtime = 0 + statistics.timing = 0 +else + statistics.starttiming(statistics) end -function input.bare_variable(str) -- assumes str is a string - -- return string.gsub(string.gsub(string.gsub(str,"%s+$",""),'^"(.+)"$',"%1"),"^'(.+)'$","%1") - return (str:gsub("\s*([\"\']?)(.+)%1\s*", "%2")) +function statistics.runtime() + statistics.stoptiming(statistics) + return statistics.formatruntime(statistics.elapsedtime(statistics)) end -if texio then - input.log = texio.write_nl -else - input.log = print +function statistics.formatruntime(runtime) + return format("%s seconds", statistics.elapsedtime(statistics)) end -function input.simple_logger(kind, name) - if name and name ~= "" then - if input.banner then - input.log(input.banner..kind..": "..name) - else - input.log("<<"..kind..": "..name..">>") - end +function statistics.timed(action,report) + local timer = { } + report = report or logs.simple + statistics.starttiming(timer) + action() + statistics.stoptiming(timer) + report("total runtime: %s",statistics.elapsedtime(timer)) +end + +-- where, not really the best spot for this: + +commands = commands or { } + +local timer + +function commands.resettimer() + statistics.resettiming(timer) + statistics.starttiming(timer) +end + +function commands.elapsedtime() + statistics.stoptiming(timer) + tex.sprint(statistics.elapsedtime(timer)) +end + +commands.resettimer() + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['trac-log'] = { + version = 1.001, + comment = "companion to trac-log.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- this is old code that needs an overhaul + +--~ io.stdout:setvbuf("no") +--~ io.stderr:setvbuf("no") + +local write_nl, write = texio.write_nl or print, texio.write or io.write +local format, gmatch = string.format, string.gmatch +local texcount = tex and tex.count + +if texlua then + write_nl = print + write = io.write +end + +--[[ldx-- +<p>This is a prelude to a more extensive logging module. For the sake +of parsing log files, in addition to the standard logging we will +provide an <l n='xml'/> structured file. Actually, any logging that +is hooked into callbacks will be \XML\ by default.</p> +--ldx]]-- + +logs = logs or { } +logs.xml = logs.xml or { } +logs.tex = logs.tex or { } + +--[[ldx-- +<p>This looks pretty ugly but we need to speed things up a bit.</p> +--ldx]]-- + +logs.moreinfo = [[ +more information about ConTeXt and the tools that come with it can be found at: + +maillist : ntg-context@ntg.nl / http://www.ntg.nl/mailman/listinfo/ntg-context +webpage : http://www.pragma-ade.nl / http://tex.aanhet.net +wiki : http://contextgarden.net +]] + +logs.levels = { + ['error'] = 1, + ['warning'] = 2, + ['info'] = 3, + ['debug'] = 4, +} + +logs.functions = { + 'report', 'start', 'stop', 'push', 'pop', 'line', 'direct', + 'start_run', 'stop_run', + 'start_page_number', 'stop_page_number', + 'report_output_pages', 'report_output_log', + 'report_tex_stat', 'report_job_stat', + 'show_open', 'show_close', 'show_load', +} + +logs.tracers = { +} + +logs.level = 0 +logs.mode = string.lower((os.getenv("MTX.LOG.MODE") or os.getenv("MTX_LOG_MODE") or "tex")) + +function logs.set_level(level) + logs.level = logs.levels[level] or level +end + +function logs.set_method(method) + for _, v in next, logs.functions do + logs[v] = logs[method][v] or function() end + end +end + +-- tex logging + +function logs.tex.report(category,fmt,...) -- new + if fmt then + write_nl(category .. " | " .. format(fmt,...)) else - if input.banner then - input.log(input.banner..kind..": no name") + write_nl(category .. " |") + end +end + +function logs.tex.line(fmt,...) -- new + if fmt then + write_nl(format(fmt,...)) + else + write_nl("") + end +end + +--~ function logs.tex.start_page_number() +--~ local real, user, sub = texcount.realpageno, texcount.userpageno, texcount.subpageno +--~ if real > 0 then +--~ if user > 0 then +--~ if sub > 0 then +--~ write(format("[%s.%s.%s",real,user,sub)) +--~ else +--~ write(format("[%s.%s",real,user)) +--~ end +--~ else +--~ write(format("[%s",real)) +--~ end +--~ else +--~ write("[-") +--~ end +--~ end + +--~ function logs.tex.stop_page_number() +--~ write("]") +--~ end + +local real, user, sub + +function logs.tex.start_page_number() + real, user, sub = texcount.realpageno, texcount.userpageno, texcount.subpageno +end + +function logs.tex.stop_page_number() + if real > 0 then + if user > 0 then + if sub > 0 then + logs.report("pages", "flushing realpage %s, userpage %s, subpage %s",real,user,sub) + else + logs.report("pages", "flushing realpage %s, userpage %s",real,user) + end else - input.log("<<"..kind..": no name>>") + logs.report("pages", "flushing realpage %s",real) end + else + logs.report("pages", "flushing page") + end + io.flush() +end + +logs.tex.report_job_stat = statistics.show_job_stat + +-- xml logging + +function logs.xml.report(category,fmt,...) -- new + if fmt then + write_nl(format("<r category='%s'>%s</r>",category,format(fmt,...))) + else + write_nl(format("<r category='%s'/>",category)) + end +end +function logs.xml.line(fmt,...) -- new + if fmt then + write_nl(format("<r>%s</r>",format(fmt,...))) + else + write_nl("<r/>") end end -function input.dummy_logger() +function logs.xml.start() if logs.level > 0 then tw("<%s>" ) end end +function logs.xml.stop () if logs.level > 0 then tw("</%s>") end end +function logs.xml.push () if logs.level > 0 then tw("<!-- ") end end +function logs.xml.pop () if logs.level > 0 then tw(" -->" ) end end + +function logs.xml.start_run() + write_nl("<?xml version='1.0' standalone='yes'?>") + write_nl("<job>") -- xmlns='www.pragma-ade.com/luatex/schemas/context-job.rng' + write_nl("") +end + +function logs.xml.stop_run() + write_nl("</job>") +end + +function logs.xml.start_page_number() + write_nl(format("<p real='%s' page='%s' sub='%s'", texcount.realpageno, texcount.userpageno, texcount.subpageno)) +end + +function logs.xml.stop_page_number() + write("/>") + write_nl("") +end + +function logs.xml.report_output_pages(p,b) + write_nl(format("<v k='pages' v='%s'/>", p)) + write_nl(format("<v k='bytes' v='%s'/>", b)) + write_nl("") +end + +function logs.xml.report_output_log() +end + +function logs.xml.report_tex_stat(k,v) + texiowrite_nl("log","<v k='"..k.."'>"..tostring(v).."</v>") +end + +local level = 0 + +function logs.xml.show_open(name) + level = level + 1 + texiowrite_nl(format("<f l='%s' n='%s'>",level,name)) +end + +function logs.xml.show_close(name) + texiowrite("</f> ") + level = level - 1 +end + +function logs.xml.show_load(name) + texiowrite_nl(format("<f l='%s' n='%s'/>",level+1,name)) end -function input.settrace(n) - input.trace = tonumber(n or 0) - if input.trace > 0 then - input.logger = input.simple_logger - input.verbose = true +-- + +local name, banner = 'report', 'context' + +local function report(category,fmt,...) + if fmt then + write_nl(format("%s | %s: %s",name,category,format(fmt,...))) + elseif category then + write_nl(format("%s | %s",name,category)) else - input.logger = function() end + write_nl(format("%s |",name)) end end -function input.report(...) -- inefficient - if input.verbose then - if input.banner then - input.log(input.banner .. table.concat({...},' ')) - elseif input.logmode() == 'xml' then - input.log("<t>"..table.concat({...},' ').."</t>") - else - input.log("<<"..table.concat({...},' ')..">>") - end +local function simple(fmt,...) + if fmt then + write_nl(format("%s | %s",name,format(fmt,...))) + else + write_nl(format("%s |",name)) end end -function input.reportlines(str) - if type(str) == "string" then - str = str:split("\n") +function logs.setprogram(_name_,_banner_,_verbose_) + name, banner = _name_, _banner_ + if _verbose_ then + trackers.enable("resolvers.locating") + end + logs.set_method("tex") + logs.report = report -- also used in libraries + logs.simple = simple -- only used in scripts ! + if utils then + utils.report = simple end - for _,v in pairs(str) do input.report(v) end + logs.verbose = _verbose_ end -input.settrace(tonumber(os.getenv("MTX.INPUT.TRACE") or os.getenv("MTX_INPUT_TRACE") or input.trace or 0)) +function logs.setverbose(what) + if what then + trackers.enable("resolvers.locating") + else + trackers.disable("resolvers.locating") + end + logs.verbose = what or false +end --- These functions can be used to test the performance, especially --- loading the database files. +function logs.extendbanner(_banner_,_verbose_) + banner = banner .. " | ".. _banner_ + if _verbose_ ~= nil then + logs.setverbose(what) + end +end -do - local clock = os.gettimeofday or os.clock +logs.verbose = false +logs.report = logs.tex.report +logs.simple = logs.tex.report - function input.starttiming(instance) - if instance then - instance.starttime = clock() - if not instance.loadtime then - instance.loadtime = 0 - end - end +function logs.reportlines(str) -- todo: <lines></lines> + for line in gmatch(str,"(.-)[\n\r]") do + logs.report(line) end +end - function input.stoptiming(instance, report) - if instance then - local starttime = instance.starttime - if starttime then - local stoptime = clock() - local loadtime = stoptime - starttime - instance.stoptime = stoptime - instance.loadtime = instance.loadtime + loadtime - if report then - input.report('load time', format("%0.3f",loadtime)) - end - return loadtime - end +function logs.reportline() -- for scripts too + logs.report() +end + +logs.simpleline = logs.reportline + +function logs.reportbanner() -- for scripts too + logs.report(banner) +end + +function logs.help(message,option) + logs.reportbanner() + logs.reportline() + logs.reportlines(message) + local moreinfo = logs.moreinfo or "" + if moreinfo ~= "" and option ~= "nomoreinfo" then + logs.reportline() + logs.reportlines(moreinfo) + end +end + +logs.set_level('error') +logs.set_method('tex') + +function logs.system(whereto,process,jobname,category,...) + for i=1,10 do + local f = io.open(whereto,"a") + if f then + f:write(format("%s %s => %s => %s => %s\r",os.date("%d/%m/%y %H:%m:%S"),process,jobname,category,format(...))) + f:close() + break + else + sleep(0.1) end - return 0 end +end + +--~ local syslogname = "oeps.xxx" +--~ +--~ for i=1,10 do +--~ logs.system(syslogname,"context","test","fonts","font %s recached due to newer version (%s)","blabla","123") +--~ end +function logs.fatal(where,...) + logs.report(where,"fatal error: %s, aborting now",format(...)) + os.exit() end -function input.elapsedtime(instance) - return format("%0.3f",(instance and instance.loadtime) or 0) + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['data-inp'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files", +} + +-- After a few years using the code the large luat-inp.lua file +-- has been split up a bit. In the process some functionality was +-- dropped: +-- +-- * support for reading lsr files +-- * selective scanning (subtrees) +-- * some public auxiliary functions were made private +-- +-- TODO: os.getenv -> os.env[] +-- TODO: instances.[hashes,cnffiles,configurations,522] +-- TODO: check escaping in find etc, too much, too slow + +-- This lib is multi-purpose and can be loaded again later on so that +-- additional functionality becomes available. We will split thislogs.report("fileio", +-- module in components once we're done with prototyping. This is the +-- first code I wrote for LuaTeX, so it needs some cleanup. Before changing +-- something in this module one can best check with Taco or Hans first; there +-- is some nasty trickery going on that relates to traditional kpse support. + +-- To be considered: hash key lowercase, first entry in table filename +-- (any case), rest paths (so no need for optimization). Or maybe a +-- separate table that matches lowercase names to mixed case when +-- present. In that case the lower() cases can go away. I will do that +-- only when we run into problems with names ... well ... Iwona-Regular. + +-- Beware, loading and saving is overloaded in luat-tmp! + +local format, gsub, find, lower, upper, match, gmatch = string.format, string.gsub, string.find, string.lower, string.upper, string.match, string.gmatch +local concat, insert, sortedkeys = table.concat, table.insert, table.sortedkeys +local next, type = next, type +local lpegmatch = lpeg.match + +local trace_locating, trace_detail, trace_expansions = false, false, false + +trackers.register("resolvers.locating", function(v) trace_locating = v end) +trackers.register("resolvers.details", function(v) trace_detail = v end) +trackers.register("resolvers.expansions", function(v) trace_expansions = v end) -- todo + +if not resolvers then + resolvers = { + suffixes = { }, + formats = { }, + dangerous = { }, + suffixmap = { }, + alternatives = { }, + locators = { }, -- locate databases + hashers = { }, -- load databases + generators = { }, -- generate databases + } end -function input.report_loadtime(instance) - if instance then - input.report('total load time', input.elapsedtime(instance)) +local resolvers = resolvers + +resolvers.locators .notfound = { nil } +resolvers.hashers .notfound = { nil } +resolvers.generators.notfound = { nil } + +resolvers.cacheversion = '1.0.1' +resolvers.cnfname = 'texmf.cnf' +resolvers.luaname = 'texmfcnf.lua' +resolvers.homedir = os.env[os.type == "windows" and 'USERPROFILE'] or os.env['HOME'] or '~' +resolvers.cnfdefault = '{$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c}' + +local dummy_path_expr = "^!*unset/*$" + +local formats = resolvers.formats +local suffixes = resolvers.suffixes +local dangerous = resolvers.dangerous +local suffixmap = resolvers.suffixmap +local alternatives = resolvers.alternatives + +formats['afm'] = 'AFMFONTS' suffixes['afm'] = { 'afm' } +formats['enc'] = 'ENCFONTS' suffixes['enc'] = { 'enc' } +formats['fmt'] = 'TEXFORMATS' suffixes['fmt'] = { 'fmt' } +formats['map'] = 'TEXFONTMAPS' suffixes['map'] = { 'map' } +formats['mp'] = 'MPINPUTS' suffixes['mp'] = { 'mp' } +formats['ocp'] = 'OCPINPUTS' suffixes['ocp'] = { 'ocp' } +formats['ofm'] = 'OFMFONTS' suffixes['ofm'] = { 'ofm', 'tfm' } +formats['otf'] = 'OPENTYPEFONTS' suffixes['otf'] = { 'otf' } -- 'ttf' +formats['opl'] = 'OPLFONTS' suffixes['opl'] = { 'opl' } +formats['otp'] = 'OTPINPUTS' suffixes['otp'] = { 'otp' } +formats['ovf'] = 'OVFFONTS' suffixes['ovf'] = { 'ovf', 'vf' } +formats['ovp'] = 'OVPFONTS' suffixes['ovp'] = { 'ovp' } +formats['tex'] = 'TEXINPUTS' suffixes['tex'] = { 'tex' } +formats['tfm'] = 'TFMFONTS' suffixes['tfm'] = { 'tfm' } +formats['ttf'] = 'TTFONTS' suffixes['ttf'] = { 'ttf', 'ttc', 'dfont' } +formats['pfb'] = 'T1FONTS' suffixes['pfb'] = { 'pfb', 'pfa' } +formats['vf'] = 'VFFONTS' suffixes['vf'] = { 'vf' } + +formats['fea'] = 'FONTFEATURES' suffixes['fea'] = { 'fea' } +formats['cid'] = 'FONTCIDMAPS' suffixes['cid'] = { 'cid', 'cidmap' } + +formats ['texmfscripts'] = 'TEXMFSCRIPTS' -- new +suffixes['texmfscripts'] = { 'rb', 'pl', 'py' } -- 'lua' + +formats ['lua'] = 'LUAINPUTS' -- new +suffixes['lua'] = { 'lua', 'luc', 'tma', 'tmc' } + +-- backward compatible ones + +alternatives['map files'] = 'map' +alternatives['enc files'] = 'enc' +alternatives['cid maps'] = 'cid' -- great, why no cid files +alternatives['font feature files'] = 'fea' -- and fea files here +alternatives['opentype fonts'] = 'otf' +alternatives['truetype fonts'] = 'ttf' +alternatives['truetype collections'] = 'ttc' +alternatives['truetype dictionary'] = 'dfont' +alternatives['type1 fonts'] = 'pfb' + +-- obscure ones + +formats ['misc fonts'] = '' +suffixes['misc fonts'] = { } + +formats ['sfd'] = 'SFDFONTS' +suffixes ['sfd'] = { 'sfd' } +alternatives['subfont definition files'] = 'sfd' + +-- lib paths + +formats ['lib'] = 'CLUAINPUTS' -- new (needs checking) +suffixes['lib'] = (os.libsuffix and { os.libsuffix }) or { 'dll', 'so' } + +-- In practice we will work within one tds tree, but i want to keep +-- the option open to build tools that look at multiple trees, which is +-- why we keep the tree specific data in a table. We used to pass the +-- instance but for practical pusposes we now avoid this and use a +-- instance variable. + +-- here we catch a few new thingies (todo: add these paths to context.tmf) +-- +-- FONTFEATURES = .;$TEXMF/fonts/fea// +-- FONTCIDMAPS = .;$TEXMF/fonts/cid// + +-- we always have one instance active + +resolvers.instance = resolvers.instance or nil -- the current one (slow access) +local instance = resolvers.instance or nil -- the current one (fast access) + +function resolvers.newinstance() + + -- store once, freeze and faster (once reset we can best use + -- instance.environment) maybe better have a register suffix + -- function + + for k, v in next, suffixes do + for i=1,#v do + local vi = v[i] + if vi then + suffixmap[vi] = k + end + end + end + + -- because vf searching is somewhat dangerous, we want to prevent + -- too liberal searching esp because we do a lookup on the current + -- path anyway; only tex (or any) is safe + + for k, v in next, formats do + dangerous[k] = true + end + dangerous.tex = nil + + -- the instance + + local newinstance = { + rootpath = '', + treepath = '', + progname = 'context', + engine = 'luatex', + format = '', + environment = { }, + variables = { }, + expansions = { }, + files = { }, + remap = { }, + configuration = { }, + setup = { }, + order = { }, + found = { }, + foundintrees = { }, + kpsevars = { }, + hashes = { }, + cnffiles = { }, + luafiles = { }, + lists = { }, + remember = true, + diskcache = true, + renewcache = false, + scandisk = true, + cachepath = nil, + loaderror = false, + sortdata = false, + savelists = true, + cleanuppaths = true, + allresults = false, + pattern = nil, -- lists + data = { }, -- only for loading + force_suffixes = true, + fakepaths = { }, + } + + local ne = newinstance.environment + + for k,v in next, os.env do + ne[k] = resolvers.bare_variable(v) end + + return newinstance + end -input.loadtime = input.elapsedtime +function resolvers.setinstance(someinstance) + instance = someinstance + resolvers.instance = someinstance + return someinstance +end -function input.env(instance,key) - return instance.environment[key] or input.osenv(instance,key) +function resolvers.reset() + return resolvers.setinstance(resolvers.newinstance()) end -function input.osenv(instance,key) +local function reset_hashes() + instance.lists = { } + instance.found = { } +end + +local function check_configuration() -- not yet ok, no time for debugging now + local ie, iv = instance.environment, instance.variables + local function fix(varname,default) + local proname = varname .. "." .. instance.progname or "crap" + local p, v = ie[proname], ie[varname] or iv[varname] + if not ((p and p ~= "") or (v and v ~= "")) then + iv[varname] = default -- or environment? + end + end + local name = os.name + if name == "windows" then + fix("OSFONTDIR", "c:/windows/fonts//") + elseif name == "macosx" then + fix("OSFONTDIR", "$HOME/Library/Fonts//;/Library/Fonts//;/System/Library/Fonts//") + else + -- bad luck + end + fix("LUAINPUTS" , ".;$TEXINPUTS;$TEXMFSCRIPTS") -- no progname, hm + -- this will go away some day + fix("FONTFEATURES", ".;$TEXMF/fonts/{data,fea}//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") + fix("FONTCIDMAPS" , ".;$TEXMF/fonts/{data,cid}//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") + -- + fix("LUATEXLIBS" , ".;$TEXMF/luatex/lua//") +end + +function resolvers.bare_variable(str) -- assumes str is a string + return (gsub(str,"\s*([\"\']?)(.+)%1\s*", "%2")) +end + +function resolvers.settrace(n) -- no longer number but: 'locating' or 'detail' + if n then + trackers.disable("resolvers.*") + trackers.enable("resolvers."..n) + end +end + +resolvers.settrace(os.getenv("MTX_INPUT_TRACE")) + +function resolvers.osenv(key) local ie = instance.environment local value = ie[key] if value == nil then @@ -3024,174 +5136,318 @@ function input.osenv(instance,key) if e == nil then -- value = "" -- false else - value = input.bare_variable(e) + value = resolvers.bare_variable(e) end ie[key] = value end return value or "" end --- we follow a rather traditional approach: --- --- (1) texmf.cnf given in TEXMFCNF --- (2) texmf.cnf searched in TEXMF/web2c +function resolvers.env(key) + return instance.environment[key] or resolvers.osenv(key) +end + -- --- for the moment we don't expect a configuration file in a zip -function input.identify_cnf(instance) - -- we no longer support treepath and rootpath (was handy for testing); - -- also we now follow the stupid route: if not set then just assume *one* - -- cnf file under texmf (i.e. distribution) - if #instance.cnffiles == 0 then - if input.env(instance,'TEXMFCNF') == "" then - local ownpath = environment.ownpath() or "." - if ownpath then - -- beware, this is tricky on my own system because at that location I do have - -- the raw tree that ends up in the zip; i.e. I cannot test this kind of mess - local function locate(filename,list) - local ownroot = input.normalize_name(file.join(ownpath,"../..")) - if not lfs.isdir(file.join(ownroot,"texmf")) then - ownroot = input.normalize_name(file.join(ownpath,"..")) - if not lfs.isdir(file.join(ownroot,"texmf")) then - input.verbose = true - input.report("error", "unable to identify cnf file") - return - end - end - local texmfcnf = file.join(ownroot,"texmf-local/web2c",filename) -- for minimals and myself - if not lfs.isfile(texmfcnf) then - texmfcnf = file.join(ownroot,"texmf/web2c",filename) - if not lfs.isfile(texmfcnf) then - input.verbose = true - input.report("error", "unable to locate",filename) - return - end - end - table.insert(list,texmfcnf) - local ie = instance.environment - if not ie['SELFAUTOPARENT'] then ie['SELFAUTOPARENT'] = ownroot end - if not ie['TEXMFCNF'] then ie['TEXMFCNF'] = file.dirname(texmfcnf) end - end - locate(input.luaname,instance.luafiles) - locate(input.cnfname,instance.cnffiles) - if #instance.luafiles == 0 and instance.cnffiles == 0 then - input.verbose = true - input.report("error", "unable to locate",filename) - os.exit() - end - -- here we also assume then TEXMF is set in the distribution, if this trickery is - -- used in the minimals, then users who don't use setuptex are on their own with - -- regards to extra trees - else - input.verbose = true - input.report("error", "unable to identify own path") - os.exit() - end +local function expand_vars(lst) -- simple vars + local variables, env = instance.variables, resolvers.env + local function resolve(a) + return variables[a] or env(a) + end + for k=1,#lst do + lst[k] = gsub(lst[k],"%$([%a%d%_%-]+)",resolve) + end +end + +local function expanded_var(var) -- simple vars + local function resolve(a) + return instance.variables[a] or resolvers.env(a) + end + return (gsub(var,"%$([%a%d%_%-]+)",resolve)) +end + +local function entry(entries,name) + if name and (name ~= "") then + name = gsub(name,'%$','') + local result = entries[name..'.'..instance.progname] or entries[name] + if result then + return result else - local t = input.split_path(input.env(instance,'TEXMFCNF')) - t = input.aux.expanded_path(instance,t) - input.aux.expand_vars(instance,t) - local function locate(filename,list) - for _,v in ipairs(t) do - local texmfcnf = input.normalize_name(file.join(v,filename)) - if lfs.isfile(texmfcnf) then - table.insert(list,texmfcnf) - end - end + result = resolvers.env(name) + if result then + instance.variables[name] = result + resolvers.expand_variables() + return instance.expansions[name] or "" end - locate(input.luaname,instance.luafiles) - locate(input.cnfname,instance.cnffiles) end end + return "" end -function input.load_cnf(instance) - local function loadoldconfigdata() - for _, fname in ipairs(instance.cnffiles) do - input.aux.load_cnf(instance,fname) +local function is_entry(entries,name) + if name and name ~= "" then + name = gsub(name,'%$','') + return (entries[name..'.'..instance.progname] or entries[name]) ~= nil + else + return false + end +end + +-- {a,b,c,d} +-- a,b,c/{p,q,r},d +-- a,b,c/{p,q,r}/d/{x,y,z}// +-- a,b,c/{p,q/{x,y,z},r},d/{p,q,r} +-- a,b,c/{p,q/{x,y,z},r},d/{p,q,r} +-- a{b,c}{d,e}f +-- {a,b,c,d} +-- {a,b,c/{p,q,r},d} +-- {a,b,c/{p,q,r}/d/{x,y,z}//} +-- {a,b,c/{p,q/{x,y,z}},d/{p,q,r}} +-- {a,b,c/{p,q/{x,y,z},w}v,d/{p,q,r}} +-- {$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c} + +-- this one is better and faster, but it took me a while to realize +-- that this kind of replacement is cleaner than messy parsing and +-- fuzzy concatenating we can probably gain a bit with selectively +-- applying lpeg, but experiments with lpeg parsing this proved not to +-- work that well; the parsing is ok, but dealing with the resulting +-- table is a pain because we need to work inside-out recursively + +local function do_first(a,b) + local t = { } + for s in gmatch(b,"[^,]+") do t[#t+1] = a .. s end + return "{" .. concat(t,",") .. "}" +end + +local function do_second(a,b) + local t = { } + for s in gmatch(a,"[^,]+") do t[#t+1] = s .. b end + return "{" .. concat(t,",") .. "}" +end + +local function do_both(a,b) + local t = { } + for sa in gmatch(a,"[^,]+") do + for sb in gmatch(b,"[^,]+") do + t[#t+1] = sa .. sb end end - -- instance.cnffiles contain complete names now ! - if #instance.cnffiles == 0 then - input.report("no cnf files found (TEXMFCNF may not be set/known)") + return "{" .. concat(t,",") .. "}" +end + +local function do_three(a,b,c) + return a .. b.. c +end + +local function splitpathexpr(str, t, validate) + -- no need for further optimization as it is only called a + -- few times, we can use lpeg for the sub + if trace_expansions then + logs.report("fileio","expanding variable '%s'",str) + end + t = t or { } + str = gsub(str,",}",",@}") + str = gsub(str,"{,","{@,") + -- str = "@" .. str .. "@" + local ok, done + while true do + done = false + while true do + str, ok = gsub(str,"([^{},]+){([^{}]+)}",do_first) + if ok > 0 then done = true else break end + end + while true do + str, ok = gsub(str,"{([^{}]+)}([^{},]+)",do_second) + if ok > 0 then done = true else break end + end + while true do + str, ok = gsub(str,"{([^{}]+)}{([^{}]+)}",do_both) + if ok > 0 then done = true else break end + end + str, ok = gsub(str,"({[^{}]*){([^{}]+)}([^{}]*})",do_three) + if ok > 0 then done = true end + if not done then break end + end + str = gsub(str,"[{}]", "") + str = gsub(str,"@","") + if validate then + for s in gmatch(str,"[^,]+") do + s = validate(s) + if s then t[#t+1] = s end + end else - instance.rootpath = instance.cnffiles[1] - for k,fname in ipairs(instance.cnffiles) do - instance.cnffiles[k] = input.normalize_name(fname:gsub("\\",'/')) + for s in gmatch(str,"[^,]+") do + t[#t+1] = s end - for i=1,3 do - instance.rootpath = file.dirname(instance.rootpath) + end + if trace_expansions then + for k=1,#t do + logs.report("fileio","% 4i: %s",k,t[k]) end - instance.rootpath = input.normalize_name(instance.rootpath) - instance.environment['SELFAUTOPARENT'] = instance.rootpath -- just to be sure - if instance.lsrmode then - loadoldconfigdata() - elseif instance.diskcache and not instance.renewcache then - input.loadoldconfig(instance,instance.cnffiles) - if instance.loaderror then - loadoldconfigdata() - input.saveoldconfig(instance) + end + return t +end + +local function expanded_path_from_list(pathlist) -- maybe not a list, just a path + -- a previous version fed back into pathlist + local newlist, ok = { }, false + for k=1,#pathlist do + if find(pathlist[k],"[{}]") then + ok = true + break + end + end + if ok then + local function validate(s) + s = file.collapse_path(s) + return s ~= "" and not find(s,dummy_path_expr) and s + end + for k=1,#pathlist do + splitpathexpr(pathlist[k],newlist,validate) + end + else + for k=1,#pathlist do + for p in gmatch(pathlist[k],"([^,]+)") do + p = file.collapse_path(p) + if p ~= "" then newlist[#newlist+1] = p end end - else - loadoldconfigdata() - if instance.renewcache then - input.saveoldconfig(instance) + end + end + return newlist +end + +-- we follow a rather traditional approach: +-- +-- (1) texmf.cnf given in TEXMFCNF +-- (2) texmf.cnf searched in default variable +-- +-- also we now follow the stupid route: if not set then just assume *one* +-- cnf file under texmf (i.e. distribution) + +local args = environment and environment.original_arguments or arg -- this needs a cleanup + +resolvers.ownbin = resolvers.ownbin or args[-2] or arg[-2] or args[-1] or arg[-1] or arg[0] or "luatex" +resolvers.ownbin = gsub(resolvers.ownbin,"\\","/") + +function resolvers.getownpath() + local ownpath = resolvers.ownpath or os.selfdir + if not ownpath or ownpath == "" or ownpath == "unset" then + ownpath = args[-1] or arg[-1] + ownpath = ownpath and file.dirname(gsub(ownpath,"\\","/")) + if not ownpath or ownpath == "" then + ownpath = args[-0] or arg[-0] + ownpath = ownpath and file.dirname(gsub(ownpath,"\\","/")) + end + local binary = resolvers.ownbin + if not ownpath or ownpath == "" then + ownpath = ownpath and file.dirname(binary) + end + if not ownpath or ownpath == "" then + if os.binsuffix ~= "" then + binary = file.replacesuffix(binary,os.binsuffix) + end + for p in gmatch(os.getenv("PATH"),"[^"..io.pathseparator.."]+") do + local b = file.join(p,binary) + if lfs.isfile(b) then + -- we assume that after changing to the path the currentdir function + -- resolves to the real location and use this side effect here; this + -- trick is needed because on the mac installations use symlinks in the + -- path instead of real locations + local olddir = lfs.currentdir() + if lfs.chdir(p) then + local pp = lfs.currentdir() + if trace_locating and p ~= pp then + logs.report("fileio","following symlink '%s' to '%s'",p,pp) + end + ownpath = pp + lfs.chdir(olddir) + else + if trace_locating then + logs.report("fileio","unable to check path '%s'",p) + end + ownpath = p + end + break + end end end - input.aux.collapse_cnf_data(instance) + if not ownpath or ownpath == "" then + ownpath = "." + logs.report("fileio","forcing fallback ownpath .") + elseif trace_locating then + logs.report("fileio","using ownpath '%s'",ownpath) + end end - input.checkconfigdata(instance) + resolvers.ownpath = ownpath + function resolvers.getownpath() + return resolvers.ownpath + end + return ownpath end -function input.load_lua(instance) - if #instance.luafiles == 0 then - -- yet harmless +local own_places = { "SELFAUTOLOC", "SELFAUTODIR", "SELFAUTOPARENT", "TEXMFCNF" } + +local function identify_own() + local ownpath = resolvers.getownpath() or dir.current() + local ie = instance.environment + if ownpath then + if resolvers.env('SELFAUTOLOC') == "" then os.env['SELFAUTOLOC'] = file.collapse_path(ownpath) end + if resolvers.env('SELFAUTODIR') == "" then os.env['SELFAUTODIR'] = file.collapse_path(ownpath .. "/..") end + if resolvers.env('SELFAUTOPARENT') == "" then os.env['SELFAUTOPARENT'] = file.collapse_path(ownpath .. "/../..") end else - instance.rootpath = instance.luafiles[1] - for k,fname in ipairs(instance.luafiles) do - instance.luafiles[k] = input.normalize_name(fname:gsub("\\",'/')) - end - for i=1,3 do - instance.rootpath = file.dirname(instance.rootpath) + logs.report("fileio","error: unable to locate ownpath") + os.exit() + end + if resolvers.env('TEXMFCNF') == "" then os.env['TEXMFCNF'] = resolvers.cnfdefault end + if resolvers.env('TEXOS') == "" then os.env['TEXOS'] = resolvers.env('SELFAUTODIR') end + if resolvers.env('TEXROOT') == "" then os.env['TEXROOT'] = resolvers.env('SELFAUTOPARENT') end + if trace_locating then + for i=1,#own_places do + local v = own_places[i] + logs.report("fileio","variable '%s' set to '%s'",v,resolvers.env(v) or "unknown") end - instance.rootpath = input.normalize_name(instance.rootpath) - instance.environment['SELFAUTOPARENT'] = instance.rootpath -- just to be sure - input.loadnewconfig(instance) - input.aux.collapse_cnf_data(instance) end - input.checkconfigdata(instance) + identify_own = function() end end -function input.aux.collapse_cnf_data(instance) -- potential optmization: pass start index (setup and configuration are shared) - for _,c in ipairs(instance.order) do - for k,v in pairs(c) do - if not instance.variables[k] then - if instance.environment[k] then - instance.variables[k] = instance.environment[k] - else - instance.kpsevars[k] = true - instance.variables[k] = input.bare_variable(v) +function resolvers.identify_cnf() + if #instance.cnffiles == 0 then + -- fallback + identify_own() + -- the real search + resolvers.expand_variables() + local t = resolvers.split_path(resolvers.env('TEXMFCNF')) + t = expanded_path_from_list(t) + expand_vars(t) -- redundant + local function locate(filename,list) + for i=1,#t do + local ti = t[i] + local texmfcnf = file.collapse_path(file.join(ti,filename)) + if lfs.isfile(texmfcnf) then + list[#list+1] = texmfcnf end end end + locate(resolvers.luaname,instance.luafiles) + locate(resolvers.cnfname,instance.cnffiles) end end -function input.aux.load_cnf(instance,fname) - fname = input.clean_path(fname) - local lname = fname:gsub("%.%a+$",input.luasuffix) - local f = io.open(lname) - if f then -- this will go - f:close() - local dname = file.dirname(fname) +local function load_cnf_file(fname) + fname = resolvers.clean_path(fname) + local lname = file.replacesuffix(fname,'lua') + if lfs.isfile(lname) then + local dname = file.dirname(fname) -- fname ? if not instance.configuration[dname] then - input.aux.load_configuration(instance,dname,lname) + resolvers.load_data(dname,'configuration',lname and file.basename(lname)) instance.order[#instance.order+1] = instance.configuration[dname] end else f = io.open(fname) if f then - input.report("loading", fname) + if trace_locating then + logs.report("fileio","loading configuration file %s", fname) + end local line, data, n, k, v local dname = file.dirname(fname) if not instance.configuration[dname] then @@ -3203,17 +5459,19 @@ function input.aux.load_cnf(instance,fname) local line, n = f:read(), 0 if line then while true do -- join lines - line, n = line:gsub("\\%s*$", "") + line, n = gsub(line,"\\%s*$", "") if n > 0 then line = line .. f:read() else break end end - if not line:find("^[%%#]") then - local k, v = (line:gsub("%s*%%.*$","")):match("%s*(.-)%s*=%s*(.-)%s*$") + if not find(line,"^[%%#]") then + local l = gsub(line,"%s*%%.*$","") + local k, v = match(l,"%s*(.-)%s*=%s*(.-)%s*$") if k and v and not data[k] then - data[k] = (v:gsub("[%%#].*",'')):gsub("~", "$HOME") + v = gsub(v,"[%%#].*",'') + data[k] = gsub(v,"~","$HOME") instance.kpsevars[k] = true end end @@ -3222,240 +5480,333 @@ function input.aux.load_cnf(instance,fname) end end f:close() + elseif trace_locating then + logs.report("fileio","skipping configuration file '%s'", fname) + end + end +end + +local function collapse_cnf_data() -- potential optimization: pass start index (setup and configuration are shared) + local order = instance.order + for i=1,#order do + local c = order[i] + for k,v in next, c do + if not instance.variables[k] then + if instance.environment[k] then + instance.variables[k] = instance.environment[k] + else + instance.kpsevars[k] = true + instance.variables[k] = resolvers.bare_variable(v) + end + end + end + end +end + +function resolvers.load_cnf() + local function loadoldconfigdata() + local cnffiles = instance.cnffiles + for i=1,#cnffiles do + load_cnf_file(cnffiles[i]) + end + end + -- instance.cnffiles contain complete names now ! + -- we still use a funny mix of cnf and new but soon + -- we will switch to lua exclusively as we only use + -- the file to collect the tree roots + if #instance.cnffiles == 0 then + if trace_locating then + logs.report("fileio","no cnf files found (TEXMFCNF may not be set/known)") + end + else + local cnffiles = instance.cnffiles + instance.rootpath = cnffiles[1] + for k=1,#cnffiles do + instance.cnffiles[k] = file.collapse_path(cnffiles[k]) + end + for i=1,3 do + instance.rootpath = file.dirname(instance.rootpath) + end + instance.rootpath = file.collapse_path(instance.rootpath) + if instance.diskcache and not instance.renewcache then + resolvers.loadoldconfig(instance.cnffiles) + if instance.loaderror then + loadoldconfigdata() + resolvers.saveoldconfig() + end else - input.report("skipping", fname) + loadoldconfigdata() + if instance.renewcache then + resolvers.saveoldconfig() + end + end + collapse_cnf_data() + end + check_configuration() +end + +function resolvers.load_lua() + if #instance.luafiles == 0 then + -- yet harmless + else + instance.rootpath = instance.luafiles[1] + local luafiles = instance.luafiles + for k=1,#luafiles do + instance.luafiles[k] = file.collapse_path(luafiles[k]) end + for i=1,3 do + instance.rootpath = file.dirname(instance.rootpath) + end + instance.rootpath = file.collapse_path(instance.rootpath) + resolvers.loadnewconfig() + collapse_cnf_data() end + check_configuration() end -- database loading -function input.load_hash(instance) - input.locatelists(instance) - if instance.lsrmode then - input.loadlists(instance) - elseif instance.diskcache and not instance.renewcache then - input.loadfiles(instance) +function resolvers.load_hash() + resolvers.locatelists() + if instance.diskcache and not instance.renewcache then + resolvers.loadfiles() if instance.loaderror then - input.loadlists(instance) - input.savefiles(instance) + resolvers.loadlists() + resolvers.savefiles() end else - input.loadlists(instance) + resolvers.loadlists() if instance.renewcache then - input.savefiles(instance) + resolvers.savefiles() end end end -function input.aux.append_hash(instance,type,tag,name) - input.logger("= hash append",tag) - table.insert(instance.hashes, { ['type']=type, ['tag']=tag, ['name']=name } ) +function resolvers.append_hash(type,tag,name) + if trace_locating then + logs.report("fileio","hash '%s' appended",tag) + end + insert(instance.hashes, { ['type']=type, ['tag']=tag, ['name']=name } ) end -function input.aux.prepend_hash(instance,type,tag,name) - input.logger("= hash prepend",tag) - table.insert(instance.hashes, 1, { ['type']=type, ['tag']=tag, ['name']=name } ) +function resolvers.prepend_hash(type,tag,name) + if trace_locating then + logs.report("fileio","hash '%s' prepended",tag) + end + insert(instance.hashes, 1, { ['type']=type, ['tag']=tag, ['name']=name } ) end -function input.aux.extend_texmf_var(instance,specification) -- crap - if instance.environment['TEXMF'] then - input.report("extending environment variable TEXMF with", specification) - instance.environment['TEXMF'] = instance.environment['TEXMF']:gsub("^%{", function() - return "{" .. specification .. "," - end) - elseif instance.variables['TEXMF'] then - input.report("extending configuration variable TEXMF with", specification) - instance.variables['TEXMF'] = instance.variables['TEXMF']:gsub("^%{", function() - return "{" .. specification .. "," - end) +function resolvers.extend_texmf_var(specification) -- crap, we could better prepend the hash +-- local t = resolvers.expanded_path_list('TEXMF') -- full expansion + local t = resolvers.split_path(resolvers.env('TEXMF')) + insert(t,1,specification) + local newspec = concat(t,";") + if instance.environment["TEXMF"] then + instance.environment["TEXMF"] = newspec + elseif instance.variables["TEXMF"] then + instance.variables["TEXMF"] = newspec else - input.report("setting configuration variable TEXMF to", specification) - instance.variables['TEXMF'] = "{" .. specification .. "}" + -- weird end - if instance.variables['TEXMF']:find("%,") and not instance.variables['TEXMF']:find("^%{") then - input.report("adding {} to complex TEXMF variable, best do that yourself") - instance.variables['TEXMF'] = "{" .. instance.variables['TEXMF'] .. "}" - end - input.expand_variables(instance) - input.reset_hashes(instance) + resolvers.expand_variables() + reset_hashes() end -- locators -function input.locatelists(instance) - for _, path in pairs(input.simplified_list(input.expansion(instance,'TEXMF'))) do - path = file.collapse_path(path) - input.report("locating list of",path) - input.locatedatabase(instance,input.normalize_name(path)) +function resolvers.locatelists() + local texmfpaths = resolvers.clean_path_list('TEXMF') + for i=1,#texmfpaths do + local path = texmfpaths[i] + if trace_locating then + logs.report("fileio","locating list of '%s'",path) + end + resolvers.locatedatabase(file.collapse_path(path)) end end -function input.locatedatabase(instance,specification) - return input.methodhandler('locators', instance, specification) +function resolvers.locatedatabase(specification) + return resolvers.methodhandler('locators', specification) end -function input.locators.tex(instance,specification) +function resolvers.locators.tex(specification) if specification and specification ~= '' and lfs.isdir(specification) then - input.logger('! tex locator', specification..' found') - input.aux.append_hash(instance,'file',specification,filename) - else - input.logger('? tex locator', specification..' not found') + if trace_locating then + logs.report("fileio","tex locator '%s' found",specification) + end + resolvers.append_hash('file',specification,filename) + elseif trace_locating then + logs.report("fileio","tex locator '%s' not found",specification) end end -- hashers -function input.hashdatabase(instance,tag,name) - return input.methodhandler('hashers',instance,tag,name) +function resolvers.hashdatabase(tag,name) + return resolvers.methodhandler('hashers',tag,name) end -function input.loadfiles(instance) +function resolvers.loadfiles() instance.loaderror = false instance.files = { } if not instance.renewcache then - for _, hash in ipairs(instance.hashes) do - input.hashdatabase(instance,hash.tag,hash.name) + local hashes = instance.hashes + for k=1,#hashes do + local hash = hashes[k] + resolvers.hashdatabase(hash.tag,hash.name) if instance.loaderror then break end end end end -function input.hashers.tex(instance,tag,name) - input.aux.load_files(instance,tag) +function resolvers.hashers.tex(tag,name) + resolvers.load_data(tag,'files') end -- generators: -function input.loadlists(instance) - for _, hash in ipairs(instance.hashes) do - input.generatedatabase(instance,hash.tag) +function resolvers.loadlists() + local hashes = instance.hashes + for i=1,#hashes do + resolvers.generatedatabase(hashes[i].tag) end end -function input.generatedatabase(instance,specification) - return input.methodhandler('generators', instance, specification) +function resolvers.generatedatabase(specification) + return resolvers.methodhandler('generators', specification) end -do +-- starting with . or .. etc or funny char - local weird = lpeg.anywhere(lpeg.S("~`!#$%^&*()={}[]:;\"\'||<>,?\n\r\t")) +local weird = lpeg.P(".")^1 + lpeg.anywhere(lpeg.S("~`!#$%^&*()={}[]:;\"\'||<>,?\n\r\t")) - function input.generators.tex(instance,specification) - local tag = specification - if not instance.lsrmode and lfs and lfs.dir then - input.report("scanning path",specification) - instance.files[tag] = { } - local files = instance.files[tag] - local n, m, r = 0, 0, 0 - local spec = specification .. '/' - local attributes = lfs.attributes - local directory = lfs.dir - local small = instance.smallcache - local function action(path) - local mode, full - if path then - full = spec .. path .. '/' - else - full = spec - end - for name in directory(full) do - if name:find("^%.") then - -- skip - -- elseif name:find("[%~%`%!%#%$%%%^%&%*%(%)%=%{%}%[%]%:%;\"\'%|%<%>%,%?\n\r\t]") then -- too much escaped - elseif weird:match(name) then - -- texio.write_nl("skipping " .. name) - -- skip - else - mode = attributes(full..name,'mode') - if mode == "directory" then - m = m + 1 - if path then - action(path..'/'..name) +--~ local l_forbidden = lpeg.S("~`!#$%^&*()={}[]:;\"\'||\\/<>,?\n\r\t") +--~ local l_confusing = lpeg.P(" ") +--~ local l_character = lpeg.patterns.utf8 +--~ local l_dangerous = lpeg.P(".") + +--~ local l_normal = (l_character - l_forbidden - l_confusing - l_dangerous) * (l_character - l_forbidden - l_confusing^2)^0 * lpeg.P(-1) +--~ ----- l_normal = l_normal * lpeg.Cc(true) + lpeg.Cc(false) + +--~ local function test(str) +--~ print(str,lpeg.match(l_normal,str)) +--~ end +--~ test("ヒラギノ明朝 Pro W3") +--~ test("..ヒラギノ明朝 Pro W3") +--~ test(":ヒラギノ明朝 Pro W3;") +--~ test("ヒラギノ明朝 /Pro W3;") +--~ test("ヒラギノ明朝 Pro W3") + +function resolvers.generators.tex(specification) + local tag = specification + if trace_locating then + logs.report("fileio","scanning path '%s'",specification) + end + instance.files[tag] = { } + local files = instance.files[tag] + local n, m, r = 0, 0, 0 + local spec = specification .. '/' + local attributes = lfs.attributes + local directory = lfs.dir + local function action(path) + local full + if path then + full = spec .. path .. '/' + else + full = spec + end + for name in directory(full) do + if not lpegmatch(weird,name) then + -- if lpegmatch(l_normal,name) then + local mode = attributes(full..name,'mode') + if mode == 'file' then + if path then + n = n + 1 + local f = files[name] + if f then + if type(f) == 'string' then + files[name] = { f, path } else - action(name) + f[#f+1] = path end - elseif path and mode == 'file' then - n = n + 1 - local f = files[name] - if f then - if not small then - if type(f) == 'string' then - files[name] = { f, path } - else - f[#f+1] = path - end - end - else - files[name] = path - local lower = name:lower() - if name ~= lower then - files["remap:"..lower] = name - r = r + 1 - end + else -- probably unique anyway + files[name] = path + local lower = lower(name) + if name ~= lower then + files["remap:"..lower] = name + r = r + 1 end end end - end - end - action() - input.report(format("%s files found on %s directories with %s uppercase remappings",n,m,r)) - else - local fullname = file.join(specification,input.lsrname) - local path = '.' - local f = io.open(fullname) - if f then - instance.files[tag] = { } - local files = instance.files[tag] - local small = instance.smallcache - input.report("loading lsr file",fullname) - -- for line in f:lines() do -- much slower then the next one - for line in (f:read("*a")):gmatch("(.-)\n") do - if line:find("^[%a%d]") then - local fl = files[line] - if fl then - if not small then - if type(fl) == 'string' then - files[line] = { fl, path } -- table - else - fl[#fl+1] = path - end - end - else - files[line] = path -- string - local lower = line:lower() - if line ~= lower then - files["remap:"..lower] = line - end - end + elseif mode == 'directory' then + m = m + 1 + if path then + action(path..'/'..name) else - path = line:match("%.%/(.-)%:$") or path -- match could be nil due to empty line + action(name) end end - f:close() end end end - + action() + if trace_locating then + logs.report("fileio","%s files found on %s directories with %s uppercase remappings",n,m,r) + end end -- savers, todo -function input.savefiles(instance) - input.aux.save_data(instance, 'files', function(k,v) - return instance.validfile(k,v) -- path, name - end) +function resolvers.savefiles() + resolvers.save_data('files') end -- A config (optionally) has the paths split in tables. Internally -- we join them and split them after the expansion has taken place. This -- is more convenient. -function input.splitconfig(instance) - for i,c in ipairs(instance) do - for k,v in pairs(c) do +--~ local checkedsplit = string.checkedsplit + +local cache = { } + +local splitter = lpeg.Ct(lpeg.splitat(lpeg.S(os.type == "windows" and ";" or ":;"))) + +local function split_kpse_path(str) -- beware, this can be either a path or a {specification} + local found = cache[str] + if not found then + if str == "" then + found = { } + else + str = gsub(str,"\\","/") +--~ local split = (find(str,";") and checkedsplit(str,";")) or checkedsplit(str,io.pathseparator) +local split = lpegmatch(splitter,str) + found = { } + for i=1,#split do + local s = split[i] + if not find(s,"^{*unset}*") then + found[#found+1] = s + end + end + if trace_expansions then + logs.report("fileio","splitting path specification '%s'",str) + for k=1,#found do + logs.report("fileio","% 4i: %s",k,found[k]) + end + end + cache[str] = found + end + end + return found +end + +resolvers.split_kpse_path = split_kpse_path + +function resolvers.splitconfig() + for i=1,#instance do + local c = instance[i] + for k,v in next, c do if type(v) == 'string' then - local t = file.split_path(v) + local t = split_kpse_path(v) if #t > 1 then c[k] = t end @@ -3463,23 +5814,28 @@ function input.splitconfig(instance) end end end -function input.joinconfig(instance) - for i,c in ipairs(instance.order) do - for k,v in pairs(c) do + +function resolvers.joinconfig() + local order = instance.order + for i=1,#order do + local c = order[i] + for k,v in next, c do -- indexed? if type(v) == 'table' then c[k] = file.join_path(v) end end end end -function input.split_path(str) + +function resolvers.split_path(str) if type(str) == 'table' then return str else - return file.split_path(str) + return split_kpse_path(str) end end -function input.join_path(str) + +function resolvers.join_path(str) if type(str) == 'table' then return file.join_path(str) else @@ -3487,48 +5843,48 @@ function input.join_path(str) end end -function input.splitexpansions(instance) - for k,v in pairs(instance.expansions) do - local t, h = { }, { } - for _,vv in pairs(file.split_path(v)) do +function resolvers.splitexpansions() + local ie = instance.expansions + for k,v in next, ie do + local t, h, p = { }, { }, split_kpse_path(v) + for kk=1,#p do + local vv = p[kk] if vv ~= "" and not h[vv] then t[#t+1] = vv h[vv] = true end end if #t > 1 then - instance.expansions[k] = t + ie[k] = t else - instance.expansions[k] = t[1] + ie[k] = t[1] end end end -- end of split/join code -function input.saveoldconfig(instance) - input.splitconfig(instance) - input.aux.save_data(instance, 'configuration', nil) - input.joinconfig(instance) +function resolvers.saveoldconfig() + resolvers.splitconfig() + resolvers.save_data('configuration') + resolvers.joinconfig() end -input.configbanner = [[ +resolvers.configbanner = [[ -- This is a Luatex configuration file created by 'luatools.lua' or -- 'luatex.exe' directly. For comment, suggestions and questions you can -- contact the ConTeXt Development Team. This configuration file is -- not copyrighted. [HH & TH] ]] -function input.serialize(files) +function resolvers.serialize(files) -- This version is somewhat optimized for the kind of -- tables that we deal with, so it's much faster than -- the generic serializer. This makes sense because -- luatools and mtxtools are called frequently. Okay, -- we pay a small price for properly tabbed tables. local t = { } - local concat = table.concat - local sorted = table.sortedkeys - local function dump(k,v,m) + local function dump(k,v,m) -- could be moved inline if type(v) == 'string' then return m .. "['" .. k .. "']='" .. v .. "'," elseif #v == 1 then @@ -3539,11 +5895,15 @@ function input.serialize(files) end t[#t+1] = "return {" if instance.sortdata then - for _, k in pairs(sorted(files)) do + local sortedfiles = sortedkeys(files) + for i=1,#sortedfiles do + local k = sortedfiles[i] local fk = files[k] if type(fk) == 'table' then t[#t+1] = "\t['" .. k .. "']={" - for _, kk in pairs(sorted(fk)) do + local sortedfk = sortedkeys(fk) + for j=1,#sortedfk do + local kk = sortedfk[j] t[#t+1] = dump(kk,fk[kk],"\t\t") end t[#t+1] = "\t}," @@ -3552,10 +5912,10 @@ function input.serialize(files) end end else - for k, v in pairs(files) do + for k, v in next, files do if type(v) == 'table' then t[#t+1] = "\t['" .. k .. "']={" - for kk,vv in pairs(v) do + for kk,vv in next, v do t[#t+1] = dump(kk,vv,"\t\t") end t[#t+1] = "\t}," @@ -3568,62 +5928,75 @@ function input.serialize(files) return concat(t,"\n") end -if not texmf then texmf = {} end -- no longer needed, at least not here +local data_state = { } + +function resolvers.data_state() + return data_state or { } +end -function input.aux.save_data(instance, dataname, check, makename) -- untested without cache overload - for cachename, files in pairs(instance[dataname]) do +function resolvers.save_data(dataname, makename) -- untested without cache overload + for cachename, files in next, instance[dataname] do local name = (makename or file.join)(cachename,dataname) - local luaname, lucname = name .. input.luasuffix, name .. input.lucsuffix - input.report("preparing " .. dataname .. " for", luaname) - for k, v in pairs(files) do - if not check or check(v,k) then -- path, name - if type(v) == "table" and #v == 1 then - files[k] = v[1] - end - else - files[k] = nil -- false + local luaname, lucname = name .. ".lua", name .. ".luc" + if trace_locating then + logs.report("fileio","preparing '%s' for '%s'",dataname,cachename) + end + for k, v in next, files do + if type(v) == "table" and #v == 1 then + files[k] = v[1] end end local data = { type = dataname, root = cachename, - version = input.cacheversion, + version = resolvers.cacheversion, date = os.date("%Y-%m-%d"), time = os.date("%H:%M:%S"), content = files, + uuid = os.uuid(), } - local f = io.open(luaname,'w') - if f then - input.report("saving " .. dataname .. " in", luaname) - f:write(input.serialize(data)) - f:close() - input.report("compiling " .. dataname .. " to", lucname) - if not utils.lua.compile(luaname,lucname) then - input.report("compiling failed for " .. dataname .. ", deleting file " .. lucname) + local ok = io.savedata(luaname,resolvers.serialize(data)) + if ok then + if trace_locating then + logs.report("fileio","'%s' saved in '%s'",dataname,luaname) + end + if utils.lua.compile(luaname,lucname,false,true) then -- no cleanup but strip + if trace_locating then + logs.report("fileio","'%s' compiled to '%s'",dataname,lucname) + end + else + if trace_locating then + logs.report("fileio","compiling failed for '%s', deleting file '%s'",dataname,lucname) + end os.remove(lucname) end - else - input.report("unable to save " .. dataname .. " in " .. name..input.luasuffix) + elseif trace_locating then + logs.report("fileio","unable to save '%s' in '%s' (access error)",dataname,luaname) end end end -function input.aux.load_data(instance,pathname,dataname,filename,makename) -- untested without cache overload +function resolvers.load_data(pathname,dataname,filename,makename) -- untested without cache overload filename = ((not filename or (filename == "")) and dataname) or filename filename = (makename and makename(dataname,filename)) or file.join(pathname,filename) - local blob = loadfile(filename .. input.lucsuffix) or loadfile(filename .. input.luasuffix) + local blob = loadfile(filename .. ".luc") or loadfile(filename .. ".lua") if blob then local data = blob() - if data and data.content and data.type == dataname and data.version == input.cacheversion then - input.report("loading",dataname,"for",pathname,"from",filename) + if data and data.content and data.type == dataname and data.version == resolvers.cacheversion then + data_state[#data_state+1] = data.uuid + if trace_locating then + logs.report("fileio","loading '%s' for '%s' from '%s'",dataname,pathname,filename) + end instance[dataname][pathname] = data.content else - input.report("skipping",dataname,"for",pathname,"from",filename) + if trace_locating then + logs.report("fileio","skipping '%s' for '%s' from '%s'",dataname,pathname,filename) + end instance[dataname][pathname] = { } instance.loaderror = true end - else - input.report("skipping",dataname,"for",pathname,"from",filename) + elseif trace_locating then + logs.report("fileio","skipping '%s' for '%s' from '%s'",dataname,pathname,filename) end end @@ -3636,195 +6009,143 @@ end -- TEXMFBOGUS = 'effe checken of dit werkt', -- } -function input.aux.load_texmfcnf(instance,dataname,pathname) - local filename = file.join(pathname,input.luaname) - local blob = loadfile(filename) - if blob then - local data = blob() - if data then - input.report("loading","configuration file",filename) - if true then - -- flatten to variable.progname - local t = { } - for k, v in pairs(data) do -- v = progname - if type(v) == "string" then - t[k] = v - else - for kk, vv in pairs(v) do -- vv = variable - if type(vv) == "string" then - t[vv.."."..v] = kk +function resolvers.resetconfig() + identify_own() + instance.configuration, instance.setup, instance.order, instance.loaderror = { }, { }, { }, false +end + +function resolvers.loadnewconfig() + local luafiles = instance.luafiles + for i=1,#luafiles do + local cnf = luafiles[i] + local pathname = file.dirname(cnf) + local filename = file.join(pathname,resolvers.luaname) + local blob = loadfile(filename) + if blob then + local data = blob() + if data then + if trace_locating then + logs.report("fileio","loading configuration file '%s'",filename) + end + if true then + -- flatten to variable.progname + local t = { } + for k, v in next, data do -- v = progname + if type(v) == "string" then + t[k] = v + else + for kk, vv in next, v do -- vv = variable + if type(vv) == "string" then + t[vv.."."..v] = kk + end end end end + instance['setup'][pathname] = t + else + instance['setup'][pathname] = data end - instance[dataname][pathname] = t else - instance[dataname][pathname] = data + if trace_locating then + logs.report("fileio","skipping configuration file '%s'",filename) + end + instance['setup'][pathname] = { } + instance.loaderror = true end - else - input.report("skipping","configuration file",filename) - instance[dataname][pathname] = { } - instance.loaderror = true + elseif trace_locating then + logs.report("fileio","skipping configuration file '%s'",filename) end - else - input.report("skipping","configuration file",filename) - end -end - -function input.aux.load_configuration(instance,dname,lname) - input.aux.load_data(instance,dname,'configuration',lname and file.basename(lname)) -end -function input.aux.load_files(instance,tag) - input.aux.load_data(instance,tag,'files') -end - -function input.resetconfig(instance) - instance.configuration, instance.setup, instance.order, instance.loaderror = { }, { }, { }, false -end - -function input.loadnewconfig(instance) - for _, cnf in ipairs(instance.luafiles) do - local dname = file.dirname(cnf) - input.aux.load_texmfcnf(instance,'setup',dname) - instance.order[#instance.order+1] = instance.setup[dname] + instance.order[#instance.order+1] = instance.setup[pathname] if instance.loaderror then break end end end -function input.loadoldconfig(instance) +function resolvers.loadoldconfig() if not instance.renewcache then - for _, cnf in ipairs(instance.cnffiles) do + local cnffiles = instance.cnffiles + for i=1,#cnffiles do + local cnf = cnffiles[i] local dname = file.dirname(cnf) - input.aux.load_configuration(instance,dname) + resolvers.load_data(dname,'configuration') instance.order[#instance.order+1] = instance.configuration[dname] if instance.loaderror then break end end end - input.joinconfig(instance) + resolvers.joinconfig() end -function input.expand_variables(instance) - instance.expansions = { } ---~ instance.environment['SELFAUTOPARENT'] = instance.environment['SELFAUTOPARENT'] or instance.rootpath - if instance.engine ~= "" then instance.environment['engine'] = instance.engine end - if instance.progname ~= "" then instance.environment['progname'] = instance.progname end - for k,v in pairs(instance.environment) do - local a, b = k:match("^(%a+)%_(.*)%s*$") +function resolvers.expand_variables() + local expansions, environment, variables = { }, instance.environment, instance.variables + local env = resolvers.env + instance.expansions = expansions + if instance.engine ~= "" then environment['engine'] = instance.engine end + if instance.progname ~= "" then environment['progname'] = instance.progname end + for k,v in next, environment do + local a, b = match(k,"^(%a+)%_(.*)%s*$") if a and b then - instance.expansions[a..'.'..b] = v + expansions[a..'.'..b] = v else - instance.expansions[k] = v + expansions[k] = v end end - for k,v in pairs(instance.environment) do -- move environment to expansions - if not instance.expansions[k] then instance.expansions[k] = v end + for k,v in next, environment do -- move environment to expansions + if not expansions[k] then expansions[k] = v end + end + for k,v in next, variables do -- move variables to expansions + if not expansions[k] then expansions[k] = v end end - for k,v in pairs(instance.variables) do -- move variables to expansions - if not instance.expansions[k] then instance.expansions[k] = v end + local busy = false + local function resolve(a) + busy = true + return expansions[a] or env(a) end while true do - local busy = false - for k,v in pairs(instance.expansions) do - local s, n = v:gsub("%$([%a%d%_%-]+)", function(a) - busy = true - return instance.expansions[a] or input.env(instance,a) - end) - local s, m = s:gsub("%$%{([%a%d%_%-]+)%}", function(a) - busy = true - return instance.expansions[a] or input.env(instance,a) - end) + busy = false + for k,v in next, expansions do + local s, n = gsub(v,"%$([%a%d%_%-]+)",resolve) + local s, m = gsub(s,"%$%{([%a%d%_%-]+)%}",resolve) if n > 0 or m > 0 then - instance.expansions[k]= s + expansions[k]= s end end if not busy then break end end - for k,v in pairs(instance.expansions) do - instance.expansions[k] = v:gsub("\\", '/') + for k,v in next, expansions do + expansions[k] = gsub(v,"\\", '/') end end -function input.aux.expand_vars(instance,lst) -- simple vars - for k,v in pairs(lst) do - lst[k] = v:gsub("%$([%a%d%_%-]+)", function(a) - return instance.variables[a] or input.env(instance,a) - end) - end +function resolvers.variable(name) + return entry(instance.variables,name) end -function input.aux.expanded_var(instance,var) -- simple vars - return var:gsub("%$([%a%d%_%-]+)", function(a) - return instance.variables[a] or input.env(instance,a) - end) +function resolvers.expansion(name) + return entry(instance.expansions,name) end -function input.aux.entry(instance,entries,name) - if name and (name ~= "") then - name = name:gsub('%$','') - local result = entries[name..'.'..instance.progname] or entries[name] - if result then - return result - else - result = input.env(instance,name) - if result then - instance.variables[name] = result - input.expand_variables(instance) - return instance.expansions[name] or "" - end - end - end - return "" -end -function input.variable(instance,name) - return input.aux.entry(instance,instance.variables,name) -end -function input.expansion(instance,name) - return input.aux.entry(instance,instance.expansions,name) +function resolvers.is_variable(name) + return is_entry(instance.variables,name) end -function input.aux.is_entry(instance,entries,name) - if name and name ~= "" then - name = name:gsub('%$','') - return (entries[name..'.'..instance.progname] or entries[name]) ~= nil - else - return false - end +function resolvers.is_expansion(name) + return is_entry(instance.expansions,name) end -function input.is_variable(instance,name) - return input.aux.is_entry(instance,instance.variables,name) -end -function input.is_expansion(instance,name) - return input.aux.is_entry(instance,instance.expansions,name) +function resolvers.unexpanded_path_list(str) + local pth = resolvers.variable(str) + local lst = resolvers.split_path(pth) + return expanded_path_from_list(lst) end -function input.simplified_list(str) - if type(str) == 'table' then - return str -- troubles ; ipv , in texmf - elseif str == '' then - return { } - else - local t = { } - for _,v in ipairs(string.splitchr(str:gsub("^\{(.+)\}$","%1"),",")) do - t[#t+1] = (v:gsub("^[%!]*(.+)[%/\\]*$","%1")) - end - return t - end +function resolvers.unexpanded_path(str) + return file.join_path(resolvers.unexpanded_path_list(str)) end -function input.unexpanded_path_list(instance,str) - local pth = input.variable(instance,str) - local lst = input.split_path(pth) - return input.aux.expanded_path(instance,lst) -end -function input.unexpanded_path(instance,str) - return file.join_path(input.unexpanded_path_list(instance,str)) -end +do -- no longer needed -do local done = { } - function input.reset_extra_path(instance) + function resolvers.reset_extra_path() local ep = instance.extra_paths if not ep then ep, done = { }, { } @@ -3834,25 +6155,25 @@ do end end - function input.register_extra_path(instance,paths,subpaths) + function resolvers.register_extra_path(paths,subpaths) local ep = instance.extra_paths or { } local n = #ep if paths and paths ~= "" then if subpaths and subpaths ~= "" then - for p in paths:gmatch("[^,]+") do + for p in gmatch(paths,"[^,]+") do -- we gmatch each step again, not that fast, but used seldom - for s in subpaths:gmatch("[^,]+") do + for s in gmatch(subpaths,"[^,]+") do local ps = p .. "/" .. s if not done[ps] then - ep[#ep+1] = input.clean_path(ps) + ep[#ep+1] = resolvers.clean_path(ps) done[ps] = true end end end else - for p in paths:gmatch("[^,]+") do + for p in gmatch(paths,"[^,]+") do if not done[p] then - ep[#ep+1] = input.clean_path(p) + ep[#ep+1] = resolvers.clean_path(p) done[p] = true end end @@ -3860,10 +6181,10 @@ do elseif subpaths and subpaths ~= "" then for i=1,n do -- we gmatch each step again, not that fast, but used seldom - for s in subpaths:gmatch("[^,]+") do + for s in gmatch(subpaths,"[^,]+") do local ps = ep[i] .. "/" .. s if not done[ps] then - ep[#ep+1] = input.clean_path(ps) + ep[#ep+1] = resolvers.clean_path(ps) done[ps] = true end end @@ -3879,311 +6200,196 @@ do end -function input.expanded_path_list(instance,str) - local function made_list(list) - local ep = instance.extra_paths - if not ep or #ep == 0 then - return list - else - local done, new = { }, { } - -- honour . .. ../.. but only when at the start - for k, v in ipairs(list) do - if not done[v] then - if v:find("^[%.%/]$") then - done[v] = true - new[#new+1] = v - else - break - end - end - end - -- first the extra paths - for k, v in ipairs(ep) do - if not done[v] then +local function made_list(instance,list) + local ep = instance.extra_paths + if not ep or #ep == 0 then + return list + else + local done, new = { }, { } + -- honour . .. ../.. but only when at the start + for k=1,#list do + local v = list[k] + if not done[v] then + if find(v,"^[%.%/]$") then done[v] = true new[#new+1] = v + else + break end end - -- next the formal paths - for k, v in ipairs(list) do - if not done[v] then - done[v] = true - new[#new+1] = v - end + end + -- first the extra paths + for k=1,#ep do + local v = ep[k] + if not done[v] then + done[v] = true + new[#new+1] = v + end + end + -- next the formal paths + for k=1,#list do + local v = list[k] + if not done[v] then + done[v] = true + new[#new+1] = v end - return new end + return new end +end + +function resolvers.clean_path_list(str) + local t = resolvers.expanded_path_list(str) + if t then + for i=1,#t do + t[i] = file.collapse_path(resolvers.clean_path(t[i])) + end + end + return t +end + +function resolvers.expand_path(str) + return file.join_path(resolvers.expanded_path_list(str)) +end + +function resolvers.expanded_path_list(str) if not str then - return ep or { } + return ep or { } -- ep ? elseif instance.savelists then -- engine+progname hash - str = str:gsub("%$","") + str = gsub(str,"%$","") if not instance.lists[str] then -- cached - local lst = made_list(input.split_path(input.expansion(instance,str))) - instance.lists[str] = input.aux.expanded_path(instance,lst) + local lst = made_list(instance,resolvers.split_path(resolvers.expansion(str))) + instance.lists[str] = expanded_path_from_list(lst) end return instance.lists[str] else - local lst = input.split_path(input.expansion(instance,str)) - return made_list(input.aux.expanded_path(instance,lst)) + local lst = resolvers.split_path(resolvers.expansion(str)) + return made_list(instance,expanded_path_from_list(lst)) end end -function input.expand_path(instance,str) - return file.join_path(input.expanded_path_list(instance,str)) -end - ---~ function input.first_writable_path(instance,name) ---~ for _,v in pairs(input.expanded_path_list(instance,name)) do ---~ if file.is_writable(file.join(v,'luatex-cache.tmp')) then ---~ return v ---~ end ---~ end ---~ return "." ---~ end - -function input.expanded_path_list_from_var(instance,str) -- brrr - local tmp = input.var_of_format_or_suffix(str:gsub("%$","")) +function resolvers.expanded_path_list_from_var(str) -- brrr + local tmp = resolvers.var_of_format_or_suffix(gsub(str,"%$","")) if tmp ~= "" then - return input.expanded_path_list(instance,str) + return resolvers.expanded_path_list(tmp) else - return input.expanded_path_list(instance,tmp) + return resolvers.expanded_path_list(str) end end -function input.expand_path_from_var(instance,str) - return file.join_path(input.expanded_path_list_from_var(instance,str)) + +function resolvers.expand_path_from_var(str) + return file.join_path(resolvers.expanded_path_list_from_var(str)) end -function input.format_of_var(str) - return input.formats[str] or input.formats[input.alternatives[str]] or '' +function resolvers.format_of_var(str) + return formats[str] or formats[alternatives[str]] or '' end -function input.format_of_suffix(str) - return input.suffixmap[file.extname(str)] or 'tex' +function resolvers.format_of_suffix(str) + return suffixmap[file.extname(str)] or 'tex' end -function input.variable_of_format(str) - return input.formats[str] or input.formats[input.alternatives[str]] or '' +function resolvers.variable_of_format(str) + return formats[str] or formats[alternatives[str]] or '' end -function input.var_of_format_or_suffix(str) - local v = input.formats[str] +function resolvers.var_of_format_or_suffix(str) + local v = formats[str] if v then return v end - v = input.formats[input.alternatives[str]] + v = formats[alternatives[str]] if v then return v end - v = input.suffixmap[file.extname(str)] + v = suffixmap[file.extname(str)] if v then - return input.formats[isf] + return formats[isf] end return '' end -function input.expand_braces(instance,str) -- output variable and brace expansion of STRING - local ori = input.variable(instance,str) - local pth = input.aux.expanded_path(instance,input.split_path(ori)) +function resolvers.expand_braces(str) -- output variable and brace expansion of STRING + local ori = resolvers.variable(str) + local pth = expanded_path_from_list(resolvers.split_path(ori)) return file.join_path(pth) end --- {a,b,c,d} --- a,b,c/{p,q,r},d --- a,b,c/{p,q,r}/d/{x,y,z}// --- a,b,c/{p,q/{x,y,z},r},d/{p,q,r} --- a,b,c/{p,q/{x,y,z},r},d/{p,q,r} --- a{b,c}{d,e}f --- {a,b,c,d} --- {a,b,c/{p,q,r},d} --- {a,b,c/{p,q,r}/d/{x,y,z}//} --- {a,b,c/{p,q/{x,y,z}},d/{p,q,r}} --- {a,b,c/{p,q/{x,y,z},w}v,d/{p,q,r}} - --- this one is better and faster, but it took me a while to realize --- that this kind of replacement is cleaner than messy parsing and --- fuzzy concatenating we can probably gain a bit with selectively --- applying lpeg, but experiments with lpeg parsing this proved not to --- work that well; the parsing is ok, but dealing with the resulting --- table is a pain because we need to work inside-out recursively +resolvers.isreadable = { } --- get rid of piecewise here, just a gmatch is ok - -function input.aux.splitpathexpr(str, t, validate) - -- no need for optimization, only called a few times, we can use lpeg for the sub - t = t or { } - local concat = table.concat - while true do - local done = false - while true do - local ok = false - str = str:gsub("([^{},]+){([^{}]-)}", function(a,b) - local t = { } - b:piecewise(",", function(s) t[#t+1] = a .. s end) - ok, done = true, true - return "{" .. concat(t,",") .. "}" - end) - if not ok then break end - end - while true do - local ok = false - str = str:gsub("{([^{}]-)}([^{},]+)", function(a,b) - local t = { } - a:piecewise(",", function(s) t[#t+1] = s .. b end) - ok, done = true, true - return "{" .. concat(t,",") .. "}" - end) - if not ok then break end - end - while true do - local ok = false - str = str:gsub("([,{]){([^{}]+)}([,}])", function(a,b,c) - ok, done = true, true - return a .. b .. c - end) - if not ok then break end - end - if not done then break end - end - while true do - local ok = false - str = str:gsub("{([^{}]-)}{([^{}]-)}", function(a,b) - local t = { } - a:piecewise(",", function(sa) - b:piecewise(",", function(sb) - t[#t+1] = sa .. sb - end) - end) - ok = true - return "{" .. concat(t,",") .. "}" - end) - if not ok then break end - end - while true do - local ok = false - str = str:gsub("{([^{}]-)}", function(a) - ok = true - return a - end) - if not ok then break end - end - if validate then - str:piecewise(",", function(s) - s = validate(s) - if s then t[#t+1] = s end - end) - else - str:piecewise(",", function(s) - t[#t+1] = s - end) - end - return t -end - -function input.aux.expanded_path(instance,pathlist) -- maybe not a list, just a path - -- a previous version fed back into pathlist - local newlist, ok = { }, false - for _,v in ipairs(pathlist) do - if v:find("[{}]") then - ok = true - break - end - end - if ok then - for _, v in ipairs(pathlist) do - input.aux.splitpathexpr(v, newlist, function(s) - s = file.collapse_path(s) - return s ~= "" and not s:find(instance.dummy_path_expr) and s - end) - end - else - for _,v in ipairs(pathlist) do - for vv in string.gmatch(v..',',"(.-),") do - vv = file.collapse_path(v) - if vv ~= "" then newlist[#newlist+1] = vv end - end - end - end - return newlist -end - -input.is_readable = { } - -function input.aux.is_readable(readable, name) - if input.trace > 2 then +function resolvers.isreadable.file(name) + local readable = lfs.isfile(name) -- brrr + if trace_detail then if readable then - input.logger("+ readable", name) + logs.report("fileio","file '%s' is readable",name) else - input.logger("- readable", name) + logs.report("fileio","file '%s' is not readable", name) end end return readable end -function input.is_readable.file(name) - -- return input.aux.is_readable(file.is_readable(name), name) - return input.aux.is_readable(input.aux.is_file(name), name) -end - -input.is_readable.tex = input.is_readable.file +resolvers.isreadable.tex = resolvers.isreadable.file -- name -- name/name -function input.aux.collect_files(instance,names) +local function collect_files(names) local filelist = { } - for _, fname in pairs(names) do - if fname then - if input.trace > 2 then - input.logger("? blobpath asked",fname) - end - local bname = file.basename(fname) - local dname = file.dirname(fname) - if dname == "" or dname:find("^%.") then - dname = false - else - dname = "/" .. dname .. "$" - end - for _, hash in ipairs(instance.hashes) do - local blobpath = hash.tag - local files = blobpath and instance.files[blobpath] - if files then - if input.trace > 2 then - input.logger('? blobpath do',blobpath .. " (" .. bname ..")") + for k=1,#names do + local fname = names[k] + if trace_detail then + logs.report("fileio","checking name '%s'",fname) + end + local bname = file.basename(fname) + local dname = file.dirname(fname) + if dname == "" or find(dname,"^%.") then + dname = false + else + dname = "/" .. dname .. "$" + end + local hashes = instance.hashes + for h=1,#hashes do + local hash = hashes[h] + local blobpath = hash.tag + local files = blobpath and instance.files[blobpath] + if files then + if trace_detail then + logs.report("fileio","deep checking '%s' (%s)",blobpath,bname) + end + local blobfile = files[bname] + if not blobfile then + local rname = "remap:"..bname + blobfile = files[rname] + if blobfile then + bname = files[rname] + blobfile = files[bname] end - local blobfile = files[bname] - if not blobfile then - local rname = "remap:"..bname - blobfile = files[rname] - if blobfile then - bname = files[rname] - blobfile = files[bname] + end + if blobfile then + if type(blobfile) == 'string' then + if not dname or find(blobfile,dname) then + filelist[#filelist+1] = { + hash.type, + file.join(blobpath,blobfile,bname), -- search + resolvers.concatinators[hash.type](blobpath,blobfile,bname) -- result + } end - end - if blobfile then - if type(blobfile) == 'string' then - if not dname or blobfile:find(dname) then + else + for kk=1,#blobfile do + local vv = blobfile[kk] + if not dname or find(vv,dname) then filelist[#filelist+1] = { hash.type, - file.join(blobpath,blobfile,bname), -- search - input.concatinators[hash.type](blobpath,blobfile,bname) -- result + file.join(blobpath,vv,bname), -- search + resolvers.concatinators[hash.type](blobpath,vv,bname) -- result } end - else - for _, vv in pairs(blobfile) do - if not dname or vv:find(dname) then - filelist[#filelist+1] = { - hash.type, - file.join(blobpath,vv,bname), -- search - input.concatinators[hash.type](blobpath,vv,bname) -- result - } - end - end end end - elseif input.trace > 1 then - input.logger('! blobpath no',blobpath .. " (" .. bname ..")" ) end + elseif trace_locating then + logs.report("fileio","no match in '%s' (%s)",blobpath,bname) end end end @@ -4194,99 +6400,146 @@ function input.aux.collect_files(instance,names) end end -function input.suffix_of_format(str) - if input.suffixes[str] then - return input.suffixes[str][1] +function resolvers.suffix_of_format(str) + if suffixes[str] then + return suffixes[str][1] else return "" end end -function input.suffixes_of_format(str) - if input.suffixes[str] then - return input.suffixes[str] +function resolvers.suffixes_of_format(str) + if suffixes[str] then + return suffixes[str] else return {} end end -do - - -- called about 700 times for an empty doc (font initializations etc) - -- i need to weed the font files for redundant calls - - local letter = lpeg.R("az","AZ") - local separator = lpeg.P("://") - - local qualified = lpeg.P(".")^0 * lpeg.P("/") + letter*lpeg.P(":") + letter^1*separator - local rootbased = lpeg.P("/") + letter*lpeg.P(":") - - -- ./name ../name /name c: :// - function input.aux.qualified_path(filename) - return qualified:match(filename) - end - function input.aux.rootbased_path(filename) - return rootbased:match(filename) - end - - function input.normalize_name(original) - return original +function resolvers.register_in_trees(name) + if not find(name,"^%.") then + instance.foundintrees[name] = (instance.foundintrees[name] or 0) + 1 -- maybe only one end - - input.normalize_name = file.collapse_path - end -function input.aux.register_in_trees(instance,name) - if not name:find("^%.") then - instance.foundintrees[name] = (instance.foundintrees[name] or 0) + 1 -- maybe only one +-- split the next one up for readability (bu this module needs a cleanup anyway) + +local function can_be_dir(name) -- can become local + local fakepaths = instance.fakepaths + if not fakepaths[name] then + if lfs.isdir(name) then + fakepaths[name] = 1 -- directory + else + fakepaths[name] = 2 -- no directory + end end + return (fakepaths[name] == 1) end --- split the next one up, better for jit - -function input.aux.find_file(instance,filename) -- todo : plugin (scanners, checkers etc) - local result = { } +local function collect_instance_files(filename,collected) -- todo : plugin (scanners, checkers etc) + local result = collected or { } local stamp = nil - filename = input.normalize_name(filename) -- elsewhere - filename = file.collapse_path(filename:gsub("\\","/")) -- elsewhere + filename = file.collapse_path(filename) -- speed up / beware: format problem if instance.remember then stamp = filename .. "--" .. instance.engine .. "--" .. instance.progname .. "--" .. instance.format if instance.found[stamp] then - input.logger('! remembered', filename) + if trace_locating then + logs.report("fileio","remembering file '%s'",filename) + end return instance.found[stamp] end end - if filename:find('%*') then - input.logger('! wildcard', filename) - result = input.find_wildcard_files(instance,filename) - elseif input.aux.qualified_path(filename) then - if input.is_readable.file(filename) then - input.logger('! qualified', filename) + if not dangerous[instance.format or "?"] then + if resolvers.isreadable.file(filename) then + if trace_detail then + logs.report("fileio","file '%s' found directly",filename) + end + instance.found[stamp] = { filename } + return { filename } + end + end + if find(filename,'%*') then + if trace_locating then + logs.report("fileio","checking wildcard '%s'", filename) + end + result = resolvers.find_wildcard_files(filename) + elseif file.is_qualified_path(filename) then + if resolvers.isreadable.file(filename) then + if trace_locating then + logs.report("fileio","qualified name '%s'", filename) + end result = { filename } else - local forcedname, ok = "", false - if file.extname(filename) == "" then + local forcedname, ok, suffix = "", false, file.extname(filename) + if suffix == "" then -- why if instance.format == "" then forcedname = filename .. ".tex" - if input.is_readable.file(forcedname) then - input.logger('! no suffix, forcing standard filetype tex') + if resolvers.isreadable.file(forcedname) then + if trace_locating then + logs.report("fileio","no suffix, forcing standard filetype 'tex'") + end result, ok = { forcedname }, true end else - for _, s in pairs(input.suffixes_of_format(instance.format)) do + local suffixes = resolvers.suffixes_of_format(instance.format) + for _, s in next, suffixes do forcedname = filename .. "." .. s - if input.is_readable.file(forcedname) then - input.logger('! no suffix, forcing format filetype', s) + if resolvers.isreadable.file(forcedname) then + if trace_locating then + logs.report("fileio","no suffix, forcing format filetype '%s'", s) + end result, ok = { forcedname }, true break end end end end - if not ok then - input.logger('? qualified', filename) + if not ok and suffix ~= "" then + -- try to find in tree (no suffix manipulation), here we search for the + -- matching last part of the name + local basename = file.basename(filename) + local pattern = gsub(filename .. "$","([%.%-])","%%%1") + local savedformat = instance.format + local format = savedformat or "" + if format == "" then + instance.format = resolvers.format_of_suffix(suffix) + end + if not format then + instance.format = "othertextfiles" -- kind of everything, maybe texinput is better + end + -- + if basename ~= filename then + local resolved = collect_instance_files(basename) + if #result == 0 then + local lowered = lower(basename) + if filename ~= lowered then + resolved = collect_instance_files(lowered) + end + end + resolvers.format = savedformat + -- + for r=1,#resolved do + local rr = resolved[r] + if find(rr,pattern) then + result[#result+1], ok = rr, true + end + end + end + -- a real wildcard: + -- + -- if not ok then + -- local filelist = collect_files({basename}) + -- for f=1,#filelist do + -- local ff = filelist[f][3] or "" + -- if find(ff,pattern) then + -- result[#result+1], ok = ff, true + -- end + -- end + -- end + end + if not ok and trace_locating then + logs.report("fileio","qualified name '%s'", filename) end end else @@ -4303,39 +6556,45 @@ function input.aux.find_file(instance,filename) -- todo : plugin (scanners, chec if ext == "" then local forcedname = filename .. '.tex' wantedfiles[#wantedfiles+1] = forcedname - filetype = input.format_of_suffix(forcedname) - input.logger('! forcing filetype',filetype) + filetype = resolvers.format_of_suffix(forcedname) + if trace_locating then + logs.report("fileio","forcing filetype '%s'",filetype) + end else - filetype = input.format_of_suffix(filename) - input.logger('! using suffix based filetype',filetype) + filetype = resolvers.format_of_suffix(filename) + if trace_locating then + logs.report("fileio","using suffix based filetype '%s'",filetype) + end end else if ext == "" then - for _, s in pairs(input.suffixes_of_format(instance.format)) do + local suffixes = resolvers.suffixes_of_format(instance.format) + for _, s in next, suffixes do wantedfiles[#wantedfiles+1] = filename .. "." .. s end end filetype = instance.format - input.logger('! using given filetype',filetype) + if trace_locating then + logs.report("fileio","using given filetype '%s'",filetype) + end end - local typespec = input.variable_of_format(filetype) - local pathlist = input.expanded_path_list(instance,typespec) + local typespec = resolvers.variable_of_format(filetype) + local pathlist = resolvers.expanded_path_list(typespec) if not pathlist or #pathlist == 0 then -- no pathlist, access check only / todo == wildcard - if input.trace > 2 then - input.logger('? filename',filename) - input.logger('? filetype',filetype or '?') - input.logger('? wanted files',table.concat(wantedfiles," | ")) + if trace_detail then + logs.report("fileio","checking filename '%s', filetype '%s', wanted files '%s'",filename, filetype or '?',concat(wantedfiles," | ")) end - for _, fname in pairs(wantedfiles) do - if fname and input.is_readable.file(fname) then + for k=1,#wantedfiles do + local fname = wantedfiles[k] + if fname and resolvers.isreadable.file(fname) then filename, done = fname, true result[#result+1] = file.join('.',fname) break end end -- this is actually 'other text files' or 'any' or 'whatever' - local filelist = input.aux.collect_files(instance,wantedfiles) + local filelist = collect_files(wantedfiles) local fl = filelist and filelist[1] if fl then filename = fl[3] @@ -4344,56 +6603,76 @@ function input.aux.find_file(instance,filename) -- todo : plugin (scanners, chec end else -- list search - local filelist = input.aux.collect_files(instance,wantedfiles) - local doscan, recurse - if input.trace > 2 then - input.logger('? filename',filename) - -- if pathlist then input.logger('? path list',table.concat(pathlist," | ")) end - -- if filelist then input.logger('? file list',table.concat(filelist," | ")) end + local filelist = collect_files(wantedfiles) + local dirlist = { } + if filelist then + for i=1,#filelist do + dirlist[i] = file.dirname(filelist[i][2]) .. "/" + end + end + if trace_detail then + logs.report("fileio","checking filename '%s'",filename) end -- a bit messy ... esp the doscan setting here - for _, path in pairs(pathlist) do - if path:find("^!!") then doscan = false else doscan = true end - if path:find("//$") then recurse = true else recurse = false end - local pathname = path:gsub("^!+", '') + local doscan + for k=1,#pathlist do + local path = pathlist[k] + if find(path,"^!!") then doscan = false else doscan = true end + local pathname = gsub(path,"^!+", '') done = false -- using file list - if filelist and not (done and not instance.allresults) and recurse then - -- compare list entries with permitted pattern - pathname = pathname:gsub("([%-%.])","%%%1") -- this also influences - pathname = pathname:gsub("/+$", '/.*') -- later usage of pathname - pathname = pathname:gsub("//", '/.-/') -- not ok for /// but harmless - local expr = "^" .. pathname - -- input.debug('?',expr) - for _, fl in ipairs(filelist) do + if filelist then + local expression + -- compare list entries with permitted pattern -- /xx /xx// + if not find(pathname,"/$") then + expression = pathname .. "/" + else + expression = pathname + end + expression = gsub(expression,"([%-%.])","%%%1") -- this also influences + expression = gsub(expression,"//+$", '/.*') -- later usage of pathname + expression = gsub(expression,"//", '/.-/') -- not ok for /// but harmless + expression = "^" .. expression .. "$" + if trace_detail then + logs.report("fileio","using pattern '%s' for path '%s'",expression,pathname) + end + for k=1,#filelist do + local fl = filelist[k] local f = fl[2] - if f:find(expr) then - -- input.debug('T',' '..f) - if input.trace > 2 then - input.logger('= found in hash',f) - end + local d = dirlist[k] + if find(d,expression) then --- todo, test for readable result[#result+1] = fl[3] - input.aux.register_in_trees(instance,f) -- for tracing used files + resolvers.register_in_trees(f) -- for tracing used files done = true - if not instance.allresults then break end - else - -- input.debug('F',' '..f) + if instance.allresults then + if trace_detail then + logs.report("fileio","match in hash for file '%s' on path '%s', continue scanning",f,d) + end + else + if trace_detail then + logs.report("fileio","match in hash for file '%s' on path '%s', quit scanning",f,d) + end + break + end + elseif trace_detail then + logs.report("fileio","no match in hash for file '%s' on path '%s'",f,d) end end end if not done and doscan then -- check if on disk / unchecked / does not work at all / also zips - if input.method_is_file(pathname) then -- ? - local pname = pathname:gsub("%.%*$",'') - if not pname:find("%*") then - local ppname = pname:gsub("/+$","") - if input.aux.can_be_dir(instance,ppname) then - for _, w in pairs(wantedfiles) do + if resolvers.splitmethod(pathname).scheme == 'file' then -- ? + local pname = gsub(pathname,"%.%*$",'') + if not find(pname,"%*") then + local ppname = gsub(pname,"/+$","") + if can_be_dir(ppname) then + for k=1,#wantedfiles do + local w = wantedfiles[k] local fname = file.join(ppname,w) - if input.is_readable.file(fname) then - if input.trace > 2 then - input.logger('= found by scanning',fname) + if resolvers.isreadable.file(fname) then + if trace_detail then + logs.report("fileio","found '%s' by scanning",fname) end result[#result+1] = fname done = true @@ -4413,8 +6692,8 @@ function input.aux.find_file(instance,filename) -- todo : plugin (scanners, chec end end end - for k,v in pairs(result) do - result[k] = file.collapse_path(v) + for k=1,#result do + result[k] = file.collapse_path(result[k]) end if instance.remember then instance.found[stamp] = result @@ -4422,48 +6701,12 @@ function input.aux.find_file(instance,filename) -- todo : plugin (scanners, chec return result end -input.aux._find_file_ = input.aux.find_file +if not resolvers.concatinators then resolvers.concatinators = { } end -function input.aux.find_file(instance,filename) -- maybe make a lowres cache too - local result = input.aux._find_file_(instance,filename) - if #result == 0 then - local lowered = filename:lower() - if filename ~= lowered then - return input.aux._find_file_(instance,lowered) - end - end - return result -end +resolvers.concatinators.tex = file.join +resolvers.concatinators.file = resolvers.concatinators.tex -if lfs and lfs.isfile then - input.aux.is_file = lfs.isfile -- to be done: use this -else - input.aux.is_file = file.is_readable -end - -if lfs and lfs.isdir then - function input.aux.can_be_dir(instance,name) - if not instance.fakepaths[name] then - if lfs.isdir(name) then - instance.fakepaths[name] = 1 -- directory - else - instance.fakepaths[name] = 2 -- no directory - end - end - return (instance.fakepaths[name] == 1) - end -else - function input.aux.can_be_dir() - return true - end -end - -if not input.concatinators then input.concatinators = { } end - -input.concatinators.tex = file.join -input.concatinators.file = input.concatinators.tex - -function input.find_files(instance,filename,filetype,mustexist) +function resolvers.find_files(filename,filetype,mustexist) if type(mustexist) == boolean then -- all set elseif type(filetype) == 'boolean' then @@ -4472,19 +6715,27 @@ function input.find_files(instance,filename,filetype,mustexist) filetype, mustexist = nil, false end instance.format = filetype or '' - local t = input.aux.find_file(instance,filename,true) + local result = collect_instance_files(filename) + if #result == 0 then + local lowered = lower(filename) + if filename ~= lowered then + return collect_instance_files(lowered) + end + end instance.format = '' - return t + return result end -function input.find_file(instance,filename,filetype,mustexist) - return (input.find_files(instance,filename,filetype,mustexist)[1] or "") +function resolvers.find_file(filename,filetype,mustexist) + return (resolvers.find_files(filename,filetype,mustexist)[1] or "") end -function input.find_given_files(instance,filename) +function resolvers.find_given_files(filename) local bname, result = file.basename(filename), { } - for k, hash in ipairs(instance.hashes) do - local files = instance.files[hash.tag] + local hashes = instance.hashes + for k=1,#hashes do + local hash = hashes[k] + local files = instance.files[hash.tag] or { } local blist = files[bname] if not blist then local rname = "remap:"..bname @@ -4496,11 +6747,12 @@ function input.find_given_files(instance,filename) end if blist then if type(blist) == 'string' then - result[#result+1] = input.concatinators[hash.type](hash.tag,blist,bname) or "" + result[#result+1] = resolvers.concatinators[hash.type](hash.tag,blist,bname) or "" if not instance.allresults then break end else - for kk,vv in pairs(blist) do - result[#result+1] = input.concatinators[hash.type](hash.tag,vv,bname) or "" + for kk=1,#blist do + local vv = blist[kk] + result[#result+1] = resolvers.concatinators[hash.type](hash.tag,vv,bname) or "" if not instance.allresults then break end end end @@ -4509,127 +6761,121 @@ function input.find_given_files(instance,filename) return result end -function input.find_given_file(instance,filename) - return (input.find_given_files(instance,filename)[1] or "") +function resolvers.find_given_file(filename) + return (resolvers.find_given_files(filename)[1] or "") end -function input.find_wildcard_files(instance,filename) -- todo: remap: +local function doit(path,blist,bname,tag,kind,result,allresults) + local done = false + if blist and kind then + if type(blist) == 'string' then + -- make function and share code + if find(lower(blist),path) then + result[#result+1] = resolvers.concatinators[kind](tag,blist,bname) or "" + done = true + end + else + for kk=1,#blist do + local vv = blist[kk] + if find(lower(vv),path) then + result[#result+1] = resolvers.concatinators[kind](tag,vv,bname) or "" + done = true + if not allresults then break end + end + end + end + end + return done +end + +function resolvers.find_wildcard_files(filename) -- todo: remap: local result = { } local bname, dname = file.basename(filename), file.dirname(filename) - local path = dname:gsub("^*/","") - path = path:gsub("*",".*") - path = path:gsub("-","%%-") + local path = gsub(dname,"^*/","") + path = gsub(path,"*",".*") + path = gsub(path,"-","%%-") if dname == "" then path = ".*" end local name = bname - name = name:gsub("*",".*") - name = name:gsub("-","%%-") - path = path:lower() - name = name:lower() - local function doit(blist,bname,hash,allresults) - local done = false - if blist then - if type(blist) == 'string' then - -- make function and share code - if (blist:lower()):find(path) then - result[#result+1] = input.concatinators[hash.type](hash.tag,blist,bname) or "" - done = true - end - else - for kk,vv in pairs(blist) do - if (vv:lower()):find(path) then - result[#result+1] = input.concatinators[hash.type](hash.tag,vv,bname) or "" - done = true - if not allresults then break end - end - end - end - end - return done - end + name = gsub(name,"*",".*") + name = gsub(name,"-","%%-") + path = lower(path) + name = lower(name) local files, allresults, done = instance.files, instance.allresults, false - if name:find("%*") then - for k, hash in ipairs(instance.hashes) do - for kk, hh in pairs(files[hash.tag]) do - if not kk:find("^remap:") then - if (kk:lower()):find(name) then - if doit(hh,kk,hash,allresults) then done = true end + if find(name,"%*") then + local hashes = instance.hashes + for k=1,#hashes do + local hash = hashes[k] + local tag, kind = hash.tag, hash.type + for kk, hh in next, files[hash.tag] do + if not find(kk,"^remap:") then + if find(lower(kk),name) then + if doit(path,hh,kk,tag,kind,result,allresults) then done = true end if done and not allresults then break end end end end end else - for k, hash in ipairs(instance.hashes) do - if doit(files[hash.tag][bname],bname,hash,allresults) then done = true end + local hashes = instance.hashes + for k=1,#hashes do + local hash = hashes[k] + local tag, kind = hash.tag, hash.type + if doit(path,files[tag][bname],bname,tag,kind,result,allresults) then done = true end if done and not allresults then break end end end + -- we can consider also searching the paths not in the database, but then + -- we end up with a messy search (all // in all path specs) return result end -function input.find_wildcard_file(instance,filename) - return (input.find_wildcard_files(instance,filename)[1] or "") +function resolvers.find_wildcard_file(filename) + return (resolvers.find_wildcard_files(filename)[1] or "") end -- main user functions -function input.save_used_files_in_trees(instance, filename,jobname) - if not filename then filename = 'luatex.jlg' end - local f = io.open(filename,'w') - if f then - f:write("<?xml version='1.0' standalone='yes'?>\n") - f:write("<rl:job>\n") - if jobname then - f:write("\t<rl:name>" .. jobname .. "</rl:name>\n") - end - f:write("\t<rl:files>\n") - for _,v in pairs(table.sortedkeys(instance.foundintrees)) do - f:write("\t\t<rl:file n='" .. instance.foundintrees[v] .. "'>" .. v .. "</rl:file>\n") - end - f:write("\t</rl:files>\n") - f:write("</rl:usedfiles>\n") - f:close() - end -end - -function input.automount(instance) +function resolvers.automount() -- implemented later end -function input.load(instance) - input.starttiming(instance) - input.resetconfig(instance) - input.identify_cnf(instance) - input.load_lua(instance) - input.expand_variables(instance) - input.load_cnf(instance) - input.expand_variables(instance) - input.load_hash(instance) - input.automount(instance) - input.stoptiming(instance) +function resolvers.load(option) + statistics.starttiming(instance) + resolvers.resetconfig() + resolvers.identify_cnf() + resolvers.load_lua() -- will become the new method + resolvers.expand_variables() + resolvers.load_cnf() -- will be skipped when we have a lua file + resolvers.expand_variables() + if option ~= "nofiles" then + resolvers.load_hash() + resolvers.automount() + end + statistics.stoptiming(instance) end -function input.for_files(instance, command, files, filetype, mustexist) +function resolvers.for_files(command, files, filetype, mustexist) if files and #files > 0 then local function report(str) - if input.verbose then - input.report(str) -- has already verbose + if trace_locating then + logs.report("fileio",str) -- has already verbose else print(str) end end - if input.verbose then - report('') + if trace_locating then + report('') -- ? end - for _, file in pairs(files) do - local result = command(instance,file,filetype,mustexist) + for f=1,#files do + local file = files[f] + local result = command(file,filetype,mustexist) if type(result) == 'string' then report(result) else - for _,v in pairs(result) do - report(v) + for i=1,#result do + report(result[i]) -- could be unpack end end end @@ -4638,22 +6884,19 @@ end -- strtab -function input.var_value(instance,str) -- output the value of variable $STRING. - return input.variable(instance,str) -end -function input.expand_var(instance,str) -- output variable expansion of STRING. - return input.expansion(instance,str) -end -function input.show_path(instance,str) -- output search path for file type NAME - return file.join_path(input.expanded_path_list(instance,input.format_of_var(str))) +resolvers.var_value = resolvers.variable -- output the value of variable $STRING. +resolvers.expand_var = resolvers.expansion -- output variable expansion of STRING. + +function resolvers.show_path(str) -- output search path for file type NAME + return file.join_path(resolvers.expanded_path_list(resolvers.format_of_var(str))) end --- input.find_file(filename) --- input.find_file(filename, filetype, mustexist) --- input.find_file(filename, mustexist) --- input.find_file(filename, filetype) +-- resolvers.find_file(filename) +-- resolvers.find_file(filename, filetype, mustexist) +-- resolvers.find_file(filename, mustexist) +-- resolvers.find_file(filename, filetype) -function input.aux.register_file(files, name, path) +function resolvers.register_file(files, name, path) if files[name] then if type(files[name]) == 'string' then files[name] = { files[name], path } @@ -4665,165 +6908,81 @@ function input.aux.register_file(files, name, path) end end -if not input.finders then input.finders = { } end -if not input.openers then input.openers = { } end -if not input.loaders then input.loaders = { } end - -input.finders.notfound = { nil } -input.openers.notfound = { nil } -input.loaders.notfound = { false, nil, 0 } - -function input.splitmethod(filename) +function resolvers.splitmethod(filename) if not filename then return { } -- safeguard elseif type(filename) == "table" then return filename -- already split - elseif not filename:find("://") then + elseif not find(filename,"://") then return { scheme="file", path = filename, original=filename } -- quick hack else return url.hashed(filename) end end -function input.method_is_file(filename) - return input.splitmethod(filename).scheme == 'file' -end - function table.sequenced(t,sep) -- temp here local s = { } - for k, v in pairs(t) do - s[#s+1] = k .. "=" .. v + for k, v in next, t do -- indexed? + s[#s+1] = k .. "=" .. tostring(v) end - return table.concat(s, sep or " | ") + return concat(s, sep or " | ") end -function input.methodhandler(what, instance, filename, filetype) -- ... - local specification = (type(filename) == "string" and input.splitmethod(filename)) or filename -- no or { }, let it bomb +function resolvers.methodhandler(what, filename, filetype) -- ... + filename = file.collapse_path(filename) + local specification = (type(filename) == "string" and resolvers.splitmethod(filename)) or filename -- no or { }, let it bomb local scheme = specification.scheme - if input[what][scheme] then - input.logger('= handler',specification.original .." -> " .. what .. " -> " .. table.sequenced(specification)) - return input[what][scheme](instance,filename,filetype) -- todo: specification - else - return input[what].tex(instance,filename,filetype) -- todo: specification - end -end - --- also inside next test? - -function input.findtexfile(instance, filename, filetype) - return input.methodhandler('finders',instance, input.normalize_name(filename), filetype) -end -function input.opentexfile(instance,filename) - return input.methodhandler('openers',instance, input.normalize_name(filename)) -end - -function input.findbinfile(instance, filename, filetype) - return input.methodhandler('finders',instance, input.normalize_name(filename), filetype) -end -function input.openbinfile(instance,filename) - return input.methodhandler('loaders',instance, input.normalize_name(filename)) -end - -function input.loadbinfile(instance, filename, filetype) - local fname = input.findbinfile(instance, input.normalize_name(filename), filetype) - if fname and fname ~= "" then - return input.openbinfile(instance,fname) - else - return unpack(input.loaders.notfound) - end -end - -function input.texdatablob(instance, filename, filetype) - local ok, data, size = input.loadbinfile(instance, filename, filetype) - return data or "" -end - -input.loadtexfile = input.texdatablob - -function input.openfile(filename) -- brrr texmf.instance here / todo ! ! ! ! ! - local fullname = input.findtexfile(texmf.instance, filename) - if fullname and (fullname ~= "") then - return input.opentexfile(texmf.instance, fullname) + if resolvers[what][scheme] then + if trace_locating then + logs.report("fileio","handler '%s' -> '%s' -> '%s'",specification.original,what,table.sequenced(specification)) + end + return resolvers[what][scheme](filename,filetype) -- todo: specification else - return nil + return resolvers[what].tex(filename,filetype) -- todo: specification end end -function input.logmode() - return (os.getenv("MTX.LOG.MODE") or os.getenv("MTX_LOG_MODE") or "tex"):lower() -end - --- this is a prelude to engine/progname specific configuration files --- in which case we can omit files meant for other programs and --- packages - ---- ctx - --- maybe texinputs + font paths --- maybe positive selection tex/context fonts/tfm|afm|vf|opentype|type1|map|enc - -input.validators = { } -input.validators.visibility = { } - -function input.validators.visibility.default(path, name) - return true -end - -function input.validators.visibility.context(path, name) - path = path[1] or path -- some day a loop - return not ( - path:find("latex") or --- path:find("doc") or - path:find("tex4ht") or - path:find("source") or --- path:find("config") or --- path:find("metafont") or - path:find("lists$") or - name:find("%.tpm$") or - name:find("%.bak$") - ) -end - --- todo: describe which functions are public (maybe input.private. ... ) - --- beware: i need to check where we still need a / on windows: - -function input.clean_path(str) ---~ return (((str:gsub("\\","/")):gsub("^!+","")):gsub("//+","//")) +function resolvers.clean_path(str) if str then - return ((str:gsub("\\","/")):gsub("^!+","")) + str = gsub(str,"\\","/") + str = gsub(str,"^!+","") + str = gsub(str,"^~",resolvers.homedir) + return str else return nil end end -function input.do_with_path(name,func) - for _, v in pairs(input.expanded_path_list(instance,name)) do - func("^"..input.clean_path(v)) +function resolvers.do_with_path(name,func) + local pathlist = resolvers.expanded_path_list(name) + for i=1,#pathlist do + func("^"..resolvers.clean_path(pathlist[i])) end end -function input.do_with_var(name,func) - func(input.aux.expanded_var(name)) +function resolvers.do_with_var(name,func) + func(expanded_var(name)) end -function input.with_files(instance,pattern,handle) - for _, hash in ipairs(instance.hashes) do +function resolvers.with_files(pattern,handle) + local hashes = instance.hashes + for i=1,#hashes do + local hash = hashes[i] local blobpath = hash.tag local blobtype = hash.type if blobpath then local files = instance.files[blobpath] if files then - for k,v in pairs(files) do - if k:find("^remap:") then + for k,v in next, files do + if find(k,"^remap:") then k = files[k] v = files[k] -- chained end - if k:find(pattern) then + if find(k,pattern) then if type(v) == "string" then handle(blobtype,blobpath,v,k) else - for _,vv in pairs(v) do + for _,vv in next, v do -- indexed handle(blobtype,blobpath,vv,k) end end @@ -4834,131 +6993,56 @@ function input.with_files(instance,pattern,handle) end end ---~ function input.update_script(oldname,newname) -- oldname -> own.name, not per se a suffix ---~ newname = file.addsuffix(newname,"lua") ---~ local newscript = input.clean_path(input.find_file(instance, newname)) ---~ local oldscript = input.clean_path(oldname) ---~ input.report("old script", oldscript) ---~ input.report("new script", newscript) ---~ if oldscript ~= newscript and (oldscript:find(file.removesuffix(newname).."$") or oldscript:find(newname.."$")) then ---~ local newdata = io.loaddata(newscript) ---~ if newdata then ---~ input.report("old script content replaced by new content") ---~ io.savedata(oldscript,newdata) ---~ end ---~ end ---~ end - -function input.update_script(instance,oldname,newname) -- oldname -> own.name, not per se a suffix - local scriptpath = "scripts/context/lua" - newname = file.addsuffix(newname,"lua") - local oldscript = input.clean_path(oldname) - input.report("to be replaced old script", oldscript) - local newscripts = input.find_files(instance, newname) or { } - if #newscripts == 0 then - input.report("unable to locate new script") - else - for _, newscript in ipairs(newscripts) do - newscript = input.clean_path(newscript) - input.report("checking new script", newscript) - if oldscript == newscript then - input.report("old and new script are the same") - elseif not newscript:find(scriptpath) then - input.report("new script should come from",scriptpath) - elseif not (oldscript:find(file.removesuffix(newname).."$") or oldscript:find(newname.."$")) then - input.report("invalid new script name") - else - local newdata = io.loaddata(newscript) - if newdata then - input.report("old script content replaced by new content") - io.savedata(oldscript,newdata) - break - else - input.report("unable to load new script") - end - end +function resolvers.locate_format(name) + local barename, fmtname = gsub(name,"%.%a+$",""), "" + if resolvers.usecache then + local path = file.join(caches.setpath("formats")) -- maybe platform + fmtname = file.join(path,barename..".fmt") or "" + end + if fmtname == "" then + fmtname = resolvers.find_files(barename..".fmt")[1] or "" + end + fmtname = resolvers.clean_path(fmtname) + if fmtname ~= "" then + local barename = file.removesuffix(fmtname) + local luaname, lucname, luiname = barename .. ".lua", barename .. ".luc", barename .. ".lui" + if lfs.isfile(luiname) then + return barename, luiname + elseif lfs.isfile(lucname) then + return barename, lucname + elseif lfs.isfile(luaname) then + return barename, luaname end end + return nil, nil end +function resolvers.boolean_variable(str,default) + local b = resolvers.expansion(str) + if b == "" then + return default + else + b = toboolean(b) + return (b == nil and default) or b + end +end ---~ print(table.serialize(input.aux.splitpathexpr("/usr/share/texmf-{texlive,tetex}", {}))) - --- command line resolver: - ---~ print(input.resolve("abc env:tmp file:cont-en.tex path:cont-en.tex full:cont-en.tex rel:zapf/one/p-chars.tex")) - -do +texconfig.kpse_init = false - local resolvers = { } +kpse = { original = kpse } setmetatable(kpse, { __index = function(k,v) return resolvers[v] end } ) - resolvers.environment = function(instance,str) - return input.clean_path(os.getenv(str) or os.getenv(str:upper()) or os.getenv(str:lower()) or "") - end - resolvers.relative = function(instance,str,n) - if io.exists(str) then - -- nothing - elseif io.exists("./" .. str) then - str = "./" .. str - else - local p = "../" - for i=1,n or 2 do - if io.exists(p .. str) then - str = p .. str - break - else - p = p .. "../" - end - end - end - return input.clean_path(str) - end - resolvers.locate = function(instance,str) - local fullname = input.find_given_file(instance,str) or "" - return input.clean_path((fullname ~= "" and fullname) or str) - end - resolvers.filename = function(instance,str) - local fullname = input.find_given_file(instance,str) or "" - return input.clean_path(file.basename((fullname ~= "" and fullname) or str)) - end - resolvers.pathname = function(instance,str) - local fullname = input.find_given_file(instance,str) or "" - return input.clean_path(file.dirname((fullname ~= "" and fullname) or str)) - end +-- for a while - resolvers.env = resolvers.environment - resolvers.rel = resolvers.relative - resolvers.loc = resolvers.locate - resolvers.kpse = resolvers.locate - resolvers.full = resolvers.locate - resolvers.file = resolvers.filename - resolvers.path = resolvers.pathname - - local function resolve(instance,str) - if type(str) == "table" then - for k, v in pairs(str) do - str[k] = resolve(instance,v) or v - end - elseif str and str ~= "" then - str = str:gsub("([a-z]+):([^ ]+)", function(method,target) - if resolvers[method] then - return resolvers[method](instance,target) - else - return method .. ":" .. target - end - end) - end - return str - end +input = resolvers - input.resolve = resolve -end +end -- of closure +do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['luat-tmp'] = { +if not modules then modules = { } end modules ['data-tmp'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -4980,63 +7064,71 @@ being written at the same time is small. We also need to extend luatools with a recache feature.</p> --ldx]]-- +local format, lower, gsub = string.format, string.lower, string.gsub + +local trace_cache = false trackers.register("resolvers.cache", function(v) trace_cache = v end) -- not used yet + caches = caches or { } -dir = dir or { } -texmf = texmf or { } - -caches.path = caches.path or nil -caches.base = caches.base or "luatex-cache" -caches.more = caches.more or "context" -caches.direct = false -- true is faster but may need huge amounts of memory -caches.trace = false -caches.tree = false -caches.paths = caches.paths or nil -caches.force = false - -input.usecache = not toboolean(os.getenv("TEXMFSHARECACHE") or "false",true) -- true - -function caches.temp(instance) - local function checkpath(cachepath) - if not cachepath or cachepath == "" then - return nil - elseif lfs.attributes(cachepath,"mode") == "directory" then -- lfs.isdir(cachepath) then - return cachepath - elseif caches.force or io.ask(string.format("Should I create the cache path %s?",cachepath), "no", { "yes", "no" }) == "yes" then - dir.mkdirs(cachepath) - return (lfs.attributes(cachepath,"mode") == "directory") and cachepath - else - return nil + +caches.path = caches.path or nil +caches.base = caches.base or "luatex-cache" +caches.more = caches.more or "context" +caches.direct = false -- true is faster but may need huge amounts of memory +caches.tree = false +caches.paths = caches.paths or nil +caches.force = false +caches.defaults = { "TEXMFCACHE", "TMPDIR", "TEMPDIR", "TMP", "TEMP", "HOME", "HOMEPATH" } + +function caches.temp() + local cachepath = nil + local function check(list,isenv) + if not cachepath then + for k=1,#list do + local v = list[k] + cachepath = (isenv and (os.env[v] or "")) or v or "" + if cachepath == "" then + -- next + else + cachepath = resolvers.clean_path(cachepath) + if lfs.isdir(cachepath) and file.iswritable(cachepath) then -- lfs.attributes(cachepath,"mode") == "directory" + break + elseif caches.force or io.ask(format("\nShould I create the cache path %s?",cachepath), "no", { "yes", "no" }) == "yes" then + dir.mkdirs(cachepath) + if lfs.isdir(cachepath) and file.iswritable(cachepath) then + break + end + end + end + cachepath = nil + end end end - local cachepath = input.expanded_path_list(instance,"TEXMFCACHE") - cachepath = cachepath and #cachepath > 0 and checkpath(cachepath[1]) - if not cachepath then - cachepath = os.getenv("TEXMFCACHE") or os.getenv("HOME") or os.getenv("HOMEPATH") or os.getenv("TMP") or os.getenv("TEMP") or os.getenv("TMPDIR") or nil - cachepath = checkpath(cachepath) - end + check(resolvers.clean_path_list("TEXMFCACHE") or { }) + check(caches.defaults,true) if not cachepath then - print("\nfatal error: there is no valid cache path defined\n") + print("\nfatal error: there is no valid (writable) cache path defined\n") os.exit() - elseif lfs.attributes(cachepath,"mode") ~= "directory" then - print(string.format("\nfatal error: cache path %s is not a directory\n",cachepath)) + elseif not lfs.isdir(cachepath) then -- lfs.attributes(cachepath,"mode") ~= "directory" + print(format("\nfatal error: cache path %s is not a directory\n",cachepath)) os.exit() end - function caches.temp(instance) + cachepath = file.collapse_path(cachepath) + function caches.temp() return cachepath end return cachepath end -function caches.configpath(instance) - return table.concat(instance.cnffiles,";") +function caches.configpath() + return table.concat(resolvers.instance.cnffiles,";") end function caches.hashed(tree) - return md5.hex((tree:lower()):gsub("[\\\/]+","/")) + return md5.hex(gsub(lower(tree),"[\\\/]+","/")) end -function caches.treehash(instance) - local tree = caches.configpath(instance) +function caches.treehash() + local tree = caches.configpath() if not tree or tree == "" then return false else @@ -5044,26 +7136,25 @@ function caches.treehash(instance) end end -function caches.setpath(instance,...) +function caches.setpath(...) if not caches.path then if not caches.path then - caches.path = caches.temp(instance) + caches.path = caches.temp() end - caches.path = input.clean_path(caches.path) -- to be sure - if lfs then - caches.tree = caches.tree or caches.treehash(instance) - if caches.tree then - caches.path = dir.mkdirs(caches.path,caches.base,caches.more,caches.tree) - else - caches.path = dir.mkdirs(caches.path,caches.base,caches.more) - end + caches.path = resolvers.clean_path(caches.path) -- to be sure + caches.tree = caches.tree or caches.treehash() + if caches.tree then + caches.path = dir.mkdirs(caches.path,caches.base,caches.more,caches.tree) + else + caches.path = dir.mkdirs(caches.path,caches.base,caches.more) end end if not caches.path then caches.path = '.' end - caches.path = input.clean_path(caches.path) - if lfs and not table.is_empty({...}) then + caches.path = resolvers.clean_path(caches.path) + local dirs = { ... } + if #dirs > 0 then local pth = dir.mkdirs(caches.path,...) return pth end @@ -5071,9 +7162,9 @@ function caches.setpath(instance,...) return caches.path end -function caches.definepath(instance,category,subcategory) +function caches.definepath(category,subcategory) return function() - return caches.setpath(instance,category,subcategory) + return caches.setpath(category,subcategory) end end @@ -5085,39 +7176,105 @@ function caches.loaddata(path,name) local tmaname, tmcname = caches.setluanames(path,name) local loader = loadfile(tmcname) or loadfile(tmaname) if loader then - return loader() + loader = loader() + collectgarbage("step") + return loader else return false end end -function caches.is_writable(filepath,filename) +--~ function caches.loaddata(path,name) +--~ local tmaname, tmcname = caches.setluanames(path,name) +--~ return dofile(tmcname) or dofile(tmaname) +--~ end + +function caches.iswritable(filepath,filename) local tmaname, tmcname = caches.setluanames(filepath,filename) - return file.is_writable(tmaname) + return file.iswritable(tmaname) end -function caches.savedata(filepath,filename,data,raw) -- raw needed for file cache +function caches.savedata(filepath,filename,data,raw) local tmaname, tmcname = caches.setluanames(filepath,filename) local reduce, simplify = true, true if raw then reduce, simplify = false, false end + data.cache_uuid = os.uuid() if caches.direct then - file.savedata(tmaname, table.serialize(data,'return',true,true)) + file.savedata(tmaname, table.serialize(data,'return',false,true,false)) -- no hex else - table.tofile (tmaname, data,'return',true,true) -- maybe not the last true + table.tofile(tmaname, data,'return',false,true,false) -- maybe not the last true end - utils.lua.compile(tmaname, tmcname) + local cleanup = resolvers.boolean_variable("PURGECACHE", false) + local strip = resolvers.boolean_variable("LUACSTRIP", true) + utils.lua.compile(tmaname, tmcname, cleanup, strip) end -- here we use the cache for format loading (texconfig.[formatname|jobname]) --~ if tex and texconfig and texconfig.formatname and texconfig.formatname == "" then -if tex and texconfig and (not texconfig.formatname or texconfig.formatname == "") and texmf.instance then +if tex and texconfig and (not texconfig.formatname or texconfig.formatname == "") and input and resolvers.instance then if not texconfig.luaname then texconfig.luaname = "cont-en.lua" end -- or luc - texconfig.formatname = caches.setpath(texmf.instance,"formats") .. "/" .. texconfig.luaname:gsub("%.lu.$",".fmt") + texconfig.formatname = caches.setpath("formats") .. "/" .. gsub(texconfig.luaname,"%.lu.$",".fmt") end + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['data-inp'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +resolvers.finders = resolvers.finders or { } +resolvers.openers = resolvers.openers or { } +resolvers.loaders = resolvers.loaders or { } + +resolvers.finders.notfound = { nil } +resolvers.openers.notfound = { nil } +resolvers.loaders.notfound = { false, nil, 0 } + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['data-out'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +outputs = outputs or { } + + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['data-con'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local format, lower, gsub = string.format, string.lower, string.gsub + +local trace_cache = false trackers.register("resolvers.cache", function(v) trace_cache = v end) +local trace_containers = false trackers.register("resolvers.containers", function(v) trace_containers = v end) +local trace_storage = false trackers.register("resolvers.storage", function(v) trace_storage = v end) + --[[ldx-- <p>Once we found ourselves defining similar cache constructs several times, containers were introduced. Containers are used @@ -5131,124 +7288,144 @@ table structures without bothering about the disk cache.</p> <p>Examples of usage can be found in the font related code.</p> --ldx]]-- -containers = { } -containers.trace = false +containers = containers or { } -do -- local report +containers.usecache = true - local function report(container,tag,name) - if caches.trace or containers.trace or container.trace then - logs.report(string.format("%s cache",container.subcategory),string.format("%s: %s",tag,name or 'invalid')) - end +local function report(container,tag,name) + if trace_cache or trace_containers then + logs.report(format("%s cache",container.subcategory),"%s: %s",tag,name or 'invalid') end +end - local allocated = { } +local allocated = { } - -- tracing +-- tracing - function containers.define(category, subcategory, version, enabled) - return function() - if category and subcategory then - local c = allocated[category] - if not c then - c = { } - allocated[category] = c - end - local s = c[subcategory] - if not s then - s = { - category = category, - subcategory = subcategory, - storage = { }, - enabled = enabled, - version = version or 1.000, - trace = false, - path = caches.setpath(texmf.instance,category,subcategory), - } - c[subcategory] = s - end - return s - else - return nil - end +function containers.define(category, subcategory, version, enabled) + return function() + if category and subcategory then + local c = allocated[category] + if not c then + c = { } + allocated[category] = c + end + local s = c[subcategory] + if not s then + s = { + category = category, + subcategory = subcategory, + storage = { }, + enabled = enabled, + version = version or 1.000, + trace = false, + path = caches and caches.setpath and caches.setpath(category,subcategory), + } + c[subcategory] = s + end + return s + else + return nil end end +end + +function containers.is_usable(container, name) + return container.enabled and caches and caches.iswritable(container.path, name) +end - function containers.is_usable(container, name) - return container.enabled and caches.is_writable(container.path, name) +function containers.is_valid(container, name) + if name and name ~= "" then + local storage = container.storage[name] + return storage and storage.cache_version == container.version + else + return false end +end - function containers.is_valid(container, name) - if name and name ~= "" then - local storage = container.storage[name] - return storage and not table.is_empty(storage) and storage.cache_version == container.version +function containers.read(container,name) + if container.enabled and caches and not container.storage[name] and containers.usecache then + container.storage[name] = caches.loaddata(container.path,name) + if containers.is_valid(container,name) then + report(container,"loaded",name) else - return false + container.storage[name] = nil end end - - function containers.read(container,name) - if container.enabled and not container.storage[name] then - container.storage[name] = caches.loaddata(container.path,name) - if containers.is_valid(container,name) then - report(container,"loaded",name) - else - container.storage[name] = nil - end - end - if container.storage[name] then - report(container,"reusing",name) - end - return container.storage[name] + if container.storage[name] then + report(container,"reusing",name) end + return container.storage[name] +end - function containers.write(container, name, data) - if data then - data.cache_version = container.version - if container.enabled then - local unique, shared = data.unique, data.shared - data.unique, data.shared = nil, nil - caches.savedata(container.path, name, data) - report(container,"saved",name) - data.unique, data.shared = unique, shared - end - report(container,"stored",name) - container.storage[name] = data +function containers.write(container, name, data) + if data then + data.cache_version = container.version + if container.enabled and caches then + local unique, shared = data.unique, data.shared + data.unique, data.shared = nil, nil + caches.savedata(container.path, name, data) + report(container,"saved",name) + data.unique, data.shared = unique, shared end - return data + report(container,"stored",name) + container.storage[name] = data end + return data +end - function containers.content(container,name) - return container.storage[name] - end +function containers.content(container,name) + return container.storage[name] +end +function containers.cleanname(name) + return (gsub(lower(name),"[^%w%d]+","-")) end + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['data-use'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local format, lower, gsub, find = string.format, string.lower, string.gsub, string.find + +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) + -- since we want to use the cache instead of the tree, we will now -- reimplement the saver. -local save_data = input.aux.save_data +local save_data = resolvers.save_data +local load_data = resolvers.load_data -input.cachepath = nil +resolvers.cachepath = nil -- public, for tracing +resolvers.usecache = true -- public, for tracing -function input.aux.save_data(instance, dataname, check) - input.cachepath = input.cachepath or caches.definepath(instance,"trees") - save_data(instance, dataname, check, function(cachename,dataname) - if input.usecache then - return file.join(input.cachepath(),caches.hashed(cachename)) +function resolvers.save_data(dataname) + save_data(dataname, function(cachename,dataname) + resolvers.usecache = not toboolean(resolvers.expansion("CACHEINTDS") or "false",true) + if resolvers.usecache then + resolvers.cachepath = resolvers.cachepath or caches.definepath("trees") + return file.join(resolvers.cachepath(),caches.hashed(cachename)) else return file.join(cachename,dataname) end end) end -local load_data = input.aux.load_data - -function input.aux.load_data(instance,pathname,dataname,filename) - input.cachepath = input.cachepath or caches.definepath(instance,"trees") - load_data(instance,pathname,dataname,filename,function(dataname,filename) - if input.usecache then - return file.join(input.cachepath(),caches.hashed(pathname)) +function resolvers.load_data(pathname,dataname,filename) + load_data(pathname,dataname,filename,function(dataname,filename) + resolvers.usecache = not toboolean(resolvers.expansion("CACHEINTDS") or "false",true) + if resolvers.usecache then + resolvers.cachepath = resolvers.cachepath or caches.definepath("trees") + return file.join(resolvers.cachepath(),caches.hashed(pathname)) else if not filename or (filename == "") then filename = dataname @@ -5260,955 +7437,327 @@ end -- we will make a better format, maybe something xml or just text or lua -input.automounted = input.automounted or { } +resolvers.automounted = resolvers.automounted or { } -function input.automount(instance,usecache) - local mountpaths = input.simplified_list(input.expansion(instance,'TEXMFMOUNT')) - if table.is_empty(mountpaths) and usecache then - mountpaths = { caches.setpath(instance,"mount") } +function resolvers.automount(usecache) + local mountpaths = resolvers.clean_path_list(resolvers.expansion('TEXMFMOUNT')) + if (not mountpaths or #mountpaths == 0) and usecache then + mountpaths = { caches.setpath("mount") } end - if not table.is_empty(mountpaths) then - input.starttiming(instance) - for k, root in pairs(mountpaths) do + if mountpaths and #mountpaths > 0 then + statistics.starttiming(resolvers.instance) + for k=1,#mountpaths do + local root = mountpaths[k] local f = io.open(root.."/url.tmi") if f then for line in f:lines() do if line then - if line:find("^[%%#%-]") then -- or %W + if find(line,"^[%%#%-]") then -- or %W -- skip - elseif line:find("^zip://") then - input.report("mounting",line) - table.insert(input.automounted,line) - input.usezipfile(instance,line) + elseif find(line,"^zip://") then + if trace_locating then + logs.report("fileio","mounting %s",line) + end + table.insert(resolvers.automounted,line) + resolvers.usezipfile(line) end end end f:close() end end - input.stoptiming(instance) + statistics.stoptiming(resolvers.instance) end end --- store info in format +-- status info -input.storage = { } -input.storage.data = { } -input.storage.min = 0 -- 500 -input.storage.max = input.storage.min - 1 -input.storage.trace = false -- true -input.storage.done = 0 -input.storage.evaluators = { } --- (evaluate,message,names) +statistics.register("used config path", function() return caches.configpath() end) +statistics.register("used cache path", function() return caches.temp() or "?" end) -function input.storage.register(...) - input.storage.data[#input.storage.data+1] = { ... } -end - -function input.storage.evaluate(name) - input.storage.evaluators[#input.storage.evaluators+1] = name -end +-- experiment (code will move) -function input.storage.finalize() -- we can prepend the string with "evaluate:" - for _, t in ipairs(input.storage.evaluators) do - for i, v in pairs(t) do - if type(v) == "string" then - t[i] = loadstring(v)() - elseif type(v) == "table" then - for _, vv in pairs(v) do - if type(vv) == "string" then - t[i] = loadstring(vv)() - end - end - end - end +function statistics.save_fmt_status(texname,formatbanner,sourcefile) -- texname == formatname + local enginebanner = status.list().banner + if formatbanner and enginebanner and sourcefile then + local luvname = file.replacesuffix(texname,"luv") + local luvdata = { + enginebanner = enginebanner, + formatbanner = formatbanner, + sourcehash = md5.hex(io.loaddata(resolvers.find_file(sourcefile)) or "unknown"), + sourcefile = sourcefile, + } + io.savedata(luvname,table.serialize(luvdata,true)) end end -function input.storage.dump() - for name, data in ipairs(input.storage.data) do - local evaluate, message, original, target = data[1], data[2], data[3] ,data[4] - local name, initialize, finalize, code = nil, "", "", "" - for str in target:gmatch("([^%.]+)") do - if name then - name = name .. "." .. str +function statistics.check_fmt_status(texname) + local enginebanner = status.list().banner + if enginebanner and texname then + local luvname = file.replacesuffix(texname,"luv") + if lfs.isfile(luvname) then + local luv = dofile(luvname) + if luv and luv.sourcefile then + local sourcehash = md5.hex(io.loaddata(resolvers.find_file(luv.sourcefile)) or "unknown") + local luvbanner = luv.enginebanner or "?" + if luvbanner ~= enginebanner then + return string.format("engine mismatch (luv:%s <> bin:%s)",luvbanner,enginebanner) + end + local luvhash = luv.sourcehash or "?" + if luvhash ~= sourcehash then + return string.format("source mismatch (luv:%s <> bin:%s)",luvhash,sourcehash) + end else - name = str + return "invalid status file" end - initialize = string.format("%s %s = %s or {} ", initialize, name, name) - end - if evaluate then - finalize = "input.storage.evaluate(" .. name .. ")" - end - input.storage.max = input.storage.max + 1 - if input.storage.trace then - logs.report('storage',string.format('saving %s in slot %s',message,input.storage.max)) - code = - initialize .. - string.format("logs.report('storage','restoring %s from slot %s') ",message,input.storage.max) .. - table.serialize(original,name) .. - finalize else - code = initialize .. table.serialize(original,name) .. finalize + return "missing status file" end - lua.bytecode[input.storage.max] = loadstring(code) - end -end - -if lua.bytecode then -- from 0 upwards - local i = input.storage.min - while lua.bytecode[i] do - lua.bytecode[i]() - lua.bytecode[i] = nil - i = i + 1 end - input.storage.done = i + return true end --- filename : luat-zip.lua --- comment : companion to luat-lib.tex --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files - -if not versions then versions = { } end versions['luat-zip'] = 1.001 +end -- of closure -if zip and input then - zip.supported = true -else - zip = { } - zip.supported = false -end - -if not zip.supported then +do -- create closure to overcome 200 locals limit - if not input then input = { } end -- will go away - - function zip.openarchive (...) return nil end -- needed ? - function zip.closenarchive (...) end -- needed ? - function input.usezipfile (...) end -- needed ? +if not modules then modules = { } end modules ['luat-kps'] = { + version = 1.001, + comment = "companion to luatools.lua", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} -else +--[[ldx-- +<p>This file is used when we want the input handlers to behave like +<type>kpsewhich</type>. What to do with the following:</p> - -- zip:///oeps.zip?name=bla/bla.tex - -- zip:///oeps.zip?tree=tex/texmf-local +<typing> +{$SELFAUTOLOC,$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,}/web2c} +$SELFAUTOLOC : /usr/tex/bin/platform +$SELFAUTODIR : /usr/tex/bin +$SELFAUTOPARENT : /usr/tex +</typing> - local function validzip(str) - if not str:find("^zip://") then - return "zip:///" .. str - else - return str - end - end +<p>How about just forgetting about them?</p> +--ldx]]-- - zip.archives = { } - zip.registeredfiles = { } +local suffixes = resolvers.suffixes +local formats = resolvers.formats + +suffixes['gf'] = { '<resolution>gf' } +suffixes['pk'] = { '<resolution>pk' } +suffixes['base'] = { 'base' } +suffixes['bib'] = { 'bib' } +suffixes['bst'] = { 'bst' } +suffixes['cnf'] = { 'cnf' } +suffixes['mem'] = { 'mem' } +suffixes['mf'] = { 'mf' } +suffixes['mfpool'] = { 'pool' } +suffixes['mft'] = { 'mft' } +suffixes['mppool'] = { 'pool' } +suffixes['graphic/figure'] = { 'eps', 'epsi' } +suffixes['texpool'] = { 'pool' } +suffixes['PostScript header'] = { 'pro' } +suffixes['ist'] = { 'ist' } +suffixes['web'] = { 'web', 'ch' } +suffixes['cweb'] = { 'w', 'web', 'ch' } +suffixes['cmap files'] = { 'cmap' } +suffixes['lig files'] = { 'lig' } +suffixes['bitmap font'] = { } +suffixes['MetaPost support'] = { } +suffixes['TeX system documentation'] = { } +suffixes['TeX system sources'] = { } +suffixes['dvips config'] = { } +suffixes['type42 fonts'] = { } +suffixes['web2c files'] = { } +suffixes['other text files'] = { } +suffixes['other binary files'] = { } +suffixes['opentype fonts'] = { 'otf' } + +suffixes['fmt'] = { 'fmt' } +suffixes['texmfscripts'] = { 'rb','lua','py','pl' } + +suffixes['pdftex config'] = { } +suffixes['Troff fonts'] = { } + +suffixes['ls-R'] = { } - function zip.openarchive(instance,name) - if not name or name == "" then - return nil - else - local arch = zip.archives[name] - if arch then - return arch - else - local full = input.find_file(instance,name) or "" - local arch = (full ~= "" and zip.open(full)) or false - zip.archives[name] = arch - return arch - end - end - end +--[[ldx-- +<p>If you wondered abou tsome of the previous mappings, how about +the next bunch:</p> +--ldx]]-- - function zip.closearchive(instance,name) - if not name or name == "" and zip.archives[name] then - zip.close(zip.archives[name]) - zip.archives[name] = nil - end - end +formats['bib'] = '' +formats['bst'] = '' +formats['mft'] = '' +formats['ist'] = '' +formats['web'] = '' +formats['cweb'] = '' +formats['MetaPost support'] = '' +formats['TeX system documentation'] = '' +formats['TeX system sources'] = '' +formats['Troff fonts'] = '' +formats['dvips config'] = '' +formats['graphic/figure'] = '' +formats['ls-R'] = '' +formats['other text files'] = '' +formats['other binary files'] = '' + +formats['gf'] = '' +formats['pk'] = '' +formats['base'] = 'MFBASES' +formats['cnf'] = '' +formats['mem'] = 'MPMEMS' +formats['mf'] = 'MFINPUTS' +formats['mfpool'] = 'MFPOOL' +formats['mppool'] = 'MPPOOL' +formats['texpool'] = 'TEXPOOL' +formats['PostScript header'] = 'TEXPSHEADERS' +formats['cmap files'] = 'CMAPFONTS' +formats['type42 fonts'] = 'T42FONTS' +formats['web2c files'] = 'WEB2C' +formats['pdftex config'] = 'PDFTEXCONFIG' +formats['texmfscripts'] = 'TEXMFSCRIPTS' +formats['bitmap font'] = '' +formats['lig files'] = 'LIGFONTS' + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['data-aux'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} - -- zip:///texmf.zip?tree=/tex/texmf - -- zip:///texmf.zip?tree=/tex/texmf-local - -- zip:///texmf-mine.zip?tree=/tex/texmf-projects +local find = string.find - function input.locators.zip(instance,specification) -- where is this used? startup zips (untested) - specification = input.splitmethod(specification) - local zipfile = specification.path - local zfile = zip.openarchive(instance,name) -- tricky, could be in to be initialized tree - if zfile then - input.logger('! zip locator', specification.original ..' found') - else - input.logger('? zip locator', specification.original ..' not found') - end - end +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) - function input.hashers.zip(instance,tag,name) - input.report("loading zip file",name,"as",tag) - input.usezipfile(instance,tag .."?tree=" .. name) +function resolvers.update_script(oldname,newname) -- oldname -> own.name, not per se a suffix + local scriptpath = "scripts/context/lua" + newname = file.addsuffix(newname,"lua") + local oldscript = resolvers.clean_path(oldname) + if trace_locating then + logs.report("fileio","to be replaced old script %s", oldscript) end - - function input.concatinators.zip(tag,path,name) - if not path or path == "" then - return tag .. '?name=' .. name - else - return tag .. '?name=' .. path .. "/" .. name + local newscripts = resolvers.find_files(newname) or { } + if #newscripts == 0 then + if trace_locating then + logs.report("fileio","unable to locate new script") end - end - - function input.is_readable.zip(name) - return true - end - - function input.finders.zip(instance,specification,filetype) - specification = input.splitmethod(specification) - if specification.path then - local q = url.query(specification.query) - if q.name then - local zfile = zip.openarchive(instance,specification.path) - if zfile then - input.logger('! zip finder',specification.path) - local dfile = zfile:open(q.name) - if dfile then - dfile = zfile:close() - input.logger('+ zip finder',q.name) - return specification.original - end - else - input.logger('? zip finder',specification.path) - end + else + for i=1,#newscripts do + local newscript = resolvers.clean_path(newscripts[i]) + if trace_locating then + logs.report("fileio","checking new script %s", newscript) end - end - input.logger('- zip finder',filename) - return unpack(input.finders.notfound) - end - - function input.openers.zip(instance,specification) - local zipspecification = input.splitmethod(specification) - if zipspecification.path then - local q = url.query(zipspecification.query) - if q.name then - local zfile = zip.openarchive(instance,zipspecification.path) - if zfile then - input.logger('+ zip starter',zipspecification.path) - local dfile = zfile:open(q.name) - if dfile then - input.show_open(specification) - return input.openers.text_opener(specification,dfile,'zip') - end - else - input.logger('- zip starter',zipspecification.path) + if oldscript == newscript then + if trace_locating then + logs.report("fileio","old and new script are the same") end - end - end - input.logger('- zip opener',filename) - return unpack(input.openers.notfound) - end - - function input.loaders.zip(instance,specification) - specification = input.splitmethod(specification) - if specification.path then - local q = url.query(specification.query) - if q.name then - local zfile = zip.openarchive(instance,specification.path) - if zfile then - input.logger('+ zip starter',specification.path) - local dfile = zfile:open(q.name) - if dfile then - input.show_load(filename) - input.logger('+ zip loader',filename) - local s = dfile:read("*all") - dfile:close() - return true, s, #s - end - else - input.logger('- zip starter',specification.path) + elseif not find(newscript,scriptpath) then + if trace_locating then + logs.report("fileio","new script should come from %s",scriptpath) end - end - end - input.logger('- zip loader',filename) - return unpack(input.openers.notfound) - end - - -- zip:///somefile.zip - -- zip:///somefile.zip?tree=texmf-local -> mount - - function input.usezipfile(instance,zipname) - zipname = validzip(zipname) - input.logger('! zip use','file '..zipname) - local specification = input.splitmethod(zipname) - local zipfile = specification.path - if zipfile and not zip.registeredfiles[zipname] then - local tree = url.query(specification.query).tree or "" - input.logger('! zip register','file '..zipname) - local z = zip.openarchive(instance,zipfile) - if z then - input.logger("= zipfile","registering "..zipname) - input.starttiming(instance) - input.aux.prepend_hash(instance,'zip',zipname,zipfile) - input.aux.extend_texmf_var(instance,zipname) -- resets hashes too - zip.registeredfiles[zipname] = z - instance.files[zipname] = input.aux.register_zip_file(z,tree or "") - input.stoptiming(instance) - else - input.logger("? zipfile","unknown "..zipname) - end - else - input.logger('! zip register','no file '..zipname) - end - end - - function input.aux.register_zip_file(z,tree) - local files, filter = { }, "" - if tree == "" then - filter = "^(.+)/(.-)$" - else - filter = "^"..tree.."/(.+)/(.-)$" - end - input.logger('= zip filter',filter) - local register, n = input.aux.register_file, 0 - for i in z:files() do - local path, name = i.filename:match(filter) - if path then - if name and name ~= '' then - register(files, name, path) - n = n + 1 - else - -- directory + elseif not (find(oldscript,file.removesuffix(newname).."$") or find(oldscript,newname.."$")) then + if trace_locating then + logs.report("fileio","invalid new script name") end else - register(files, i.filename, '') - n = n + 1 - end - end - input.report('= zip entries',n) - return files - end - -end - - --- filename : luat-zip.lua --- comment : companion to luat-lib.tex --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files - -if not versions then versions = { } end versions['luat-tex'] = 1.001 - --- special functions that deal with io - -if texconfig and not texlua then - - input.level = input.level or 0 - - if input.logmode() == 'xml' then - function input.show_open(name) - input.level = input.level + 1 - texio.write_nl("<f l='"..input.level.."' n='"..name.."'>") - end - function input.show_close(name) - texio.write("</f> ") - input.level = input.level - 1 - end - function input.show_load(name) - texio.write_nl("<f l='"..(input.level+1).."' n='"..name.."'/>") -- level? - end - else - function input.show_open () end - function input.show_close() end - function input.show_load () end - end - - function input.finders.generic(instance,tag,filename,filetype) - local foundname = input.find_file(instance,filename,filetype) - if foundname and foundname ~= "" then - input.logger('+ ' .. tag .. ' finder',filename,'filetype') - return foundname - else - input.logger('- ' .. tag .. ' finder',filename,'filetype') - return unpack(input.finders.notfound) - end - end - - input.filters.dynamic_translator = nil - input.filters.frozen_translator = nil - input.filters.utf_translator = nil - - function input.openers.text_opener(filename,file_handle,tag) - local u = unicode.utftype(file_handle) - local t = { } - if u > 0 then - input.logger('+ ' .. tag .. ' opener (' .. unicode.utfname[u] .. ')',filename) - local l - if u > 2 then - l = unicode.utf32_to_utf8(file_handle:read("*a"),u==4) - else - l = unicode.utf16_to_utf8(file_handle:read("*a"),u==2) - end - file_handle:close() - t = { - utftype = u, -- may go away - lines = l, - current = 0, -- line number, not really needed - handle = nil, - noflines = #l, - close = function() - input.logger('= ' .. tag .. ' closer (' .. unicode.utfname[u] .. ')',filename) - input.show_close(filename) - end, ---~ getline = function(n) ---~ local line = t.lines[n] ---~ if not line or line == "" then ---~ return "" ---~ else ---~ local translator = input.filters.utf_translator ---~ return (translator and translator(line)) or line ---~ end ---~ end, - reader = function(self) - self = self or t - local current, lines = self.current, self.lines - if current >= #lines then - return nil - else - current = current + 1 - self.current = current - local line = lines[current] - if line == "" then - return "" - else - local translator = input.filters.utf_translator - -- return (translator and translator(line)) or line - if translator then - return translator(line) - else - return line - end - end - end - end - } - else - input.logger('+ ' .. tag .. ' opener',filename) - -- todo: file;name -> freeze / eerste regel scannen -> freeze - local filters = input.filters - t = { - reader = function(self) - local line = file_handle:read() - if line == "" then - return "" - end - local translator = filters.utf_translator - if translator then - return translator(line) - end - translator = filters.dynamic_translator - if translator then - return translator(line) + local newdata = io.loaddata(newscript) + if newdata then + if trace_locating then + logs.report("fileio","old script content replaced by new content") end - return line - end, - close = function() - input.logger('= ' .. tag .. ' closer',filename) - input.show_close(filename) - file_handle:close() - end, - handle = function() - return file_handle - end, - noflines = function() - t.noflines = io.noflines(file_handle) - return t.noflines - end - } - end - return t - end - - function input.openers.generic(instance,tag,filename) - if filename and filename ~= "" then - local f = io.open(filename,"r") - if f then - input.show_open(filename) - return input.openers.text_opener(filename,f,tag) - end - end - input.logger('- ' .. tag .. ' opener',filename) - return unpack(input.openers.notfound) - end - - function input.loaders.generic(instance,tag,filename) - if filename and filename ~= "" then - local f = io.open(filename,"rb") - if f then - input.show_load(filename) - input.logger('+ ' .. tag .. ' loader',filename) - local s = f:read("*a") - f:close() - if s then - return true, s, #s + io.savedata(oldscript,newdata) + break + elseif trace_locating then + logs.report("fileio","unable to load new script") end end end - input.logger('- ' .. tag .. ' loader',filename) - return unpack(input.loaders.notfound) end - - function input.finders.tex(instance,filename,filetype) - return input.finders.generic(instance,'tex',filename,filetype) - end - function input.openers.tex(instance,filename) - return input.openers.generic(instance,'tex',filename) - end - function input.loaders.tex(instance,filename) - return input.loaders.generic(instance,'tex',filename) - end - end --- callback into the file io and related things; disabling kpse +end -- of closure -if texconfig and not texlua then do +do -- create closure to overcome 200 locals limit - -- this is not the right place, because we refer to quite some not yet defined tables, but who cares ... - - ctx = ctx or { } - - local ss = { } +if not modules then modules = { } end modules ['data-lst'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} - function ctx.writestatus(a,b) - local s = ss[a] - if not ss[a] then - s = a:rpadd(15) .. ": " - ss[a] = s - end - texio.write_nl(s .. b .. "\n") - end +-- used in mtxrun - -- this will become: ctx.install_statistics(fnc() return ..,.. end) etc +local find, concat, upper, format = string.find, table.concat, string.upper, string.format - local statusinfo, n = { }, 0 +resolvers.listers = resolvers.listers or { } - function ctx.register_statistics(tag,pattern,fnc) - statusinfo[#statusinfo+1] = { tag, pattern, fnc } - if #tag > n then n = #tag end - end - - function ctx.show_statistics() -- todo: move calls - if caches then - ctx.register_statistics("used config path", "%s", function() return caches.configpath(texmf.instance) end) - ctx.register_statistics("used cache path", "%s", function() return caches.path end) - end - if status.luabytecodes > 0 and input.storage and input.storage.done then - ctx.register_statistics("modules/dumps/instances", "%s/%s/%s", function() return status.luabytecodes-500, input.storage.done, status.luastates end) - end - if texmf.instance then - ctx.register_statistics("input load time", "%s seconds", function() return input.loadtime(texmf.instance) end) - end - if fonts then - ctx.register_statistics("fonts load time","%s seconds", function() return input.loadtime(fonts) end) - end - if xml then - ctx.register_statistics("xml load time", "%s seconds, backreferences: %i, outer filtering time: %s", function() return input.loadtime(xml), #lxml.self, input.loadtime(lxml) end) - end - if mptopdf then - ctx.register_statistics("mps conversion time", "%s seconds", function() return input.loadtime(mptopdf) end) - end - if nodes then - ctx.register_statistics("node processing time", "%s seconds (including kernel)", function() return input.loadtime(nodes) end) - end - if kernel then - ctx.register_statistics("kernel processing time", "%s seconds", function() return input.loadtime(kernel) end) - end - if attributes then - ctx.register_statistics("attribute processing time", "%s seconds", function() return input.loadtime(attributes) end) - end - if languages then - ctx.register_statistics("language load time", "%s seconds, n=%s", function() return input.loadtime(languages), languages.hyphenation.n() end) - end - if figures then - ctx.register_statistics("graphics processing time", "%s seconds, n=%s (including tex)", function() return input.loadtime(figures), figures.n or "?" end) - end - if metapost then - ctx.register_statistics("metapost processing time", "%s seconds, loading: %s seconds, execution: %s seconds, n: %s", function() return input.loadtime(metapost), input.loadtime(mplib), input.loadtime(metapost.exectime), metapost.n end) - end - if status.luastate_bytes then - ctx.register_statistics("current memory usage", "%s bytes", function() return status.luastate_bytes end) - end - if nodes then - ctx.register_statistics("cleaned up reserved nodes", "%s nodes, %s lists of %s", function() return nodes.cleanup_reserved(tex.count[24]) end) -- \topofboxstack - end - if status.node_mem_usage then - ctx.register_statistics("node memory usage", "%s", function() return status.node_mem_usage end) - end - if languages then - ctx.register_statistics("loaded patterns", "%s", function() return languages.logger.report() end) - end - if fonts then - ctx.register_statistics("loaded fonts", "%s", function() return fonts.logger.report() end) - end - if xml then -- so we are in mkiv, we need a different check - ctx.register_statistics("runtime", "%s seconds, %i processed pages, %i shipped pages, %.3f pages/second", function() - input.stoptiming(texmf) - local runtime = input.loadtime(texmf) - local shipped = tex.count['nofshipouts'] - local pages = tex.count['realpageno'] - 1 - local persecond = shipped / runtime - return runtime, pages, shipped, persecond - end) - end - for _, t in ipairs(statusinfo) do - local tag, pattern, fnc = t[1], t[2], t[3] - ctx.writestatus("mkiv lua stats", string.format("%s - %s", tag:rpadd(n," "), pattern:format(fnc()))) - end +local function tabstr(str) + if type(str) == 'table' then + return concat(str," | ") + else + return str end +end -end end - -if texconfig and not texlua then - - texconfig.kpse_init = false - texconfig.trace_file_names = input.logmode() == 'tex' - texconfig.max_print_line = 100000 - - -- if still present, we overload kpse (put it off-line so to say) - - if not texmf then texmf = { } end - - input.starttiming(texmf) - - if not texmf.instance then - - if not texmf.instance then -- prevent a second loading - - texmf.instance = input.reset() - texmf.instance.progname = environment.progname or 'context' - texmf.instance.engine = environment.engine or 'luatex' - texmf.instance.validfile = input.validctxfile - - input.load(texmf.instance) - - end - - if callback then - callback.register('find_read_file' , function(id,name) return input.findtexfile(texmf.instance,name) end) - callback.register('open_read_file' , function( name) return input.opentexfile(texmf.instance,name) end) - end - - if callback then - callback.register('find_data_file' , function(name) return input.findbinfile(texmf.instance,name,"tex") end) - callback.register('find_enc_file' , function(name) return input.findbinfile(texmf.instance,name,"enc") end) - callback.register('find_font_file' , function(name) return input.findbinfile(texmf.instance,name,"tfm") end) - callback.register('find_format_file' , function(name) return input.findbinfile(texmf.instance,name,"fmt") end) - callback.register('find_image_file' , function(name) return input.findbinfile(texmf.instance,name,"tex") end) - callback.register('find_map_file' , function(name) return input.findbinfile(texmf.instance,name,"map") end) - callback.register('find_ocp_file' , function(name) return input.findbinfile(texmf.instance,name,"ocp") end) - callback.register('find_opentype_file' , function(name) return input.findbinfile(texmf.instance,name,"otf") end) - callback.register('find_output_file' , function(name) return name end) - callback.register('find_pk_file' , function(name) return input.findbinfile(texmf.instance,name,"pk") end) - callback.register('find_sfd_file' , function(name) return input.findbinfile(texmf.instance,name,"sfd") end) - callback.register('find_truetype_file' , function(name) return input.findbinfile(texmf.instance,name,"ttf") end) - callback.register('find_type1_file' , function(name) return input.findbinfile(texmf.instance,name,"pfb") end) - callback.register('find_vf_file' , function(name) return input.findbinfile(texmf.instance,name,"vf") end) - - callback.register('read_data_file' , function(file) return input.loadbinfile(texmf.instance,file,"tex") end) - callback.register('read_enc_file' , function(file) return input.loadbinfile(texmf.instance,file,"enc") end) - callback.register('read_font_file' , function(file) return input.loadbinfile(texmf.instance,file,"tfm") end) - -- format - -- image - callback.register('read_map_file' , function(file) return input.loadbinfile(texmf.instance,file,"map") end) - callback.register('read_ocp_file' , function(file) return input.loadbinfile(texmf.instance,file,"ocp") end) - callback.register('read_opentype_file' , function(file) return input.loadbinfile(texmf.instance,file,"otf") end) - -- output - callback.register('read_pk_file' , function(file) return input.loadbinfile(texmf.instance,file,"pk") end) - callback.register('read_sfd_file' , function(file) return input.loadbinfile(texmf.instance,file,"sfd") end) - callback.register('read_truetype_file' , function(file) return input.loadbinfile(texmf.instance,file,"ttf") end) - callback.register('read_type1_file' , function(file) return input.loadbinfile(texmf.instance,file,"pfb") end) - callback.register('read_vf_file' , function(file) return input.loadbinfile(texmf.instance,file,"vf" ) end) - end - - if callback and environment.aleph_mode then - callback.register('find_font_file' , function(name) return input.findbinfile(texmf.instance,name,"ofm") end) - callback.register('read_font_file' , function(file) return input.loadbinfile(texmf.instance,file,"ofm") end) - callback.register('find_vf_file' , function(name) return input.findbinfile(texmf.instance,name,"ovf") end) - callback.register('read_vf_file' , function(file) return input.loadbinfile(texmf.instance,file,"ovf") end) - end - - if callback then - callback.register('find_write_file' , function(id,name) return name end) - end - - if callback and (not config or (#config == 0)) then - callback.register('find_format_file' , function(name) return name end) - end - - if callback and false then - for k, v in pairs(callback.list()) do - if not v then texio.write_nl("<w>callback "..k.." is not set</w>") end - end - end - - if callback then - - input.start_actions = { } - input.stop_actions = { } - - function input.register_start_actions(f) table.insert(input.start_actions, f) end - function input.register_stop_actions (f) table.insert(input.stop_actions, f) end - - --~ callback.register('start_run', function() for _, a in pairs(input.start_actions) do a() end end) - --~ callback.register('stop_run' , function() for _, a in pairs(input.stop_actions ) do a() end end) - - end - - if callback then - - if input.logmode() == 'xml' then - - function input.start_page_number() - texio.write_nl("<p real='" .. tex.count[0] .. "' page='"..tex.count[1].."' sub='"..tex.count[2].."'") - end - function input.stop_page_number() - texio.write("/>") - texio.write_nl("") - end - - callback.register('start_page_number' , input.start_page_number) - callback.register('stop_page_number' , input.stop_page_number ) - - function input.report_output_pages(p,b) - texio.write_nl("<v k='pages'>"..p.."</v>") - texio.write_nl("<v k='bytes'>"..b.."</v>") - texio.write_nl("") - end - function input.report_output_log() - end - - callback.register('report_output_pages', input.report_output_pages) - callback.register('report_output_log' , input.report_output_log ) - - function input.start_run() - texio.write_nl("<?xml version='1.0' standalone='yes'?>") - texio.write_nl("<job xmlns='www.tug.org/luatex/schemas/context-job.rng'>") - texio.write_nl("") - end - function input.stop_run() - texio.write_nl("</job>") - end - function input.show_statistics() - for k,v in pairs(status.list()) do - texio.write_nl("log","<v k='"..k.."'>"..tostring(v).."</v>") - end +local function list(list,report) + local instance = resolvers.instance + local pat = upper(pattern or "","") + local report = report or texio.write_nl + local sorted = table.sortedkeys(list) + for i=1,#sorted do + local key = sorted[i] + if instance.pattern == "" or find(upper(key),pat) then + if instance.kpseonly then + if instance.kpsevars[key] then + report(format("%s=%s",key,tabstr(list[key]))) end - - table.insert(input.start_actions, input.start_run) - table.insert(input.stop_actions , input.show_statistics) - table.insert(input.stop_actions , input.stop_run) - else - table.insert(input.stop_actions , input.show_statistics) + report(format('%s %s=%s',(instance.kpsevars[key] and 'K') or 'E',key,tabstr(list[key]))) end - - callback.register('start_run', function() for _, a in pairs(input.start_actions) do a() end end) - callback.register('stop_run' , function() for _, a in pairs(input.stop_actions ) do a() end ctx.show_statistics() end) - end - end - - if kpse then - - function kpse.find_file(filename,filetype,mustexist) - return input.find_file(texmf.instance,filename,filetype,mustexist) - end - function kpse.expand_path(variable) - return input.expand_path(texmf.instance,variable) - end - function kpse.expand_var(variable) - return input.expand_var(texmf.instance,variable) - end - function kpse.expand_braces(variable) - return input.expand_braces(texmf.instance,variable) - end - - end - end --- program specific configuration (memory settings and alike) - -if texconfig and not texlua then - - luatex = luatex or { } +function resolvers.listers.variables () list(resolvers.instance.variables ) end +function resolvers.listers.expansions() list(resolvers.instance.expansions) end - luatex.variablenames = { - 'main_memory', 'extra_mem_bot', 'extra_mem_top', - 'buf_size','expand_depth', - 'font_max', 'font_mem_size', - 'hash_extra', 'max_strings', 'pool_free', 'pool_size', 'string_vacancies', - 'obj_tab_size', 'pdf_mem_size', 'dest_names_size', - 'nest_size', 'param_size', 'save_size', 'stack_size', - 'trie_size', 'hyph_size', 'max_in_open', - 'ocp_stack_size', 'ocp_list_size', 'ocp_buf_size' - } - - function luatex.variables() - local t, x = { }, nil - for _,v in pairs(luatex.variablenames) do - x = input.var_value(texmf.instance,v) - if x and x:find("^%d+$") then - t[v] = tonumber(x) +function resolvers.listers.configurations(report) + local report = report or texio.write_nl + local instance = resolvers.instance + local sorted = table.sortedkeys(instance.kpsevars) + for i=1,#sorted do + local key = sorted[i] + if not instance.pattern or (instance.pattern=="") or find(key,instance.pattern) then + report(format("%s\n",key)) + local order = instance.order + for i=1,#order do + local str = order[i][key] + if str then + report(format("\t%s\t%s",i,str)) + end end - end - return t - end - - function luatex.setvariables(tab) - for k,v in pairs(luatex.variables()) do - tab[k] = v + report("") end end - - if not luatex.variables_set then - luatex.setvariables(texconfig) - luatex.variables_set = true - end - - texconfig.max_print_line = 100000 - texconfig.max_in_open = 127 - end --- some tex basics - -if not cs then cs = { } end - -function cs.def(k,v) - tex.sprint(tex.texcatcodes, "\\def\\" .. k .. "{" .. v .. "}") -end - -function cs.chardef(k,v) - tex.sprint(tex.texcatcodes, "\\chardef\\" .. k .. "=" .. v .. "\\relax") -end - -function cs.boolcase(b) - if b then tex.write(1) else tex.write(0) end -end - -function cs.testcase(b) - if b then - tex.sprint(tex.texcatcodes, "\\firstoftwoarguments") - else - tex.sprint(tex.texcatcodes, "\\secondoftwoarguments") - end -end - - -if not modules then modules = { } end modules ['luat-kps'] = { - version = 1.001, - comment = "companion to luatools.lua", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - ---[[ldx-- -<p>This file is used when we want the input handlers to behave like -<type>kpsewhich</type>. What to do with the following:</p> - -<typing> -{$SELFAUTOLOC,$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,}/web2c} -$SELFAUTOLOC : /usr/tex/bin/platform -$SELFAUTODIR : /usr/tex/bin -$SELFAUTOPARENT : /usr/tex -</typing> - -<p>How about just forgetting abou them?</p> ---ldx]]-- - -input = input or { } -input.suffixes = input.suffixes or { } -input.formats = input.formats or { } - -input.suffixes['gf'] = { '<resolution>gf' } -input.suffixes['pk'] = { '<resolution>pk' } -input.suffixes['base'] = { 'base' } -input.suffixes['bib'] = { 'bib' } -input.suffixes['bst'] = { 'bst' } -input.suffixes['cnf'] = { 'cnf' } -input.suffixes['mem'] = { 'mem' } -input.suffixes['mf'] = { 'mf' } -input.suffixes['mfpool'] = { 'pool' } -input.suffixes['mft'] = { 'mft' } -input.suffixes['mppool'] = { 'pool' } -input.suffixes['graphic/figure'] = { 'eps', 'epsi' } -input.suffixes['texpool'] = { 'pool' } -input.suffixes['PostScript header'] = { 'pro' } -input.suffixes['ist'] = { 'ist' } -input.suffixes['web'] = { 'web', 'ch' } -input.suffixes['cweb'] = { 'w', 'web', 'ch' } -input.suffixes['cmap files'] = { 'cmap' } -input.suffixes['lig files'] = { 'lig' } -input.suffixes['bitmap font'] = { } -input.suffixes['MetaPost support'] = { } -input.suffixes['TeX system documentation'] = { } -input.suffixes['TeX system sources'] = { } -input.suffixes['dvips config'] = { } -input.suffixes['type42 fonts'] = { } -input.suffixes['web2c files'] = { } -input.suffixes['other text files'] = { } -input.suffixes['other binary files'] = { } -input.suffixes['opentype fonts'] = { 'otf' } - -input.suffixes['fmt'] = { 'fmt' } -input.suffixes['texmfscripts'] = { 'rb','lua','py','pl' } - -input.suffixes['pdftex config'] = { } -input.suffixes['Troff fonts'] = { } - -input.suffixes['ls-R'] = { } - ---[[ldx-- -<p>If you wondered abou tsome of the previous mappings, how about -the next bunch:</p> ---ldx]]-- - -input.formats['bib'] = '' -input.formats['bst'] = '' -input.formats['mft'] = '' -input.formats['ist'] = '' -input.formats['web'] = '' -input.formats['cweb'] = '' -input.formats['MetaPost support'] = '' -input.formats['TeX system documentation'] = '' -input.formats['TeX system sources'] = '' -input.formats['Troff fonts'] = '' -input.formats['dvips config'] = '' -input.formats['graphic/figure'] = '' -input.formats['ls-R'] = '' -input.formats['other text files'] = '' -input.formats['other binary files'] = '' - -input.formats['gf'] = '' -input.formats['pk'] = '' -input.formats['base'] = 'MFBASES' -input.formats['cnf'] = '' -input.formats['mem'] = 'MPMEMS' -input.formats['mf'] = 'MFINPUTS' -input.formats['mfpool'] = 'MFPOOL' -input.formats['mppool'] = 'MPPOOL' -input.formats['texpool'] = 'TEXPOOL' -input.formats['PostScript header'] = 'TEXPSHEADERS' -input.formats['cmap files'] = 'CMAPFONTS' -input.formats['type42 fonts'] = 'T42FONTS' -input.formats['web2c files'] = 'WEB2C' -input.formats['pdftex config'] = 'PDFTEXCONFIG' -input.formats['texmfscripts'] = 'TEXMFSCRIPTS' -input.formats['bitmap font'] = '' -input.formats['lig files'] = 'LIGFONTS' +end -- of closure -- end library merge -- We initialize some characteristics of this program. We need to @@ -6226,19 +7775,34 @@ own.libs = { -- todo: check which ones are really needed 'l-number.lua', 'l-set.lua', 'l-os.lua', - 'l-md5.lua', 'l-file.lua', + 'l-md5.lua', 'l-url.lua', 'l-dir.lua', 'l-boolean.lua', 'l-unicode.lua', + 'l-math.lua', 'l-utils.lua', - 'luat-lib.lua', - 'luat-inp.lua', - 'luat-tmp.lua', - 'luat-zip.lua', - 'luat-tex.lua', - 'luat-kps.lua', + 'l-aux.lua', + 'trac-tra.lua', + 'luat-env.lua', + 'trac-inf.lua', + 'trac-log.lua', + 'data-res.lua', + 'data-tmp.lua', +-- 'data-pre.lua', + 'data-inp.lua', + 'data-out.lua', + 'data-con.lua', + 'data-use.lua', +-- 'data-tex.lua', +-- 'data-bin.lua', +-- 'data-zip.lua', +-- 'data-crl.lua', +-- 'data-lua.lua', + 'data-kps.lua', -- so that we can replace kpsewhich + 'data-aux.lua', -- updater + 'data-lst.lua', -- lister } -- We need this hack till luatex is fixed. @@ -6275,11 +7839,11 @@ function locate_libs() end end -if not input then +if not resolvers then locate_libs() end -if not input then +if not resolvers then print("") print("Luatools is unable to start up due to lack of libraries. You may") print("try to run 'lua luatools.lua --selfmerge' in the path where this") @@ -6288,61 +7852,71 @@ if not input then os.exit() end -instance = input.reset() -input.verbose = environment.arguments["verbose"] or false -input.banner = 'LuaTools | ' -utils.report = input.report +logs.setprogram('LuaTools',"TDS Management Tool 1.32",environment.arguments["verbose"] or false) -input.defaultlibs = { -- not all are needed - 'l-string.lua', 'l-lpeg.lua', 'l-table.lua', 'l-boolean.lua', 'l-number.lua', 'l-set.lua', 'l-unicode.lua', - 'l-md5.lua', 'l-os.lua', 'l-io.lua', 'l-file.lua', 'l-url.lua', 'l-dir.lua', 'l-utils.lua', 'l-tex.lua', - 'luat-env.lua', 'luat-lib.lua', 'luat-inp.lua', 'luat-tmp.lua', 'luat-zip.lua', 'luat-tex.lua' -} +local instance = resolvers.reset() --- todo: use environment.argument() instead of environment.arguments[] +resolvers.defaultlibs = { -- not all are needed (this will become: context.lus (lua spec) + 'l-string.lua', + 'l-lpeg.lua', + 'l-table.lua', + 'l-boolean.lua', + 'l-number.lua', + 'l-unicode.lua', + 'l-os.lua', + 'l-io.lua', + 'l-file.lua', + 'l-md5.lua', + 'l-url.lua', + 'l-dir.lua', + 'l-utils.lua', + 'l-dimen.lua', + 'trac-inf.lua', + 'trac-tra.lua', + 'trac-log.lua', + 'luat-env.lua', -- here ? + 'data-res.lua', + 'data-inp.lua', + 'data-out.lua', + 'data-tmp.lua', + 'data-con.lua', + 'data-use.lua', +-- 'data-pre.lua', + 'data-tex.lua', + 'data-bin.lua', +-- 'data-zip.lua', +-- 'data-clr.lua', + 'data-lua.lua', + 'data-ctx.lua', + 'luat-fio.lua', + 'luat-cnf.lua', +} instance.engine = environment.arguments["engine"] or 'luatex' instance.progname = environment.arguments["progname"] or 'context' instance.luaname = environment.arguments["luafile"] or "" -- environment.ownname or "" -instance.lualibs = environment.arguments["lualibs"] or table.concat(input.defaultlibs,",") +instance.lualibs = environment.arguments["lualibs"] or table.concat(resolvers.defaultlibs,",") instance.allresults = environment.arguments["all"] or false instance.pattern = environment.arguments["pattern"] or nil instance.sortdata = environment.arguments["sort"] or false instance.kpseonly = not environment.arguments["all"] or false instance.my_format = environment.arguments["format"] or instance.format -instance.lsrmode = environment.arguments["lsr"] or false if type(instance.pattern) == 'boolean' then - input.report("invalid pattern specification") -- toto, force verbose for one message + logs.simple("invalid pattern specification") instance.pattern = nil end -if environment.arguments["trace"] then input.settrace(environment.arguments["trace"]) end - -if environment.arguments["minimize"] then - if input.validators.visibility[instance.progname] then - instance.validfile = input.validators.visibility[instance.progname] - end -end +if environment.arguments["trace"] then resolvers.settrace(environment.arguments["trace"]) end -function input.my_prepare_a(instance) - input.resetconfig(instance) - input.identify_cnf(instance) - input.load_lua(instance) - input.expand_variables(instance) - input.load_cnf(instance) - input.expand_variables(instance) -end +local trackspec = environment.argument("trackers") or environment.argument("track") -function input.my_prepare_b(instance) - input.my_prepare_a(instance) - input.load_hash(instance) - input.automount(instance) +if trackspec then + trackers.enable(trackspec) end --- barename - -if not messages then messages = { } end +runners = runners or { } +messages = messages or { } messages.no_ini_file = [[ There is no lua initialization file found. This file can be forced by the @@ -6368,21 +7942,20 @@ messages.help = [[ --luafile=str lua inifile (default is <progname>.lua) --lualibs=list libraries to assemble (optional when --compile) --compile assemble and compile lua inifile ---mkii force context mkii mode (only for testing, not usable!) --verbose give a bit more info ---minimize optimize lists for format --all show all found files --sort sort cached data --engine=str target engine --progname=str format or backend --pattern=str filter variables ---lsr use lsr and cnf directly +--trackers=list enable given trackers ]] -function input.my_make_format(instance,texname) +function runners.make_format(texname) + local instance = resolvers.instance if texname and texname ~= "" then - if input.usecache then - local path = file.join(caches.setpath(instance,"formats")) -- maybe platform + if resolvers.usecache then + local path = file.join(caches.setpath("formats")) -- maybe platform if path and lfs then lfs.chdir(path) end @@ -6391,22 +7964,22 @@ function input.my_make_format(instance,texname) if barename == texname then texname = texname .. ".tex" end - local fullname = input.find_files(instance,texname)[1] or "" + local fullname = resolvers.find_files(texname)[1] or "" if fullname == "" then - input.report("no tex file with name",texname) + logs.simple("no tex file with name: %s",texname) else local luaname, lucname, luapath, lualibs = "", "", "", { } -- the following is optional, since context.lua can also -- handle this collect and compile business if environment.arguments["compile"] then if luaname == "" then luaname = barename end - input.report("creating initialization file " .. luaname) + logs.simple("creating initialization file: %s",luaname) luapath = file.dirname(luaname) if luapath == "" then luapath = file.dirname(texname) end if luapath == "" then - luapath = file.dirname(input.find_files(instance,texname)[1] or "") + luapath = file.dirname(resolvers.find_files(texname)[1] or "") end lualibs = string.split(instance.lualibs,",") luaname = file.basename(barename .. ".lua") @@ -6416,74 +7989,88 @@ function input.my_make_format(instance,texname) if lualibs[1] then local firstlib = file.join(luapath,lualibs[1]) if not lfs.isfile(firstlib) then - local foundname = input.find_files(instance,lualibs[1])[1] + local foundname = resolvers.find_files(lualibs[1])[1] if foundname then - input.report("located library path : " .. luapath) + logs.simple("located library path: %s",luapath) luapath = file.dirname(foundname) end end end - input.report("using library path : " .. luapath) - input.report("using lua libraries: " .. table.join(lualibs," ")) + logs.simple("using library path: %s",luapath) + logs.simple("using lua libraries: %s",table.join(lualibs," ")) utils.merger.selfcreate(lualibs,luapath,luaname) - if utils.lua.compile(luaname, lucname) and io.exists(lucname) then + local strip = resolvers.boolean_variable("LUACSTRIP", true) + if utils.lua.compile(luaname,lucname,false,strip) and io.exists(lucname) then luaname = lucname - input.report("using compiled initialization file " .. lucname) + logs.simple("using compiled initialization file: %s",lucname) else - input.report("using uncompiled initialization file " .. luaname) + logs.simple("using uncompiled initialization file: %s",luaname) end else - for _, v in pairs({instance.luaname, instance.progname, barename}) do - v = string.gsub(v..".lua","%.lua%.lua$",".lua") + local what = { instance.luaname, instance.progname, barename } + for k=1,#what do + local v = string.gsub(what[k]..".lua","%.lua%.lua$",".lua") if v and (v ~= "") then - luaname = input.find_files(instance,v)[1] or "" + luaname = resolvers.find_files(v)[1] or "" if luaname ~= "" then break end end end end + if environment.arguments["noluc"] then + luaname = luaname:gsub("%.luc$",".lua") -- make this an option + end if luaname == "" then - input.reportlines(messages.no_ini_file) - input.report("texname : " .. texname) - input.report("luaname : " .. instance.luaname) - input.report("progname : " .. instance.progname) - input.report("barename : " .. barename) + if logs.verbose then + logs.simplelines(messages.no_ini_file) + logs.simple("texname : %s",texname) + logs.simple("luaname : %s",instance.luaname) + logs.simple("progname: %s",instance.progname) + logs.simple("barename: %s",barename) + end else - input.report("using lua initialization file " .. luaname) - local flags = { "--ini" } - if environment.arguments["mkii"] then - flags[#flags+1] = "--progname=" .. instance.progname - else - flags[#flags+1] = "--lua=" .. string.quote(luaname) + logs.simple("using lua initialization file: %s",luaname) + local mp = dir.glob(file.removesuffix(file.basename(luaname)).."-*.mem") + if mp and #mp > 0 then + for i=1,#mp do + local name = mp[i] + logs.simple("removing related mplib format %s", file.basename(name)) + os.remove(name) + end end - local bs = (environment.platform == "unix" and "\\\\") or "\\" -- todo: make a function + local flags = { + "--ini", + "--lua=" .. string.quote(luaname) + } + local bs = (os.platform == "unix" and "\\\\") or "\\" -- todo: make a function local command = "luatex ".. table.concat(flags," ") .. " " .. string.quote(fullname) .. " " .. bs .. "dump" - input.report("running command: " .. command .. "\n") + logs.simple("running command: %s\n",command) os.spawn(command) + -- todo: do a dummy run that generates the related metafun and mfplain formats end end else - input.report("no tex file given") + logs.simple("no tex file given") end end -function input.my_run_format(instance,name,data,more) +function runners.run_format(name,data,more) -- hm, rather old code here; we can now use the file.whatever functions if name and (name ~= "") then local barename = name:gsub("%.%a+$","") local fmtname = "" - if input.usecache then - local path = file.join(caches.setpath(instance,"formats")) -- maybe platform + if resolvers.usecache then + local path = file.join(caches.setpath("formats")) -- maybe platform fmtname = file.join(path,barename..".fmt") or "" end if fmtname == "" then - fmtname = input.find_files(instance,barename..".fmt")[1] or "" + fmtname = resolvers.find_files(barename..".fmt")[1] or "" end - fmtname = input.clean_path(fmtname) + fmtname = resolvers.clean_path(fmtname) barename = fmtname:gsub("%.%a+$","") if fmtname == "" then - input.report("no format with name",name) + logs.simple("no format with name: %s",name) else local luaname = barename .. ".luc" local f = io.open(luaname) @@ -6493,164 +8080,106 @@ function input.my_run_format(instance,name,data,more) end if f then f:close() - local command = "luatex --fmt=" .. string.quote(barename) .. " --lua=" .. string.quote(luaname) .. " " .. string.quote(data) .. " " .. string.quote(more) - input.report("running command: " .. command) + local command = "luatex --fmt=" .. string.quote(barename) .. " --lua=" .. string.quote(luaname) .. " " .. string.quote(data) .. " " .. (more ~= "" and string.quote(more) or "") + logs.simple("running command: %s",command) os.spawn(command) else - input.report("using format name",fmtname) - input.report("no luc/lua with name",barename) + logs.simple("using format name: %s",fmtname) + logs.simple("no luc/lua with name: %s",barename) end end end end --- helpers for verbose lists - -input.listers = input.listers or { } - -local function tabstr(str) - if type(str) == 'table' then - return table.concat(str," | ") - else - return str - end -end - -local function list(instance,list) - local pat = string.upper(instance.pattern or "","") - for _,key in pairs(table.sortedkeys(list)) do - if instance.pattern == "" or string.find(key:upper(),pat) then - if instance.kpseonly then - if instance.kpsevars[key] then - print(format("%s=%s",key,tabstr(list[key]))) - end - else - print(format('%s %s=%s',(instance.kpsevars[key] and 'K') or 'E',key,tabstr(list[key]))) - end - end - end -end - -function input.listers.variables (instance) list(instance,instance.variables ) end -function input.listers.expansions(instance) list(instance,instance.expansions) end - -function input.listers.configurations(instance) - for _,key in pairs(table.sortedkeys(instance.kpsevars)) do - if not instance.pattern or (instance.pattern=="") or key:find(instance.pattern) then - print(key.."\n") - for i,c in ipairs(instance.order) do - local str = c[key] - if str then - print(format("\t%s\t\t%s",i,input.aux.tabstr(str))) - end - end - print() - end - end -end - -input.report(banner,"\n") - local ok = true +-- private option --noluc for testing errors in the stub + if environment.arguments["find-file"] then - input.my_prepare_b(instance) + resolvers.load() instance.format = environment.arguments["format"] or instance.format if instance.pattern then instance.allresults = true - input.for_files(instance, input.find_files, { instance.pattern }, instance.my_format) + resolvers.for_files(resolvers.find_files, { instance.pattern }, instance.my_format) else - input.for_files(instance, input.find_files, environment.files, instance.my_format) + resolvers.for_files(resolvers.find_files, environment.files, instance.my_format) end elseif environment.arguments["find-path"] then - input.my_prepare_b(instance) - local path = input.find_file(instance, environment.files[1], instance.my_format) - if input.verbose then - input.report(file.dirname(path)) + resolvers.load() + local path = resolvers.find_file(environment.files[1], instance.my_format) + if logs.verbose then + logs.simple(file.dirname(path)) else print(file.dirname(path)) end ---~ elseif environment.arguments["first-writable-path"] then ---~ input.my_prepare_b(instance) ---~ input.report(input.first_writable_path(instance,environment.files[1] or ".")) elseif environment.arguments["run"] then - input.my_prepare_a(instance) -- ! no need for loading databases - input.verbose = true - input.my_run_format(instance,environment.files[1] or "",environment.files[2] or "",environment.files[3] or "") + resolvers.load("nofiles") -- ! no need for loading databases + logs.setverbose(true) + runners.run_format(environment.files[1] or "",environment.files[2] or "",environment.files[3] or "") elseif environment.arguments["fmt"] then - input.my_prepare_a(instance) -- ! no need for loading databases - input.verbose = true - input.my_run_format(instance,environment.arguments["fmt"], environment.files[1] or "",environment.files[2] or "") + resolvers.load("nofiles") -- ! no need for loading databases + logs.setverbose(true) + runners.run_format(environment.arguments["fmt"], environment.files[1] or "",environment.files[2] or "") elseif environment.arguments["expand-braces"] then - input.my_prepare_a(instance) - input.for_files(instance, input.expand_braces, environment.files) + resolvers.load("nofiles") + resolvers.for_files(resolvers.expand_braces, environment.files) elseif environment.arguments["expand-path"] then - input.my_prepare_a(instance) - input.for_files(instance, input.expand_path, environment.files) + resolvers.load("nofiles") + resolvers.for_files(resolvers.expand_path, environment.files) elseif environment.arguments["expand-var"] or environment.arguments["expand-variable"] then - input.my_prepare_a(instance) - input.for_files(instance, input.expand_var, environment.files) + resolvers.load("nofiles") + resolvers.for_files(resolvers.expand_var, environment.files) elseif environment.arguments["show-path"] or environment.arguments["path-value"] then - input.my_prepare_a(instance) - input.for_files(instance, input.show_path, environment.files) + resolvers.load("nofiles") + resolvers.for_files(resolvers.show_path, environment.files) elseif environment.arguments["var-value"] or environment.arguments["show-value"] then - input.my_prepare_a(instance) - input.for_files(instance, input.var_value, environment.files) + resolvers.load("nofiles") + resolvers.for_files(resolvers.var_value, environment.files) elseif environment.arguments["format-path"] then - input.my_prepare_b(instance) - input.report(caches.setpath(instance,"format")) + resolvers.load() + logs.simple(caches.setpath("format")) elseif instance.pattern then -- brrr - input.my_prepare_b(instance) + resolvers.load() instance.format = environment.arguments["format"] or instance.format instance.allresults = true - input.for_files(instance, input.find_files, { instance.pattern }, instance.my_format) + resolvers.for_files(resolvers.find_files, { instance.pattern }, instance.my_format) elseif environment.arguments["generate"] then instance.renewcache = true - input.verbose = true - input.my_prepare_b(instance) + logs.setverbose(true) + resolvers.load() elseif environment.arguments["make"] or environment.arguments["ini"] or environment.arguments["compile"] then - input.my_prepare_b(instance) - input.verbose = true - input.my_make_format(instance,environment.files[1] or "") + resolvers.load() + logs.setverbose(true) + runners.make_format(environment.files[1] or "") elseif environment.arguments["selfmerge"] then utils.merger.selfmerge(own.name,own.libs,own.list) elseif environment.arguments["selfclean"] then utils.merger.selfclean(own.name) elseif environment.arguments["selfupdate"] then - input.my_prepare_b(instance) - input.verbose = true - input.update_script(instance,own.name,"luatools") + resolvers.load() + logs.setverbose(true) + resolvers.update_script(own.name,"luatools") elseif environment.arguments["variables"] or environment.arguments["show-variables"] then - input.my_prepare_a(instance) - input.listers.variables(instance) + resolvers.load("nofiles") + resolvers.listers.variables() elseif environment.arguments["expansions"] or environment.arguments["show-expansions"] then - input.my_prepare_a(instance) - input.listers.expansions(instance) + resolvers.load("nofiles") + resolvers.listers.expansions() elseif environment.arguments["configurations"] or environment.arguments["show-configurations"] then - input.my_prepare_a(instance) - input.listers.configurations(instance) + resolvers.load("nofiles") + resolvers.listers.configurations() elseif environment.arguments["help"] or (environment.files[1]=='help') or (#environment.files==0) then - if not input.verbose then - input.verbose = true - input.report(banner,"\n") - end - input.reportlines(messages.help) + logs.help(messages.help) else - input.my_prepare_b(instance) - input.for_files(instance, input.find_files, environment.files, instance.my_format) + resolvers.load() + resolvers.for_files(resolvers.find_files, environment.files, instance.my_format) end -if input.verbose then - input.report("") - input.report(string.format("runtime: %0.3f seconds",os.runtime())) +if logs.verbose then + logs.simpleline() + logs.simple("runtime: %0.3f seconds",os.runtime()) end ---~ if ok then ---~ input.report("exit code: 0") os.exit(0) ---~ else ---~ input.report("exit code: 1") os.exit(1) ---~ end - -if environment.platform == "unix" then +if os.platform == "unix" then io.write("\n") end diff --git a/Master/bin/win32/makempy.bat b/Master/bin/win32/makempy.bat deleted file mode 100755 index 9622d96a5f3..00000000000 --- a/Master/bin/win32/makempy.bat +++ /dev/null @@ -1,2 +0,0 @@ -@echo off
-texmfstart makempy.pl %*
diff --git a/Master/bin/win32/makempy.exe b/Master/bin/win32/makempy.exe Binary files differdeleted file mode 100755 index 5777d90a17a..00000000000 --- a/Master/bin/win32/makempy.exe +++ /dev/null diff --git a/Master/bin/win32/mpstools.bat b/Master/bin/win32/mpstools.bat deleted file mode 100755 index 10ec3f1d687..00000000000 --- a/Master/bin/win32/mpstools.bat +++ /dev/null @@ -1,2 +0,0 @@ -@echo off
-texmfstart mpstools.rb %*
diff --git a/Master/bin/win32/mpstools.exe b/Master/bin/win32/mpstools.exe Binary files differdeleted file mode 100755 index 5777d90a17a..00000000000 --- a/Master/bin/win32/mpstools.exe +++ /dev/null diff --git a/Master/bin/win32/mptopdf.bat b/Master/bin/win32/mptopdf.bat deleted file mode 100755 index 757ea58f9ce..00000000000 --- a/Master/bin/win32/mptopdf.bat +++ /dev/null @@ -1,2 +0,0 @@ -@echo off
-texmfstart mptopdf.pl %*
diff --git a/Master/bin/win32/mptopdf.exe b/Master/bin/win32/mptopdf.exe Binary files differdeleted file mode 100755 index 5777d90a17a..00000000000 --- a/Master/bin/win32/mptopdf.exe +++ /dev/null diff --git a/Master/bin/win32/mtxrun.exe b/Master/bin/win32/mtxrun.exe Binary files differindex 5777d90a17a..745eaf22464 100755 --- a/Master/bin/win32/mtxrun.exe +++ b/Master/bin/win32/mtxrun.exe diff --git a/Master/bin/win32/mtxrun.lua b/Master/bin/win32/mtxrun.lua index 040214178b8..b99327692d7 100755 --- a/Master/bin/win32/mtxrun.lua +++ b/Master/bin/win32/mtxrun.lua @@ -8,6 +8,7 @@ if not modules then modules = { } end modules ['mtxrun'] = { license = "see context related readme files" } + -- one can make a stub: -- -- #!/bin/sh @@ -26,7 +27,7 @@ if not modules then modules = { } end modules ['mtxrun'] = { -- one. Interesting is that using a scripting language instead of c does -- not have a speed penalty. Actually the lua variant is more efficient, -- especially when multiple calls to kpsewhich are involved. The lua --- library also gives way more ocntrol. +-- library also gives way more control. -- to be done / considered -- @@ -37,161 +38,46 @@ if not modules then modules = { } end modules ['mtxrun'] = { -- remember for subruns: _CTX_K_S_#{original}_ -- remember for subruns: TEXMFSTART.#{original} [tex.rb texmfstart.rb] -banner = "version 1.0.2 - 2007+ - PRAGMA ADE / CONTEXT" texlua = true -- begin library merge --- filename : l-string.lua --- comment : split off from luat-lib --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files -if not versions then versions = { } end versions['l-string'] = 1.001 ---~ function string.split(str, pat) -- taken from the lua wiki ---~ local t = {n = 0} -- so this table has a length field, traverse with ipairs then! ---~ local fpat = "(.-)"..pat ---~ local last_end = 1 ---~ local s, e, cap = string.find(str, fpat, 1) ---~ while s ~= nil do ---~ if s~=1 or cap~="" then ---~ table.insert(t,cap) ---~ end ---~ last_end = e+1 ---~ s, e, cap = string.find(str, fpat, last_end) ---~ end ---~ if last_end<=string.len(str) then ---~ table.insert(t,(string.sub(str,last_end))) ---~ end ---~ return t ---~ end ---~ function string:split(pat) -- taken from the lua wiki but adapted ---~ local t = { } -- self and colon usage (faster) ---~ local fpat = "(.-)"..pat ---~ local last_end = 1 ---~ local s, e, cap = self:find(fpat, 1) ---~ while s ~= nil do ---~ if s~=1 or cap~="" then ---~ t[#t+1] = cap ---~ end ---~ last_end = e+1 ---~ s, e, cap = self:find(fpat, last_end) ---~ end ---~ if last_end <= #self then ---~ t[#t+1] = self:sub(last_end) ---~ end ---~ return t ---~ end +do -- create closure to overcome 200 locals limit ---~ a piece of brilliant code by Rici Lake (posted on lua list) -- only names changed ---~ ---~ function string:splitter(pat) ---~ local st, g = 1, self:gmatch("()"..pat.."()") ---~ local function splitter(self) ---~ if st then ---~ local s, f = g() ---~ local rv = self:sub(st, (s or 0)-1) ---~ st = f ---~ return rv ---~ end ---~ end ---~ return splitter, self ---~ end +if not modules then modules = { } end modules ['l-string'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} -function string:splitter(pat) - -- by Rici Lake (posted on lua list) -- only names changed - -- p 79 ref man: () returns position of match - local st, g = 1, self:gmatch("()("..pat..")") - local function strgetter(self, segs, seps, sep, cap1, ...) - st = sep and seps + #sep - return self:sub(segs, (seps or 0) - 1), cap1 or sep, ... - end - local function strsplitter(self) - if st then return strgetter(self, st, g()) end - end - return strsplitter, self -end +local sub, gsub, find, match, gmatch, format, char, byte, rep, lower = string.sub, string.gsub, string.find, string.match, string.gmatch, string.format, string.char, string.byte, string.rep, string.lower +local lpegmatch = lpeg.match -function string:split(separator) - local t = {} - for k in self:splitter(separator) do t[#t+1] = k end - return t -end +-- some functions may disappear as they are not used anywhere --- faster than a string:split: +if not string.split then -function string:splitchr(chr) - if #self > 0 then - local t = { } - for s in string.gmatch(self..chr,"(.-)"..chr) do - t[#t+1] = s + -- this will be overloaded by a faster lpeg variant + + function string:split(pattern) + if #self > 0 then + local t = { } + for s in gmatch(self..pattern,"(.-)"..pattern) do + t[#t+1] = s + end + return t + else + return { } end - return t - else - return { } end -end ---~ function string.piecewise(str, pat, fnc) -- variant of split ---~ local fpat = "(.-)"..pat ---~ local last_end = 1 ---~ local s, e, cap = string.find(str, fpat, 1) ---~ while s ~= nil do ---~ if s~=1 or cap~="" then ---~ fnc(cap) ---~ end ---~ last_end = e+1 ---~ s, e, cap = string.find(str, fpat, last_end) ---~ end ---~ if last_end <= #str then ---~ fnc((string.sub(str,last_end))) ---~ end ---~ end - -function string.piecewise(str, pat, fnc) -- variant of split - for k in string.splitter(str,pat) do fnc(k) end end ---~ function string.piecewise(str, pat, fnc) -- variant of split ---~ for k in str:splitter(pat) do fnc(k) end ---~ end - ---~ do if lpeg then - ---~ -- this alternative is 30% faster esp when we cache them ---~ -- problem: no expressions - ---~ splitters = { } - ---~ function string:split(separator) ---~ if #self > 0 then ---~ local split = splitters[separator] ---~ if not split then ---~ -- based on code by Roberto ---~ local p = lpeg.P(separator) ---~ local c = lpeg.C((1-p)^0) ---~ split = lpeg.Ct(c*(p*c)^0) ---~ splitters[separator] = split ---~ end ---~ return split:match(self) ---~ else ---~ return { } ---~ end ---~ end - ---~ string.splitchr = string.split - ---~ function string:piecewise(separator,fnc) ---~ for _,v in pairs(self:split(separator)) do ---~ fnc(v) ---~ end ---~ end - ---~ end end - -string.chr_to_esc = { +local chr_to_esc = { ["%"] = "%%", ["."] = "%.", ["+"] = "%+", ["-"] = "%-", ["*"] = "%*", @@ -201,21 +87,31 @@ string.chr_to_esc = { ["{"] = "%{", ["}"] = "%}" } +string.chr_to_esc = chr_to_esc + function string:esc() -- variant 2 - return (self:gsub("(.)",string.chr_to_esc)) + return (gsub(self,"(.)",chr_to_esc)) end -function string.unquote(str) - return (str:gsub("^([\"\'])(.*)%1$","%2")) +function string:unquote() + return (gsub(self,"^([\"\'])(.*)%1$","%2")) end -function string.quote(str) - return '"' .. str:unquote() .. '"' +--~ function string:unquote() +--~ if find(self,"^[\'\"]") then +--~ return sub(self,2,-2) +--~ else +--~ return self +--~ end +--~ end + +function string:quote() -- we could use format("%q") + return format("%q",self) end function string:count(pattern) -- variant 3 local n = 0 - for _ in self:gmatch(pattern) do + for _ in gmatch(self,pattern) do n = n + 1 end return n @@ -224,29 +120,36 @@ end function string:limit(n,sentinel) if #self > n then sentinel = sentinel or " ..." - return self:sub(1,(n-#sentinel)) .. sentinel + return sub(self,1,(n-#sentinel)) .. sentinel else return self end end -function string:strip() - return (self:gsub("^%s*(.-)%s*$", "%1")) -end - ---~ function string.strip(str) -- slightly different ---~ return (string.gsub(string.gsub(str,"^%s*(.-)%s*$","%1"),"%s+"," ")) +--~ function string:strip() -- the .- is quite efficient +--~ -- return match(self,"^%s*(.-)%s*$") or "" +--~ -- return match(self,'^%s*(.*%S)') or '' -- posted on lua list +--~ return find(s,'^%s*$') and '' or match(s,'^%s*(.*%S)') --~ end +do -- roberto's variant: + local space = lpeg.S(" \t\v\n") + local nospace = 1 - space + local stripper = space^0 * lpeg.C((space^0 * nospace^1)^0) + function string.strip(str) + return lpegmatch(stripper,str) or "" + end +end + function string:is_empty() - return not self:find("%S") + return not find(self,"%S") end function string:enhance(pattern,action) local ok, n = true, 0 while ok do ok = false - self = self:gsub(pattern, function(...) + self = gsub(self,pattern, function(...) ok, n = true, n + 1 return action(...) end) @@ -254,73 +157,33 @@ function string:enhance(pattern,action) return self, n end ---~ function string:enhance(pattern,action) ---~ local ok, n = 0, 0 ---~ repeat ---~ self, ok = self:gsub(pattern, function(...) ---~ n = n + 1 ---~ return action(...) ---~ end) ---~ until ok == 0 ---~ return self, n ---~ end - ---~ function string:to_hex() ---~ if self then ---~ return (self:gsub("(.)",function(c) ---~ return string.format("%02X",c:byte()) ---~ end)) ---~ else ---~ return "" ---~ end ---~ end - ---~ function string:from_hex() ---~ if self then ---~ return (self:gsub("(..)",function(c) ---~ return string.char(tonumber(c,16)) ---~ end)) ---~ else ---~ return "" ---~ end ---~ end - -string.chr_to_hex = { } -string.hex_to_chr = { } +local chr_to_hex, hex_to_chr = { }, { } for i=0,255 do - local c, h = string.char(i), string.format("%02X",i) - string.chr_to_hex[c], string.hex_to_chr[h] = h, c + local c, h = char(i), format("%02X",i) + chr_to_hex[c], hex_to_chr[h] = h, c end ---~ function string:to_hex() ---~ if self then return (self:gsub("(.)",string.chr_to_hex)) else return "" end ---~ end - ---~ function string:from_hex() ---~ if self then return (self:gsub("(..)",string.hex_to_chr)) else return "" end ---~ end - function string:to_hex() - return ((self or ""):gsub("(.)",string.chr_to_hex)) + return (gsub(self or "","(.)",chr_to_hex)) end function string:from_hex() - return ((self or ""):gsub("(..)",string.hex_to_chr)) + return (gsub(self or "","(..)",hex_to_chr)) end if not string.characters then local function nextchar(str, index) index = index + 1 - return (index <= #str) and index or nil, str:sub(index,index) + return (index <= #str) and index or nil, sub(str,index,index) end function string:characters() return nextchar, self, 0 end local function nextbyte(str, index) index = index + 1 - return (index <= #str) and index or nil, string.byte(str:sub(index,index)) + return (index <= #str) and index or nil, byte(sub(str,index,index)) end function string:bytes() return nextbyte, self, 0 @@ -328,14 +191,12 @@ if not string.characters then end ---~ function string:padd(n,chr) ---~ return self .. self.rep(chr or " ",n-#self) ---~ end +-- we can use format for this (neg n) function string:rpadd(n,chr) local m = n-#self if m > 0 then - return self .. self.rep(chr or " ",m) + return self .. rep(chr or " ",m) else return self end @@ -344,7 +205,7 @@ end function string:lpadd(n,chr) local m = n-#self if m > 0 then - return self.rep(chr or " ",m) .. self + return rep(chr or " ",m) .. self else return self end @@ -352,8 +213,8 @@ end string.padd = string.rpadd -function is_number(str) - return str:find("^[%-%+]?[%d]-%.?[%d+]$") == 1 +function is_number(str) -- tonumber + return find(str,"^[%-%+]?[%d]-%.?[%d+]$") == 1 end --~ print(is_number("1")) @@ -365,9 +226,9 @@ end --~ print(is_number("+.1")) function string:split_settings() -- no {} handling, see l-aux for lpeg variant - if self:find("=") then + if find(self,"=") then local t = { } - for k,v in self:gmatch("(%a+)=([^%,]*)") do + for k,v in gmatch(self,"(%a+)=([^%,]*)") do t[k] = v end return t @@ -389,93 +250,335 @@ local patterns_escapes = { } function string:pattesc() - return (self:gsub(".",patterns_escapes)) + return (gsub(self,".",patterns_escapes)) +end + +local simple_escapes = { + ["-"] = "%-", + ["."] = "%.", + ["?"] = ".", + ["*"] = ".*", +} + +function string:simpleesc() + return (gsub(self,".",simple_escapes)) end function string:tohash() local t = { } - for s in self:gmatch("([^, ]+)") do -- lpeg + for s in gmatch(self,"([^, ]+)") do -- lpeg t[s] = true end return t end +local pattern = lpeg.Ct(lpeg.C(1)^0) --- filename : l-lpeg.lua --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +function string:totable() + return lpegmatch(pattern,self) +end + +--~ local t = { +--~ "1234567123456712345671234567", +--~ "a\tb\tc", +--~ "aa\tbb\tcc", +--~ "aaa\tbbb\tccc", +--~ "aaaa\tbbbb\tcccc", +--~ "aaaaa\tbbbbb\tccccc", +--~ "aaaaaa\tbbbbbb\tcccccc", +--~ } +--~ for k,v do +--~ print(string.tabtospace(t[k])) +--~ end + +function string.tabtospace(str,tab) + -- we don't handle embedded newlines + while true do + local s = find(str,"\t") + if s then + if not tab then tab = 7 end -- only when found + local d = tab-(s-1) % tab + if d > 0 then + str = gsub(str,"\t",rep(" ",d),1) + else + str = gsub(str,"\t","",1) + end + else + break + end + end + return str +end -if not versions then versions = { } end versions['l-lpeg'] = 1.001 +function string:compactlong() -- strips newlines and leading spaces + self = gsub(self,"[\n\r]+ *","") + self = gsub(self,"^ *","") + return self +end ---~ l-lpeg.lua : +function string:striplong() -- strips newlines and leading spaces + self = gsub(self,"^%s*","") + self = gsub(self,"[\n\r]+ *","\n") + return self +end ---~ lpeg.digit = lpeg.R('09')^1 ---~ lpeg.sign = lpeg.S('+-')^1 ---~ lpeg.cardinal = lpeg.P(lpeg.sign^0 * lpeg.digit^1) ---~ lpeg.integer = lpeg.P(lpeg.sign^0 * lpeg.digit^1) ---~ lpeg.float = lpeg.P(lpeg.sign^0 * lpeg.digit^0 * lpeg.P('.') * lpeg.digit^1) ---~ lpeg.number = lpeg.float + lpeg.integer ---~ lpeg.oct = lpeg.P("0") * lpeg.R('07')^1 ---~ lpeg.hex = lpeg.P("0x") * (lpeg.R('09') + lpeg.R('AF'))^1 ---~ lpeg.uppercase = lpeg.P("AZ") ---~ lpeg.lowercase = lpeg.P("az") +function string:topattern(lowercase,strict) + if lowercase then + self = lower(self) + end + self = gsub(self,".",simple_escapes) + if self == "" then + self = ".*" + elseif strict then + self = "^" .. self .. "$" + end + return self +end ---~ lpeg.eol = lpeg.S('\r\n\f')^1 -- includes formfeed ---~ lpeg.space = lpeg.S(' ')^1 ---~ lpeg.nonspace = lpeg.P(1-lpeg.space)^1 ---~ lpeg.whitespace = lpeg.S(' \r\n\f\t')^1 ---~ lpeg.nonwhitespace = lpeg.P(1-lpeg.whitespace)^1 -local hash = { } +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['l-lpeg'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local lpeg = require("lpeg") + +lpeg.patterns = lpeg.patterns or { } -- so that we can share +local patterns = lpeg.patterns + +local P, R, S, Ct, C, Cs, Cc, V = lpeg.P, lpeg.R, lpeg.S, lpeg.Ct, lpeg.C, lpeg.Cs, lpeg.Cc, lpeg.V +local match = lpeg.match + +local digit, sign = R('09'), S('+-') +local cr, lf, crlf = P("\r"), P("\n"), P("\r\n") +local utf8byte = R("\128\191") + +patterns.utf8byte = utf8byte +patterns.utf8one = R("\000\127") +patterns.utf8two = R("\194\223") * utf8byte +patterns.utf8three = R("\224\239") * utf8byte * utf8byte +patterns.utf8four = R("\240\244") * utf8byte * utf8byte * utf8byte + +patterns.digit = digit +patterns.sign = sign +patterns.cardinal = sign^0 * digit^1 +patterns.integer = sign^0 * digit^1 +patterns.float = sign^0 * digit^0 * P('.') * digit^1 +patterns.number = patterns.float + patterns.integer +patterns.oct = P("0") * R("07")^1 +patterns.octal = patterns.oct +patterns.HEX = P("0x") * R("09","AF")^1 +patterns.hex = P("0x") * R("09","af")^1 +patterns.hexadecimal = P("0x") * R("09","AF","af")^1 +patterns.lowercase = R("az") +patterns.uppercase = R("AZ") +patterns.letter = patterns.lowercase + patterns.uppercase +patterns.space = S(" ") +patterns.eol = S("\n\r") +patterns.spacer = S(" \t\f\v") -- + string.char(0xc2, 0xa0) if we want utf (cf mail roberto) +patterns.newline = crlf + cr + lf +patterns.nonspace = 1 - patterns.space +patterns.nonspacer = 1 - patterns.spacer +patterns.whitespace = patterns.eol + patterns.spacer +patterns.nonwhitespace = 1 - patterns.whitespace +patterns.utf8 = patterns.utf8one + patterns.utf8two + patterns.utf8three + patterns.utf8four +patterns.utfbom = P('\000\000\254\255') + P('\255\254\000\000') + P('\255\254') + P('\254\255') + P('\239\187\191') function lpeg.anywhere(pattern) --slightly adapted from website - return lpeg.P { lpeg.P(pattern) + 1 * lpeg.V(1) } + return P { P(pattern) + 1 * V(1) } -- why so complex? end -function lpeg.startswith(pattern) --slightly adapted - return lpeg.P(pattern) +function lpeg.splitter(pattern, action) + return (((1-P(pattern))^1)/action+1)^0 end ---~ g = lpeg.splitter(" ",function(s) ... end) -- gmatch:lpeg = 3:2 +local spacing = patterns.spacer^0 * patterns.newline -- sort of strip +local empty = spacing * Cc("") +local nonempty = Cs((1-spacing)^1) * spacing^-1 +local content = (empty + nonempty)^1 -function lpeg.splitter(pattern, action) - return (((1-lpeg.P(pattern))^1)/action+1)^0 +local capture = Ct(content^0) + +function string:splitlines() + return match(capture,self) end -local crlf = lpeg.P("\r\n") -local cr = lpeg.P("\r") -local lf = lpeg.P("\n") -local space = lpeg.S(" \t\f\v") -local newline = crlf + cr + lf -local spacing = space^0 * newline +patterns.textline = content -local empty = spacing * lpeg.Cc("") -local nonempty = lpeg.Cs((1-spacing)^1) * spacing^-1 -local content = (empty + nonempty)^1 +--~ local p = lpeg.splitat("->",false) print(match(p,"oeps->what->more")) -- oeps what more +--~ local p = lpeg.splitat("->",true) print(match(p,"oeps->what->more")) -- oeps what->more +--~ local p = lpeg.splitat("->",false) print(match(p,"oeps")) -- oeps +--~ local p = lpeg.splitat("->",true) print(match(p,"oeps")) -- oeps -local capture = lpeg.Ct(content^0) +local splitters_s, splitters_m = { }, { } -function string:splitlines() - return capture:match(self) +local function splitat(separator,single) + local splitter = (single and splitters_s[separator]) or splitters_m[separator] + if not splitter then + separator = P(separator) + if single then + local other, any = C((1 - separator)^0), P(1) + splitter = other * (separator * C(any^0) + "") -- ? + splitters_s[separator] = splitter + else + local other = C((1 - separator)^0) + splitter = other * (separator * other)^0 + splitters_m[separator] = splitter + end + end + return splitter end +lpeg.splitat = splitat --- filename : l-table.lua --- comment : split off from luat-lib --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +local cache = { } -if not versions then versions = { } end versions['l-table'] = 1.001 +function lpeg.split(separator,str) + local c = cache[separator] + if not c then + c = Ct(splitat(separator)) + cache[separator] = c + end + return match(c,str) +end + +function string:split(separator) + local c = cache[separator] + if not c then + c = Ct(splitat(separator)) + cache[separator] = c + end + return match(c,self) +end + +lpeg.splitters = cache + +local cache = { } + +function lpeg.checkedsplit(separator,str) + local c = cache[separator] + if not c then + separator = P(separator) + local other = C((1 - separator)^0) + c = Ct(separator^0 * other * (separator^1 * other)^0) + cache[separator] = c + end + return match(c,str) +end + +function string:checkedsplit(separator) + local c = cache[separator] + if not c then + separator = P(separator) + local other = C((1 - separator)^0) + c = Ct(separator^0 * other * (separator^1 * other)^0) + cache[separator] = c + end + return match(c,self) +end + +--~ function lpeg.append(list,pp) +--~ local p = pp +--~ for l=1,#list do +--~ if p then +--~ p = p + P(list[l]) +--~ else +--~ p = P(list[l]) +--~ end +--~ end +--~ return p +--~ end + +--~ from roberto's site: + +local f1 = string.byte + +local function f2(s) local c1, c2 = f1(s,1,2) return c1 * 64 + c2 - 12416 end +local function f3(s) local c1, c2, c3 = f1(s,1,3) return (c1 * 64 + c2) * 64 + c3 - 925824 end +local function f4(s) local c1, c2, c3, c4 = f1(s,1,4) return ((c1 * 64 + c2) * 64 + c3) * 64 + c4 - 63447168 end + +patterns.utf8byte = patterns.utf8one/f1 + patterns.utf8two/f2 + patterns.utf8three/f3 + patterns.utf8four/f4 + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['l-table'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} table.join = table.concat +local concat, sort, insert, remove = table.concat, table.sort, table.insert, table.remove +local format, find, gsub, lower, dump, match = string.format, string.find, string.gsub, string.lower, string.dump, string.match +local getmetatable, setmetatable = getmetatable, setmetatable +local type, next, tostring, tonumber, ipairs = type, next, tostring, tonumber, ipairs + +-- Starting with version 5.2 Lua no longer provide ipairs, which makes +-- sense. As we already used the for loop and # in most places the +-- impact on ConTeXt was not that large; the remaining ipairs already +-- have been replaced. In a similar fashio we also hardly used pairs. +-- +-- Just in case, we provide the fallbacks as discussed in Programming +-- in Lua (http://www.lua.org/pil/7.3.html): + +if not ipairs then + + -- for k, v in ipairs(t) do ... end + -- for k=1,#t do local v = t[k] ... end + + local function iterate(a,i) + i = i + 1 + local v = a[i] + if v ~= nil then + return i, v --, nil + end + end + + function ipairs(a) + return iterate, a, 0 + end + +end + +if not pairs then + + -- for k, v in pairs(t) do ... end + -- for k, v in next, t do ... end + + function pairs(t) + return next, t -- , nil + end + +end + +-- Also, unpack has been moved to the table table, and for compatiility +-- reasons we provide both now. + +if not table.unpack then + table.unpack = _G.unpack +elseif not unpack then + _G.unpack = table.unpack +end + +-- extra functions, some might go (when not used) + function table.strip(tab) local lst = { } - for k, v in ipairs(tab) do - -- s = string.gsub(v, "^%s*(.-)%s*$", "%1") - s = v:gsub("^%s*(.-)%s*$", "%1") + for i=1,#tab do + local s = gsub(tab[i],"^%s*(.-)%s*$","%1") if s == "" then -- skip this one else @@ -485,18 +588,21 @@ function table.strip(tab) return lst end ---~ function table.sortedkeys(tab) ---~ local srt = { } ---~ for key,_ in pairs(tab) do ---~ srt[#srt+1] = key ---~ end ---~ table.sort(srt) ---~ return srt ---~ end +function table.keys(t) + local k = { } + for key, _ in next, t do + k[#k+1] = key + end + return k +end -function table.sortedkeys(tab) +local function compare(a,b) + return (tostring(a) < tostring(b)) +end + +local function sortedkeys(tab) local srt, kind = { }, 0 -- 0=unknown 1=string, 2=number 3=mixed - for key,_ in pairs(tab) do + for key,_ in next, tab do srt[#srt+1] = key if kind == 3 then -- no further check @@ -514,22 +620,47 @@ function table.sortedkeys(tab) end end if kind == 0 or kind == 3 then - table.sort(srt,function(a,b) return (tostring(a) < tostring(b)) end) + sort(srt,compare) else - table.sort(srt) + sort(srt) end return srt end +local function sortedhashkeys(tab) -- fast one + local srt = { } + for key,_ in next, tab do + srt[#srt+1] = key + end + sort(srt) + return srt +end + +table.sortedkeys = sortedkeys +table.sortedhashkeys = sortedhashkeys + +function table.sortedhash(t) + local s = sortedhashkeys(t) -- maybe just sortedkeys + local n = 0 + local function kv(s) + n = n + 1 + local k = s[n] + return k, t[k] + end + return kv, s +end + +table.sortedpairs = table.sortedhash + function table.append(t, list) - for _,v in pairs(list) do - table.insert(t,v) + for _,v in next, list do + insert(t,v) end end function table.prepend(t, list) - for k,v in pairs(list) do - table.insert(t,k,v) + for k,v in next, list do + insert(t,k,v) end end @@ -537,7 +668,7 @@ function table.merge(t, ...) -- first one is target t = t or {} local lst = {...} for i=1,#lst do - for k, v in pairs(lst[i]) do + for k, v in next, lst[i] do t[k] = v end end @@ -547,7 +678,7 @@ end function table.merged(...) local tmp, lst = { }, {...} for i=1,#lst do - for k, v in pairs(lst[i]) do + for k, v in next, lst[i] do tmp[k] = v end end @@ -576,436 +707,584 @@ function table.imerged(...) return tmp end -if not table.fastcopy then do - - local type, pairs, getmetatable, setmetatable = type, pairs, getmetatable, setmetatable - - local function fastcopy(old) -- fast one - if old then - local new = { } - for k,v in pairs(old) do - if type(v) == "table" then - new[k] = fastcopy(v) -- was just table.copy - else - new[k] = v - end - end - local mt = getmetatable(old) - if mt then - setmetatable(new,mt) +local function fastcopy(old) -- fast one + if old then + local new = { } + for k,v in next, old do + if type(v) == "table" then + new[k] = fastcopy(v) -- was just table.copy + else + new[k] = v end - return new - else - return { } end + -- optional second arg + local mt = getmetatable(old) + if mt then + setmetatable(new,mt) + end + return new + else + return { } end +end - table.fastcopy = fastcopy - -end end - -if not table.copy then do - - local type, pairs, getmetatable, setmetatable = type, pairs, getmetatable, setmetatable - - local function copy(t, tables) -- taken from lua wiki, slightly adapted - tables = tables or { } - local tcopy = {} - if not tables[t] then - tables[t] = tcopy - end - for i,v in pairs(t) do -- brrr, what happens with sparse indexed - if type(i) == "table" then - if tables[i] then - i = tables[i] - else - i = copy(i, tables) - end - end - if type(v) ~= "table" then - tcopy[i] = v - elseif tables[v] then - tcopy[i] = tables[v] +local function copy(t, tables) -- taken from lua wiki, slightly adapted + tables = tables or { } + local tcopy = {} + if not tables[t] then + tables[t] = tcopy + end + for i,v in next, t do -- brrr, what happens with sparse indexed + if type(i) == "table" then + if tables[i] then + i = tables[i] else - tcopy[i] = copy(v, tables) + i = copy(i, tables) end end - local mt = getmetatable(t) - if mt then - setmetatable(tcopy,mt) + if type(v) ~= "table" then + tcopy[i] = v + elseif tables[v] then + tcopy[i] = tables[v] + else + tcopy[i] = copy(v, tables) end - return tcopy end + local mt = getmetatable(t) + if mt then + setmetatable(tcopy,mt) + end + return tcopy +end - table.copy = copy - -end end +table.fastcopy = fastcopy +table.copy = copy --- rougly: copy-loop : unpack : sub == 0.9 : 0.4 : 0.45 (so in critical apps, use unpack) +-- roughly: copy-loop : unpack : sub == 0.9 : 0.4 : 0.45 (so in critical apps, use unpack) function table.sub(t,i,j) return { unpack(t,i,j) } end function table.replace(a,b) - for k,v in pairs(b) do + for k,v in next, b do a[k] = v end end -- slower than #t on indexed tables (#t only returns the size of the numerically indexed slice) -function table.is_empty(t) +function table.is_empty(t) -- obolete, use inline code instead return not t or not next(t) end -function table.one_entry(t) +function table.one_entry(t) -- obolete, use inline code instead local n = next(t) return n and not next(t,n) end -function table.starts_at(t) - return ipairs(t,1)(t,0) -end +--~ function table.starts_at(t) -- obsolete, not nice anyway +--~ return ipairs(t,1)(t,0) +--~ end -do +function table.tohash(t,value) + local h = { } + if t then + if value == nil then value = true end + for _, v in next, t do -- no ipairs here + h[v] = value + end + end + return h +end - -- one of my first exercises in lua ... +function table.fromhash(t) + local h = { } + for k, v in next, t do -- no ipairs here + if v then h[#h+1] = k end + end + return h +end - -- 34.055.092 32.403.326 arabtype.tma - -- 1.620.614 1.513.863 lmroman10-italic.tma - -- 1.325.585 1.233.044 lmroman10-regular.tma - -- 1.248.157 1.158.903 lmsans10-regular.tma - -- 194.646 153.120 lmtypewriter10-regular.tma - -- 1.771.678 1.658.461 palatinosanscom-bold.tma - -- 1.695.251 1.584.491 palatinosanscom-regular.tma - -- 13.736.534 13.409.446 zapfinoextraltpro.tma +--~ print(table.serialize(t), "\n") +--~ print(table.serialize(t,"name"), "\n") +--~ print(table.serialize(t,false), "\n") +--~ print(table.serialize(t,true), "\n") +--~ print(table.serialize(t,"name",true), "\n") +--~ print(table.serialize(t,"name",true,true), "\n") - -- 13.679.038 11.774.106 arabtype.tmc - -- 886.248 754.944 lmroman10-italic.tmc - -- 729.828 466.864 lmroman10-regular.tmc - -- 688.482 441.962 lmsans10-regular.tmc - -- 128.685 95.853 lmtypewriter10-regular.tmc - -- 715.929 582.985 palatinosanscom-bold.tmc - -- 669.942 540.126 palatinosanscom-regular.tmc - -- 1.560.588 1.317.000 zapfinoextraltpro.tmc +table.serialize_functions = true +table.serialize_compact = true +table.serialize_inline = true - table.serialize_functions = true - table.serialize_compact = true - table.serialize_inline = true +local noquotes, hexify, handle, reduce, compact, inline, functions - local function key(k) - if type(k) == "number" then -- or k:find("^%d+$") then - return "["..k.."]" - elseif noquotes and k:find("^%a[%a%d%_]*$") then - return k - else - return '["'..k..'"]' - end - end +local reserved = table.tohash { -- intercept a language flaw, no reserved words as key + 'and', 'break', 'do', 'else', 'elseif', 'end', 'false', 'for', 'function', 'if', + 'in', 'local', 'nil', 'not', 'or', 'repeat', 'return', 'then', 'true', 'until', 'while', +} - local function simple_table(t) - if #t > 0 then - local n = 0 - for _,v in pairs(t) do - n = n + 1 - end - if n == #t then - local tt = { } - for i=1,#t do - local v = t[i] - local tv = type(v) - if tv == "number" or tv == "boolean" then - tt[#tt+1] = tostring(v) - elseif tv == "string" then - tt[#tt+1] = ("%q"):format(v) +local function simple_table(t) + if #t > 0 then + local n = 0 + for _,v in next, t do + n = n + 1 + end + if n == #t then + local tt = { } + for i=1,#t do + local v = t[i] + local tv = type(v) + if tv == "number" then + if hexify then + tt[#tt+1] = format("0x%04X",v) else - tt = nil - break + tt[#tt+1] = tostring(v) -- tostring not needed end + elseif tv == "boolean" then + tt[#tt+1] = tostring(v) + elseif tv == "string" then + tt[#tt+1] = format("%q",v) + else + tt = nil + break end - return tt end + return tt end - return nil end + return nil +end - local function serialize(root,name,handle,depth,level,reduce,noquotes,indexed) - handle = handle or print - reduce = reduce or false - if depth then - depth = depth .. " " - if indexed then - handle(("%s{"):format(depth)) +-- Because this is a core function of mkiv I moved some function calls +-- inline. +-- +-- twice as fast in a test: +-- +-- local propername = lpeg.P(lpeg.R("AZ","az","__") * lpeg.R("09","AZ","az", "__")^0 * lpeg.P(-1) ) + +-- problem: there no good number_to_string converter with the best resolution + +local function do_serialize(root,name,depth,level,indexed) + if level > 0 then + depth = depth .. " " + if indexed then + handle(format("%s{",depth)) + elseif name then + --~ handle(format("%s%s={",depth,key(name))) + if type(name) == "number" then -- or find(k,"^%d+$") then + if hexify then + handle(format("%s[0x%04X]={",depth,name)) + else + handle(format("%s[%s]={",depth,name)) + end + elseif noquotes and not reserved[name] and find(name,"^%a[%w%_]*$") then + handle(format("%s%s={",depth,name)) else - handle(("%s%s={"):format(depth,key(name))) + handle(format("%s[%q]={",depth,name)) end else - depth = "" - local tname = type(name) - if tname == "string" then - if name == "return" then - handle("return {") + handle(format("%s{",depth)) + end + end + -- we could check for k (index) being number (cardinal) + if root and next(root) then + local first, last = nil, 0 -- #root cannot be trusted here (will be ok in 5.2 when ipairs is gone) + if compact then + -- NOT: for k=1,#root do (we need to quit at nil) + for k,v in ipairs(root) do -- can we use next? + if not first then first = k end + last = last + 1 + end + end + local sk = sortedkeys(root) + for i=1,#sk do + local k = sk[i] + local v = root[k] + --~ if v == root then + -- circular + --~ else + local t = type(v) + if compact and first and type(k) == "number" and k >= first and k <= last then + if t == "number" then + if hexify then + handle(format("%s 0x%04X,",depth,v)) + else + handle(format("%s %s,",depth,v)) -- %.99g + end + elseif t == "string" then + if reduce and tonumber(v) then + handle(format("%s %s,",depth,v)) + else + handle(format("%s %q,",depth,v)) + end + elseif t == "table" then + if not next(v) then + handle(format("%s {},",depth)) + elseif inline then -- and #t > 0 + local st = simple_table(v) + if st then + handle(format("%s { %s },",depth,concat(st,", "))) + else + do_serialize(v,k,depth,level+1,true) + end + else + do_serialize(v,k,depth,level+1,true) + end + elseif t == "boolean" then + handle(format("%s %s,",depth,tostring(v))) + elseif t == "function" then + if functions then + handle(format('%s loadstring(%q),',depth,dump(v))) + else + handle(format('%s "function",',depth)) + end else - handle(name .. "={") + handle(format("%s %q,",depth,tostring(v))) + end + elseif k == "__p__" then -- parent + if false then + handle(format("%s __p__=nil,",depth)) end - elseif tname == "number" then - handle("[" .. name .. "]={") - elseif tname == "boolean" then - if name then - handle("return {") + elseif t == "number" then + --~ if hexify then + --~ handle(format("%s %s=0x%04X,",depth,key(k),v)) + --~ else + --~ handle(format("%s %s=%s,",depth,key(k),v)) -- %.99g + --~ end + if type(k) == "number" then -- or find(k,"^%d+$") then + if hexify then + handle(format("%s [0x%04X]=0x%04X,",depth,k,v)) + else + handle(format("%s [%s]=%s,",depth,k,v)) -- %.99g + end + elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then + if hexify then + handle(format("%s %s=0x%04X,",depth,k,v)) + else + handle(format("%s %s=%s,",depth,k,v)) -- %.99g + end else - handle("{") + if hexify then + handle(format("%s [%q]=0x%04X,",depth,k,v)) + else + handle(format("%s [%q]=%s,",depth,k,v)) -- %.99g + end end - else - handle("t={") - end - end - if root and next(root) then - local compact = table.serialize_compact - local inline = compact and table.serialize_inline - local first, last = nil, 0 -- #root cannot be trusted here - if compact then - for k,v in ipairs(root) do -- NOT: for k=1,#root do (why) - if not first then first = k end - last = last + 1 - end - end - for _,k in pairs(table.sortedkeys(root)) do - local v = root[k] - local t = type(v) - if compact and first and type(k) == "number" and k >= first and k <= last then - if t == "number" then - handle(("%s %s,"):format(depth,v)) - elseif t == "string" then - if reduce and (v:find("^[%-%+]?[%d]-%.?[%d+]$") == 1) then - handle(("%s %s,"):format(depth,v)) + elseif t == "string" then + if reduce and tonumber(v) then + --~ handle(format("%s %s=%s,",depth,key(k),v)) + if type(k) == "number" then -- or find(k,"^%d+$") then + if hexify then + handle(format("%s [0x%04X]=%s,",depth,k,v)) else - handle(("%s %q,"):format(depth,v)) + handle(format("%s [%s]=%s,",depth,k,v)) end - elseif t == "table" then - if not next(v) then - handle(("%s {},"):format(depth)) - elseif inline then - local st = simple_table(v) - if st then - handle(("%s { %s },"):format(depth,table.concat(st,", "))) - else - serialize(v,k,handle,depth,level+1,reduce,noquotes,true) - end + elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then + handle(format("%s %s=%s,",depth,k,v)) + else + handle(format("%s [%q]=%s,",depth,k,v)) + end + else + --~ handle(format("%s %s=%q,",depth,key(k),v)) + if type(k) == "number" then -- or find(k,"^%d+$") then + if hexify then + handle(format("%s [0x%04X]=%q,",depth,k,v)) else - serialize(v,k,handle,depth,level+1,reduce,noquotes,true) + handle(format("%s [%s]=%q,",depth,k,v)) end - elseif t == "boolean" then - handle(("%s %s,"):format(depth,tostring(v))) - elseif t == "function" then - if table.serialize_functions then - handle(('%s loadstring(%q),'):format(depth,string.dump(v))) + elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then + handle(format("%s %s=%q,",depth,k,v)) + else + handle(format("%s [%q]=%q,",depth,k,v)) + end + end + elseif t == "table" then + if not next(v) then + --~ handle(format("%s %s={},",depth,key(k))) + if type(k) == "number" then -- or find(k,"^%d+$") then + if hexify then + handle(format("%s [0x%04X]={},",depth,k)) else - handle(('%s "function",'):format(depth)) + handle(format("%s [%s]={},",depth,k)) end + elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then + handle(format("%s %s={},",depth,k)) else - handle(("%s %q,"):format(depth,tostring(v))) + handle(format("%s [%q]={},",depth,k)) end - elseif k == "__p__" then -- parent - if false then - handle(("%s __p__=nil,"):format(depth)) + elseif inline then + local st = simple_table(v) + if st then + --~ handle(format("%s %s={ %s },",depth,key(k),concat(st,", "))) + if type(k) == "number" then -- or find(k,"^%d+$") then + if hexify then + handle(format("%s [0x%04X]={ %s },",depth,k,concat(st,", "))) + else + handle(format("%s [%s]={ %s },",depth,k,concat(st,", "))) + end + elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then + handle(format("%s %s={ %s },",depth,k,concat(st,", "))) + else + handle(format("%s [%q]={ %s },",depth,k,concat(st,", "))) + end + else + do_serialize(v,k,depth,level+1) end - elseif t == "number" then - handle(("%s %s=%s,"):format(depth,key(k),v)) - elseif t == "string" then - if reduce and (v:find("^[%-%+]?[%d]-%.?[%d+]$") == 1) then - handle(("%s %s=%s,"):format(depth,key(k),v)) + else + do_serialize(v,k,depth,level+1) + end + elseif t == "boolean" then + --~ handle(format("%s %s=%s,",depth,key(k),tostring(v))) + if type(k) == "number" then -- or find(k,"^%d+$") then + if hexify then + handle(format("%s [0x%04X]=%s,",depth,k,tostring(v))) else - handle(("%s %s=%q,"):format(depth,key(k),v)) + handle(format("%s [%s]=%s,",depth,k,tostring(v))) end - elseif t == "table" then - if not next(v) then - handle(("%s %s={},"):format(depth,key(k))) - elseif inline then - local st = simple_table(v) - if st then - handle(("%s %s={ %s },"):format(depth,key(k),table.concat(st,", "))) + elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then + handle(format("%s %s=%s,",depth,k,tostring(v))) + else + handle(format("%s [%q]=%s,",depth,k,tostring(v))) + end + elseif t == "function" then + if functions then + --~ handle(format('%s %s=loadstring(%q),',depth,key(k),dump(v))) + if type(k) == "number" then -- or find(k,"^%d+$") then + if hexify then + handle(format("%s [0x%04X]=loadstring(%q),",depth,k,dump(v))) else - serialize(v,k,handle,depth,level+1,reduce,noquotes) + handle(format("%s [%s]=loadstring(%q),",depth,k,dump(v))) end + elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then + handle(format("%s %s=loadstring(%q),",depth,k,dump(v))) else - serialize(v,k,handle,depth,level+1,reduce,noquotes) + handle(format("%s [%q]=loadstring(%q),",depth,k,dump(v))) end - elseif t == "boolean" then - handle(("%s %s=%s,"):format(depth,key(k),tostring(v))) - elseif t == "function" then - if table.serialize_functions then - handle(('%s %s=loadstring(%q),'):format(depth,key(k),string.dump(v))) + end + else + --~ handle(format("%s %s=%q,",depth,key(k),tostring(v))) + if type(k) == "number" then -- or find(k,"^%d+$") then + if hexify then + handle(format("%s [0x%04X]=%q,",depth,k,tostring(v))) else - handle(('%s %s="function",'):format(depth,key(k))) + handle(format("%s [%s]=%q,",depth,k,tostring(v))) end + elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then + handle(format("%s %s=%q,",depth,k,tostring(v))) else - handle(("%s %s=%q,"):format(depth,key(k),tostring(v))) - -- handle(('%s %s=loadstring(%q),'):format(depth,key(k),string.dump(function() return v end))) + handle(format("%s [%q]=%q,",depth,k,tostring(v))) end end - if level > 0 then - handle(("%s},"):format(depth)) - else - handle(("%s}"):format(depth)) - end - else - handle(("%s}"):format(depth)) + --~ end end end + if level > 0 then + handle(format("%s},",depth)) + end +end - --~ name: - --~ - --~ true : return { } - --~ false : { } - --~ nil : t = { } - --~ string : string = { } - --~ 'return' : return { } - --~ number : [number] = { } +-- replacing handle by a direct t[#t+1] = ... (plus test) is not much +-- faster (0.03 on 1.00 for zapfino.tma) - function table.serialize(root,name,reduce,noquotes) - local t = { } - local function flush(s) - t[#t+1] = s +local function serialize(root,name,_handle,_reduce,_noquotes,_hexify) + noquotes = _noquotes + hexify = _hexify + handle = _handle or print + reduce = _reduce or false + compact = table.serialize_compact + inline = compact and table.serialize_inline + functions = table.serialize_functions + local tname = type(name) + if tname == "string" then + if name == "return" then + handle("return {") + else + handle(name .. "={") end - serialize(root, name, flush, nil, 0, reduce, noquotes) - return table.concat(t,"\n") + elseif tname == "number" then + if hexify then + handle(format("[0x%04X]={",name)) + else + handle("[" .. name .. "]={") + end + elseif tname == "boolean" then + if name then + handle("return {") + else + handle("{") + end + else + handle("t={") + end + if root and next(root) then + do_serialize(root,name,"",0,indexed) end + handle("}") +end - function table.tohandle(handle,root,name,reduce,noquotes) - serialize(root, name, handle, nil, 0, reduce, noquotes) +--~ name: +--~ +--~ true : return { } +--~ false : { } +--~ nil : t = { } +--~ string : string = { } +--~ 'return' : return { } +--~ number : [number] = { } + +function table.serialize(root,name,reduce,noquotes,hexify) + local t = { } + local function flush(s) + t[#t+1] = s end + serialize(root,name,flush,reduce,noquotes,hexify) + return concat(t,"\n") +end - -- sometimes tables are real use (zapfino extra pro is some 85M) in which - -- case a stepwise serialization is nice; actually, we could consider: - -- - -- for line in table.serializer(root,name,reduce,noquotes) do - -- ...(line) - -- end - -- - -- so this is on the todo list +function table.tohandle(handle,root,name,reduce,noquotes,hexify) + serialize(root,name,handle,reduce,noquotes,hexify) +end - table.tofile_maxtab = 2*1024 +-- sometimes tables are real use (zapfino extra pro is some 85M) in which +-- case a stepwise serialization is nice; actually, we could consider: +-- +-- for line in table.serializer(root,name,reduce,noquotes) do +-- ...(line) +-- end +-- +-- so this is on the todo list - function table.tofile(filename,root,name,reduce,noquotes) - local f = io.open(filename,'w') - if f then - local concat = table.concat - local maxtab = table.tofile_maxtab - if maxtab > 1 then - local t = { } - local function flush(s) - t[#t+1] = s - if #t > maxtab then - f:write(concat(t,"\n"),"\n") -- hm, write(sometable) should be nice - t = { } - end - end - serialize(root, name, flush, nil, 0, reduce, noquotes) - f:write(concat(t,"\n"),"\n") - else - local function flush(s) - f:write(s,"\n") +table.tofile_maxtab = 2*1024 + +function table.tofile(filename,root,name,reduce,noquotes,hexify) + local f = io.open(filename,'w') + if f then + local maxtab = table.tofile_maxtab + if maxtab > 1 then + local t = { } + local function flush(s) + t[#t+1] = s + if #t > maxtab then + f:write(concat(t,"\n"),"\n") -- hm, write(sometable) should be nice + t = { } end - serialize(root, name, flush, nil, 0, reduce, noquotes) end - f:close() + serialize(root,name,flush,reduce,noquotes,hexify) + f:write(concat(t,"\n"),"\n") + else + local function flush(s) + f:write(s,"\n") + end + serialize(root,name,flush,reduce,noquotes,hexify) end + f:close() end - end ---~ t = { ---~ b = "123", ---~ a = "x", ---~ c = 1.23, ---~ d = "1.23", ---~ e = true, ---~ f = { ---~ d = "1.23", ---~ a = "x", ---~ b = "123", ---~ c = 1.23, ---~ e = true, ---~ f = { ---~ e = true, ---~ f = { ---~ e = true ---~ }, ---~ }, ---~ }, ---~ g = function() end ---~ } - ---~ print(table.serialize(t), "\n") ---~ print(table.serialize(t,"name"), "\n") ---~ print(table.serialize(t,false), "\n") ---~ print(table.serialize(t,true), "\n") ---~ print(table.serialize(t,"name",true), "\n") ---~ print(table.serialize(t,"name",true,true), "\n") - -do - - local function flatten(t,f,complete) - for i=1,#t do - local v = t[i] - if type(v) == "table" then - if complete or type(v[1]) == "table" then - flatten(v,f,complete) - else - f[#f+1] = v - end +local function flatten(t,f,complete) -- is this used? meybe a variant with next, ... + for i=1,#t do + local v = t[i] + if type(v) == "table" then + if complete or type(v[1]) == "table" then + flatten(v,f,complete) else f[#f+1] = v end + else + f[#f+1] = v end end +end - function table.flatten(t) - local f = { } - flatten(t,f,true) - return f - end +function table.flatten(t) + local f = { } + flatten(t,f,true) + return f +end - function table.unnest(t) -- bad name - local f = { } - flatten(t,f,false) - return f +function table.unnest(t) -- bad name + local f = { } + flatten(t,f,false) + return f +end + +table.flatten_one_level = table.unnest + +-- a better one: + +local function flattened(t,f) + if not f then + f = { } + end + for k, v in next, t do + if type(v) == "table" then + flattened(v,f) + else + f[k] = v + end end + return f +end + +table.flattened = flattened - table.flatten_one_level = table.unnest +-- the next three may disappear +function table.remove_value(t,value) -- todo: n + if value then + for i=1,#t do + if t[i] == value then + remove(t,i) + -- remove all, so no: return + end + end + end end function table.insert_before_value(t,value,str) - for i=1,#t do - if t[i] == value then - table.insert(t,i,str) - return + if str then + if value then + for i=1,#t do + if t[i] == value then + insert(t,i,str) + return + end + end end + insert(t,1,str) + elseif value then + insert(t,1,value) end - table.insert(t,1,str) end function table.insert_after_value(t,value,str) - for i=1,#t do - if t[i] == value then - table.insert(t,i+1,str) - return + if str then + if value then + for i=1,#t do + if t[i] == value then + insert(t,i+1,str) + return + end + end end + t[#t+1] = str + elseif value then + t[#t+1] = value end - t[#t+1] = str end -function table.are_equal(a,b,n,m) - if #a == #b then +local function are_equal(a,b,n,m) -- indexed + if a and b and #a == #b then n = n or 1 m = m or #a for i=n,m do local ai, bi = a[i], b[i] - if (ai==bi) or (type(ai)=="table" and type(bi)=="table" and table.are_equal(ai,bi)) then - -- continue + if ai==bi then + -- same + elseif type(ai)=="table" and type(bi)=="table" then + if not are_equal(ai,bi) then + return false + end else return false end @@ -1016,37 +1295,42 @@ function table.are_equal(a,b,n,m) end end -function table.compact(t) - if t then - for k,v in pairs(t) do - if not next(v) then - t[k] = nil +local function identical(a,b) -- assumes same structure + for ka, va in next, a do + local vb = b[k] + if va == vb then + -- same + elseif type(va) == "table" and type(vb) == "table" then + if not identical(va,vb) then + return false end + else + return false end end + return true end -function table.tohash(t) - local h = { } - for _, v in pairs(t) do -- no ipairs here - h[v] = true - end - return h -end +table.are_equal = are_equal +table.identical = identical -function table.fromhash(t) - local h = { } - for k, v in pairs(t) do -- no ipairs here - if v then h[#h+1] = k end +-- maybe also make a combined one + +function table.compact(t) + if t then + for k,v in next, t do + if not next(v) then + t[k] = nil + end + end end - return h end function table.contains(t, v) if t then for i=1, #t do if t[i] == v then - return true + return i end end end @@ -1063,7 +1347,7 @@ end function table.swapped(t) local s = { } - for k, v in pairs(t) do + for k, v in next, t do s[v] = k end return s @@ -1079,21 +1363,20 @@ function table.clone(t,p) -- t is optional or nil or table elseif not t then t = { } end - setmetatable(t, { __index = function(_,key) return p[key] end }) + setmetatable(t, { __index = function(_,key) return p[key] end }) -- why not __index = p ? return t end - function table.hexed(t,seperator) local tt = { } - for i=1,#t do tt[i] = string.format("0x%04X",t[i]) end - return table.concat(tt,seperator or " ") + for i=1,#t do tt[i] = format("0x%04X",t[i]) end + return concat(tt,seperator or " ") end function table.reverse_hash(h) local r = { } - for k,v in pairs(h) do - r[v] = (k:gsub(" ","")):lower() + for k,v in next, h do + r[v] = lower(gsub(k," ","")) end return r end @@ -1108,14 +1391,51 @@ function table.reverse(t) return tt end +function table.insert_before_value(t,value,extra) + for i=1,#t do + if t[i] == extra then + remove(t,i) + end + end + for i=1,#t do + if t[i] == value then + insert(t,i,extra) + return + end + end + insert(t,1,extra) +end + +function table.insert_after_value(t,value,extra) + for i=1,#t do + if t[i] == extra then + remove(t,i) + end + end + for i=1,#t do + if t[i] == value then + insert(t,i+1,extra) + return + end + end + insert(t,#t+1,extra) +end --- filename : l-io.lua --- comment : split off from luat-lib --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files -if not versions then versions = { } end versions['l-io'] = 1.001 + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['l-io'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local byte, find, gsub = string.byte, string.find, string.gsub if string.find(os.getenv("PATH"),";") then io.fileseparator, io.pathseparator = "\\", ";" @@ -1123,10 +1443,12 @@ else io.fileseparator, io.pathseparator = "/" , ":" end -function io.loaddata(filename) - local f = io.open(filename,'rb') +function io.loaddata(filename,textmode) + local f = io.open(filename,(textmode and 'r') or 'rb') if f then + -- collectgarbage("step") -- sometimes makes a big difference in mem consumption local data = f:read('*all') + -- garbagecollector.check(data) f:close() return data else @@ -1135,16 +1457,19 @@ function io.loaddata(filename) end function io.savedata(filename,data,joiner) - local f = io.open(filename, "wb") + local f = io.open(filename,"wb") if f then if type(data) == "table" then f:write(table.join(data,joiner or "")) elseif type(data) == "function" then data(f) else - f:write(data) + f:write(data or "") end f:close() + return true + else + return false end end @@ -1178,146 +1503,83 @@ function io.noflines(f) return n end -do - - local sb = string.byte - - local nextchar = { - [ 4] = function(f) - return f:read(1,1,1,1) - end, - [ 2] = function(f) - return f:read(1,1) - end, - [ 1] = function(f) - return f:read(1) - end, - [-2] = function(f) - local a, b = f:read(1,1) - return b, a - end, - [-4] = function(f) - local a, b, c, d = f:read(1,1,1,1) - return d, c, b, a - end - } - - function io.characters(f,n) - if f then - return nextchar[n or 1], f - else - return nil, nil - end +local nextchar = { + [ 4] = function(f) + return f:read(1,1,1,1) + end, + [ 2] = function(f) + return f:read(1,1) + end, + [ 1] = function(f) + return f:read(1) + end, + [-2] = function(f) + local a, b = f:read(1,1) + return b, a + end, + [-4] = function(f) + local a, b, c, d = f:read(1,1,1,1) + return d, c, b, a end +} +function io.characters(f,n) + if f then + return nextchar[n or 1], f + else + return nil, nil + end end -do - - local sb = string.byte - ---~ local nextbyte = { ---~ [4] = function(f) ---~ local a = f:read(1) ---~ local b = f:read(1) ---~ local c = f:read(1) ---~ local d = f:read(1) ---~ if d then ---~ return sb(a), sb(b), sb(c), sb(d) ---~ else ---~ return nil, nil, nil, nil ---~ end ---~ end, ---~ [2] = function(f) ---~ local a = f:read(1) ---~ local b = f:read(1) ---~ if b then ---~ return sb(a), sb(b) ---~ else ---~ return nil, nil ---~ end ---~ end, ---~ [1] = function (f) ---~ local a = f:read(1) ---~ if a then ---~ return sb(a) ---~ else ---~ return nil ---~ end ---~ end, ---~ [-2] = function (f) ---~ local a = f:read(1) ---~ local b = f:read(1) ---~ if b then ---~ return sb(b), sb(a) ---~ else ---~ return nil, nil ---~ end ---~ end, ---~ [-4] = function(f) ---~ local a = f:read(1) ---~ local b = f:read(1) ---~ local c = f:read(1) ---~ local d = f:read(1) ---~ if d then ---~ return sb(d), sb(c), sb(b), sb(a) ---~ else ---~ return nil, nil, nil, nil ---~ end ---~ end ---~ } - - local nextbyte = { - [4] = function(f) - local a, b, c, d = f:read(1,1,1,1) - if d then - return sb(a), sb(b), sb(c), sb(d) - else - return nil, nil, nil, nil - end - end, - [2] = function(f) - local a, b = f:read(1,1) - if b then - return sb(a), sb(b) - else - return nil, nil - end - end, - [1] = function (f) - local a = f:read(1) - if a then - return sb(a) - else - return nil - end - end, - [-2] = function (f) - local a, b = f:read(1,1) - if b then - return sb(b), sb(a) - else - return nil, nil - end - end, - [-4] = function(f) - local a, b, c, d = f:read(1,1,1,1) - if d then - return sb(d), sb(c), sb(b), sb(a) - else - return nil, nil, nil, nil - end +local nextbyte = { + [4] = function(f) + local a, b, c, d = f:read(1,1,1,1) + if d then + return byte(a), byte(b), byte(c), byte(d) + else + return nil, nil, nil, nil end - } - - function io.bytes(f,n) - if f then - return nextbyte[n or 1], f + end, + [2] = function(f) + local a, b = f:read(1,1) + if b then + return byte(a), byte(b) + else + return nil, nil + end + end, + [1] = function (f) + local a = f:read(1) + if a then + return byte(a) + else + return nil + end + end, + [-2] = function (f) + local a, b = f:read(1,1) + if b then + return byte(b), byte(a) else return nil, nil end + end, + [-4] = function(f) + local a, b, c, d = f:read(1,1,1,1) + if d then + return byte(d), byte(c), byte(b), byte(a) + else + return nil, nil, nil, nil + end end +} +function io.bytes(f,n) + if f then + return nextbyte[n or 1], f + else + return nil, nil + end end function io.ask(question,default,options) @@ -1331,20 +1593,21 @@ function io.ask(question,default,options) end io.write(string.format(" ")) local answer = io.read() - answer = answer:gsub("^%s*(.*)%s*$","%1") + answer = gsub(answer,"^%s*(.*)%s*$","%1") if answer == "" and default then return default elseif not options then return answer else - for _,v in pairs(options) do - if v == answer then + for k=1,#options do + if options[k] == answer then return answer end end local pattern = "^" .. answer - for _,v in pairs(options) do - if v:find(pattern) then + for k=1,#options do + local v = options[k] + if find(v,pattern) then return v end end @@ -1353,44 +1616,30 @@ function io.ask(question,default,options) end --- filename : l-md5.lua --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files - -if not versions then versions = { } end versions['l-md5'] = 1.001 - -if md5 then do - - local function convert(str,fmt) - return (string.gsub(md5.sum(str),".",function(chr) return string.format(fmt,string.byte(chr)) end)) - end - - if not md5.HEX then function md5.HEX(str) return convert(str,"%02X") end end - if not md5.hex then function md5.hex(str) return convert(str,"%02x") end end - if not md5.dec then function md5.dec(str) return convert(str,"%03i") end end - -end end +end -- of closure +do -- create closure to overcome 200 locals limit --- filename : l-number.lua --- comment : split off from luat-lib --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +if not modules then modules = { } end modules ['l-number'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} -if not versions then versions = { } end versions['l-number'] = 1.001 +local tostring = tostring +local format, floor, insert, match = string.format, math.floor, table.insert, string.match +local lpegmatch = lpeg.match -if not number then number = { } end +number = number or { } -- a,b,c,d,e,f = number.toset(100101) function number.toset(n) - return (tostring(n)):match("(.?)(.?)(.?)(.?)(.?)(.?)(.?)(.?)") + return match(tostring(n),"(.?)(.?)(.?)(.?)(.?)(.?)(.?)(.?)") end -local format = string.format - function number.toevenhex(n) local s = format("%X",n) if #s % 2 == 0 then @@ -1411,72 +1660,100 @@ end -- -- of course dedicated "(.)(.)(.)(.)" matches are even faster -do - local one = lpeg.C(1-lpeg.S(''))^1 +local one = lpeg.C(1-lpeg.S(''))^1 - function number.toset(n) - return one:match(tostring(n)) - end +function number.toset(n) + return lpegmatch(one,tostring(n)) end +function number.bits(n,zero) + local t, i = { }, (zero and 0) or 1 + while n > 0 do + local m = n % 2 + if m > 0 then + insert(t,1,i) + end + n = floor(n/2) + i = i + 1 + end + return t +end --- filename : l-set.lua --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +end -- of closure -if not versions then versions = { } end versions['l-set'] = 1.001 +do -- create closure to overcome 200 locals limit -if not set then set = { } end +if not modules then modules = { } end modules ['l-set'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} -do +set = set or { } - local nums = { } - local tabs = { } - local concat = table.concat +local nums = { } +local tabs = { } +local concat = table.concat +local next, type = next, type - set.create = table.tohash +set.create = table.tohash - function set.tonumber(t) - if next(t) then - local s = "" - -- we could save mem by sorting, but it slows down - for k, v in pairs(t) do - if v then - -- why bother about the leading space - s = s .. " " .. k - end - end - if not nums[s] then - tabs[#tabs+1] = t - nums[s] = #tabs +function set.tonumber(t) + if next(t) then + local s = "" + -- we could save mem by sorting, but it slows down + for k, v in next, t do + if v then + -- why bother about the leading space + s = s .. " " .. k end - return nums[s] - else - return 0 end + local n = nums[s] + if not n then + n = #tabs + 1 + tabs[n] = t + nums[s] = n + end + return n + else + return 0 end +end - function set.totable(n) - if n == 0 then - return { } - else - return tabs[n] or { } - end +function set.totable(n) + if n == 0 then + return { } + else + return tabs[n] or { } end +end - function set.contains(n,s) - if type(n) == "table" then - return n[s] - elseif n == 0 then - return false - else - local t = tabs[n] - return t and t[s] +function set.tolist(n) + if n == 0 or not tabs[n] then + return "" + else + local t = { } + for k, v in next, tabs[n] do + if v then + t[#t+1] = k + end end + return concat(t," ") end +end +function set.contains(n,s) + if type(n) == "table" then + return n[s] + elseif n == 0 then + return false + else + local t = tabs[n] + return t and t[s] + end end --~ local c = set.create{'aap','noot','mies'} @@ -1493,46 +1770,69 @@ end --- filename : l-os.lua --- comment : split off from luat-lib --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['l-os'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- maybe build io.flush in os.execute + +local find, format, gsub = string.find, string.format, string.gsub +local random, ceil = math.random, math.ceil + +local execute, spawn, exec, ioflush = os.execute, os.spawn or os.execute, os.exec or os.execute, io.flush -if not versions then versions = { } end versions['l-os'] = 1.001 +function os.execute(...) ioflush() return execute(...) end +function os.spawn (...) ioflush() return spawn (...) end +function os.exec (...) ioflush() return exec (...) end function os.resultof(command) - return io.popen(command,"r"):read("*all") + ioflush() -- else messed up logging + local handle = io.popen(command,"r") + if not handle then + -- print("unknown command '".. command .. "' in os.resultof") + return "" + else + return handle:read("*all") or "" + end end -if not os.exec then os.exec = os.execute end -if not os.spawn then os.spawn = os.execute end - ---~ os.type : windows | unix (new, we already guessed os.platform) ---~ os.name : windows | msdos | linux | macosx | solaris | .. | generic (new) +--~ os.type : windows | unix (new, we already guessed os.platform) +--~ os.name : windows | msdos | linux | macosx | solaris | .. | generic (new) +--~ os.platform : extended os.name with architecture if not io.fileseparator then - if string.find(os.getenv("PATH"),";") then - io.fileseparator, io.pathseparator, os.platform = "\\", ";", os.type or "windows" + if find(os.getenv("PATH"),";") then + io.fileseparator, io.pathseparator, os.type = "\\", ";", os.type or "mswin" else - io.fileseparator, io.pathseparator, os.platform = "/" , ":", os.type or "unix" + io.fileseparator, io.pathseparator, os.type = "/" , ":", os.type or "unix" end end -os.platform = os.platform or os.type or (io.pathseparator == ";" and "windows") or "unix" +os.type = os.type or (io.pathseparator == ";" and "windows") or "unix" +os.name = os.name or (os.type == "windows" and "mswin" ) or "linux" + +if os.type == "windows" then + os.libsuffix, os.binsuffix = 'dll', 'exe' +else + os.libsuffix, os.binsuffix = 'so', '' +end function os.launch(str) - if os.platform == "windows" then + if os.type == "windows" then os.execute("start " .. str) -- os.spawn ? else os.execute(str .. " &") -- os.spawn ? end end -if not os.setenv then - function os.setenv() return false end -end - if not os.times then -- utime = user time -- stime = system time @@ -1550,11 +1850,10 @@ end os.gettimeofday = os.gettimeofday or os.clock -do - local startuptime = os.gettimeofday() - function os.runtime() - return os.gettimeofday() - startuptime - end +local startuptime = os.gettimeofday() + +function os.runtime() + return os.gettimeofday() - startuptime end --~ print(os.gettimeofday()-os.time()) @@ -1563,23 +1862,249 @@ end --~ print(os.date("%H:%M:%S",os.gettimeofday())) --~ print(os.date("%H:%M:%S",os.time())) +-- no need for function anymore as we have more clever code and helpers now +-- this metatable trickery might as well disappear --- filename : l-file.lua --- comment : split off from luat-lib --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +os.resolvers = os.resolvers or { } + +local resolvers = os.resolvers + +local osmt = getmetatable(os) or { __index = function(t,k) t[k] = "unset" return "unset" end } -- maybe nil +local osix = osmt.__index + +osmt.__index = function(t,k) + return (resolvers[k] or osix)(t,k) +end + +setmetatable(os,osmt) + +if not os.setenv then + + -- we still store them but they won't be seen in + -- child processes although we might pass them some day + -- using command concatination + + local env, getenv = { }, os.getenv + + function os.setenv(k,v) + env[k] = v + end + + function os.getenv(k) + return env[k] or getenv(k) + end + +end + +-- we can use HOSTTYPE on some platforms + +local name, platform = os.name or "linux", os.getenv("MTX_PLATFORM") or "" + +local function guess() + local architecture = os.resultof("uname -m") or "" + if architecture ~= "" then + return architecture + end + architecture = os.getenv("HOSTTYPE") or "" + if architecture ~= "" then + return architecture + end + return os.resultof("echo $HOSTTYPE") or "" +end + +if platform ~= "" then + + os.platform = platform + +elseif os.type == "windows" then + + -- we could set the variable directly, no function needed here + + function os.resolvers.platform(t,k) + local platform, architecture = "", os.getenv("PROCESSOR_ARCHITECTURE") or "" + if find(architecture,"AMD64") then + platform = "mswin-64" + else + platform = "mswin" + end + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +elseif name == "linux" then + + function os.resolvers.platform(t,k) + -- we sometims have HOSTTYPE set so let's check that first + local platform, architecture = "", os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" + if find(architecture,"x86_64") then + platform = "linux-64" + elseif find(architecture,"ppc") then + platform = "linux-ppc" + else + platform = "linux" + end + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +elseif name == "macosx" then + + --[[ + Identifying the architecture of OSX is quite a mess and this + is the best we can come up with. For some reason $HOSTTYPE is + a kind of pseudo environment variable, not known to the current + environment. And yes, uname cannot be trusted either, so there + is a change that you end up with a 32 bit run on a 64 bit system. + Also, some proper 64 bit intel macs are too cheap (low-end) and + therefore not permitted to run the 64 bit kernel. + ]]-- + + function os.resolvers.platform(t,k) + -- local platform, architecture = "", os.getenv("HOSTTYPE") or "" + -- if architecture == "" then + -- architecture = os.resultof("echo $HOSTTYPE") or "" + -- end + local platform, architecture = "", os.resultof("echo $HOSTTYPE") or "" + if architecture == "" then + -- print("\nI have no clue what kind of OSX you're running so let's assume an 32 bit intel.\n") + platform = "osx-intel" + elseif find(architecture,"i386") then + platform = "osx-intel" + elseif find(architecture,"x86_64") then + platform = "osx-64" + else + platform = "osx-ppc" + end + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +elseif name == "sunos" then + + function os.resolvers.platform(t,k) + local platform, architecture = "", os.resultof("uname -m") or "" + if find(architecture,"sparc") then + platform = "solaris-sparc" + else -- if architecture == 'i86pc' + platform = "solaris-intel" + end + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +elseif name == "freebsd" then + + function os.resolvers.platform(t,k) + local platform, architecture = "", os.resultof("uname -m") or "" + if find(architecture,"amd64") then + platform = "freebsd-amd64" + else + platform = "freebsd" + end + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +elseif name == "kfreebsd" then + + function os.resolvers.platform(t,k) + -- we sometims have HOSTTYPE set so let's check that first + local platform, architecture = "", os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" + if find(architecture,"x86_64") then + platform = "kfreebsd-64" + else + platform = "kfreebsd-i386" + end + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +else + + -- platform = "linux" + -- os.setenv("MTX_PLATFORM",platform) + -- os.platform = platform + + function os.resolvers.platform(t,k) + local platform = "linux" + os.setenv("MTX_PLATFORM",platform) + os.platform = platform + return platform + end + +end + +-- beware, we set the randomseed + +-- from wikipedia: Version 4 UUIDs use a scheme relying only on random numbers. This algorithm sets the +-- version number as well as two reserved bits. All other bits are set using a random or pseudorandom +-- data source. Version 4 UUIDs have the form xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx with hexadecimal +-- digits x and hexadecimal digits 8, 9, A, or B for y. e.g. f47ac10b-58cc-4372-a567-0e02b2c3d479. +-- +-- as we don't call this function too often there is not so much risk on repetition + +local t = { 8, 9, "a", "b" } + +function os.uuid() + return format("%04x%04x-4%03x-%s%03x-%04x-%04x%04x%04x", + random(0xFFFF),random(0xFFFF), + random(0x0FFF), + t[ceil(random(4))] or 8,random(0x0FFF), + random(0xFFFF), + random(0xFFFF),random(0xFFFF),random(0xFFFF) + ) +end + +local d -if not versions then versions = { } end versions['l-file'] = 1.001 +function os.timezone(delta) + d = d or tonumber(tonumber(os.date("%H")-os.date("!%H"))) + if delta then + if d > 0 then + return format("+%02i:00",d) + else + return format("-%02i:00",-d) + end + else + return 1 + end +end -if not file then file = { } end + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['l-file'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- needs a cleanup + +file = file or { } + +local concat = table.concat +local find, gmatch, match, gsub, sub, char = string.find, string.gmatch, string.match, string.gsub, string.sub, string.char +local lpegmatch = lpeg.match function file.removesuffix(filename) - return filename:gsub("%.[%a%d]+$", "") + return (gsub(filename,"%.[%a%d]+$","")) end function file.addsuffix(filename, suffix) - if not filename:find("%.[%a%d]+$") then + if not suffix or suffix == "" then + return filename + elseif not find(filename,"%.[%a%d]+$") then return filename .. "." .. suffix else return filename @@ -1587,141 +2112,145 @@ function file.addsuffix(filename, suffix) end function file.replacesuffix(filename, suffix) - if not filename:find("%.[%a%d]+$") then - return filename .. "." .. suffix - else - return (filename:gsub("%.[%a%d]+$","."..suffix)) - end + return (gsub(filename,"%.[%a%d]+$","")) .. "." .. suffix end -function file.dirname(name) - return name:match("^(.+)[/\\].-$") or "" +function file.dirname(name,default) + return match(name,"^(.+)[/\\].-$") or (default or "") end function file.basename(name) - return name:match("^.+[/\\](.-)$") or name + return match(name,"^.+[/\\](.-)$") or name end function file.nameonly(name) - return ((name:match("^.+[/\\](.-)$") or name):gsub("%..*$","")) + return (gsub(match(name,"^.+[/\\](.-)$") or name,"%..*$","")) end -function file.extname(name) - return name:match("^.+%.([^/\\]-)$") or "" +function file.extname(name,default) + return match(name,"^.+%.([^/\\]-)$") or default or "" end file.suffix = file.extname -function file.stripsuffix(name) - return (name:gsub("%.[%a%d]+$","")) -end - --~ function file.join(...) ---~ local t = { ... } ---~ for i=1,#t do ---~ t[i] = (t[i]:gsub("\\","/")):gsub("/+$","") +--~ local pth = concat({...},"/") +--~ pth = gsub(pth,"\\","/") +--~ local a, b = match(pth,"^(.*://)(.*)$") +--~ if a and b then +--~ return a .. gsub(b,"//+","/") --~ end ---~ return table.concat(t,"/") +--~ a, b = match(pth,"^(//)(.*)$") +--~ if a and b then +--~ return a .. gsub(b,"//+","/") +--~ end +--~ return (gsub(pth,"//+","/")) --~ end ---~ print(file.join("x/","/y")) ---~ print(file.join("http://","/y")) ---~ print(file.join("http://a","/y")) ---~ print(file.join("http:///a","/y")) ---~ print(file.join("//nas-1","/y")) +local trick_1 = char(1) +local trick_2 = "^" .. trick_1 .. "/+" function file.join(...) - local pth = table.concat({...},"/") - pth = pth:gsub("\\","/") - local a, b = pth:match("^(.*://)(.*)$") + local lst = { ... } + local a, b = lst[1], lst[2] + if a == "" then + lst[1] = trick_1 + elseif b and find(a,"^/+$") and find(b,"^/") then + lst[1] = "" + lst[2] = gsub(b,"^/+","") + end + local pth = concat(lst,"/") + pth = gsub(pth,"\\","/") + local a, b = match(pth,"^(.*://)(.*)$") if a and b then - return a .. b:gsub("//+","/") + return a .. gsub(b,"//+","/") end - a, b = pth:match("^(//)(.*)$") + a, b = match(pth,"^(//)(.*)$") if a and b then - return a .. b:gsub("//+","/") + return a .. gsub(b,"//+","/") end - return (pth:gsub("//+","/")) + pth = gsub(pth,trick_2,"") + return (gsub(pth,"//+","/")) end -function file.is_writable(name) - local f = io.open(name, 'w') - if f then - f:close() - return true - else - return false - end +--~ print(file.join("//","/y")) +--~ print(file.join("/","/y")) +--~ print(file.join("","/y")) +--~ print(file.join("/x/","/y")) +--~ print(file.join("x/","/y")) +--~ print(file.join("http://","/y")) +--~ print(file.join("http://a","/y")) +--~ print(file.join("http:///a","/y")) +--~ print(file.join("//nas-1","/y")) + +function file.iswritable(name) + local a = lfs.attributes(name) or lfs.attributes(file.dirname(name,".")) + return a and sub(a.permissions,2,2) == "w" end -function file.is_readable(name) - local f = io.open(name,'r') - if f then - f:close() - return true - else - return false - end +function file.isreadable(name) + local a = lfs.attributes(name) + return a and sub(a.permissions,1,1) == "r" end +file.is_readable = file.isreadable +file.is_writable = file.iswritable + +-- todo: lpeg + --~ function file.split_path(str) ---~ if str:find(';') then ---~ return str:splitchr(";") ---~ else ---~ return str:splitchr(io.pathseparator) +--~ local t = { } +--~ str = gsub(str,"\\", "/") +--~ str = gsub(str,"(%a):([;/])", "%1\001%2") +--~ for name in gmatch(str,"([^;:]+)") do +--~ if name ~= "" then +--~ t[#t+1] = gsub(name,"\001",":") +--~ end --~ end +--~ return t --~ end --- todo: lpeg +local checkedsplit = string.checkedsplit -function file.split_path(str) - local t = { } - str = str:gsub("\\", "/") - str = str:gsub("(%a):([;/])", "%1\001%2") - for name in str:gmatch("([^;:]+)") do - if name ~= "" then - name = name:gsub("\001",":") - t[#t+1] = name - end - end - return t +function file.split_path(str,separator) + str = gsub(str,"\\","/") + return checkedsplit(str,separator or io.pathseparator) end function file.join_path(tab) - return table.concat(tab,io.pathseparator) -- can have trailing // -end - ---~ print('test' .. " == " .. file.collapse_path("test")) ---~ print("test/test" .. " == " .. file.collapse_path("test/test")) ---~ print("test/test/test" .. " == " .. file.collapse_path("test/test/test")) ---~ print("test/test" .. " == " .. file.collapse_path("test/../test/test")) ---~ print("test" .. " == " .. file.collapse_path("test/../test")) ---~ print("../test" .. " == " .. file.collapse_path("../test")) ---~ print("../test/" .. " == " .. file.collapse_path("../test/")) ---~ print("a/a" .. " == " .. file.collapse_path("a/b/c/../../a")) - ---~ function file.collapse_path(str) ---~ local ok, n = false, 0 ---~ while not ok do ---~ ok = true ---~ str, n = str:gsub("[^%./]+/%.%./", function(s) ---~ ok = false ---~ return "" ---~ end) ---~ end ---~ return (str:gsub("/%./","/")) ---~ end + return concat(tab,io.pathseparator) -- can have trailing // +end + +-- we can hash them weakly function file.collapse_path(str) - local n = 1 - while n > 0 do - str, n = str:gsub("([^/%.]+/%.%./)","") - end - return (str:gsub("/%./","/")) -end + str = gsub(str,"\\","/") + if find(str,"/") then + str = gsub(str,"^%./",(gsub(lfs.currentdir(),"\\","/")) .. "/") -- ./xx in qualified + str = gsub(str,"/%./","/") + local n, m = 1, 1 + while n > 0 or m > 0 do + str, n = gsub(str,"[^/%.]+/%.%.$","") + str, m = gsub(str,"[^/%.]+/%.%./","") + end + str = gsub(str,"([^/])/$","%1") + -- str = gsub(str,"^%./","") -- ./xx in qualified + str = gsub(str,"/%.$","") + end + if str == "" then str = "." end + return str +end + +--~ print(file.collapse_path("/a")) +--~ print(file.collapse_path("a/./b/..")) +--~ print(file.collapse_path("a/aa/../b/bb")) +--~ print(file.collapse_path("a/../..")) +--~ print(file.collapse_path("a/.././././b/..")) +--~ print(file.collapse_path("a/./././b/..")) +--~ print(file.collapse_path("a/b/c/../..")) function file.robustname(str) - return (str:gsub("[^%a%d%/%-%.\\]+","-")) + return (gsub(str,"[^%a%d%/%-%.\\]+","-")) end file.readdata = io.loaddata @@ -1731,152 +2260,502 @@ function file.copy(oldname,newname) file.savedata(newname,io.loaddata(oldname)) end +-- lpeg variants, slightly faster, not always --- filename : l-dir.lua --- comment : split off from luat-lib --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +--~ local period = lpeg.P(".") +--~ local slashes = lpeg.S("\\/") +--~ local noperiod = 1-period +--~ local noslashes = 1-slashes +--~ local name = noperiod^1 -if not versions then versions = { } end versions['l-dir'] = 1.001 +--~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * lpeg.C(noperiod^1) * -1 -dir = { } +--~ function file.extname(name) +--~ return lpegmatch(pattern,name) or "" +--~ end --- optimizing for no string.find (*) does not save time +--~ local pattern = lpeg.Cs(((period * noperiod^1 * -1)/"" + 1)^1) -if lfs then do +--~ function file.removesuffix(name) +--~ return lpegmatch(pattern,name) +--~ end ---~ local attributes = lfs.attributes ---~ local walkdir = lfs.dir ---~ ---~ local function glob_pattern(path,patt,recurse,action) ---~ local ok, scanner = xpcall(function() return walkdir(path) end, function() end) -- kepler safe ---~ if ok and type(scanner) == "function" then ---~ if not path:find("/$") then path = path .. '/' end ---~ for name in scanner do ---~ local full = path .. name ---~ local mode = attributes(full,'mode') ---~ if mode == 'file' then ---~ if name:find(patt) then ---~ action(full) ---~ end ---~ elseif recurse and (mode == "directory") and (name ~= '.') and (name ~= "..") then ---~ glob_pattern(full,patt,recurse,action) ---~ end ---~ end ---~ end +--~ local pattern = (noslashes^0 * slashes)^1 * lpeg.C(noslashes^1) * -1 + +--~ function file.basename(name) +--~ return lpegmatch(pattern,name) or name +--~ end + +--~ local pattern = (noslashes^0 * slashes)^1 * lpeg.Cp() * noslashes^1 * -1 + +--~ function file.dirname(name) +--~ local p = lpegmatch(pattern,name) +--~ if p then +--~ return sub(name,1,p-2) +--~ else +--~ return "" --~ end ---~ ---~ dir.glob_pattern = glob_pattern ---~ ---~ local function glob(pattern, action) ---~ local t = { } ---~ local action = action or function(name) t[#t+1] = name end ---~ local path, patt = pattern:match("^(.*)/*%*%*/*(.-)$") ---~ local recurse = path and patt ---~ if not recurse then ---~ path, patt = pattern:match("^(.*)/(.-)$") ---~ if not (path and patt) then ---~ path, patt = '.', pattern ---~ end ---~ end ---~ patt = patt:gsub("([%.%-%+])", "%%%1") ---~ patt = patt:gsub("%*", ".*") ---~ patt = patt:gsub("%?", ".") ---~ patt = "^" .. patt .. "$" ---~ -- print('path: ' .. path .. ' | pattern: ' .. patt .. ' | recurse: ' .. tostring(recurse)) ---~ glob_pattern(path,patt,recurse,action) ---~ return t +--~ end + +--~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * lpeg.Cp() * noperiod^1 * -1 + +--~ function file.addsuffix(name, suffix) +--~ local p = lpegmatch(pattern,name) +--~ if p then +--~ return name +--~ else +--~ return name .. "." .. suffix --~ end +--~ end + +--~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * lpeg.Cp() * noperiod^1 * -1 + +--~ function file.replacesuffix(name,suffix) +--~ local p = lpegmatch(pattern,name) +--~ if p then +--~ return sub(name,1,p-2) .. "." .. suffix +--~ else +--~ return name .. "." .. suffix +--~ end +--~ end + +--~ local pattern = (noslashes^0 * slashes)^0 * lpeg.Cp() * ((noperiod^1 * period)^1 * lpeg.Cp() + lpeg.P(true)) * noperiod^1 * -1 + +--~ function file.nameonly(name) +--~ local a, b = lpegmatch(pattern,name) +--~ if b then +--~ return sub(name,a,b-2) +--~ elseif a then +--~ return sub(name,a) +--~ else +--~ return name +--~ end +--~ end + +--~ local test = file.extname +--~ local test = file.basename +--~ local test = file.dirname +--~ local test = file.addsuffix +--~ local test = file.replacesuffix +--~ local test = file.nameonly + +--~ print(1,test("./a/b/c/abd.def.xxx","!!!")) +--~ print(2,test("./../b/c/abd.def.xxx","!!!")) +--~ print(3,test("a/b/c/abd.def.xxx","!!!")) +--~ print(4,test("a/b/c/def.xxx","!!!")) +--~ print(5,test("a/b/c/def","!!!")) +--~ print(6,test("def","!!!")) +--~ print(7,test("def.xxx","!!!")) + +--~ local tim = os.clock() for i=1,250000 do local ext = test("abd.def.xxx","!!!") end print(os.clock()-tim) + +-- also rewrite previous + +local letter = lpeg.R("az","AZ") + lpeg.S("_-+") +local separator = lpeg.P("://") + +local qualified = lpeg.P(".")^0 * lpeg.P("/") + letter*lpeg.P(":") + letter^1*separator + letter^1 * lpeg.P("/") +local rootbased = lpeg.P("/") + letter*lpeg.P(":") + +-- ./name ../name /name c: :// name/name + +function file.is_qualified_path(filename) + return lpegmatch(qualified,filename) ~= nil +end + +function file.is_rootbased_path(filename) + return lpegmatch(rootbased,filename) ~= nil +end + +local slash = lpeg.S("\\/") +local period = lpeg.P(".") +local drive = lpeg.C(lpeg.R("az","AZ")) * lpeg.P(":") +local path = lpeg.C(((1-slash)^0 * slash)^0) +local suffix = period * lpeg.C(lpeg.P(1-period)^0 * lpeg.P(-1)) +local base = lpeg.C((1-suffix)^0) + +local pattern = (drive + lpeg.Cc("")) * (path + lpeg.Cc("")) * (base + lpeg.Cc("")) * (suffix + lpeg.Cc("")) + +function file.splitname(str) -- returns drive, path, base, suffix + return lpegmatch(pattern,str) +end + +-- function test(t) for k, v in next, t do print(v, "=>", file.splitname(v)) end end +-- +-- test { "c:", "c:/aa", "c:/aa/bb", "c:/aa/bb/cc", "c:/aa/bb/cc.dd", "c:/aa/bb/cc.dd.ee" } +-- test { "c:", "c:aa", "c:aa/bb", "c:aa/bb/cc", "c:aa/bb/cc.dd", "c:aa/bb/cc.dd.ee" } +-- test { "/aa", "/aa/bb", "/aa/bb/cc", "/aa/bb/cc.dd", "/aa/bb/cc.dd.ee" } +-- test { "aa", "aa/bb", "aa/bb/cc", "aa/bb/cc.dd", "aa/bb/cc.dd.ee" } + +--~ -- todo: --~ ---~ dir.glob = glob +--~ if os.type == "windows" then +--~ local currentdir = lfs.currentdir +--~ function lfs.currentdir() +--~ return (gsub(currentdir(),"\\","/")) +--~ end +--~ end - local attributes = lfs.attributes - local walkdir = lfs.dir - local function glob_pattern(path,patt,recurse,action) - local ok, scanner - if path == "/" then - ok, scanner = xpcall(function() return walkdir(path..".") end, function() end) -- kepler safe - else - ok, scanner = xpcall(function() return walkdir(path) end, function() end) -- kepler safe +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['l-md5'] = { + version = 1.001, + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- This also provides file checksums and checkers. + +local gsub, format, byte = string.gsub, string.format, string.byte + +local function convert(str,fmt) + return (gsub(md5.sum(str),".",function(chr) return format(fmt,byte(chr)) end)) +end + +if not md5.HEX then function md5.HEX(str) return convert(str,"%02X") end end +if not md5.hex then function md5.hex(str) return convert(str,"%02x") end end +if not md5.dec then function md5.dec(str) return convert(str,"%03i") end end + +--~ if not md5.HEX then +--~ local function remap(chr) return format("%02X",byte(chr)) end +--~ function md5.HEX(str) return (gsub(md5.sum(str),".",remap)) end +--~ end +--~ if not md5.hex then +--~ local function remap(chr) return format("%02x",byte(chr)) end +--~ function md5.hex(str) return (gsub(md5.sum(str),".",remap)) end +--~ end +--~ if not md5.dec then +--~ local function remap(chr) return format("%03i",byte(chr)) end +--~ function md5.dec(str) return (gsub(md5.sum(str),".",remap)) end +--~ end + +file.needs_updating_threshold = 1 + +function file.needs_updating(oldname,newname) -- size modification access change + local oldtime = lfs.attributes(oldname, modification) + local newtime = lfs.attributes(newname, modification) + if newtime >= oldtime then + return false + elseif oldtime - newtime < file.needs_updating_threshold then + return false + else + return true + end +end + +function file.checksum(name) + if md5 then + local data = io.loaddata(name) + if data then + return md5.HEX(data) end - if ok and type(scanner) == "function" then - if not path:find("/$") then path = path .. '/' end - for name in scanner do - local full = path .. name - local mode = attributes(full,'mode') - if mode == 'file' then - if full:find(patt) then - action(full) - end - elseif recurse and (mode == "directory") and (name ~= '.') and (name ~= "..") then - glob_pattern(full,patt,recurse,action) - end - end - end - end - - dir.glob_pattern = glob_pattern - - --~ local function glob(pattern, action) - --~ local t = { } - --~ local path, rest, patt, recurse - --~ local action = action or function(name) t[#t+1] = name end - --~ local pattern = pattern:gsub("^%*%*","./**") - --~ local pattern = pattern:gsub("/%*/","/**/") - --~ path, rest = pattern:match("^(/)(.-)$") - --~ if path then - --~ path = path - --~ else - --~ path, rest = pattern:match("^([^/]*)/(.-)$") - --~ end - --~ if rest then - --~ patt = rest:gsub("([%.%-%+])", "%%%1") - --~ end - --~ patt = patt:gsub("%*", "[^/]*") - --~ patt = patt:gsub("%?", "[^/]") - --~ patt = patt:gsub("%[%^/%]%*%[%^/%]%*", ".*") - --~ if path == "" then path = "." end - --~ recurse = patt:find("%.%*/") ~= nil - --~ glob_pattern(path,patt,recurse,action) - --~ return t - --~ end + end + return nil +end + +function file.loadchecksum(name) + if md5 then + local data = io.loaddata(name .. ".md5") + return data and (gsub(data,"%s","")) + end + return nil +end + +function file.savechecksum(name, checksum) + if not checksum then checksum = file.checksum(name) end + if checksum then + io.savedata(name .. ".md5",checksum) + return checksum + end + return nil +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['l-url'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local char, gmatch, gsub = string.char, string.gmatch, string.gsub +local tonumber, type = tonumber, type +local lpegmatch = lpeg.match + +-- from the spec (on the web): +-- +-- foo://example.com:8042/over/there?name=ferret#nose +-- \_/ \______________/\_________/ \_________/ \__/ +-- | | | | | +-- scheme authority path query fragment +-- | _____________________|__ +-- / \ / \ +-- urn:example:animal:ferret:nose + +url = url or { } + +local function tochar(s) + return char(tonumber(s,16)) +end - local P, S, R, C, Cc, Cs, Ct, Cv, V = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.Cc, lpeg.Cs, lpeg.Ct, lpeg.Cv, lpeg.V +local colon, qmark, hash, slash, percent, endofstring = lpeg.P(":"), lpeg.P("?"), lpeg.P("#"), lpeg.P("/"), lpeg.P("%"), lpeg.P(-1) - local pattern = Ct { - [1] = (C(P(".") + P("/")^1) + C(R("az","AZ") * P(":") * P("/")^0) + Cc("./")) * V(2) * V(3), - [2] = C(((1-S("*?/"))^0 * P("/"))^0), - [3] = C(P(1)^0) +local hexdigit = lpeg.R("09","AF","af") +local plus = lpeg.P("+") +local escaped = (plus / " ") + (percent * lpeg.C(hexdigit * hexdigit) / tochar) + +-- we assume schemes with more than 1 character (in order to avoid problems with windows disks) + +local scheme = lpeg.Cs((escaped+(1-colon-slash-qmark-hash))^2) * colon + lpeg.Cc("") +local authority = slash * slash * lpeg.Cs((escaped+(1- slash-qmark-hash))^0) + lpeg.Cc("") +local path = slash * lpeg.Cs((escaped+(1- qmark-hash))^0) + lpeg.Cc("") +local query = qmark * lpeg.Cs((escaped+(1- hash))^0) + lpeg.Cc("") +local fragment = hash * lpeg.Cs((escaped+(1- endofstring))^0) + lpeg.Cc("") + +local parser = lpeg.Ct(scheme * authority * path * query * fragment) + +-- todo: reconsider Ct as we can as well have five return values (saves a table) +-- so we can have two parsers, one with and one without + +function url.split(str) + return (type(str) == "string" and lpegmatch(parser,str)) or str +end + +-- todo: cache them + +function url.hashed(str) + local s = url.split(str) + local somescheme = s[1] ~= "" + return { + scheme = (somescheme and s[1]) or "file", + authority = s[2], + path = s[3], + query = s[4], + fragment = s[5], + original = str, + noscheme = not somescheme, } +end + +function url.hasscheme(str) + return url.split(str)[1] ~= "" +end + +function url.addscheme(str,scheme) + return (url.hasscheme(str) and str) or ((scheme or "file:///") .. str) +end + +function url.construct(hash) + local fullurl = hash.sheme .. "://".. hash.authority .. hash.path + if hash.query then + fullurl = fullurl .. "?".. hash.query + end + if hash.fragment then + fullurl = fullurl .. "?".. hash.fragment + end + return fullurl +end - local filter = Cs ( ( - P("**") / ".*" + - P("*") / "[^/]*" + - P("?") / "[^/]" + - P(".") / "%%." + - P("+") / "%%+" + - P("-") / "%%-" + - P(1) - )^0 ) - - local function glob(str,t) +function url.filename(filename) + local t = url.hashed(filename) + return (t.scheme == "file" and (gsub(t.path,"^/([a-zA-Z])([:|])/)","%1:"))) or filename +end + +function url.query(str) + if type(str) == "string" then + local t = { } + for k, v in gmatch(str,"([^&=]*)=([^&=]*)") do + t[k] = v + end + return t + else + return str + end +end + +--~ print(url.filename("file:///c:/oeps.txt")) +--~ print(url.filename("c:/oeps.txt")) +--~ print(url.filename("file:///oeps.txt")) +--~ print(url.filename("file:///etc/test.txt")) +--~ print(url.filename("/oeps.txt")) + +--~ from the spec on the web (sort of): +--~ +--~ function test(str) +--~ print(table.serialize(url.hashed(str))) +--~ end +--~ +--~ test("%56pass%20words") +--~ test("file:///c:/oeps.txt") +--~ test("file:///c|/oeps.txt") +--~ test("file:///etc/oeps.txt") +--~ test("file://./etc/oeps.txt") +--~ test("file:////etc/oeps.txt") +--~ test("ftp://ftp.is.co.za/rfc/rfc1808.txt") +--~ test("http://www.ietf.org/rfc/rfc2396.txt") +--~ test("ldap://[2001:db8::7]/c=GB?objectClass?one#what") +--~ test("mailto:John.Doe@example.com") +--~ test("news:comp.infosystems.www.servers.unix") +--~ test("tel:+1-816-555-1212") +--~ test("telnet://192.0.2.16:80/") +--~ test("urn:oasis:names:specification:docbook:dtd:xml:4.1.2") +--~ test("/etc/passwords") +--~ test("http://www.pragma-ade.com/spaced%20name") + +--~ test("zip:///oeps/oeps.zip#bla/bla.tex") +--~ test("zip:///oeps/oeps.zip?bla/bla.tex") + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['l-dir'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- dir.expand_name will be merged with cleanpath and collapsepath + +local type = type +local find, gmatch, match, gsub = string.find, string.gmatch, string.match, string.gsub +local lpegmatch = lpeg.match + +dir = dir or { } + +-- handy + +function dir.current() + return (gsub(lfs.currentdir(),"\\","/")) +end + +-- optimizing for no string.find (*) does not save time + +local attributes = lfs.attributes +local walkdir = lfs.dir + +local function glob_pattern(path,patt,recurse,action) + local ok, scanner + if path == "/" then + ok, scanner = xpcall(function() return walkdir(path..".") end, function() end) -- kepler safe + else + ok, scanner = xpcall(function() return walkdir(path) end, function() end) -- kepler safe + end + if ok and type(scanner) == "function" then + if not find(path,"/$") then path = path .. '/' end + for name in scanner do + local full = path .. name + local mode = attributes(full,'mode') + if mode == 'file' then + if find(full,patt) then + action(full) + end + elseif recurse and (mode == "directory") and (name ~= '.') and (name ~= "..") then + glob_pattern(full,patt,recurse,action) + end + end + end +end + +dir.glob_pattern = glob_pattern + +local function collect_pattern(path,patt,recurse,result) + local ok, scanner + result = result or { } + if path == "/" then + ok, scanner = xpcall(function() return walkdir(path..".") end, function() end) -- kepler safe + else + ok, scanner = xpcall(function() return walkdir(path) end, function() end) -- kepler safe + end + if ok and type(scanner) == "function" then + if not find(path,"/$") then path = path .. '/' end + for name in scanner do + local full = path .. name + local attr = attributes(full) + local mode = attr.mode + if mode == 'file' then + if find(full,patt) then + result[name] = attr + end + elseif recurse and (mode == "directory") and (name ~= '.') and (name ~= "..") then + attr.list = collect_pattern(full,patt,recurse) + result[name] = attr + end + end + end + return result +end + +dir.collect_pattern = collect_pattern + +local P, S, R, C, Cc, Cs, Ct, Cv, V = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.Cc, lpeg.Cs, lpeg.Ct, lpeg.Cv, lpeg.V + +local pattern = Ct { + [1] = (C(P(".") + P("/")^1) + C(R("az","AZ") * P(":") * P("/")^0) + Cc("./")) * V(2) * V(3), + [2] = C(((1-S("*?/"))^0 * P("/"))^0), + [3] = C(P(1)^0) +} + +local filter = Cs ( ( + P("**") / ".*" + + P("*") / "[^/]*" + + P("?") / "[^/]" + + P(".") / "%%." + + P("+") / "%%+" + + P("-") / "%%-" + + P(1) +)^0 ) + +local function glob(str,t) + if type(t) == "function" then + if type(str) == "table" then + for s=1,#str do + glob(str[s],t) + end + elseif lfs.isfile(str) then + t(str) + else + local split = lpegmatch(pattern,str) + if split then + local root, path, base = split[1], split[2], split[3] + local recurse = find(base,"%*%*") + local start = root .. path + local result = lpegmatch(filter,start .. base) + glob_pattern(start,result,recurse,t) + end + end + else if type(str) == "table" then local t = t or { } - for _, s in ipairs(str) do - glob(s,t) + for s=1,#str do + glob(str[s],t) end return t + elseif lfs.isfile(str) then + local t = t or { } + t[#t+1] = str + return t else - local split = pattern:match(str) + local split = lpegmatch(pattern,str) if split then local t = t or { } local action = action or function(name) t[#t+1] = name end local root, path, base = split[1], split[2], split[3] - local recurse = base:find("**") + local recurse = find(base,"%*%*") local start = root .. path - local result = filter:match(start .. base) + local result = lpegmatch(filter,start .. base) glob_pattern(start,result,recurse,action) return t else @@ -1884,113 +2763,120 @@ if lfs then do end end end +end - dir.glob = glob +dir.glob = glob - --~ list = dir.glob("**/*.tif") - --~ list = dir.glob("/**/*.tif") - --~ list = dir.glob("./**/*.tif") - --~ list = dir.glob("oeps/**/*.tif") - --~ list = dir.glob("/oeps/**/*.tif") +--~ list = dir.glob("**/*.tif") +--~ list = dir.glob("/**/*.tif") +--~ list = dir.glob("./**/*.tif") +--~ list = dir.glob("oeps/**/*.tif") +--~ list = dir.glob("/oeps/**/*.tif") - local function globfiles(path,recurse,func,files) -- func == pattern or function - if type(func) == "string" then - local s = func -- alas, we need this indirect way - func = function(name) return name:find(s) end - end - files = files or { } - for name in walkdir(path) do - if name:find("^%.") then - --- skip - elseif attributes(name,'mode') == "directory" then +local function globfiles(path,recurse,func,files) -- func == pattern or function + if type(func) == "string" then + local s = func -- alas, we need this indirect way + func = function(name) return find(name,s) end + end + files = files or { } + for name in walkdir(path) do + if find(name,"^%.") then + --- skip + else + local mode = attributes(name,'mode') + if mode == "directory" then if recurse then globfiles(path .. "/" .. name,recurse,func,files) end - elseif func then - if func(name) then + elseif mode == "file" then + if func then + if func(name) then + files[#files+1] = path .. "/" .. name + end + else files[#files+1] = path .. "/" .. name end - else - files[#files+1] = path .. "/" .. name end end - return files end + return files +end - dir.globfiles = globfiles +dir.globfiles = globfiles - -- t = dir.glob("c:/data/develop/context/sources/**/????-*.tex") - -- t = dir.glob("c:/data/develop/tex/texmf/**/*.tex") - -- t = dir.glob("c:/data/develop/context/texmf/**/*.tex") - -- t = dir.glob("f:/minimal/tex/**/*") - -- print(dir.ls("f:/minimal/tex/**/*")) - -- print(dir.ls("*.tex")) +-- t = dir.glob("c:/data/develop/context/sources/**/????-*.tex") +-- t = dir.glob("c:/data/develop/tex/texmf/**/*.tex") +-- t = dir.glob("c:/data/develop/context/texmf/**/*.tex") +-- t = dir.glob("f:/minimal/tex/**/*") +-- print(dir.ls("f:/minimal/tex/**/*")) +-- print(dir.ls("*.tex")) - function dir.ls(pattern) - return table.concat(glob(pattern),"\n") - end +function dir.ls(pattern) + return table.concat(glob(pattern),"\n") +end - --~ mkdirs("temp") - --~ mkdirs("a/b/c") - --~ mkdirs(".","/a/b/c") - --~ mkdirs("a","b","c") +--~ mkdirs("temp") +--~ mkdirs("a/b/c") +--~ mkdirs(".","/a/b/c") +--~ mkdirs("a","b","c") - local make_indeed = true -- false +local make_indeed = true -- false - if string.find(os.getenv("PATH"),";") then +if string.find(os.getenv("PATH"),";") then -- os.type == "windows" - function dir.mkdirs(...) - local str, pth = "", "" - for _, s in ipairs({...}) do - if s ~= "" then - if str ~= "" then - str = str .. "/" .. s - else - str = s - end + function dir.mkdirs(...) + local str, pth, t = "", "", { ... } + for i=1,#t do + local s = t[i] + if s ~= "" then + if str ~= "" then + str = str .. "/" .. s + else + str = s end end - local first, middle, last - local drive = false - first, middle, last = str:match("^(//)(//*)(.*)$") + end + local first, middle, last + local drive = false + first, middle, last = match(str,"^(//)(//*)(.*)$") + if first then + -- empty network path == local path + else + first, last = match(str,"^(//)/*(.-)$") if first then - -- empty network path == local path + middle, last = match(str,"([^/]+)/+(.-)$") + if middle then + pth = "//" .. middle + else + pth = "//" .. last + last = "" + end else - first, last = str:match("^(//)/*(.-)$") + first, middle, last = match(str,"^([a-zA-Z]:)(/*)(.-)$") if first then - middle, last = str:match("([^/]+)/+(.-)$") - if middle then - pth = "//" .. middle - else - pth = "//" .. last - last = "" - end + pth, drive = first .. middle, true else - first, middle, last = str:match("^([a-zA-Z]:)(/*)(.-)$") - if first then - pth, drive = first .. middle, true - else - middle, last = str:match("^(/*)(.-)$") - if not middle then - last = str - end + middle, last = match(str,"^(/*)(.-)$") + if not middle then + last = str end end end - for s in last:gmatch("[^/]+") do - if pth == "" then - pth = s - elseif drive then - pth, drive = pth .. s, false - else - pth = pth .. "/" .. s - end - if make_indeed and not lfs.isdir(pth) then - lfs.mkdir(pth) - end + end + for s in gmatch(last,"[^/]+") do + if pth == "" then + pth = s + elseif drive then + pth, drive = pth .. s, false + else + pth = pth .. "/" .. s + end + if make_indeed and not lfs.isdir(pth) then + lfs.mkdir(pth) end - return pth, (lfs.isdir(pth) == true) end + return pth, (lfs.isdir(pth) == true) + end --~ print(dir.mkdirs("","","a","c")) --~ print(dir.mkdirs("a")) @@ -2004,79 +2890,77 @@ if lfs then do --~ print(dir.mkdirs("///a/b/c")) --~ print(dir.mkdirs("a/bbb//ccc/")) - function dir.expand_name(str) - local first, nothing, last = str:match("^(//)(//*)(.*)$") - if first then - first = lfs.currentdir() .. "/" - first = first:gsub("\\","/") - end - if not first then - first, last = str:match("^(//)/*(.*)$") - end - if not first then - first, last = str:match("^([a-zA-Z]:)(.*)$") - if first and not last:find("^/") then - local d = lfs.currentdir() - if lfs.chdir(first) then - first = lfs.currentdir() - first = first:gsub("\\","/") - end - lfs.chdir(d) + function dir.expand_name(str) -- will be merged with cleanpath and collapsepath + local first, nothing, last = match(str,"^(//)(//*)(.*)$") + if first then + first = dir.current() .. "/" + end + if not first then + first, last = match(str,"^(//)/*(.*)$") + end + if not first then + first, last = match(str,"^([a-zA-Z]:)(.*)$") + if first and not find(last,"^/") then + local d = lfs.currentdir() + if lfs.chdir(first) then + first = dir.current() end - end - if not first then - first, last = lfs.currentdir(), str - first = first:gsub("\\","/") - end - last = last:gsub("//","/") - last = last:gsub("/%./","/") - last = last:gsub("^/*","") - first = first:gsub("/*$","") - if last == "" then - return first - else - return first .. "/" .. last + lfs.chdir(d) end end + if not first then + first, last = dir.current(), str + end + last = gsub(last,"//","/") + last = gsub(last,"/%./","/") + last = gsub(last,"^/*","") + first = gsub(first,"/*$","") + if last == "" then + return first + else + return first .. "/" .. last + end + end - else +else - function dir.mkdirs(...) - local str, pth = "", "" - for _, s in ipairs({...}) do - if s ~= "" then - if str ~= "" then - str = str .. "/" .. s - else - str = s - end + function dir.mkdirs(...) + local str, pth, t = "", "", { ... } + for i=1,#t do + local s = t[i] + if s ~= "" then + if str ~= "" then + str = str .. "/" .. s + else + str = s end end - str = str:gsub("/+","/") - if str:find("^/") then - pth = "/" - for s in str:gmatch("[^/]+") do - local first = (pth == "/") - if first then - pth = pth .. s - else - pth = pth .. "/" .. s - end - if make_indeed and not first and not lfs.isdir(pth) then - lfs.mkdir(pth) - end - end - else - pth = "." - for s in str:gmatch("[^/]+") do + end + str = gsub(str,"/+","/") + if find(str,"^/") then + pth = "/" + for s in gmatch(str,"[^/]+") do + local first = (pth == "/") + if first then + pth = pth .. s + else pth = pth .. "/" .. s - if make_indeed and not lfs.isdir(pth) then - lfs.mkdir(pth) - end + end + if make_indeed and not first and not lfs.isdir(pth) then + lfs.mkdir(pth) + end + end + else + pth = "." + for s in gmatch(str,"[^/]+") do + pth = pth .. "/" .. s + if make_indeed and not lfs.isdir(pth) then + lfs.mkdir(pth) end end - return pth, (lfs.isdir(pth) == true) end + return pth, (lfs.isdir(pth) == true) + end --~ print(dir.mkdirs("","","a","c")) --~ print(dir.mkdirs("a")) @@ -2086,30 +2970,35 @@ if lfs then do --~ print(dir.mkdirs("///a/b/c")) --~ print(dir.mkdirs("a/bbb//ccc/")) - function dir.expand_name(str) - if not str:find("^/") then - str = lfs.currentdir() .. "/" .. str - end - str = str:gsub("//","/") - str = str:gsub("/%./","/") - return str + function dir.expand_name(str) -- will be merged with cleanpath and collapsepath + if not find(str,"^/") then + str = lfs.currentdir() .. "/" .. str end - + str = gsub(str,"//","/") + str = gsub(str,"/%./","/") + return str end - dir.makedirs = dir.mkdirs +end -end end +dir.makedirs = dir.mkdirs --- filename : l-boolean.lua --- comment : split off from luat-lib --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['l-boolean'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +boolean = boolean or { } -if not versions then versions = { } end versions['l-boolean'] = 1.001 -if not boolean then boolean = { } end +local type, tonumber = type, tonumber function boolean.tonumber(b) if b then return 1 else return 0 end @@ -2119,7 +3008,7 @@ function toboolean(str,tolerant) if tolerant then local tstr = type(str) if tstr == "string" then - return str == "true" or str == "yes" or str == "on" or str == "1" + return str == "true" or str == "yes" or str == "on" or str == "1" or str == "t" elseif tstr == "number" then return tonumber(str) ~= 0 elseif tstr == "nil" then @@ -2138,9 +3027,9 @@ end function string.is_boolean(str) if type(str) == "string" then - if str == "true" or str == "yes" or str == "on" then + if str == "true" or str == "yes" or str == "on" or str == "t" then return true - elseif str == "false" or str == "no" or str == "off" then + elseif str == "false" or str == "no" or str == "off" or str == "f" then return false end end @@ -2156,7 +3045,854 @@ function boolean.falsetrue() end -if not modules then modules = { } end modules ['l-xml'] = { +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['l-math'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local floor, sin, cos, tan = math.floor, math.sin, math.cos, math.tan + +if not math.round then + function math.round(x) + return floor(x + 0.5) + end +end + +if not math.div then + function math.div(n,m) + return floor(n/m) + end +end + +if not math.mod then + function math.mod(n,m) + return n % m + end +end + +local pipi = 2*math.pi/360 + +function math.sind(d) + return sin(d*pipi) +end + +function math.cosd(d) + return cos(d*pipi) +end + +function math.tand(d) + return tan(d*pipi) +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['l-utils'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- hm, quite unreadable + +local gsub = string.gsub +local concat = table.concat +local type, next = type, next + +if not utils then utils = { } end +if not utils.merger then utils.merger = { } end +if not utils.lua then utils.lua = { } end + +utils.merger.m_begin = "begin library merge" +utils.merger.m_end = "end library merge" +utils.merger.pattern = + "%c+" .. + "%-%-%s+" .. utils.merger.m_begin .. + "%c+(.-)%c+" .. + "%-%-%s+" .. utils.merger.m_end .. + "%c+" + +function utils.merger._self_fake_() + return + "-- " .. "created merged file" .. "\n\n" .. + "-- " .. utils.merger.m_begin .. "\n\n" .. + "-- " .. utils.merger.m_end .. "\n\n" +end + +function utils.report(...) + print(...) +end + +utils.merger.strip_comment = true + +function utils.merger._self_load_(name) + local f, data = io.open(name), "" + if f then + utils.report("reading merge from %s",name) + data = f:read("*all") + f:close() + else + utils.report("unknown file to merge %s",name) + end + if data and utils.merger.strip_comment then + -- saves some 20K + data = gsub(data,"%-%-~[^\n\r]*[\r\n]", "") + end + return data or "" +end + +function utils.merger._self_save_(name, data) + if data ~= "" then + local f = io.open(name,'w') + if f then + utils.report("saving merge from %s",name) + f:write(data) + f:close() + end + end +end + +function utils.merger._self_swap_(data,code) + if data ~= "" then + return (gsub(data,utils.merger.pattern, function(s) + return "\n\n" .. "-- "..utils.merger.m_begin .. "\n" .. code .. "\n" .. "-- "..utils.merger.m_end .. "\n\n" + end, 1)) + else + return "" + end +end + +--~ stripper: +--~ +--~ data = gsub(data,"%-%-~[^\n]*\n","") +--~ data = gsub(data,"\n\n+","\n") + +function utils.merger._self_libs_(libs,list) + local result, f, frozen = { }, nil, false + result[#result+1] = "\n" + if type(libs) == 'string' then libs = { libs } end + if type(list) == 'string' then list = { list } end + local foundpath = nil + for i=1,#libs do + local lib = libs[i] + for j=1,#list do + local pth = gsub(list[j],"\\","/") -- file.clean_path + utils.report("checking library path %s",pth) + local name = pth .. "/" .. lib + if lfs.isfile(name) then + foundpath = pth + end + end + if foundpath then break end + end + if foundpath then + utils.report("using library path %s",foundpath) + local right, wrong = { }, { } + for i=1,#libs do + local lib = libs[i] + local fullname = foundpath .. "/" .. lib + if lfs.isfile(fullname) then + -- right[#right+1] = lib + utils.report("merging library %s",fullname) + result[#result+1] = "do -- create closure to overcome 200 locals limit" + result[#result+1] = io.loaddata(fullname,true) + result[#result+1] = "end -- of closure" + else + -- wrong[#wrong+1] = lib + utils.report("no library %s",fullname) + end + end + if #right > 0 then + utils.report("merged libraries: %s",concat(right," ")) + end + if #wrong > 0 then + utils.report("skipped libraries: %s",concat(wrong," ")) + end + else + utils.report("no valid library path found") + end + return concat(result, "\n\n") +end + +function utils.merger.selfcreate(libs,list,target) + if target then + utils.merger._self_save_( + target, + utils.merger._self_swap_( + utils.merger._self_fake_(), + utils.merger._self_libs_(libs,list) + ) + ) + end +end + +function utils.merger.selfmerge(name,libs,list,target) + utils.merger._self_save_( + target or name, + utils.merger._self_swap_( + utils.merger._self_load_(name), + utils.merger._self_libs_(libs,list) + ) + ) +end + +function utils.merger.selfclean(name) + utils.merger._self_save_( + name, + utils.merger._self_swap_( + utils.merger._self_load_(name), + "" + ) + ) +end + +function utils.lua.compile(luafile, lucfile, cleanup, strip) -- defaults: cleanup=false strip=true + -- utils.report("compiling",luafile,"into",lucfile) + os.remove(lucfile) + local command = "-o " .. string.quote(lucfile) .. " " .. string.quote(luafile) + if strip ~= false then + command = "-s " .. command + end + local done = (os.spawn("texluac " .. command) == 0) or (os.spawn("luac " .. command) == 0) + if done and cleanup == true and lfs.isfile(lucfile) and lfs.isfile(luafile) then + -- utils.report("removing",luafile) + os.remove(luafile) + end + return done +end + + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['l-aux'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- for inline, no store split : for s in string.gmatch(str,",* *([^,]+)") do .. end + +aux = aux or { } + +local concat, format, gmatch = table.concat, string.format, string.gmatch +local tostring, type = tostring, type +local lpegmatch = lpeg.match + +local P, R, V = lpeg.P, lpeg.R, lpeg.V + +local escape, left, right = P("\\"), P('{'), P('}') + +lpeg.patterns.balanced = P { + [1] = ((escape * (left+right)) + (1 - (left+right)) + V(2))^0, + [2] = left * V(1) * right +} + +local space = lpeg.P(' ') +local equal = lpeg.P("=") +local comma = lpeg.P(",") +local lbrace = lpeg.P("{") +local rbrace = lpeg.P("}") +local nobrace = 1 - (lbrace+rbrace) +local nested = lpeg.P { lbrace * (nobrace + lpeg.V(1))^0 * rbrace } +local spaces = space^0 + +local value = lpeg.P(lbrace * lpeg.C((nobrace + nested)^0) * rbrace) + lpeg.C((nested + (1-comma))^0) + +local key = lpeg.C((1-equal-comma)^1) +local pattern_a = (space+comma)^0 * (key * equal * value + key * lpeg.C("")) +local pattern_c = (space+comma)^0 * (key * equal * value) + +local key = lpeg.C((1-space-equal-comma)^1) +local pattern_b = spaces * comma^0 * spaces * (key * ((spaces * equal * spaces * value) + lpeg.C(""))) + +-- "a=1, b=2, c=3, d={a{b,c}d}, e=12345, f=xx{a{b,c}d}xx, g={}" : outer {} removes, leading spaces ignored + +local hash = { } + +local function set(key,value) -- using Carg is slower here + hash[key] = value +end + +local pattern_a_s = (pattern_a/set)^1 +local pattern_b_s = (pattern_b/set)^1 +local pattern_c_s = (pattern_c/set)^1 + +aux.settings_to_hash_pattern_a = pattern_a_s +aux.settings_to_hash_pattern_b = pattern_b_s +aux.settings_to_hash_pattern_c = pattern_c_s + +function aux.make_settings_to_hash_pattern(set,how) + if how == "strict" then + return (pattern_c/set)^1 + elseif how == "tolerant" then + return (pattern_b/set)^1 + else + return (pattern_a/set)^1 + end +end + +function aux.settings_to_hash(str,existing) + if str and str ~= "" then + hash = existing or { } + if moretolerant then + lpegmatch(pattern_b_s,str) + else + lpegmatch(pattern_a_s,str) + end + return hash + else + return { } + end +end + +function aux.settings_to_hash_tolerant(str,existing) + if str and str ~= "" then + hash = existing or { } + lpegmatch(pattern_b_s,str) + return hash + else + return { } + end +end + +function aux.settings_to_hash_strict(str,existing) + if str and str ~= "" then + hash = existing or { } + lpegmatch(pattern_c_s,str) + return next(hash) and hash + else + return nil + end +end + +local separator = comma * space^0 +local value = lpeg.P(lbrace * lpeg.C((nobrace + nested)^0) * rbrace) + lpeg.C((nested + (1-comma))^0) +local pattern = lpeg.Ct(value*(separator*value)^0) + +-- "aap, {noot}, mies" : outer {} removes, leading spaces ignored + +aux.settings_to_array_pattern = pattern + +-- we could use a weak table as cache + +function aux.settings_to_array(str) + if not str or str == "" then + return { } + else + return lpegmatch(pattern,str) + end +end + +local function set(t,v) + t[#t+1] = v +end + +local value = lpeg.P(lpeg.Carg(1)*value) / set +local pattern = value*(separator*value)^0 * lpeg.Carg(1) + +function aux.add_settings_to_array(t,str) + return lpegmatch(pattern,str,nil,t) +end + +function aux.hash_to_string(h,separator,yes,no,strict,omit) + if h then + local t, s = { }, table.sortedkeys(h) + omit = omit and table.tohash(omit) + for i=1,#s do + local key = s[i] + if not omit or not omit[key] then + local value = h[key] + if type(value) == "boolean" then + if yes and no then + if value then + t[#t+1] = key .. '=' .. yes + elseif not strict then + t[#t+1] = key .. '=' .. no + end + elseif value or not strict then + t[#t+1] = key .. '=' .. tostring(value) + end + else + t[#t+1] = key .. '=' .. value + end + end + end + return concat(t,separator or ",") + else + return "" + end +end + +function aux.array_to_string(a,separator) + if a then + return concat(a,separator or ",") + else + return "" + end +end + +function aux.settings_to_set(str,t) + t = t or { } + for s in gmatch(str,"%s*([^,]+)") do + t[s] = true + end + return t +end + +local value = lbrace * lpeg.C((nobrace + nested)^0) * rbrace +local pattern = lpeg.Ct((space + value)^0) + +function aux.arguments_to_table(str) + return lpegmatch(pattern,str) +end + +-- temporary here + +function aux.getparameters(self,class,parentclass,settings) + local sc = self[class] + if not sc then + sc = table.clone(self[parent]) + self[class] = sc + end + aux.settings_to_hash(settings,sc) +end + +-- temporary here + +local digit = lpeg.R("09") +local period = lpeg.P(".") +local zero = lpeg.P("0") +local trailingzeros = zero^0 * -digit -- suggested by Roberto R +local case_1 = period * trailingzeros / "" +local case_2 = period * (digit - trailingzeros)^1 * (trailingzeros / "") +local number = digit^1 * (case_1 + case_2) +local stripper = lpeg.Cs((number + 1)^0) + +--~ local sample = "bla 11.00 bla 11 bla 0.1100 bla 1.00100 bla 0.00 bla 0.001 bla 1.1100 bla 0.100100100 bla 0.00100100100" +--~ collectgarbage("collect") +--~ str = string.rep(sample,10000) +--~ local ts = os.clock() +--~ lpegmatch(stripper,str) +--~ print(#str, os.clock()-ts, lpegmatch(stripper,sample)) + +lpeg.patterns.strip_zeros = stripper + +function aux.strip_zeros(str) + return lpegmatch(stripper,str) +end + +function aux.definetable(target) -- defines undefined tables + local composed, t = nil, { } + for name in gmatch(target,"([^%.]+)") do + if composed then + composed = composed .. "." .. name + else + composed = name + end + t[#t+1] = format("%s = %s or { }",composed,composed) + end + return concat(t,"\n") +end + +function aux.accesstable(target) + local t = _G + for name in gmatch(target,"([^%.]+)") do + t = t[name] + end + return t +end + +--~ function string.commaseparated(str) +--~ return gmatch(str,"([^,%s]+)") +--~ end + +-- as we use this a lot ... + +--~ function aux.cachefunction(action,weak) +--~ local cache = { } +--~ if weak then +--~ setmetatable(cache, { __mode = "kv" } ) +--~ end +--~ local function reminder(str) +--~ local found = cache[str] +--~ if not found then +--~ found = action(str) +--~ cache[str] = found +--~ end +--~ return found +--~ end +--~ return reminder, cache +--~ end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['trac-tra'] = { + version = 1.001, + comment = "companion to trac-tra.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- the <anonymous> tag is kind of generic and used for functions that are not +-- bound to a variable, like node.new, node.copy etc (contrary to for instance +-- node.has_attribute which is bound to a has_attribute local variable in mkiv) + +local debug = require "debug" + +local getinfo = debug.getinfo +local type, next = type, next +local concat = table.concat +local format, find, lower, gmatch, gsub = string.format, string.find, string.lower, string.gmatch, string.gsub + +debugger = debugger or { } + +local counters = { } +local names = { } + +-- one + +local function hook() + local f = getinfo(2,"f").func + local n = getinfo(2,"Sn") +-- if n.what == "C" and n.name then print (n.namewhat .. ': ' .. n.name) end + if f then + local cf = counters[f] + if cf == nil then + counters[f] = 1 + names[f] = n + else + counters[f] = cf + 1 + end + end +end +local function getname(func) + local n = names[func] + if n then + if n.what == "C" then + return n.name or '<anonymous>' + else + -- source short_src linedefined what name namewhat nups func + local name = n.name or n.namewhat or n.what + if not name or name == "" then name = "?" end + return format("%s : %s : %s", n.short_src or "unknown source", n.linedefined or "--", name) + end + else + return "unknown" + end +end +function debugger.showstats(printer,threshold) + printer = printer or texio.write or print + threshold = threshold or 0 + local total, grandtotal, functions = 0, 0, 0 + printer("\n") -- ugly but ok + -- table.sort(counters) + for func, count in next, counters do + if count > threshold then + local name = getname(func) + if not find(name,"for generator") then + printer(format("%8i %s", count, name)) + total = total + count + end + end + grandtotal = grandtotal + count + functions = functions + 1 + end + printer(format("functions: %s, total: %s, grand total: %s, threshold: %s\n", functions, total, grandtotal, threshold)) +end + +-- two + +--~ local function hook() +--~ local n = getinfo(2) +--~ if n.what=="C" and not n.name then +--~ local f = tostring(debug.traceback()) +--~ local cf = counters[f] +--~ if cf == nil then +--~ counters[f] = 1 +--~ names[f] = n +--~ else +--~ counters[f] = cf + 1 +--~ end +--~ end +--~ end +--~ function debugger.showstats(printer,threshold) +--~ printer = printer or texio.write or print +--~ threshold = threshold or 0 +--~ local total, grandtotal, functions = 0, 0, 0 +--~ printer("\n") -- ugly but ok +--~ -- table.sort(counters) +--~ for func, count in next, counters do +--~ if count > threshold then +--~ printer(format("%8i %s", count, func)) +--~ total = total + count +--~ end +--~ grandtotal = grandtotal + count +--~ functions = functions + 1 +--~ end +--~ printer(format("functions: %s, total: %s, grand total: %s, threshold: %s\n", functions, total, grandtotal, threshold)) +--~ end + +-- rest + +function debugger.savestats(filename,threshold) + local f = io.open(filename,'w') + if f then + debugger.showstats(function(str) f:write(str) end,threshold) + f:close() + end +end + +function debugger.enable() + debug.sethook(hook,"c") +end + +function debugger.disable() + debug.sethook() +--~ counters[debug.getinfo(2,"f").func] = nil +end + +function debugger.tracing() + local n = tonumber(os.env['MTX.TRACE.CALLS']) or tonumber(os.env['MTX_TRACE_CALLS']) or 0 + if n > 0 then + function debugger.tracing() return true end ; return true + else + function debugger.tracing() return false end ; return false + end +end + +--~ debugger.enable() + +--~ print(math.sin(1*.5)) +--~ print(math.sin(1*.5)) +--~ print(math.sin(1*.5)) +--~ print(math.sin(1*.5)) +--~ print(math.sin(1*.5)) + +--~ debugger.disable() + +--~ print("") +--~ debugger.showstats() +--~ print("") +--~ debugger.showstats(print,3) + +setters = setters or { } +setters.data = setters.data or { } + +--~ local function set(t,what,value) +--~ local data, done = t.data, t.done +--~ if type(what) == "string" then +--~ what = aux.settings_to_array(what) -- inefficient but ok +--~ end +--~ for i=1,#what do +--~ local w = what[i] +--~ for d, f in next, data do +--~ if done[d] then +--~ -- prevent recursion due to wildcards +--~ elseif find(d,w) then +--~ done[d] = true +--~ for i=1,#f do +--~ f[i](value) +--~ end +--~ end +--~ end +--~ end +--~ end + +local function set(t,what,value) + local data, done = t.data, t.done + if type(what) == "string" then + what = aux.settings_to_hash(what) -- inefficient but ok + end + for w, v in next, what do + if v == "" then + v = value + else + v = toboolean(v) + end + for d, f in next, data do + if done[d] then + -- prevent recursion due to wildcards + elseif find(d,w) then + done[d] = true + for i=1,#f do + f[i](v) + end + end + end + end +end + +local function reset(t) + for d, f in next, t.data do + for i=1,#f do + f[i](false) + end + end +end + +local function enable(t,what) + set(t,what,true) +end + +local function disable(t,what) + local data = t.data + if not what or what == "" then + t.done = { } + reset(t) + else + set(t,what,false) + end +end + +function setters.register(t,what,...) + local data = t.data + what = lower(what) + local w = data[what] + if not w then + w = { } + data[what] = w + end + for _, fnc in next, { ... } do + local typ = type(fnc) + if typ == "function" then + w[#w+1] = fnc + elseif typ == "string" then + w[#w+1] = function(value) set(t,fnc,value,nesting) end + end + end +end + +function setters.enable(t,what) + local e = t.enable + t.enable, t.done = enable, { } + enable(t,string.simpleesc(tostring(what))) + t.enable, t.done = e, { } +end + +function setters.disable(t,what) + local e = t.disable + t.disable, t.done = disable, { } + disable(t,string.simpleesc(tostring(what))) + t.disable, t.done = e, { } +end + +function setters.reset(t) + t.done = { } + reset(t) +end + +function setters.list(t) -- pattern + local list = table.sortedkeys(t.data) + local user, system = { }, { } + for l=1,#list do + local what = list[l] + if find(what,"^%*") then + system[#system+1] = what + else + user[#user+1] = what + end + end + return user, system +end + +function setters.show(t) + commands.writestatus("","") + local list = setters.list(t) + for k=1,#list do + commands.writestatus(t.name,list[k]) + end + commands.writestatus("","") +end + +-- we could have used a bit of oo and the trackers:enable syntax but +-- there is already a lot of code around using the singular tracker + +-- we could make this into a module + +function setters.new(name) + local t + t = { + data = { }, + name = name, + enable = function(...) setters.enable (t,...) end, + disable = function(...) setters.disable (t,...) end, + register = function(...) setters.register(t,...) end, + list = function(...) setters.list (t,...) end, + show = function(...) setters.show (t,...) end, + } + setters.data[name] = t + return t +end + +trackers = setters.new("trackers") +directives = setters.new("directives") +experiments = setters.new("experiments") + +-- nice trick: we overload two of the directives related functions with variants that +-- do tracing (itself using a tracker) .. proof of concept + +local trace_directives = false local trace_directives = false trackers.register("system.directives", function(v) trace_directives = v end) +local trace_experiments = false local trace_experiments = false trackers.register("system.experiments", function(v) trace_experiments = v end) + +local e = directives.enable +local d = directives.disable + +function directives.enable(...) + commands.writestatus("directives","enabling: %s",concat({...}," ")) + e(...) +end + +function directives.disable(...) + commands.writestatus("directives","disabling: %s",concat({...}," ")) + d(...) +end + +local e = experiments.enable +local d = experiments.disable + +function experiments.enable(...) + commands.writestatus("experiments","enabling: %s",concat({...}," ")) + e(...) +end + +function experiments.disable(...) + commands.writestatus("experiments","disabling: %s",concat({...}," ")) + d(...) +end + +-- a useful example + +directives.register("system.nostatistics", function(v) + statistics.enable = not v +end) + + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['lxml-tab'] = { version = 1.001, comment = "this module is the basis for the lxml-* ones", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", @@ -2164,46 +3900,34 @@ if not modules then modules = { } end modules ['l-xml'] = { license = "see context related readme files" } --- RJ: key=value ... lpeg.Ca(lpeg.Cc({}) * (pattern-producing-key-and-value / rawset)^0) +-- this module needs a cleanup: check latest lpeg, passing args, (sub)grammar, etc etc +-- stripping spaces from e.g. cont-en.xml saves .2 sec runtime so it's not worth the +-- trouble --- some code may move to l-xmlext +local trace_entities = false trackers.register("xml.entities", function(v) trace_entities = v end) --[[ldx-- <p>The parser used here is inspired by the variant discussed in the lua book, but handles comment and processing instructions, has a different structure, provides -parent access; a first version used different tricky but was less optimized to we +parent access; a first version used different trickery but was less optimized to we went this route. First we had a find based parser, now we have an <l n='lpeg'/> based one. The find based parser can be found in l-xml-edu.lua along with other older code.</p> -<p>Expecially the lpath code is experimental, we will support some of xpath, but -only things that make sense for us; as compensation it is possible to hook in your -own functions. Apart from preprocessing content for <l n='context'/> we also need -this module for process management, like handling <l n='ctx'/> and <l n='rlx'/> -files.</p> - -<typing> -a/b/c /*/c -a/b/c/first() a/b/c/last() a/b/c/index(n) a/b/c/index(-n) -a/b/c/text() a/b/c/text(1) a/b/c/text(-1) a/b/c/text(n) -</typing> - <p>Beware, the interface may change. For instance at, ns, tg, dt may get more verbose names. Once the code is stable we will also remove some tracing and optimize the code.</p> --ldx]]-- xml = xml or { } -tex = tex or { } - -xml.trace_lpath = false -xml.trace_print = false -xml.trace_remap = false -local format, concat = string.format, table.concat +--~ local xml = xml ---~ local pairs, next, type = pairs, next, type - --- todo: some things per xml file, liek namespace remapping +local concat, remove, insert = table.concat, table.remove, table.insert +local type, next, setmetatable, getmetatable, tonumber = type, next, setmetatable, getmetatable, tonumber +local format, lower, find, match, gsub = string.format, string.lower, string.find, string.match, string.gsub +local utfchar = unicode.utf8.char +local lpegmatch = lpeg.match +local P, S, R, C, V, C, Cs = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.C, lpeg.Cs --[[ldx-- <p>First a hack to enable namespace resolving. A namespace is characterized by @@ -2215,65 +3939,61 @@ much cleaner.</p> xml.xmlns = xml.xmlns or { } -do - - local check = lpeg.P(false) - local parse = check +local check = P(false) +local parse = check - --[[ldx-- - <p>The next function associates a namespace prefix with an <l n='url'/>. This - normally happens independent of parsing.</p> +--[[ldx-- +<p>The next function associates a namespace prefix with an <l n='url'/>. This +normally happens independent of parsing.</p> - <typing> - xml.registerns("mml","mathml") - </typing> - --ldx]]-- +<typing> +xml.registerns("mml","mathml") +</typing> +--ldx]]-- - function xml.registerns(namespace, pattern) -- pattern can be an lpeg - check = check + lpeg.C(lpeg.P(pattern:lower())) / namespace - parse = lpeg.P { lpeg.P(check) + 1 * lpeg.V(1) } - end +function xml.registerns(namespace, pattern) -- pattern can be an lpeg + check = check + C(P(lower(pattern))) / namespace + parse = P { P(check) + 1 * V(1) } +end - --[[ldx-- - <p>The next function also registers a namespace, but this time we map a - given namespace prefix onto a registered one, using the given - <l n='url'/>. This used for attributes like <t>xmlns:m</t>.</p> +--[[ldx-- +<p>The next function also registers a namespace, but this time we map a +given namespace prefix onto a registered one, using the given +<l n='url'/>. This used for attributes like <t>xmlns:m</t>.</p> - <typing> - xml.checkns("m","http://www.w3.org/mathml") - </typing> - --ldx]]-- +<typing> +xml.checkns("m","http://www.w3.org/mathml") +</typing> +--ldx]]-- - function xml.checkns(namespace,url) - local ns = parse:match(url:lower()) - if ns and namespace ~= ns then - xml.xmlns[namespace] = ns - end +function xml.checkns(namespace,url) + local ns = lpegmatch(parse,lower(url)) + if ns and namespace ~= ns then + xml.xmlns[namespace] = ns end +end - --[[ldx-- - <p>Next we provide a way to turn an <l n='url'/> into a registered - namespace. This used for the <t>xmlns</t> attribute.</p> - - <typing> - resolvedns = xml.resolvens("http://www.w3.org/mathml") - </typing> - - This returns <t>mml</t>. - --ldx]]-- +--[[ldx-- +<p>Next we provide a way to turn an <l n='url'/> into a registered +namespace. This used for the <t>xmlns</t> attribute.</p> - function xml.resolvens(url) - return parse:match(url:lower()) or "" - end +<typing> +resolvedns = xml.resolvens("http://www.w3.org/mathml") +</typing> - --[[ldx-- - <p>A namespace in an element can be remapped onto the registered - one efficiently by using the <t>xml.xmlns</t> table.</p> - --ldx]]-- +This returns <t>mml</t>. +--ldx]]-- +function xml.resolvens(url) + return lpegmatch(parse,lower(url)) or "" end --[[ldx-- +<p>A namespace in an element can be remapped onto the registered +one efficiently by using the <t>xml.xmlns</t> table.</p> +--ldx]]-- + +--[[ldx-- <p>This version uses <l n='lpeg'/>. We follow the same approach as before, stack and top and such. This version is about twice as fast which is mostly due to the fact that we don't have to prepare the stream for cdata, doctype etc etc. This variant is @@ -2304,272 +4024,570 @@ local x = xml.convert(somestring) <p>An optional second boolean argument tells this function not to create a root element.</p> + +<p>Valid entities are:</p> + +<typing> +<!ENTITY xxxx SYSTEM "yyyy" NDATA zzzz> +<!ENTITY xxxx PUBLIC "yyyy" > +<!ENTITY xxxx "yyyy" > +</typing> --ldx]]-- -xml.strip_cm_and_dt = false -- an extra global flag, in case we have many includes +-- not just one big nested table capture (lpeg overflow) -do +local nsremap, resolvens = xml.xmlns, xml.resolvens - -- not just one big nested table capture (lpeg overflow) +local stack, top, dt, at, xmlns, errorstr, entities = { }, { }, { }, { }, { }, nil, { } +local strip, cleanup, utfize, resolve, resolve_predefined, unify_predefined = false, false, false, false, false, false +local dcache, hcache, acache = { }, { }, { } - local remove, nsremap, resolvens = table.remove, xml.xmlns, xml.resolvens +local mt = { } - local stack, top, dt, at, xmlns, errorstr, entities = {}, {}, {}, {}, {}, nil, {} +function initialize_mt(root) + mt = { __index = root } -- will be redefined later +end - local mt = { __tostring = xml.text } +function xml.setproperty(root,k,v) + getmetatable(root).__index[k] = v +end - function xml.check_error(top,toclose) - return "" +function xml.check_error(top,toclose) + return "" +end + +local function add_attribute(namespace,tag,value) + if cleanup and #value > 0 then + value = cleanup(value) -- new + end + if tag == "xmlns" then + xmlns[#xmlns+1] = resolvens(value) + at[tag] = value + elseif namespace == "" then + at[tag] = value + elseif namespace == "xmlns" then + xml.checkns(tag,value) + at["xmlns:" .. tag] = value + else + -- for the moment this way: + at[namespace .. ":" .. tag] = value + end +end + +local function add_empty(spacing, namespace, tag) + if #spacing > 0 then + dt[#dt+1] = spacing + end + local resolved = (namespace == "" and xmlns[#xmlns]) or nsremap[namespace] or namespace + top = stack[#stack] + dt = top.dt + local t = { ns=namespace or "", rn=resolved, tg=tag, at=at, dt={}, __p__ = top } + dt[#dt+1] = t + setmetatable(t, mt) + if at.xmlns then + remove(xmlns) + end + at = { } +end + +local function add_begin(spacing, namespace, tag) + if #spacing > 0 then + dt[#dt+1] = spacing + end + local resolved = (namespace == "" and xmlns[#xmlns]) or nsremap[namespace] or namespace + top = { ns=namespace or "", rn=resolved, tg=tag, at=at, dt={}, __p__ = stack[#stack] } + setmetatable(top, mt) + dt = top.dt + stack[#stack+1] = top + at = { } +end + +local function add_end(spacing, namespace, tag) + if #spacing > 0 then + dt[#dt+1] = spacing + end + local toclose = remove(stack) + top = stack[#stack] + if #stack < 1 then + errorstr = format("nothing to close with %s %s", tag, xml.check_error(top,toclose) or "") + elseif toclose.tg ~= tag then -- no namespace check + errorstr = format("unable to close %s with %s %s", toclose.tg, tag, xml.check_error(top,toclose) or "") + end + dt = top.dt + dt[#dt+1] = toclose + -- dt[0] = top -- nasty circular reference when serializing table + if toclose.at.xmlns then + remove(xmlns) + end +end + +local function add_text(text) + if cleanup and #text > 0 then + dt[#dt+1] = cleanup(text) + else + dt[#dt+1] = text + end +end + +local function add_special(what, spacing, text) + if #spacing > 0 then + dt[#dt+1] = spacing + end + if strip and (what == "@cm@" or what == "@dt@") then + -- forget it + else + dt[#dt+1] = { special=true, ns="", tg=what, dt={ text } } + end +end + +local function set_message(txt) + errorstr = "garbage at the end of the file: " .. gsub(txt,"([ \n\r\t]*)","") +end + +local reported_attribute_errors = { } + +local function attribute_value_error(str) + if not reported_attribute_errors[str] then + logs.report("xml","invalid attribute value: %q",str) + reported_attribute_errors[str] = true + at._error_ = str + end + return str +end +local function attribute_specification_error(str) + if not reported_attribute_errors[str] then + logs.report("xml","invalid attribute specification: %q",str) + reported_attribute_errors[str] = true + at._error_ = str end + return str +end - local strip = false - local cleanup = false +function xml.unknown_dec_entity_format(str) return (str == "" and "&error;") or format("&%s;",str) end +function xml.unknown_hex_entity_format(str) return format("&#x%s;",str) end +function xml.unknown_any_entity_format(str) return format("&#x%s;",str) end - function xml.set_text_cleanup(fnc) - cleanup = fnc +local function fromhex(s) + local n = tonumber(s,16) + if n then + return utfchar(n) + else + return format("h:%s",s), true + end +end + +local function fromdec(s) + local n = tonumber(s) + if n then + return utfchar(n) + else + return format("d:%s",s), true end +end + +-- one level expansion (simple case), no checking done + +local rest = (1-P(";"))^0 +local many = P(1)^0 + +local parsedentity = + P("&") * (P("#x")*(rest/fromhex) + P("#")*(rest/fromdec)) * P(";") * P(-1) + + (P("#x")*(many/fromhex) + P("#")*(many/fromdec)) + +-- parsing in the xml file + +local predefined_unified = { + [38] = "&", + [42] = """, + [47] = "'", + [74] = "<", + [76] = "&gr;", +} + +local predefined_simplified = { + [38] = "&", amp = "&", + [42] = '"', quot = '"', + [47] = "'", apos = "'", + [74] = "<", lt = "<", + [76] = ">", gt = ">", +} - local function add_attribute(namespace,tag,value) - if tag == "xmlns" then - xmlns[#xmlns+1] = resolvens(value) - at[tag] = value - elseif namespace == "xmlns" then - xml.checkns(tag,value) - at["xmlns:" .. tag] = value +local function handle_hex_entity(str) + local h = hcache[str] + if not h then + local n = tonumber(str,16) + h = unify_predefined and predefined_unified[n] + if h then + if trace_entities then + logs.report("xml","utfize, converting hex entity &#x%s; into %s",str,h) + end + elseif utfize then + h = (n and utfchar(n)) or xml.unknown_hex_entity_format(str) or "" + if not n then + logs.report("xml","utfize, ignoring hex entity &#x%s;",str) + elseif trace_entities then + logs.report("xml","utfize, converting hex entity &#x%s; into %s",str,h) + end else - at[tag] = value - end - end - local function add_begin(spacing, namespace, tag) - if #spacing > 0 then - dt[#dt+1] = spacing - end - local resolved = (namespace == "" and xmlns[#xmlns]) or nsremap[namespace] or namespace - top = { ns=namespace or "", rn=resolved, tg=tag, at=at, dt={}, __p__ = stack[#stack] } - setmetatable(top, mt) - dt = top.dt - stack[#stack+1] = top - at = { } - end - local function add_end(spacing, namespace, tag) - if #spacing > 0 then - dt[#dt+1] = spacing - end - local toclose = remove(stack) - top = stack[#stack] - if #stack < 1 then - errorstr = format("nothing to close with %s %s", tag, xml.check_error(top,toclose) or "") - elseif toclose.tg ~= tag then -- no namespace check - errorstr = format("unable to close %s with %s %s", toclose.tg, tag, xml.check_error(top,toclose) or "") - end - dt = top.dt - dt[#dt+1] = toclose - if at.xmlns then - remove(xmlns) - end - end - local function add_empty(spacing, namespace, tag) - if #spacing > 0 then - dt[#dt+1] = spacing - end - local resolved = (namespace == "" and xmlns[#xmlns]) or nsremap[namespace] or namespace - top = stack[#stack] - dt = top.dt - local t = { ns=namespace or "", rn=resolved, tg=tag, at=at, dt={}, __p__ = top } - dt[#dt+1] = t - setmetatable(t, mt) - at = { } - if at.xmlns then - remove(xmlns) - end - end - local function add_text(text) - if cleanup and #text > 0 then - dt[#dt+1] = cleanup(text) + if trace_entities then + logs.report("xml","found entity &#x%s;",str) + end + h = "&#x" .. str .. ";" + end + hcache[str] = h + end + return h +end + +local function handle_dec_entity(str) + local d = dcache[str] + if not d then + local n = tonumber(str) + d = unify_predefined and predefined_unified[n] + if d then + if trace_entities then + logs.report("xml","utfize, converting dec entity &#%s; into %s",str,d) + end + elseif utfize then + d = (n and utfchar(n)) or xml.unknown_dec_entity_format(str) or "" + if not n then + logs.report("xml","utfize, ignoring dec entity &#%s;",str) + elseif trace_entities then + logs.report("xml","utfize, converting dec entity &#%s; into %s",str,h) + end else - dt[#dt+1] = text + if trace_entities then + logs.report("xml","found entity &#%s;",str) + end + d = "&#" .. str .. ";" end + dcache[str] = d end - local function add_special(what, spacing, text) - if #spacing > 0 then - dt[#dt+1] = spacing + return d +end + +xml.parsedentitylpeg = parsedentity + +local function handle_any_entity(str) + if resolve then + local a = acache[str] -- per instance ! todo + if not a then + a = resolve_predefined and predefined_simplified[str] + if a then + -- one of the predefined + elseif type(resolve) == "function" then + a = resolve(str) or entities[str] + else + a = entities[str] + end + if a then + if trace_entities then + logs.report("xml","resolved entity &%s; -> %s (internal)",str,a) + end + a = lpegmatch(parsedentity,a) or a + else + if xml.unknown_any_entity_format then + a = xml.unknown_any_entity_format(str) or "" + end + if a then + if trace_entities then + logs.report("xml","resolved entity &%s; -> %s (external)",str,a) + end + else + if trace_entities then + logs.report("xml","keeping entity &%s;",str) + end + if str == "" then + a = "&error;" + else + a = "&" .. str .. ";" + end + end + end + acache[str] = a + elseif trace_entities then + if not acache[str] then + logs.report("xml","converting entity &%s; into %s",str,a) + acache[str] = a + end end - if strip and (what == "@cm@" or what == "@dt@") then - -- forget it - else - dt[#dt+1] = { special=true, ns="", tg=what, dt={text} } - end - end - local function set_message(txt) - errorstr = "garbage at the end of the file: " .. txt:gsub("([ \n\r\t]*)","") - end - - local P, S, R, C, V = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V - - local space = S(' \r\n\t') - local open = P('<') - local close = P('>') - local squote = S("'") - local dquote = S('"') - local equal = P('=') - local slash = P('/') - local colon = P(':') - local valid = R('az', 'AZ', '09') + S('_-.') - local name_yes = C(valid^1) * colon * C(valid^1) - local name_nop = C(P(true)) * C(valid^1) - local name = name_yes + name_nop - - local utfbom = P('\000\000\254\255') + P('\255\254\000\000') + - P('\255\254') + P('\254\255') + P('\239\187\191') -- no capture - - local spacing = C(space^0) - local justtext = C((1-open)^1) - local somespace = space^1 - local optionalspace = space^0 - - local value = (squote * C((1 - squote)^0) * squote) + (dquote * C((1 - dquote)^0) * dquote) - local attribute = (somespace * name * optionalspace * equal * optionalspace * value) / add_attribute - local attributes = attribute^0 - - local text = justtext / add_text - local balanced = P { "[" * ((1 - S"[]") + V(1))^0 * "]" } -- taken from lpeg manual, () example - - local emptyelement = (spacing * open * name * attributes * optionalspace * slash * close) / add_empty - local beginelement = (spacing * open * name * attributes * optionalspace * close) / add_begin - local endelement = (spacing * open * slash * name * optionalspace * close) / add_end - - local begincomment = open * P("!--") - local endcomment = P("--") * close - local begininstruction = open * P("?") - local endinstruction = P("?") * close - local begincdata = open * P("![CDATA[") - local endcdata = P("]]") * close - - local someinstruction = C((1 - endinstruction)^0) - local somecomment = C((1 - endcomment )^0) - local somecdata = C((1 - endcdata )^0) - - function entity(k,v) entities[k] = v end - - local begindoctype = open * P("!DOCTYPE") - local enddoctype = close - local beginset = P("[") - local endset = P("]") - local doctypename = C((1-somespace)^0) - local elementdoctype = optionalspace * P("<!ELEMENT") * (1-close)^0 * close - local entitydoctype = optionalspace * P("<!ENTITY") * somespace * (doctypename * somespace * value)/entity * optionalspace * close - local publicdoctype = doctypename * somespace * P("PUBLIC") * somespace * value * somespace * value * somespace - local systemdoctype = doctypename * somespace * P("SYSTEM") * somespace * value * somespace - local definitiondoctype= doctypename * somespace * beginset * P(elementdoctype + entitydoctype)^0 * optionalspace * endset - local simpledoctype = (1-close)^1 -- * balanced^0 - local somedoctype = C((somespace * (publicdoctype + systemdoctype + definitiondoctype + simpledoctype) * optionalspace)^0) - - local instruction = (spacing * begininstruction * someinstruction * endinstruction) / function(...) add_special("@pi@",...) end - local comment = (spacing * begincomment * somecomment * endcomment ) / function(...) add_special("@cm@",...) end - local cdata = (spacing * begincdata * somecdata * endcdata ) / function(...) add_special("@cd@",...) end - local doctype = (spacing * begindoctype * somedoctype * enddoctype ) / function(...) add_special("@dt@",...) end - - -- nicer but slower: - -- - -- local instruction = (lpeg.Cc("@pi@") * spacing * begininstruction * someinstruction * endinstruction) / add_special - -- local comment = (lpeg.Cc("@cm@") * spacing * begincomment * somecomment * endcomment ) / add_special - -- local cdata = (lpeg.Cc("@cd@") * spacing * begincdata * somecdata * endcdata ) / add_special - -- local doctype = (lpeg.Cc("@dt@") * spacing * begindoctype * somedoctype * enddoctype ) / add_special + return a + else + local a = acache[str] + if not a then + if trace_entities then + logs.report("xml","found entity &%s;",str) + end + a = resolve_predefined and predefined_simplified[str] + if a then + -- one of the predefined + acache[str] = a + elseif str == "" then + a = "&error;" + acache[str] = a + else + a = "&" .. str .. ";" + acache[str] = a + end + end + return a + end +end + +local function handle_end_entity(chr) + logs.report("xml","error in entity, %q found instead of ';'",chr) +end + +local space = S(' \r\n\t') +local open = P('<') +local close = P('>') +local squote = S("'") +local dquote = S('"') +local equal = P('=') +local slash = P('/') +local colon = P(':') +local semicolon = P(';') +local ampersand = P('&') +local valid = R('az', 'AZ', '09') + S('_-.') +local name_yes = C(valid^1) * colon * C(valid^1) +local name_nop = C(P(true)) * C(valid^1) +local name = name_yes + name_nop +local utfbom = lpeg.patterns.utfbom -- no capture +local spacing = C(space^0) + +----- entitycontent = (1-open-semicolon)^0 +local anyentitycontent = (1-open-semicolon-space-close)^0 +local hexentitycontent = R("AF","af","09")^0 +local decentitycontent = R("09")^0 +local parsedentity = P("#")/"" * ( + P("x")/"" * (hexentitycontent/handle_hex_entity) + + (decentitycontent/handle_dec_entity) + ) + (anyentitycontent/handle_any_entity) +local entity = ampersand/"" * parsedentity * ( (semicolon/"") + #(P(1)/handle_end_entity)) + +local text_unparsed = C((1-open)^1) +local text_parsed = Cs(((1-open-ampersand)^1 + entity)^1) + +local somespace = space^1 +local optionalspace = space^0 + +----- value = (squote * C((1 - squote)^0) * squote) + (dquote * C((1 - dquote)^0) * dquote) -- ampersand and < also invalid in value +local value = (squote * Cs((entity + (1 - squote))^0) * squote) + (dquote * Cs((entity + (1 - dquote))^0) * dquote) -- ampersand and < also invalid in value + +local endofattributes = slash * close + close -- recovery of flacky html +local whatever = space * name * optionalspace * equal +local wrongvalue = C(P(1-whatever-close)^1 + P(1-close)^1) / attribute_value_error +----- wrongvalue = C(P(1-whatever-endofattributes)^1 + P(1-endofattributes)^1) / attribute_value_error +----- wrongvalue = C(P(1-space-endofattributes)^1) / attribute_value_error +local wrongvalue = Cs(P(entity + (1-space-endofattributes))^1) / attribute_value_error + +local attributevalue = value + wrongvalue + +local attribute = (somespace * name * optionalspace * equal * optionalspace * attributevalue) / add_attribute +----- attributes = (attribute)^0 + +local attributes = (attribute + somespace^-1 * (((1-endofattributes)^1)/attribute_specification_error))^0 + +local parsedtext = text_parsed / add_text +local unparsedtext = text_unparsed / add_text +local balanced = P { "[" * ((1 - S"[]") + V(1))^0 * "]" } -- taken from lpeg manual, () example + +local emptyelement = (spacing * open * name * attributes * optionalspace * slash * close) / add_empty +local beginelement = (spacing * open * name * attributes * optionalspace * close) / add_begin +local endelement = (spacing * open * slash * name * optionalspace * close) / add_end + +local begincomment = open * P("!--") +local endcomment = P("--") * close +local begininstruction = open * P("?") +local endinstruction = P("?") * close +local begincdata = open * P("![CDATA[") +local endcdata = P("]]") * close + +local someinstruction = C((1 - endinstruction)^0) +local somecomment = C((1 - endcomment )^0) +local somecdata = C((1 - endcdata )^0) + +local function normalentity(k,v ) entities[k] = v end +local function systementity(k,v,n) entities[k] = v end +local function publicentity(k,v,n) entities[k] = v end + +local begindoctype = open * P("!DOCTYPE") +local enddoctype = close +local beginset = P("[") +local endset = P("]") +local doctypename = C((1-somespace-close)^0) +local elementdoctype = optionalspace * P("<!ELEMENT") * (1-close)^0 * close + +local normalentitytype = (doctypename * somespace * value)/normalentity +local publicentitytype = (doctypename * somespace * P("PUBLIC") * somespace * value)/publicentity +local systementitytype = (doctypename * somespace * P("SYSTEM") * somespace * value * somespace * P("NDATA") * somespace * doctypename)/systementity +local entitydoctype = optionalspace * P("<!ENTITY") * somespace * (systementitytype + publicentitytype + normalentitytype) * optionalspace * close + +local doctypeset = beginset * optionalspace * P(elementdoctype + entitydoctype + space)^0 * optionalspace * endset +local definitiondoctype= doctypename * somespace * doctypeset +local publicdoctype = doctypename * somespace * P("PUBLIC") * somespace * value * somespace * value * somespace * doctypeset +local systemdoctype = doctypename * somespace * P("SYSTEM") * somespace * value * somespace * doctypeset +local simpledoctype = (1-close)^1 -- * balanced^0 +local somedoctype = C((somespace * (publicdoctype + systemdoctype + definitiondoctype + simpledoctype) * optionalspace)^0) + +local instruction = (spacing * begininstruction * someinstruction * endinstruction) / function(...) add_special("@pi@",...) end +local comment = (spacing * begincomment * somecomment * endcomment ) / function(...) add_special("@cm@",...) end +local cdata = (spacing * begincdata * somecdata * endcdata ) / function(...) add_special("@cd@",...) end +local doctype = (spacing * begindoctype * somedoctype * enddoctype ) / function(...) add_special("@dt@",...) end + +-- nicer but slower: +-- +-- local instruction = (Cc("@pi@") * spacing * begininstruction * someinstruction * endinstruction) / add_special +-- local comment = (Cc("@cm@") * spacing * begincomment * somecomment * endcomment ) / add_special +-- local cdata = (Cc("@cd@") * spacing * begincdata * somecdata * endcdata ) / add_special +-- local doctype = (Cc("@dt@") * spacing * begindoctype * somedoctype * enddoctype ) / add_special - local trailer = space^0 * (justtext/set_message)^0 +local trailer = space^0 * (text_unparsed/set_message)^0 - -- comment + emptyelement + text + cdata + instruction + V("parent"), -- 6.5 seconds on 40 MB database file - -- text + comment + emptyelement + cdata + instruction + V("parent"), -- 5.8 - -- text + V("parent") + emptyelement + comment + cdata + instruction, -- 5.5 +-- comment + emptyelement + text + cdata + instruction + V("parent"), -- 6.5 seconds on 40 MB database file +-- text + comment + emptyelement + cdata + instruction + V("parent"), -- 5.8 +-- text + V("parent") + emptyelement + comment + cdata + instruction, -- 5.5 - local grammar = P { "preamble", - preamble = utfbom^0 * instruction^0 * (doctype + comment + instruction)^0 * V("parent") * trailer, - parent = beginelement * V("children")^0 * endelement, - children = text + V("parent") + emptyelement + comment + cdata + instruction, - } +local grammar_parsed_text = P { "preamble", + preamble = utfbom^0 * instruction^0 * (doctype + comment + instruction)^0 * V("parent") * trailer, + parent = beginelement * V("children")^0 * endelement, + children = parsedtext + V("parent") + emptyelement + comment + cdata + instruction, +} - -- todo: xml.new + properties like entities and strip and such (store in root) - - function xml.convert(data, no_root, strip_cm_and_dt, given_entities) -- maybe use table met k/v (given_entities may disapear) - strip = strip_cm_and_dt or xml.strip_cm_and_dt - stack, top, at, xmlns, errorstr, result, entities = {}, {}, {}, {}, nil, nil, given_entities or {} - stack[#stack+1] = top - top.dt = { } - dt = top.dt - if not data or data == "" then - errorstr = "empty xml file" - elseif not grammar:match(data) then - errorstr = "invalid xml file" +local grammar_unparsed_text = P { "preamble", + preamble = utfbom^0 * instruction^0 * (doctype + comment + instruction)^0 * V("parent") * trailer, + parent = beginelement * V("children")^0 * endelement, + children = unparsedtext + V("parent") + emptyelement + comment + cdata + instruction, +} + +-- maybe we will add settinsg to result as well + +local function xmlconvert(data, settings) + settings = settings or { } -- no_root strip_cm_and_dt given_entities parent_root error_handler + strip = settings.strip_cm_and_dt + utfize = settings.utfize_entities + resolve = settings.resolve_entities + resolve_predefined = settings.resolve_predefined_entities -- in case we have escaped entities + unify_predefined = settings.unify_predefined_entities -- & -> & + cleanup = settings.text_cleanup + stack, top, at, xmlns, errorstr, result, entities = { }, { }, { }, { }, nil, nil, settings.entities or { } + acache, hcache, dcache = { }, { }, { } -- not stored + reported_attribute_errors = { } + if settings.parent_root then + mt = getmetatable(settings.parent_root) + else + initialize_mt(top) + end + stack[#stack+1] = top + top.dt = { } + dt = top.dt + if not data or data == "" then + errorstr = "empty xml file" + elseif utfize or resolve then + if lpegmatch(grammar_parsed_text,data) then + errorstr = "" else + errorstr = "invalid xml file - parsed text" + end + elseif type(data) == "string" then + if lpegmatch(grammar_unparsed_text,data) then errorstr = "" + else + errorstr = "invalid xml file - unparsed text" end - if errorstr and errorstr ~= "" then - result = { dt = { { ns = "", tg = "error", dt = { errorstr }, at={}, er = true } }, error = true } - setmetatable(stack, mt) - if xml.error_handler then xml.error_handler("load",errorstr) end + else + errorstr = "invalid xml file - no text at all" + end + if errorstr and errorstr ~= "" then + result = { dt = { { ns = "", tg = "error", dt = { errorstr }, at={ }, er = true } } } + setmetatable(stack, mt) + local error_handler = settings.error_handler + if error_handler == false then + -- no error message else - result = stack[1] - end - if not no_root then - result = { special = true, ns = "", tg = '@rt@', dt = result.dt, at={}, entities = entities } - setmetatable(result, mt) - local rdt = result.dt - for k=1,#rdt do - local v = rdt[k] - if type(v) == "table" and not v.special then -- always table -) - result.ri = k -- rootindex - break - end + error_handler = error_handler or xml.error_handler + if error_handler then + xml.error_handler("load",errorstr) end end - return result + else + result = stack[1] + end + if not settings.no_root then + result = { special = true, ns = "", tg = '@rt@', dt = result.dt, at={ }, entities = entities, settings = settings } + setmetatable(result, mt) + local rdt = result.dt + for k=1,#rdt do + local v = rdt[k] + if type(v) == "table" and not v.special then -- always table -) + result.ri = k -- rootindex +v.__p__ = result -- new, experiment, else we cannot go back to settings, we need to test this ! + break + end + end + end + if errorstr and errorstr ~= "" then + result.error = true end + return result +end - --[[ldx-- - <p>Packaging data in an xml like table is done with the following - function. Maybe it will go away (when not used).</p> - --ldx]]-- +xml.convert = xmlconvert - function xml.is_valid(root) - return root and root.dt and root.dt[1] and type(root.dt[1]) == "table" and not root.dt[1].er - end +function xml.inheritedconvert(data,xmldata) + local settings = xmldata.settings + settings.parent_root = xmldata -- to be tested + -- settings.no_root = true + local xc = xmlconvert(data,settings) + -- xc.settings = nil + -- xc.entities = nil + -- xc.special = nil + -- xc.ri = nil + -- print(xc.tg) + return xc +end - function xml.package(tag,attributes,data) - local ns, tg = tag:match("^(.-):?([^:]+)$") - local t = { ns = ns, tg = tg, dt = data or "", at = attributes or {} } - setmetatable(t, mt) - return t - end +--[[ldx-- +<p>Packaging data in an xml like table is done with the following +function. Maybe it will go away (when not used).</p> +--ldx]]-- - function xml.is_valid(root) - return root and not root.error - end +function xml.is_valid(root) + return root and root.dt and root.dt[1] and type(root.dt[1]) == "table" and not root.dt[1].er +end - xml.error_handler = (logs and logs.report) or (input and input.report) or print +function xml.package(tag,attributes,data) + local ns, tg = match(tag,"^(.-):?([^:]+)$") + local t = { ns = ns, tg = tg, dt = data or "", at = attributes or {} } + setmetatable(t, mt) + return t +end +function xml.is_valid(root) + return root and not root.error end +xml.error_handler = (logs and logs.report) or (input and logs.report) or print + --[[ldx-- <p>We cannot load an <l n='lpeg'/> from a filehandle so we need to load the whole file first. The function accepts a string representing a filename or a file handle.</p> --ldx]]-- -function xml.load(filename) +function xml.load(filename,settings) + local data = "" if type(filename) == "string" then + -- local data = io.loaddata(filename) - -todo: check type in io.loaddata local f = io.open(filename,'r') if f then - local root = xml.convert(f:read("*all")) + data = f:read("*all") f:close() - return root - else - return xml.convert("") end elseif filename then -- filehandle - return xml.convert(filename:read("*all")) - else - return xml.convert("") + data = filename:read("*all") end + return xmlconvert(data,settings) end --[[ldx-- @@ -2577,9 +4595,11 @@ end valid trees, which is what the next function does.</p> --ldx]]-- +local no_root = { no_root = true } + function xml.toxml(data) if type(data) == "string" then - local root = { xml.convert(data,true) } + local root = { xmlconvert(data,no_root) } return (#root > 1 and root) or root[1] else return data @@ -2592,32 +4612,28 @@ generic table copier. Since we know what we're dealing with we can speed up things a bit. The second argument is not to be used!</p> --ldx]]-- -do - - function copy(old,tables) - if old then - tables = tables or { } - local new = { } - if not tables[old] then - tables[old] = new - end - for k,v in pairs(old) do - new[k] = (type(v) == "table" and (tables[v] or copy(v, tables))) or v - end - local mt = getmetatable(old) - if mt then - setmetatable(new,mt) - end - return new - else - return { } +local function copy(old,tables) + if old then + tables = tables or { } + local new = { } + if not tables[old] then + tables[old] = new end + for k,v in next, old do + new[k] = (type(v) == "table" and (tables[v] or copy(v, tables))) or v + end + local mt = getmetatable(old) + if mt then + setmetatable(new,mt) + end + return new + else + return { } end - - xml.copy = copy - end +xml.copy = copy + --[[ldx-- <p>In <l n='context'/> serializing the tree or parts of the tree is a major actitivity which is why the following function is pretty optimized resulting @@ -2626,217 +4642,306 @@ function for all components is about 15% slower than the concatinating alternative.</p> --ldx]]-- -do +-- todo: add <?xml version='1.0' standalone='yes'?> when not present - -- todo: add <?xml version='1.0' standalone='yes'?> when not present - - local fallbackhandle = (tex and tex.sprint) or io.write - - local function serialize(e, handle, textconverter, attributeconverter, specialconverter, nocommands) - if not e then - return - elseif not nocommands then - local ec = e.command - if ec ~= nil then -- we can have all kind of types - -if e.special then -- todo test for true/false - local etg, edt = e.tg, e.dt - local spc = specialconverter and specialconverter[etg] - if spc then ---~ print("SPECIAL",etg,table.serialize(specialconverter), spc) - local result = spc(edt[1]) - if result then - handle(result) - return - else - -- no need to handle any further +function xml.checkbom(root) -- can be made faster + if root.ri then + local dt, found = root.dt, false + for k=1,#dt do + local v = dt[k] + if type(v) == "table" and v.special and v.tg == "@pi@" and find(v.dt[1],"xml.*version=") then + found = true + break + end + end + if not found then + insert(dt, 1, { special=true, ns="", tg="@pi@", dt = { "xml version='1.0' standalone='yes'"} } ) + insert(dt, 2, "\n" ) end end end - local xc = xml.command - if xc then - xc(e,ec) - return +--[[ldx-- +<p>At the cost of some 25% runtime overhead you can first convert the tree to a string +and then handle the lot.</p> +--ldx]]-- + +-- new experimental reorganized serialize + +local function verbose_element(e,handlers) + local handle = handlers.handle + local serialize = handlers.serialize + local ens, etg, eat, edt, ern = e.ns, e.tg, e.at, e.dt, e.rn + local ats = eat and next(eat) and { } + if ats then + for k,v in next, eat do + ats[#ats+1] = format('%s=%q',k,v) + end + end + if ern and trace_remap and ern ~= ens then + ens = ern + end + if ens ~= "" then + if edt and #edt > 0 then + if ats then + handle("<",ens,":",etg," ",concat(ats," "),">") + else + handle("<",ens,":",etg,">") + end + for i=1,#edt do + local e = edt[i] + if type(e) == "string" then + handle(e) + else + serialize(e,handlers) end end + handle("</",ens,":",etg,">") + else + if ats then + handle("<",ens,":",etg," ",concat(ats," "),"/>") + else + handle("<",ens,":",etg,"/>") + end end - handle = handle or fallbackhandle - local etg = e.tg - if etg then - if e.special then - local edt = e.dt - local spc = specialconverter and specialconverter[etg] - if spc then - local result = spc(edt[1]) - if result then - handle(result) - else - -- no need to handle any further - end - elseif etg == "@pi@" then - -- handle(format("<?%s?>",edt[1])) - handle("<?" .. edt[1] .. "?>") - elseif etg == "@cm@" then - -- handle(format("<!--%s-->",edt[1])) - handle("<!--" .. edt[1] .. "-->") - elseif etg == "@cd@" then - -- handle(format("<![CDATA[%s]]>",edt[1])) - handle("<![CDATA[" .. edt[1] .. "]]>") - elseif etg == "@dt@" then - -- handle(format("<!DOCTYPE %s>",edt[1])) - handle("<!DOCTYPE " .. edt[1] .. ">") - elseif etg == "@rt@" then - serialize(edt,handle,textconverter,attributeconverter,specialconverter,nocommands) - end + else + if edt and #edt > 0 then + if ats then + handle("<",etg," ",concat(ats," "),">") else - local ens, eat, edt, ern = e.ns, e.at, e.dt, e.rn - local ats = eat and next(eat) and { } -- type test maybe faster - if ats then - if attributeconverter then - for k,v in pairs(eat) do - ats[#ats+1] = format('%s=%q',k,attributeconverter(v)) - end - else - for k,v in pairs(eat) do - ats[#ats+1] = format('%s=%q',k,v) - end - end - end - if ern and xml.trace_remap and ern ~= ens then ---~ if ats then ---~ ats[#ats+1] = format("xmlns:remapped='%s'",ern) ---~ else ---~ ats = { format("xmlns:remapped='%s'",ern) } ---~ end ---~ if ats then ---~ ats[#ats+1] = format("remappedns='%s'",ens or '-') ---~ else ---~ ats = { format("remappedns='%s'",ens or '-') } ---~ end -ens = ern - end - if ens ~= "" then - if edt and #edt > 0 then - if ats then - -- handle(format("<%s:%s %s>",ens,etg,concat(ats," "))) - handle("<" .. ens .. ":" .. etg .. " " .. concat(ats," ") .. ">") - else - -- handle(format("<%s:%s>",ens,etg)) - handle("<" .. ens .. ":" .. etg .. ">") - end - for i=1,#edt do - local e = edt[i] - if type(e) == "string" then - if textconverter then - handle(textconverter(e)) - else - handle(e) - end - else - serialize(e,handle,textconverter,attributeconverter,specialconverter,nocommands) - end - end - -- handle(format("</%s:%s>",ens,etg)) - handle("</" .. ens .. ":" .. etg .. ">") - else - if ats then - -- handle(format("<%s:%s %s/>",ens,etg,concat(ats," "))) - handle("<" .. ens .. ":" .. etg .. " " .. concat(ats," ") .. "/>") - else - -- handle(format("<%s:%s/>",ens,etg)) - handle("<" .. ens .. ":" .. etg .. "/>") - end - end + handle("<",etg,">") + end + for i=1,#edt do + local ei = edt[i] + if type(ei) == "string" then + handle(ei) else - if edt and #edt > 0 then - if ats then - -- handle(format("<%s %s>",etg,concat(ats," "))) - handle("<" .. etg .. " " .. concat(ats," ") .. ">") - else - -- handle(format("<%s>",etg)) - handle("<" .. etg .. ">") - end - for i=1,#edt do - serialize(edt[i],handle,textconverter,attributeconverter,specialconverter,nocommands) - end - -- handle(format("</%s>",etg)) - handle("</" .. etg .. ">") - else - if ats then - -- handle(format("<%s %s/>",etg,concat(ats," "))) - handle("<" .. etg .. " " .. concat(ats," ") .. "/>") - else - -- handle(format("<%s/>",etg)) - handle("<" .. etg .. "/>") - end - end + serialize(ei,handlers) end end - elseif type(e) == "string" then - if textconverter then - handle(textconverter(e)) + handle("</",etg,">") + else + if ats then + handle("<",etg," ",concat(ats," "),"/>") else - handle(e) + handle("<",etg,"/>") end + end + end +end + +local function verbose_pi(e,handlers) + handlers.handle("<?",e.dt[1],"?>") +end + +local function verbose_comment(e,handlers) + handlers.handle("<!--",e.dt[1],"-->") +end + +local function verbose_cdata(e,handlers) + handlers.handle("<![CDATA[", e.dt[1],"]]>") +end + +local function verbose_doctype(e,handlers) + handlers.handle("<!DOCTYPE ",e.dt[1],">") +end + +local function verbose_root(e,handlers) + handlers.serialize(e.dt,handlers) +end + +local function verbose_text(e,handlers) + handlers.handle(e) +end + +local function verbose_document(e,handlers) + local serialize = handlers.serialize + local functions = handlers.functions + for i=1,#e do + local ei = e[i] + if type(ei) == "string" then + functions["@tx@"](ei,handlers) else - for i=1,#e do - serialize(e[i],handle,textconverter,attributeconverter,specialconverter,nocommands) - end + serialize(ei,handlers) end end +end - xml.serialize = serialize +local function serialize(e,handlers,...) + local initialize = handlers.initialize + local finalize = handlers.finalize + local functions = handlers.functions + if initialize then + local state = initialize(...) + if not state == true then + return state + end + end + local etg = e.tg + if etg then + (functions[etg] or functions["@el@"])(e,handlers) + -- elseif type(e) == "string" then + -- functions["@tx@"](e,handlers) + else + functions["@dc@"](e,handlers) + end + if finalize then + return finalize() + end +end - function xml.checkbom(root) -- can be made faster - if root.ri then - local dt, found = root.dt, false - for k,v in ipairs(dt) do - if type(v) == "table" and v.special and v.tg == "@pi" and v.dt:find("xml.*version=") then - found = true - break +local function xserialize(e,handlers) + local functions = handlers.functions + local etg = e.tg + if etg then + (functions[etg] or functions["@el@"])(e,handlers) + -- elseif type(e) == "string" then + -- functions["@tx@"](e,handlers) + else + functions["@dc@"](e,handlers) + end +end + +local handlers = { } + +local function newhandlers(settings) + local t = table.copy(handlers.verbose or { }) -- merge + if settings then + for k,v in next, settings do + if type(v) == "table" then + tk = t[k] if not tk then tk = { } t[k] = tk end + for kk,vv in next, v do + tk[kk] = vv end + else + t[k] = v end - if not found then - table.insert(dt, 1, { special=true, ns="", tg="@pi@", dt = { "xml version='1.0' standalone='yes'"} } ) - table.insert(dt, 2, "\n" ) - end + end + if settings.name then + handlers[settings.name] = t end end + return t +end - --[[ldx-- - <p>At the cost of some 25% runtime overhead you can first convert the tree to a string - and then handle the lot.</p> - --ldx]]-- +local nofunction = function() end - function xml.tostring(root) -- 25% overhead due to collecting - if root then +function xml.sethandlersfunction(handler,name,fnc) + handler.functions[name] = fnc or nofunction +end + +function xml.gethandlersfunction(handler,name) + return handler.functions[name] +end + +function xml.gethandlers(name) + return handlers[name] +end + +newhandlers { + name = "verbose", + initialize = false, -- faster than nil and mt lookup + finalize = false, -- faster than nil and mt lookup + serialize = xserialize, + handle = print, + functions = { + ["@dc@"] = verbose_document, + ["@dt@"] = verbose_doctype, + ["@rt@"] = verbose_root, + ["@el@"] = verbose_element, + ["@pi@"] = verbose_pi, + ["@cm@"] = verbose_comment, + ["@cd@"] = verbose_cdata, + ["@tx@"] = verbose_text, + } +} + +--[[ldx-- +<p>How you deal with saving data depends on your preferences. For a 40 MB database +file the timing on a 2.3 Core Duo are as follows (time in seconds):</p> + +<lines> +1.3 : load data from file to string +6.1 : convert string into tree +5.3 : saving in file using xmlsave +6.8 : converting to string using xml.tostring +3.6 : saving converted string in file +</lines> + +<p>Beware, these were timing with the old routine but measurements will not be that +much different I guess.</p> +--ldx]]-- + +-- maybe this will move to lxml-xml + +local result + +local xmlfilehandler = newhandlers { + name = "file", + initialize = function(name) result = io.open(name,"wb") return result end, + finalize = function() result:close() return true end, + handle = function(...) result:write(...) end, +} + +-- no checking on writeability here but not faster either +-- +-- local xmlfilehandler = newhandlers { +-- initialize = function(name) io.output(name,"wb") return true end, +-- finalize = function() io.close() return true end, +-- handle = io.write, +-- } + + +function xml.save(root,name) + serialize(root,xmlfilehandler,name) +end + +local result + +local xmlstringhandler = newhandlers { + name = "string", + initialize = function() result = { } return result end, + finalize = function() return concat(result) end, + handle = function(...) result[#result+1] = concat { ... } end +} + +local function xmltostring(root) -- 25% overhead due to collecting + if root then if type(root) == 'string' then return root - elseif next(root) then -- next is faster than type (and >0 test) - local result = { } - serialize(root,function(s) result[#result+1] = s end) - return concat(result,"") + else -- if next(root) then -- next is faster than type (and >0 test) + return serialize(root,xmlstringhandler) or "" end end - return "" - end + return "" +end +local function xmltext(root) -- inline + return (root and xmltostring(root)) or "" end +function initialize_mt(root) + mt = { __tostring = xmltext, __index = root } +end + +xml.defaulthandlers = handlers +xml.newhandlers = newhandlers +xml.serialize = serialize +xml.tostring = xmltostring + --[[ldx-- <p>The next function operated on the content only and needs a handle function that accepts a string.</p> --ldx]]-- -function xml.string(e,handle) +local function xmlstring(e,handle) if not handle or (e.special and e.tg ~= "@rt@") then -- nothing elseif e.tg then local edt = e.dt if edt then for i=1,#edt do - xml.string(edt[i],handle) + xmlstring(edt[i],handle) end end else @@ -2844,61 +4949,68 @@ function xml.string(e,handle) end end ---[[ldx-- -<p>How you deal with saving data depends on your preferences. For a 40 MB database -file the timing on a 2.3 Core Duo are as follows (time in seconds):</p> +xml.string = xmlstring -<lines> -1.3 : load data from file to string -6.1 : convert string into tree -5.3 : saving in file using xmlsave -6.8 : converting to string using xml.tostring -3.6 : saving converted string in file -</lines> - -<p>The save function is given below.</p> +--[[ldx-- +<p>A few helpers:</p> --ldx]]-- -function xml.save(root,name) - local f = io.open(name,"w") - if f then - xml.serialize(root,function(s) f:write(s) end) - f:close() +--~ xmlsetproperty(root,"settings",settings) + +function xml.settings(e) + while e do + local s = e.settings + if s then + return s + else + e = e.__p__ + end end + return nil end ---[[ldx-- -<p>A few helpers:</p> ---ldx]]-- +function xml.root(e) + local r = e + while e do + e = e.__p__ + if e then + r = e + end + end + return r +end -function xml.body(root) - return (root.ri and root.dt[root.ri]) or root +function xml.parent(root) + return root.__p__ end -function xml.text(root) - return (root and xml.tostring(root)) or "" +function xml.body(root) + return (root.ri and root.dt[root.ri]) or root -- not ok yet end -function xml.content(root) -- bugged - return (root and root.dt and xml.tostring(root.dt)) or "" +function xml.name(root) + if not root then + return "" + elseif root.ns == "" then + return root.tg + else + return root.ns .. ":" .. root.tg + end end --[[ldx-- <p>The next helper erases an element but keeps the table as it is, and since empty strings are not serialized (effectively) it does not harm. Copying the table would take more time. Usage:</p> - -<typing> -dt[k] = xml.empty() or xml.empty(dt,k) -</typing> --ldx]]-- -function xml.empty(dt,k) - if dt and k then - dt[k] = "" - return dt[k] - else - return "" +function xml.erase(dt,k) + if dt then + if k then + dt[k] = "" + else for k=1,#dt do + dt[1] = { "" } + end end end end @@ -2919,1777 +5031,2323 @@ function xml.assign(dt,k,root) end end +-- the following helpers may move + +--[[ldx-- +<p>The next helper assigns a tree (or string). Usage:</p> +<typing> +xml.tocdata(e) +xml.tocdata(e,"error") +</typing> +--ldx]]-- + +function xml.tocdata(e,wrapper) + local whatever = xmltostring(e.dt) + if wrapper then + whatever = format("<%s>%s</%s>",wrapper,whatever,wrapper) + end + local t = { special = true, ns = "", tg = "@cd@", at = {}, rn = "", dt = { whatever }, __p__ = e } + setmetatable(t,getmetatable(e)) + e.dt = { t } +end + +function xml.makestandalone(root) + if root.ri then + local dt = root.dt + for k=1,#dt do + local v = dt[k] + if type(v) == "table" and v.special and v.tg == "@pi@" then + local txt = v.dt[1] + if find(txt,"xml.*version=") then + v.dt[1] = txt .. " standalone='yes'" + break + end + end + end + end +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['lxml-pth'] = { + version = 1.001, + comment = "this module is the basis for the lxml-* ones", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- e.ni is only valid after a filter run + +local concat, remove, insert = table.concat, table.remove, table.insert +local type, next, tonumber, tostring, setmetatable, loadstring = type, next, tonumber, tostring, setmetatable, loadstring +local format, upper, lower, gmatch, gsub, find, rep = string.format, string.upper, string.lower, string.gmatch, string.gsub, string.find, string.rep +local lpegmatch = lpeg.match + +-- beware, this is not xpath ... e.g. position is different (currently) and +-- we have reverse-sibling as reversed preceding sibling + +--[[ldx-- +<p>This module can be used stand alone but also inside <l n='mkiv'/> in +which case it hooks into the tracker code. Therefore we provide a few +functions that set the tracers. Here we overload a previously defined +function.</p> +<p>If I can get in the mood I will make a variant that is XSLT compliant +but I wonder if it makes sense.</P> +--ldx]]-- + +--[[ldx-- +<p>Expecially the lpath code is experimental, we will support some of xpath, but +only things that make sense for us; as compensation it is possible to hook in your +own functions. Apart from preprocessing content for <l n='context'/> we also need +this module for process management, like handling <l n='ctx'/> and <l n='rlx'/> +files.</p> + +<typing> +a/b/c /*/c +a/b/c/first() a/b/c/last() a/b/c/index(n) a/b/c/index(-n) +a/b/c/text() a/b/c/text(1) a/b/c/text(-1) a/b/c/text(n) +</typing> +--ldx]]-- + +local trace_lpath = false if trackers then trackers.register("xml.path", function(v) trace_lpath = v end) end +local trace_lparse = false if trackers then trackers.register("xml.parse", function(v) trace_lparse = v end) end +local trace_lprofile = false if trackers then trackers.register("xml.profile", function(v) trace_lpath = v trace_lparse = v trace_lprofile = v end) end + --[[ldx-- -<p>We've now arrived at an intersting part: accessing the tree using a subset +<p>We've now arrived at an interesting part: accessing the tree using a subset of <l n='xpath'/> and since we're not compatible we call it <l n='lpath'/>. We will explain more about its usage in other documents.</p> --ldx]]-- -do - - xml.functions = xml.functions or { } - - local functions = xml.functions - - local actions = { - [10] = "stay", - [11] = "parent", - [12] = "subtree root", - [13] = "document root", - [14] = "any", - [15] = "many", - [16] = "initial", - [20] = "match", - [21] = "match one of", - [22] = "match and attribute eq", - [23] = "match and attribute ne", - [24] = "match one of and attribute eq", - [25] = "match one of and attribute ne", - [27] = "has attribute", - [28] = "has value", - [29] = "fast match", - [30] = "select", - [31] = "expression", - [40] = "processing instruction", - } +local lpathcalls = 0 function xml.lpathcalls () return lpathcalls end +local lpathcached = 0 function xml.lpathcached() return lpathcached end - --~ local function make_expression(str) --could also be an lpeg - --~ str = str:gsub("@([a-zA-Z%-_]+)", "(a['%1'] or '')") - --~ str = str:gsub("position%(%)", "i") - --~ str = str:gsub("text%(%)", "t") - --~ str = str:gsub("!=", "~=") - --~ str = str:gsub("([^=!~<>])=([^=!~<>])", "%1==%2") - --~ str = str:gsub("([a-zA-Z%-_]+)%(", "functions.%1(") - --~ return str, loadstring(format("return function(functions,i,a,t) return %s end", str))() - --~ end - - -- a rather dumb lpeg - - local P, S, R, C, V, Cc = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc - - local lp_position = P("position()") / "id" - local lp_text = P("text()") / "tx" - local lp_name = P("name()") / "((rt.ns~='' and rt.ns..':'..rt.tg) or '')" - local lp_tag = P("tag()") / "(rt.tg or '')" - local lp_ns = P("ns()") / "(rt.ns or '')" - local lp_noequal = P("!=") / "~=" + P("<=") + P(">=") + P("==") - local lp_doequal = P("=") / "==" - local lp_attribute = P("@") / "" * Cc("(at['") * R("az","AZ","--","__")^1 * Cc("'] or '')") - - local lp_function = C(R("az","AZ","--","__")^1) * P("(") / function(t) - if functions[t] then - return "functions." .. t .. "(" - else - return "functions.error(" - end +xml.functions = xml.functions or { } -- internal +xml.expressions = xml.expressions or { } -- in expressions +xml.finalizers = xml.finalizers or { } -- fast do-with ... (with return value other than collection) +xml.specialhandler = xml.specialhandler or { } + +local functions = xml.functions +local expressions = xml.expressions +local finalizers = xml.finalizers + +finalizers.xml = finalizers.xml or { } +finalizers.tex = finalizers.tex or { } + +local function fallback (t, name) + local fn = finalizers[name] + if fn then + t[name] = fn + else + logs.report("xml","unknown sub finalizer '%s'",tostring(name)) + fn = function() end end + return fn +end - local lparent = lpeg.P("(") - local rparent = lpeg.P(")") - local noparent = 1 - (lparent+rparent) - local nested = lpeg.P{lparent * (noparent + lpeg.V(1))^0 * rparent} - local value = lpeg.P(lparent * lpeg.C((noparent + nested)^0) * rparent) +setmetatable(finalizers.xml, { __index = fallback }) +setmetatable(finalizers.tex, { __index = fallback }) ---~ local value = P { "(" * C(((1 - S("()")) + V(1))^0) * ")" } +xml.defaultprotocol = "xml" - local lp_special = (C(P("name")+P("text")+P("tag"))) * value / function(t,s) - if functions[t] then - if s then - return "functions." .. t .. "(rt,k," .. s ..")" - else - return "functions." .. t .. "(rt,k)" - end - else - return "functions.error(" .. t .. ")" - end - end - - local converter = lpeg.Cs ( ( - lp_position + - lp_text + lp_name + -- fast one - lp_special + - lp_noequal + lp_doequal + - lp_attribute + - lp_function + - 1 )^1 ) - - local function make_expression(str) - str = converter:match(str) - return str, loadstring(format("return function(functions,id,at,tx,rt,k) return %s end", str))() - end - - local map = { } - - local space = S(' \r\n\t') - local squote = S("'") - local dquote = S('"') - local lparent = P('(') - local rparent = P(')') - local atsign = P('@') - local lbracket = P('[') - local rbracket = P(']') - local exclam = P('!') - local period = P('.') - local eq = P('==') + P('=') - local ne = P('<>') + P('!=') - local star = P('*') - local slash = P('/') - local colon = P(':') - local bar = P('|') - local hat = P('^') - local valid = R('az', 'AZ', '09') + S('_-') ---~ local name_yes = C(valid^1 + star) * colon * C(valid^1 + star) -- permits ns:* *:tg *:* ---~ local name_nop = C(P(true)) * C(valid^1) - local name_yes = C(valid^1 + star) * colon * C(valid^1 + star) -- permits ns:* *:tg *:* - local name_nop = Cc("*") * C(valid^1) - local name = name_yes + name_nop - local number = C((S('+-')^0 * R('09')^1)) / tonumber - local names = (bar^0 * name)^1 - local morenames = name * (bar^0 * name)^1 - local instructiontag = P('pi::') - local spacing = C(space^0) - local somespace = space^1 - local optionalspace = space^0 - local text = C(valid^0) - local value = (squote * C((1 - squote)^0) * squote) + (dquote * C((1 - dquote)^0) * dquote) - local empty = 1-slash - - local is_eq = lbracket * atsign * name * eq * value * rbracket - local is_ne = lbracket * atsign * name * ne * value * rbracket - local is_attribute = lbracket * atsign * name * rbracket - local is_value = lbracket * value * rbracket - local is_number = lbracket * number * rbracket - - local nobracket = 1-(lbracket+rbracket) -- must be improved - local is_expression = lbracket * C(((C(nobracket^1))/make_expression)) * rbracket - - local is_expression = lbracket * (C(nobracket^1))/make_expression * rbracket - - local is_one = name - local is_none = exclam * name - local is_one_of = ((lparent * names * rparent) + morenames) - local is_none_of = exclam * ((lparent * names * rparent) + morenames) - - local stay = (period ) - local parent = (period * period ) / function( ) map[#map+1] = { 11 } end - local subtreeroot = (slash + hat ) / function( ) map[#map+1] = { 12 } end - local documentroot = (hat * hat ) / function( ) map[#map+1] = { 13 } end - local any = (star ) / function( ) map[#map+1] = { 14 } end - local many = (star * star ) / function( ) map[#map+1] = { 15 } end - local initial = (hat * hat * hat ) / function( ) map[#map+1] = { 16 } end - - local match = (is_one ) / function(...) map[#map+1] = { 20, true , ... } end - local match_one_of = (is_one_of ) / function(...) map[#map+1] = { 21, true , ... } end - local dont_match = (is_none ) / function(...) map[#map+1] = { 20, false, ... } end - local dont_match_one_of = (is_none_of ) / function(...) map[#map+1] = { 21, false, ... } end - - local match_and_eq = (is_one * is_eq ) / function(...) map[#map+1] = { 22, true , ... } end - local match_and_ne = (is_one * is_ne ) / function(...) map[#map+1] = { 23, true , ... } end - local dont_match_and_eq = (is_none * is_eq ) / function(...) map[#map+1] = { 22, false, ... } end - local dont_match_and_ne = (is_none * is_ne ) / function(...) map[#map+1] = { 23, false, ... } end - - local match_one_of_and_eq = (is_one_of * is_eq ) / function(...) map[#map+1] = { 24, true , ... } end - local match_one_of_and_ne = (is_one_of * is_ne ) / function(...) map[#map+1] = { 25, true , ... } end - local dont_match_one_of_and_eq = (is_none_of * is_eq ) / function(...) map[#map+1] = { 24, false, ... } end - local dont_match_one_of_and_ne = (is_none_of * is_ne ) / function(...) map[#map+1] = { 25, false, ... } end - - local has_attribute = (is_one * is_attribute) / function(...) map[#map+1] = { 27, true , ... } end - local has_value = (is_one * is_value ) / function(...) map[#map+1] = { 28, true , ... } end - local dont_has_attribute = (is_none * is_attribute) / function(...) map[#map+1] = { 27, false, ... } end - local dont_has_value = (is_none * is_value ) / function(...) map[#map+1] = { 28, false, ... } end - local position = (is_one * is_number ) / function(...) map[#map+1] = { 30, true, ... } end - local dont_position = (is_none * is_number ) / function(...) map[#map+1] = { 30, false, ... } end - - local expression = (is_one * is_expression)/ function(...) map[#map+1] = { 31, true, ... } end - local dont_expression = (is_none * is_expression)/ function(...) map[#map+1] = { 31, false, ... } end - - local self_expression = ( is_expression) / function(...) if #map == 0 then map[#map+1] = { 11 } end - map[#map+1] = { 31, true, "*", "*", ... } end - local dont_self_expression = (exclam * is_expression) / function(...) if #map == 0 then map[#map+1] = { 11 } end - map[#map+1] = { 31, false, "*", "*", ... } end - - local instruction = (instructiontag * text ) / function(...) map[#map+1] = { 40, ... } end - local nothing = (empty ) / function( ) map[#map+1] = { 15 } end -- 15 ? - local crap = (1-slash)^1 - - -- a few ugly goodies: - - local docroottag = P('^^') / function( ) map[#map+1] = { 12 } end - local subroottag = P('^') / function( ) map[#map+1] = { 13 } end - local roottag = P('root::') / function( ) map[#map+1] = { 12 } end - local parenttag = P('parent::') / function( ) map[#map+1] = { 11 } end - local childtag = P('child::') - local selftag = P('self::') - - -- there will be more and order will be optimized - - local selector = ( - instruction + - many + any + - parent + stay + - dont_position + position + - dont_match_one_of_and_eq + dont_match_one_of_and_ne + - match_one_of_and_eq + match_one_of_and_ne + - dont_match_and_eq + dont_match_and_ne + - match_and_eq + match_and_ne + - dont_expression + expression + - dont_self_expression + self_expression + - has_attribute + has_value + - dont_match_one_of + match_one_of + - dont_match + match + - crap + empty - ) +-- as xsl does not follow xpath completely here we will also +-- be more liberal especially with regards to the use of | and +-- the rootpath: +-- +-- test : all 'test' under current +-- /test : 'test' relative to current +-- a|b|c : set of names +-- (a|b|c) : idem +-- ! : not +-- +-- after all, we're not doing transformations but filtering. in +-- addition we provide filter functions (last bit) +-- +-- todo: optimizer +-- +-- .. : parent +-- * : all kids +-- / : anchor here +-- // : /**/ +-- ** : all in between +-- +-- so far we had (more practical as we don't transform) +-- +-- {/test} : kids 'test' under current node +-- {test} : any kid with tag 'test' +-- {//test} : same as above - local grammar = P { "startup", - startup = (initial + documentroot + subtreeroot + roottag + docroottag + subroottag)^0 * V("followup"), - followup = ((slash + parenttag + childtag + selftag)^0 * selector)^1, - } +-- evaluator (needs to be redone, for the moment copied) - local function compose(str) - if not str or str == "" then - -- wildcard - return true - elseif str == '/' then - -- root - return false - else - map = { } - grammar:match(str) - if #map == 0 then - return true - else - local m = map[1][1] - if #map == 1 then - if m == 14 or m == 15 then - -- wildcard - return true - elseif m == 12 then - -- root - return false - end - elseif #map == 2 and m == 12 and map[2][1] == 20 then - -- return { { 29, map[2][2], map[2][3], map[2][4], map[2][5] } } - map[2][1] = 29 - return { map[2] } - end - if m ~= 11 and m ~= 12 and m ~= 13 and m ~= 14 and m ~= 15 and m ~= 16 then - table.insert(map, 1, { 16 }) - end - -- print((table.serialize(map)):gsub("[ \n]+"," ")) - return map +-- todo: apply_axis(list,notable) and collection vs single + +local apply_axis = { } + +apply_axis['root'] = function(list) + local collected = { } + for l=1,#list do + local ll = list[l] + local rt = ll + while ll do + ll = ll.__p__ + if ll then + rt = ll end end + collected[#collected+1] = rt end + return collected +end - local cache = { } +apply_axis['self'] = function(list) +--~ local collected = { } +--~ for l=1,#list do +--~ collected[#collected+1] = list[l] +--~ end +--~ return collected + return list +end - function xml.lpath(pattern,trace) - if type(pattern) == "string" then - local result = cache[pattern] - if not result then - result = compose(pattern) - cache[pattern] = result - end - if trace or xml.trace_lpath then - xml.lshow(result) +apply_axis['child'] = function(list) + local collected = { } + for l=1,#list do + local ll = list[l] + local dt = ll.dt + local en = 0 + for k=1,#dt do + local dk = dt[k] + if dk.tg then + collected[#collected+1] = dk + dk.ni = k -- refresh + en = en + 1 + dk.ei = en end - return result - else - return pattern end + ll.en = en end + return collected +end - local fallbackreport = (texio and texio.write) or io.write - - function xml.lshow(pattern,report) - report = report or fallbackreport - local lp = xml.lpath(pattern) - if lp == false then - report(" -: root\n") - elseif lp == true then - report(" -: wildcard\n") - else - if type(pattern) == "string" then - report(format("pattern: %s\n",pattern)) - end - for k,v in ipairs(lp) do - if #v > 1 then - local t = { } - for i=2,#v do - local vv = v[i] - if type(vv) == "string" then - t[#t+1] = (vv ~= "" and vv) or "#" - elseif type(vv) == "boolean" then - t[#t+1] = (vv and "==") or "<>" - end - end - report(format("%2i: %s %s -> %s\n", k,v[1],actions[v[1]],concat(t," "))) - else - report(format("%2i: %s %s\n", k,v[1],actions[v[1]])) - end +local function collect(list,collected) + local dt = list.dt + if dt then + local en = 0 + for k=1,#dt do + local dk = dt[k] + if dk.tg then + collected[#collected+1] = dk + dk.ni = k -- refresh + en = en + 1 + dk.ei = en + collect(dk,collected) end end + list.en = en end +end +apply_axis['descendant'] = function(list) + local collected = { } + for l=1,#list do + collect(list[l],collected) + end + return collected +end - function xml.xshow(e,...) -- also handy when report is given, use () to isolate first e - local t = { ... } - local report = (type(t[#t]) == "function" and t[#t]) or fallbackreport - if e == nil then - report("<!-- no element -->\n") - elseif type(e) ~= "table" then - report(tostring(e)) - elseif e.tg then - report(tostring(e) .. "\n") - else - for i=1,#e do - report(tostring(e[i]) .. "\n") +local function collect(list,collected) + local dt = list.dt + if dt then + local en = 0 + for k=1,#dt do + local dk = dt[k] + if dk.tg then + collected[#collected+1] = dk + dk.ni = k -- refresh + en = en + 1 + dk.ei = en + collect(dk,collected) end end + list.en = en end - +end +apply_axis['descendant-or-self'] = function(list) + local collected = { } + for l=1,#list do + local ll = list[l] + if ll.special ~= true then -- catch double root + collected[#collected+1] = ll + end + collect(ll,collected) + end + return collected end ---[[ldx-- -<p>An <l n='lpath'/> is converted to a table with instructions for traversing the -tree. Hoever, simple cases are signaled by booleans. Because we don't know in -advance what we want to do with the found element the handle gets three arguments:</p> +apply_axis['ancestor'] = function(list) + local collected = { } + for l=1,#list do + local ll = list[l] + while ll do + ll = ll.__p__ + if ll then + collected[#collected+1] = ll + end + end + end + return collected +end -<lines> -<t>r</t> : the root element of the data table -<t>d</t> : the data table of the result -<t>t</t> : the index in the data table of the result -</lines> +apply_axis['ancestor-or-self'] = function(list) + local collected = { } + for l=1,#list do + local ll = list[l] + collected[#collected+1] = ll + while ll do + ll = ll.__p__ + if ll then + collected[#collected+1] = ll + end + end + end + return collected +end -<p> Access to the root and data table makes it possible to construct insert and delete -functions.</p> ---ldx]]-- +apply_axis['parent'] = function(list) + local collected = { } + for l=1,#list do + local pl = list[l].__p__ + if pl then + collected[#collected+1] = pl + end + end + return collected +end -do +apply_axis['attribute'] = function(list) + return { } +end - local functions = xml.functions +apply_axis['namespace'] = function(list) + return { } +end - functions.contains = string.find - functions.find = string.find - functions.upper = string.upper - functions.lower = string.lower - functions.number = tonumber - functions.boolean = toboolean +apply_axis['following'] = function(list) -- incomplete +--~ local collected = { } +--~ for l=1,#list do +--~ local ll = list[l] +--~ local p = ll.__p__ +--~ local d = p.dt +--~ for i=ll.ni+1,#d do +--~ local di = d[i] +--~ if type(di) == "table" then +--~ collected[#collected+1] = di +--~ break +--~ end +--~ end +--~ end +--~ return collected + return { } +end + +apply_axis['preceding'] = function(list) -- incomplete +--~ local collected = { } +--~ for l=1,#list do +--~ local ll = list[l] +--~ local p = ll.__p__ +--~ local d = p.dt +--~ for i=ll.ni-1,1,-1 do +--~ local di = d[i] +--~ if type(di) == "table" then +--~ collected[#collected+1] = di +--~ break +--~ end +--~ end +--~ end +--~ return collected + return { } +end - functions.oneof = function(s,...) -- slow - local t = {...} for i=1,#t do if s == t[i] then return true end end return false - end - functions.error = function(str) - xml.error_handler("unknown function in lpath expression",str) - return false - end - functions.text = function(root,k,n) -- unchecked, maybe one deeper - local t = type(t) - if t == "string" then - return t - else -- todo n - local rdt = root.dt - return (rdt and rdt[k]) or root[k] or "" - end - end - functions.name = function(root,k,n) - -- way too fuzzy - local found - if not k or not n then - local ns, tg = root.rn or root.ns or "", root.tg - if not tg then - for i=1,#root do - local e = root[i] - if type(e) == "table" then - found = e - break - end - end - elseif ns ~= "" then - return ns .. ":" .. tg - else - return tg - end - elseif n == 0 then - local e = root[k] - if type(e) ~= "table" then - found = e - end - elseif n < 0 then - for i=k-1,1,-1 do - local e = root[i] - if type(e) == "table" then - if n == -1 then - found = e - break - else - n = n + 1 - end - end +apply_axis['following-sibling'] = function(list) + local collected = { } + for l=1,#list do + local ll = list[l] + local p = ll.__p__ + local d = p.dt + for i=ll.ni+1,#d do + local di = d[i] + if type(di) == "table" then + collected[#collected+1] = di end - else ---~ print(k,n,#root) - for i=k+1,#root,1 do - local e = root[i] - if type(e) == "table" then - if n == 1 then - found = e - break - else - n = n - 1 - end - end + end + end + return collected +end + +apply_axis['preceding-sibling'] = function(list) + local collected = { } + for l=1,#list do + local ll = list[l] + local p = ll.__p__ + local d = p.dt + for i=1,ll.ni-1 do + local di = d[i] + if type(di) == "table" then + collected[#collected+1] = di end end - if found then - local ns, tg = found.rn or found.ns or "", found.tg - if ns ~= "" then - return ns .. ":" .. tg - else - return tg + end + return collected +end + +apply_axis['reverse-sibling'] = function(list) -- reverse preceding + local collected = { } + for l=1,#list do + local ll = list[l] + local p = ll.__p__ + local d = p.dt + for i=ll.ni-1,1,-1 do + local di = d[i] + if type(di) == "table" then + collected[#collected+1] = di end - else - return "" end end + return collected +end - local function traverse(root,pattern,handle,reverse,index,parent,wildcard) -- multiple only for tags, not for namespaces - if not root then -- error - return false - elseif pattern == false then -- root - handle(root,root.dt,root.ri) - return false - elseif pattern == true then -- wildcard - local rootdt = root.dt - if rootdt then - local start, stop, step = 1, #rootdt, 1 - if reverse then - start, stop, step = stop, start, -1 - end - for k=start,stop,step do - if handle(root,rootdt,root.ri or k) then return false end - if not traverse(rootdt[k],true,handle,reverse) then return false end - end +apply_axis['auto-descendant-or-self'] = apply_axis['descendant-or-self'] +apply_axis['auto-descendant'] = apply_axis['descendant'] +apply_axis['auto-child'] = apply_axis['child'] +apply_axis['auto-self'] = apply_axis['self'] +apply_axis['initial-child'] = apply_axis['child'] + +local function apply_nodes(list,directive,nodes) + -- todo: nodes[1] etc ... negated node name in set ... when needed + -- ... currently ignored + local maxn = #nodes + if maxn == 3 then --optimized loop + local nns, ntg = nodes[2], nodes[3] + if not nns and not ntg then -- wildcard + if directive then + return list + else + return { } end - return false - elseif root.dt then - index = index or 1 - local action = pattern[index] - local command = action[1] - if command == 29 then -- fast case /oeps - local rootdt = root.dt - for k=1,#rootdt do - local e = rootdt[k] - local tg = e.tg - if e.tg then - local ns = e.rn or e.ns - local ns_a, tg_a = action[3], action[4] - local matched = (ns_a == "*" or ns == ns_a) and (tg_a == "*" or tg == tg_a) - if not action[2] then matched = not matched end - if matched then - if handle(root,rootdt,k) then return false end + else + local collected, m, p = { }, 0, nil + if not nns then -- only check tag + for l=1,#list do + local ll = list[l] + local ltg = ll.tg + if ltg then + if directive then + if ntg == ltg then + local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end + collected[#collected+1], ll.mi = ll, m + end + elseif ntg ~= ltg then + local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end + collected[#collected+1], ll.mi = ll, m end end end - elseif command == 11 then -- parent - local ep = root.__p__ or parent - if index < #pattern then - if not traverse(ep,pattern,handle,reverse,index+1,root) then return false end - elseif handle(root,rootdt,k) then - return false - end - else - if (command == 16 or command == 12) and index == 1 then -- initial - -- wildcard = true - wildcard = command == 16 -- ok? - index = index + 1 - action = pattern[index] - command = action and action[1] or 0 -- something is wrong - end - if command == 11 then -- parent - local ep = root.__p__ or parent - if index < #pattern then - if not traverse(ep,pattern,handle,reverse,index+1,root) then return false end - elseif handle(root,rootdt,k) then - return false - end - else - local rootdt = root.dt - local start, stop, step, n, dn = 1, #rootdt, 1, 0, 1 - if command == 30 then - if action[5] < 0 then - start, stop, step = stop, start, -1 - dn = -1 + elseif not ntg then -- only check namespace + for l=1,#list do + local ll = list[l] + local lns = ll.rn or ll.ns + if lns then + if directive then + if lns == nns then + local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end + collected[#collected+1], ll.mi = ll, m + end + elseif lns ~= nns then + local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end + collected[#collected+1], ll.mi = ll, m end - elseif reverse and index == #pattern then - start, stop, step = stop, start, -1 end - local idx = 0 - for k=start,stop,step do -- we used to have functions for all but a case is faster - local e = rootdt[k] - local ns, tg = e.rn or e.ns, e.tg - if tg then - idx = idx + 1 - if command == 30 then - local ns_a, tg_a = action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - if not action[2] then matched = not matched end - if matched then - n = n + dn - if n == action[5] then - if index == #pattern then - if handle(root,rootdt,root.ri or k) then return false end - else - if not traverse(e,pattern,handle,reverse,index+1,root) then return false end - end - break - end - elseif wildcard then - if not traverse(e,pattern,handle,reverse,index,root,true) then return false end - end - else - local matched, multiple = false, false - if command == 20 then -- match - local ns_a, tg_a = action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - if not action[2] then matched = not matched end - elseif command == 21 then -- match one of - multiple = true - for i=3,#action,2 do - local ns_a, tg_a = action[i], action[i+1] - if (ns_a == "*" or ns == ns_a) and (tg == "*" or tg == tg_a) then - matched = true - break - end - end - if not action[2] then matched = not matched end - elseif command == 22 then -- eq - local ns_a, tg_a = action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - matched = matched and e.at[action[6]] == action[7] - elseif command == 23 then -- ne - local ns_a, tg_a = action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - if not action[2] then matched = not matched end - matched = mached and e.at[action[6]] ~= action[7] - elseif command == 24 then -- one of eq - multiple = true - for i=3,#action-2,2 do - local ns_a, tg_a = action[i], action[i+1] - if (ns_a == "*" or ns == ns_a) and (tg == "*" or tg == tg_a) then - matched = true - break - end - end - if not action[2] then matched = not matched end - matched = matched and e.at[action[#action-1]] == action[#action] - elseif command == 25 then -- one of ne - multiple = true - for i=3,#action-2,2 do - local ns_a, tg_a = action[i], action[i+1] - if (ns_a == "*" or ns == ns_a) and (tg == "*" or tg == tg_a) then - matched = true - break - end - end - if not action[2] then matched = not matched end - matched = matched and e.at[action[#action-1]] ~= action[#action] - elseif command == 27 then -- has attribute - local ns_a, tg_a = action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - if not action[2] then matched = not matched end - matched = matched and e.at[action[5]] - elseif command == 28 then -- has value - local edt, ns_a, tg_a = e.dt, action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - if not action[2] then matched = not matched end - matched = matched and edt and edt[1] == action[5] - elseif command == 31 then - local edt, ns_a, tg_a = e.dt, action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - if not action[2] then matched = not matched end - if matched then - matched = action[6](functions,idx,e.at or { },edt[1],rootdt,k) - end - end - if matched then -- combine tg test and at test - if index == #pattern then - if handle(root,rootdt,root.ri or k) then return false end - if wildcard then - if multiple then - if not traverse(e,pattern,handle,reverse,index,root,true) then return false end - else - -- maybe or multiple; anyhow, check on (section|title) vs just section and title in example in lxml - if not traverse(e,pattern,handle,reverse,index,root) then return false end - end - end - else - if not traverse(e,pattern,handle,reverse,index+1,root) then return false end - end - elseif command == 14 then -- any - if index == #pattern then - if handle(root,rootdt,root.ri or k) then return false end - else - if not traverse(e,pattern,handle,reverse,index+1,root) then return false end - end - elseif command == 15 then -- many - if index == #pattern then - if handle(root,rootdt,root.ri or k) then return false end - else - if not traverse(e,pattern,handle,reverse,index+1,root,true) then return false end - end - -- not here : 11 - elseif command == 11 then -- parent - local ep = e.__p__ or parent - if index < #pattern then - if not traverse(ep,pattern,handle,reverse,root,index+1) then return false end - elseif handle(root,rootdt,k) then - return false - end - elseif command == 40 and e.special and tg == "@pi@" then -- pi - local pi = action[2] - if pi ~= "" then - local pt = e.dt[1] - if pt and pt:find(pi) then - if handle(root,rootdt,k) then - return false - end - end - elseif handle(root,rootdt,k) then - return false - end - elseif wildcard then - if not traverse(e,pattern,handle,reverse,index,root,true) then return false end - end - end - else - -- not here : 11 - if command == 11 then -- parent - local ep = e.__p__ or parent - if index < #pattern then - if not traverse(ep,pattern,handle,reverse,index+1,root) then return false end - elseif handle(root,rootdt,k) then - return false - end - break -- else loop + end + else -- check both + for l=1,#list do + local ll = list[l] + local ltg = ll.tg + if ltg then + local lns = ll.rn or ll.ns + local ok = ltg == ntg and lns == nns + if directive then + if ok then + local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end + collected[#collected+1], ll.mi = ll, m end + elseif not ok then + local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end + collected[#collected+1], ll.mi = ll, m end end end end + return collected end - return true + else + local collected, m, p = { }, 0, nil + for l=1,#list do + local ll = list[l] + local ltg = ll.tg + if ltg then + local lns = ll.rn or ll.ns + local ok = false + for n=1,maxn,3 do + local nns, ntg = nodes[n+1], nodes[n+2] + ok = (not ntg or ltg == ntg) and (not nns or lns == nns) + if ok then + break + end + end + if directive then + if ok then + local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end + collected[#collected+1], ll.mi = ll, m + end + elseif not ok then + local llp = ll.__p__ ; if llp ~= p then p, m = llp, 1 else m = m + 1 end + collected[#collected+1], ll.mi = ll, m + end + end + end + return collected end - - xml.traverse = traverse - end ---[[ldx-- -<p>Next come all kind of locators and manipulators. The most generic function here -is <t>xml.filter(root,pattern)</t>. All registers functions in the filters namespace -can be path of a search path, as in:</p> +local quit_expression = false -<typing> -local r, d, k = xml.filter(root,"/a/b/c/position(4)" -</typing> ---ldx]]-- +local function apply_expression(list,expression,order) + local collected = { } + quit_expression = false + for l=1,#list do + local ll = list[l] + if expression(list,ll,l,order) then -- nasty, order alleen valid als n=1 + collected[#collected+1] = ll + end + if quit_expression then + break + end + end + return collected +end + +local P, V, C, Cs, Cc, Ct, R, S, Cg, Cb = lpeg.P, lpeg.V, lpeg.C, lpeg.Cs, lpeg.Cc, lpeg.Ct, lpeg.R, lpeg.S, lpeg.Cg, lpeg.Cb + +local spaces = S(" \n\r\t\f")^0 +local lp_space = S(" \n\r\t\f") +local lp_any = P(1) +local lp_noequal = P("!=") / "~=" + P("<=") + P(">=") + P("==") +local lp_doequal = P("=") / "==" +local lp_or = P("|") / " or " +local lp_and = P("&") / " and " + +local lp_builtin = P ( + P("firstindex") / "1" + + P("lastindex") / "(#ll.__p__.dt or 1)" + + P("firstelement") / "1" + + P("lastelement") / "(ll.__p__.en or 1)" + + P("first") / "1" + + P("last") / "#list" + + P("rootposition") / "order" + + P("position") / "l" + -- is element in finalizer + P("order") / "order" + + P("element") / "(ll.ei or 1)" + + P("index") / "(ll.ni or 1)" + + P("match") / "(ll.mi or 1)" + + P("text") / "(ll.dt[1] or '')" + + -- P("name") / "(ll.ns~='' and ll.ns..':'..ll.tg)" + + P("name") / "((ll.ns~='' and ll.ns..':'..ll.tg) or ll.tg)" + + P("tag") / "ll.tg" + + P("ns") / "ll.ns" + ) * ((spaces * P("(") * spaces * P(")"))/"") + +local lp_attribute = (P("@") + P("attribute::")) / "" * Cc("(ll.at and ll.at['") * R("az","AZ","--","__")^1 * Cc("'])") +local lp_fastpos_p = ((P("+")^0 * R("09")^1 * P(-1)) / function(s) return "l==" .. s end) +local lp_fastpos_n = ((P("-") * R("09")^1 * P(-1)) / function(s) return "(" .. s .. "<0 and (#list+".. s .. "==l))" end) +local lp_fastpos = lp_fastpos_n + lp_fastpos_p +local lp_reserved = C("and") + C("or") + C("not") + C("div") + C("mod") + C("true") + C("false") + +local lp_lua_function = C(R("az","AZ","__")^1 * (P(".") * R("az","AZ","__")^1)^1) * ("(") / function(t) -- todo: better . handling + return t .. "(" +end + +local lp_function = C(R("az","AZ","__")^1) * P("(") / function(t) -- todo: better . handling + if expressions[t] then + return "expr." .. t .. "(" + else + return "expr.error(" + end +end -do +local lparent = lpeg.P("(") +local rparent = lpeg.P(")") +local noparent = 1 - (lparent+rparent) +local nested = lpeg.P{lparent * (noparent + lpeg.V(1))^0 * rparent} +local value = lpeg.P(lparent * lpeg.C((noparent + nested)^0) * rparent) -- lpeg.P{"("*C(((1-S("()"))+V(1))^0)*")"} - local traverse, lpath, convert = xml.traverse, xml.lpath, xml.convert +local lp_child = Cc("expr.child(ll,'") * R("az","AZ","--","__")^1 * Cc("')") +local lp_number = S("+-") * R("09")^1 +local lp_string = Cc("'") * R("az","AZ","--","__")^1 * Cc("'") +local lp_content = (P("'") * (1-P("'"))^0 * P("'") + P('"') * (1-P('"'))^0 * P('"')) - xml.filters = { } +local cleaner - function xml.filters.default(root,pattern) - local rt, dt, dk - traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end) - return dt and dt[dk], rt, dt, dk - end - function xml.filters.attributes(root,pattern,arguments) - local rt, dt, dk - traverse(root, lpath(pattern), function(r,d,k) rt, dt, dk = r, d, k return true end) - local ekat = (dt and dt[dk] and dt[dk].at) or (rt and rt.at) - if ekat then - if arguments then - return ekat[arguments] or "", rt, dt, dk - else - return ekat, rt, dt, dk - end +local lp_special = (C(P("name")+P("text")+P("tag")+P("count")+P("child"))) * value / function(t,s) + if expressions[t] then + s = s and s ~= "" and lpegmatch(cleaner,s) + if s and s ~= "" then + return "expr." .. t .. "(ll," .. s ..")" else - return { }, rt, dt, dk + return "expr." .. t .. "(ll)" end + else + return "expr.error(" .. t .. ")" end - function xml.filters.reverse(root,pattern) - local rt, dt, dk - traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end, 'reverse') - return dt and dt[dk], rt, dt, dk - end - function xml.filters.count(root,pattern,everything) - local n = 0 - traverse(root, lpath(pattern), function(r,d,t) - if everything or type(d[t]) == "table" then - n = n + 1 - end - end) - return n - end - function xml.filters.elements(root, pattern) -- == all - local t = { } - traverse(root, lpath(pattern), function(r,d,k) - local e = d[k] - if e then - t[#t+1] = e - end - end) - return t - end - function xml.filters.texts(root, pattern) - local t = { } - traverse(root, lpath(pattern), function(r,d,k) - local e = d[k] - if e and e.dt then - t[#t+1] = e.dt - end - end) - return t +end + +local content = + lp_builtin + + lp_attribute + + lp_special + + lp_noequal + lp_doequal + + lp_or + lp_and + + lp_reserved + + lp_lua_function + lp_function + + lp_content + -- too fragile + lp_child + + lp_any + +local converter = Cs ( + lp_fastpos + (P { lparent * (V(1))^0 * rparent + content } )^0 +) + +cleaner = Cs ( ( +--~ lp_fastpos + + lp_reserved + + lp_number + + lp_string + +1 )^1 ) + + +--~ expr + +local template_e = [[ + local expr = xml.expressions + return function(list,ll,l,order) + return %s end - function xml.filters.first(root,pattern) - local rt, dt, dk - traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end) - return dt and dt[dk], rt, dt, dk - end - function xml.filters.last(root,pattern) - local rt, dt, dk - traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end, 'reverse') - return dt and dt[dk], rt, dt, dk - end - function xml.filters.index(root,pattern,arguments) - local rt, dt, dk, reverse, i = nil, nil, nil, false, tonumber(arguments or '1') or 1 - if i and i ~= 0 then - if i < 0 then - reverse, i = true, -i - end - traverse(root, lpath(pattern), function(r,d,k) rt, dt, dk, i = r, d, k, i-1 return i == 0 end, reverse) - if i == 0 then - return dt and dt[dk], rt, dt, dk - end - end - return nil, nil, nil, nil - end - function xml.filters.attribute(root,pattern,arguments) - local rt, dt, dk - traverse(root, lpath(pattern), function(r,d,k) rt, dt, dk = r, d, k return true end) - local ekat = (dt and dt[dk] and dt[dk].at) or (rt and rt.at) - return (ekat and (ekat[arguments] or ekat[arguments:gsub("^([\"\'])(.*)%1$","%2")])) or "" - end - function xml.filters.text(root,pattern,arguments) -- ?? why index, tostring slow - local dtk, rt, dt, dk = xml.filters.index(root,pattern,arguments) - if dtk then -- n - local dtkdt = dtk.dt - if not dtkdt then - return "", rt, dt, dk - elseif #dtkdt == 1 and type(dtkdt[1]) == "string" then - return dtkdt[1], rt, dt, dk - else - return xml.tostring(dtkdt), rt, dt, dk - end - else - return "", rt, dt, dk - end +]] + +local template_f_y = [[ + local finalizer = xml.finalizers['%s']['%s'] + return function(collection) + return finalizer(collection,%s) end - function xml.filters.tag(root,pattern,n) - local tag = "" - traverse(root, lpath(pattern), function(r,d,k) - tag = xml.functions.tag(d,k,n and tonumber(n)) - return true - end) - return tag +]] + +local template_f_n = [[ + return xml.finalizers['%s']['%s'] +]] + +-- + +local register_self = { kind = "axis", axis = "self" } -- , apply = apply_axis["self"] } +local register_parent = { kind = "axis", axis = "parent" } -- , apply = apply_axis["parent"] } +local register_descendant = { kind = "axis", axis = "descendant" } -- , apply = apply_axis["descendant"] } +local register_child = { kind = "axis", axis = "child" } -- , apply = apply_axis["child"] } +local register_descendant_or_self = { kind = "axis", axis = "descendant-or-self" } -- , apply = apply_axis["descendant-or-self"] } +local register_root = { kind = "axis", axis = "root" } -- , apply = apply_axis["root"] } +local register_ancestor = { kind = "axis", axis = "ancestor" } -- , apply = apply_axis["ancestor"] } +local register_ancestor_or_self = { kind = "axis", axis = "ancestor-or-self" } -- , apply = apply_axis["ancestor-or-self"] } +local register_attribute = { kind = "axis", axis = "attribute" } -- , apply = apply_axis["attribute"] } +local register_namespace = { kind = "axis", axis = "namespace" } -- , apply = apply_axis["namespace"] } +local register_following = { kind = "axis", axis = "following" } -- , apply = apply_axis["following"] } +local register_following_sibling = { kind = "axis", axis = "following-sibling" } -- , apply = apply_axis["following-sibling"] } +local register_preceding = { kind = "axis", axis = "preceding" } -- , apply = apply_axis["preceding"] } +local register_preceding_sibling = { kind = "axis", axis = "preceding-sibling" } -- , apply = apply_axis["preceding-sibling"] } +local register_reverse_sibling = { kind = "axis", axis = "reverse-sibling" } -- , apply = apply_axis["reverse-sibling"] } + +local register_auto_descendant_or_self = { kind = "axis", axis = "auto-descendant-or-self" } -- , apply = apply_axis["auto-descendant-or-self"] } +local register_auto_descendant = { kind = "axis", axis = "auto-descendant" } -- , apply = apply_axis["auto-descendant"] } +local register_auto_self = { kind = "axis", axis = "auto-self" } -- , apply = apply_axis["auto-self"] } +local register_auto_child = { kind = "axis", axis = "auto-child" } -- , apply = apply_axis["auto-child"] } + +local register_initial_child = { kind = "axis", axis = "initial-child" } -- , apply = apply_axis["initial-child"] } + +local register_all_nodes = { kind = "nodes", nodetest = true, nodes = { true, false, false } } + +local skip = { } + +local function errorrunner_e(str,cnv) + if not skip[str] then + logs.report("lpath","error in expression: %s => %s",str,cnv) + skip[str] = cnv or str end - function xml.filters.name(root,pattern,n) - local tag = "" - traverse(root, lpath(pattern), function(r,d,k) - tag = xml.functions.name(d,k,n and tonumber(n)) - return true - end) - return tag + return false +end +local function errorrunner_f(str,arg) + logs.report("lpath","error in finalizer: %s(%s)",str,arg or "") + return false +end + +local function register_nodes(nodetest,nodes) + return { kind = "nodes", nodetest = nodetest, nodes = nodes } +end + +local function register_expression(expression) + local converted = lpegmatch(converter,expression) + local runner = loadstring(format(template_e,converted)) + runner = (runner and runner()) or function() errorrunner_e(expression,converted) end + return { kind = "expression", expression = expression, converted = converted, evaluator = runner } +end + +local function register_finalizer(protocol,name,arguments) + local runner + if arguments and arguments ~= "" then + runner = loadstring(format(template_f_y,protocol or xml.defaultprotocol,name,arguments)) + else + runner = loadstring(format(template_f_n,protocol or xml.defaultprotocol,name)) end + runner = (runner and runner()) or function() errorrunner_f(name,arguments) end + return { kind = "finalizer", name = name, arguments = arguments, finalizer = runner } +end - --[[ldx-- - <p>For splitting the filter function from the path specification, we can - use string matching or lpeg matching. Here the difference in speed is - neglectable but the lpeg variant is more robust.</p> - --ldx]]-- +local expression = P { "ex", + ex = "[" * C((V("sq") + V("dq") + (1 - S("[]")) + V("ex"))^0) * "]", + sq = "'" * (1 - S("'"))^0 * "'", + dq = '"' * (1 - S('"'))^0 * '"', +} - -- not faster but hipper ... although ... i can't get rid of the trailing / in the path +local arguments = P { "ar", + ar = "(" * Cs((V("sq") + V("dq") + V("nq") + P(1-P(")")))^0) * ")", + nq = ((1 - S("),'\""))^1) / function(s) return format("%q",s) end, + sq = P("'") * (1 - P("'"))^0 * P("'"), + dq = P('"') * (1 - P('"'))^0 * P('"'), +} - local P, S, R, C, V, Cc = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc +-- todo: better arg parser - local slash = P('/') - local name = (R("az","AZ","--","__"))^1 - local path = C(((1-slash)^0 * slash)^1) - local argument = P { "(" * C(((1 - S("()")) + V(1))^0) * ")" } - local action = Cc(1) * path * C(name) * argument - local attribute = Cc(2) * path * P('@') * C(name) - local direct = Cc(3) * Cc("../*") * slash^0 * C(name) * argument +local function register_error(str) + return { kind = "error", error = format("unparsed: %s",str) } +end - local parser = direct + action + attribute +-- there is a difference in * and /*/ and so we need to catch a few special cases - local filters = xml.filters - local attribute_filter = xml.filters.attributes - local default_filter = xml.filters.default +local special_1 = P("*") * Cc(register_auto_descendant) * Cc(register_all_nodes) -- last one not needed +local special_2 = P("/") * Cc(register_auto_self) +local special_3 = P("") * Cc(register_auto_self) - -- todo: also hash, could be gc'd +local parser = Ct { "patterns", -- can be made a bit faster by moving pattern outside - function xml.filter(root,pattern) - local kind, a, b, c = parser:match(pattern) ---~ if xml.trace_lpath then ---~ print(pattern,kind,a,b,c) ---~ end - if kind == 1 or kind == 3 then - return (filters[b] or default_filter)(root,a,c) - elseif kind == 2 then - return attribute_filter(root,a,b) - else - return default_filter(root,pattern) - end - end - - --~ slightly faster, but first we need a proper test file - --~ - --~ local hash = { } - --~ - --~ function xml.filter(root,pattern) - --~ local h = hash[pattern] - --~ if not h then - --~ local kind, a, b, c = parser:match(pattern) - --~ if kind == 1 then - --~ h = { kind, filters[b] or default_filter, a, b, c } - --~ elseif kind == 2 then - --~ h = { kind, attribute_filter, a, b, c } - --~ else - --~ h = { kind, default_filter, a, b, c } - --~ end - --~ hash[pattern] = h - --~ end - --~ local kind = h[1] - --~ if kind == 1 then - --~ return h[2](root,h[2],h[4]) - --~ elseif kind == 2 then - --~ return h[2](root,h[2],h[3]) - --~ else - --~ return h[2](root,pattern) - --~ end - --~ end - - --[[ldx-- - <p>The following functions collect elements and texts.</p> - --ldx]]-- - - -- still somewhat bugged - - function xml.collect_elements(root, pattern, ignorespaces) - local rr, dd = { }, { } - traverse(root, lpath(pattern), function(r,d,k) - local dk = d and d[k] - if dk then - if ignorespaces and type(dk) == "string" and dk:find("[^%S]") then - -- ignore - else - local n = #rr+1 - rr[n], dd[n] = r, dk - end - end - end) - return dd, rr - end - - function xml.collect_texts(root, pattern, flatten) - local t = { } -- no r collector - traverse(root, lpath(pattern), function(r,d,k) - if d then - local ek = d[k] - local tx = ek and ek.dt - if flatten then - if tx then - t[#t+1] = xml.tostring(tx) or "" - else - t[#t+1] = "" - end - else - t[#t+1] = tx or "" - end - else - t[#t+1] = "" - end - end) - return t - end + patterns = spaces * V("protocol") * spaces * ( + ( V("special") * spaces * P(-1) ) + + ( V("initial") * spaces * V("step") * spaces * (P("/") * spaces * V("step") * spaces)^0 ) + ), - function xml.collect_tags(root, pattern, nonamespace) - local t = { } - xml.traverse(root, xml.lpath(pattern), function(r,d,k) - local dk = d and d[k] - if dk and type(dk) == "table" then - local ns, tg = e.ns, e.tg - if nonamespace then - t[#t+1] = tg -- if needed we can return an extra table - elseif ns == "" then - t[#t+1] = tg - else - t[#t+1] = ns .. ":" .. tg - end - end - end) - return #t > 0 and {} - end + protocol = Cg(V("letters"),"protocol") * P("://") + Cg(Cc(nil),"protocol"), - --[[ldx-- - <p>Often using an iterators looks nicer in the code than passing handler - functions. The <l n='lua'/> book describes how to use coroutines for that - purpose (<url href='http://www.lua.org/pil/9.3.html'/>). This permits - code like:</p> + -- the / is needed for // as descendant or self is somewhat special + -- step = (V("shortcuts") + V("axis") * spaces * V("nodes")^0 + V("error")) * spaces * V("expressions")^0 * spaces * V("finalizer")^0, + step = ((V("shortcuts") + P("/") + V("axis")) * spaces * V("nodes")^0 + V("error")) * spaces * V("expressions")^0 * spaces * V("finalizer")^0, - <typing> - for r, d, k in xml.elements(xml.load('text.xml'),"title") do - print(d[k]) - end - </typing> + axis = V("descendant") + V("child") + V("parent") + V("self") + V("root") + V("ancestor") + + V("descendant_or_self") + V("following_sibling") + V("following") + + V("reverse_sibling") + V("preceding_sibling") + V("preceding") + V("ancestor_or_self") + + #(1-P(-1)) * Cc(register_auto_child), - <p>Which will print all the titles in the document. The iterator variant takes - 1.5 times the runtime of the function variant which is due to the overhead in - creating the wrapper. So, instead of:</p> + special = special_1 + special_2 + special_3, - <typing> - function xml.filters.first(root,pattern) - for rt,dt,dk in xml.elements(root,pattern) - return dt and dt[dk], rt, dt, dk - end - return nil, nil, nil, nil - end - </typing> + initial = (P("/") * spaces * Cc(register_initial_child))^-1, - <p>We use the function variants in the filters.</p> - --ldx]]-- + error = (P(1)^1) / register_error, - local wrap, yield = coroutine.wrap, coroutine.yield + shortcuts_a = V("s_descendant_or_self") + V("s_descendant") + V("s_child") + V("s_parent") + V("s_self") + V("s_root") + V("s_ancestor"), - function xml.elements(root,pattern,reverse) - return wrap(function() traverse(root, lpath(pattern), yield, reverse) end) - end + shortcuts = V("shortcuts_a") * (spaces * "/" * spaces * V("shortcuts_a"))^0, + + s_descendant_or_self = (P("***/") + P("/")) * Cc(register_descendant_or_self), --- *** is a bonus + -- s_descendant_or_self = P("/") * Cc(register_descendant_or_self), + s_descendant = P("**") * Cc(register_descendant), + s_child = P("*") * #(1-P(":")) * Cc(register_child ), +-- s_child = P("*") * #(P("/")+P(-1)) * Cc(register_child ), + s_parent = P("..") * Cc(register_parent ), + s_self = P("." ) * Cc(register_self ), + s_root = P("^^") * Cc(register_root ), + s_ancestor = P("^") * Cc(register_ancestor ), + + descendant = P("descendant::") * Cc(register_descendant ), + child = P("child::") * Cc(register_child ), + parent = P("parent::") * Cc(register_parent ), + self = P("self::") * Cc(register_self ), + root = P('root::') * Cc(register_root ), + ancestor = P('ancestor::') * Cc(register_ancestor ), + descendant_or_self = P('descendant-or-self::') * Cc(register_descendant_or_self ), + ancestor_or_self = P('ancestor-or-self::') * Cc(register_ancestor_or_self ), + -- attribute = P('attribute::') * Cc(register_attribute ), + -- namespace = P('namespace::') * Cc(register_namespace ), + following = P('following::') * Cc(register_following ), + following_sibling = P('following-sibling::') * Cc(register_following_sibling ), + preceding = P('preceding::') * Cc(register_preceding ), + preceding_sibling = P('preceding-sibling::') * Cc(register_preceding_sibling ), + reverse_sibling = P('reverse-sibling::') * Cc(register_reverse_sibling ), + + nodes = (V("nodefunction") * spaces * P("(") * V("nodeset") * P(")") + V("nodetest") * V("nodeset")) / register_nodes, - function xml.elements_only(root,pattern,reverse) - return wrap(function() traverse(root, lpath(pattern), function(r,d,k) yield(d[k]) end, reverse) end) + expressions = expression / register_expression, + + letters = R("az")^1, + name = (1-lpeg.S("/[]()|:*!"))^1, + negate = P("!") * Cc(false), + + nodefunction = V("negate") + P("not") * Cc(false) + Cc(true), + nodetest = V("negate") + Cc(true), + nodename = (V("negate") + Cc(true)) * spaces * ((V("wildnodename") * P(":") * V("wildnodename")) + (Cc(false) * V("wildnodename"))), + wildnodename = (C(V("name")) + P("*") * Cc(false)) * #(1-P("(")), + nodeset = spaces * Ct(V("nodename") * (spaces * P("|") * spaces * V("nodename"))^0) * spaces, + + finalizer = (Cb("protocol") * P("/")^-1 * C(V("name")) * arguments * P(-1)) / register_finalizer, + +} + +local cache = { } + +local function nodesettostring(set,nodetest) + local t = { } + for i=1,#set,3 do + local directive, ns, tg = set[i], set[i+1], set[i+2] + if not ns or ns == "" then ns = "*" end + if not tg or tg == "" then tg = "*" end + tg = (tg == "@rt@" and "[root]") or format("%s:%s",ns,tg) + t[#t+1] = (directive and tg) or format("not(%s)",tg) + end + if nodetest == false then + return format("not(%s)",concat(t,"|")) + else + return concat(t,"|") end +end - function xml.each_element(root, pattern, handle, reverse) - local ok - traverse(root, lpath(pattern), function(r,d,k) ok = true handle(r,d,k) end, reverse) - return ok +local function tagstostring(list) + if #list == 0 then + return "no elements" + else + local t = { } + for i=1, #list do + local li = list[i] + local ns, tg = li.ns, li.tg + if not ns or ns == "" then ns = "*" end + if not tg or tg == "" then tg = "*" end + t[#t+1] = (tg == "@rt@" and "[root]") or format("%s:%s",ns,tg) + end + return concat(t," ") end +end - function xml.process_elements(root, pattern, handle) - traverse(root, lpath(pattern), function(r,d,k) - local dkdt = d[k].dt - if dkdt then - for i=1,#dkdt do - local v = dkdt[i] - if v.tg then handle(v) end - end - end - end) +xml.nodesettostring = nodesettostring + +local parse_pattern -- we have a harmless kind of circular reference + +local function lshow(parsed) + if type(parsed) == "string" then + parsed = parse_pattern(parsed) end + local s = table.serialize_functions -- ugly + table.serialize_functions = false -- ugly + logs.report("lpath","%s://%s => %s",parsed.protocol or xml.defaultprotocol,parsed.pattern,table.serialize(parsed,false)) + table.serialize_functions = s -- ugly +end - function xml.process_attributes(root, pattern, handle) - traverse(root, lpath(pattern), function(r,d,k) - local ek = d[k] - local a = ek.at or { } - handle(a) - if next(a) then -- next is faster than type (and >0 test) - ek.at = a - else - ek.at = nil - end - end) +xml.lshow = lshow + +local function add_comment(p,str) + local pc = p.comment + if not pc then + p.comment = { str } + else + pc[#pc+1] = str end +end - --[[ldx-- - <p>We've now arrives at the functions that manipulate the tree.</p> - --ldx]]-- - - function xml.inject_element(root, pattern, element, prepend) - if root and element then - local matches, collect = { }, nil - if type(element) == "string" then - element = convert(element,true) - end - if element then - collect = function(r,d,k) matches[#matches+1] = { r, d, k, element } end - traverse(root, lpath(pattern), collect) - for i=1,#matches do - local m = matches[i] - local r, d, k, element, edt = m[1], m[2], m[3], m[4], nil - if element.ri then - element = element.dt[element.ri].dt - else - element = element.dt - end - if r.ri then - edt = r.dt[r.ri].dt - else - edt = d and d[k] and d[k].dt - end - if edt then - local be, af - if prepend then - be, af = xml.copy(element), edt +parse_pattern = function (pattern) -- the gain of caching is rather minimal + lpathcalls = lpathcalls + 1 + if type(pattern) == "table" then + return pattern + else + local parsed = cache[pattern] + if parsed then + lpathcached = lpathcached + 1 + else + parsed = lpegmatch(parser,pattern) + if parsed then + parsed.pattern = pattern + local np = #parsed + if np == 0 then + parsed = { pattern = pattern, register_self, state = "parsing error" } + logs.report("lpath","parsing error in '%s'",pattern) + lshow(parsed) + else + -- we could have done this with a more complex parser but this + -- is cleaner + local pi = parsed[1] + if pi.axis == "auto-child" then + if false then + add_comment(parsed, "auto-child replaced by auto-descendant-or-self") + parsed[1] = register_auto_descendant_or_self else - be, af = edt, xml.copy(element) - end - for i=1,#af do - be[#be+1] = af[i] + add_comment(parsed, "auto-child replaced by auto-descendant") + parsed[1] = register_auto_descendant end - if r.ri then - r.dt[r.ri].dt = be - else - d[k].dt = be + elseif pi.axis == "initial-child" and np > 1 and parsed[2].axis then + add_comment(parsed, "initial-child removed") -- we could also make it a auto-self + remove(parsed,1) + end + local np = #parsed -- can have changed + if np > 1 then + local pnp = parsed[np] + if pnp.kind == "nodes" and pnp.nodetest == true then + local nodes = pnp.nodes + if nodes[1] == true and nodes[2] == false and nodes[3] == false then + add_comment(parsed, "redundant final wildcard filter removed") + remove(parsed,np) + end end - else - -- r.dt = element.dt -- todo end end + else + parsed = { pattern = pattern } + end + cache[pattern] = parsed + if trace_lparse and not trace_lprofile then + lshow(parsed) end end + return parsed end +end - -- todo: copy ! +-- we can move all calls inline and then merge the trace back +-- technically we can combine axis and the next nodes which is +-- what we did before but this a bit cleaner (but slower too) +-- but interesting is that it's not that much faster when we +-- go inline +-- +-- beware: we need to return a collection even when we filter +-- else the (simple) cache gets messed up - function xml.insert_element(root, pattern, element, before) -- todo: element als functie - if root and element then - if pattern == "/" then - xml.inject_element(root, pattern, element, before) - else - local matches, collect = { }, nil - if type(element) == "string" then - element = convert(element,true) - end - if element and element.ri then - element = element.dt[element.ri] - end - if element then - collect = function(r,d,k) matches[#matches+1] = { r, d, k, element } end - traverse(root, lpath(pattern), collect) - for i=#matches,1,-1 do - local m = matches[i] - local r, d, k, element = m[1], m[2], m[3], m[4] - if not before then k = k + 1 end - if element.tg then - table.insert(d,k,element) -- untested - elseif element.dt then - for _,v in ipairs(element.dt) do -- i added - table.insert(d,k,v) - k = k + 1 - end - end - end - end +-- caching found lookups saves not that much (max .1 sec on a 8 sec run) +-- and it also messes up finalizers + +-- watch out: when there is a finalizer, it's always called as there +-- can be cases that a finalizer returns (or does) something in case +-- there is no match; an example of this is count() + +local profiled = { } xml.profiled = profiled + +local function profiled_apply(list,parsed,nofparsed,order) + local p = profiled[parsed.pattern] + if p then + p.tested = p.tested + 1 + else + p = { tested = 1, matched = 0, finalized = 0 } + profiled[parsed.pattern] = p + end + local collected = list + for i=1,nofparsed do + local pi = parsed[i] + local kind = pi.kind + if kind == "axis" then + collected = apply_axis[pi.axis](collected) + elseif kind == "nodes" then + collected = apply_nodes(collected,pi.nodetest,pi.nodes) + elseif kind == "expression" then + collected = apply_expression(collected,pi.evaluator,order) + elseif kind == "finalizer" then + collected = pi.finalizer(collected) + p.matched = p.matched + 1 + p.finalized = p.finalized + 1 + return collected + end + if not collected or #collected == 0 then + local pn = i < nofparsed and parsed[nofparsed] + if pn and pn.kind == "finalizer" then + collected = pn.finalizer(collected) + p.finalized = p.finalized + 1 + return collected end + return nil end end - - xml.insert_element_after = xml.insert_element - xml.insert_element_before = function(r,p,e) xml.insert_element(r,p,e,true) end - xml.inject_element_after = xml.inject_element - xml.inject_element_before = function(r,p,e) xml.inject_element(r,p,e,true) end - - function xml.delete_element(root, pattern) - local matches, deleted = { }, { } - local collect = function(r,d,k) matches[#matches+1] = { r, d, k } end - traverse(root, lpath(pattern), collect) - for i=#matches,1,-1 do - local m = matches[i] - deleted[#deleted+1] = table.remove(m[2],m[3]) + if collected then + p.matched = p.matched + 1 + end + return collected +end + +local function traced_apply(list,parsed,nofparsed,order) + if trace_lparse then + lshow(parsed) + end + logs.report("lpath", "collecting : %s",parsed.pattern) + logs.report("lpath", " root tags : %s",tagstostring(list)) + logs.report("lpath", " order : %s",order or "unset") + local collected = list + for i=1,nofparsed do + local pi = parsed[i] + local kind = pi.kind + if kind == "axis" then + collected = apply_axis[pi.axis](collected) + logs.report("lpath", "% 10i : ax : %s",(collected and #collected) or 0,pi.axis) + elseif kind == "nodes" then + collected = apply_nodes(collected,pi.nodetest,pi.nodes) + logs.report("lpath", "% 10i : ns : %s",(collected and #collected) or 0,nodesettostring(pi.nodes,pi.nodetest)) + elseif kind == "expression" then + collected = apply_expression(collected,pi.evaluator,order) + logs.report("lpath", "% 10i : ex : %s -> %s",(collected and #collected) or 0,pi.expression,pi.converted) + elseif kind == "finalizer" then + collected = pi.finalizer(collected) + logs.report("lpath", "% 10i : fi : %s : %s(%s)",(type(collected) == "table" and #collected) or 0,parsed.protocol or xml.defaultprotocol,pi.name,pi.arguments or "") + return collected + end + if not collected or #collected == 0 then + local pn = i < nofparsed and parsed[nofparsed] + if pn and pn.kind == "finalizer" then + collected = pn.finalizer(collected) + logs.report("lpath", "% 10i : fi : %s : %s(%s)",(type(collected) == "table" and #collected) or 0,parsed.protocol or xml.defaultprotocol,pn.name,pn.arguments or "") + return collected + end + return nil end - return deleted end + return collected +end - function xml.replace_element(root, pattern, element) - if type(element) == "string" then - element = convert(element,true) +local function normal_apply(list,parsed,nofparsed,order) + local collected = list + for i=1,nofparsed do + local pi = parsed[i] + local kind = pi.kind + if kind == "axis" then + local axis = pi.axis + if axis ~= "self" then + collected = apply_axis[axis](collected) + end + elseif kind == "nodes" then + collected = apply_nodes(collected,pi.nodetest,pi.nodes) + elseif kind == "expression" then + collected = apply_expression(collected,pi.evaluator,order) + elseif kind == "finalizer" then + return pi.finalizer(collected) end - if element and element.ri then - element = element.dt[element.ri] + if not collected or #collected == 0 then + local pf = i < nofparsed and parsed[nofparsed].finalizer + if pf then + return pf(collected) -- can be anything + end + return nil end - if element then - traverse(root, lpath(pattern), function(rm, d, k) - d[k] = element.dt -- maybe not clever enough - end) + end + return collected +end + +local function parse_apply(list,pattern) + -- we avoid an extra call + local parsed = cache[pattern] + if parsed then + lpathcalls = lpathcalls + 1 + lpathcached = lpathcached + 1 + elseif type(pattern) == "table" then + lpathcalls = lpathcalls + 1 + parsed = pattern + else + parsed = parse_pattern(pattern) or pattern + end + if not parsed then + return + end + local nofparsed = #parsed + if nofparsed == 0 then + return -- something is wrong + end + local one = list[1] + if not one then + return -- something is wrong + elseif not trace_lpath then + return normal_apply(list,parsed,nofparsed,one.mi) + elseif trace_lprofile then + return profiled_apply(list,parsed,nofparsed,one.mi) + else + return traced_apply(list,parsed,nofparsed,one.mi) + end +end + +-- internal (parsed) + +expressions.child = function(e,pattern) + return parse_apply({ e },pattern) -- todo: cache +end +expressions.count = function(e,pattern) + local collected = parse_apply({ e },pattern) -- todo: cache + return (collected and #collected) or 0 +end + +-- external + +expressions.oneof = function(s,...) -- slow + local t = {...} for i=1,#t do if s == t[i] then return true end end return false +end +expressions.error = function(str) + xml.error_handler("unknown function in lpath expression",tostring(str or "?")) + return false +end +expressions.undefined = function(s) + return s == nil +end + +expressions.quit = function(s) + if s or s == nil then + quit_expression = true + end + return true +end + +expressions.print = function(...) + print(...) + return true +end + +expressions.contains = find +expressions.find = find +expressions.upper = upper +expressions.lower = lower +expressions.number = tonumber +expressions.boolean = toboolean + +-- user interface + +local function traverse(root,pattern,handle) + logs.report("xml","use 'xml.selection' instead for '%s'",pattern) + local collected = parse_apply({ root },pattern) + if collected then + for c=1,#collected do + local e = collected[c] + local r = e.__p__ + handle(r,r.dt,e.ni) end end +end - local function load_data(name) -- == io.loaddata - local f, data = io.open(name), "" - if f then - data = f:read("*all",'b') -- 'b' ? - f:close() +local function selection(root,pattern,handle) + local collected = parse_apply({ root },pattern) + if collected then + if handle then + for c=1,#collected do + handle(collected[c]) + end + else + return collected end - return data end +end - function xml.include(xmldata,pattern,attribute,recursive,loaddata) - -- parse="text" (default: xml), encoding="" (todo) - -- attribute = attribute or 'href' - pattern = pattern or 'include' - loaddata = loaddata or load_data - local function include(r,d,k) - local ek, name = d[k], nil - if not attribute or attribute == "" then - local ekdt = ek.dt - name = (type(ekdt) == "table" and ekdt[1]) or ekdt +xml.parse_parser = parser +xml.parse_pattern = parse_pattern +xml.parse_apply = parse_apply +xml.traverse = traverse -- old method, r, d, k +xml.selection = selection -- new method, simple handle + +local lpath = parse_pattern + +xml.lpath = lpath + +function xml.cached_patterns() + return cache +end + +-- generic function finalizer (independant namespace) + +local function dofunction(collected,fnc) + if collected then + local f = functions[fnc] + if f then + for c=1,#collected do + f(collected[c]) end - if not name then - if ek.at then - for a in (attribute or "href"):gmatch("([^|]+)") do - name = ek.at[a] - if name then break end - end + else + logs.report("xml","unknown function '%s'",fnc) + end + end +end + +xml.finalizers.xml["function"] = dofunction +xml.finalizers.tex["function"] = dofunction + +-- functions + +expressions.text = function(e,n) + local rdt = e.__p__.dt + return (rdt and rdt[n]) or "" +end + +expressions.name = function(e,n) -- ns + tg + local found = false + n = tonumber(n) or 0 + if n == 0 then + found = type(e) == "table" and e + elseif n < 0 then + local d, k = e.__p__.dt, e.ni + for i=k-1,1,-1 do + local di = d[i] + if type(di) == "table" then + if n == -1 then + found = di + break + else + n = n + 1 end end - local data = (name and name ~= "" and loaddata(name)) or "" - if data == "" then - xml.empty(d,k) - elseif ek.at["parse"] == "text" then -- for the moment hard coded - d[k] = xml.escaped(data) - else - local xi = xml.convert(data) - if not xi then - xml.empty(d,k) + end + else + local d, k = e.__p__.dt, e.ni + for i=k+1,#d,1 do + local di = d[i] + if type(di) == "table" then + if n == 1 then + found = di + break else - if recursive then - xml.include(xi,pattern,attribute,recursive,loaddata) - end - xml.assign(d,k,xi) + n = n - 1 end end end - xml.each_element(xmldata, pattern, include) end + if found then + local ns, tg = found.rn or found.ns or "", found.tg + if ns ~= "" then + return ns .. ":" .. tg + else + return tg + end + else + return "" + end +end - function xml.strip_whitespace(root, pattern) - traverse(root, lpath(pattern), function(r,d,k) - local dkdt = d[k].dt - if dkdt then -- can be optimized - local t = { } - for i=1,#dkdt do - local str = dkdt[i] - if type(str) == "string" and str:find("^[ \n\r\t]*$") then - -- stripped +expressions.tag = function(e,n) -- only tg + if not e then + return "" + else + local found = false + n = tonumber(n) or 0 + if n == 0 then + found = (type(e) == "table") and e -- seems to fail + elseif n < 0 then + local d, k = e.__p__.dt, e.ni + for i=k-1,1,-1 do + local di = d[i] + if type(di) == "table" then + if n == -1 then + found = di + break else - t[#t+1] = str + n = n + 1 end end - d[k].dt = t end - end) - end - - function xml.rename_space(root, oldspace, newspace) -- fast variant - local ndt = #root.dt - local rename = xml.rename_space - for i=1,ndt or 0 do - local e = root[i] - if type(e) == "table" then - if e.ns == oldspace then - e.ns = newspace - if e.rn then - e.rn = newspace + else + local d, k = e.__p__.dt, e.ni + for i=k+1,#d,1 do + local di = d[i] + if type(di) == "table" then + if n == 1 then + found = di + break + else + n = n - 1 end end - local edt = e.dt - if edt then - rename(edt, oldspace, newspace) - end end end + return (found and found.tg) or "" end +end - function xml.remap_tag(root, pattern, newtg) - traverse(root, lpath(pattern), function(r,d,k) - d[k].tg = newtg - end) - end - function xml.remap_namespace(root, pattern, newns) - traverse(root, lpath(pattern), function(r,d,k) - d[k].ns = newns - end) - end - function xml.check_namespace(root, pattern, newns) - traverse(root, lpath(pattern), function(r,d,k) - local dk = d[k] - if (not dk.rn or dk.rn == "") and dk.ns == "" then - dk.rn = newns - end - end) - end - function xml.remap_name(root, pattern, newtg, newns, newrn) - traverse(root, lpath(pattern), function(r,d,k) - local dk = d[k] - dk.tg = newtg - dk.ns = newns - dk.rn = newrn - end) - end - - function xml.filters.found(root,pattern,check_content) - local found = false - traverse(root, lpath(pattern), function(r,d,k) - if check_content then - local dk = d and d[k] - found = dk and dk.dt and next(dk.dt) and true - else - found = true - end - return true - end) - return found - end +--[[ldx-- +<p>This is the main filter function. It returns whatever is asked for.</p> +--ldx]]-- +function xml.filter(root,pattern) -- no longer funny attribute handling here + return parse_apply({ root },pattern) end --[[ldx-- -<p>Here are a few synonyms.</p> +<p>Often using an iterators looks nicer in the code than passing handler +functions. The <l n='lua'/> book describes how to use coroutines for that +purpose (<url href='http://www.lua.org/pil/9.3.html'/>). This permits +code like:</p> + +<typing> +for r, d, k in xml.elements(xml.load('text.xml'),"title") do + print(d[k]) -- old method +end +for e in xml.collected(xml.load('text.xml'),"title") do + print(e) -- new one +end +</typing> --ldx]]-- -xml.filters.position = xml.filters.index +local wrap, yield = coroutine.wrap, coroutine.yield -xml.count = xml.filters.count -xml.index = xml.filters.index -xml.position = xml.filters.index -xml.first = xml.filters.first -xml.last = xml.filters.last -xml.found = xml.filters.found +function xml.elements(root,pattern,reverse) -- r, d, k + local collected = parse_apply({ root },pattern) + if collected then + if reverse then + return wrap(function() for c=#collected,1,-1 do + local e = collected[c] local r = e.__p__ yield(r,r.dt,e.ni) + end end) + else + return wrap(function() for c=1,#collected do + local e = collected[c] local r = e.__p__ yield(r,r.dt,e.ni) + end end) + end + end + return wrap(function() end) +end -xml.each = xml.each_element -xml.process = xml.process_element -xml.strip = xml.strip_whitespace -xml.collect = xml.collect_elements -xml.all = xml.collect_elements +function xml.collected(root,pattern,reverse) -- e + local collected = parse_apply({ root },pattern) + if collected then + if reverse then + return wrap(function() for c=#collected,1,-1 do yield(collected[c]) end end) + else + return wrap(function() for c=1,#collected do yield(collected[c]) end end) + end + end + return wrap(function() end) +end -xml.insert = xml.insert_element_after -xml.inject = xml.inject_element_after -xml.after = xml.insert_element_after -xml.before = xml.insert_element_before -xml.delete = xml.delete_element -xml.replace = xml.replace_element + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['lxml-mis'] = { + version = 1.001, + comment = "this module is the basis for the lxml-* ones", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local concat = table.concat +local type, next, tonumber, tostring, setmetatable, loadstring = type, next, tonumber, tostring, setmetatable, loadstring +local format, gsub, match = string.format, string.gsub, string.match +local lpegmatch = lpeg.match --[[ldx-- -<p>The following helper functions best belong to the <t>lmxl-ini</t> +<p>The following helper functions best belong to the <t>lxml-ini</t> module. Some are here because we need then in the <t>mk</t> document and other manuals, others came up when playing with this module. Since this module is also used in <l n='mtxrun'/> we've put them here instead of loading mode modules there then needed.</p> --ldx]]-- -function xml.gsub(t,old,new) - if t.dt then - for k,v in ipairs(t.dt) do +local function xmlgsub(t,old,new) -- will be replaced + local dt = t.dt + if dt then + for k=1,#dt do + local v = dt[k] if type(v) == "string" then - t.dt[k] = v:gsub(old,new) + dt[k] = gsub(v,old,new) else - xml.gsub(v,old,new) + xmlgsub(v,old,new) end end end end +--~ xml.gsub = xmlgsub + function xml.strip_leading_spaces(dk,d,k) -- cosmetic, for manual - if d and k and d[k-1] and type(d[k-1]) == "string" then - local s = d[k-1]:match("\n(%s+)") - xml.gsub(dk,"\n"..string.rep(" ",#s),"\n") + if d and k then + local dkm = d[k-1] + if dkm and type(dkm) == "string" then + local s = match(dkm,"\n(%s+)") + xmlgsub(dk,"\n"..rep(" ",#s),"\n") + end end end -function xml.serialize_path(root,lpath,handle) - local dk, r, d, k = xml.first(root,lpath) - dk = xml.copy(dk) - xml.strip_leading_spaces(dk,d,k) - xml.serialize(dk,handle) -end - --~ xml.escapes = { ['&'] = '&', ['<'] = '<', ['>'] = '>', ['"'] = '"' } ---~ xml.unescapes = { } for k,v in pairs(xml.escapes) do xml.unescapes[v] = k end +--~ xml.unescapes = { } for k,v in next, xml.escapes do xml.unescapes[v] = k end ---~ function xml.escaped (str) return str:gsub("(.)" , xml.escapes ) end ---~ function xml.unescaped(str) return str:gsub("(&.-;)", xml.unescapes) end ---~ function xml.cleansed (str) return str:gsub("<.->" , '' ) end -- "%b<>" +--~ function xml.escaped (str) return (gsub(str,"(.)" , xml.escapes )) end +--~ function xml.unescaped(str) return (gsub(str,"(&.-;)", xml.unescapes)) end +--~ function xml.cleansed (str) return (gsub(str,"<.->" , '' )) end -- "%b<>" -do +local P, S, R, C, V, Cc, Cs = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc, lpeg.Cs - local P, S, R, C, V, Cc, Cs = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc, lpeg.Cs +-- 100 * 2500 * "oeps< oeps> oeps&" : gsub:lpeg|lpeg|lpeg +-- +-- 1021:0335:0287:0247 - -- 100 * 2500 * "oeps< oeps> oeps&" : gsub:lpeg|lpeg|lpeg - -- - -- 1021:0335:0287:0247 +-- 10 * 1000 * "oeps< oeps> oeps& asfjhalskfjh alskfjh alskfjh alskfjh ;al J;LSFDJ" +-- +-- 1559:0257:0288:0190 (last one suggested by roberto) - -- 10 * 1000 * "oeps< oeps> oeps& asfjhalskfjh alskfjh alskfjh alskfjh ;al J;LSFDJ" - -- - -- 1559:0257:0288:0190 (last one suggested by roberto) +-- escaped = Cs((S("<&>") / xml.escapes + 1)^0) +-- escaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0) +local normal = (1 - S("<&>"))^0 +local special = P("<")/"<" + P(">")/">" + P("&")/"&" +local escaped = Cs(normal * (special * normal)^0) - -- escaped = Cs((S("<&>") / xml.escapes + 1)^0) - -- escaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0) - local normal = (1 - S("<&>"))^0 - local special = P("<")/"<" + P(">")/">" + P("&")/"&" - local escaped = Cs(normal * (special * normal)^0) +-- 100 * 1000 * "oeps< oeps> oeps&" : gsub:lpeg == 0153:0280:0151:0080 (last one by roberto) - -- 100 * 1000 * "oeps< oeps> oeps&" : gsub:lpeg == 0153:0280:0151:0080 (last one by roberto) +local normal = (1 - S"&")^0 +local special = P("<")/"<" + P(">")/">" + P("&")/"&" +local unescaped = Cs(normal * (special * normal)^0) - -- unescaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0) - -- unescaped = Cs((((P("&")/"") * (P("lt")/"<" + P("gt")/">" + P("amp")/"&") * (P(";")/"")) + 1)^0) - local normal = (1 - S"&")^0 - local special = P("<")/"<" + P(">")/">" + P("&")/"&" - local unescaped = Cs(normal * (special * normal)^0) +-- 100 * 5000 * "oeps <oeps bla='oeps' foo='bar'> oeps </oeps> oeps " : gsub:lpeg == 623:501 msec (short tags, less difference) - -- 100 * 5000 * "oeps <oeps bla='oeps' foo='bar'> oeps </oeps> oeps " : gsub:lpeg == 623:501 msec (short tags, less difference) +local cleansed = Cs(((P("<") * (1-P(">"))^0 * P(">"))/"" + 1)^0) - local cleansed = Cs(((P("<") * (1-P(">"))^0 * P(">"))/"" + 1)^0) +xml.escaped_pattern = escaped +xml.unescaped_pattern = unescaped +xml.cleansed_pattern = cleansed - function xml.escaped (str) return escaped :match(str) end - function xml.unescaped(str) return unescaped:match(str) end - function xml.cleansed (str) return cleansed :match(str) end +function xml.escaped (str) return lpegmatch(escaped,str) end +function xml.unescaped(str) return lpegmatch(unescaped,str) end +function xml.cleansed (str) return lpegmatch(cleansed,str) end +-- this might move + +function xml.fillin(root,pattern,str,check) + local e = xml.first(root,pattern) + if e then + local n = #e.dt + if not check or n == 0 or (n == 1 and e.dt[1] == "") then + e.dt = { str } + end + end end -function xml.join(t,separator,lastseparator) - if #t > 0 then - local result = { } - for k,v in pairs(t) do - result[k] = xml.tostring(v) + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['lxml-aux'] = { + version = 1.001, + comment = "this module is the basis for the lxml-* ones", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- not all functions here make sense anymore vbut we keep them for +-- compatibility reasons + +local trace_manipulations = false trackers.register("lxml.manipulations", function(v) trace_manipulations = v end) + +local xmlparseapply, xmlconvert, xmlcopy, xmlname = xml.parse_apply, xml.convert, xml.copy, xml.name +local xmlinheritedconvert = xml.inheritedconvert + +local type = type +local insert, remove = table.insert, table.remove +local gmatch, gsub = string.gmatch, string.gsub + +local function report(what,pattern,c,e) + logs.report("xml","%s element '%s' (root: '%s', position: %s, index: %s, pattern: %s)",what,xmlname(e),xmlname(e.__p__),c,e.ni,pattern) +end + +local function withelements(e,handle,depth) + if e and handle then + local edt = e.dt + if edt then + depth = depth or 0 + for i=1,#edt do + local e = edt[i] + if type(e) == "table" then + handle(e,depth) + withelements(e,handle,depth+1) + end + end + end + end +end + +xml.withelements = withelements + +function xml.withelement(e,n,handle) -- slow + if e and n ~= 0 and handle then + local edt = e.dt + if edt then + if n > 0 then + for i=1,#edt do + local ei = edt[i] + if type(ei) == "table" then + if n == 1 then + handle(ei) + return + else + n = n - 1 + end + end + end + elseif n < 0 then + for i=#edt,1,-1 do + local ei = edt[i] + if type(ei) == "table" then + if n == -1 then + handle(ei) + return + else + n = n + 1 + end + end + end + end end - if lastseparator then - return concat(result,separator or "",1,#result-1) .. (lastseparator or "") .. result[#result] + end +end + +xml.elements_only = xml.collected + +function xml.each_element(root,pattern,handle,reverse) + local collected = xmlparseapply({ root },pattern) + if collected then + if reverse then + for c=#collected,1,-1 do + handle(collected[c]) + end else - return concat(result,separator) + for c=1,#collected do + handle(collected[c]) + end end - else - return "" + return collected end end +xml.process_elements = xml.each_element + +function xml.process_attributes(root,pattern,handle) + local collected = xmlparseapply({ root },pattern) + if collected and handle then + for c=1,#collected do + handle(collected[c].at) + end + end + return collected +end --[[ldx-- -<p>We provide (at least here) two entity handlers. The more extensive -resolver consults a hash first, tries to convert to <l n='utf'/> next, -and finaly calls a handler when defines. When this all fails, the -original entity is returned.</p> +<p>The following functions collect elements and texts.</p> --ldx]]-- -do if unicode and unicode.utf8 then +-- are these still needed -> lxml-cmp.lua - xml.entities = xml.entities or { } -- xml.entities.handler == function +function xml.collect_elements(root, pattern) + return xmlparseapply({ root },pattern) +end - function xml.entities.handler(e) - return format("[%s]",e) +function xml.collect_texts(root, pattern, flatten) -- todo: variant with handle + local collected = xmlparseapply({ root },pattern) + if collected and flatten then + local xmltostring = xml.tostring + for c=1,#collected do + collected[c] = xmltostring(collected[c].dt) + end end + return collected or { } +end - local char = unicode.utf8.char +function xml.collect_tags(root, pattern, nonamespace) + local collected = xmlparseapply({ root },pattern) + if collected then + local t = { } + for c=1,#collected do + local e = collected[c] + local ns, tg = e.ns, e.tg + if nonamespace then + t[#t+1] = tg + elseif ns == "" then + t[#t+1] = tg + else + t[#t+1] = ns .. ":" .. tg + end + end + return t + end +end + +--[[ldx-- +<p>We've now arrived at the functions that manipulate the tree.</p> +--ldx]]-- + +local no_root = { no_root = true } - local function toutf(s) - return char(tonumber(s,16)) +function xml.redo_ni(d) + for k=1,#d do + local dk = d[k] + if type(dk) == "table" then + dk.ni = k + end + end +end + +local function xmltoelement(whatever,root) + if not whatever then + return nil + end + local element + if type(whatever) == "string" then + element = xmlinheritedconvert(whatever,root) + else + element = whatever -- we assume a table + end + if element.error then + return whatever -- string end + if element then + --~ if element.ri then + --~ element = element.dt[element.ri].dt + --~ else + --~ element = element.dt + --~ end + end + return element +end - local entities = xml.entities -- global entities +xml.toelement = xmltoelement - function utfize(root) - local d = root.dt - for k=1,#d do - local dk = d[k] - if type(dk) == "string" then - -- test prevents copying if no match - if dk:find("&#x.-;") then - d[k] = dk:gsub("&#x(.-);",toutf) +local function copiedelement(element,newparent) + if type(element) == "string" then + return element + else + element = xmlcopy(element).dt + if newparent and type(element) == "table" then + element.__p__ = newparent + end + return element + end +end + +function xml.delete_element(root,pattern) + local collected = xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + local e = collected[c] + local p = e.__p__ + if p then + if trace_manipulations then + report('deleting',pattern,c,e) end - else - utfize(dk) + local d = p.dt + remove(d,e.ni) + xml.redo_ni(d) -- can be made faster and inlined end end end +end - xml.utfize = utfize +function xml.replace_element(root,pattern,whatever) + local element = root and xmltoelement(whatever,root) + local collected = element and xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + local e = collected[c] + local p = e.__p__ + if p then + if trace_manipulations then + report('replacing',pattern,c,e) + end + local d = p.dt + d[e.ni] = copiedelement(element,p) + xml.redo_ni(d) -- probably not needed + end + end + end +end - local function resolve(e) -- hex encoded always first, just to avoid mkii fallbacks - if e:find("#x") then - return char(tonumber(e:sub(3),16)) - else - local ee = entities[e] - if ee then - return ee - else - local h = xml.entities.handler - return (h and h(e)) or "&" .. e .. ";" +local function inject_element(root,pattern,whatever,prepend) + local element = root and xmltoelement(whatever,root) + local collected = element and xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + local e = collected[c] + local r = e.__p__ + local d, k, rri = r.dt, e.ni, r.ri + local edt = (rri and d[rri].dt) or (d and d[k] and d[k].dt) + if edt then + local be, af + local cp = copiedelement(element,e) + if prepend then + be, af = cp, edt + else + be, af = edt, cp + end + for i=1,#af do + be[#be+1] = af[i] + end + if rri then + r.dt[rri].dt = be + else + d[k].dt = be + end + xml.redo_ni(d) end end end +end - local function resolve_entities(root) - if not root.special or root.tg == "@rt@" then - local d = root.dt - for k=1,#d do - local dk = d[k] - if type(dk) == "string" then - if dk:find("&.-;") then - d[k] = dk:gsub("&(.-);",resolve) - end +local function insert_element(root,pattern,whatever,before) -- todo: element als functie + local element = root and xmltoelement(whatever,root) + local collected = element and xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + local e = collected[c] + local r = e.__p__ + local d, k = r.dt, e.ni + if not before then + k = k + 1 + end + insert(d,k,copiedelement(element,r)) + xml.redo_ni(d) + end + end +end + +xml.insert_element = insert_element +xml.insert_element_after = insert_element +xml.insert_element_before = function(r,p,e) insert_element(r,p,e,true) end +xml.inject_element = inject_element +xml.inject_element_after = inject_element +xml.inject_element_before = function(r,p,e) inject_element(r,p,e,true) end + +local function include(xmldata,pattern,attribute,recursive,loaddata) + -- parse="text" (default: xml), encoding="" (todo) + -- attribute = attribute or 'href' + pattern = pattern or 'include' + loaddata = loaddata or io.loaddata + local collected = xmlparseapply({ xmldata },pattern) + if collected then + for c=1,#collected do + local ek = collected[c] + local name = nil + local ekdt = ek.dt + local ekat = ek.at + local epdt = ek.__p__.dt + if not attribute or attribute == "" then + name = (type(ekdt) == "table" and ekdt[1]) or ekdt -- ckeck, probably always tab or str + end + if not name then + for a in gmatch(attribute or "href","([^|]+)") do + name = ekat[a] + if name then break end + end + end + local data = (name and name ~= "" and loaddata(name)) or "" + if data == "" then + epdt[ek.ni] = "" -- xml.empty(d,k) + elseif ekat["parse"] == "text" then + -- for the moment hard coded + epdt[ek.ni] = xml.escaped(data) -- d[k] = xml.escaped(data) + else +--~ local settings = xmldata.settings +--~ settings.parent_root = xmldata -- to be tested +--~ local xi = xmlconvert(data,settings) + local xi = xmlinheritedconvert(data,xmldata) + if not xi then + epdt[ek.ni] = "" -- xml.empty(d,k) else - resolve_entities(dk) + if recursive then + include(xi,pattern,attribute,recursive,loaddata) + end + epdt[ek.ni] = xml.body(xi) -- xml.assign(d,k,xi) end end end end +end - xml.resolve_entities = resolve_entities +xml.include = include - function xml.utfize_text(str) - if str:find("&#") then - return (str:gsub("&#x(.-);",toutf)) - else - return str +--~ local function manipulate(xmldata,pattern,manipulator) -- untested and might go away +--~ local collected = xmlparseapply({ xmldata },pattern) +--~ if collected then +--~ local xmltostring = xml.tostring +--~ for c=1,#collected do +--~ local e = collected[c] +--~ local data = manipulator(xmltostring(e)) +--~ if data == "" then +--~ epdt[e.ni] = "" +--~ else +--~ local xi = xmlinheritedconvert(data,xmldata) +--~ if not xi then +--~ epdt[e.ni] = "" +--~ else +--~ epdt[e.ni] = xml.body(xi) -- xml.assign(d,k,xi) +--~ end +--~ end +--~ end +--~ end +--~ end + +--~ xml.manipulate = manipulate + +function xml.strip_whitespace(root, pattern, nolines) -- strips all leading and trailing space ! + local collected = xmlparseapply({ root },pattern) + if collected then + for i=1,#collected do + local e = collected[i] + local edt = e.dt + if edt then + local t = { } + for i=1,#edt do + local str = edt[i] + if type(str) == "string" then + if str == "" then + -- stripped + else + if nolines then + str = gsub(str,"[ \n\r\t]+"," ") + end + if str == "" then + -- stripped + else + t[#t+1] = str + end + end + else + --~ str.ni = i + t[#t+1] = str + end + end + e.dt = t + end end end +end - function xml.resolve_text_entities(str) -- maybe an lpeg. maybe resolve inline - if str:find("&") then - return (str:gsub("&(.-);",resolve)) - else - return str +function xml.strip_whitespace(root, pattern, nolines, anywhere) -- strips all leading and trailing spacing + local collected = xmlparseapply({ root },pattern) -- beware, indices no longer are valid now + if collected then + for i=1,#collected do + local e = collected[i] + local edt = e.dt + if edt then + if anywhere then + local t = { } + for e=1,#edt do + local str = edt[e] + if type(str) ~= "string" then + t[#t+1] = str + elseif str ~= "" then + -- todo: lpeg for each case + if nolines then + str = gsub(str,"%s+"," ") + end + str = gsub(str,"^%s*(.-)%s*$","%1") + if str ~= "" then + t[#t+1] = str + end + end + end + e.dt = t + else + -- we can assume a regular sparse xml table with no successive strings + -- otherwise we should use a while loop + if #edt > 0 then + -- strip front + local str = edt[1] + if type(str) ~= "string" then + -- nothing + elseif str == "" then + remove(edt,1) + else + if nolines then + str = gsub(str,"%s+"," ") + end + str = gsub(str,"^%s+","") + if str == "" then + remove(edt,1) + else + edt[1] = str + end + end + end + if #edt > 1 then + -- strip end + local str = edt[#edt] + if type(str) ~= "string" then + -- nothing + elseif str == "" then + remove(edt) + else + if nolines then + str = gsub(str,"%s+"," ") + end + str = gsub(str,"%s+$","") + if str == "" then + remove(edt) + else + edt[#edt] = str + end + end + end + end + end end end +end - function xml.show_text_entities(str) - if str:find("&") then - return (str:gsub("&(.-);","[%1]")) - else - return str +local function rename_space(root, oldspace, newspace) -- fast variant + local ndt = #root.dt + for i=1,ndt or 0 do + local e = root[i] + if type(e) == "table" then + if e.ns == oldspace then + e.ns = newspace + if e.rn then + e.rn = newspace + end + end + local edt = e.dt + if edt then + rename_space(edt, oldspace, newspace) + end end end +end + +xml.rename_space = rename_space + +function xml.remap_tag(root, pattern, newtg) + local collected = xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + collected[c].tg = newtg + end + end +end - -- experimental, this will be done differently +function xml.remap_namespace(root, pattern, newns) + local collected = xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + collected[c].ns = newns + end + end +end - function xml.merge_entities(root) - local documententities = root.entities - local allentities = xml.entities - if documententities then - for k, v in pairs(documententities) do - allentities[k] = v +function xml.check_namespace(root, pattern, newns) + local collected = xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + local e = collected[c] + if (not e.rn or e.rn == "") and e.ns == "" then + e.rn = newns end end end +end -end end +function xml.remap_name(root, pattern, newtg, newns, newrn) + local collected = xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + local e = collected[c] + e.tg, e.ns, e.rn = newtg, newns, newrn + end + end +end --- xml.set_text_cleanup(xml.show_text_entities) --- xml.set_text_cleanup(xml.resolve_text_entities) +--[[ldx-- +<p>Here are a few synonyms.</p> +--ldx]]-- ---~ xml.lshow("/../../../a/(b|c)[@d='e']/f") ---~ xml.lshow("/../../../a/!(b|c)[@d='e']/f") ---~ xml.lshow("/../../../a/!b[@d!='e']/f") +xml.each = xml.each_element +xml.process = xml.process_element +xml.strip = xml.strip_whitespace +xml.collect = xml.collect_elements +xml.all = xml.collect_elements ---~ x = xml.convert([[ ---~ <a> ---~ <b n='01'>01</b> ---~ <b n='02'>02</b> ---~ <b n='03'>03</b> ---~ <b n='04'>OK</b> ---~ <b n='05'>05</b> ---~ <b n='06'>06</b> ---~ <b n='07'>ALSO OK</b> ---~ </a> ---~ ]]) +xml.insert = xml.insert_element_after +xml.inject = xml.inject_element_after +xml.after = xml.insert_element_after +xml.before = xml.insert_element_before +xml.delete = xml.delete_element +xml.replace = xml.replace_element ---~ xml.trace_lpath = true ---~ xml.xshow(xml.first(x,"b[position() > 2 and position() < 5 and text() == 'ok']")) ---~ xml.xshow(xml.first(x,"b[position() > 2 and position() < 5 and text() == upper('ok')]")) ---~ xml.xshow(xml.first(x,"b[@n=='03' or @n=='08']")) ---~ xml.xshow(xml.all (x,"b[number(@n)>2 and number(@n)<6]")) ---~ xml.xshow(xml.first(x,"b[find(text(),'ALSO')]")) +end -- of closure ---~ str = [[ ---~ <?xml version="1.0" encoding="utf-8"?> ---~ <story line='mojca'> ---~ <windows>my secret</mouse> ---~ </story> ---~ ]] +do -- create closure to overcome 200 locals limit ---~ x = xml.convert([[ ---~ <a><b n='01'>01</b><b n='02'>02</b><x>xx</x><b n='03'>03</b><b n='04'>OK</b></a> ---~ ]]) ---~ xml.xshow(xml.first(x,"b[tag(2) == 'x']")) ---~ xml.xshow(xml.first(x,"b[tag(1) == 'x']")) ---~ xml.xshow(xml.first(x,"b[tag(-1) == 'x']")) ---~ xml.xshow(xml.first(x,"b[tag(-2) == 'x']")) +if not modules then modules = { } end modules ['lxml-xml'] = { + version = 1.001, + comment = "this module is the basis for the lxml-* ones", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} ---~ print(xml.filter(x,"b/tag(2)")) ---~ print(xml.filter(x,"b/tag(1)")) +local finalizers = xml.finalizers.xml +local xmlfilter = xml.filter -- we could inline this one for speed +local xmltostring = xml.tostring +local xmlserialize = xml.serialize +local function first(collected) -- wrong ? + return collected and collected[1] +end --- filename : l-utils.lua --- comment : split off from luat-lib --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +local function last(collected) + return collected and collected[#collected] +end -if not versions then versions = { } end versions['l-utils'] = 1.001 +local function all(collected) + return collected +end -if not utils then utils = { } end -if not utils.merger then utils.merger = { } end -if not utils.lua then utils.lua = { } end +local function reverse(collected) + if collected then + local reversed = { } + for c=#collected,1,-1 do + reversed[#reversed+1] = collected[c] + end + return reversed + end +end -utils.merger.m_begin = "begin library merge" -utils.merger.m_end = "end library merge" -utils.merger.pattern = - "%c+" .. - "%-%-%s+" .. utils.merger.m_begin .. - "%c+(.-)%c+" .. - "%-%-%s+" .. utils.merger.m_end .. - "%c+" +local function attribute(collected,name) + if collected and #collected > 0 then + local at = collected[1].at + return at and at[name] + end +end -function utils.merger._self_fake_() - return - "-- " .. "created merged file" .. "\n\n" .. - "-- " .. utils.merger.m_begin .. "\n\n" .. - "-- " .. utils.merger.m_end .. "\n\n" +local function att(id,name) + local at = id.at + return at and at[name] end -function utils.report(...) - print(...) +local function count(collected) + return (collected and #collected) or 0 end -function utils.merger._self_load_(name) - local f, data = io.open(name), "" - if f then - data = f:read("*all") - f:close() +local function position(collected,n) + if collected then + n = tonumber(n) or 0 + if n < 0 then + return collected[#collected + n + 1] + elseif n > 0 then + return collected[n] + else + return collected[1].mi or 0 + end end - return data or "" end -function utils.merger._self_save_(name, data) - if data ~= "" then - local f = io.open(name,'w') - if f then - f:write(data) - f:close() - end +local function match(collected) + return (collected and collected[1].mi) or 0 -- match +end + +local function index(collected) + if collected then + return collected[1].ni end end -function utils.merger._self_swap_(data,code) - if data ~= "" then - return (data:gsub(utils.merger.pattern, function(s) - return "\n\n" .. "-- "..utils.merger.m_begin .. "\n" .. code .. "\n" .. "-- "..utils.merger.m_end .. "\n\n" - end, 1)) - else - return "" +local function attributes(collected,arguments) + if collected then + local at = collected[1].at + if arguments then + return at[arguments] + elseif next(at) then + return at -- all of them + end end end -function utils.merger._self_libs_(libs,list) - local result, f = { }, nil - if type(libs) == 'string' then libs = { libs } end - if type(list) == 'string' then list = { list } end - for _, lib in ipairs(libs) do - for _, pth in ipairs(list) do - local name = string.gsub(pth .. "/" .. lib,"\\","/") - f = io.open(name) - if f then - -- utils.report("merging library",name) - result[#result+1] = f:read("*all") - f:close() - list = { pth } -- speed up the search - break +local function chainattribute(collected,arguments) -- todo: optional levels + if collected then + local e = collected[1] + while e do + local at = e.at + if at then + local a = at[arguments] + if a then + return a + end else - -- utils.report("no library",name) + break -- error end + e = e.__p__ end end - return table.concat(result, "\n\n") + return "" end -function utils.merger.selfcreate(libs,list,target) - if target then - utils.merger._self_save_( - target, - utils.merger._self_swap_( - utils.merger._self_fake_(), - utils.merger._self_libs_(libs,list) - ) - ) +local function raw(collected) -- hybrid + if collected then + local e = collected[1] or collected + return (e and xmlserialize(e)) or "" -- only first as we cannot concat function + else + return "" end end -function utils.merger.selfmerge(name,libs,list,target) - utils.merger._self_save_( - target or name, - utils.merger._self_swap_( - utils.merger._self_load_(name), - utils.merger._self_libs_(libs,list) - ) - ) +local function text(collected) -- hybrid + if collected then + local e = collected[1] or collected + return (e and xmltostring(e.dt)) or "" + else + return "" + end end -function utils.merger.selfclean(name) - utils.merger._self_save_( - name, - utils.merger._self_swap_( - utils.merger._self_load_(name), - "" - ) - ) +local function texts(collected) + if collected then + local t = { } + for c=1,#collected do + local e = collection[c] + if e and e.dt then + t[#t+1] = e.dt + end + end + return t + end end -utils.lua.compile_strip = true - -function utils.lua.compile(luafile, lucfile) - -- utils.report("compiling",luafile,"into",lucfile) - os.remove(lucfile) - local command = "-o " .. string.quote(lucfile) .. " " .. string.quote(luafile) - if utils.lua.compile_strip then - command = "-s " .. command +local function tag(collected,n) + if collected then + local c + if n == 0 or not n then + c = collected[1] + elseif n > 1 then + c = collected[n] + else + c = collected[#collected-n+1] + end + return c and c.tg end - if os.spawn("texluac " .. command) == 0 then - return true - elseif os.spawn("luac " .. command) == 0 then - return true - else - return false +end + +local function name(collected,n) + if collected then + local c + if n == 0 or not n then + c = collected[1] + elseif n > 1 then + c = collected[n] + else + c = collected[#collected-n+1] + end + if c then + if c.ns == "" then + return c.tg + else + return c.ns .. ":" .. c.tg + end + end end end +local function tags(collected,nonamespace) + if collected then + local t = { } + for c=1,#collected do + local e = collected[c] + local ns, tg = e.ns, e.tg + if nonamespace or ns == "" then + t[#t+1] = tg + else + t[#t+1] = ns .. ":" .. tg + end + end + return t + end +end +local function empty(collected) + if collected then + for c=1,#collected do + local e = collected[c] + if e then + local edt = e.dt + if edt then + local n = #edt + if n == 1 then + local edk = edt[1] + local typ = type(edk) + if typ == "table" then + return false + elseif edk ~= "" then -- maybe an extra tester for spacing only + return false + end + elseif n > 1 then + return false + end + end + end + end + end + return true +end --- filename : luat-lib.lua --- comment : companion to luat-lib.tex --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +finalizers.first = first +finalizers.last = last +finalizers.all = all +finalizers.reverse = reverse +finalizers.elements = all +finalizers.default = all +finalizers.attribute = attribute +finalizers.att = att +finalizers.count = count +finalizers.position = position +finalizers.match = match +finalizers.index = index +finalizers.attributes = attributes +finalizers.chainattribute = chainattribute +finalizers.text = text +finalizers.texts = texts +finalizers.tag = tag +finalizers.name = name +finalizers.tags = tags +finalizers.empty = empty -if not versions then versions = { } end versions['luat-lib'] = 1.001 +-- shortcuts -- we could support xmlfilter(id,pattern,first) --- mostcode moved to the l-*.lua and other luat-*.lua files +function xml.first(id,pattern) + return first(xmlfilter(id,pattern)) +end --- os / io +function xml.last(id,pattern) + return last(xmlfilter(id,pattern)) +end -os.setlocale(nil,nil) -- useless feature and even dangerous in luatex +function xml.count(id,pattern) + return count(xmlfilter(id,pattern)) +end --- os.platform +function xml.attribute(id,pattern,a,default) + return attribute(xmlfilter(id,pattern),a,default) +end --- mswin|bccwin|mingw|cygwin windows --- darwin|rhapsody|nextstep macosx --- netbsd|unix unix --- linux linux +function xml.raw(id,pattern) + if pattern then + return raw(xmlfilter(id,pattern)) + else + return raw(id) + end +end -if not io.fileseparator then - if string.find(os.getenv("PATH"),";") then - io.fileseparator, io.pathseparator, os.platform = "\\", ";", os.type or "windows" +function xml.text(id,pattern) + if pattern then + -- return text(xmlfilter(id,pattern)) + local collected = xmlfilter(id,pattern) + return (collected and xmltostring(collected[1].dt)) or "" + elseif id then + -- return text(id) + return xmltostring(id.dt) or "" else - io.fileseparator, io.pathseparator, os.platform = "/" , ":", os.type or "unix" + return "" end end -os.platform = os.platform or os.type or (io.pathseparator == ";" and "windows") or "unix" +xml.content = text --- arg normalization --- --- for k,v in pairs(arg) do print(k,v) end +function xml.position(id,pattern,n) -- element + return position(xmlfilter(id,pattern),n) +end --- environment +function xml.match(id,pattern) -- number + return match(xmlfilter(id,pattern)) +end -if not environment then environment = { } end +function xml.empty(id,pattern) + return empty(xmlfilter(id,pattern)) +end -environment.ownbin = environment.ownbin or arg[-2] or arg[-1] or arg[0] or "luatex" +xml.all = xml.filter +xml.index = xml.position +xml.found = xml.filter -local ownpath = nil -- we could use a metatable here -function environment.ownpath() - if not ownpath then - for p in string.gmatch(os.getenv("PATH"),"[^"..io.pathseparator.."]+") do - local b = file.join(p,environment.ownbin) - if lfs.isfile(b..".exe") or lfs.isfile(b) then - ownpath = p - break - end - end - if not ownpath then ownpath = '.' end - end - return ownpath +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['luat-env'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- A former version provided functionality for non embeded core +-- scripts i.e. runtime library loading. Given the amount of +-- Lua code we use now, this no longer makes sense. Much of this +-- evolved before bytecode arrays were available and so a lot of +-- code has disappeared already. + +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) + +local format, sub, match, gsub, find = string.format, string.sub, string.match, string.gsub, string.find +local unquote, quote = string.unquote, string.quote + +-- precautions + +os.setlocale(nil,nil) -- useless feature and even dangerous in luatex + +function os.setlocale() + -- no way you can mess with it end +-- dirty tricks + if arg and (arg[0] == 'luatex' or arg[0] == 'luatex.exe') and arg[1] == "--luaonly" then arg[-1]=arg[0] arg[0]=arg[2] for k=3,#arg do arg[k-2]=arg[k] end arg[#arg]=nil arg[#arg]=nil end -environment.arguments = { } -environment.files = { } -environment.sorted_argument_keys = nil +if profiler and os.env["MTX_PROFILE_RUN"] == "YES" then + profiler.start("luatex-profile.log") +end + +-- environment -environment.platform = os.platform +environment = environment or { } +environment.arguments = { } +environment.files = { } +environment.sortedflags = nil + +if not environment.jobname or environment.jobname == "" then if tex then environment.jobname = tex.jobname end end +if not environment.version or environment.version == "" then environment.version = "unknown" end +if not environment.jobname then environment.jobname = "unknown" end function environment.initialize_arguments(arg) - environment.arguments = { } - environment.files = { } - environment.sorted_argument_keys = nil - for index, argument in pairs(arg) do + local arguments, files = { }, { } + environment.arguments, environment.files, environment.sortedflags = arguments, files, nil + for index=1,#arg do + local argument = arg[index] if index > 0 then - local flag, value = argument:match("^%-+(.+)=(.-)$") + local flag, value = match(argument,"^%-+(.-)=(.-)$") if flag then - environment.arguments[flag] = string.unquote(value or "") + arguments[flag] = unquote(value or "") else - flag = argument:match("^%-+(.+)") + flag = match(argument,"^%-+(.+)") if flag then - environment.arguments[flag] = true + arguments[flag] = true else - environment.files[#environment.files+1] = argument + files[#files+1] = argument end end end @@ -4697,41 +7355,45 @@ function environment.initialize_arguments(arg) environment.ownname = environment.ownname or arg[0] or 'unknown.lua' end -function environment.showarguments() - for k,v in pairs(environment.arguments) do - print(k .. " : " .. tostring(v)) - end - if #environment.files > 0 then - print("files : " .. table.concat(environment.files, " ")) - end -end - function environment.setargument(name,value) environment.arguments[name] = value end -function environment.argument(name) - if environment.arguments[name] then - return environment.arguments[name] - else - if not environment.sorted_argument_keys then - environment.sorted_argument_keys = { } - for _,v in pairs(table.sortedkeys(environment.arguments)) do - table.insert(environment.sorted_argument_keys, "^" .. v) +-- todo: defaults, better checks e.g on type (boolean versus string) +-- +-- tricky: too many hits when we support partials unless we add +-- a registration of arguments so from now on we have 'partial' + +function environment.argument(name,partial) + local arguments, sortedflags = environment.arguments, environment.sortedflags + if arguments[name] then + return arguments[name] + elseif partial then + if not sortedflags then + sortedflags = table.sortedkeys(arguments) + for k=1,#sortedflags do + sortedflags[k] = "^" .. sortedflags[k] end + environment.sortedflags = sortedflags end - for _,v in pairs(environment.sorted_argument_keys) do - if name:find(v) then - return environment.arguments[v:sub(2,#v)] + -- example of potential clash: ^mode ^modefile + for k=1,#sortedflags do + local v = sortedflags[k] + if find(name,v) then + return arguments[sub(v,2,#v)] end end end return nil end +environment.argument("x",true) + function environment.split_arguments(separator) -- rather special, cut-off before separator local done, before, after = false, { }, { } - for _,v in ipairs(environment.original_arguments) do + local original_arguments = environment.original_arguments + for k=1,#original_arguments do + local v = original_arguments[k] if not done and v == separator then done = true elseif done then @@ -4743,355 +7405,979 @@ function environment.split_arguments(separator) -- rather special, cut-off befor return before, after end -function environment.reconstruct_commandline(arg) - if not arg then arg = environment.original_arguments end - local result = { } - for _,a in ipairs(arg) do -- ipairs 1 .. #n - local kk, vv = a:match("^(%-+.-)=(.+)$") - if kk and vv then - if vv:find(" ") then - result[#result+1] = kk .. "=" .. string.quote(vv) +function environment.reconstruct_commandline(arg,noquote) + arg = arg or environment.original_arguments + if noquote and #arg == 1 then + local a = arg[1] + a = resolvers.resolve(a) + a = unquote(a) + return a + elseif #arg > 0 then + local result = { } + for i=1,#arg do + local a = arg[i] + a = resolvers.resolve(a) + a = unquote(a) + a = gsub(a,'"','\\"') -- tricky + if find(a," ") then + result[#result+1] = quote(a) else result[#result+1] = a end - elseif a:find(" ") then - result[#result+1] = string.quote(a) - else - result[#result+1] = a end + return table.join(result," ") + else + return "" end - return table.join(result," ") end if arg then - environment.initialize_arguments(arg) - environment.original_arguments = arg + + -- new, reconstruct quoted snippets (maybe better just remove the " then and add them later) + local newarg, instring = { }, false + + for index=1,#arg do + local argument = arg[index] + if find(argument,"^\"") then + newarg[#newarg+1] = gsub(argument,"^\"","") + if not find(argument,"\"$") then + instring = true + end + elseif find(argument,"\"$") then + newarg[#newarg] = newarg[#newarg] .. " " .. gsub(argument,"\"$","") + instring = false + elseif instring then + newarg[#newarg] = newarg[#newarg] .. " " .. argument + else + newarg[#newarg+1] = argument + end + end + for i=1,-5,-1 do + newarg[i] = arg[i] + end + + environment.initialize_arguments(newarg) + environment.original_arguments = newarg + environment.raw_arguments = arg + arg = { } -- prevent duplicate handling + end +-- weird place ... depends on a not yet loaded module --- filename : luat-inp.lua --- comment : companion to luat-lib.tex --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +function environment.texfile(filename) + return resolvers.find_file(filename,'tex') +end --- This lib is multi-purpose and can be loaded again later on so that --- additional functionality becomes available. We will split this --- module in components when we're done with prototyping. +function environment.luafile(filename) + local resolved = resolvers.find_file(filename,'tex') or "" + if resolved ~= "" then + return resolved + end + resolved = resolvers.find_file(filename,'texmfscripts') or "" + if resolved ~= "" then + return resolved + end + return resolvers.find_file(filename,'luatexlibs') or "" +end --- TODO: os.getenv -> os.env[] --- TODO: instances.[hashes,cnffiles,configurations,522] -> ipairs (alles check, sneller) --- TODO: check escaping in find etc, too much, too slow +environment.loadedluacode = loadfile -- can be overloaded --- This is the first code I wrote for LuaTeX, so it needs some cleanup. +--~ function environment.loadedluacode(name) +--~ if os.spawn("texluac -s -o texluac.luc " .. name) == 0 then +--~ local chunk = loadstring(io.loaddata("texluac.luc")) +--~ os.remove("texluac.luc") +--~ return chunk +--~ else +--~ environment.loadedluacode = loadfile -- can be overloaded +--~ return loadfile(name) +--~ end +--~ end --- To be considered: hash key lowercase, first entry in table filename --- (any case), rest paths (so no need for optimization). Or maybe a --- separate table that matches lowercase names to mixed case when --- present. In that case the lower() cases can go away. I will do that --- only when we run into problems with names ... well ... Iwona-Regular. +function environment.luafilechunk(filename) -- used for loading lua bytecode in the format + filename = file.replacesuffix(filename, "lua") + local fullname = environment.luafile(filename) + if fullname and fullname ~= "" then + if trace_locating then + logs.report("fileio","loading file %s", fullname) + end + return environment.loadedluacode(fullname) + else + if trace_locating then + logs.report("fileio","unknown file %s", filename) + end + return nil + end +end --- Beware, loading and saving is overloaded in luat-tmp! +-- the next ones can use the previous ones / combine -if not versions then versions = { } end versions['luat-inp'] = 1.001 -if not environment then environment = { } end -if not file then file = { } end +function environment.loadluafile(filename, version) + local lucname, luaname, chunk + local basename = file.removesuffix(filename) + if basename == filename then + lucname, luaname = basename .. ".luc", basename .. ".lua" + else + lucname, luaname = nil, basename -- forced suffix + end + -- when not overloaded by explicit suffix we look for a luc file first + local fullname = (lucname and environment.luafile(lucname)) or "" + if fullname ~= "" then + if trace_locating then + logs.report("fileio","loading %s", fullname) + end + chunk = loadfile(fullname) -- this way we don't need a file exists check + end + if chunk then + assert(chunk)() + if version then + -- we check of the version number of this chunk matches + local v = version -- can be nil + if modules and modules[filename] then + v = modules[filename].version -- new method + elseif versions and versions[filename] then + v = versions[filename] -- old method + end + if v == version then + return true + else + if trace_locating then + logs.report("fileio","version mismatch for %s: lua=%s, luc=%s", filename, v, version) + end + environment.loadluafile(filename) + end + else + return true + end + end + fullname = (luaname and environment.luafile(luaname)) or "" + if fullname ~= "" then + if trace_locating then + logs.report("fileio","loading %s", fullname) + end + chunk = loadfile(fullname) -- this way we don't need a file exists check + if not chunk then + if trace_locating then + logs.report("fileio","unknown file %s", filename) + end + else + assert(chunk)() + return true + end + end + return false +end -if environment.aleph_mode == nil then environment.aleph_mode = true end -- temp hack -if not input then input = { } end -if not input.suffixes then input.suffixes = { } end -if not input.formats then input.formats = { } end -if not input.aux then input.aux = { } end +end -- of closure -if not input.suffixmap then input.suffixmap = { } end +do -- create closure to overcome 200 locals limit -if not input.locators then input.locators = { } end -- locate databases -if not input.hashers then input.hashers = { } end -- load databases -if not input.generators then input.generators = { } end -- generate databases -if not input.filters then input.filters = { } end -- conversion filters +if not modules then modules = { } end modules ['trac-inf'] = { + version = 1.001, + comment = "companion to trac-inf.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} local format = string.format -input.locators.notfound = { nil } -input.hashers.notfound = { nil } -input.generators.notfound = { nil } - -input.cacheversion = '1.0.1' -input.banner = nil -input.verbose = false -input.debug = false -input.cnfname = 'texmf.cnf' -input.luaname = 'texmfcnf.lua' -input.lsrname = 'ls-R' -input.luasuffix = '.tma' -input.lucsuffix = '.tmc' - --- we use a cleaned up list / format=any is a wildcard, as is *name - -input.formats['afm'] = 'AFMFONTS' input.suffixes['afm'] = { 'afm' } -input.formats['enc'] = 'ENCFONTS' input.suffixes['enc'] = { 'enc' } -input.formats['fmt'] = 'TEXFORMATS' input.suffixes['fmt'] = { 'fmt' } -input.formats['map'] = 'TEXFONTMAPS' input.suffixes['map'] = { 'map' } -input.formats['mp'] = 'MPINPUTS' input.suffixes['mp'] = { 'mp' } -input.formats['ocp'] = 'OCPINPUTS' input.suffixes['ocp'] = { 'ocp' } -input.formats['ofm'] = 'OFMFONTS' input.suffixes['ofm'] = { 'ofm', 'tfm' } -input.formats['otf'] = 'OPENTYPEFONTS' input.suffixes['otf'] = { 'otf' } -- 'ttf' -input.formats['opl'] = 'OPLFONTS' input.suffixes['opl'] = { 'opl' } -input.formats['otp'] = 'OTPINPUTS' input.suffixes['otp'] = { 'otp' } -input.formats['ovf'] = 'OVFFONTS' input.suffixes['ovf'] = { 'ovf', 'vf' } -input.formats['ovp'] = 'OVPFONTS' input.suffixes['ovp'] = { 'ovp' } -input.formats['tex'] = 'TEXINPUTS' input.suffixes['tex'] = { 'tex' } -input.formats['tfm'] = 'TFMFONTS' input.suffixes['tfm'] = { 'tfm' } -input.formats['ttf'] = 'TTFONTS' input.suffixes['ttf'] = { 'ttf', 'ttc' } -input.formats['pfb'] = 'T1FONTS' input.suffixes['pfb'] = { 'pfb', 'pfa' } -input.formats['vf'] = 'VFFONTS' input.suffixes['vf'] = { 'vf' } - -input.formats['fea'] = 'FONTFEATURES' input.suffixes['fea'] = { 'fea' } -input.formats['cid'] = 'FONTCIDMAPS' input.suffixes['cid'] = { 'cid', 'cidmap' } - -input.formats ['texmfscripts'] = 'TEXMFSCRIPTS' -- new -input.suffixes['texmfscripts'] = { 'rb', 'pl', 'py' } -- 'lua' - -input.formats ['lua'] = 'LUAINPUTS' -- new -input.suffixes['lua'] = { 'lua', 'luc', 'tma', 'tmc' } +local statusinfo, n, registered = { }, 0, { } --- here we catch a few new thingies (todo: add these paths to context.tmf) --- --- FONTFEATURES = .;$TEXMF/fonts/fea// --- FONTCIDMAPS = .;$TEXMF/fonts/cid// +statistics = statistics or { } -function input.checkconfigdata(instance) -- not yet ok, no time for debugging now - local function fix(varname,default) - local proname = varname .. "." .. instance.progname or "crap" - local p = instance.environment[proname] - local v = instance.environment[varname] - if not ((p and p ~= "") or (v and v ~= "")) then - instance.variables[varname] = default -- or environment? +statistics.enable = true +statistics.threshold = 0.05 + +-- timing functions + +local clock = os.gettimeofday or os.clock + +local notimer + +function statistics.hastimer(instance) + return instance and instance.starttime +end + +function statistics.resettiming(instance) + if not instance then + notimer = { timing = 0, loadtime = 0 } + else + instance.timing, instance.loadtime = 0, 0 + end +end + +function statistics.starttiming(instance) + if not instance then + notimer = { } + instance = notimer + end + local it = instance.timing + if not it then + it = 0 + end + if it == 0 then + instance.starttime = clock() + if not instance.loadtime then + instance.loadtime = 0 end + else +--~ logs.report("system","nested timing (%s)",tostring(instance)) end - fix("LUAINPUTS" , ".;$TEXINPUTS;$TEXMFSCRIPTS") - fix("FONTFEATURES", ".;$TEXMF/fonts/fea//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") - fix("FONTCIDMAPS" , ".;$TEXMF/fonts/cid//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") + instance.timing = it + 1 end --- backward compatible ones +function statistics.stoptiming(instance, report) + if not instance then + instance = notimer + end + if instance then + local it = instance.timing + if it > 1 then + instance.timing = it - 1 + else + local starttime = instance.starttime + if starttime then + local stoptime = clock() + local loadtime = stoptime - starttime + instance.stoptime = stoptime + instance.loadtime = instance.loadtime + loadtime + if report then + statistics.report("load time %0.3f",loadtime) + end + instance.timing = 0 + return loadtime + end + end + end + return 0 +end -input.alternatives = { } +function statistics.elapsedtime(instance) + if not instance then + instance = notimer + end + return format("%0.3f",(instance and instance.loadtime) or 0) +end -input.alternatives['map files'] = 'map' -input.alternatives['enc files'] = 'enc' -input.alternatives['cid files'] = 'cid' -input.alternatives['fea files'] = 'fea' -input.alternatives['opentype fonts'] = 'otf' -input.alternatives['truetype fonts'] = 'ttf' -input.alternatives['truetype collections'] = 'ttc' -input.alternatives['type1 fonts'] = 'pfb' +function statistics.elapsedindeed(instance) + if not instance then + instance = notimer + end + local t = (instance and instance.loadtime) or 0 + return t > statistics.threshold +end --- obscure ones +function statistics.elapsedseconds(instance,rest) -- returns nil if 0 seconds + if statistics.elapsedindeed(instance) then + return format("%s seconds %s", statistics.elapsedtime(instance),rest or "") + end +end -input.formats ['misc fonts'] = '' -input.suffixes['misc fonts'] = { } - -input.formats ['sfd'] = 'SFDFONTS' -input.suffixes ['sfd'] = { 'sfd' } -input.alternatives['subfont definition files'] = 'sfd' - -function input.reset() - - local instance = { } - - instance.rootpath = '' - instance.treepath = '' - instance.progname = environment.progname or 'context' - instance.engine = environment.engine or 'luatex' - instance.format = '' - instance.environment = { } - instance.variables = { } - instance.expansions = { } - instance.files = { } - instance.remap = { } - instance.configuration = { } - instance.setup = { } - instance.order = { } - instance.found = { } - instance.foundintrees = { } - instance.kpsevars = { } - instance.hashes = { } - instance.cnffiles = { } - instance.luafiles = { } - instance.lists = { } - instance.remember = true - instance.diskcache = true - instance.renewcache = false - instance.scandisk = true - instance.cachepath = nil - instance.loaderror = false - instance.smallcache = false - instance.savelists = true - instance.cleanuppaths = true - instance.allresults = false - instance.pattern = nil -- lists - instance.kpseonly = false -- lists - instance.cachefile = 'tmftools' - instance.loadtime = 0 - instance.starttime = 0 - instance.stoptime = 0 - instance.validfile = function(path,name) return true end - instance.data = { } -- only for loading - instance.force_suffixes = true - instance.dummy_path_expr = "^!*unset/*$" - instance.fakepaths = { } - instance.lsrmode = false - - if os.env then - -- store once, freeze and faster - for k,v in pairs(os.env) do - instance.environment[k] = input.bare_variable(v) - end - else - -- we will access os.env frequently - for k,v in pairs({'HOME','TEXMF','TEXMFCNF'}) do - local e = os.getenv(v) - if e then - -- input.report("setting",v,"to",input.bare_variable(e)) - instance.environment[v] = input.bare_variable(e) +-- general function + +function statistics.register(tag,fnc) + if statistics.enable and type(fnc) == "function" then + local rt = registered[tag] or (#statusinfo + 1) + statusinfo[rt] = { tag, fnc } + registered[tag] = rt + if #tag > n then n = #tag end + end +end + +function statistics.show(reporter) + if statistics.enable then + if not reporter then reporter = function(tag,data,n) texio.write_nl(tag .. " " .. data) end end + -- this code will move + local register = statistics.register + register("luatex banner", function() + return string.lower(status.banner) + end) + register("control sequences", function() + return format("%s of %s", status.cs_count, status.hash_size+status.hash_extra) + end) + register("callbacks", function() + local total, indirect = status.callbacks or 0, status.indirect_callbacks or 0 + return format("direct: %s, indirect: %s, total: %s", total-indirect, indirect, total) + end) + register("current memory usage", statistics.memused) + register("runtime",statistics.runtime) +-- -- + for i=1,#statusinfo do + local s = statusinfo[i] + local r = s[2]() + if r then + reporter(s[1],r,n) end end + texio.write_nl("") -- final newline + statistics.enable = false + end +end + +function statistics.show_job_stat(tag,data,n) + texio.write_nl(format("%-15s: %s - %s","mkiv lua stats",tag:rpadd(n," "),data)) +end + +function statistics.memused() -- no math.round yet -) + local round = math.round or math.floor + return format("%s MB (ctx: %s MB)",round(collectgarbage("count")/1000), round(status.luastate_bytes/1000000)) +end + +if statistics.runtime then + -- already loaded and set +elseif luatex and luatex.starttime then + statistics.starttime = luatex.starttime + statistics.loadtime = 0 + statistics.timing = 0 +else + statistics.starttiming(statistics) +end + +function statistics.runtime() + statistics.stoptiming(statistics) + return statistics.formatruntime(statistics.elapsedtime(statistics)) +end + +function statistics.formatruntime(runtime) + return format("%s seconds", statistics.elapsedtime(statistics)) +end + +function statistics.timed(action,report) + local timer = { } + report = report or logs.simple + statistics.starttiming(timer) + action() + statistics.stoptiming(timer) + report("total runtime: %s",statistics.elapsedtime(timer)) +end + +-- where, not really the best spot for this: + +commands = commands or { } + +local timer + +function commands.resettimer() + statistics.resettiming(timer) + statistics.starttiming(timer) +end + +function commands.elapsedtime() + statistics.stoptiming(timer) + tex.sprint(statistics.elapsedtime(timer)) +end + +commands.resettimer() + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['trac-log'] = { + version = 1.001, + comment = "companion to trac-log.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- this is old code that needs an overhaul + +--~ io.stdout:setvbuf("no") +--~ io.stderr:setvbuf("no") + +local write_nl, write = texio.write_nl or print, texio.write or io.write +local format, gmatch = string.format, string.gmatch +local texcount = tex and tex.count + +if texlua then + write_nl = print + write = io.write +end + +--[[ldx-- +<p>This is a prelude to a more extensive logging module. For the sake +of parsing log files, in addition to the standard logging we will +provide an <l n='xml'/> structured file. Actually, any logging that +is hooked into callbacks will be \XML\ by default.</p> +--ldx]]-- + +logs = logs or { } +logs.xml = logs.xml or { } +logs.tex = logs.tex or { } + +--[[ldx-- +<p>This looks pretty ugly but we need to speed things up a bit.</p> +--ldx]]-- + +logs.moreinfo = [[ +more information about ConTeXt and the tools that come with it can be found at: + +maillist : ntg-context@ntg.nl / http://www.ntg.nl/mailman/listinfo/ntg-context +webpage : http://www.pragma-ade.nl / http://tex.aanhet.net +wiki : http://contextgarden.net +]] + +logs.levels = { + ['error'] = 1, + ['warning'] = 2, + ['info'] = 3, + ['debug'] = 4, +} + +logs.functions = { + 'report', 'start', 'stop', 'push', 'pop', 'line', 'direct', + 'start_run', 'stop_run', + 'start_page_number', 'stop_page_number', + 'report_output_pages', 'report_output_log', + 'report_tex_stat', 'report_job_stat', + 'show_open', 'show_close', 'show_load', +} + +logs.tracers = { +} + +logs.level = 0 +logs.mode = string.lower((os.getenv("MTX.LOG.MODE") or os.getenv("MTX_LOG_MODE") or "tex")) + +function logs.set_level(level) + logs.level = logs.levels[level] or level +end + +function logs.set_method(method) + for _, v in next, logs.functions do + logs[v] = logs[method][v] or function() end + end +end + +-- tex logging + +function logs.tex.report(category,fmt,...) -- new + if fmt then + write_nl(category .. " | " .. format(fmt,...)) + else + write_nl(category .. " |") + end +end + +function logs.tex.line(fmt,...) -- new + if fmt then + write_nl(format(fmt,...)) + else + write_nl("") end +end - -- cross referencing +--~ function logs.tex.start_page_number() +--~ local real, user, sub = texcount.realpageno, texcount.userpageno, texcount.subpageno +--~ if real > 0 then +--~ if user > 0 then +--~ if sub > 0 then +--~ write(format("[%s.%s.%s",real,user,sub)) +--~ else +--~ write(format("[%s.%s",real,user)) +--~ end +--~ else +--~ write(format("[%s",real)) +--~ end +--~ else +--~ write("[-") +--~ end +--~ end - for k, v in pairs(input.suffixes) do - for _, vv in pairs(v) do - if vv then - input.suffixmap[vv] = k +--~ function logs.tex.stop_page_number() +--~ write("]") +--~ end + +local real, user, sub + +function logs.tex.start_page_number() + real, user, sub = texcount.realpageno, texcount.userpageno, texcount.subpageno +end + +function logs.tex.stop_page_number() + if real > 0 then + if user > 0 then + if sub > 0 then + logs.report("pages", "flushing realpage %s, userpage %s, subpage %s",real,user,sub) + else + logs.report("pages", "flushing realpage %s, userpage %s",real,user) end + else + logs.report("pages", "flushing realpage %s",real) end + else + logs.report("pages", "flushing page") end + io.flush() +end + +logs.tex.report_job_stat = statistics.show_job_stat - return instance +-- xml logging +function logs.xml.report(category,fmt,...) -- new + if fmt then + write_nl(format("<r category='%s'>%s</r>",category,format(fmt,...))) + else + write_nl(format("<r category='%s'/>",category)) + end +end +function logs.xml.line(fmt,...) -- new + if fmt then + write_nl(format("<r>%s</r>",format(fmt,...))) + else + write_nl("<r/>") + end end -function input.reset_hashes(instance) - instance.lists = { } - instance.found = { } +function logs.xml.start() if logs.level > 0 then tw("<%s>" ) end end +function logs.xml.stop () if logs.level > 0 then tw("</%s>") end end +function logs.xml.push () if logs.level > 0 then tw("<!-- ") end end +function logs.xml.pop () if logs.level > 0 then tw(" -->" ) end end + +function logs.xml.start_run() + write_nl("<?xml version='1.0' standalone='yes'?>") + write_nl("<job>") -- xmlns='www.pragma-ade.com/luatex/schemas/context-job.rng' + write_nl("") end -function input.bare_variable(str) -- assumes str is a string - -- return string.gsub(string.gsub(string.gsub(str,"%s+$",""),'^"(.+)"$',"%1"),"^'(.+)'$","%1") - return (str:gsub("\s*([\"\']?)(.+)%1\s*", "%2")) +function logs.xml.stop_run() + write_nl("</job>") end -if texio then - input.log = texio.write_nl -else - input.log = print +function logs.xml.start_page_number() + write_nl(format("<p real='%s' page='%s' sub='%s'", texcount.realpageno, texcount.userpageno, texcount.subpageno)) end -function input.simple_logger(kind, name) - if name and name ~= "" then - if input.banner then - input.log(input.banner..kind..": "..name) - else - input.log("<<"..kind..": "..name..">>") - end +function logs.xml.stop_page_number() + write("/>") + write_nl("") +end + +function logs.xml.report_output_pages(p,b) + write_nl(format("<v k='pages' v='%s'/>", p)) + write_nl(format("<v k='bytes' v='%s'/>", b)) + write_nl("") +end + +function logs.xml.report_output_log() +end + +function logs.xml.report_tex_stat(k,v) + texiowrite_nl("log","<v k='"..k.."'>"..tostring(v).."</v>") +end + +local level = 0 + +function logs.xml.show_open(name) + level = level + 1 + texiowrite_nl(format("<f l='%s' n='%s'>",level,name)) +end + +function logs.xml.show_close(name) + texiowrite("</f> ") + level = level - 1 +end + +function logs.xml.show_load(name) + texiowrite_nl(format("<f l='%s' n='%s'/>",level+1,name)) +end + +-- + +local name, banner = 'report', 'context' + +local function report(category,fmt,...) + if fmt then + write_nl(format("%s | %s: %s",name,category,format(fmt,...))) + elseif category then + write_nl(format("%s | %s",name,category)) else - if input.banner then - input.log(input.banner..kind..": no name") - else - input.log("<<"..kind..": no name>>") - end + write_nl(format("%s |",name)) end end -function input.dummy_logger() +local function simple(fmt,...) + if fmt then + write_nl(format("%s | %s",name,format(fmt,...))) + else + write_nl(format("%s |",name)) + end end -function input.settrace(n) - input.trace = tonumber(n or 0) - if input.trace > 0 then - input.logger = input.simple_logger - input.verbose = true +function logs.setprogram(_name_,_banner_,_verbose_) + name, banner = _name_, _banner_ + if _verbose_ then + trackers.enable("resolvers.locating") + end + logs.set_method("tex") + logs.report = report -- also used in libraries + logs.simple = simple -- only used in scripts ! + if utils then + utils.report = simple + end + logs.verbose = _verbose_ +end + +function logs.setverbose(what) + if what then + trackers.enable("resolvers.locating") else - input.logger = function() end + trackers.disable("resolvers.locating") end + logs.verbose = what or false end -function input.report(...) -- inefficient - if input.verbose then - if input.banner then - input.log(input.banner .. table.concat({...},' ')) - elseif input.logmode() == 'xml' then - input.log("<t>"..table.concat({...},' ').."</t>") - else - input.log("<<"..table.concat({...},' ')..">>") - end +function logs.extendbanner(_banner_,_verbose_) + banner = banner .. " | ".. _banner_ + if _verbose_ ~= nil then + logs.setverbose(what) end end -function input.reportlines(str) - if type(str) == "string" then - str = str:split("\n") +logs.verbose = false +logs.report = logs.tex.report +logs.simple = logs.tex.report + +function logs.reportlines(str) -- todo: <lines></lines> + for line in gmatch(str,"(.-)[\n\r]") do + logs.report(line) end - for _,v in pairs(str) do input.report(v) end end -input.settrace(tonumber(os.getenv("MTX.INPUT.TRACE") or os.getenv("MTX_INPUT_TRACE") or input.trace or 0)) +function logs.reportline() -- for scripts too + logs.report() +end --- These functions can be used to test the performance, especially --- loading the database files. +logs.simpleline = logs.reportline -do - local clock = os.gettimeofday or os.clock +function logs.reportbanner() -- for scripts too + logs.report(banner) +end - function input.starttiming(instance) - if instance then - instance.starttime = clock() - if not instance.loadtime then - instance.loadtime = 0 - end - end +function logs.help(message,option) + logs.reportbanner() + logs.reportline() + logs.reportlines(message) + local moreinfo = logs.moreinfo or "" + if moreinfo ~= "" and option ~= "nomoreinfo" then + logs.reportline() + logs.reportlines(moreinfo) end +end - function input.stoptiming(instance, report) - if instance then - local starttime = instance.starttime - if starttime then - local stoptime = clock() - local loadtime = stoptime - starttime - instance.stoptime = stoptime - instance.loadtime = instance.loadtime + loadtime - if report then - input.report('load time', format("%0.3f",loadtime)) - end - return loadtime - end +logs.set_level('error') +logs.set_method('tex') + +function logs.system(whereto,process,jobname,category,...) + for i=1,10 do + local f = io.open(whereto,"a") + if f then + f:write(format("%s %s => %s => %s => %s\r",os.date("%d/%m/%y %H:%m:%S"),process,jobname,category,format(...))) + f:close() + break + else + sleep(0.1) end - return 0 end +end +--~ local syslogname = "oeps.xxx" +--~ +--~ for i=1,10 do +--~ logs.system(syslogname,"context","test","fonts","font %s recached due to newer version (%s)","blabla","123") +--~ end + +function logs.fatal(where,...) + logs.report(where,"fatal error: %s, aborting now",format(...)) + os.exit() end -function input.elapsedtime(instance) - return format("%0.3f",(instance and instance.loadtime) or 0) + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['data-inp'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files", +} + +-- After a few years using the code the large luat-inp.lua file +-- has been split up a bit. In the process some functionality was +-- dropped: +-- +-- * support for reading lsr files +-- * selective scanning (subtrees) +-- * some public auxiliary functions were made private +-- +-- TODO: os.getenv -> os.env[] +-- TODO: instances.[hashes,cnffiles,configurations,522] +-- TODO: check escaping in find etc, too much, too slow + +-- This lib is multi-purpose and can be loaded again later on so that +-- additional functionality becomes available. We will split thislogs.report("fileio", +-- module in components once we're done with prototyping. This is the +-- first code I wrote for LuaTeX, so it needs some cleanup. Before changing +-- something in this module one can best check with Taco or Hans first; there +-- is some nasty trickery going on that relates to traditional kpse support. + +-- To be considered: hash key lowercase, first entry in table filename +-- (any case), rest paths (so no need for optimization). Or maybe a +-- separate table that matches lowercase names to mixed case when +-- present. In that case the lower() cases can go away. I will do that +-- only when we run into problems with names ... well ... Iwona-Regular. + +-- Beware, loading and saving is overloaded in luat-tmp! + +local format, gsub, find, lower, upper, match, gmatch = string.format, string.gsub, string.find, string.lower, string.upper, string.match, string.gmatch +local concat, insert, sortedkeys = table.concat, table.insert, table.sortedkeys +local next, type = next, type +local lpegmatch = lpeg.match + +local trace_locating, trace_detail, trace_expansions = false, false, false + +trackers.register("resolvers.locating", function(v) trace_locating = v end) +trackers.register("resolvers.details", function(v) trace_detail = v end) +trackers.register("resolvers.expansions", function(v) trace_expansions = v end) -- todo + +if not resolvers then + resolvers = { + suffixes = { }, + formats = { }, + dangerous = { }, + suffixmap = { }, + alternatives = { }, + locators = { }, -- locate databases + hashers = { }, -- load databases + generators = { }, -- generate databases + } end -function input.report_loadtime(instance) - if instance then - input.report('total load time', input.elapsedtime(instance)) +local resolvers = resolvers + +resolvers.locators .notfound = { nil } +resolvers.hashers .notfound = { nil } +resolvers.generators.notfound = { nil } + +resolvers.cacheversion = '1.0.1' +resolvers.cnfname = 'texmf.cnf' +resolvers.luaname = 'texmfcnf.lua' +resolvers.homedir = os.env[os.type == "windows" and 'USERPROFILE'] or os.env['HOME'] or '~' +resolvers.cnfdefault = '{$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c}' + +local dummy_path_expr = "^!*unset/*$" + +local formats = resolvers.formats +local suffixes = resolvers.suffixes +local dangerous = resolvers.dangerous +local suffixmap = resolvers.suffixmap +local alternatives = resolvers.alternatives + +formats['afm'] = 'AFMFONTS' suffixes['afm'] = { 'afm' } +formats['enc'] = 'ENCFONTS' suffixes['enc'] = { 'enc' } +formats['fmt'] = 'TEXFORMATS' suffixes['fmt'] = { 'fmt' } +formats['map'] = 'TEXFONTMAPS' suffixes['map'] = { 'map' } +formats['mp'] = 'MPINPUTS' suffixes['mp'] = { 'mp' } +formats['ocp'] = 'OCPINPUTS' suffixes['ocp'] = { 'ocp' } +formats['ofm'] = 'OFMFONTS' suffixes['ofm'] = { 'ofm', 'tfm' } +formats['otf'] = 'OPENTYPEFONTS' suffixes['otf'] = { 'otf' } -- 'ttf' +formats['opl'] = 'OPLFONTS' suffixes['opl'] = { 'opl' } +formats['otp'] = 'OTPINPUTS' suffixes['otp'] = { 'otp' } +formats['ovf'] = 'OVFFONTS' suffixes['ovf'] = { 'ovf', 'vf' } +formats['ovp'] = 'OVPFONTS' suffixes['ovp'] = { 'ovp' } +formats['tex'] = 'TEXINPUTS' suffixes['tex'] = { 'tex' } +formats['tfm'] = 'TFMFONTS' suffixes['tfm'] = { 'tfm' } +formats['ttf'] = 'TTFONTS' suffixes['ttf'] = { 'ttf', 'ttc', 'dfont' } +formats['pfb'] = 'T1FONTS' suffixes['pfb'] = { 'pfb', 'pfa' } +formats['vf'] = 'VFFONTS' suffixes['vf'] = { 'vf' } + +formats['fea'] = 'FONTFEATURES' suffixes['fea'] = { 'fea' } +formats['cid'] = 'FONTCIDMAPS' suffixes['cid'] = { 'cid', 'cidmap' } + +formats ['texmfscripts'] = 'TEXMFSCRIPTS' -- new +suffixes['texmfscripts'] = { 'rb', 'pl', 'py' } -- 'lua' + +formats ['lua'] = 'LUAINPUTS' -- new +suffixes['lua'] = { 'lua', 'luc', 'tma', 'tmc' } + +-- backward compatible ones + +alternatives['map files'] = 'map' +alternatives['enc files'] = 'enc' +alternatives['cid maps'] = 'cid' -- great, why no cid files +alternatives['font feature files'] = 'fea' -- and fea files here +alternatives['opentype fonts'] = 'otf' +alternatives['truetype fonts'] = 'ttf' +alternatives['truetype collections'] = 'ttc' +alternatives['truetype dictionary'] = 'dfont' +alternatives['type1 fonts'] = 'pfb' + +-- obscure ones + +formats ['misc fonts'] = '' +suffixes['misc fonts'] = { } + +formats ['sfd'] = 'SFDFONTS' +suffixes ['sfd'] = { 'sfd' } +alternatives['subfont definition files'] = 'sfd' + +-- lib paths + +formats ['lib'] = 'CLUAINPUTS' -- new (needs checking) +suffixes['lib'] = (os.libsuffix and { os.libsuffix }) or { 'dll', 'so' } + +-- In practice we will work within one tds tree, but i want to keep +-- the option open to build tools that look at multiple trees, which is +-- why we keep the tree specific data in a table. We used to pass the +-- instance but for practical pusposes we now avoid this and use a +-- instance variable. + +-- here we catch a few new thingies (todo: add these paths to context.tmf) +-- +-- FONTFEATURES = .;$TEXMF/fonts/fea// +-- FONTCIDMAPS = .;$TEXMF/fonts/cid// + +-- we always have one instance active + +resolvers.instance = resolvers.instance or nil -- the current one (slow access) +local instance = resolvers.instance or nil -- the current one (fast access) + +function resolvers.newinstance() + + -- store once, freeze and faster (once reset we can best use + -- instance.environment) maybe better have a register suffix + -- function + + for k, v in next, suffixes do + for i=1,#v do + local vi = v[i] + if vi then + suffixmap[vi] = k + end + end + end + + -- because vf searching is somewhat dangerous, we want to prevent + -- too liberal searching esp because we do a lookup on the current + -- path anyway; only tex (or any) is safe + + for k, v in next, formats do + dangerous[k] = true + end + dangerous.tex = nil + + -- the instance + + local newinstance = { + rootpath = '', + treepath = '', + progname = 'context', + engine = 'luatex', + format = '', + environment = { }, + variables = { }, + expansions = { }, + files = { }, + remap = { }, + configuration = { }, + setup = { }, + order = { }, + found = { }, + foundintrees = { }, + kpsevars = { }, + hashes = { }, + cnffiles = { }, + luafiles = { }, + lists = { }, + remember = true, + diskcache = true, + renewcache = false, + scandisk = true, + cachepath = nil, + loaderror = false, + sortdata = false, + savelists = true, + cleanuppaths = true, + allresults = false, + pattern = nil, -- lists + data = { }, -- only for loading + force_suffixes = true, + fakepaths = { }, + } + + local ne = newinstance.environment + + for k,v in next, os.env do + ne[k] = resolvers.bare_variable(v) end + + return newinstance + end -input.loadtime = input.elapsedtime +function resolvers.setinstance(someinstance) + instance = someinstance + resolvers.instance = someinstance + return someinstance +end -function input.env(instance,key) - return instance.environment[key] or input.osenv(instance,key) +function resolvers.reset() + return resolvers.setinstance(resolvers.newinstance()) end -function input.osenv(instance,key) +local function reset_hashes() + instance.lists = { } + instance.found = { } +end + +local function check_configuration() -- not yet ok, no time for debugging now + local ie, iv = instance.environment, instance.variables + local function fix(varname,default) + local proname = varname .. "." .. instance.progname or "crap" + local p, v = ie[proname], ie[varname] or iv[varname] + if not ((p and p ~= "") or (v and v ~= "")) then + iv[varname] = default -- or environment? + end + end + local name = os.name + if name == "windows" then + fix("OSFONTDIR", "c:/windows/fonts//") + elseif name == "macosx" then + fix("OSFONTDIR", "$HOME/Library/Fonts//;/Library/Fonts//;/System/Library/Fonts//") + else + -- bad luck + end + fix("LUAINPUTS" , ".;$TEXINPUTS;$TEXMFSCRIPTS") -- no progname, hm + -- this will go away some day + fix("FONTFEATURES", ".;$TEXMF/fonts/{data,fea}//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") + fix("FONTCIDMAPS" , ".;$TEXMF/fonts/{data,cid}//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") + -- + fix("LUATEXLIBS" , ".;$TEXMF/luatex/lua//") +end + +function resolvers.bare_variable(str) -- assumes str is a string + return (gsub(str,"\s*([\"\']?)(.+)%1\s*", "%2")) +end + +function resolvers.settrace(n) -- no longer number but: 'locating' or 'detail' + if n then + trackers.disable("resolvers.*") + trackers.enable("resolvers."..n) + end +end + +resolvers.settrace(os.getenv("MTX_INPUT_TRACE")) + +function resolvers.osenv(key) local ie = instance.environment local value = ie[key] if value == nil then @@ -5100,174 +8386,318 @@ function input.osenv(instance,key) if e == nil then -- value = "" -- false else - value = input.bare_variable(e) + value = resolvers.bare_variable(e) end ie[key] = value end return value or "" end --- we follow a rather traditional approach: --- --- (1) texmf.cnf given in TEXMFCNF --- (2) texmf.cnf searched in TEXMF/web2c +function resolvers.env(key) + return instance.environment[key] or resolvers.osenv(key) +end + -- --- for the moment we don't expect a configuration file in a zip -function input.identify_cnf(instance) - -- we no longer support treepath and rootpath (was handy for testing); - -- also we now follow the stupid route: if not set then just assume *one* - -- cnf file under texmf (i.e. distribution) - if #instance.cnffiles == 0 then - if input.env(instance,'TEXMFCNF') == "" then - local ownpath = environment.ownpath() or "." - if ownpath then - -- beware, this is tricky on my own system because at that location I do have - -- the raw tree that ends up in the zip; i.e. I cannot test this kind of mess - local function locate(filename,list) - local ownroot = input.normalize_name(file.join(ownpath,"../..")) - if not lfs.isdir(file.join(ownroot,"texmf")) then - ownroot = input.normalize_name(file.join(ownpath,"..")) - if not lfs.isdir(file.join(ownroot,"texmf")) then - input.verbose = true - input.report("error", "unable to identify cnf file") - return - end - end - local texmfcnf = file.join(ownroot,"texmf-local/web2c",filename) -- for minimals and myself - if not lfs.isfile(texmfcnf) then - texmfcnf = file.join(ownroot,"texmf/web2c",filename) - if not lfs.isfile(texmfcnf) then - input.verbose = true - input.report("error", "unable to locate",filename) - return - end - end - table.insert(list,texmfcnf) - local ie = instance.environment - if not ie['SELFAUTOPARENT'] then ie['SELFAUTOPARENT'] = ownroot end - if not ie['TEXMFCNF'] then ie['TEXMFCNF'] = file.dirname(texmfcnf) end - end - locate(input.luaname,instance.luafiles) - locate(input.cnfname,instance.cnffiles) - if #instance.luafiles == 0 and instance.cnffiles == 0 then - input.verbose = true - input.report("error", "unable to locate",filename) - os.exit() - end - -- here we also assume then TEXMF is set in the distribution, if this trickery is - -- used in the minimals, then users who don't use setuptex are on their own with - -- regards to extra trees - else - input.verbose = true - input.report("error", "unable to identify own path") - os.exit() - end +local function expand_vars(lst) -- simple vars + local variables, env = instance.variables, resolvers.env + local function resolve(a) + return variables[a] or env(a) + end + for k=1,#lst do + lst[k] = gsub(lst[k],"%$([%a%d%_%-]+)",resolve) + end +end + +local function expanded_var(var) -- simple vars + local function resolve(a) + return instance.variables[a] or resolvers.env(a) + end + return (gsub(var,"%$([%a%d%_%-]+)",resolve)) +end + +local function entry(entries,name) + if name and (name ~= "") then + name = gsub(name,'%$','') + local result = entries[name..'.'..instance.progname] or entries[name] + if result then + return result else - local t = input.split_path(input.env(instance,'TEXMFCNF')) - t = input.aux.expanded_path(instance,t) - input.aux.expand_vars(instance,t) - local function locate(filename,list) - for _,v in ipairs(t) do - local texmfcnf = input.normalize_name(file.join(v,filename)) - if lfs.isfile(texmfcnf) then - table.insert(list,texmfcnf) - end - end + result = resolvers.env(name) + if result then + instance.variables[name] = result + resolvers.expand_variables() + return instance.expansions[name] or "" end - locate(input.luaname,instance.luafiles) - locate(input.cnfname,instance.cnffiles) end end + return "" end -function input.load_cnf(instance) - local function loadoldconfigdata() - for _, fname in ipairs(instance.cnffiles) do - input.aux.load_cnf(instance,fname) +local function is_entry(entries,name) + if name and name ~= "" then + name = gsub(name,'%$','') + return (entries[name..'.'..instance.progname] or entries[name]) ~= nil + else + return false + end +end + +-- {a,b,c,d} +-- a,b,c/{p,q,r},d +-- a,b,c/{p,q,r}/d/{x,y,z}// +-- a,b,c/{p,q/{x,y,z},r},d/{p,q,r} +-- a,b,c/{p,q/{x,y,z},r},d/{p,q,r} +-- a{b,c}{d,e}f +-- {a,b,c,d} +-- {a,b,c/{p,q,r},d} +-- {a,b,c/{p,q,r}/d/{x,y,z}//} +-- {a,b,c/{p,q/{x,y,z}},d/{p,q,r}} +-- {a,b,c/{p,q/{x,y,z},w}v,d/{p,q,r}} +-- {$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c} + +-- this one is better and faster, but it took me a while to realize +-- that this kind of replacement is cleaner than messy parsing and +-- fuzzy concatenating we can probably gain a bit with selectively +-- applying lpeg, but experiments with lpeg parsing this proved not to +-- work that well; the parsing is ok, but dealing with the resulting +-- table is a pain because we need to work inside-out recursively + +local function do_first(a,b) + local t = { } + for s in gmatch(b,"[^,]+") do t[#t+1] = a .. s end + return "{" .. concat(t,",") .. "}" +end + +local function do_second(a,b) + local t = { } + for s in gmatch(a,"[^,]+") do t[#t+1] = s .. b end + return "{" .. concat(t,",") .. "}" +end + +local function do_both(a,b) + local t = { } + for sa in gmatch(a,"[^,]+") do + for sb in gmatch(b,"[^,]+") do + t[#t+1] = sa .. sb end end - -- instance.cnffiles contain complete names now ! - if #instance.cnffiles == 0 then - input.report("no cnf files found (TEXMFCNF may not be set/known)") + return "{" .. concat(t,",") .. "}" +end + +local function do_three(a,b,c) + return a .. b.. c +end + +local function splitpathexpr(str, t, validate) + -- no need for further optimization as it is only called a + -- few times, we can use lpeg for the sub + if trace_expansions then + logs.report("fileio","expanding variable '%s'",str) + end + t = t or { } + str = gsub(str,",}",",@}") + str = gsub(str,"{,","{@,") + -- str = "@" .. str .. "@" + local ok, done + while true do + done = false + while true do + str, ok = gsub(str,"([^{},]+){([^{}]+)}",do_first) + if ok > 0 then done = true else break end + end + while true do + str, ok = gsub(str,"{([^{}]+)}([^{},]+)",do_second) + if ok > 0 then done = true else break end + end + while true do + str, ok = gsub(str,"{([^{}]+)}{([^{}]+)}",do_both) + if ok > 0 then done = true else break end + end + str, ok = gsub(str,"({[^{}]*){([^{}]+)}([^{}]*})",do_three) + if ok > 0 then done = true end + if not done then break end + end + str = gsub(str,"[{}]", "") + str = gsub(str,"@","") + if validate then + for s in gmatch(str,"[^,]+") do + s = validate(s) + if s then t[#t+1] = s end + end else - instance.rootpath = instance.cnffiles[1] - for k,fname in ipairs(instance.cnffiles) do - instance.cnffiles[k] = input.normalize_name(fname:gsub("\\",'/')) + for s in gmatch(str,"[^,]+") do + t[#t+1] = s end - for i=1,3 do - instance.rootpath = file.dirname(instance.rootpath) + end + if trace_expansions then + for k=1,#t do + logs.report("fileio","% 4i: %s",k,t[k]) end - instance.rootpath = input.normalize_name(instance.rootpath) - instance.environment['SELFAUTOPARENT'] = instance.rootpath -- just to be sure - if instance.lsrmode then - loadoldconfigdata() - elseif instance.diskcache and not instance.renewcache then - input.loadoldconfig(instance,instance.cnffiles) - if instance.loaderror then - loadoldconfigdata() - input.saveoldconfig(instance) + end + return t +end + +local function expanded_path_from_list(pathlist) -- maybe not a list, just a path + -- a previous version fed back into pathlist + local newlist, ok = { }, false + for k=1,#pathlist do + if find(pathlist[k],"[{}]") then + ok = true + break + end + end + if ok then + local function validate(s) + s = file.collapse_path(s) + return s ~= "" and not find(s,dummy_path_expr) and s + end + for k=1,#pathlist do + splitpathexpr(pathlist[k],newlist,validate) + end + else + for k=1,#pathlist do + for p in gmatch(pathlist[k],"([^,]+)") do + p = file.collapse_path(p) + if p ~= "" then newlist[#newlist+1] = p end end - else - loadoldconfigdata() - if instance.renewcache then - input.saveoldconfig(instance) + end + end + return newlist +end + +-- we follow a rather traditional approach: +-- +-- (1) texmf.cnf given in TEXMFCNF +-- (2) texmf.cnf searched in default variable +-- +-- also we now follow the stupid route: if not set then just assume *one* +-- cnf file under texmf (i.e. distribution) + +local args = environment and environment.original_arguments or arg -- this needs a cleanup + +resolvers.ownbin = resolvers.ownbin or args[-2] or arg[-2] or args[-1] or arg[-1] or arg[0] or "luatex" +resolvers.ownbin = gsub(resolvers.ownbin,"\\","/") + +function resolvers.getownpath() + local ownpath = resolvers.ownpath or os.selfdir + if not ownpath or ownpath == "" or ownpath == "unset" then + ownpath = args[-1] or arg[-1] + ownpath = ownpath and file.dirname(gsub(ownpath,"\\","/")) + if not ownpath or ownpath == "" then + ownpath = args[-0] or arg[-0] + ownpath = ownpath and file.dirname(gsub(ownpath,"\\","/")) + end + local binary = resolvers.ownbin + if not ownpath or ownpath == "" then + ownpath = ownpath and file.dirname(binary) + end + if not ownpath or ownpath == "" then + if os.binsuffix ~= "" then + binary = file.replacesuffix(binary,os.binsuffix) + end + for p in gmatch(os.getenv("PATH"),"[^"..io.pathseparator.."]+") do + local b = file.join(p,binary) + if lfs.isfile(b) then + -- we assume that after changing to the path the currentdir function + -- resolves to the real location and use this side effect here; this + -- trick is needed because on the mac installations use symlinks in the + -- path instead of real locations + local olddir = lfs.currentdir() + if lfs.chdir(p) then + local pp = lfs.currentdir() + if trace_locating and p ~= pp then + logs.report("fileio","following symlink '%s' to '%s'",p,pp) + end + ownpath = pp + lfs.chdir(olddir) + else + if trace_locating then + logs.report("fileio","unable to check path '%s'",p) + end + ownpath = p + end + break + end end end - input.aux.collapse_cnf_data(instance) + if not ownpath or ownpath == "" then + ownpath = "." + logs.report("fileio","forcing fallback ownpath .") + elseif trace_locating then + logs.report("fileio","using ownpath '%s'",ownpath) + end end - input.checkconfigdata(instance) + resolvers.ownpath = ownpath + function resolvers.getownpath() + return resolvers.ownpath + end + return ownpath end -function input.load_lua(instance) - if #instance.luafiles == 0 then - -- yet harmless +local own_places = { "SELFAUTOLOC", "SELFAUTODIR", "SELFAUTOPARENT", "TEXMFCNF" } + +local function identify_own() + local ownpath = resolvers.getownpath() or dir.current() + local ie = instance.environment + if ownpath then + if resolvers.env('SELFAUTOLOC') == "" then os.env['SELFAUTOLOC'] = file.collapse_path(ownpath) end + if resolvers.env('SELFAUTODIR') == "" then os.env['SELFAUTODIR'] = file.collapse_path(ownpath .. "/..") end + if resolvers.env('SELFAUTOPARENT') == "" then os.env['SELFAUTOPARENT'] = file.collapse_path(ownpath .. "/../..") end else - instance.rootpath = instance.luafiles[1] - for k,fname in ipairs(instance.luafiles) do - instance.luafiles[k] = input.normalize_name(fname:gsub("\\",'/')) - end - for i=1,3 do - instance.rootpath = file.dirname(instance.rootpath) + logs.report("fileio","error: unable to locate ownpath") + os.exit() + end + if resolvers.env('TEXMFCNF') == "" then os.env['TEXMFCNF'] = resolvers.cnfdefault end + if resolvers.env('TEXOS') == "" then os.env['TEXOS'] = resolvers.env('SELFAUTODIR') end + if resolvers.env('TEXROOT') == "" then os.env['TEXROOT'] = resolvers.env('SELFAUTOPARENT') end + if trace_locating then + for i=1,#own_places do + local v = own_places[i] + logs.report("fileio","variable '%s' set to '%s'",v,resolvers.env(v) or "unknown") end - instance.rootpath = input.normalize_name(instance.rootpath) - instance.environment['SELFAUTOPARENT'] = instance.rootpath -- just to be sure - input.loadnewconfig(instance) - input.aux.collapse_cnf_data(instance) end - input.checkconfigdata(instance) + identify_own = function() end end -function input.aux.collapse_cnf_data(instance) -- potential optmization: pass start index (setup and configuration are shared) - for _,c in ipairs(instance.order) do - for k,v in pairs(c) do - if not instance.variables[k] then - if instance.environment[k] then - instance.variables[k] = instance.environment[k] - else - instance.kpsevars[k] = true - instance.variables[k] = input.bare_variable(v) +function resolvers.identify_cnf() + if #instance.cnffiles == 0 then + -- fallback + identify_own() + -- the real search + resolvers.expand_variables() + local t = resolvers.split_path(resolvers.env('TEXMFCNF')) + t = expanded_path_from_list(t) + expand_vars(t) -- redundant + local function locate(filename,list) + for i=1,#t do + local ti = t[i] + local texmfcnf = file.collapse_path(file.join(ti,filename)) + if lfs.isfile(texmfcnf) then + list[#list+1] = texmfcnf end end end + locate(resolvers.luaname,instance.luafiles) + locate(resolvers.cnfname,instance.cnffiles) end end -function input.aux.load_cnf(instance,fname) - fname = input.clean_path(fname) - local lname = fname:gsub("%.%a+$",input.luasuffix) - local f = io.open(lname) - if f then -- this will go - f:close() - local dname = file.dirname(fname) +local function load_cnf_file(fname) + fname = resolvers.clean_path(fname) + local lname = file.replacesuffix(fname,'lua') + if lfs.isfile(lname) then + local dname = file.dirname(fname) -- fname ? if not instance.configuration[dname] then - input.aux.load_configuration(instance,dname,lname) + resolvers.load_data(dname,'configuration',lname and file.basename(lname)) instance.order[#instance.order+1] = instance.configuration[dname] end else f = io.open(fname) if f then - input.report("loading", fname) + if trace_locating then + logs.report("fileio","loading configuration file %s", fname) + end local line, data, n, k, v local dname = file.dirname(fname) if not instance.configuration[dname] then @@ -5279,17 +8709,19 @@ function input.aux.load_cnf(instance,fname) local line, n = f:read(), 0 if line then while true do -- join lines - line, n = line:gsub("\\%s*$", "") + line, n = gsub(line,"\\%s*$", "") if n > 0 then line = line .. f:read() else break end end - if not line:find("^[%%#]") then - local k, v = (line:gsub("%s*%%.*$","")):match("%s*(.-)%s*=%s*(.-)%s*$") + if not find(line,"^[%%#]") then + local l = gsub(line,"%s*%%.*$","") + local k, v = match(l,"%s*(.-)%s*=%s*(.-)%s*$") if k and v and not data[k] then - data[k] = (v:gsub("[%%#].*",'')):gsub("~", "$HOME") + v = gsub(v,"[%%#].*",'') + data[k] = gsub(v,"~","$HOME") instance.kpsevars[k] = true end end @@ -5298,240 +8730,333 @@ function input.aux.load_cnf(instance,fname) end end f:close() + elseif trace_locating then + logs.report("fileio","skipping configuration file '%s'", fname) + end + end +end + +local function collapse_cnf_data() -- potential optimization: pass start index (setup and configuration are shared) + local order = instance.order + for i=1,#order do + local c = order[i] + for k,v in next, c do + if not instance.variables[k] then + if instance.environment[k] then + instance.variables[k] = instance.environment[k] + else + instance.kpsevars[k] = true + instance.variables[k] = resolvers.bare_variable(v) + end + end + end + end +end + +function resolvers.load_cnf() + local function loadoldconfigdata() + local cnffiles = instance.cnffiles + for i=1,#cnffiles do + load_cnf_file(cnffiles[i]) + end + end + -- instance.cnffiles contain complete names now ! + -- we still use a funny mix of cnf and new but soon + -- we will switch to lua exclusively as we only use + -- the file to collect the tree roots + if #instance.cnffiles == 0 then + if trace_locating then + logs.report("fileio","no cnf files found (TEXMFCNF may not be set/known)") + end + else + local cnffiles = instance.cnffiles + instance.rootpath = cnffiles[1] + for k=1,#cnffiles do + instance.cnffiles[k] = file.collapse_path(cnffiles[k]) + end + for i=1,3 do + instance.rootpath = file.dirname(instance.rootpath) + end + instance.rootpath = file.collapse_path(instance.rootpath) + if instance.diskcache and not instance.renewcache then + resolvers.loadoldconfig(instance.cnffiles) + if instance.loaderror then + loadoldconfigdata() + resolvers.saveoldconfig() + end else - input.report("skipping", fname) + loadoldconfigdata() + if instance.renewcache then + resolvers.saveoldconfig() + end end + collapse_cnf_data() end + check_configuration() +end + +function resolvers.load_lua() + if #instance.luafiles == 0 then + -- yet harmless + else + instance.rootpath = instance.luafiles[1] + local luafiles = instance.luafiles + for k=1,#luafiles do + instance.luafiles[k] = file.collapse_path(luafiles[k]) + end + for i=1,3 do + instance.rootpath = file.dirname(instance.rootpath) + end + instance.rootpath = file.collapse_path(instance.rootpath) + resolvers.loadnewconfig() + collapse_cnf_data() + end + check_configuration() end -- database loading -function input.load_hash(instance) - input.locatelists(instance) - if instance.lsrmode then - input.loadlists(instance) - elseif instance.diskcache and not instance.renewcache then - input.loadfiles(instance) +function resolvers.load_hash() + resolvers.locatelists() + if instance.diskcache and not instance.renewcache then + resolvers.loadfiles() if instance.loaderror then - input.loadlists(instance) - input.savefiles(instance) + resolvers.loadlists() + resolvers.savefiles() end else - input.loadlists(instance) + resolvers.loadlists() if instance.renewcache then - input.savefiles(instance) + resolvers.savefiles() end end end -function input.aux.append_hash(instance,type,tag,name) - input.logger("= hash append",tag) - table.insert(instance.hashes, { ['type']=type, ['tag']=tag, ['name']=name } ) +function resolvers.append_hash(type,tag,name) + if trace_locating then + logs.report("fileio","hash '%s' appended",tag) + end + insert(instance.hashes, { ['type']=type, ['tag']=tag, ['name']=name } ) end -function input.aux.prepend_hash(instance,type,tag,name) - input.logger("= hash prepend",tag) - table.insert(instance.hashes, 1, { ['type']=type, ['tag']=tag, ['name']=name } ) +function resolvers.prepend_hash(type,tag,name) + if trace_locating then + logs.report("fileio","hash '%s' prepended",tag) + end + insert(instance.hashes, 1, { ['type']=type, ['tag']=tag, ['name']=name } ) end -function input.aux.extend_texmf_var(instance,specification) -- crap - if instance.environment['TEXMF'] then - input.report("extending environment variable TEXMF with", specification) - instance.environment['TEXMF'] = instance.environment['TEXMF']:gsub("^%{", function() - return "{" .. specification .. "," - end) - elseif instance.variables['TEXMF'] then - input.report("extending configuration variable TEXMF with", specification) - instance.variables['TEXMF'] = instance.variables['TEXMF']:gsub("^%{", function() - return "{" .. specification .. "," - end) +function resolvers.extend_texmf_var(specification) -- crap, we could better prepend the hash +-- local t = resolvers.expanded_path_list('TEXMF') -- full expansion + local t = resolvers.split_path(resolvers.env('TEXMF')) + insert(t,1,specification) + local newspec = concat(t,";") + if instance.environment["TEXMF"] then + instance.environment["TEXMF"] = newspec + elseif instance.variables["TEXMF"] then + instance.variables["TEXMF"] = newspec else - input.report("setting configuration variable TEXMF to", specification) - instance.variables['TEXMF'] = "{" .. specification .. "}" - end - if instance.variables['TEXMF']:find("%,") and not instance.variables['TEXMF']:find("^%{") then - input.report("adding {} to complex TEXMF variable, best do that yourself") - instance.variables['TEXMF'] = "{" .. instance.variables['TEXMF'] .. "}" + -- weird end - input.expand_variables(instance) - input.reset_hashes(instance) + resolvers.expand_variables() + reset_hashes() end -- locators -function input.locatelists(instance) - for _, path in pairs(input.simplified_list(input.expansion(instance,'TEXMF'))) do - path = file.collapse_path(path) - input.report("locating list of",path) - input.locatedatabase(instance,input.normalize_name(path)) +function resolvers.locatelists() + local texmfpaths = resolvers.clean_path_list('TEXMF') + for i=1,#texmfpaths do + local path = texmfpaths[i] + if trace_locating then + logs.report("fileio","locating list of '%s'",path) + end + resolvers.locatedatabase(file.collapse_path(path)) end end -function input.locatedatabase(instance,specification) - return input.methodhandler('locators', instance, specification) +function resolvers.locatedatabase(specification) + return resolvers.methodhandler('locators', specification) end -function input.locators.tex(instance,specification) +function resolvers.locators.tex(specification) if specification and specification ~= '' and lfs.isdir(specification) then - input.logger('! tex locator', specification..' found') - input.aux.append_hash(instance,'file',specification,filename) - else - input.logger('? tex locator', specification..' not found') + if trace_locating then + logs.report("fileio","tex locator '%s' found",specification) + end + resolvers.append_hash('file',specification,filename) + elseif trace_locating then + logs.report("fileio","tex locator '%s' not found",specification) end end -- hashers -function input.hashdatabase(instance,tag,name) - return input.methodhandler('hashers',instance,tag,name) +function resolvers.hashdatabase(tag,name) + return resolvers.methodhandler('hashers',tag,name) end -function input.loadfiles(instance) +function resolvers.loadfiles() instance.loaderror = false instance.files = { } if not instance.renewcache then - for _, hash in ipairs(instance.hashes) do - input.hashdatabase(instance,hash.tag,hash.name) + local hashes = instance.hashes + for k=1,#hashes do + local hash = hashes[k] + resolvers.hashdatabase(hash.tag,hash.name) if instance.loaderror then break end end end end -function input.hashers.tex(instance,tag,name) - input.aux.load_files(instance,tag) +function resolvers.hashers.tex(tag,name) + resolvers.load_data(tag,'files') end -- generators: -function input.loadlists(instance) - for _, hash in ipairs(instance.hashes) do - input.generatedatabase(instance,hash.tag) +function resolvers.loadlists() + local hashes = instance.hashes + for i=1,#hashes do + resolvers.generatedatabase(hashes[i].tag) end end -function input.generatedatabase(instance,specification) - return input.methodhandler('generators', instance, specification) +function resolvers.generatedatabase(specification) + return resolvers.methodhandler('generators', specification) end -do +-- starting with . or .. etc or funny char - local weird = lpeg.anywhere(lpeg.S("~`!#$%^&*()={}[]:;\"\'||<>,?\n\r\t")) +local weird = lpeg.P(".")^1 + lpeg.anywhere(lpeg.S("~`!#$%^&*()={}[]:;\"\'||<>,?\n\r\t")) - function input.generators.tex(instance,specification) - local tag = specification - if not instance.lsrmode and lfs and lfs.dir then - input.report("scanning path",specification) - instance.files[tag] = { } - local files = instance.files[tag] - local n, m, r = 0, 0, 0 - local spec = specification .. '/' - local attributes = lfs.attributes - local directory = lfs.dir - local small = instance.smallcache - local function action(path) - local mode, full - if path then - full = spec .. path .. '/' - else - full = spec - end - for name in directory(full) do - if name:find("^%.") then - -- skip - -- elseif name:find("[%~%`%!%#%$%%%^%&%*%(%)%=%{%}%[%]%:%;\"\'%|%<%>%,%?\n\r\t]") then -- too much escaped - elseif weird:match(name) then - -- texio.write_nl("skipping " .. name) - -- skip - else - mode = attributes(full..name,'mode') - if mode == "directory" then - m = m + 1 - if path then - action(path..'/'..name) +--~ local l_forbidden = lpeg.S("~`!#$%^&*()={}[]:;\"\'||\\/<>,?\n\r\t") +--~ local l_confusing = lpeg.P(" ") +--~ local l_character = lpeg.patterns.utf8 +--~ local l_dangerous = lpeg.P(".") + +--~ local l_normal = (l_character - l_forbidden - l_confusing - l_dangerous) * (l_character - l_forbidden - l_confusing^2)^0 * lpeg.P(-1) +--~ ----- l_normal = l_normal * lpeg.Cc(true) + lpeg.Cc(false) + +--~ local function test(str) +--~ print(str,lpeg.match(l_normal,str)) +--~ end +--~ test("ヒラギノ明朝 Pro W3") +--~ test("..ヒラギノ明朝 Pro W3") +--~ test(":ヒラギノ明朝 Pro W3;") +--~ test("ヒラギノ明朝 /Pro W3;") +--~ test("ヒラギノ明朝 Pro W3") + +function resolvers.generators.tex(specification) + local tag = specification + if trace_locating then + logs.report("fileio","scanning path '%s'",specification) + end + instance.files[tag] = { } + local files = instance.files[tag] + local n, m, r = 0, 0, 0 + local spec = specification .. '/' + local attributes = lfs.attributes + local directory = lfs.dir + local function action(path) + local full + if path then + full = spec .. path .. '/' + else + full = spec + end + for name in directory(full) do + if not lpegmatch(weird,name) then + -- if lpegmatch(l_normal,name) then + local mode = attributes(full..name,'mode') + if mode == 'file' then + if path then + n = n + 1 + local f = files[name] + if f then + if type(f) == 'string' then + files[name] = { f, path } else - action(name) + f[#f+1] = path end - elseif path and mode == 'file' then - n = n + 1 - local f = files[name] - if f then - if not small then - if type(f) == 'string' then - files[name] = { f, path } - else - f[#f+1] = path - end - end - else - files[name] = path - local lower = name:lower() - if name ~= lower then - files["remap:"..lower] = name - r = r + 1 - end + else -- probably unique anyway + files[name] = path + local lower = lower(name) + if name ~= lower then + files["remap:"..lower] = name + r = r + 1 end end end - end - end - action() - input.report(format("%s files found on %s directories with %s uppercase remappings",n,m,r)) - else - local fullname = file.join(specification,input.lsrname) - local path = '.' - local f = io.open(fullname) - if f then - instance.files[tag] = { } - local files = instance.files[tag] - local small = instance.smallcache - input.report("loading lsr file",fullname) - -- for line in f:lines() do -- much slower then the next one - for line in (f:read("*a")):gmatch("(.-)\n") do - if line:find("^[%a%d]") then - local fl = files[line] - if fl then - if not small then - if type(fl) == 'string' then - files[line] = { fl, path } -- table - else - fl[#fl+1] = path - end - end - else - files[line] = path -- string - local lower = line:lower() - if line ~= lower then - files["remap:"..lower] = line - end - end + elseif mode == 'directory' then + m = m + 1 + if path then + action(path..'/'..name) else - path = line:match("%.%/(.-)%:$") or path -- match could be nil due to empty line + action(name) end end - f:close() end end end - + action() + if trace_locating then + logs.report("fileio","%s files found on %s directories with %s uppercase remappings",n,m,r) + end end -- savers, todo -function input.savefiles(instance) - input.aux.save_data(instance, 'files', function(k,v) - return instance.validfile(k,v) -- path, name - end) +function resolvers.savefiles() + resolvers.save_data('files') end -- A config (optionally) has the paths split in tables. Internally -- we join them and split them after the expansion has taken place. This -- is more convenient. -function input.splitconfig(instance) - for i,c in ipairs(instance) do - for k,v in pairs(c) do +--~ local checkedsplit = string.checkedsplit + +local cache = { } + +local splitter = lpeg.Ct(lpeg.splitat(lpeg.S(os.type == "windows" and ";" or ":;"))) + +local function split_kpse_path(str) -- beware, this can be either a path or a {specification} + local found = cache[str] + if not found then + if str == "" then + found = { } + else + str = gsub(str,"\\","/") +--~ local split = (find(str,";") and checkedsplit(str,";")) or checkedsplit(str,io.pathseparator) +local split = lpegmatch(splitter,str) + found = { } + for i=1,#split do + local s = split[i] + if not find(s,"^{*unset}*") then + found[#found+1] = s + end + end + if trace_expansions then + logs.report("fileio","splitting path specification '%s'",str) + for k=1,#found do + logs.report("fileio","% 4i: %s",k,found[k]) + end + end + cache[str] = found + end + end + return found +end + +resolvers.split_kpse_path = split_kpse_path + +function resolvers.splitconfig() + for i=1,#instance do + local c = instance[i] + for k,v in next, c do if type(v) == 'string' then - local t = file.split_path(v) + local t = split_kpse_path(v) if #t > 1 then c[k] = t end @@ -5539,23 +9064,28 @@ function input.splitconfig(instance) end end end -function input.joinconfig(instance) - for i,c in ipairs(instance.order) do - for k,v in pairs(c) do + +function resolvers.joinconfig() + local order = instance.order + for i=1,#order do + local c = order[i] + for k,v in next, c do -- indexed? if type(v) == 'table' then c[k] = file.join_path(v) end end end end -function input.split_path(str) + +function resolvers.split_path(str) if type(str) == 'table' then return str else - return file.split_path(str) + return split_kpse_path(str) end end -function input.join_path(str) + +function resolvers.join_path(str) if type(str) == 'table' then return file.join_path(str) else @@ -5563,48 +9093,48 @@ function input.join_path(str) end end -function input.splitexpansions(instance) - for k,v in pairs(instance.expansions) do - local t, h = { }, { } - for _,vv in pairs(file.split_path(v)) do +function resolvers.splitexpansions() + local ie = instance.expansions + for k,v in next, ie do + local t, h, p = { }, { }, split_kpse_path(v) + for kk=1,#p do + local vv = p[kk] if vv ~= "" and not h[vv] then t[#t+1] = vv h[vv] = true end end if #t > 1 then - instance.expansions[k] = t + ie[k] = t else - instance.expansions[k] = t[1] + ie[k] = t[1] end end end -- end of split/join code -function input.saveoldconfig(instance) - input.splitconfig(instance) - input.aux.save_data(instance, 'configuration', nil) - input.joinconfig(instance) +function resolvers.saveoldconfig() + resolvers.splitconfig() + resolvers.save_data('configuration') + resolvers.joinconfig() end -input.configbanner = [[ +resolvers.configbanner = [[ -- This is a Luatex configuration file created by 'luatools.lua' or -- 'luatex.exe' directly. For comment, suggestions and questions you can -- contact the ConTeXt Development Team. This configuration file is -- not copyrighted. [HH & TH] ]] -function input.serialize(files) +function resolvers.serialize(files) -- This version is somewhat optimized for the kind of -- tables that we deal with, so it's much faster than -- the generic serializer. This makes sense because -- luatools and mtxtools are called frequently. Okay, -- we pay a small price for properly tabbed tables. local t = { } - local concat = table.concat - local sorted = table.sortedkeys - local function dump(k,v,m) + local function dump(k,v,m) -- could be moved inline if type(v) == 'string' then return m .. "['" .. k .. "']='" .. v .. "'," elseif #v == 1 then @@ -5615,11 +9145,15 @@ function input.serialize(files) end t[#t+1] = "return {" if instance.sortdata then - for _, k in pairs(sorted(files)) do + local sortedfiles = sortedkeys(files) + for i=1,#sortedfiles do + local k = sortedfiles[i] local fk = files[k] if type(fk) == 'table' then t[#t+1] = "\t['" .. k .. "']={" - for _, kk in pairs(sorted(fk)) do + local sortedfk = sortedkeys(fk) + for j=1,#sortedfk do + local kk = sortedfk[j] t[#t+1] = dump(kk,fk[kk],"\t\t") end t[#t+1] = "\t}," @@ -5628,10 +9162,10 @@ function input.serialize(files) end end else - for k, v in pairs(files) do + for k, v in next, files do if type(v) == 'table' then t[#t+1] = "\t['" .. k .. "']={" - for kk,vv in pairs(v) do + for kk,vv in next, v do t[#t+1] = dump(kk,vv,"\t\t") end t[#t+1] = "\t}," @@ -5644,62 +9178,75 @@ function input.serialize(files) return concat(t,"\n") end -if not texmf then texmf = {} end -- no longer needed, at least not here +local data_state = { } + +function resolvers.data_state() + return data_state or { } +end -function input.aux.save_data(instance, dataname, check, makename) -- untested without cache overload - for cachename, files in pairs(instance[dataname]) do +function resolvers.save_data(dataname, makename) -- untested without cache overload + for cachename, files in next, instance[dataname] do local name = (makename or file.join)(cachename,dataname) - local luaname, lucname = name .. input.luasuffix, name .. input.lucsuffix - input.report("preparing " .. dataname .. " for", luaname) - for k, v in pairs(files) do - if not check or check(v,k) then -- path, name - if type(v) == "table" and #v == 1 then - files[k] = v[1] - end - else - files[k] = nil -- false + local luaname, lucname = name .. ".lua", name .. ".luc" + if trace_locating then + logs.report("fileio","preparing '%s' for '%s'",dataname,cachename) + end + for k, v in next, files do + if type(v) == "table" and #v == 1 then + files[k] = v[1] end end local data = { type = dataname, root = cachename, - version = input.cacheversion, + version = resolvers.cacheversion, date = os.date("%Y-%m-%d"), time = os.date("%H:%M:%S"), content = files, + uuid = os.uuid(), } - local f = io.open(luaname,'w') - if f then - input.report("saving " .. dataname .. " in", luaname) - f:write(input.serialize(data)) - f:close() - input.report("compiling " .. dataname .. " to", lucname) - if not utils.lua.compile(luaname,lucname) then - input.report("compiling failed for " .. dataname .. ", deleting file " .. lucname) + local ok = io.savedata(luaname,resolvers.serialize(data)) + if ok then + if trace_locating then + logs.report("fileio","'%s' saved in '%s'",dataname,luaname) + end + if utils.lua.compile(luaname,lucname,false,true) then -- no cleanup but strip + if trace_locating then + logs.report("fileio","'%s' compiled to '%s'",dataname,lucname) + end + else + if trace_locating then + logs.report("fileio","compiling failed for '%s', deleting file '%s'",dataname,lucname) + end os.remove(lucname) end - else - input.report("unable to save " .. dataname .. " in " .. name..input.luasuffix) + elseif trace_locating then + logs.report("fileio","unable to save '%s' in '%s' (access error)",dataname,luaname) end end end -function input.aux.load_data(instance,pathname,dataname,filename,makename) -- untested without cache overload +function resolvers.load_data(pathname,dataname,filename,makename) -- untested without cache overload filename = ((not filename or (filename == "")) and dataname) or filename filename = (makename and makename(dataname,filename)) or file.join(pathname,filename) - local blob = loadfile(filename .. input.lucsuffix) or loadfile(filename .. input.luasuffix) + local blob = loadfile(filename .. ".luc") or loadfile(filename .. ".lua") if blob then local data = blob() - if data and data.content and data.type == dataname and data.version == input.cacheversion then - input.report("loading",dataname,"for",pathname,"from",filename) + if data and data.content and data.type == dataname and data.version == resolvers.cacheversion then + data_state[#data_state+1] = data.uuid + if trace_locating then + logs.report("fileio","loading '%s' for '%s' from '%s'",dataname,pathname,filename) + end instance[dataname][pathname] = data.content else - input.report("skipping",dataname,"for",pathname,"from",filename) + if trace_locating then + logs.report("fileio","skipping '%s' for '%s' from '%s'",dataname,pathname,filename) + end instance[dataname][pathname] = { } instance.loaderror = true end - else - input.report("skipping",dataname,"for",pathname,"from",filename) + elseif trace_locating then + logs.report("fileio","skipping '%s' for '%s' from '%s'",dataname,pathname,filename) end end @@ -5712,195 +9259,143 @@ end -- TEXMFBOGUS = 'effe checken of dit werkt', -- } -function input.aux.load_texmfcnf(instance,dataname,pathname) - local filename = file.join(pathname,input.luaname) - local blob = loadfile(filename) - if blob then - local data = blob() - if data then - input.report("loading","configuration file",filename) - if true then - -- flatten to variable.progname - local t = { } - for k, v in pairs(data) do -- v = progname - if type(v) == "string" then - t[k] = v - else - for kk, vv in pairs(v) do -- vv = variable - if type(vv) == "string" then - t[vv.."."..v] = kk +function resolvers.resetconfig() + identify_own() + instance.configuration, instance.setup, instance.order, instance.loaderror = { }, { }, { }, false +end + +function resolvers.loadnewconfig() + local luafiles = instance.luafiles + for i=1,#luafiles do + local cnf = luafiles[i] + local pathname = file.dirname(cnf) + local filename = file.join(pathname,resolvers.luaname) + local blob = loadfile(filename) + if blob then + local data = blob() + if data then + if trace_locating then + logs.report("fileio","loading configuration file '%s'",filename) + end + if true then + -- flatten to variable.progname + local t = { } + for k, v in next, data do -- v = progname + if type(v) == "string" then + t[k] = v + else + for kk, vv in next, v do -- vv = variable + if type(vv) == "string" then + t[vv.."."..v] = kk + end end end end + instance['setup'][pathname] = t + else + instance['setup'][pathname] = data end - instance[dataname][pathname] = t else - instance[dataname][pathname] = data + if trace_locating then + logs.report("fileio","skipping configuration file '%s'",filename) + end + instance['setup'][pathname] = { } + instance.loaderror = true end - else - input.report("skipping","configuration file",filename) - instance[dataname][pathname] = { } - instance.loaderror = true + elseif trace_locating then + logs.report("fileio","skipping configuration file '%s'",filename) end - else - input.report("skipping","configuration file",filename) - end -end - -function input.aux.load_configuration(instance,dname,lname) - input.aux.load_data(instance,dname,'configuration',lname and file.basename(lname)) -end -function input.aux.load_files(instance,tag) - input.aux.load_data(instance,tag,'files') -end - -function input.resetconfig(instance) - instance.configuration, instance.setup, instance.order, instance.loaderror = { }, { }, { }, false -end - -function input.loadnewconfig(instance) - for _, cnf in ipairs(instance.luafiles) do - local dname = file.dirname(cnf) - input.aux.load_texmfcnf(instance,'setup',dname) - instance.order[#instance.order+1] = instance.setup[dname] + instance.order[#instance.order+1] = instance.setup[pathname] if instance.loaderror then break end end end -function input.loadoldconfig(instance) +function resolvers.loadoldconfig() if not instance.renewcache then - for _, cnf in ipairs(instance.cnffiles) do + local cnffiles = instance.cnffiles + for i=1,#cnffiles do + local cnf = cnffiles[i] local dname = file.dirname(cnf) - input.aux.load_configuration(instance,dname) + resolvers.load_data(dname,'configuration') instance.order[#instance.order+1] = instance.configuration[dname] if instance.loaderror then break end end end - input.joinconfig(instance) + resolvers.joinconfig() end -function input.expand_variables(instance) - instance.expansions = { } ---~ instance.environment['SELFAUTOPARENT'] = instance.environment['SELFAUTOPARENT'] or instance.rootpath - if instance.engine ~= "" then instance.environment['engine'] = instance.engine end - if instance.progname ~= "" then instance.environment['progname'] = instance.progname end - for k,v in pairs(instance.environment) do - local a, b = k:match("^(%a+)%_(.*)%s*$") +function resolvers.expand_variables() + local expansions, environment, variables = { }, instance.environment, instance.variables + local env = resolvers.env + instance.expansions = expansions + if instance.engine ~= "" then environment['engine'] = instance.engine end + if instance.progname ~= "" then environment['progname'] = instance.progname end + for k,v in next, environment do + local a, b = match(k,"^(%a+)%_(.*)%s*$") if a and b then - instance.expansions[a..'.'..b] = v + expansions[a..'.'..b] = v else - instance.expansions[k] = v + expansions[k] = v end end - for k,v in pairs(instance.environment) do -- move environment to expansions - if not instance.expansions[k] then instance.expansions[k] = v end + for k,v in next, environment do -- move environment to expansions + if not expansions[k] then expansions[k] = v end + end + for k,v in next, variables do -- move variables to expansions + if not expansions[k] then expansions[k] = v end end - for k,v in pairs(instance.variables) do -- move variables to expansions - if not instance.expansions[k] then instance.expansions[k] = v end + local busy = false + local function resolve(a) + busy = true + return expansions[a] or env(a) end while true do - local busy = false - for k,v in pairs(instance.expansions) do - local s, n = v:gsub("%$([%a%d%_%-]+)", function(a) - busy = true - return instance.expansions[a] or input.env(instance,a) - end) - local s, m = s:gsub("%$%{([%a%d%_%-]+)%}", function(a) - busy = true - return instance.expansions[a] or input.env(instance,a) - end) + busy = false + for k,v in next, expansions do + local s, n = gsub(v,"%$([%a%d%_%-]+)",resolve) + local s, m = gsub(s,"%$%{([%a%d%_%-]+)%}",resolve) if n > 0 or m > 0 then - instance.expansions[k]= s + expansions[k]= s end end if not busy then break end end - for k,v in pairs(instance.expansions) do - instance.expansions[k] = v:gsub("\\", '/') + for k,v in next, expansions do + expansions[k] = gsub(v,"\\", '/') end end -function input.aux.expand_vars(instance,lst) -- simple vars - for k,v in pairs(lst) do - lst[k] = v:gsub("%$([%a%d%_%-]+)", function(a) - return instance.variables[a] or input.env(instance,a) - end) - end +function resolvers.variable(name) + return entry(instance.variables,name) end -function input.aux.expanded_var(instance,var) -- simple vars - return var:gsub("%$([%a%d%_%-]+)", function(a) - return instance.variables[a] or input.env(instance,a) - end) +function resolvers.expansion(name) + return entry(instance.expansions,name) end -function input.aux.entry(instance,entries,name) - if name and (name ~= "") then - name = name:gsub('%$','') - local result = entries[name..'.'..instance.progname] or entries[name] - if result then - return result - else - result = input.env(instance,name) - if result then - instance.variables[name] = result - input.expand_variables(instance) - return instance.expansions[name] or "" - end - end - end - return "" -end -function input.variable(instance,name) - return input.aux.entry(instance,instance.variables,name) -end -function input.expansion(instance,name) - return input.aux.entry(instance,instance.expansions,name) +function resolvers.is_variable(name) + return is_entry(instance.variables,name) end -function input.aux.is_entry(instance,entries,name) - if name and name ~= "" then - name = name:gsub('%$','') - return (entries[name..'.'..instance.progname] or entries[name]) ~= nil - else - return false - end +function resolvers.is_expansion(name) + return is_entry(instance.expansions,name) end -function input.is_variable(instance,name) - return input.aux.is_entry(instance,instance.variables,name) -end -function input.is_expansion(instance,name) - return input.aux.is_entry(instance,instance.expansions,name) +function resolvers.unexpanded_path_list(str) + local pth = resolvers.variable(str) + local lst = resolvers.split_path(pth) + return expanded_path_from_list(lst) end -function input.simplified_list(str) - if type(str) == 'table' then - return str -- troubles ; ipv , in texmf - elseif str == '' then - return { } - else - local t = { } - for _,v in ipairs(string.splitchr(str:gsub("^\{(.+)\}$","%1"),",")) do - t[#t+1] = (v:gsub("^[%!]*(.+)[%/\\]*$","%1")) - end - return t - end +function resolvers.unexpanded_path(str) + return file.join_path(resolvers.unexpanded_path_list(str)) end -function input.unexpanded_path_list(instance,str) - local pth = input.variable(instance,str) - local lst = input.split_path(pth) - return input.aux.expanded_path(instance,lst) -end -function input.unexpanded_path(instance,str) - return file.join_path(input.unexpanded_path_list(instance,str)) -end +do -- no longer needed -do local done = { } - function input.reset_extra_path(instance) + function resolvers.reset_extra_path() local ep = instance.extra_paths if not ep then ep, done = { }, { } @@ -5910,25 +9405,25 @@ do end end - function input.register_extra_path(instance,paths,subpaths) + function resolvers.register_extra_path(paths,subpaths) local ep = instance.extra_paths or { } local n = #ep if paths and paths ~= "" then if subpaths and subpaths ~= "" then - for p in paths:gmatch("[^,]+") do + for p in gmatch(paths,"[^,]+") do -- we gmatch each step again, not that fast, but used seldom - for s in subpaths:gmatch("[^,]+") do + for s in gmatch(subpaths,"[^,]+") do local ps = p .. "/" .. s if not done[ps] then - ep[#ep+1] = input.clean_path(ps) + ep[#ep+1] = resolvers.clean_path(ps) done[ps] = true end end end else - for p in paths:gmatch("[^,]+") do + for p in gmatch(paths,"[^,]+") do if not done[p] then - ep[#ep+1] = input.clean_path(p) + ep[#ep+1] = resolvers.clean_path(p) done[p] = true end end @@ -5936,10 +9431,10 @@ do elseif subpaths and subpaths ~= "" then for i=1,n do -- we gmatch each step again, not that fast, but used seldom - for s in subpaths:gmatch("[^,]+") do + for s in gmatch(subpaths,"[^,]+") do local ps = ep[i] .. "/" .. s if not done[ps] then - ep[#ep+1] = input.clean_path(ps) + ep[#ep+1] = resolvers.clean_path(ps) done[ps] = true end end @@ -5955,311 +9450,196 @@ do end -function input.expanded_path_list(instance,str) - local function made_list(list) - local ep = instance.extra_paths - if not ep or #ep == 0 then - return list - else - local done, new = { }, { } - -- honour . .. ../.. but only when at the start - for k, v in ipairs(list) do - if not done[v] then - if v:find("^[%.%/]$") then - done[v] = true - new[#new+1] = v - else - break - end - end - end - -- first the extra paths - for k, v in ipairs(ep) do - if not done[v] then +local function made_list(instance,list) + local ep = instance.extra_paths + if not ep or #ep == 0 then + return list + else + local done, new = { }, { } + -- honour . .. ../.. but only when at the start + for k=1,#list do + local v = list[k] + if not done[v] then + if find(v,"^[%.%/]$") then done[v] = true new[#new+1] = v + else + break end end - -- next the formal paths - for k, v in ipairs(list) do - if not done[v] then - done[v] = true - new[#new+1] = v - end + end + -- first the extra paths + for k=1,#ep do + local v = ep[k] + if not done[v] then + done[v] = true + new[#new+1] = v + end + end + -- next the formal paths + for k=1,#list do + local v = list[k] + if not done[v] then + done[v] = true + new[#new+1] = v end - return new end + return new end +end + +function resolvers.clean_path_list(str) + local t = resolvers.expanded_path_list(str) + if t then + for i=1,#t do + t[i] = file.collapse_path(resolvers.clean_path(t[i])) + end + end + return t +end + +function resolvers.expand_path(str) + return file.join_path(resolvers.expanded_path_list(str)) +end + +function resolvers.expanded_path_list(str) if not str then - return ep or { } + return ep or { } -- ep ? elseif instance.savelists then -- engine+progname hash - str = str:gsub("%$","") + str = gsub(str,"%$","") if not instance.lists[str] then -- cached - local lst = made_list(input.split_path(input.expansion(instance,str))) - instance.lists[str] = input.aux.expanded_path(instance,lst) + local lst = made_list(instance,resolvers.split_path(resolvers.expansion(str))) + instance.lists[str] = expanded_path_from_list(lst) end return instance.lists[str] else - local lst = input.split_path(input.expansion(instance,str)) - return made_list(input.aux.expanded_path(instance,lst)) + local lst = resolvers.split_path(resolvers.expansion(str)) + return made_list(instance,expanded_path_from_list(lst)) end end -function input.expand_path(instance,str) - return file.join_path(input.expanded_path_list(instance,str)) -end - ---~ function input.first_writable_path(instance,name) ---~ for _,v in pairs(input.expanded_path_list(instance,name)) do ---~ if file.is_writable(file.join(v,'luatex-cache.tmp')) then ---~ return v ---~ end ---~ end ---~ return "." ---~ end - -function input.expanded_path_list_from_var(instance,str) -- brrr - local tmp = input.var_of_format_or_suffix(str:gsub("%$","")) +function resolvers.expanded_path_list_from_var(str) -- brrr + local tmp = resolvers.var_of_format_or_suffix(gsub(str,"%$","")) if tmp ~= "" then - return input.expanded_path_list(instance,str) + return resolvers.expanded_path_list(tmp) else - return input.expanded_path_list(instance,tmp) + return resolvers.expanded_path_list(str) end end -function input.expand_path_from_var(instance,str) - return file.join_path(input.expanded_path_list_from_var(instance,str)) + +function resolvers.expand_path_from_var(str) + return file.join_path(resolvers.expanded_path_list_from_var(str)) end -function input.format_of_var(str) - return input.formats[str] or input.formats[input.alternatives[str]] or '' +function resolvers.format_of_var(str) + return formats[str] or formats[alternatives[str]] or '' end -function input.format_of_suffix(str) - return input.suffixmap[file.extname(str)] or 'tex' +function resolvers.format_of_suffix(str) + return suffixmap[file.extname(str)] or 'tex' end -function input.variable_of_format(str) - return input.formats[str] or input.formats[input.alternatives[str]] or '' +function resolvers.variable_of_format(str) + return formats[str] or formats[alternatives[str]] or '' end -function input.var_of_format_or_suffix(str) - local v = input.formats[str] +function resolvers.var_of_format_or_suffix(str) + local v = formats[str] if v then return v end - v = input.formats[input.alternatives[str]] + v = formats[alternatives[str]] if v then return v end - v = input.suffixmap[file.extname(str)] + v = suffixmap[file.extname(str)] if v then - return input.formats[isf] + return formats[isf] end return '' end -function input.expand_braces(instance,str) -- output variable and brace expansion of STRING - local ori = input.variable(instance,str) - local pth = input.aux.expanded_path(instance,input.split_path(ori)) +function resolvers.expand_braces(str) -- output variable and brace expansion of STRING + local ori = resolvers.variable(str) + local pth = expanded_path_from_list(resolvers.split_path(ori)) return file.join_path(pth) end --- {a,b,c,d} --- a,b,c/{p,q,r},d --- a,b,c/{p,q,r}/d/{x,y,z}// --- a,b,c/{p,q/{x,y,z},r},d/{p,q,r} --- a,b,c/{p,q/{x,y,z},r},d/{p,q,r} --- a{b,c}{d,e}f --- {a,b,c,d} --- {a,b,c/{p,q,r},d} --- {a,b,c/{p,q,r}/d/{x,y,z}//} --- {a,b,c/{p,q/{x,y,z}},d/{p,q,r}} --- {a,b,c/{p,q/{x,y,z},w}v,d/{p,q,r}} - --- this one is better and faster, but it took me a while to realize --- that this kind of replacement is cleaner than messy parsing and --- fuzzy concatenating we can probably gain a bit with selectively --- applying lpeg, but experiments with lpeg parsing this proved not to --- work that well; the parsing is ok, but dealing with the resulting --- table is a pain because we need to work inside-out recursively - --- get rid of piecewise here, just a gmatch is ok +resolvers.isreadable = { } -function input.aux.splitpathexpr(str, t, validate) - -- no need for optimization, only called a few times, we can use lpeg for the sub - t = t or { } - local concat = table.concat - while true do - local done = false - while true do - local ok = false - str = str:gsub("([^{},]+){([^{}]-)}", function(a,b) - local t = { } - b:piecewise(",", function(s) t[#t+1] = a .. s end) - ok, done = true, true - return "{" .. concat(t,",") .. "}" - end) - if not ok then break end - end - while true do - local ok = false - str = str:gsub("{([^{}]-)}([^{},]+)", function(a,b) - local t = { } - a:piecewise(",", function(s) t[#t+1] = s .. b end) - ok, done = true, true - return "{" .. concat(t,",") .. "}" - end) - if not ok then break end - end - while true do - local ok = false - str = str:gsub("([,{]){([^{}]+)}([,}])", function(a,b,c) - ok, done = true, true - return a .. b .. c - end) - if not ok then break end - end - if not done then break end - end - while true do - local ok = false - str = str:gsub("{([^{}]-)}{([^{}]-)}", function(a,b) - local t = { } - a:piecewise(",", function(sa) - b:piecewise(",", function(sb) - t[#t+1] = sa .. sb - end) - end) - ok = true - return "{" .. concat(t,",") .. "}" - end) - if not ok then break end - end - while true do - local ok = false - str = str:gsub("{([^{}]-)}", function(a) - ok = true - return a - end) - if not ok then break end - end - if validate then - str:piecewise(",", function(s) - s = validate(s) - if s then t[#t+1] = s end - end) - else - str:piecewise(",", function(s) - t[#t+1] = s - end) - end - return t -end - -function input.aux.expanded_path(instance,pathlist) -- maybe not a list, just a path - -- a previous version fed back into pathlist - local newlist, ok = { }, false - for _,v in ipairs(pathlist) do - if v:find("[{}]") then - ok = true - break - end - end - if ok then - for _, v in ipairs(pathlist) do - input.aux.splitpathexpr(v, newlist, function(s) - s = file.collapse_path(s) - return s ~= "" and not s:find(instance.dummy_path_expr) and s - end) - end - else - for _,v in ipairs(pathlist) do - for vv in string.gmatch(v..',',"(.-),") do - vv = file.collapse_path(v) - if vv ~= "" then newlist[#newlist+1] = vv end - end - end - end - return newlist -end - -input.is_readable = { } - -function input.aux.is_readable(readable, name) - if input.trace > 2 then +function resolvers.isreadable.file(name) + local readable = lfs.isfile(name) -- brrr + if trace_detail then if readable then - input.logger("+ readable", name) + logs.report("fileio","file '%s' is readable",name) else - input.logger("- readable", name) + logs.report("fileio","file '%s' is not readable", name) end end return readable end -function input.is_readable.file(name) - -- return input.aux.is_readable(file.is_readable(name), name) - return input.aux.is_readable(input.aux.is_file(name), name) -end - -input.is_readable.tex = input.is_readable.file +resolvers.isreadable.tex = resolvers.isreadable.file -- name -- name/name -function input.aux.collect_files(instance,names) +local function collect_files(names) local filelist = { } - for _, fname in pairs(names) do - if fname then - if input.trace > 2 then - input.logger("? blobpath asked",fname) - end - local bname = file.basename(fname) - local dname = file.dirname(fname) - if dname == "" or dname:find("^%.") then - dname = false - else - dname = "/" .. dname .. "$" - end - for _, hash in ipairs(instance.hashes) do - local blobpath = hash.tag - local files = blobpath and instance.files[blobpath] - if files then - if input.trace > 2 then - input.logger('? blobpath do',blobpath .. " (" .. bname ..")") + for k=1,#names do + local fname = names[k] + if trace_detail then + logs.report("fileio","checking name '%s'",fname) + end + local bname = file.basename(fname) + local dname = file.dirname(fname) + if dname == "" or find(dname,"^%.") then + dname = false + else + dname = "/" .. dname .. "$" + end + local hashes = instance.hashes + for h=1,#hashes do + local hash = hashes[h] + local blobpath = hash.tag + local files = blobpath and instance.files[blobpath] + if files then + if trace_detail then + logs.report("fileio","deep checking '%s' (%s)",blobpath,bname) + end + local blobfile = files[bname] + if not blobfile then + local rname = "remap:"..bname + blobfile = files[rname] + if blobfile then + bname = files[rname] + blobfile = files[bname] end - local blobfile = files[bname] - if not blobfile then - local rname = "remap:"..bname - blobfile = files[rname] - if blobfile then - bname = files[rname] - blobfile = files[bname] + end + if blobfile then + if type(blobfile) == 'string' then + if not dname or find(blobfile,dname) then + filelist[#filelist+1] = { + hash.type, + file.join(blobpath,blobfile,bname), -- search + resolvers.concatinators[hash.type](blobpath,blobfile,bname) -- result + } end - end - if blobfile then - if type(blobfile) == 'string' then - if not dname or blobfile:find(dname) then + else + for kk=1,#blobfile do + local vv = blobfile[kk] + if not dname or find(vv,dname) then filelist[#filelist+1] = { hash.type, - file.join(blobpath,blobfile,bname), -- search - input.concatinators[hash.type](blobpath,blobfile,bname) -- result + file.join(blobpath,vv,bname), -- search + resolvers.concatinators[hash.type](blobpath,vv,bname) -- result } end - else - for _, vv in pairs(blobfile) do - if not dname or vv:find(dname) then - filelist[#filelist+1] = { - hash.type, - file.join(blobpath,vv,bname), -- search - input.concatinators[hash.type](blobpath,vv,bname) -- result - } - end - end end end - elseif input.trace > 1 then - input.logger('! blobpath no',blobpath .. " (" .. bname ..")" ) end + elseif trace_locating then + logs.report("fileio","no match in '%s' (%s)",blobpath,bname) end end end @@ -6270,99 +9650,146 @@ function input.aux.collect_files(instance,names) end end -function input.suffix_of_format(str) - if input.suffixes[str] then - return input.suffixes[str][1] +function resolvers.suffix_of_format(str) + if suffixes[str] then + return suffixes[str][1] else return "" end end -function input.suffixes_of_format(str) - if input.suffixes[str] then - return input.suffixes[str] +function resolvers.suffixes_of_format(str) + if suffixes[str] then + return suffixes[str] else return {} end end -do - - -- called about 700 times for an empty doc (font initializations etc) - -- i need to weed the font files for redundant calls - - local letter = lpeg.R("az","AZ") - local separator = lpeg.P("://") - - local qualified = lpeg.P(".")^0 * lpeg.P("/") + letter*lpeg.P(":") + letter^1*separator - local rootbased = lpeg.P("/") + letter*lpeg.P(":") - - -- ./name ../name /name c: :// - function input.aux.qualified_path(filename) - return qualified:match(filename) - end - function input.aux.rootbased_path(filename) - return rootbased:match(filename) - end - - function input.normalize_name(original) - return original +function resolvers.register_in_trees(name) + if not find(name,"^%.") then + instance.foundintrees[name] = (instance.foundintrees[name] or 0) + 1 -- maybe only one end - - input.normalize_name = file.collapse_path - end -function input.aux.register_in_trees(instance,name) - if not name:find("^%.") then - instance.foundintrees[name] = (instance.foundintrees[name] or 0) + 1 -- maybe only one +-- split the next one up for readability (bu this module needs a cleanup anyway) + +local function can_be_dir(name) -- can become local + local fakepaths = instance.fakepaths + if not fakepaths[name] then + if lfs.isdir(name) then + fakepaths[name] = 1 -- directory + else + fakepaths[name] = 2 -- no directory + end end + return (fakepaths[name] == 1) end --- split the next one up, better for jit - -function input.aux.find_file(instance,filename) -- todo : plugin (scanners, checkers etc) - local result = { } +local function collect_instance_files(filename,collected) -- todo : plugin (scanners, checkers etc) + local result = collected or { } local stamp = nil - filename = input.normalize_name(filename) -- elsewhere - filename = file.collapse_path(filename:gsub("\\","/")) -- elsewhere + filename = file.collapse_path(filename) -- speed up / beware: format problem if instance.remember then stamp = filename .. "--" .. instance.engine .. "--" .. instance.progname .. "--" .. instance.format if instance.found[stamp] then - input.logger('! remembered', filename) + if trace_locating then + logs.report("fileio","remembering file '%s'",filename) + end return instance.found[stamp] end end - if filename:find('%*') then - input.logger('! wildcard', filename) - result = input.find_wildcard_files(instance,filename) - elseif input.aux.qualified_path(filename) then - if input.is_readable.file(filename) then - input.logger('! qualified', filename) + if not dangerous[instance.format or "?"] then + if resolvers.isreadable.file(filename) then + if trace_detail then + logs.report("fileio","file '%s' found directly",filename) + end + instance.found[stamp] = { filename } + return { filename } + end + end + if find(filename,'%*') then + if trace_locating then + logs.report("fileio","checking wildcard '%s'", filename) + end + result = resolvers.find_wildcard_files(filename) + elseif file.is_qualified_path(filename) then + if resolvers.isreadable.file(filename) then + if trace_locating then + logs.report("fileio","qualified name '%s'", filename) + end result = { filename } else - local forcedname, ok = "", false - if file.extname(filename) == "" then + local forcedname, ok, suffix = "", false, file.extname(filename) + if suffix == "" then -- why if instance.format == "" then forcedname = filename .. ".tex" - if input.is_readable.file(forcedname) then - input.logger('! no suffix, forcing standard filetype tex') + if resolvers.isreadable.file(forcedname) then + if trace_locating then + logs.report("fileio","no suffix, forcing standard filetype 'tex'") + end result, ok = { forcedname }, true end else - for _, s in pairs(input.suffixes_of_format(instance.format)) do + local suffixes = resolvers.suffixes_of_format(instance.format) + for _, s in next, suffixes do forcedname = filename .. "." .. s - if input.is_readable.file(forcedname) then - input.logger('! no suffix, forcing format filetype', s) + if resolvers.isreadable.file(forcedname) then + if trace_locating then + logs.report("fileio","no suffix, forcing format filetype '%s'", s) + end result, ok = { forcedname }, true break end end end end - if not ok then - input.logger('? qualified', filename) + if not ok and suffix ~= "" then + -- try to find in tree (no suffix manipulation), here we search for the + -- matching last part of the name + local basename = file.basename(filename) + local pattern = gsub(filename .. "$","([%.%-])","%%%1") + local savedformat = instance.format + local format = savedformat or "" + if format == "" then + instance.format = resolvers.format_of_suffix(suffix) + end + if not format then + instance.format = "othertextfiles" -- kind of everything, maybe texinput is better + end + -- + if basename ~= filename then + local resolved = collect_instance_files(basename) + if #result == 0 then + local lowered = lower(basename) + if filename ~= lowered then + resolved = collect_instance_files(lowered) + end + end + resolvers.format = savedformat + -- + for r=1,#resolved do + local rr = resolved[r] + if find(rr,pattern) then + result[#result+1], ok = rr, true + end + end + end + -- a real wildcard: + -- + -- if not ok then + -- local filelist = collect_files({basename}) + -- for f=1,#filelist do + -- local ff = filelist[f][3] or "" + -- if find(ff,pattern) then + -- result[#result+1], ok = ff, true + -- end + -- end + -- end + end + if not ok and trace_locating then + logs.report("fileio","qualified name '%s'", filename) end end else @@ -6379,39 +9806,45 @@ function input.aux.find_file(instance,filename) -- todo : plugin (scanners, chec if ext == "" then local forcedname = filename .. '.tex' wantedfiles[#wantedfiles+1] = forcedname - filetype = input.format_of_suffix(forcedname) - input.logger('! forcing filetype',filetype) + filetype = resolvers.format_of_suffix(forcedname) + if trace_locating then + logs.report("fileio","forcing filetype '%s'",filetype) + end else - filetype = input.format_of_suffix(filename) - input.logger('! using suffix based filetype',filetype) + filetype = resolvers.format_of_suffix(filename) + if trace_locating then + logs.report("fileio","using suffix based filetype '%s'",filetype) + end end else if ext == "" then - for _, s in pairs(input.suffixes_of_format(instance.format)) do + local suffixes = resolvers.suffixes_of_format(instance.format) + for _, s in next, suffixes do wantedfiles[#wantedfiles+1] = filename .. "." .. s end end filetype = instance.format - input.logger('! using given filetype',filetype) + if trace_locating then + logs.report("fileio","using given filetype '%s'",filetype) + end end - local typespec = input.variable_of_format(filetype) - local pathlist = input.expanded_path_list(instance,typespec) + local typespec = resolvers.variable_of_format(filetype) + local pathlist = resolvers.expanded_path_list(typespec) if not pathlist or #pathlist == 0 then -- no pathlist, access check only / todo == wildcard - if input.trace > 2 then - input.logger('? filename',filename) - input.logger('? filetype',filetype or '?') - input.logger('? wanted files',table.concat(wantedfiles," | ")) + if trace_detail then + logs.report("fileio","checking filename '%s', filetype '%s', wanted files '%s'",filename, filetype or '?',concat(wantedfiles," | ")) end - for _, fname in pairs(wantedfiles) do - if fname and input.is_readable.file(fname) then + for k=1,#wantedfiles do + local fname = wantedfiles[k] + if fname and resolvers.isreadable.file(fname) then filename, done = fname, true result[#result+1] = file.join('.',fname) break end end -- this is actually 'other text files' or 'any' or 'whatever' - local filelist = input.aux.collect_files(instance,wantedfiles) + local filelist = collect_files(wantedfiles) local fl = filelist and filelist[1] if fl then filename = fl[3] @@ -6420,56 +9853,76 @@ function input.aux.find_file(instance,filename) -- todo : plugin (scanners, chec end else -- list search - local filelist = input.aux.collect_files(instance,wantedfiles) - local doscan, recurse - if input.trace > 2 then - input.logger('? filename',filename) - -- if pathlist then input.logger('? path list',table.concat(pathlist," | ")) end - -- if filelist then input.logger('? file list',table.concat(filelist," | ")) end + local filelist = collect_files(wantedfiles) + local dirlist = { } + if filelist then + for i=1,#filelist do + dirlist[i] = file.dirname(filelist[i][2]) .. "/" + end + end + if trace_detail then + logs.report("fileio","checking filename '%s'",filename) end -- a bit messy ... esp the doscan setting here - for _, path in pairs(pathlist) do - if path:find("^!!") then doscan = false else doscan = true end - if path:find("//$") then recurse = true else recurse = false end - local pathname = path:gsub("^!+", '') + local doscan + for k=1,#pathlist do + local path = pathlist[k] + if find(path,"^!!") then doscan = false else doscan = true end + local pathname = gsub(path,"^!+", '') done = false -- using file list - if filelist and not (done and not instance.allresults) and recurse then - -- compare list entries with permitted pattern - pathname = pathname:gsub("([%-%.])","%%%1") -- this also influences - pathname = pathname:gsub("/+$", '/.*') -- later usage of pathname - pathname = pathname:gsub("//", '/.-/') -- not ok for /// but harmless - local expr = "^" .. pathname - -- input.debug('?',expr) - for _, fl in ipairs(filelist) do + if filelist then + local expression + -- compare list entries with permitted pattern -- /xx /xx// + if not find(pathname,"/$") then + expression = pathname .. "/" + else + expression = pathname + end + expression = gsub(expression,"([%-%.])","%%%1") -- this also influences + expression = gsub(expression,"//+$", '/.*') -- later usage of pathname + expression = gsub(expression,"//", '/.-/') -- not ok for /// but harmless + expression = "^" .. expression .. "$" + if trace_detail then + logs.report("fileio","using pattern '%s' for path '%s'",expression,pathname) + end + for k=1,#filelist do + local fl = filelist[k] local f = fl[2] - if f:find(expr) then - -- input.debug('T',' '..f) - if input.trace > 2 then - input.logger('= found in hash',f) - end + local d = dirlist[k] + if find(d,expression) then --- todo, test for readable result[#result+1] = fl[3] - input.aux.register_in_trees(instance,f) -- for tracing used files + resolvers.register_in_trees(f) -- for tracing used files done = true - if not instance.allresults then break end - else - -- input.debug('F',' '..f) + if instance.allresults then + if trace_detail then + logs.report("fileio","match in hash for file '%s' on path '%s', continue scanning",f,d) + end + else + if trace_detail then + logs.report("fileio","match in hash for file '%s' on path '%s', quit scanning",f,d) + end + break + end + elseif trace_detail then + logs.report("fileio","no match in hash for file '%s' on path '%s'",f,d) end end end if not done and doscan then -- check if on disk / unchecked / does not work at all / also zips - if input.method_is_file(pathname) then -- ? - local pname = pathname:gsub("%.%*$",'') - if not pname:find("%*") then - local ppname = pname:gsub("/+$","") - if input.aux.can_be_dir(instance,ppname) then - for _, w in pairs(wantedfiles) do + if resolvers.splitmethod(pathname).scheme == 'file' then -- ? + local pname = gsub(pathname,"%.%*$",'') + if not find(pname,"%*") then + local ppname = gsub(pname,"/+$","") + if can_be_dir(ppname) then + for k=1,#wantedfiles do + local w = wantedfiles[k] local fname = file.join(ppname,w) - if input.is_readable.file(fname) then - if input.trace > 2 then - input.logger('= found by scanning',fname) + if resolvers.isreadable.file(fname) then + if trace_detail then + logs.report("fileio","found '%s' by scanning",fname) end result[#result+1] = fname done = true @@ -6489,8 +9942,8 @@ function input.aux.find_file(instance,filename) -- todo : plugin (scanners, chec end end end - for k,v in pairs(result) do - result[k] = file.collapse_path(v) + for k=1,#result do + result[k] = file.collapse_path(result[k]) end if instance.remember then instance.found[stamp] = result @@ -6498,48 +9951,12 @@ function input.aux.find_file(instance,filename) -- todo : plugin (scanners, chec return result end -input.aux._find_file_ = input.aux.find_file - -function input.aux.find_file(instance,filename) -- maybe make a lowres cache too - local result = input.aux._find_file_(instance,filename) - if #result == 0 then - local lowered = filename:lower() - if filename ~= lowered then - return input.aux._find_file_(instance,lowered) - end - end - return result -end - -if lfs and lfs.isfile then - input.aux.is_file = lfs.isfile -- to be done: use this -else - input.aux.is_file = file.is_readable -end - -if lfs and lfs.isdir then - function input.aux.can_be_dir(instance,name) - if not instance.fakepaths[name] then - if lfs.isdir(name) then - instance.fakepaths[name] = 1 -- directory - else - instance.fakepaths[name] = 2 -- no directory - end - end - return (instance.fakepaths[name] == 1) - end -else - function input.aux.can_be_dir() - return true - end -end - -if not input.concatinators then input.concatinators = { } end +if not resolvers.concatinators then resolvers.concatinators = { } end -input.concatinators.tex = file.join -input.concatinators.file = input.concatinators.tex +resolvers.concatinators.tex = file.join +resolvers.concatinators.file = resolvers.concatinators.tex -function input.find_files(instance,filename,filetype,mustexist) +function resolvers.find_files(filename,filetype,mustexist) if type(mustexist) == boolean then -- all set elseif type(filetype) == 'boolean' then @@ -6548,19 +9965,27 @@ function input.find_files(instance,filename,filetype,mustexist) filetype, mustexist = nil, false end instance.format = filetype or '' - local t = input.aux.find_file(instance,filename,true) + local result = collect_instance_files(filename) + if #result == 0 then + local lowered = lower(filename) + if filename ~= lowered then + return collect_instance_files(lowered) + end + end instance.format = '' - return t + return result end -function input.find_file(instance,filename,filetype,mustexist) - return (input.find_files(instance,filename,filetype,mustexist)[1] or "") +function resolvers.find_file(filename,filetype,mustexist) + return (resolvers.find_files(filename,filetype,mustexist)[1] or "") end -function input.find_given_files(instance,filename) +function resolvers.find_given_files(filename) local bname, result = file.basename(filename), { } - for k, hash in ipairs(instance.hashes) do - local files = instance.files[hash.tag] + local hashes = instance.hashes + for k=1,#hashes do + local hash = hashes[k] + local files = instance.files[hash.tag] or { } local blist = files[bname] if not blist then local rname = "remap:"..bname @@ -6572,11 +9997,12 @@ function input.find_given_files(instance,filename) end if blist then if type(blist) == 'string' then - result[#result+1] = input.concatinators[hash.type](hash.tag,blist,bname) or "" + result[#result+1] = resolvers.concatinators[hash.type](hash.tag,blist,bname) or "" if not instance.allresults then break end else - for kk,vv in pairs(blist) do - result[#result+1] = input.concatinators[hash.type](hash.tag,vv,bname) or "" + for kk=1,#blist do + local vv = blist[kk] + result[#result+1] = resolvers.concatinators[hash.type](hash.tag,vv,bname) or "" if not instance.allresults then break end end end @@ -6585,127 +10011,121 @@ function input.find_given_files(instance,filename) return result end -function input.find_given_file(instance,filename) - return (input.find_given_files(instance,filename)[1] or "") +function resolvers.find_given_file(filename) + return (resolvers.find_given_files(filename)[1] or "") +end + +local function doit(path,blist,bname,tag,kind,result,allresults) + local done = false + if blist and kind then + if type(blist) == 'string' then + -- make function and share code + if find(lower(blist),path) then + result[#result+1] = resolvers.concatinators[kind](tag,blist,bname) or "" + done = true + end + else + for kk=1,#blist do + local vv = blist[kk] + if find(lower(vv),path) then + result[#result+1] = resolvers.concatinators[kind](tag,vv,bname) or "" + done = true + if not allresults then break end + end + end + end + end + return done end -function input.find_wildcard_files(instance,filename) -- todo: remap: +function resolvers.find_wildcard_files(filename) -- todo: remap: local result = { } local bname, dname = file.basename(filename), file.dirname(filename) - local path = dname:gsub("^*/","") - path = path:gsub("*",".*") - path = path:gsub("-","%%-") + local path = gsub(dname,"^*/","") + path = gsub(path,"*",".*") + path = gsub(path,"-","%%-") if dname == "" then path = ".*" end local name = bname - name = name:gsub("*",".*") - name = name:gsub("-","%%-") - path = path:lower() - name = name:lower() - local function doit(blist,bname,hash,allresults) - local done = false - if blist then - if type(blist) == 'string' then - -- make function and share code - if (blist:lower()):find(path) then - result[#result+1] = input.concatinators[hash.type](hash.tag,blist,bname) or "" - done = true - end - else - for kk,vv in pairs(blist) do - if (vv:lower()):find(path) then - result[#result+1] = input.concatinators[hash.type](hash.tag,vv,bname) or "" - done = true - if not allresults then break end - end - end - end - end - return done - end + name = gsub(name,"*",".*") + name = gsub(name,"-","%%-") + path = lower(path) + name = lower(name) local files, allresults, done = instance.files, instance.allresults, false - if name:find("%*") then - for k, hash in ipairs(instance.hashes) do - for kk, hh in pairs(files[hash.tag]) do - if not kk:find("^remap:") then - if (kk:lower()):find(name) then - if doit(hh,kk,hash,allresults) then done = true end + if find(name,"%*") then + local hashes = instance.hashes + for k=1,#hashes do + local hash = hashes[k] + local tag, kind = hash.tag, hash.type + for kk, hh in next, files[hash.tag] do + if not find(kk,"^remap:") then + if find(lower(kk),name) then + if doit(path,hh,kk,tag,kind,result,allresults) then done = true end if done and not allresults then break end end end end end else - for k, hash in ipairs(instance.hashes) do - if doit(files[hash.tag][bname],bname,hash,allresults) then done = true end + local hashes = instance.hashes + for k=1,#hashes do + local hash = hashes[k] + local tag, kind = hash.tag, hash.type + if doit(path,files[tag][bname],bname,tag,kind,result,allresults) then done = true end if done and not allresults then break end end end + -- we can consider also searching the paths not in the database, but then + -- we end up with a messy search (all // in all path specs) return result end -function input.find_wildcard_file(instance,filename) - return (input.find_wildcard_files(instance,filename)[1] or "") +function resolvers.find_wildcard_file(filename) + return (resolvers.find_wildcard_files(filename)[1] or "") end -- main user functions -function input.save_used_files_in_trees(instance, filename,jobname) - if not filename then filename = 'luatex.jlg' end - local f = io.open(filename,'w') - if f then - f:write("<?xml version='1.0' standalone='yes'?>\n") - f:write("<rl:job>\n") - if jobname then - f:write("\t<rl:name>" .. jobname .. "</rl:name>\n") - end - f:write("\t<rl:files>\n") - for _,v in pairs(table.sortedkeys(instance.foundintrees)) do - f:write("\t\t<rl:file n='" .. instance.foundintrees[v] .. "'>" .. v .. "</rl:file>\n") - end - f:write("\t</rl:files>\n") - f:write("</rl:usedfiles>\n") - f:close() - end -end - -function input.automount(instance) +function resolvers.automount() -- implemented later end -function input.load(instance) - input.starttiming(instance) - input.resetconfig(instance) - input.identify_cnf(instance) - input.load_lua(instance) - input.expand_variables(instance) - input.load_cnf(instance) - input.expand_variables(instance) - input.load_hash(instance) - input.automount(instance) - input.stoptiming(instance) +function resolvers.load(option) + statistics.starttiming(instance) + resolvers.resetconfig() + resolvers.identify_cnf() + resolvers.load_lua() -- will become the new method + resolvers.expand_variables() + resolvers.load_cnf() -- will be skipped when we have a lua file + resolvers.expand_variables() + if option ~= "nofiles" then + resolvers.load_hash() + resolvers.automount() + end + statistics.stoptiming(instance) end -function input.for_files(instance, command, files, filetype, mustexist) +function resolvers.for_files(command, files, filetype, mustexist) if files and #files > 0 then local function report(str) - if input.verbose then - input.report(str) -- has already verbose + if trace_locating then + logs.report("fileio",str) -- has already verbose else print(str) end end - if input.verbose then - report('') + if trace_locating then + report('') -- ? end - for _, file in pairs(files) do - local result = command(instance,file,filetype,mustexist) + for f=1,#files do + local file = files[f] + local result = command(file,filetype,mustexist) if type(result) == 'string' then report(result) else - for _,v in pairs(result) do - report(v) + for i=1,#result do + report(result[i]) -- could be unpack end end end @@ -6714,22 +10134,19 @@ end -- strtab -function input.var_value(instance,str) -- output the value of variable $STRING. - return input.variable(instance,str) -end -function input.expand_var(instance,str) -- output variable expansion of STRING. - return input.expansion(instance,str) -end -function input.show_path(instance,str) -- output search path for file type NAME - return file.join_path(input.expanded_path_list(instance,input.format_of_var(str))) +resolvers.var_value = resolvers.variable -- output the value of variable $STRING. +resolvers.expand_var = resolvers.expansion -- output variable expansion of STRING. + +function resolvers.show_path(str) -- output search path for file type NAME + return file.join_path(resolvers.expanded_path_list(resolvers.format_of_var(str))) end --- input.find_file(filename) --- input.find_file(filename, filetype, mustexist) --- input.find_file(filename, mustexist) --- input.find_file(filename, filetype) +-- resolvers.find_file(filename) +-- resolvers.find_file(filename, filetype, mustexist) +-- resolvers.find_file(filename, mustexist) +-- resolvers.find_file(filename, filetype) -function input.aux.register_file(files, name, path) +function resolvers.register_file(files, name, path) if files[name] then if type(files[name]) == 'string' then files[name] = { files[name], path } @@ -6741,165 +10158,81 @@ function input.aux.register_file(files, name, path) end end -if not input.finders then input.finders = { } end -if not input.openers then input.openers = { } end -if not input.loaders then input.loaders = { } end - -input.finders.notfound = { nil } -input.openers.notfound = { nil } -input.loaders.notfound = { false, nil, 0 } - -function input.splitmethod(filename) +function resolvers.splitmethod(filename) if not filename then return { } -- safeguard elseif type(filename) == "table" then return filename -- already split - elseif not filename:find("://") then + elseif not find(filename,"://") then return { scheme="file", path = filename, original=filename } -- quick hack else return url.hashed(filename) end end -function input.method_is_file(filename) - return input.splitmethod(filename).scheme == 'file' -end - function table.sequenced(t,sep) -- temp here local s = { } - for k, v in pairs(t) do - s[#s+1] = k .. "=" .. v + for k, v in next, t do -- indexed? + s[#s+1] = k .. "=" .. tostring(v) end - return table.concat(s, sep or " | ") + return concat(s, sep or " | ") end -function input.methodhandler(what, instance, filename, filetype) -- ... - local specification = (type(filename) == "string" and input.splitmethod(filename)) or filename -- no or { }, let it bomb +function resolvers.methodhandler(what, filename, filetype) -- ... + filename = file.collapse_path(filename) + local specification = (type(filename) == "string" and resolvers.splitmethod(filename)) or filename -- no or { }, let it bomb local scheme = specification.scheme - if input[what][scheme] then - input.logger('= handler',specification.original .." -> " .. what .. " -> " .. table.sequenced(specification)) - return input[what][scheme](instance,filename,filetype) -- todo: specification - else - return input[what].tex(instance,filename,filetype) -- todo: specification - end -end - --- also inside next test? - -function input.findtexfile(instance, filename, filetype) - return input.methodhandler('finders',instance, input.normalize_name(filename), filetype) -end -function input.opentexfile(instance,filename) - return input.methodhandler('openers',instance, input.normalize_name(filename)) -end - -function input.findbinfile(instance, filename, filetype) - return input.methodhandler('finders',instance, input.normalize_name(filename), filetype) -end -function input.openbinfile(instance,filename) - return input.methodhandler('loaders',instance, input.normalize_name(filename)) -end - -function input.loadbinfile(instance, filename, filetype) - local fname = input.findbinfile(instance, input.normalize_name(filename), filetype) - if fname and fname ~= "" then - return input.openbinfile(instance,fname) - else - return unpack(input.loaders.notfound) - end -end - -function input.texdatablob(instance, filename, filetype) - local ok, data, size = input.loadbinfile(instance, filename, filetype) - return data or "" -end - -input.loadtexfile = input.texdatablob - -function input.openfile(filename) -- brrr texmf.instance here / todo ! ! ! ! ! - local fullname = input.findtexfile(texmf.instance, filename) - if fullname and (fullname ~= "") then - return input.opentexfile(texmf.instance, fullname) + if resolvers[what][scheme] then + if trace_locating then + logs.report("fileio","handler '%s' -> '%s' -> '%s'",specification.original,what,table.sequenced(specification)) + end + return resolvers[what][scheme](filename,filetype) -- todo: specification else - return nil + return resolvers[what].tex(filename,filetype) -- todo: specification end end -function input.logmode() - return (os.getenv("MTX.LOG.MODE") or os.getenv("MTX_LOG_MODE") or "tex"):lower() -end - --- this is a prelude to engine/progname specific configuration files --- in which case we can omit files meant for other programs and --- packages - ---- ctx - --- maybe texinputs + font paths --- maybe positive selection tex/context fonts/tfm|afm|vf|opentype|type1|map|enc - -input.validators = { } -input.validators.visibility = { } - -function input.validators.visibility.default(path, name) - return true -end - -function input.validators.visibility.context(path, name) - path = path[1] or path -- some day a loop - return not ( - path:find("latex") or --- path:find("doc") or - path:find("tex4ht") or - path:find("source") or --- path:find("config") or --- path:find("metafont") or - path:find("lists$") or - name:find("%.tpm$") or - name:find("%.bak$") - ) -end - --- todo: describe which functions are public (maybe input.private. ... ) - --- beware: i need to check where we still need a / on windows: - -function input.clean_path(str) ---~ return (((str:gsub("\\","/")):gsub("^!+","")):gsub("//+","//")) +function resolvers.clean_path(str) if str then - return ((str:gsub("\\","/")):gsub("^!+","")) + str = gsub(str,"\\","/") + str = gsub(str,"^!+","") + str = gsub(str,"^~",resolvers.homedir) + return str else return nil end end -function input.do_with_path(name,func) - for _, v in pairs(input.expanded_path_list(instance,name)) do - func("^"..input.clean_path(v)) +function resolvers.do_with_path(name,func) + local pathlist = resolvers.expanded_path_list(name) + for i=1,#pathlist do + func("^"..resolvers.clean_path(pathlist[i])) end end -function input.do_with_var(name,func) - func(input.aux.expanded_var(name)) +function resolvers.do_with_var(name,func) + func(expanded_var(name)) end -function input.with_files(instance,pattern,handle) - for _, hash in ipairs(instance.hashes) do +function resolvers.with_files(pattern,handle) + local hashes = instance.hashes + for i=1,#hashes do + local hash = hashes[i] local blobpath = hash.tag local blobtype = hash.type if blobpath then local files = instance.files[blobpath] if files then - for k,v in pairs(files) do - if k:find("^remap:") then + for k,v in next, files do + if find(k,"^remap:") then k = files[k] v = files[k] -- chained end - if k:find(pattern) then + if find(k,pattern) then if type(v) == "string" then handle(blobtype,blobpath,v,k) else - for _,vv in pairs(v) do + for _,vv in next, v do -- indexed handle(blobtype,blobpath,vv,k) end end @@ -6910,131 +10243,56 @@ function input.with_files(instance,pattern,handle) end end ---~ function input.update_script(oldname,newname) -- oldname -> own.name, not per se a suffix ---~ newname = file.addsuffix(newname,"lua") ---~ local newscript = input.clean_path(input.find_file(instance, newname)) ---~ local oldscript = input.clean_path(oldname) ---~ input.report("old script", oldscript) ---~ input.report("new script", newscript) ---~ if oldscript ~= newscript and (oldscript:find(file.removesuffix(newname).."$") or oldscript:find(newname.."$")) then ---~ local newdata = io.loaddata(newscript) ---~ if newdata then ---~ input.report("old script content replaced by new content") ---~ io.savedata(oldscript,newdata) ---~ end ---~ end ---~ end - -function input.update_script(instance,oldname,newname) -- oldname -> own.name, not per se a suffix - local scriptpath = "scripts/context/lua" - newname = file.addsuffix(newname,"lua") - local oldscript = input.clean_path(oldname) - input.report("to be replaced old script", oldscript) - local newscripts = input.find_files(instance, newname) or { } - if #newscripts == 0 then - input.report("unable to locate new script") - else - for _, newscript in ipairs(newscripts) do - newscript = input.clean_path(newscript) - input.report("checking new script", newscript) - if oldscript == newscript then - input.report("old and new script are the same") - elseif not newscript:find(scriptpath) then - input.report("new script should come from",scriptpath) - elseif not (oldscript:find(file.removesuffix(newname).."$") or oldscript:find(newname.."$")) then - input.report("invalid new script name") - else - local newdata = io.loaddata(newscript) - if newdata then - input.report("old script content replaced by new content") - io.savedata(oldscript,newdata) - break - else - input.report("unable to load new script") - end - end +function resolvers.locate_format(name) + local barename, fmtname = gsub(name,"%.%a+$",""), "" + if resolvers.usecache then + local path = file.join(caches.setpath("formats")) -- maybe platform + fmtname = file.join(path,barename..".fmt") or "" + end + if fmtname == "" then + fmtname = resolvers.find_files(barename..".fmt")[1] or "" + end + fmtname = resolvers.clean_path(fmtname) + if fmtname ~= "" then + local barename = file.removesuffix(fmtname) + local luaname, lucname, luiname = barename .. ".lua", barename .. ".luc", barename .. ".lui" + if lfs.isfile(luiname) then + return barename, luiname + elseif lfs.isfile(lucname) then + return barename, lucname + elseif lfs.isfile(luaname) then + return barename, luaname end end + return nil, nil end +function resolvers.boolean_variable(str,default) + local b = resolvers.expansion(str) + if b == "" then + return default + else + b = toboolean(b) + return (b == nil and default) or b + end +end ---~ print(table.serialize(input.aux.splitpathexpr("/usr/share/texmf-{texlive,tetex}", {}))) - --- command line resolver: - ---~ print(input.resolve("abc env:tmp file:cont-en.tex path:cont-en.tex full:cont-en.tex rel:zapf/one/p-chars.tex")) - -do +texconfig.kpse_init = false - local resolvers = { } +kpse = { original = kpse } setmetatable(kpse, { __index = function(k,v) return resolvers[v] end } ) - resolvers.environment = function(instance,str) - return input.clean_path(os.getenv(str) or os.getenv(str:upper()) or os.getenv(str:lower()) or "") - end - resolvers.relative = function(instance,str,n) - if io.exists(str) then - -- nothing - elseif io.exists("./" .. str) then - str = "./" .. str - else - local p = "../" - for i=1,n or 2 do - if io.exists(p .. str) then - str = p .. str - break - else - p = p .. "../" - end - end - end - return input.clean_path(str) - end - resolvers.locate = function(instance,str) - local fullname = input.find_given_file(instance,str) or "" - return input.clean_path((fullname ~= "" and fullname) or str) - end - resolvers.filename = function(instance,str) - local fullname = input.find_given_file(instance,str) or "" - return input.clean_path(file.basename((fullname ~= "" and fullname) or str)) - end - resolvers.pathname = function(instance,str) - local fullname = input.find_given_file(instance,str) or "" - return input.clean_path(file.dirname((fullname ~= "" and fullname) or str)) - end - - resolvers.env = resolvers.environment - resolvers.rel = resolvers.relative - resolvers.loc = resolvers.locate - resolvers.kpse = resolvers.locate - resolvers.full = resolvers.locate - resolvers.file = resolvers.filename - resolvers.path = resolvers.pathname +-- for a while - local function resolve(instance,str) - if type(str) == "table" then - for k, v in pairs(str) do - str[k] = resolve(instance,v) or v - end - elseif str and str ~= "" then - str = str:gsub("([a-z]+):([^ ]+)", function(method,target) - if resolvers[method] then - return resolvers[method](instance,target) - else - return method .. ":" .. target - end - end) - end - return str - end +input = resolvers - input.resolve = resolve -end +end -- of closure +do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['luat-tmp'] = { +if not modules then modules = { } end modules ['data-tmp'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -7056,63 +10314,71 @@ being written at the same time is small. We also need to extend luatools with a recache feature.</p> --ldx]]-- +local format, lower, gsub = string.format, string.lower, string.gsub + +local trace_cache = false trackers.register("resolvers.cache", function(v) trace_cache = v end) -- not used yet + caches = caches or { } -dir = dir or { } -texmf = texmf or { } - -caches.path = caches.path or nil -caches.base = caches.base or "luatex-cache" -caches.more = caches.more or "context" -caches.direct = false -- true is faster but may need huge amounts of memory -caches.trace = false -caches.tree = false -caches.paths = caches.paths or nil -caches.force = false - -input.usecache = not toboolean(os.getenv("TEXMFSHARECACHE") or "false",true) -- true - -function caches.temp(instance) - local function checkpath(cachepath) - if not cachepath or cachepath == "" then - return nil - elseif lfs.attributes(cachepath,"mode") == "directory" then -- lfs.isdir(cachepath) then - return cachepath - elseif caches.force or io.ask(string.format("Should I create the cache path %s?",cachepath), "no", { "yes", "no" }) == "yes" then - dir.mkdirs(cachepath) - return (lfs.attributes(cachepath,"mode") == "directory") and cachepath - else - return nil + +caches.path = caches.path or nil +caches.base = caches.base or "luatex-cache" +caches.more = caches.more or "context" +caches.direct = false -- true is faster but may need huge amounts of memory +caches.tree = false +caches.paths = caches.paths or nil +caches.force = false +caches.defaults = { "TEXMFCACHE", "TMPDIR", "TEMPDIR", "TMP", "TEMP", "HOME", "HOMEPATH" } + +function caches.temp() + local cachepath = nil + local function check(list,isenv) + if not cachepath then + for k=1,#list do + local v = list[k] + cachepath = (isenv and (os.env[v] or "")) or v or "" + if cachepath == "" then + -- next + else + cachepath = resolvers.clean_path(cachepath) + if lfs.isdir(cachepath) and file.iswritable(cachepath) then -- lfs.attributes(cachepath,"mode") == "directory" + break + elseif caches.force or io.ask(format("\nShould I create the cache path %s?",cachepath), "no", { "yes", "no" }) == "yes" then + dir.mkdirs(cachepath) + if lfs.isdir(cachepath) and file.iswritable(cachepath) then + break + end + end + end + cachepath = nil + end end end - local cachepath = input.expanded_path_list(instance,"TEXMFCACHE") - cachepath = cachepath and #cachepath > 0 and checkpath(cachepath[1]) + check(resolvers.clean_path_list("TEXMFCACHE") or { }) + check(caches.defaults,true) if not cachepath then - cachepath = os.getenv("TEXMFCACHE") or os.getenv("HOME") or os.getenv("HOMEPATH") or os.getenv("TMP") or os.getenv("TEMP") or os.getenv("TMPDIR") or nil - cachepath = checkpath(cachepath) - end - if not cachepath then - print("\nfatal error: there is no valid cache path defined\n") + print("\nfatal error: there is no valid (writable) cache path defined\n") os.exit() - elseif lfs.attributes(cachepath,"mode") ~= "directory" then - print(string.format("\nfatal error: cache path %s is not a directory\n",cachepath)) + elseif not lfs.isdir(cachepath) then -- lfs.attributes(cachepath,"mode") ~= "directory" + print(format("\nfatal error: cache path %s is not a directory\n",cachepath)) os.exit() end - function caches.temp(instance) + cachepath = file.collapse_path(cachepath) + function caches.temp() return cachepath end return cachepath end -function caches.configpath(instance) - return table.concat(instance.cnffiles,";") +function caches.configpath() + return table.concat(resolvers.instance.cnffiles,";") end function caches.hashed(tree) - return md5.hex((tree:lower()):gsub("[\\\/]+","/")) + return md5.hex(gsub(lower(tree),"[\\\/]+","/")) end -function caches.treehash(instance) - local tree = caches.configpath(instance) +function caches.treehash() + local tree = caches.configpath() if not tree or tree == "" then return false else @@ -7120,26 +10386,25 @@ function caches.treehash(instance) end end -function caches.setpath(instance,...) +function caches.setpath(...) if not caches.path then if not caches.path then - caches.path = caches.temp(instance) + caches.path = caches.temp() end - caches.path = input.clean_path(caches.path) -- to be sure - if lfs then - caches.tree = caches.tree or caches.treehash(instance) - if caches.tree then - caches.path = dir.mkdirs(caches.path,caches.base,caches.more,caches.tree) - else - caches.path = dir.mkdirs(caches.path,caches.base,caches.more) - end + caches.path = resolvers.clean_path(caches.path) -- to be sure + caches.tree = caches.tree or caches.treehash() + if caches.tree then + caches.path = dir.mkdirs(caches.path,caches.base,caches.more,caches.tree) + else + caches.path = dir.mkdirs(caches.path,caches.base,caches.more) end end if not caches.path then caches.path = '.' end - caches.path = input.clean_path(caches.path) - if lfs and not table.is_empty({...}) then + caches.path = resolvers.clean_path(caches.path) + local dirs = { ... } + if #dirs > 0 then local pth = dir.mkdirs(caches.path,...) return pth end @@ -7147,9 +10412,9 @@ function caches.setpath(instance,...) return caches.path end -function caches.definepath(instance,category,subcategory) +function caches.definepath(category,subcategory) return function() - return caches.setpath(instance,category,subcategory) + return caches.setpath(category,subcategory) end end @@ -7161,39 +10426,220 @@ function caches.loaddata(path,name) local tmaname, tmcname = caches.setluanames(path,name) local loader = loadfile(tmcname) or loadfile(tmaname) if loader then - return loader() + loader = loader() + collectgarbage("step") + return loader else return false end end -function caches.is_writable(filepath,filename) +--~ function caches.loaddata(path,name) +--~ local tmaname, tmcname = caches.setluanames(path,name) +--~ return dofile(tmcname) or dofile(tmaname) +--~ end + +function caches.iswritable(filepath,filename) local tmaname, tmcname = caches.setluanames(filepath,filename) - return file.is_writable(tmaname) + return file.iswritable(tmaname) end -function caches.savedata(filepath,filename,data,raw) -- raw needed for file cache +function caches.savedata(filepath,filename,data,raw) local tmaname, tmcname = caches.setluanames(filepath,filename) local reduce, simplify = true, true if raw then reduce, simplify = false, false end + data.cache_uuid = os.uuid() if caches.direct then - file.savedata(tmaname, table.serialize(data,'return',true,true)) + file.savedata(tmaname, table.serialize(data,'return',false,true,false)) -- no hex else - table.tofile (tmaname, data,'return',true,true) -- maybe not the last true + table.tofile(tmaname, data,'return',false,true,false) -- maybe not the last true end - utils.lua.compile(tmaname, tmcname) + local cleanup = resolvers.boolean_variable("PURGECACHE", false) + local strip = resolvers.boolean_variable("LUACSTRIP", true) + utils.lua.compile(tmaname, tmcname, cleanup, strip) end -- here we use the cache for format loading (texconfig.[formatname|jobname]) --~ if tex and texconfig and texconfig.formatname and texconfig.formatname == "" then -if tex and texconfig and (not texconfig.formatname or texconfig.formatname == "") and texmf.instance then +if tex and texconfig and (not texconfig.formatname or texconfig.formatname == "") and input and resolvers.instance then if not texconfig.luaname then texconfig.luaname = "cont-en.lua" end -- or luc - texconfig.formatname = caches.setpath(texmf.instance,"formats") .. "/" .. texconfig.luaname:gsub("%.lu.$",".fmt") + texconfig.formatname = caches.setpath("formats") .. "/" .. gsub(texconfig.luaname,"%.lu.$",".fmt") +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['data-res'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +--~ print(resolvers.resolve("abc env:tmp file:cont-en.tex path:cont-en.tex full:cont-en.tex rel:zapf/one/p-chars.tex")) + +local upper, lower, gsub = string.upper, string.lower, string.gsub + +local prefixes = { } + +prefixes.environment = function(str) + return resolvers.clean_path(os.getenv(str) or os.getenv(upper(str)) or os.getenv(lower(str)) or "") end +prefixes.relative = function(str,n) + if io.exists(str) then + -- nothing + elseif io.exists("./" .. str) then + str = "./" .. str + else + local p = "../" + for i=1,n or 2 do + if io.exists(p .. str) then + str = p .. str + break + else + p = p .. "../" + end + end + end + return resolvers.clean_path(str) +end + +prefixes.auto = function(str) + local fullname = prefixes.relative(str) + if not lfs.isfile(fullname) then + fullname = prefixes.locate(str) + end + return fullname +end + +prefixes.locate = function(str) + local fullname = resolvers.find_given_file(str) or "" + return resolvers.clean_path((fullname ~= "" and fullname) or str) +end + +prefixes.filename = function(str) + local fullname = resolvers.find_given_file(str) or "" + return resolvers.clean_path(file.basename((fullname ~= "" and fullname) or str)) +end + +prefixes.pathname = function(str) + local fullname = resolvers.find_given_file(str) or "" + return resolvers.clean_path(file.dirname((fullname ~= "" and fullname) or str)) +end + +prefixes.env = prefixes.environment +prefixes.rel = prefixes.relative +prefixes.loc = prefixes.locate +prefixes.kpse = prefixes.locate +prefixes.full = prefixes.locate +prefixes.file = prefixes.filename +prefixes.path = prefixes.pathname + +function resolvers.allprefixes(separator) + local all = table.sortedkeys(prefixes) + if separator then + for i=1,#all do + all[i] = all[i] .. ":" + end + end + return all +end + +local function _resolve_(method,target) + if prefixes[method] then + return prefixes[method](target) + else + return method .. ":" .. target + end +end + +local function resolve(str) + if type(str) == "table" then + for k=1,#str do + local v = str[k] + str[k] = resolve(v) or v + end + elseif str and str ~= "" then + str = gsub(str,"([a-z]+):([^ \"\']*)",_resolve_) + end + return str +end + +resolvers.resolve = resolve + +if os.uname then + + for k, v in next, os.uname() do + if not prefixes[k] then + prefixes[k] = function() return v end + end + end + +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['data-inp'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +resolvers.finders = resolvers.finders or { } +resolvers.openers = resolvers.openers or { } +resolvers.loaders = resolvers.loaders or { } + +resolvers.finders.notfound = { nil } +resolvers.openers.notfound = { nil } +resolvers.loaders.notfound = { false, nil, 0 } + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['data-out'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +outputs = outputs or { } + + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['data-con'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local format, lower, gsub = string.format, string.lower, string.gsub + +local trace_cache = false trackers.register("resolvers.cache", function(v) trace_cache = v end) +local trace_containers = false trackers.register("resolvers.containers", function(v) trace_containers = v end) +local trace_storage = false trackers.register("resolvers.storage", function(v) trace_storage = v end) + --[[ldx-- <p>Once we found ourselves defining similar cache constructs several times, containers were introduced. Containers are used @@ -7207,124 +10653,144 @@ table structures without bothering about the disk cache.</p> <p>Examples of usage can be found in the font related code.</p> --ldx]]-- -containers = { } -containers.trace = false +containers = containers or { } -do -- local report +containers.usecache = true - local function report(container,tag,name) - if caches.trace or containers.trace or container.trace then - logs.report(string.format("%s cache",container.subcategory),string.format("%s: %s",tag,name or 'invalid')) - end +local function report(container,tag,name) + if trace_cache or trace_containers then + logs.report(format("%s cache",container.subcategory),"%s: %s",tag,name or 'invalid') end +end - local allocated = { } +local allocated = { } - -- tracing +-- tracing - function containers.define(category, subcategory, version, enabled) - return function() - if category and subcategory then - local c = allocated[category] - if not c then - c = { } - allocated[category] = c - end - local s = c[subcategory] - if not s then - s = { - category = category, - subcategory = subcategory, - storage = { }, - enabled = enabled, - version = version or 1.000, - trace = false, - path = caches.setpath(texmf.instance,category,subcategory), - } - c[subcategory] = s - end - return s - else - return nil - end +function containers.define(category, subcategory, version, enabled) + return function() + if category and subcategory then + local c = allocated[category] + if not c then + c = { } + allocated[category] = c + end + local s = c[subcategory] + if not s then + s = { + category = category, + subcategory = subcategory, + storage = { }, + enabled = enabled, + version = version or 1.000, + trace = false, + path = caches and caches.setpath and caches.setpath(category,subcategory), + } + c[subcategory] = s + end + return s + else + return nil end end +end + +function containers.is_usable(container, name) + return container.enabled and caches and caches.iswritable(container.path, name) +end - function containers.is_usable(container, name) - return container.enabled and caches.is_writable(container.path, name) +function containers.is_valid(container, name) + if name and name ~= "" then + local storage = container.storage[name] + return storage and storage.cache_version == container.version + else + return false end +end - function containers.is_valid(container, name) - if name and name ~= "" then - local storage = container.storage[name] - return storage and not table.is_empty(storage) and storage.cache_version == container.version +function containers.read(container,name) + if container.enabled and caches and not container.storage[name] and containers.usecache then + container.storage[name] = caches.loaddata(container.path,name) + if containers.is_valid(container,name) then + report(container,"loaded",name) else - return false + container.storage[name] = nil end end - - function containers.read(container,name) - if container.enabled and not container.storage[name] then - container.storage[name] = caches.loaddata(container.path,name) - if containers.is_valid(container,name) then - report(container,"loaded",name) - else - container.storage[name] = nil - end - end - if container.storage[name] then - report(container,"reusing",name) - end - return container.storage[name] + if container.storage[name] then + report(container,"reusing",name) end + return container.storage[name] +end - function containers.write(container, name, data) - if data then - data.cache_version = container.version - if container.enabled then - local unique, shared = data.unique, data.shared - data.unique, data.shared = nil, nil - caches.savedata(container.path, name, data) - report(container,"saved",name) - data.unique, data.shared = unique, shared - end - report(container,"stored",name) - container.storage[name] = data +function containers.write(container, name, data) + if data then + data.cache_version = container.version + if container.enabled and caches then + local unique, shared = data.unique, data.shared + data.unique, data.shared = nil, nil + caches.savedata(container.path, name, data) + report(container,"saved",name) + data.unique, data.shared = unique, shared end - return data + report(container,"stored",name) + container.storage[name] = data end + return data +end - function containers.content(container,name) - return container.storage[name] - end +function containers.content(container,name) + return container.storage[name] +end +function containers.cleanname(name) + return (gsub(lower(name),"[^%w%d]+","-")) end + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['data-use'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local format, lower, gsub, find = string.format, string.lower, string.gsub, string.find + +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) + -- since we want to use the cache instead of the tree, we will now -- reimplement the saver. -local save_data = input.aux.save_data +local save_data = resolvers.save_data +local load_data = resolvers.load_data -input.cachepath = nil +resolvers.cachepath = nil -- public, for tracing +resolvers.usecache = true -- public, for tracing -function input.aux.save_data(instance, dataname, check) - input.cachepath = input.cachepath or caches.definepath(instance,"trees") - save_data(instance, dataname, check, function(cachename,dataname) - if input.usecache then - return file.join(input.cachepath(),caches.hashed(cachename)) +function resolvers.save_data(dataname) + save_data(dataname, function(cachename,dataname) + resolvers.usecache = not toboolean(resolvers.expansion("CACHEINTDS") or "false",true) + if resolvers.usecache then + resolvers.cachepath = resolvers.cachepath or caches.definepath("trees") + return file.join(resolvers.cachepath(),caches.hashed(cachename)) else return file.join(cachename,dataname) end end) end -local load_data = input.aux.load_data - -function input.aux.load_data(instance,pathname,dataname,filename) - input.cachepath = input.cachepath or caches.definepath(instance,"trees") - load_data(instance,pathname,dataname,filename,function(dataname,filename) - if input.usecache then - return file.join(input.cachepath(),caches.hashed(pathname)) +function resolvers.load_data(pathname,dataname,filename) + load_data(pathname,dataname,filename,function(dataname,filename) + resolvers.usecache = not toboolean(resolvers.expansion("CACHEINTDS") or "false",true) + if resolvers.usecache then + resolvers.cachepath = resolvers.cachepath or caches.definepath("trees") + return file.join(resolvers.cachepath(),caches.hashed(pathname)) else if not filename or (filename == "") then filename = dataname @@ -7336,241 +10802,812 @@ end -- we will make a better format, maybe something xml or just text or lua -input.automounted = input.automounted or { } +resolvers.automounted = resolvers.automounted or { } -function input.automount(instance,usecache) - local mountpaths = input.simplified_list(input.expansion(instance,'TEXMFMOUNT')) - if table.is_empty(mountpaths) and usecache then - mountpaths = { caches.setpath(instance,"mount") } +function resolvers.automount(usecache) + local mountpaths = resolvers.clean_path_list(resolvers.expansion('TEXMFMOUNT')) + if (not mountpaths or #mountpaths == 0) and usecache then + mountpaths = { caches.setpath("mount") } end - if not table.is_empty(mountpaths) then - input.starttiming(instance) - for k, root in pairs(mountpaths) do + if mountpaths and #mountpaths > 0 then + statistics.starttiming(resolvers.instance) + for k=1,#mountpaths do + local root = mountpaths[k] local f = io.open(root.."/url.tmi") if f then for line in f:lines() do if line then - if line:find("^[%%#%-]") then -- or %W + if find(line,"^[%%#%-]") then -- or %W -- skip - elseif line:find("^zip://") then - input.report("mounting",line) - table.insert(input.automounted,line) - input.usezipfile(instance,line) + elseif find(line,"^zip://") then + if trace_locating then + logs.report("fileio","mounting %s",line) + end + table.insert(resolvers.automounted,line) + resolvers.usezipfile(line) end end end f:close() end end - input.stoptiming(instance) + statistics.stoptiming(resolvers.instance) end end --- store info in format +-- status info + +statistics.register("used config path", function() return caches.configpath() end) +statistics.register("used cache path", function() return caches.temp() or "?" end) -input.storage = { } -input.storage.data = { } -input.storage.min = 0 -- 500 -input.storage.max = input.storage.min - 1 -input.storage.trace = false -- true -input.storage.done = 0 -input.storage.evaluators = { } --- (evaluate,message,names) +-- experiment (code will move) -function input.storage.register(...) - input.storage.data[#input.storage.data+1] = { ... } +function statistics.save_fmt_status(texname,formatbanner,sourcefile) -- texname == formatname + local enginebanner = status.list().banner + if formatbanner and enginebanner and sourcefile then + local luvname = file.replacesuffix(texname,"luv") + local luvdata = { + enginebanner = enginebanner, + formatbanner = formatbanner, + sourcehash = md5.hex(io.loaddata(resolvers.find_file(sourcefile)) or "unknown"), + sourcefile = sourcefile, + } + io.savedata(luvname,table.serialize(luvdata,true)) + end end -function input.storage.evaluate(name) - input.storage.evaluators[#input.storage.evaluators+1] = name +function statistics.check_fmt_status(texname) + local enginebanner = status.list().banner + if enginebanner and texname then + local luvname = file.replacesuffix(texname,"luv") + if lfs.isfile(luvname) then + local luv = dofile(luvname) + if luv and luv.sourcefile then + local sourcehash = md5.hex(io.loaddata(resolvers.find_file(luv.sourcefile)) or "unknown") + local luvbanner = luv.enginebanner or "?" + if luvbanner ~= enginebanner then + return string.format("engine mismatch (luv:%s <> bin:%s)",luvbanner,enginebanner) + end + local luvhash = luv.sourcehash or "?" + if luvhash ~= sourcehash then + return string.format("source mismatch (luv:%s <> bin:%s)",luvhash,sourcehash) + end + else + return "invalid status file" + end + else + return "missing status file" + end + end + return true end -function input.storage.finalize() -- we can prepend the string with "evaluate:" - for _, t in ipairs(input.storage.evaluators) do - for i, v in pairs(t) do - if type(v) == "string" then - t[i] = loadstring(v)() - elseif type(v) == "table" then - for _, vv in pairs(v) do - if type(vv) == "string" then - t[i] = loadstring(vv)() + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['data-zip'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local format, find, match = string.format, string.find, string.match +local unpack = unpack or table.unpack + +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) + +-- zip:///oeps.zip?name=bla/bla.tex +-- zip:///oeps.zip?tree=tex/texmf-local +-- zip:///texmf.zip?tree=/tex/texmf +-- zip:///texmf.zip?tree=/tex/texmf-local +-- zip:///texmf-mine.zip?tree=/tex/texmf-projects + +zip = zip or { } +zip.archives = zip.archives or { } +zip.registeredfiles = zip.registeredfiles or { } + +local finders, openers, loaders = resolvers.finders, resolvers.openers, resolvers.loaders +local locators, hashers, concatinators = resolvers.locators, resolvers.hashers, resolvers.concatinators + +local archives = zip.archives + +local function validzip(str) -- todo: use url splitter + if not find(str,"^zip://") then + return "zip:///" .. str + else + return str + end +end + +function zip.openarchive(name) + if not name or name == "" then + return nil + else + local arch = archives[name] + if not arch then + local full = resolvers.find_file(name) or "" + arch = (full ~= "" and zip.open(full)) or false + archives[name] = arch + end + return arch + end +end + +function zip.closearchive(name) + if not name or (name == "" and archives[name]) then + zip.close(archives[name]) + archives[name] = nil + end +end + +function locators.zip(specification) -- where is this used? startup zips (untested) + specification = resolvers.splitmethod(specification) + local zipfile = specification.path + local zfile = zip.openarchive(name) -- tricky, could be in to be initialized tree + if trace_locating then + if zfile then + logs.report("fileio","zip locator, archive '%s' found",specification.original) + else + logs.report("fileio","zip locator, archive '%s' not found",specification.original) + end + end +end + +function hashers.zip(tag,name) + if trace_locating then + logs.report("fileio","loading zip file '%s' as '%s'",name,tag) + end + resolvers.usezipfile(format("%s?tree=%s",tag,name)) +end + +function concatinators.zip(tag,path,name) + if not path or path == "" then + return format('%s?name=%s',tag,name) + else + return format('%s?name=%s/%s',tag,path,name) + end +end + +function resolvers.isreadable.zip(name) + return true +end + +function finders.zip(specification,filetype) + specification = resolvers.splitmethod(specification) + if specification.path then + local q = url.query(specification.query) + if q.name then + local zfile = zip.openarchive(specification.path) + if zfile then + if trace_locating then + logs.report("fileio","zip finder, archive '%s' found",specification.path) + end + local dfile = zfile:open(q.name) + if dfile then + dfile = zfile:close() + if trace_locating then + logs.report("fileio","zip finder, file '%s' found",q.name) end + return specification.original + elseif trace_locating then + logs.report("fileio","zip finder, file '%s' not found",q.name) end + elseif trace_locating then + logs.report("fileio","zip finder, unknown archive '%s'",specification.path) end end end + if trace_locating then + logs.report("fileio","zip finder, '%s' not found",filename) + end + return unpack(finders.notfound) end -function input.storage.dump() - for name, data in ipairs(input.storage.data) do - local evaluate, message, original, target = data[1], data[2], data[3] ,data[4] - local name, initialize, finalize, code = nil, "", "", "" - for str in target:gmatch("([^%.]+)") do - if name then - name = name .. "." .. str - else - name = str +function openers.zip(specification) + local zipspecification = resolvers.splitmethod(specification) + if zipspecification.path then + local q = url.query(zipspecification.query) + if q.name then + local zfile = zip.openarchive(zipspecification.path) + if zfile then + if trace_locating then + logs.report("fileio","zip opener, archive '%s' opened",zipspecification.path) + end + local dfile = zfile:open(q.name) + if dfile then + logs.show_open(specification) + if trace_locating then + logs.report("fileio","zip opener, file '%s' found",q.name) + end + return openers.text_opener(specification,dfile,'zip') + elseif trace_locating then + logs.report("fileio","zip opener, file '%s' not found",q.name) + end + elseif trace_locating then + logs.report("fileio","zip opener, unknown archive '%s'",zipspecification.path) end - initialize = string.format("%s %s = %s or {} ", initialize, name, name) end - if evaluate then - finalize = "input.storage.evaluate(" .. name .. ")" + end + if trace_locating then + logs.report("fileio","zip opener, '%s' not found",filename) + end + return unpack(openers.notfound) +end + +function loaders.zip(specification) + specification = resolvers.splitmethod(specification) + if specification.path then + local q = url.query(specification.query) + if q.name then + local zfile = zip.openarchive(specification.path) + if zfile then + if trace_locating then + logs.report("fileio","zip loader, archive '%s' opened",specification.path) + end + local dfile = zfile:open(q.name) + if dfile then + logs.show_load(filename) + if trace_locating then + logs.report("fileio","zip loader, file '%s' loaded",filename) + end + local s = dfile:read("*all") + dfile:close() + return true, s, #s + elseif trace_locating then + logs.report("fileio","zip loader, file '%s' not found",q.name) + end + elseif trace_locating then + logs.report("fileio","zip loader, unknown archive '%s'",specification.path) + end end - input.storage.max = input.storage.max + 1 - if input.storage.trace then - logs.report('storage',string.format('saving %s in slot %s',message,input.storage.max)) - code = - initialize .. - string.format("logs.report('storage','restoring %s from slot %s') ",message,input.storage.max) .. - table.serialize(original,name) .. - finalize - else - code = initialize .. table.serialize(original,name) .. finalize + end + if trace_locating then + logs.report("fileio","zip loader, '%s' not found",filename) + end + return unpack(openers.notfound) +end + +-- zip:///somefile.zip +-- zip:///somefile.zip?tree=texmf-local -> mount + +function resolvers.usezipfile(zipname) + zipname = validzip(zipname) + local specification = resolvers.splitmethod(zipname) + local zipfile = specification.path + if zipfile and not zip.registeredfiles[zipname] then + local tree = url.query(specification.query).tree or "" + local z = zip.openarchive(zipfile) + if z then + local instance = resolvers.instance + if trace_locating then + logs.report("fileio","zip registering, registering archive '%s'",zipname) + end + statistics.starttiming(instance) + resolvers.prepend_hash('zip',zipname,zipfile) + resolvers.extend_texmf_var(zipname) -- resets hashes too + zip.registeredfiles[zipname] = z + instance.files[zipname] = resolvers.register_zip_file(z,tree or "") + statistics.stoptiming(instance) + elseif trace_locating then + logs.report("fileio","zip registering, unknown archive '%s'",zipname) end - lua.bytecode[input.storage.max] = loadstring(code) + elseif trace_locating then + logs.report("fileio","zip registering, '%s' not found",zipname) end end -if lua.bytecode then -- from 0 upwards - local i = input.storage.min - while lua.bytecode[i] do - lua.bytecode[i]() - lua.bytecode[i] = nil - i = i + 1 +function resolvers.register_zip_file(z,tree) + local files, filter = { }, "" + if tree == "" then + filter = "^(.+)/(.-)$" + else + filter = format("^%s/(.+)/(.-)$",tree) + end + if trace_locating then + logs.report("fileio","zip registering, using filter '%s'",filter) end - input.storage.done = i + local register, n = resolvers.register_file, 0 + for i in z:files() do + local path, name = match(i.filename,filter) + if path then + if name and name ~= '' then + register(files, name, path) + n = n + 1 + else + -- directory + end + else + register(files, i.filename, '') + n = n + 1 + end + end + logs.report("fileio","zip registering, %s files registered",n) + return files end -if not modules then modules = { } end modules ['luat-log'] = { +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['data-crl'] = { version = 1.001, - comment = "companion to luat-lib.tex", + comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } ---[[ldx-- -<p>This is a prelude to a more extensive logging module. For the sake -of parsing log files, in addition to the standard logging we will -provide an <l n='xml'/> structured file. Actually, any logging that -is hooked into callbacks will be \XML\ by default.</p> ---ldx]]-- +local gsub = string.gsub -input = input or { } -logs = logs or { } +curl = curl or { } ---[[ldx-- -<p>This looks pretty ugly but we need to speed things up a bit.</p> ---ldx]]-- +curl.cached = { } +curl.cachepath = caches.definepath("curl") -logs.levels = { - ['error'] = 1, - ['warning'] = 2, - ['info'] = 3, - ['debug'] = 4 -} +local finders, openers, loaders = resolvers.finders, resolvers.openers, resolvers.loaders -logs.functions = { - 'error', 'warning', 'info', 'debug', 'report', - 'start', 'stop', 'push', 'pop' -} +function curl.fetch(protocol, name) + local cachename = curl.cachepath() .. "/" .. gsub(name,"[^%a%d%.]+","-") +-- cachename = gsub(cachename,"[\\/]", io.fileseparator) + cachename = gsub(cachename,"[\\]", "/") -- cleanup + if not curl.cached[name] then + if not io.exists(cachename) then + curl.cached[name] = cachename + local command = "curl --silent --create-dirs --output " .. cachename .. " " .. name -- no protocol .. "://" + os.spawn(command) + end + if io.exists(cachename) then + curl.cached[name] = cachename + else + curl.cached[name] = "" + end + end + return curl.cached[name] +end + +function finders.curl(protocol,filename) + local foundname = curl.fetch(protocol, filename) + return finders.generic(protocol,foundname,filetype) +end + +function openers.curl(protocol,filename) + return openers.generic(protocol,filename) +end + +function loaders.curl(protocol,filename) + return loaders.generic(protocol,filename) +end + +-- todo: metamethod + +function curl.install(protocol) + finders[protocol] = function (filename,filetype) return finders.curl(protocol,filename) end + openers[protocol] = function (filename) return openers.curl(protocol,filename) end + loaders[protocol] = function (filename) return loaders.curl(protocol,filename) end +end + +curl.install('http') +curl.install('https') +curl.install('ftp') + + +end -- of closure + +do -- create closure to overcome 200 locals limit -logs.callbacks = { - 'start_page_number', - 'stop_page_number', - 'report_output_pages', - 'report_output_log' +if not modules then modules = { } end modules ['data-lua'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" } -logs.xml = logs.xml or { } -logs.tex = logs.tex or { } +-- some loading stuff ... we might move this one to slot 2 depending +-- on the developments (the loaders must not trigger kpse); we could +-- of course use a more extensive lib path spec -logs.level = 0 +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) -do - local write_nl, write, format = texio.write_nl or print, texio.write or io.write, string.format +local gsub, insert = string.gsub, table.insert +local unpack = unpack or table.unpack - if texlua then - write_nl = print - write = io.write - end +local libformats = { 'luatexlibs', 'tex', 'texmfscripts', 'othertextfiles' } -- 'luainputs' +local clibformats = { 'lib' } - function logs.xml.debug(category,str) - if logs.level > 3 then write_nl(format("<d category='%s'>%s</d>",category,str)) end - end - function logs.xml.info(category,str) - if logs.level > 2 then write_nl(format("<i category='%s'>%s</i>",category,str)) end +local _path_, libpaths, _cpath_, clibpaths + +function package.libpaths() + if not _path_ or package.path ~= _path_ then + _path_ = package.path + libpaths = file.split_path(_path_,";") end - function logs.xml.warning(category,str) - if logs.level > 1 then write_nl(format("<w category='%s'>%s</w>",category,str)) end + return libpaths +end + +function package.clibpaths() + if not _cpath_ or package.cpath ~= _cpath_ then + _cpath_ = package.cpath + clibpaths = file.split_path(_cpath_,";") end - function logs.xml.error(category,str) - if logs.level > 0 then write_nl(format("<e category='%s'>%s</e>",category,str)) end + return clibpaths +end + +local function thepath(...) + local t = { ... } t[#t+1] = "?.lua" + local path = file.join(unpack(t)) + if trace_locating then + logs.report("fileio","! appending '%s' to 'package.path'",path) end - function logs.xml.report(category,str) - write_nl(format("<r category='%s'>%s</r>",category,str)) + return path +end + +local p_libpaths, a_libpaths = { }, { } + +function package.append_libpath(...) + insert(a_libpath,thepath(...)) +end + +function package.prepend_libpath(...) + insert(p_libpaths,1,thepath(...)) +end + +-- beware, we need to return a loadfile result ! + +local function loaded(libpaths,name,simple) + for i=1,#libpaths do -- package.path, might become option + local libpath = libpaths[i] + local resolved = gsub(libpath,"%?",simple) + if trace_locating then -- more detail + logs.report("fileio","! checking for '%s' on 'package.path': '%s' => '%s'",simple,libpath,resolved) + end + if resolvers.isreadable.file(resolved) then + if trace_locating then + logs.report("fileio","! lib '%s' located via 'package.path': '%s'",name,resolved) + end + return loadfile(resolved) + end end +end - function logs.xml.start() if logs.level > 0 then tw("<%s>" ) end end - function logs.xml.stop () if logs.level > 0 then tw("</%s>") end end - function logs.xml.push () if logs.level > 0 then tw("<!-- ") end end - function logs.xml.pop () if logs.level > 0 then tw(" -->" ) end end - function logs.tex.debug(category,str) - if logs.level > 3 then write_nl(format("debug >> %s: %s" ,category,str)) end +package.loaders[2] = function(name) -- was [#package.loaders+1] + if trace_locating then -- mode detail + logs.report("fileio","! locating '%s'",name) + end + for i=1,#libformats do + local format = libformats[i] + local resolved = resolvers.find_file(name,format) or "" + if trace_locating then -- mode detail + logs.report("fileio","! checking for '%s' using 'libformat path': '%s'",name,format) + end + if resolved ~= "" then + if trace_locating then + logs.report("fileio","! lib '%s' located via environment: '%s'",name,resolved) + end + return loadfile(resolved) + end end - function logs.tex.info(category,str) - if logs.level > 2 then write_nl(format("info >> %s: %s" ,category,str)) end + -- libpaths + local libpaths, clibpaths = package.libpaths(), package.clibpaths() + local simple = gsub(name,"%.lua$","") + local simple = gsub(simple,"%.","/") + local resolved = loaded(p_libpaths,name,simple) or loaded(libpaths,name,simple) or loaded(a_libpaths,name,simple) + if resolved then + return resolved end - function logs.tex.warning(category,str) - if logs.level > 1 then write_nl(format("warning >> %s: %s",category,str)) end + -- + local libname = file.addsuffix(simple,os.libsuffix) + for i=1,#clibformats do + -- better have a dedicated loop + local format = clibformats[i] + local paths = resolvers.expanded_path_list_from_var(format) + for p=1,#paths do + local path = paths[p] + local resolved = file.join(path,libname) + if trace_locating then -- mode detail + logs.report("fileio","! checking for '%s' using 'clibformat path': '%s'",libname,path) + end + if resolvers.isreadable.file(resolved) then + if trace_locating then + logs.report("fileio","! lib '%s' located via 'clibformat': '%s'",libname,resolved) + end + return package.loadlib(resolved,name) + end + end end - function logs.tex.error(category,str) - if logs.level > 0 then write_nl(format("error >> %s: %s" ,category,str)) end + for i=1,#clibpaths do -- package.path, might become option + local libpath = clibpaths[i] + local resolved = gsub(libpath,"?",simple) + if trace_locating then -- more detail + logs.report("fileio","! checking for '%s' on 'package.cpath': '%s'",simple,libpath) + end + if resolvers.isreadable.file(resolved) then + if trace_locating then + logs.report("fileio","! lib '%s' located via 'package.cpath': '%s'",name,resolved) + end + return package.loadlib(resolved,name) + end end - function logs.tex.report(category,str) - write_nl(format("report >> %s: %s" ,category,str)) + -- just in case the distribution is messed up + if trace_loading then -- more detail + logs.report("fileio","! checking for '%s' using 'luatexlibs': '%s'",name) end - - function logs.set_level(level) - logs.level = logs.levels[level] or level + local resolved = resolvers.find_file(file.basename(name),'luatexlibs') or "" + if resolved ~= "" then + if trace_locating then + logs.report("fileio","! lib '%s' located by basename via environment: '%s'",name,resolved) + end + return loadfile(resolved) + end + if trace_locating then + logs.report("fileio",'? unable to locate lib: %s',name) end +-- return "unable to locate " .. name +end + +resolvers.loadlualib = require - function logs.set_method(method) - for _, v in pairs(logs.functions) do - logs[v] = logs[method][v] or function() end + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['luat-kps'] = { + version = 1.001, + comment = "companion to luatools.lua", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +--[[ldx-- +<p>This file is used when we want the input handlers to behave like +<type>kpsewhich</type>. What to do with the following:</p> + +<typing> +{$SELFAUTOLOC,$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,}/web2c} +$SELFAUTOLOC : /usr/tex/bin/platform +$SELFAUTODIR : /usr/tex/bin +$SELFAUTOPARENT : /usr/tex +</typing> + +<p>How about just forgetting about them?</p> +--ldx]]-- + +local suffixes = resolvers.suffixes +local formats = resolvers.formats + +suffixes['gf'] = { '<resolution>gf' } +suffixes['pk'] = { '<resolution>pk' } +suffixes['base'] = { 'base' } +suffixes['bib'] = { 'bib' } +suffixes['bst'] = { 'bst' } +suffixes['cnf'] = { 'cnf' } +suffixes['mem'] = { 'mem' } +suffixes['mf'] = { 'mf' } +suffixes['mfpool'] = { 'pool' } +suffixes['mft'] = { 'mft' } +suffixes['mppool'] = { 'pool' } +suffixes['graphic/figure'] = { 'eps', 'epsi' } +suffixes['texpool'] = { 'pool' } +suffixes['PostScript header'] = { 'pro' } +suffixes['ist'] = { 'ist' } +suffixes['web'] = { 'web', 'ch' } +suffixes['cweb'] = { 'w', 'web', 'ch' } +suffixes['cmap files'] = { 'cmap' } +suffixes['lig files'] = { 'lig' } +suffixes['bitmap font'] = { } +suffixes['MetaPost support'] = { } +suffixes['TeX system documentation'] = { } +suffixes['TeX system sources'] = { } +suffixes['dvips config'] = { } +suffixes['type42 fonts'] = { } +suffixes['web2c files'] = { } +suffixes['other text files'] = { } +suffixes['other binary files'] = { } +suffixes['opentype fonts'] = { 'otf' } + +suffixes['fmt'] = { 'fmt' } +suffixes['texmfscripts'] = { 'rb','lua','py','pl' } + +suffixes['pdftex config'] = { } +suffixes['Troff fonts'] = { } + +suffixes['ls-R'] = { } + +--[[ldx-- +<p>If you wondered abou tsome of the previous mappings, how about +the next bunch:</p> +--ldx]]-- + +formats['bib'] = '' +formats['bst'] = '' +formats['mft'] = '' +formats['ist'] = '' +formats['web'] = '' +formats['cweb'] = '' +formats['MetaPost support'] = '' +formats['TeX system documentation'] = '' +formats['TeX system sources'] = '' +formats['Troff fonts'] = '' +formats['dvips config'] = '' +formats['graphic/figure'] = '' +formats['ls-R'] = '' +formats['other text files'] = '' +formats['other binary files'] = '' + +formats['gf'] = '' +formats['pk'] = '' +formats['base'] = 'MFBASES' +formats['cnf'] = '' +formats['mem'] = 'MPMEMS' +formats['mf'] = 'MFINPUTS' +formats['mfpool'] = 'MFPOOL' +formats['mppool'] = 'MPPOOL' +formats['texpool'] = 'TEXPOOL' +formats['PostScript header'] = 'TEXPSHEADERS' +formats['cmap files'] = 'CMAPFONTS' +formats['type42 fonts'] = 'T42FONTS' +formats['web2c files'] = 'WEB2C' +formats['pdftex config'] = 'PDFTEXCONFIG' +formats['texmfscripts'] = 'TEXMFSCRIPTS' +formats['bitmap font'] = '' +formats['lig files'] = 'LIGFONTS' + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['data-aux'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local find = string.find + +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) + +function resolvers.update_script(oldname,newname) -- oldname -> own.name, not per se a suffix + local scriptpath = "scripts/context/lua" + newname = file.addsuffix(newname,"lua") + local oldscript = resolvers.clean_path(oldname) + if trace_locating then + logs.report("fileio","to be replaced old script %s", oldscript) + end + local newscripts = resolvers.find_files(newname) or { } + if #newscripts == 0 then + if trace_locating then + logs.report("fileio","unable to locate new script") end - if callback and input[method] then - for _, cb in pairs(logs.callbacks) do - callback.register(cb, input[method][cb]) + else + for i=1,#newscripts do + local newscript = resolvers.clean_path(newscripts[i]) + if trace_locating then + logs.report("fileio","checking new script %s", newscript) + end + if oldscript == newscript then + if trace_locating then + logs.report("fileio","old and new script are the same") + end + elseif not find(newscript,scriptpath) then + if trace_locating then + logs.report("fileio","new script should come from %s",scriptpath) + end + elseif not (find(oldscript,file.removesuffix(newname).."$") or find(oldscript,newname.."$")) then + if trace_locating then + logs.report("fileio","invalid new script name") + end + else + local newdata = io.loaddata(newscript) + if newdata then + if trace_locating then + logs.report("fileio","old script content replaced by new content") + end + io.savedata(oldscript,newdata) + break + elseif trace_locating then + logs.report("fileio","unable to load new script") + end end end end +end - function logs.xml.start_page_number() - write_nl(format("<p real='%s' page='%s' sub='%s'", tex.count[0], tex.count[1], tex.count[2])) - end - function logs.xml.stop_page_number() - write("/>") - write_nl("") - end +end -- of closure - function logs.xml.report_output_pages(p,b) - write_nl(format("<v k='pages' v='%s'/>", p)) - write_nl(format("<v k='bytes' v='%s'/>", b)) - write_nl("") - end +do -- create closure to overcome 200 locals limit - function logs.xml.report_output_log() +if not modules then modules = { } end modules ['data-tmf'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local find, gsub, match = string.find, string.gsub, string.match +local getenv, setenv = os.getenv, os.setenv + +-- loads *.tmf files in minimal tree roots (to be optimized and documented) + +function resolvers.check_environment(tree) + logs.simpleline() + setenv('TMP', getenv('TMP') or getenv('TEMP') or getenv('TMPDIR') or getenv('HOME')) + setenv('TEXOS', getenv('TEXOS') or ("texmf-" .. os.platform)) + setenv('TEXPATH', gsub(tree or "tex","\/+$",'')) + setenv('TEXMFOS', getenv('TEXPATH') .. "/" .. getenv('TEXOS')) + logs.simpleline() + logs.simple("preset : TEXPATH => %s", getenv('TEXPATH')) + logs.simple("preset : TEXOS => %s", getenv('TEXOS')) + logs.simple("preset : TEXMFOS => %s", getenv('TEXMFOS')) + logs.simple("preset : TMP => %s", getenv('TMP')) + logs.simple('') +end + +function resolvers.load_environment(name) -- todo: key=value as well as lua + local f = io.open(name) + if f then + for line in f:lines() do + if find(line,"^[%%%#]") then + -- skip comment + else + local key, how, value = match(line,"^(.-)%s*([<=>%?]+)%s*(.*)%s*$") + if how then + value = gsub(value,"%%(.-)%%", function(v) return getenv(v) or "" end) + if how == "=" or how == "<<" then + setenv(key,value) + elseif how == "?" or how == "??" then + setenv(key,getenv(key) or value) + elseif how == "<" or how == "+=" then + if getenv(key) then + setenv(key,getenv(key) .. io.fileseparator .. value) + else + setenv(key,value) + end + elseif how == ">" or how == "=+" then + if getenv(key) then + setenv(key,value .. io.pathseparator .. getenv(key)) + else + setenv(key,value) + end + end + end + end + end + f:close() end +end +function resolvers.load_tree(tree) + if tree and tree ~= "" then + local setuptex = 'setuptex.tmf' + if lfs.attributes(tree, "mode") == "directory" then -- check if not nil + setuptex = tree .. "/" .. setuptex + else + setuptex = tree + end + if io.exists(setuptex) then + resolvers.check_environment(tree) + resolvers.load_environment(setuptex) + end + end end -logs.set_level('error') -logs.set_method('tex') +end -- of closure + +do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['luat-sta'] = { version = 1.001, @@ -7579,6 +11616,11 @@ if not modules then modules = { } end modules ['luat-sta'] = { license = "see context related readme files" } +-- this code is used in the updater + +local gmatch, match = string.gmatch, string.match +local type = type + states = states or { } states.data = states.data or { } states.hash = states.hash or { } @@ -7605,27 +11647,36 @@ end function states.set_by_tag(tag,key,value,default,persistent) local d, h = states.data[tag], states.hash[tag] if d then - local dkey, hkey = key, key - local pre, post = key:match("(.+)%.([^%.]+)$") - if pre and post then - for k in pre:gmatch("[^%.]+") do - local dk = d[k] - if not dk then - dk = { } - d[k] = dk + if type(d) == "table" then + local dkey, hkey = key, key + local pre, post = match(key,"(.+)%.([^%.]+)$") + if pre and post then + for k in gmatch(pre,"[^%.]+") do + local dk = d[k] + if not dk then + dk = { } + d[k] = dk + elseif type(dk) == "string" then + -- invalid table, unable to upgrade structure + -- hope for the best or delete the state file + break + end + d = dk end - d = dk + dkey, hkey = post, key end - dkey, hkey = post, key - end - if type(value) == nil then - value = value or default - elseif persistent then - value = value or d[dkey] or default - else - value = value or default + if type(value) == nil then + value = value or default + elseif persistent then + value = value or d[dkey] or default + else + value = value or default + end + d[dkey], h[hkey] = value, value + elseif type(d) == "string" then + -- weird + states.data[tag], states.hash[tag] = value, value end - d[dkey], h[hkey] = value, value end end @@ -7636,7 +11687,7 @@ function states.get_by_tag(tag,key,default) else local d = states.data[tag] if d then - for k in key:gmatch("[^%.]+") do + for k in gmatch(key,"[^%.]+") do local dk = d[k] if dk then d = dk @@ -7745,7 +11796,6 @@ end --~ }, --~ } - --~ states.save("teststate", "update") --~ states.load("teststate", "update") @@ -7756,34 +11806,58 @@ end --~ states.load("teststate", "update") --~ print(states.get_by_tag("update","rsync.server","unknown")) + +end -- of closure -- end library merge -own = { } +own = { } -- not local own.libs = { -- todo: check which ones are really needed 'l-string.lua', 'l-lpeg.lua', 'l-table.lua', 'l-io.lua', - 'l-md5.lua', 'l-number.lua', 'l-set.lua', 'l-os.lua', 'l-file.lua', + 'l-md5.lua', + 'l-url.lua', 'l-dir.lua', 'l-boolean.lua', - 'l-xml.lua', + 'l-math.lua', -- 'l-unicode.lua', - 'l-utils.lua', -- 'l-tex.lua', - 'luat-lib.lua', - 'luat-inp.lua', --- 'luat-zip.lua', --- 'luat-tex.lua', --- 'luat-kps.lua', - 'luat-tmp.lua', - 'luat-log.lua', - 'luat-sta.lua', + 'l-utils.lua', + 'l-aux.lua', +-- 'l-xml.lua', + 'trac-tra.lua', + 'lxml-tab.lua', + 'lxml-lpt.lua', +-- 'lxml-ent.lua', + 'lxml-mis.lua', + 'lxml-aux.lua', + 'lxml-xml.lua', + 'luat-env.lua', + 'trac-inf.lua', + 'trac-log.lua', + 'data-res.lua', + 'data-tmp.lua', + 'data-pre.lua', + 'data-inp.lua', + 'data-out.lua', + 'data-con.lua', + 'data-use.lua', +-- 'data-tex.lua', +-- 'data-bin.lua', + 'data-zip.lua', + 'data-crl.lua', + 'data-lua.lua', + 'data-kps.lua', -- so that we can replace kpsewhich + 'data-aux.lua', -- updater + 'data-tmf.lua', -- tree files + -- needed ? + 'luat-sta.lua', -- states } -- We need this hack till luatex is fixed. @@ -7796,7 +11870,8 @@ end -- End of hack. -own.name = (environment and environment.ownname) or arg[0] or 'luatools.lua' +own.name = (environment and environment.ownname) or arg[0] or 'luatools.lua' + own.path = string.match(own.name,"^(.+)[\\/].-$") or "." own.list = { '.' } @@ -7807,7 +11882,7 @@ table.insert(own.list,own.path.."/../../../tex/context/base") table.insert(own.list,own.path.."/mtx") table.insert(own.list,own.path.."/../sources") -function locate_libs() +local function locate_libs() for _, lib in pairs(own.libs) do for _, pth in pairs(own.list) do local filename = string.gsub(pth .. "/" .. lib,"\\","/") @@ -7821,11 +11896,11 @@ function locate_libs() end end -if not input then +if not resolvers then locate_libs() end -if not input then +if not resolvers then print("") print("Mtxrun is unable to start up due to lack of libraries. You may") print("try to run 'lua mtxrun.lua --selfmerge' in the path where this") @@ -7834,227 +11909,81 @@ if not input then os.exit() end -instance = input.reset() -input.verbose = environment.argument("verbose") or false -input.banner = 'MtxRun | ' -utils.report = input.report +logs.setprogram('MTXrun',"TDS Runner Tool 1.24",environment.arguments["verbose"] or false) -instance.engine = environment.argument("engine") or 'luatex' -instance.progname = environment.argument("progname") or 'context' -instance.lsrmode = environment.argument("lsr") or false +local instance = resolvers.reset() --- use os.env or environment when available - ---~ function input.check_environment(tree) ---~ input.report('') ---~ os.setenv('TMP', os.getenv('TMP') or os.getenv('TEMP') or os.getenv('TMPDIR') or os.getenv('HOME')) ---~ if os.platform == 'linux' then ---~ os.setenv('TEXOS', os.getenv('TEXOS') or 'texmf-linux') ---~ elseif os.platform == 'windows' then ---~ os.setenv('TEXOS', os.getenv('TEXOS') or 'texmf-windows') ---~ elseif os.platform == 'macosx' then ---~ os.setenv('TEXOS', os.getenv('TEXOS') or 'texmf-macosx') ---~ end ---~ os.setenv('TEXOS', string.gsub(string.gsub(os.getenv('TEXOS'),"^[\\\/]*", ''),"[\\\/]*$", '')) ---~ os.setenv('TEXPATH', string.gsub(tree,"\/+$",'')) ---~ os.setenv('TEXMFOS', os.getenv('TEXPATH') .. "/" .. os.getenv('TEXOS')) ---~ input.report('') ---~ input.report("preset : TEXPATH => " .. os.getenv('TEXPATH')) ---~ input.report("preset : TEXOS => " .. os.getenv('TEXOS')) ---~ input.report("preset : TEXMFOS => " .. os.getenv('TEXMFOS')) ---~ input.report("preset : TMP => " .. os.getenv('TMP')) ---~ input.report('') ---~ end +local trackspec = environment.argument("trackers") or environment.argument("track") -function input.check_environment(tree) - input.report('') - os.setenv('TMP', os.getenv('TMP') or os.getenv('TEMP') or os.getenv('TMPDIR') or os.getenv('HOME')) - os.setenv('TEXOS', os.getenv('TEXOS') or ("texmf-" .. os.currentplatform())) - os.setenv('TEXPATH', (tree or "tex"):gsub("\/+$",'')) - os.setenv('TEXMFOS', os.getenv('TEXPATH') .. "/" .. os.getenv('TEXOS')) - input.report('') - input.report("preset : TEXPATH => " .. os.getenv('TEXPATH')) - input.report("preset : TEXOS => " .. os.getenv('TEXOS')) - input.report("preset : TEXMFOS => " .. os.getenv('TEXMFOS')) - input.report("preset : TMP => " .. os.getenv('TMP')) - input.report('') -end - -function input.load_environment(name) -- todo: key=value as well as lua - local f = io.open(name) - if f then - for line in f:lines() do - if line:find("^[%%%#]") then - -- skip comment - else - local key, how, value = line:match("^(.-)%s*([<=>%?]+)%s*(.*)%s*$") - if how then - value = value:gsub("%%(.-)%%", function(v) return os.getenv(v) or "" end) - if how == "=" or how == "<<" then - os.setenv(key,value) - elseif how == "?" or how == "??" then - os.setenv(key,os.getenv(key) or value) - elseif how == "<" or how == "+=" then - if os.getenv(key) then - os.setenv(key,os.getenv(key) .. io.fileseparator .. value) - else - os.setenv(key,value) - end - elseif how == ">" or how == "=+" then - if os.getenv(key) then - os.setenv(key,value .. io.pathseparator .. os.getenv(key)) - else - os.setenv(key,value) - end - end - end - end - end - f:close() - end +if trackspec then + trackers.enable(trackspec) end -function input.load_tree(tree) - if tree and tree ~= "" then - local setuptex = 'setuptex.tmf' - if lfs.attributes(tree, "mode") == "directory" then -- check if not nil - setuptex = tree .. "/" .. setuptex - else - setuptex = tree - end - if io.exists(setuptex) then - input.check_environment(tree) - input.load_environment(setuptex) - end - end -end - --- md5 extensions - --- maybe md.md5 md.md5hex md.md5HEX - -if not md5 then md5 = { } end - -if not md5.sum then - function md5.sum(k) - return string.rep("x",16) - end -end - -function md5.hexsum(k) - return (string.gsub(md5.sum(k), ".", function(c) return string.format("%02x", string.byte(c)) end)) -end - -function md5.HEXsum(k) - return (string.gsub(md5.sum(k), ".", function(c) return string.format("%02X", string.byte(c)) end)) -end - --- file extensions +runners = runners or { } -- global +messages = messages or { } -file.needs_updating_threshold = 1 - -function file.needs_updating(oldname,newname) -- size modification access change - local oldtime = lfs.attributes(oldname, modification) - local newtime = lfs.attributes(newname, modification) - if newtime >= oldtime then - return false - elseif oldtime - newtime < file.needs_updating_threshold then - return false - else - return true - end -end +messages.help = [[ +--script run an mtx script (lua prefered method) (--noquotes), no script gives list +--execute run a script or program (texmfstart method) (--noquotes) +--resolve resolve prefixed arguments +--ctxlua run internally (using preloaded libs) +--internal run script using built in libraries (same as --ctxlua) +--locate locate given filename -function file.mdchecksum(name) - if md5 then - local data = io.loadall(name) - if data then - return md5.HEXsum(data) - end - end - return nil -end +--autotree use texmf tree cf. env 'texmfstart_tree' or 'texmfstarttree' +--tree=pathtotree use given texmf tree (default file: 'setuptex.tmf') +--environment=name use given (tmf) environment file +--path=runpath go to given path before execution +--ifchanged=filename only execute when given file has changed (md checksum) +--iftouched=old,new only execute when given file has changed (time stamp) -function file.loadchecksum(name) - if md then - local data = io.loadall(name .. ".md5") - if data then - return string.gsub(md5.HEXsum(data),"%s$","") - end - end - return nil -end +--make create stubs for (context related) scripts +--remove remove stubs (context related) scripts +--stubpath=binpath paths where stubs wil be written +--windows create windows (mswin) stubs +--unix create unix (linux) stubs -function file.savechecksum(name, checksum) - if not checksum then checksum = file.mdchecksum(name) end - if checksum then - local f = io.open(name .. ".md5","w") - if f then - f:write(checksum) - f:close() - return checksum - end - end - return nil -end +--verbose give a bit more info +--trackers=list enable given trackers +--engine=str target engine +--progname=str format or backend -os.arch = os.arch or function() - return os.resultof("uname -m") or "linux" -end +--edit launch editor with found file +--launch (--all) launch files like manuals, assumes os support -function os.currentplatform(name, default) - local name = os.name or os.platform or name -- os.name is built in, os.platform is mine - if name then - if name == "windows" or name == "mswin" or name == "win32" or name == "msdos" then - return "mswin" - elseif name == "linux" then - local architecture = os.arch() - if architecture:find("x86_64") then - return "linux-64" - else - return "linux" - end - elseif name == "macosx" then - local architecture = os.arch() - if architecture:find("i386") then - return "osx-intel" - else - return "osx-ppc" - end - elseif name == "freebsd" then - return "freebsd" - end - end - return default or name -end +--timedrun run a script an time its run +--autogenerate regenerate databases if needed (handy when used to run context in an editor) --- it starts here +--usekpse use kpse as fallback (when no mkiv and cache installed, often slower) +--forcekpse force using kpse (handy when no mkiv and cache installed but less functionality) -input.runners = { } -input.runners.applications = { } +--prefixes show supported prefixes +]] -input.runners.applications.lua = "luatex --luaonly" -input.runners.applications.pl = "perl" -input.runners.applications.py = "python" -input.runners.applications.rb = "ruby" +runners.applications = { + ["lua"] = "luatex --luaonly", + ["luc"] = "luatex --luaonly", + ["pl"] = "perl", + ["py"] = "python", + ["rb"] = "ruby", +} -input.runners.suffixes = { +runners.suffixes = { 'rb', 'lua', 'py', 'pl' } -input.runners.registered = { - texexec = { 'texexec.rb', true }, - texutil = { 'texutil.rb', true }, - texfont = { 'texfont.pl', true }, - texshow = { 'texshow.pl', false }, - +runners.registered = { + texexec = { 'texexec.rb', false }, -- context mkii runner (only tool not to be luafied) + texutil = { 'texutil.rb', true }, -- old perl based index sorter for mkii (old versions need it) + texfont = { 'texfont.pl', true }, -- perl script that makes mkii font metric files + texfind = { 'texfind.pl', false }, -- perltk based tex searching tool, mostly used at pragma + texshow = { 'texshow.pl', false }, -- perltk based context help system, will be luafied + -- texwork = { 'texwork.pl', false }, -- perltk based editing environment, only used at pragma makempy = { 'makempy.pl', true }, mptopdf = { 'mptopdf.pl', true }, - pstopdf = { 'pstopdf.rb', true }, - - examplex = { 'examplex.rb', false }, + pstopdf = { 'pstopdf.rb', true }, -- converts ps (and some more) images, does some cleaning (replaced) +-- examplex = { 'examplex.rb', false }, concheck = { 'concheck.rb', false }, - runtools = { 'runtools.rb', true }, textools = { 'textools.rb', true }, tmftools = { 'tmftools.rb', true }, @@ -8062,81 +11991,45 @@ input.runners.registered = { rlxtools = { 'rlxtools.rb', true }, pdftools = { 'pdftools.rb', true }, mpstools = { 'mpstools.rb', true }, - exatools = { 'exatools.rb', true }, +-- exatools = { 'exatools.rb', true }, xmltools = { 'xmltools.rb', true }, - luatools = { 'luatools.lua', true }, +-- luatools = { 'luatools.lua', true }, mtxtools = { 'mtxtools.rb', true }, - pdftrimwhite = { 'pdftrimwhite.pl', false } } -if not messages then messages = { } end - -messages.help = [[ ---script run an mtx script ---execute run a script or program ---resolve resolve prefixed arguments ---ctxlua run internally (using preloaded libs) ---locate locate given filename - ---autotree use texmf tree cf. env 'texmfstart_tree' or 'texmfstarttree' ---tree=pathtotree use given texmf tree (default file: 'setuptex.tmf') ---environment=name use given (tmf) environment file ---path=runpath go to given path before execution ---ifchanged=filename only execute when given file has changed (md checksum) ---iftouched=old,new only execute when given file has changed (time stamp) - ---make create stubs for (context related) scripts ---remove remove stubs (context related) scripts ---stubpath=binpath paths where stubs wil be written ---windows create windows (mswin) stubs ---unix create unix (linux) stubs - ---verbose give a bit more info ---engine=str target engine ---progname=str format or backend - ---edit launch editor with found file ---launch (--all) launch files (assume os support) - ---intern run script using built in libraries -]] +runners.launchers = { + windows = { }, + unix = { } +} -function input.runners.my_prepare_a(instance) - input.resetconfig(instance) - input.identify_cnf(instance) - input.load_lua(instance) - input.expand_variables(instance) - input.load_cnf(instance) - input.expand_variables(instance) -end +-- like runners.libpath("framework"): looks on script's subpath -function input.runners.my_prepare_b(instance) - input.runners.my_prepare_a(instance) - input.load_hash(instance) - input.automount(instance) +function runners.libpath(...) + package.prepend_libpath(file.dirname(environment.ownscript),...) + package.prepend_libpath(file.dirname(environment.ownname) ,...) end -function input.runners.prepare(instance) +function runners.prepare() local checkname = environment.argument("ifchanged") if checkname and checkname ~= "" then local oldchecksum = file.loadchecksum(checkname) local newchecksum = file.checksum(checkname) if oldchecksum == newchecksum then - report("file '" .. checkname .. "' is unchanged") + logs.simple("file '%s' is unchanged",checkname) return "skip" else - report("file '" .. checkname .. "' is changed, processing started") + logs.simple("file '%s' is changed, processing started",checkname) end file.savechecksum(checkname) end local oldname, newname = string.split(environment.argument("iftouched") or "", ",") if oldname and newname and oldname ~= "" and newname ~= "" then if not file.needs_updating(oldname,newname) then - report("file '" .. oldname .. "' and '" .. newname .. "'have same age") + logs.simple("file '%s' and '%s' have same age",oldname,newname) return "skip" else - report("file '" .. newname .. "' is older than '" .. oldname .. "'") + logs.simple("file '%s' is older than '%s'",oldname,newname) end end local tree = environment.argument('tree') or "" @@ -8144,26 +12037,27 @@ function input.runners.prepare(instance) tree = os.getenv('TEXMFSTART_TREE') or os.getenv('TEXMFSTARTTREE') or tree end if tree and tree ~= "" then - input.load_tree(tree) + resolvers.load_tree(tree) end local env = environment.argument('environment') or "" if env and env ~= "" then for _,e in pairs(string.split(env)) do -- maybe force suffix when not given - input.load_tree(e) + resolvers.load_tree(e) end end local runpath = environment.argument("path") - if runpath and not dir.chdir(runpath) then - input.report("unable to change to path '" .. runpath .. "'") + if runpath and not lfs.chdir(runpath) then + logs.simple("unable to change to path '%s'",runpath) return "error" end return "run" end -function input.runners.execute_script(instance,fullname,internal) +function runners.execute_script(fullname,internal,nosplit) + local noquote = environment.argument("noquotes") if fullname and fullname ~= "" then - local state = input.runners.prepare(instance) + local state = runners.prepare() if state == 'error' then return false elseif state == 'skip' then @@ -8180,43 +12074,65 @@ function input.runners.execute_script(instance,fullname,internal) return "" end ) name = name:gsub("^script:","") - if suffix == "" and input.runners.registered[name] and input.runners.registered[name][1] then - name = input.runners.registered[name][1] + if suffix == "" and runners.registered[name] and runners.registered[name][1] then + name = runners.registered[name][1] suffix = file.extname(name) end if suffix == "" then -- loop over known suffixes - for _,s in pairs(input.runners.suffixes) do - result = input.find_file(instance, name .. "." .. s, 'texmfscripts') + for _,s in pairs(runners.suffixes) do + result = resolvers.find_file(name .. "." .. s, 'texmfscripts') if result ~= "" then break end end - elseif input.runners.applications[suffix] then - result = input.find_file(instance, name, 'texmfscripts') + elseif runners.applications[suffix] then + result = resolvers.find_file(name, 'texmfscripts') else -- maybe look on path - result = input.find_file(instance, name, 'other text files') + result = resolvers.find_file(name, 'other text files') end end if result and result ~= "" then + if not no_split then + local before, after = environment.split_arguments(fullname) -- already done + environment.arguments_before, environment.arguments_after = before, after + end if internal then - local before, after = environment.split_arguments(fullname) - arg = { } for _,v in pairs(after) do arg[#arg+1] = v end + arg = { } for _,v in pairs(environment.arguments_after) do arg[#arg+1] = v end + environment.ownscript = result dofile(result) else - local binary = input.runners.applications[file.extname(result)] + local binary = runners.applications[file.extname(result)] if binary and binary ~= "" then result = binary .. " " .. result end - local before, after = environment.split_arguments(fullname) - local command = result .. " " .. environment.reconstruct_commandline(after) - input.report("") - input.report("executing: " .. command) - input.report("\n \n") - io.flush() - local code = os.exec(command) -- maybe spawn - return code == 0 + local command = result .. " " .. environment.reconstruct_commandline(environment.arguments_after,noquote) + if logs.verbose then + logs.simpleline() + logs.simple("executing: %s",command) + logs.simpleline() + logs.simpleline() + io.flush() + end + -- no os.exec because otherwise we get the wrong return value + local code = os.execute(command) -- maybe spawn + if code == 0 then + return true + else + if binary then + binary = file.addsuffix(binary,os.binsuffix) + for p in string.gmatch(os.getenv("PATH"),"[^"..io.pathseparator.."]+") do + if lfs.isfile(file.join(p,binary)) then + return false + end + end + logs.simpleline() + logs.simple("This script needs '%s' which seems not to be installed.",binary) + logs.simpleline() + end + return false + end end end end @@ -8224,9 +12140,10 @@ function input.runners.execute_script(instance,fullname,internal) return false end -function input.runners.execute_program(instance,fullname) +function runners.execute_program(fullname) + local noquote = environment.argument("noquotes") if fullname and fullname ~= "" then - local state = input.runners.prepare(instance) + local state = runners.prepare() if state == 'error' then return false elseif state == 'skip' then @@ -8235,10 +12152,11 @@ function input.runners.execute_program(instance,fullname) local before, after = environment.split_arguments(fullname) environment.initialize_arguments(after) fullname = fullname:gsub("^bin:","") - local command = fullname .. " " .. environment.reconstruct_commandline(after) - input.report("") - input.report("executing: " .. command) - input.report("\n \n") + local command = fullname .. " " .. (environment.reconstruct_commandline(after or "",noquote) or "") + logs.simpleline() + logs.simple("executing: %s",command) + logs.simpleline() + logs.simpleline() io.flush() local code = os.exec(command) -- (fullname,unpack(after)) does not work / maybe spawn return code == 0 @@ -8247,90 +12165,102 @@ function input.runners.execute_program(instance,fullname) return false end -function input.runners.handle_stubs(instance,create) - local stubpath = environment.argument('stubpath') or '.' -- 'auto' no longer supported +-- the --usekpse flag will fallback on kpse (hm, we can better update mtx-stubs) + +local windows_stub = '@echo off\013\010setlocal\013\010set ownpath=%%~dp0%%\013\010texlua "%%ownpath%%mtxrun.lua" --usekpse --execute %s %%*\013\010endlocal\013\010' +local unix_stub = '#!/bin/sh\010mtxrun --usekpse --execute %s \"$@\"\010' + +function runners.handle_stubs(create) + local stubpath = environment.argument('stubpath') or '.' -- 'auto' no longer subpathssupported local windows = environment.argument('windows') or environment.argument('mswin') or false local unix = environment.argument('unix') or environment.argument('linux') or false if not windows and not unix then - if environment.platform == "unix" then + if os.platform == "unix" then unix = true else windows = true end end - for _,v in pairs(input.runners.registered) do + for _,v in pairs(runners.registered) do local name, doit = v[1], v[2] if doit then local base = string.gsub(file.basename(name), "%.(.-)$", "") if create then - -- direct local command = input.runners.applications[file.extname(name)] .. " " .. name - local command = "luatex --luaonly mtxrun.lua " .. name if windows then - io.savedata(base..".bat", {"@echo off", command.." %*"}, "\013\010") - input.report("windows stub for '" .. base .. "' created") + io.savedata(file.join(stubpath,base..".bat"),string.format(windows_stub,name)) + logs.simple("windows stub for '%s' created",base) end if unix then - io.savedata(base, {"#!/bin/sh", command..' "$@"'}, "\010") - input.report("unix stub for '" .. base .. "' created") + io.savedata(file.join(stubpath,base),string.format(unix_stub,name)) + logs.simple("unix stub for '%s' created",base) end else - if windows and (os.remove(base..'.bat') or os.remove(base..'.cmd')) then - input.report("windows stub for '" .. base .. "' removed") + if windows and (os.remove(file.join(stubpath,base..'.bat')) or os.remove(file.join(stubpath,base..'.cmd'))) then + logs.simple("windows stub for '%s' removed", base) end - if unix and (os.remove(base) or os.remove(base..'.sh')) then - input.report("unix stub for '" .. base .. "' removed") + if unix and (os.remove(file.join(stubpath,base)) or os.remove(file.join(stubpath,base..'.sh'))) then + logs.simple("unix stub for '%s' removed",base) end end end end end -function input.runners.resolve_string(instance,filename) +function runners.resolve_string(filename) if filename and filename ~= "" then - input.runners.report_location(instance,input.resolve(instance,filename)) + runners.report_location(resolvers.resolve(filename)) end end -function input.runners.locate_file(instance,filename) +function runners.locate_file(filename) + -- differs from texmfstart where locate appends .com .exe .bat ... todo if filename and filename ~= "" then - input.runners.report_location(instance,input.find_given_file(instance,filename)) + runners.report_location(resolvers.find_given_file(filename)) end end -function input.runners.locate_platform(instance) - input.runners.report_location(instance,os.currentplatform()) +function runners.locate_platform() + runners.report_location(os.platform) end -function input.runners.report_location(instance,result) - if input.verbose then - input.report("") +function runners.report_location(result) + if logs.verbose then + logs.simpleline() if result and result ~= "" then - input.report(result) + logs.simple(result) else - input.report("not found") + logs.simple("not found") end else io.write(result) end end -function input.runners.edit_script(instance,filename) - local editor = os.getenv("MTXRUN_EDITOR") or os.getenv("TEXMFSTART_EDITOR") or os.getenv("EDITOR") or 'scite' - local rest = input.resolve(instance,filename) +function runners.edit_script(filename) -- we assume that vim is present on most systems + local editor = os.getenv("MTXRUN_EDITOR") or os.getenv("TEXMFSTART_EDITOR") or os.getenv("EDITOR") or 'vim' + local rest = resolvers.resolve(filename) if rest ~= "" then - os.launch(editor .. " " .. rest) + local command = editor .. " " .. rest + if logs.verbose then + logs.simpleline() + logs.simple("starting editor: %s",command) + logs.simple_line() + logs.simple_line() + end + os.launch(command) end end -function input.runners.save_script_session(filename, list) +function runners.save_script_session(filename, list) local t = { } - for _, key in ipairs(list) do + for i=1,#list do + local key = list[i] t[key] = environment.arguments[key] end io.savedata(filename,table.serialize(t,true)) end -function input.runners.load_script_session(filename) +function runners.load_script_session(filename) if lfs.isfile(filename) then local t = io.loaddata(filename) if t then @@ -8343,15 +12273,10 @@ function input.runners.load_script_session(filename) end end -input.runners.launchers = { - windows = { }, - unix = { } -} - -function input.launch(str) +function resolvers.launch(str) -- maybe we also need to test on mtxrun.launcher.suffix environment -- variable or on windows consult the assoc and ftype vars and such - local launchers = input.runners.launchers[os.platform] if launchers then + local launchers = runners.launchers[os.platform] if launchers then local suffix = file.extname(str) if suffix then local runner = launchers[suffix] if runner then str = runner .. " " .. str @@ -8361,35 +12286,40 @@ function input.launch(str) os.launch(str) end -function input.runners.launch_file(instance,filename) +function runners.launch_file(filename) instance.allresults = true - input.verbose = true + logs.setverbose(true) local pattern = environment.arguments["pattern"] if not pattern or pattern == "" then pattern = filename end if not pattern or pattern == "" then - input.report("provide name or --pattern=") + logs.simple("provide name or --pattern=") else - local t = input.find_files(instance,pattern) - -- local t = input.aux.find_file(instance,"*/" .. pattern,true) + local t = resolvers.find_files(pattern) + if not t or #t == 0 then + t = resolvers.find_files("*/" .. pattern) + end + if not t or #t == 0 then + t = resolvers.find_files("*/" .. pattern .. "*") + end if t and #t > 0 then if environment.arguments["all"] then for _, v in pairs(t) do - input.report("launching", v) - input.launch(v) + logs.simple("launching %s", v) + resolvers.launch(v) end else - input.report("launching", t[1]) - input.launch(t[1]) + logs.simple("launching %s", t[1]) + resolvers.launch(t[1]) end else - input.report("no match for", pattern) + logs.simple("no match for %s", pattern) end end end -function input.runners.execute_ctx_script(instance,filename,arguments) +function runners.find_mtx_script(filename) local function found(name) local path = file.dirname(name) if path and path ~= "" then @@ -8399,72 +12329,156 @@ function input.runners.execute_ctx_script(instance,filename,arguments) return io.exists(fullname) and fullname end end - local suffix = "" - if not filename:find("%.lua$") then suffix = ".lua" end - local fullname = filename - -- just <filename> - fullname = filename .. suffix - fullname = input.find_file(instance,fullname) - -- mtx-<filename> - if not fullname or fullname == "" then - fullname = "mtx-" .. filename .. suffix - fullname = found(fullname) or input.find_file(instance,fullname) + filename = file.addsuffix(filename,"lua") + local basename = file.removesuffix(file.basename(filename)) + local suffix = file.extname(filename) + -- qualified path, raw name + local fullname = file.is_qualified_path(filename) and io.exists(filename) and filename + if fullname and fullname ~= "" then + return fullname end - -- mtx-<filename>s - if not fullname or fullname == "" then - fullname = "mtx-" .. filename .. "s" .. suffix - fullname = found(fullname) or input.find_file(instance,fullname) + -- current path, raw name + fullname = "./" .. filename + fullname = io.exists(fullname) and fullname + if fullname and fullname ~= "" then + return fullname end - -- mtx-<filename minus trailing s> - if not fullname or fullname == "" then - fullname = "mtx-" .. filename:gsub("s$","") .. suffix - fullname = found(fullname) or input.find_file(instance,fullname) + -- mtx- prefix checking + local mtxprefix = (filename:find("^mtx%-") and "") or "mtx-" + -- context namespace, mtx-<filename> + fullname = mtxprefix .. filename + fullname = found(fullname) or resolvers.find_file(fullname) + if fullname and fullname ~= "" then + return fullname end - -- that should do it + -- context namespace, mtx-<filename>s + fullname = mtxprefix .. basename .. "s" .. "." .. suffix + fullname = found(fullname) or resolvers.find_file(fullname) + if fullname and fullname ~= "" then + return fullname + end + -- context namespace, mtx-<filename minus trailing s> + fullname = mtxprefix .. basename:gsub("s$","") .. "." .. suffix + fullname = found(fullname) or resolvers.find_file(fullname) if fullname and fullname ~= "" then - local state = input.runners.prepare(instance) + return fullname + end + -- context namespace, just <filename> + fullname = resolvers.find_file(filename) + return fullname +end + +function runners.execute_ctx_script(filename) + local arguments = environment.arguments_after + local fullname = runners.find_mtx_script(filename) or "" + if file.extname(fullname) == "cld" then + -- handy in editors where we force --autopdf + logs.simple("running cld script: %s",filename) + table.insert(arguments,1,fullname) + table.insert(arguments,"--autopdf") + fullname = runners.find_mtx_script("context") or "" + end + -- retry after generate but only if --autogenerate + if fullname == "" and environment.argument("autogenerate") then -- might become the default + instance.renewcache = true + logs.setverbose(true) + resolvers.load() + -- + fullname = runners.find_mtx_script(filename) or "" + end + -- that should do it + if fullname ~= "" then + local state = runners.prepare() if state == 'error' then return false elseif state == 'skip' then return true elseif state == "run" then -- load and save ... kind of undocumented - arg = { } for _,v in pairs(arguments) do arg[#arg+1] = v end + arg = { } for _,v in pairs(arguments) do arg[#arg+1] = resolvers.resolve(v) end environment.initialize_arguments(arg) local loadname = environment.arguments['load'] if loadname then if type(loadname) ~= "string" then loadname = file.basename(fullname) end loadname = file.replacesuffix(loadname,"cfg") - input.runners.load_script_session(loadname) + runners.load_script_session(loadname) end filename = environment.files[1] - if input.verbose then - input.report("using script: " .. fullname) + if logs.verbose then + logs.simple("using script: %s\n",fullname) end + environment.ownscript = fullname dofile(fullname) local savename = environment.arguments['save'] - if savename and input.runners.save_list and not table.is_empty(input.runners.save_list or { }) then - if type(savename) ~= "string" then savename = file.basename(fullname) end - savename = file.replacesuffix(savename,"cfg") - input.runners.save_script_session(savename, input.runners.save_list) + if savename then + local save_list = runners.save_list + if save_list and next(save_list) then + if type(savename) ~= "string" then savename = file.basename(fullname) end + savename = file.replacesuffix(savename,"cfg") + runners.save_script_session(savename,save_list) + end end return true end else - input.verbose = true - input.report("unknown script: " .. filename) + -- logs.setverbose(true) + if filename == "" or filename == "help" then + local context = resolvers.find_file("mtx-context.lua") + logs.setverbose(true) + if context ~= "" then + local result = dir.glob((string.gsub(context,"mtx%-context","mtx-*"))) -- () needed + local valid = { } + table.sort(result) + for i=1,#result do + local scriptname = result[i] + local scriptbase = string.match(scriptname,".*mtx%-([^%-]-)%.lua") + if scriptbase then + local data = io.loaddata(scriptname) + local banner, version = string.match(data,"[\n\r]logs%.extendbanner%s*%(%s*[\"\']([^\n\r]+)%s*(%d+%.%d+)") + if banner then + valid[#valid+1] = { scriptbase, version, banner } + end + end + end + if #valid > 0 then + logs.reportbanner() + logs.reportline() + logs.simple("no script name given, known scripts:") + logs.simple() + for k=1,#valid do + local v = valid[k] + logs.simple("%-12s %4s %s",v[1],v[2],v[3]) + end + end + else + logs.simple("no script name given") + end + else + filename = file.addsuffix(filename,"lua") + if file.is_qualified_path(filename) then + logs.simple("unknown script '%s'",filename) + else + logs.simple("unknown script '%s' or 'mtx-%s'",filename,filename) + end + end return false end end -input.report(banner,"\n") +function runners.prefixes() + logs.reportbanner() + logs.reportline() + logs.simple(table.concat(resolvers.allprefixes(true)," ")) +end -function input.help(banner,message) - if not input.verbose then - input.verbose = true - input.report(banner,"\n") +function runners.timedrun(filename) -- just for me + if filename and filename ~= "" then + runners.timed(function() os.execute(filename) end) end - input.reportlines(message) +end + +function runners.timed(action) + statistics.timed(action) end -- this is a bit dirty ... first we store the first filename and next we @@ -8475,12 +12489,80 @@ local filename = environment.files[1] or "" local ok = true local before, after = environment.split_arguments(filename) +environment.arguments_before, environment.arguments_after = before, after +environment.initialize_arguments(before) -input.runners.my_prepare_b(instance) -before = input.resolve(instance,before) -- experimental here -after = input.resolve(instance,after) -- experimental here +instance.engine = environment.argument("engine") or 'luatex' +instance.progname = environment.argument("progname") or 'context' +instance.lsrmode = environment.argument("lsr") or false -environment.initialize_arguments(before) +-- maybe the unset has to go to this level + +local is_mkii_stub = runners.registered[file.removesuffix(file.basename(filename))] + +if environment.argument("usekpse") or environment.argument("forcekpse") or is_mkii_stub then + + os.setenv("engine","") + os.setenv("progname","") + + local remapper = { + otf = "opentype fonts", + ttf = "truetype fonts", + ttc = "truetype fonts", + pfb = "type1 fonts", + other = "other text files", + } + + local function kpse_initialized() + texconfig.kpse_init = true + local t = os.clock() + local k = kpse.original.new("luatex",instance.progname) + local dummy = k:find_file("mtxrun.lua") -- so that we're initialized + logs.simple("kpse fallback with progname '%s' initialized in %s seconds",instance.progname,os.clock()-t) + kpse_initialized = function() return k end + return k + end + + local find_file = resolvers.find_file + local show_path = resolvers.show_path + + if environment.argument("forcekpse") then + + function resolvers.find_file(name,kind) + return (kpse_initialized():find_file(resolvers.clean_path(name),(kind ~= "" and (remapper[kind] or kind)) or "tex") or "") or "" + end + function resolvers.show_path(name) + return (kpse_initialized():show_path(name)) or "" + end + + elseif environment.argument("usekpse") or is_mkii_stub then + + resolvers.load() + + function resolvers.find_file(name,kind) + local found = find_file(name,kind) or "" + if found ~= "" then + return found + else + return (kpse_initialized():find_file(resolvers.clean_path(name),(kind ~= "" and (remapper[kind] or kind)) or "tex") or "") or "" + end + end + function resolvers.show_path(name) + local found = show_path(name) or "" + if found ~= "" then + return found + else + return (kpse_initialized():show_path(name)) or "" + end + end + + end + +else + + resolvers.load() + +end if environment.argument("selfmerge") then -- embed used libraries @@ -8489,61 +12571,69 @@ elseif environment.argument("selfclean") then -- remove embedded libraries utils.merger.selfclean(own.name) elseif environment.argument("selfupdate") then - input.verbose = true - input.update_script(instance,own.name,"mtxrun") + logs.setverbose(true) + resolvers.update_script(own.name,"mtxrun") elseif environment.argument("ctxlua") or environment.argument("internal") then -- run a script by loading it (using libs) - ok = input.runners.execute_script(instance,filename,true) -elseif environment.argument("script") then + ok = runners.execute_script(filename,true) +elseif environment.argument("script") or environment.argument("scripts") then -- run a script by loading it (using libs), pass args - ok = input.runners.execute_ctx_script(instance,filename,after) + if is_mkii_stub then + -- execute mkii script + ok = runners.execute_script(filename,false,true) + else + ok = runners.execute_ctx_script(filename) + end elseif environment.argument("execute") then -- execute script - ok = input.runners.execute_script(instance,filename) + ok = runners.execute_script(filename) elseif environment.argument("direct") then -- equals bin: - ok = input.runners.execute_program(instance,filename) + ok = runners.execute_program(filename) elseif environment.argument("edit") then -- edit file - input.runners.edit_script(instance,filename) + runners.edit_script(filename) elseif environment.argument("launch") then - input.runners.launch_file(instance,filename) + runners.launch_file(filename) elseif environment.argument("make") then -- make stubs - input.runners.handle_stubs(instance,true) + runners.handle_stubs(true) elseif environment.argument("remove") then -- remove stub - input.runners.handle_stubs(instance,false) + runners.handle_stubs(false) elseif environment.argument("resolve") then -- resolve string - input.runners.resolve_string(instance,filename) + runners.resolve_string(filename) elseif environment.argument("locate") then -- locate file - input.runners.locate_file(instance,filename) + runners.locate_file(filename) elseif environment.argument("platform")then -- locate platform - input.runners.locate_platform(instance) + runners.locate_platform() +elseif environment.argument("prefixes") then + runners.prefixes() +elseif environment.argument("timedrun") then + -- locate platform + runners.timedrun(filename) elseif environment.argument("help") or filename=='help' or filename == "" then - input.help(banner,messages.help) + logs.help(messages.help) -- execute script - if filename:find("^bin:") then - ok = input.runners.execute_program(instance,filename) - else - ok = input.runners.execute_script(instance,filename) +elseif filename:find("^bin:") then + ok = runners.execute_program(filename) +elseif is_mkii_stub then + -- execute mkii script + ok = runners.execute_script(filename,false,true) +else + ok = runners.execute_ctx_script(filename) + if not ok then + ok = runners.execute_script(filename) end end ---~ if input.verbose then ---~ input.report("") ---~ input.report(string.format("runtime: %0.3f seconds",os.runtime())) ---~ end - ---~ if ok then ---~ input.report("exit code: 0") os.exit(0) ---~ else ---~ input.report("exit code: 1") os.exit(1) ---~ end - -if environment.platform == "unix" then +if os.platform == "unix" then io.write("\n") end + +if ok == false then ok = 1 elseif ok == true then ok = 0 end + +os.exit(ok) diff --git a/Master/bin/win32/mtxtools.bat b/Master/bin/win32/mtxtools.bat deleted file mode 100755 index cb0b1e39e47..00000000000 --- a/Master/bin/win32/mtxtools.bat +++ /dev/null @@ -1,2 +0,0 @@ -@echo off
-texmfstart mtxtools.rb %*
diff --git a/Master/bin/win32/mtxtools.exe b/Master/bin/win32/mtxtools.exe Binary files differdeleted file mode 100755 index 5777d90a17a..00000000000 --- a/Master/bin/win32/mtxtools.exe +++ /dev/null diff --git a/Master/bin/win32/pdftools.bat b/Master/bin/win32/pdftools.bat deleted file mode 100755 index 71662d3f766..00000000000 --- a/Master/bin/win32/pdftools.bat +++ /dev/null @@ -1,2 +0,0 @@ -@echo off
-texmfstart pdftools.rb %*
diff --git a/Master/bin/win32/pdftools.exe b/Master/bin/win32/pdftools.exe Binary files differdeleted file mode 100755 index 5777d90a17a..00000000000 --- a/Master/bin/win32/pdftools.exe +++ /dev/null diff --git a/Master/bin/win32/pstopdf.bat b/Master/bin/win32/pstopdf.bat deleted file mode 100755 index 091b0d4d7bd..00000000000 --- a/Master/bin/win32/pstopdf.bat +++ /dev/null @@ -1,2 +0,0 @@ -@echo off
-texmfstart pstopdf.rb %*
diff --git a/Master/bin/win32/pstopdf.exe b/Master/bin/win32/pstopdf.exe Binary files differdeleted file mode 100755 index 5777d90a17a..00000000000 --- a/Master/bin/win32/pstopdf.exe +++ /dev/null diff --git a/Master/bin/win32/rlxtools.exe b/Master/bin/win32/rlxtools.exe Binary files differindex 5777d90a17a..745eaf22464 100755 --- a/Master/bin/win32/rlxtools.exe +++ b/Master/bin/win32/rlxtools.exe diff --git a/Master/bin/win32/runtools.bat b/Master/bin/win32/runtools.bat deleted file mode 100755 index 37df398ab37..00000000000 --- a/Master/bin/win32/runtools.bat +++ /dev/null @@ -1,2 +0,0 @@ -@echo off
-texmfstart runtools.rb %*
diff --git a/Master/bin/win32/runtools.exe b/Master/bin/win32/runtools.exe Binary files differdeleted file mode 100755 index 5777d90a17a..00000000000 --- a/Master/bin/win32/runtools.exe +++ /dev/null diff --git a/Master/bin/win32/texexec.exe b/Master/bin/win32/texexec.exe Binary files differindex 5777d90a17a..2d45f27494d 100755 --- a/Master/bin/win32/texexec.exe +++ b/Master/bin/win32/texexec.exe diff --git a/Master/bin/win32/texfont.bat b/Master/bin/win32/texfont.bat deleted file mode 100755 index 9dd50309ff3..00000000000 --- a/Master/bin/win32/texfont.bat +++ /dev/null @@ -1,2 +0,0 @@ -@echo off
-texmfstart texfont.pl %*
diff --git a/Master/bin/win32/texfont.exe b/Master/bin/win32/texfont.exe Binary files differdeleted file mode 100755 index 5777d90a17a..00000000000 --- a/Master/bin/win32/texfont.exe +++ /dev/null diff --git a/Master/bin/win32/texmfstart.exe b/Master/bin/win32/texmfstart.exe Binary files differindex 6ab218e8656..2d45f27494d 100755 --- a/Master/bin/win32/texmfstart.exe +++ b/Master/bin/win32/texmfstart.exe diff --git a/Master/bin/win32/textools.bat b/Master/bin/win32/textools.bat deleted file mode 100755 index 4fc5c549255..00000000000 --- a/Master/bin/win32/textools.bat +++ /dev/null @@ -1,2 +0,0 @@ -@echo off
-texmfstart textools.rb %*
diff --git a/Master/bin/win32/textools.exe b/Master/bin/win32/textools.exe Binary files differdeleted file mode 100755 index 5777d90a17a..00000000000 --- a/Master/bin/win32/textools.exe +++ /dev/null diff --git a/Master/bin/win32/texutil.bat b/Master/bin/win32/texutil.bat deleted file mode 100755 index 61e90ca6509..00000000000 --- a/Master/bin/win32/texutil.bat +++ /dev/null @@ -1,2 +0,0 @@ -@echo off
-texmfstart texutil.rb %*
diff --git a/Master/bin/win32/texutil.exe b/Master/bin/win32/texutil.exe Binary files differdeleted file mode 100755 index 5777d90a17a..00000000000 --- a/Master/bin/win32/texutil.exe +++ /dev/null diff --git a/Master/bin/win32/tmftools.bat b/Master/bin/win32/tmftools.bat deleted file mode 100755 index a7053043d22..00000000000 --- a/Master/bin/win32/tmftools.bat +++ /dev/null @@ -1,2 +0,0 @@ -@echo off
-texmfstart tmftools.rb %*
diff --git a/Master/bin/win32/tmftools.exe b/Master/bin/win32/tmftools.exe Binary files differdeleted file mode 100755 index 5777d90a17a..00000000000 --- a/Master/bin/win32/tmftools.exe +++ /dev/null diff --git a/Master/bin/win32/xmltools.bat b/Master/bin/win32/xmltools.bat deleted file mode 100755 index 9721095b298..00000000000 --- a/Master/bin/win32/xmltools.bat +++ /dev/null @@ -1,2 +0,0 @@ -@echo off
-texmfstart xmltools.rb %*
diff --git a/Master/bin/win32/xmltools.exe b/Master/bin/win32/xmltools.exe Binary files differdeleted file mode 100755 index 5777d90a17a..00000000000 --- a/Master/bin/win32/xmltools.exe +++ /dev/null |