diff options
| author | Taco Hoekwater <taco@elvenkind.com> | 2008-06-12 10:42:53 +0000 |
|---|---|---|
| committer | Taco Hoekwater <taco@elvenkind.com> | 2008-06-12 10:42:53 +0000 |
| commit | 0d01365d53c456d246da0ca1f0b3cd9868f02b35 (patch) | |
| tree | 01a655c8028e17cfb371456b299c1848fe08c05b /Master/texmf-dist/scripts/context | |
| parent | 44f3714442da07fdfc36a7f2a8dcd5d4294c5d26 (diff) | |
ConTeXt release 2008.05.21
git-svn-id: svn://tug.org/texlive/trunk@8691 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Master/texmf-dist/scripts/context')
71 files changed, 34641 insertions, 1267 deletions
diff --git a/Master/texmf-dist/scripts/context/lua/luatools.lua b/Master/texmf-dist/scripts/context/lua/luatools.lua new file mode 100644 index 00000000000..89e5e0eb4db --- /dev/null +++ b/Master/texmf-dist/scripts/context/lua/luatools.lua @@ -0,0 +1,6656 @@ +#!/usr/bin/env texlua + +-- one can make a stub: +-- +-- #!/bin/sh +-- env LUATEXDIR=/....../texmf/scripts/context/lua luatex --luaonly=luatools.lua "$@" +-- filename : luatools.lua +-- comment : companion to context.tex +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files +-- Although this script is part of the ConTeXt distribution it is +-- relatively indepent of ConTeXt. The same is true for some of +-- the luat files. We may may make them even less dependent in +-- the future. As long as Luatex is under development the +-- interfaces and names of functions may change. + +banner = "version 1.2.0 - 2006+ - PRAGMA ADE / CONTEXT" +texlua = true + +-- For the sake of independence we optionally can merge the library +-- code here. It's too much code, but that does not harm. Much of the +-- library code is used elsewhere. We don't want dependencies on +-- Lua library paths simply because these scripts are located in the +-- texmf tree and not in some Lua path. Normally this merge is not +-- needed when texmfstart is used, or when the proper stub is used or +-- when (windows) suffix binding is active. + +-- begin library merge +-- filename : l-string.lua +-- comment : split off from luat-lib +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-string'] = 1.001 + +--~ function string.split(str, pat) -- taken from the lua wiki +--~ local t = {n = 0} -- so this table has a length field, traverse with ipairs then! +--~ local fpat = "(.-)"..pat +--~ local last_end = 1 +--~ local s, e, cap = string.find(str, fpat, 1) +--~ while s ~= nil do +--~ if s~=1 or cap~="" then +--~ table.insert(t,cap) +--~ end +--~ last_end = e+1 +--~ s, e, cap = string.find(str, fpat, last_end) +--~ end +--~ if last_end<=string.len(str) then +--~ table.insert(t,(string.sub(str,last_end))) +--~ end +--~ return t +--~ end + +--~ function string:split(pat) -- taken from the lua wiki but adapted +--~ local t = { } -- self and colon usage (faster) +--~ local fpat = "(.-)"..pat +--~ local last_end = 1 +--~ local s, e, cap = self:find(fpat, 1) +--~ while s ~= nil do +--~ if s~=1 or cap~="" then +--~ t[#t+1] = cap +--~ end +--~ last_end = e+1 +--~ s, e, cap = self:find(fpat, last_end) +--~ end +--~ if last_end <= #self then +--~ t[#t+1] = self:sub(last_end) +--~ end +--~ return t +--~ end + +--~ a piece of brilliant code by Rici Lake (posted on lua list) -- only names changed +--~ +--~ function string:splitter(pat) +--~ local st, g = 1, self:gmatch("()"..pat.."()") +--~ local function splitter(self) +--~ if st then +--~ local s, f = g() +--~ local rv = self:sub(st, (s or 0)-1) +--~ st = f +--~ return rv +--~ end +--~ end +--~ return splitter, self +--~ end + +function string:splitter(pat) + -- by Rici Lake (posted on lua list) -- only names changed + -- p 79 ref man: () returns position of match + local st, g = 1, self:gmatch("()("..pat..")") + local function strgetter(self, segs, seps, sep, cap1, ...) + st = sep and seps + #sep + return self:sub(segs, (seps or 0) - 1), cap1 or sep, ... + end + local function strsplitter(self) + if st then return strgetter(self, st, g()) end + end + return strsplitter, self +end + +function string:split(separator) + local t = {} + for k in self:splitter(separator) do t[#t+1] = k end + return t +end + +-- faster than a string:split: + +function string:splitchr(chr) + if #self > 0 then + local t = { } + for s in string.gmatch(self..chr,"(.-)"..chr) do + t[#t+1] = s + end + return t + else + return { } + end +end + +--~ function string.piecewise(str, pat, fnc) -- variant of split +--~ local fpat = "(.-)"..pat +--~ local last_end = 1 +--~ local s, e, cap = string.find(str, fpat, 1) +--~ while s ~= nil do +--~ if s~=1 or cap~="" then +--~ fnc(cap) +--~ end +--~ last_end = e+1 +--~ s, e, cap = string.find(str, fpat, last_end) +--~ end +--~ if last_end <= #str then +--~ fnc((string.sub(str,last_end))) +--~ end +--~ end + +function string.piecewise(str, pat, fnc) -- variant of split + for k in string.splitter(str,pat) do fnc(k) end +end + +--~ function string.piecewise(str, pat, fnc) -- variant of split +--~ for k in str:splitter(pat) do fnc(k) end +--~ end + +--~ do if lpeg then + +--~ -- this alternative is 30% faster esp when we cache them +--~ -- problem: no expressions + +--~ splitters = { } + +--~ function string:split(separator) +--~ if #self > 0 then +--~ local split = splitters[separator] +--~ if not split then +--~ -- based on code by Roberto +--~ local p = lpeg.P(separator) +--~ local c = lpeg.C((1-p)^0) +--~ split = lpeg.Ct(c*(p*c)^0) +--~ splitters[separator] = split +--~ end +--~ return split:match(self) +--~ else +--~ return { } +--~ end +--~ end + +--~ string.splitchr = string.split + +--~ function string:piecewise(separator,fnc) +--~ for _,v in pairs(self:split(separator)) do +--~ fnc(v) +--~ end +--~ end + +--~ end end + +string.chr_to_esc = { + ["%"] = "%%", + ["."] = "%.", + ["+"] = "%+", ["-"] = "%-", ["*"] = "%*", + ["^"] = "%^", ["$"] = "%$", + ["["] = "%[", ["]"] = "%]", + ["("] = "%(", [")"] = "%)", + ["{"] = "%{", ["}"] = "%}" +} + +function string:esc() -- variant 2 + return (self:gsub("(.)",string.chr_to_esc)) +end + +function string.unquote(str) + return (str:gsub("^([\"\'])(.*)%1$","%2")) +end + +function string.quote(str) + return '"' .. str:unquote() .. '"' +end + +function string:count(pattern) -- variant 3 + local n = 0 + for _ in self:gmatch(pattern) do + n = n + 1 + end + return n +end + +function string:limit(n,sentinel) + if #self > n then + sentinel = sentinel or " ..." + return self:sub(1,(n-#sentinel)) .. sentinel + else + return self + end +end + +function string:strip() + return (self:gsub("^%s*(.-)%s*$", "%1")) +end + +--~ function string.strip(str) -- slightly different +--~ return (string.gsub(string.gsub(str,"^%s*(.-)%s*$","%1"),"%s+"," ")) +--~ end + +function string:is_empty() + return not self:find("%S") +end + +function string:enhance(pattern,action) + local ok, n = true, 0 + while ok do + ok = false + self = self:gsub(pattern, function(...) + ok, n = true, n + 1 + return action(...) + end) + end + return self, n +end + +--~ function string:enhance(pattern,action) +--~ local ok, n = 0, 0 +--~ repeat +--~ self, ok = self:gsub(pattern, function(...) +--~ n = n + 1 +--~ return action(...) +--~ end) +--~ until ok == 0 +--~ return self, n +--~ end + +--~ function string:to_hex() +--~ if self then +--~ return (self:gsub("(.)",function(c) +--~ return string.format("%02X",c:byte()) +--~ end)) +--~ else +--~ return "" +--~ end +--~ end + +--~ function string:from_hex() +--~ if self then +--~ return (self:gsub("(..)",function(c) +--~ return string.char(tonumber(c,16)) +--~ end)) +--~ else +--~ return "" +--~ end +--~ end + +string.chr_to_hex = { } +string.hex_to_chr = { } + +for i=0,255 do + local c, h = string.char(i), string.format("%02X",i) + string.chr_to_hex[c], string.hex_to_chr[h] = h, c +end + +--~ function string:to_hex() +--~ if self then return (self:gsub("(.)",string.chr_to_hex)) else return "" end +--~ end + +--~ function string:from_hex() +--~ if self then return (self:gsub("(..)",string.hex_to_chr)) else return "" end +--~ end + +function string:to_hex() + return ((self or ""):gsub("(.)",string.chr_to_hex)) +end + +function string:from_hex() + return ((self or ""):gsub("(..)",string.hex_to_chr)) +end + +if not string.characters then + + local function nextchar(str, index) + index = index + 1 + return (index <= #str) and index or nil, str:sub(index,index) + end + function string:characters() + return nextchar, self, 0 + end + local function nextbyte(str, index) + index = index + 1 + return (index <= #str) and index or nil, string.byte(str:sub(index,index)) + end + function string:bytes() + return nextbyte, self, 0 + end + +end + +--~ function string:padd(n,chr) +--~ return self .. self.rep(chr or " ",n-#self) +--~ end + +function string:rpadd(n,chr) + local m = n-#self + if m > 0 then + return self .. self.rep(chr or " ",m) + else + return self + end +end + +function string:lpadd(n,chr) + local m = n-#self + if m > 0 then + return self.rep(chr or " ",m) .. self + else + return self + end +end + +string.padd = string.rpadd + +function is_number(str) + return str:find("^[%-%+]?[%d]-%.?[%d+]$") == 1 +end + +--~ print(is_number("1")) +--~ print(is_number("1.1")) +--~ print(is_number(".1")) +--~ print(is_number("-0.1")) +--~ print(is_number("+0.1")) +--~ print(is_number("-.1")) +--~ print(is_number("+.1")) + +function string:split_settings() -- no {} handling, see l-aux for lpeg variant + if self:find("=") then + local t = { } + for k,v in self:gmatch("(%a+)=([^%,]*)") do + t[k] = v + end + return t + else + return nil + end +end + +local patterns_escapes = { + ["-"] = "%-", + ["."] = "%.", + ["+"] = "%+", + ["*"] = "%*", + ["%"] = "%%", + ["("] = "%)", + [")"] = "%)", + ["["] = "%[", + ["]"] = "%]", +} + +function string:pattesc() + return (self:gsub(".",patterns_escapes)) +end + +function string:tohash() + local t = { } + for s in self:gmatch("([^, ]+)") do -- lpeg + t[s] = true + end + return t +end + + +-- filename : l-lpeg.lua +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-lpeg'] = 1.001 + +--~ l-lpeg.lua : + +--~ lpeg.digit = lpeg.R('09')^1 +--~ lpeg.sign = lpeg.S('+-')^1 +--~ lpeg.cardinal = lpeg.P(lpeg.sign^0 * lpeg.digit^1) +--~ lpeg.integer = lpeg.P(lpeg.sign^0 * lpeg.digit^1) +--~ lpeg.float = lpeg.P(lpeg.sign^0 * lpeg.digit^0 * lpeg.P('.') * lpeg.digit^1) +--~ lpeg.number = lpeg.float + lpeg.integer +--~ lpeg.oct = lpeg.P("0") * lpeg.R('07')^1 +--~ lpeg.hex = lpeg.P("0x") * (lpeg.R('09') + lpeg.R('AF'))^1 +--~ lpeg.uppercase = lpeg.P("AZ") +--~ lpeg.lowercase = lpeg.P("az") + +--~ lpeg.eol = lpeg.S('\r\n\f')^1 -- includes formfeed +--~ lpeg.space = lpeg.S(' ')^1 +--~ lpeg.nonspace = lpeg.P(1-lpeg.space)^1 +--~ lpeg.whitespace = lpeg.S(' \r\n\f\t')^1 +--~ lpeg.nonwhitespace = lpeg.P(1-lpeg.whitespace)^1 + +local hash = { } + +function lpeg.anywhere(pattern) --slightly adapted from website + return lpeg.P { lpeg.P(pattern) + 1 * lpeg.V(1) } +end + +function lpeg.startswith(pattern) --slightly adapted + return lpeg.P(pattern) +end + +--~ g = lpeg.splitter(" ",function(s) ... end) -- gmatch:lpeg = 3:2 + +function lpeg.splitter(pattern, action) + return (((1-lpeg.P(pattern))^1)/action+1)^0 +end + +local crlf = lpeg.P("\r\n") +local cr = lpeg.P("\r") +local lf = lpeg.P("\n") +local space = lpeg.S(" \t\f\v") +local newline = crlf + cr + lf +local spacing = space^0 * newline + +local empty = spacing * lpeg.Cc("") +local nonempty = lpeg.Cs((1-spacing)^1) * spacing^-1 +local content = (empty + nonempty)^1 + +local capture = lpeg.Ct(content^0) + +function string:splitlines() + return capture:match(self) +end + + +-- filename : l-table.lua +-- comment : split off from luat-lib +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-table'] = 1.001 + +table.join = table.concat + +function table.strip(tab) + local lst = { } + for k, v in ipairs(tab) do + -- s = string.gsub(v, "^%s*(.-)%s*$", "%1") + s = v:gsub("^%s*(.-)%s*$", "%1") + if s == "" then + -- skip this one + else + lst[#lst+1] = s + end + end + return lst +end + +--~ function table.sortedkeys(tab) +--~ local srt = { } +--~ for key,_ in pairs(tab) do +--~ srt[#srt+1] = key +--~ end +--~ table.sort(srt) +--~ return srt +--~ end + +function table.sortedkeys(tab) + local srt, kind = { }, 0 -- 0=unknown 1=string, 2=number 3=mixed + for key,_ in pairs(tab) do + srt[#srt+1] = key + if kind == 3 then + -- no further check + else + local tkey = type(key) + if tkey == "string" then + -- if kind == 2 then kind = 3 else kind = 1 end + kind = (kind == 2 and 3) or 1 + elseif tkey == "number" then + -- if kind == 1 then kind = 3 else kind = 2 end + kind = (kind == 1 and 3) or 2 + else + kind = 3 + end + end + end + if kind == 0 or kind == 3 then + table.sort(srt,function(a,b) return (tostring(a) < tostring(b)) end) + else + table.sort(srt) + end + return srt +end + +function table.append(t, list) + for _,v in pairs(list) do + table.insert(t,v) + end +end + +function table.prepend(t, list) + for k,v in pairs(list) do + table.insert(t,k,v) + end +end + +function table.merge(t, ...) -- first one is target + t = t or {} + local lst = {...} + for i=1,#lst do + for k, v in pairs(lst[i]) do + t[k] = v + end + end + return t +end + +function table.merged(...) + local tmp, lst = { }, {...} + for i=1,#lst do + for k, v in pairs(lst[i]) do + tmp[k] = v + end + end + return tmp +end + +function table.imerge(t, ...) + local lst = {...} + for i=1,#lst do + local nst = lst[i] + for j=1,#nst do + t[#t+1] = nst[j] + end + end + return t +end + +function table.imerged(...) + local tmp, lst = { }, {...} + for i=1,#lst do + local nst = lst[i] + for j=1,#nst do + tmp[#tmp+1] = nst[j] + end + end + return tmp +end + +if not table.fastcopy then do + + local type, pairs, getmetatable, setmetatable = type, pairs, getmetatable, setmetatable + + local function fastcopy(old) -- fast one + if old then + local new = { } + for k,v in pairs(old) do + if type(v) == "table" then + new[k] = fastcopy(v) -- was just table.copy + else + new[k] = v + end + end + local mt = getmetatable(old) + if mt then + setmetatable(new,mt) + end + return new + else + return { } + end + end + + table.fastcopy = fastcopy + +end end + +if not table.copy then do + + local type, pairs, getmetatable, setmetatable = type, pairs, getmetatable, setmetatable + + local function copy(t, tables) -- taken from lua wiki, slightly adapted + tables = tables or { } + local tcopy = {} + if not tables[t] then + tables[t] = tcopy + end + for i,v in pairs(t) do -- brrr, what happens with sparse indexed + if type(i) == "table" then + if tables[i] then + i = tables[i] + else + i = copy(i, tables) + end + end + if type(v) ~= "table" then + tcopy[i] = v + elseif tables[v] then + tcopy[i] = tables[v] + else + tcopy[i] = copy(v, tables) + end + end + local mt = getmetatable(t) + if mt then + setmetatable(tcopy,mt) + end + return tcopy + end + + table.copy = copy + +end end + +-- rougly: copy-loop : unpack : sub == 0.9 : 0.4 : 0.45 (so in critical apps, use unpack) + +function table.sub(t,i,j) + return { unpack(t,i,j) } +end + +function table.replace(a,b) + for k,v in pairs(b) do + a[k] = v + end +end + +-- slower than #t on indexed tables (#t only returns the size of the numerically indexed slice) + +function table.is_empty(t) + return not t or not next(t) +end + +function table.one_entry(t) + local n = next(t) + return n and not next(t,n) +end + +function table.starts_at(t) + return ipairs(t,1)(t,0) +end + +do + + -- one of my first exercises in lua ... + + -- 34.055.092 32.403.326 arabtype.tma + -- 1.620.614 1.513.863 lmroman10-italic.tma + -- 1.325.585 1.233.044 lmroman10-regular.tma + -- 1.248.157 1.158.903 lmsans10-regular.tma + -- 194.646 153.120 lmtypewriter10-regular.tma + -- 1.771.678 1.658.461 palatinosanscom-bold.tma + -- 1.695.251 1.584.491 palatinosanscom-regular.tma + -- 13.736.534 13.409.446 zapfinoextraltpro.tma + + -- 13.679.038 11.774.106 arabtype.tmc + -- 886.248 754.944 lmroman10-italic.tmc + -- 729.828 466.864 lmroman10-regular.tmc + -- 688.482 441.962 lmsans10-regular.tmc + -- 128.685 95.853 lmtypewriter10-regular.tmc + -- 715.929 582.985 palatinosanscom-bold.tmc + -- 669.942 540.126 palatinosanscom-regular.tmc + -- 1.560.588 1.317.000 zapfinoextraltpro.tmc + + table.serialize_functions = true + table.serialize_compact = true + table.serialize_inline = true + + local function key(k) + if type(k) == "number" then -- or k:find("^%d+$") then + return "["..k.."]" + elseif noquotes and k:find("^%a[%a%d%_]*$") then + return k + else + return '["'..k..'"]' + end + end + + local function simple_table(t) + if #t > 0 then + local n = 0 + for _,v in pairs(t) do + n = n + 1 + end + if n == #t then + local tt = { } + for i=1,#t do + local v = t[i] + local tv = type(v) + if tv == "number" or tv == "boolean" then + tt[#tt+1] = tostring(v) + elseif tv == "string" then + tt[#tt+1] = ("%q"):format(v) + else + tt = nil + break + end + end + return tt + end + end + return nil + end + + local function serialize(root,name,handle,depth,level,reduce,noquotes,indexed) + handle = handle or print + reduce = reduce or false + if depth then + depth = depth .. " " + if indexed then + handle(("%s{"):format(depth)) + else + handle(("%s%s={"):format(depth,key(name))) + end + else + depth = "" + local tname = type(name) + if tname == "string" then + if name == "return" then + handle("return {") + else + handle(name .. "={") + end + elseif tname == "number" then + handle("[" .. name .. "]={") + elseif tname == "boolean" then + if name then + handle("return {") + else + handle("{") + end + else + handle("t={") + end + end + if root and next(root) then + local compact = table.serialize_compact + local inline = compact and table.serialize_inline + local first, last = nil, 0 -- #root cannot be trusted here + if compact then + for k,v in ipairs(root) do -- NOT: for k=1,#root do (why) + if not first then first = k end + last = last + 1 + end + end + for _,k in pairs(table.sortedkeys(root)) do + local v = root[k] + local t = type(v) + if compact and first and type(k) == "number" and k >= first and k <= last then + if t == "number" then + handle(("%s %s,"):format(depth,v)) + elseif t == "string" then + if reduce and (v:find("^[%-%+]?[%d]-%.?[%d+]$") == 1) then + handle(("%s %s,"):format(depth,v)) + else + handle(("%s %q,"):format(depth,v)) + end + elseif t == "table" then + if not next(v) then + handle(("%s {},"):format(depth)) + elseif inline then + local st = simple_table(v) + if st then + handle(("%s { %s },"):format(depth,table.concat(st,", "))) + else + serialize(v,k,handle,depth,level+1,reduce,noquotes,true) + end + else + serialize(v,k,handle,depth,level+1,reduce,noquotes,true) + end + elseif t == "boolean" then + handle(("%s %s,"):format(depth,tostring(v))) + elseif t == "function" then + if table.serialize_functions then + handle(('%s loadstring(%q),'):format(depth,string.dump(v))) + else + handle(('%s "function",'):format(depth)) + end + else + handle(("%s %q,"):format(depth,tostring(v))) + end + elseif k == "__p__" then -- parent + if false then + handle(("%s __p__=nil,"):format(depth)) + end + elseif t == "number" then + handle(("%s %s=%s,"):format(depth,key(k),v)) + elseif t == "string" then + if reduce and (v:find("^[%-%+]?[%d]-%.?[%d+]$") == 1) then + handle(("%s %s=%s,"):format(depth,key(k),v)) + else + handle(("%s %s=%q,"):format(depth,key(k),v)) + end + elseif t == "table" then + if not next(v) then + handle(("%s %s={},"):format(depth,key(k))) + elseif inline then + local st = simple_table(v) + if st then + handle(("%s %s={ %s },"):format(depth,key(k),table.concat(st,", "))) + else + serialize(v,k,handle,depth,level+1,reduce,noquotes) + end + else + serialize(v,k,handle,depth,level+1,reduce,noquotes) + end + elseif t == "boolean" then + handle(("%s %s=%s,"):format(depth,key(k),tostring(v))) + elseif t == "function" then + if table.serialize_functions then + handle(('%s %s=loadstring(%q),'):format(depth,key(k),string.dump(v))) + else + handle(('%s %s="function",'):format(depth,key(k))) + end + else + handle(("%s %s=%q,"):format(depth,key(k),tostring(v))) + -- handle(('%s %s=loadstring(%q),'):format(depth,key(k),string.dump(function() return v end))) + end + end + if level > 0 then + handle(("%s},"):format(depth)) + else + handle(("%s}"):format(depth)) + end + else + handle(("%s}"):format(depth)) + end + end + + --~ name: + --~ + --~ true : return { } + --~ false : { } + --~ nil : t = { } + --~ string : string = { } + --~ 'return' : return { } + --~ number : [number] = { } + + function table.serialize(root,name,reduce,noquotes) + local t = { } + local function flush(s) + t[#t+1] = s + end + serialize(root, name, flush, nil, 0, reduce, noquotes) + return table.concat(t,"\n") + end + + function table.tohandle(handle,root,name,reduce,noquotes) + serialize(root, name, handle, nil, 0, reduce, noquotes) + end + + -- sometimes tables are real use (zapfino extra pro is some 85M) in which + -- case a stepwise serialization is nice; actually, we could consider: + -- + -- for line in table.serializer(root,name,reduce,noquotes) do + -- ...(line) + -- end + -- + -- so this is on the todo list + + table.tofile_maxtab = 2*1024 + + function table.tofile(filename,root,name,reduce,noquotes) + local f = io.open(filename,'w') + if f then + local concat = table.concat + local maxtab = table.tofile_maxtab + if maxtab > 1 then + local t = { } + local function flush(s) + t[#t+1] = s + if #t > maxtab then + f:write(concat(t,"\n"),"\n") -- hm, write(sometable) should be nice + t = { } + end + end + serialize(root, name, flush, nil, 0, reduce, noquotes) + f:write(concat(t,"\n"),"\n") + else + local function flush(s) + f:write(s,"\n") + end + serialize(root, name, flush, nil, 0, reduce, noquotes) + end + f:close() + end + end + +end + +--~ t = { +--~ b = "123", +--~ a = "x", +--~ c = 1.23, +--~ d = "1.23", +--~ e = true, +--~ f = { +--~ d = "1.23", +--~ a = "x", +--~ b = "123", +--~ c = 1.23, +--~ e = true, +--~ f = { +--~ e = true, +--~ f = { +--~ e = true +--~ }, +--~ }, +--~ }, +--~ g = function() end +--~ } + +--~ print(table.serialize(t), "\n") +--~ print(table.serialize(t,"name"), "\n") +--~ print(table.serialize(t,false), "\n") +--~ print(table.serialize(t,true), "\n") +--~ print(table.serialize(t,"name",true), "\n") +--~ print(table.serialize(t,"name",true,true), "\n") + +do + + local function flatten(t,f,complete) + for i=1,#t do + local v = t[i] + if type(v) == "table" then + if complete or type(v[1]) == "table" then + flatten(v,f,complete) + else + f[#f+1] = v + end + else + f[#f+1] = v + end + end + end + + function table.flatten(t) + local f = { } + flatten(t,f,true) + return f + end + + function table.unnest(t) -- bad name + local f = { } + flatten(t,f,false) + return f + end + + table.flatten_one_level = table.unnest + +end + +function table.insert_before_value(t,value,str) + for i=1,#t do + if t[i] == value then + table.insert(t,i,str) + return + end + end + table.insert(t,1,str) +end + +function table.insert_after_value(t,value,str) + for i=1,#t do + if t[i] == value then + table.insert(t,i+1,str) + return + end + end + t[#t+1] = str +end + +function table.are_equal(a,b,n,m) + if #a == #b then + n = n or 1 + m = m or #a + for i=n,m do + local ai, bi = a[i], b[i] + if (ai==bi) or (type(ai)=="table" and type(bi)=="table" and table.are_equal(ai,bi)) then + -- continue + else + return false + end + end + return true + else + return false + end +end + +function table.compact(t) + if t then + for k,v in pairs(t) do + if not next(v) then + t[k] = nil + end + end + end +end + +function table.tohash(t) + local h = { } + for _, v in pairs(t) do -- no ipairs here + h[v] = true + end + return h +end + +function table.fromhash(t) + local h = { } + for k, v in pairs(t) do -- no ipairs here + if v then h[#h+1] = k end + end + return h +end + +function table.contains(t, v) + if t then + for i=1, #t do + if t[i] == v then + return true + end + end + end + return false +end + +function table.count(t) + local n, e = 0, next(t) + while e do + n, e = n + 1, next(t,e) + end + return n +end + +function table.swapped(t) + local s = { } + for k, v in pairs(t) do + s[v] = k + end + return s +end + +--~ function table.are_equal(a,b) +--~ return table.serialize(a) == table.serialize(b) +--~ end + +function table.clone(t,p) -- t is optional or nil or table + if not p then + t, p = { }, t or { } + elseif not t then + t = { } + end + setmetatable(t, { __index = function(_,key) return p[key] end }) + return t +end + + +function table.hexed(t,seperator) + local tt = { } + for i=1,#t do tt[i] = string.format("0x%04X",t[i]) end + return table.concat(tt,seperator or " ") +end + +function table.reverse_hash(h) + local r = { } + for k,v in pairs(h) do + r[v] = (k:gsub(" ","")):lower() + end + return r +end + +function table.reverse(t) + local tt = { } + if #t > 0 then + for i=#t,1,-1 do + tt[#tt+1] = t[i] + end + end + return tt +end + + +-- filename : l-io.lua +-- comment : split off from luat-lib +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-io'] = 1.001 + +if string.find(os.getenv("PATH"),";") then + io.fileseparator, io.pathseparator = "\\", ";" +else + io.fileseparator, io.pathseparator = "/" , ":" +end + +function io.loaddata(filename) + local f = io.open(filename,'rb') + if f then + local data = f:read('*all') + f:close() + return data + else + return nil + end +end + +function io.savedata(filename,data,joiner) + local f = io.open(filename, "wb") + if f then + if type(data) == "table" then + f:write(table.join(data,joiner or "")) + elseif type(data) == "function" then + data(f) + else + f:write(data) + end + f:close() + end +end + +function io.exists(filename) + local f = io.open(filename) + if f == nil then + return false + else + assert(f:close()) + return true + end +end + +function io.size(filename) + local f = io.open(filename) + if f == nil then + return 0 + else + local s = f:seek("end") + assert(f:close()) + return s + end +end + +function io.noflines(f) + local n = 0 + for _ in f:lines() do + n = n + 1 + end + f:seek('set',0) + return n +end + +do + + local sb = string.byte + + local nextchar = { + [ 4] = function(f) + return f:read(1,1,1,1) + end, + [ 2] = function(f) + return f:read(1,1) + end, + [ 1] = function(f) + return f:read(1) + end, + [-2] = function(f) + local a, b = f:read(1,1) + return b, a + end, + [-4] = function(f) + local a, b, c, d = f:read(1,1,1,1) + return d, c, b, a + end + } + + function io.characters(f,n) + if f then + return nextchar[n or 1], f + else + return nil, nil + end + end + +end + +do + + local sb = string.byte + +--~ local nextbyte = { +--~ [4] = function(f) +--~ local a = f:read(1) +--~ local b = f:read(1) +--~ local c = f:read(1) +--~ local d = f:read(1) +--~ if d then +--~ return sb(a), sb(b), sb(c), sb(d) +--~ else +--~ return nil, nil, nil, nil +--~ end +--~ end, +--~ [2] = function(f) +--~ local a = f:read(1) +--~ local b = f:read(1) +--~ if b then +--~ return sb(a), sb(b) +--~ else +--~ return nil, nil +--~ end +--~ end, +--~ [1] = function (f) +--~ local a = f:read(1) +--~ if a then +--~ return sb(a) +--~ else +--~ return nil +--~ end +--~ end, +--~ [-2] = function (f) +--~ local a = f:read(1) +--~ local b = f:read(1) +--~ if b then +--~ return sb(b), sb(a) +--~ else +--~ return nil, nil +--~ end +--~ end, +--~ [-4] = function(f) +--~ local a = f:read(1) +--~ local b = f:read(1) +--~ local c = f:read(1) +--~ local d = f:read(1) +--~ if d then +--~ return sb(d), sb(c), sb(b), sb(a) +--~ else +--~ return nil, nil, nil, nil +--~ end +--~ end +--~ } + + local nextbyte = { + [4] = function(f) + local a, b, c, d = f:read(1,1,1,1) + if d then + return sb(a), sb(b), sb(c), sb(d) + else + return nil, nil, nil, nil + end + end, + [2] = function(f) + local a, b = f:read(1,1) + if b then + return sb(a), sb(b) + else + return nil, nil + end + end, + [1] = function (f) + local a = f:read(1) + if a then + return sb(a) + else + return nil + end + end, + [-2] = function (f) + local a, b = f:read(1,1) + if b then + return sb(b), sb(a) + else + return nil, nil + end + end, + [-4] = function(f) + local a, b, c, d = f:read(1,1,1,1) + if d then + return sb(d), sb(c), sb(b), sb(a) + else + return nil, nil, nil, nil + end + end + } + + function io.bytes(f,n) + if f then + return nextbyte[n or 1], f + else + return nil, nil + end + end + +end + +function io.ask(question,default,options) + while true do + io.write(question) + if options then + io.write(string.format(" [%s]",table.concat(options,"|"))) + end + if default then + io.write(string.format(" [%s]",default)) + end + io.write(string.format(" ")) + local answer = io.read() + answer = answer:gsub("^%s*(.*)%s*$","%1") + if answer == "" and default then + return default + elseif not options then + return answer + else + for _,v in pairs(options) do + if v == answer then + return answer + end + end + local pattern = "^" .. answer + for _,v in pairs(options) do + if v:find(pattern) then + return v + end + end + end + end +end + + +-- filename : l-number.lua +-- comment : split off from luat-lib +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-number'] = 1.001 + +if not number then number = { } end + +-- a,b,c,d,e,f = number.toset(100101) + +function number.toset(n) + return (tostring(n)):match("(.?)(.?)(.?)(.?)(.?)(.?)(.?)(.?)") +end + +local format = string.format + +function number.toevenhex(n) + local s = format("%X",n) + if #s % 2 == 0 then + return s + else + return "0" .. s + end +end + +-- the lpeg way is slower on 8 digits, but faster on 4 digits, some 7.5% +-- on +-- +-- for i=1,1000000 do +-- local a,b,c,d,e,f,g,h = number.toset(12345678) +-- local a,b,c,d = number.toset(1234) +-- local a,b,c = number.toset(123) +-- end +-- +-- of course dedicated "(.)(.)(.)(.)" matches are even faster + +do + local one = lpeg.C(1-lpeg.S(''))^1 + + function number.toset(n) + return one:match(tostring(n)) + end +end + + + +-- filename : l-set.lua +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-set'] = 1.001 + +if not set then set = { } end + +do + + local nums = { } + local tabs = { } + local concat = table.concat + + set.create = table.tohash + + function set.tonumber(t) + if next(t) then + local s = "" + -- we could save mem by sorting, but it slows down + for k, v in pairs(t) do + if v then + -- why bother about the leading space + s = s .. " " .. k + end + end + if not nums[s] then + tabs[#tabs+1] = t + nums[s] = #tabs + end + return nums[s] + else + return 0 + end + end + + function set.totable(n) + if n == 0 then + return { } + else + return tabs[n] or { } + end + end + + function set.contains(n,s) + if type(n) == "table" then + return n[s] + elseif n == 0 then + return false + else + local t = tabs[n] + return t and t[s] + end + end + +end + +--~ local c = set.create{'aap','noot','mies'} +--~ local s = set.tonumber(c) +--~ local t = set.totable(s) +--~ print(t['aap']) +--~ local c = set.create{'zus','wim','jet'} +--~ local s = set.tonumber(c) +--~ local t = set.totable(s) +--~ print(t['aap']) +--~ print(t['jet']) +--~ print(set.contains(t,'jet')) +--~ print(set.contains(t,'aap')) + + + +-- filename : l-os.lua +-- comment : split off from luat-lib +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-os'] = 1.001 + +function os.resultof(command) + return io.popen(command,"r"):read("*all") +end + +if not os.exec then os.exec = os.execute end +if not os.spawn then os.spawn = os.execute end + +--~ os.type : windows | unix (new, we already guessed os.platform) +--~ os.name : windows | msdos | linux | macosx | solaris | .. | generic (new) + +if not io.fileseparator then + if string.find(os.getenv("PATH"),";") then + io.fileseparator, io.pathseparator, os.platform = "\\", ";", os.type or "windows" + else + io.fileseparator, io.pathseparator, os.platform = "/" , ":", os.type or "unix" + end +end + +os.platform = os.platform or os.type or (io.pathseparator == ";" and "windows") or "unix" + +function os.launch(str) + if os.platform == "windows" then + os.execute("start " .. str) -- os.spawn ? + else + os.execute(str .. " &") -- os.spawn ? + end +end + +if not os.setenv then + function os.setenv() return false end +end + +if not os.times then + -- utime = user time + -- stime = system time + -- cutime = children user time + -- cstime = children system time + function os.times() + return { + utime = os.gettimeofday(), -- user + stime = 0, -- system + cutime = 0, -- children user + cstime = 0, -- children system + } + end +end + +os.gettimeofday = os.gettimeofday or os.clock + +do + local startuptime = os.gettimeofday() + function os.runtime() + return os.gettimeofday() - startuptime + end +end + +--~ print(os.gettimeofday()-os.time()) +--~ os.sleep(1.234) +--~ print (">>",os.runtime()) +--~ print(os.date("%H:%M:%S",os.gettimeofday())) +--~ print(os.date("%H:%M:%S",os.time())) + + +-- filename : l-md5.lua +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-md5'] = 1.001 + +if md5 then do + + local function convert(str,fmt) + return (string.gsub(md5.sum(str),".",function(chr) return string.format(fmt,string.byte(chr)) end)) + end + + if not md5.HEX then function md5.HEX(str) return convert(str,"%02X") end end + if not md5.hex then function md5.hex(str) return convert(str,"%02x") end end + if not md5.dec then function md5.dec(str) return convert(str,"%03i") end end + +end end + + +-- filename : l-file.lua +-- comment : split off from luat-lib +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-file'] = 1.001 + +if not file then file = { } end + +function file.removesuffix(filename) + return filename:gsub("%.[%a%d]+$", "") +end + +function file.addsuffix(filename, suffix) + if not filename:find("%.[%a%d]+$") then + return filename .. "." .. suffix + else + return filename + end +end + +function file.replacesuffix(filename, suffix) + if not filename:find("%.[%a%d]+$") then + return filename .. "." .. suffix + else + return (filename:gsub("%.[%a%d]+$","."..suffix)) + end +end + +function file.dirname(name) + return name:match("^(.+)[/\\].-$") or "" +end + +function file.basename(name) + return name:match("^.+[/\\](.-)$") or name +end + +function file.nameonly(name) + return ((name:match("^.+[/\\](.-)$") or name):gsub("%..*$","")) +end + +function file.extname(name) + return name:match("^.+%.([^/\\]-)$") or "" +end + +file.suffix = file.extname + +function file.stripsuffix(name) + return (name:gsub("%.[%a%d]+$","")) +end + +--~ function file.join(...) +--~ local t = { ... } +--~ for i=1,#t do +--~ t[i] = (t[i]:gsub("\\","/")):gsub("/+$","") +--~ end +--~ return table.concat(t,"/") +--~ end + +--~ print(file.join("x/","/y")) +--~ print(file.join("http://","/y")) +--~ print(file.join("http://a","/y")) +--~ print(file.join("http:///a","/y")) +--~ print(file.join("//nas-1","/y")) + +function file.join(...) + local pth = table.concat({...},"/") + pth = pth:gsub("\\","/") + local a, b = pth:match("^(.*://)(.*)$") + if a and b then + return a .. b:gsub("//+","/") + end + a, b = pth:match("^(//)(.*)$") + if a and b then + return a .. b:gsub("//+","/") + end + return (pth:gsub("//+","/")) +end + +function file.is_writable(name) + local f = io.open(name, 'w') + if f then + f:close() + return true + else + return false + end +end + +function file.is_readable(name) + local f = io.open(name,'r') + if f then + f:close() + return true + else + return false + end +end + +--~ function file.split_path(str) +--~ if str:find(';') then +--~ return str:splitchr(";") +--~ else +--~ return str:splitchr(io.pathseparator) +--~ end +--~ end + +-- todo: lpeg + +function file.split_path(str) + local t = { } + str = str:gsub("\\", "/") + str = str:gsub("(%a):([;/])", "%1\001%2") + for name in str:gmatch("([^;:]+)") do + if name ~= "" then + name = name:gsub("\001",":") + t[#t+1] = name + end + end + return t +end + +function file.join_path(tab) + return table.concat(tab,io.pathseparator) -- can have trailing // +end + +--~ print('test' .. " == " .. file.collapse_path("test")) +--~ print("test/test" .. " == " .. file.collapse_path("test/test")) +--~ print("test/test/test" .. " == " .. file.collapse_path("test/test/test")) +--~ print("test/test" .. " == " .. file.collapse_path("test/../test/test")) +--~ print("test" .. " == " .. file.collapse_path("test/../test")) +--~ print("../test" .. " == " .. file.collapse_path("../test")) +--~ print("../test/" .. " == " .. file.collapse_path("../test/")) +--~ print("a/a" .. " == " .. file.collapse_path("a/b/c/../../a")) + +--~ function file.collapse_path(str) +--~ local ok, n = false, 0 +--~ while not ok do +--~ ok = true +--~ str, n = str:gsub("[^%./]+/%.%./", function(s) +--~ ok = false +--~ return "" +--~ end) +--~ end +--~ return (str:gsub("/%./","/")) +--~ end + +function file.collapse_path(str) + local n = 1 + while n > 0 do + str, n = str:gsub("([^/%.]+/%.%./)","") + end + return (str:gsub("/%./","/")) +end + +function file.robustname(str) + return (str:gsub("[^%a%d%/%-%.\\]+","-")) +end + +file.readdata = io.loaddata +file.savedata = io.savedata + +function file.copy(oldname,newname) + file.savedata(newname,io.loaddata(oldname)) +end + + +-- filename : l-url.lua +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-url'] = 1.001 +if not url then url = { } end + +-- from the spec (on the web): +-- +-- foo://example.com:8042/over/there?name=ferret#nose +-- \_/ \______________/\_________/ \_________/ \__/ +-- | | | | | +-- scheme authority path query fragment +-- | _____________________|__ +-- / \ / \ +-- urn:example:animal:ferret:nose + +do + + local function tochar(s) + return string.char(tonumber(s,16)) + end + + local colon, qmark, hash, slash, percent, endofstring = lpeg.P(":"), lpeg.P("?"), lpeg.P("#"), lpeg.P("/"), lpeg.P("%"), lpeg.P(-1) + + local hexdigit = lpeg.R("09","AF","af") + local escaped = percent * lpeg.C(hexdigit * hexdigit) / tochar + + local scheme = lpeg.Cs((escaped+(1-colon-slash-qmark-hash))^0) * colon + lpeg.Cc("") + local authority = slash * slash * lpeg.Cs((escaped+(1- slash-qmark-hash))^0) + lpeg.Cc("") + local path = slash * lpeg.Cs((escaped+(1- qmark-hash))^0) + lpeg.Cc("") + local query = qmark * lpeg.Cs((escaped+(1- hash))^0) + lpeg.Cc("") + local fragment = hash * lpeg.Cs((escaped+(1- endofstring))^0) + lpeg.Cc("") + + local parser = lpeg.Ct(scheme * authority * path * query * fragment) + + function url.split(str) + return (type(str) == "string" and parser:match(str)) or str + end + +end + +function url.hashed(str) + local s = url.split(str) + return { + scheme = (s[1] ~= "" and s[1]) or "file", + authority = s[2], + path = s[3], + query = s[4], + fragment = s[5], + original = str + } +end + +function url.filename(filename) + local t = url.hashed(filename) + return (t.scheme == "file" and t.path:gsub("^/([a-zA-Z])([:|])/)","%1:")) or filename +end + +function url.query(str) + if type(str) == "string" then + local t = { } + for k, v in str:gmatch("([^&=]*)=([^&=]*)") do + t[k] = v + end + return t + else + return str + end +end + +--~ print(url.filename("file:///c:/oeps.txt")) +--~ print(url.filename("c:/oeps.txt")) +--~ print(url.filename("file:///oeps.txt")) +--~ print(url.filename("file:///etc/test.txt")) +--~ print(url.filename("/oeps.txt")) + +-- from the spec on the web (sort of): +--~ +--~ function test(str) +--~ print(table.serialize(url.hashed(str))) +--~ end +---~ +--~ test("%56pass%20words") +--~ test("file:///c:/oeps.txt") +--~ test("file:///c|/oeps.txt") +--~ test("file:///etc/oeps.txt") +--~ test("file://./etc/oeps.txt") +--~ test("file:////etc/oeps.txt") +--~ test("ftp://ftp.is.co.za/rfc/rfc1808.txt") +--~ test("http://www.ietf.org/rfc/rfc2396.txt") +--~ test("ldap://[2001:db8::7]/c=GB?objectClass?one#what") +--~ test("mailto:John.Doe@example.com") +--~ test("news:comp.infosystems.www.servers.unix") +--~ test("tel:+1-816-555-1212") +--~ test("telnet://192.0.2.16:80/") +--~ test("urn:oasis:names:specification:docbook:dtd:xml:4.1.2") +--~ test("/etc/passwords") +--~ test("http://www.pragma-ade.com/spaced%20name") + +--~ test("zip:///oeps/oeps.zip#bla/bla.tex") +--~ test("zip:///oeps/oeps.zip?bla/bla.tex") + + +-- filename : l-dir.lua +-- comment : split off from luat-lib +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-dir'] = 1.001 + +dir = { } + +-- optimizing for no string.find (*) does not save time + +if lfs then do + +--~ local attributes = lfs.attributes +--~ local walkdir = lfs.dir +--~ +--~ local function glob_pattern(path,patt,recurse,action) +--~ local ok, scanner = xpcall(function() return walkdir(path) end, function() end) -- kepler safe +--~ if ok and type(scanner) == "function" then +--~ if not path:find("/$") then path = path .. '/' end +--~ for name in scanner do +--~ local full = path .. name +--~ local mode = attributes(full,'mode') +--~ if mode == 'file' then +--~ if name:find(patt) then +--~ action(full) +--~ end +--~ elseif recurse and (mode == "directory") and (name ~= '.') and (name ~= "..") then +--~ glob_pattern(full,patt,recurse,action) +--~ end +--~ end +--~ end +--~ end +--~ +--~ dir.glob_pattern = glob_pattern +--~ +--~ local function glob(pattern, action) +--~ local t = { } +--~ local action = action or function(name) t[#t+1] = name end +--~ local path, patt = pattern:match("^(.*)/*%*%*/*(.-)$") +--~ local recurse = path and patt +--~ if not recurse then +--~ path, patt = pattern:match("^(.*)/(.-)$") +--~ if not (path and patt) then +--~ path, patt = '.', pattern +--~ end +--~ end +--~ patt = patt:gsub("([%.%-%+])", "%%%1") +--~ patt = patt:gsub("%*", ".*") +--~ patt = patt:gsub("%?", ".") +--~ patt = "^" .. patt .. "$" +--~ -- print('path: ' .. path .. ' | pattern: ' .. patt .. ' | recurse: ' .. tostring(recurse)) +--~ glob_pattern(path,patt,recurse,action) +--~ return t +--~ end +--~ +--~ dir.glob = glob + + local attributes = lfs.attributes + local walkdir = lfs.dir + + local function glob_pattern(path,patt,recurse,action) + local ok, scanner + if path == "/" then + ok, scanner = xpcall(function() return walkdir(path..".") end, function() end) -- kepler safe + else + ok, scanner = xpcall(function() return walkdir(path) end, function() end) -- kepler safe + end + if ok and type(scanner) == "function" then + if not path:find("/$") then path = path .. '/' end + for name in scanner do + local full = path .. name + local mode = attributes(full,'mode') + if mode == 'file' then + if full:find(patt) then + action(full) + end + elseif recurse and (mode == "directory") and (name ~= '.') and (name ~= "..") then + glob_pattern(full,patt,recurse,action) + end + end + end + end + + dir.glob_pattern = glob_pattern + + --~ local function glob(pattern, action) + --~ local t = { } + --~ local path, rest, patt, recurse + --~ local action = action or function(name) t[#t+1] = name end + --~ local pattern = pattern:gsub("^%*%*","./**") + --~ local pattern = pattern:gsub("/%*/","/**/") + --~ path, rest = pattern:match("^(/)(.-)$") + --~ if path then + --~ path = path + --~ else + --~ path, rest = pattern:match("^([^/]*)/(.-)$") + --~ end + --~ if rest then + --~ patt = rest:gsub("([%.%-%+])", "%%%1") + --~ end + --~ patt = patt:gsub("%*", "[^/]*") + --~ patt = patt:gsub("%?", "[^/]") + --~ patt = patt:gsub("%[%^/%]%*%[%^/%]%*", ".*") + --~ if path == "" then path = "." end + --~ recurse = patt:find("%.%*/") ~= nil + --~ glob_pattern(path,patt,recurse,action) + --~ return t + --~ end + + local P, S, R, C, Cc, Cs, Ct, Cv, V = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.Cc, lpeg.Cs, lpeg.Ct, lpeg.Cv, lpeg.V + + local pattern = Ct { + [1] = (C(P(".") + P("/")^1) + C(R("az","AZ") * P(":") * P("/")^0) + Cc("./")) * V(2) * V(3), + [2] = C(((1-S("*?/"))^0 * P("/"))^0), + [3] = C(P(1)^0) + } + + local filter = Cs ( ( + P("**") / ".*" + + P("*") / "[^/]*" + + P("?") / "[^/]" + + P(".") / "%%." + + P("+") / "%%+" + + P("-") / "%%-" + + P(1) + )^0 ) + + local function glob(str,t) + if type(str) == "table" then + local t = t or { } + for _, s in ipairs(str) do + glob(s,t) + end + return t + else + local split = pattern:match(str) + if split then + local t = t or { } + local action = action or function(name) t[#t+1] = name end + local root, path, base = split[1], split[2], split[3] + local recurse = base:find("**") + local start = root .. path + local result = filter:match(start .. base) + glob_pattern(start,result,recurse,action) + return t + else + return { } + end + end + end + + dir.glob = glob + + --~ list = dir.glob("**/*.tif") + --~ list = dir.glob("/**/*.tif") + --~ list = dir.glob("./**/*.tif") + --~ list = dir.glob("oeps/**/*.tif") + --~ list = dir.glob("/oeps/**/*.tif") + + local function globfiles(path,recurse,func,files) -- func == pattern or function + if type(func) == "string" then + local s = func -- alas, we need this indirect way + func = function(name) return name:find(s) end + end + files = files or { } + for name in walkdir(path) do + if name:find("^%.") then + --- skip + elseif attributes(name,'mode') == "directory" then + if recurse then + globfiles(path .. "/" .. name,recurse,func,files) + end + elseif func then + if func(name) then + files[#files+1] = path .. "/" .. name + end + else + files[#files+1] = path .. "/" .. name + end + end + return files + end + + dir.globfiles = globfiles + + -- t = dir.glob("c:/data/develop/context/sources/**/????-*.tex") + -- t = dir.glob("c:/data/develop/tex/texmf/**/*.tex") + -- t = dir.glob("c:/data/develop/context/texmf/**/*.tex") + -- t = dir.glob("f:/minimal/tex/**/*") + -- print(dir.ls("f:/minimal/tex/**/*")) + -- print(dir.ls("*.tex")) + + function dir.ls(pattern) + return table.concat(glob(pattern),"\n") + end + + --~ mkdirs("temp") + --~ mkdirs("a/b/c") + --~ mkdirs(".","/a/b/c") + --~ mkdirs("a","b","c") + + local make_indeed = true -- false + + if string.find(os.getenv("PATH"),";") then + + function dir.mkdirs(...) + local str, pth = "", "" + for _, s in ipairs({...}) do + if s ~= "" then + if str ~= "" then + str = str .. "/" .. s + else + str = s + end + end + end + local first, middle, last + local drive = false + first, middle, last = str:match("^(//)(//*)(.*)$") + if first then + -- empty network path == local path + else + first, last = str:match("^(//)/*(.-)$") + if first then + middle, last = str:match("([^/]+)/+(.-)$") + if middle then + pth = "//" .. middle + else + pth = "//" .. last + last = "" + end + else + first, middle, last = str:match("^([a-zA-Z]:)(/*)(.-)$") + if first then + pth, drive = first .. middle, true + else + middle, last = str:match("^(/*)(.-)$") + if not middle then + last = str + end + end + end + end + for s in last:gmatch("[^/]+") do + if pth == "" then + pth = s + elseif drive then + pth, drive = pth .. s, false + else + pth = pth .. "/" .. s + end + if make_indeed and not lfs.isdir(pth) then + lfs.mkdir(pth) + end + end + return pth, (lfs.isdir(pth) == true) + end + +--~ print(dir.mkdirs("","","a","c")) +--~ print(dir.mkdirs("a")) +--~ print(dir.mkdirs("a:")) +--~ print(dir.mkdirs("a:/b/c")) +--~ print(dir.mkdirs("a:b/c")) +--~ print(dir.mkdirs("a:/bbb/c")) +--~ print(dir.mkdirs("/a/b/c")) +--~ print(dir.mkdirs("/aaa/b/c")) +--~ print(dir.mkdirs("//a/b/c")) +--~ print(dir.mkdirs("///a/b/c")) +--~ print(dir.mkdirs("a/bbb//ccc/")) + + function dir.expand_name(str) + local first, nothing, last = str:match("^(//)(//*)(.*)$") + if first then + first = lfs.currentdir() .. "/" + first = first:gsub("\\","/") + end + if not first then + first, last = str:match("^(//)/*(.*)$") + end + if not first then + first, last = str:match("^([a-zA-Z]:)(.*)$") + if first and not last:find("^/") then + local d = lfs.currentdir() + if lfs.chdir(first) then + first = lfs.currentdir() + first = first:gsub("\\","/") + end + lfs.chdir(d) + end + end + if not first then + first, last = lfs.currentdir(), str + first = first:gsub("\\","/") + end + last = last:gsub("//","/") + last = last:gsub("/%./","/") + last = last:gsub("^/*","") + first = first:gsub("/*$","") + if last == "" then + return first + else + return first .. "/" .. last + end + end + + else + + function dir.mkdirs(...) + local str, pth = "", "" + for _, s in ipairs({...}) do + if s ~= "" then + if str ~= "" then + str = str .. "/" .. s + else + str = s + end + end + end + str = str:gsub("/+","/") + if str:find("^/") then + pth = "/" + for s in str:gmatch("[^/]+") do + local first = (pth == "/") + if first then + pth = pth .. s + else + pth = pth .. "/" .. s + end + if make_indeed and not first and not lfs.isdir(pth) then + lfs.mkdir(pth) + end + end + else + pth = "." + for s in str:gmatch("[^/]+") do + pth = pth .. "/" .. s + if make_indeed and not lfs.isdir(pth) then + lfs.mkdir(pth) + end + end + end + return pth, (lfs.isdir(pth) == true) + end + +--~ print(dir.mkdirs("","","a","c")) +--~ print(dir.mkdirs("a")) +--~ print(dir.mkdirs("/a/b/c")) +--~ print(dir.mkdirs("/aaa/b/c")) +--~ print(dir.mkdirs("//a/b/c")) +--~ print(dir.mkdirs("///a/b/c")) +--~ print(dir.mkdirs("a/bbb//ccc/")) + + function dir.expand_name(str) + if not str:find("^/") then + str = lfs.currentdir() .. "/" .. str + end + str = str:gsub("//","/") + str = str:gsub("/%./","/") + return str + end + + end + + dir.makedirs = dir.mkdirs + +end end + + +-- filename : l-boolean.lua +-- comment : split off from luat-lib +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-boolean'] = 1.001 +if not boolean then boolean = { } end + +function boolean.tonumber(b) + if b then return 1 else return 0 end +end + +function toboolean(str,tolerant) + if tolerant then + local tstr = type(str) + if tstr == "string" then + return str == "true" or str == "yes" or str == "on" or str == "1" + elseif tstr == "number" then + return tonumber(str) ~= 0 + elseif tstr == "nil" then + return false + else + return str + end + elseif str == "true" then + return true + elseif str == "false" then + return false + else + return str + end +end + +function string.is_boolean(str) + if type(str) == "string" then + if str == "true" or str == "yes" or str == "on" then + return true + elseif str == "false" or str == "no" or str == "off" then + return false + end + end + return nil +end + +function boolean.alwaystrue() + return true +end + +function boolean.falsetrue() + return false +end + + +-- filename : l-unicode.lua +-- comment : split off from luat-inp +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-unicode'] = 1.001 +if not unicode then unicode = { } end + +if not garbagecollector then + garbagecollector = { + push = function() collectgarbage("stop") end, + pop = function() collectgarbage("restart") end, + } +end + +-- 0 EF BB BF UTF-8 +-- 1 FF FE UTF-16-little-endian +-- 2 FE FF UTF-16-big-endian +-- 3 FF FE 00 00 UTF-32-little-endian +-- 4 00 00 FE FF UTF-32-big-endian + +unicode.utfname = { + [0] = 'utf-8', + [1] = 'utf-16-le', + [2] = 'utf-16-be', + [3] = 'utf-32-le', + [4] = 'utf-32-be' +} + +function unicode.utftype(f) -- \000 fails ! + local str = f:read(4) + if not str then + f:seek('set') + return 0 + elseif str:find("^%z%z\254\255") then + return 4 + elseif str:find("^\255\254%z%z") then + return 3 + elseif str:find("^\254\255") then + f:seek('set',2) + return 2 + elseif str:find("^\255\254") then + f:seek('set',2) + return 1 + elseif str:find("^\239\187\191") then + f:seek('set',3) + return 0 + else + f:seek('set') + return 0 + end +end + +function unicode.utf16_to_utf8(str, endian) + garbagecollector.push() + local result = { } + local tc, uc = table.concat, unicode.utf8.char + local tmp, n, m, p = { }, 0, 0, 0 + -- lf | cr | crlf / (cr:13, lf:10) + local function doit() + if n == 10 then + if p ~= 13 then + result[#result+1] = tc(tmp,"") + tmp = { } + p = 0 + end + elseif n == 13 then + result[#result+1] = tc(tmp,"") + tmp = { } + p = n + else + tmp[#tmp+1] = uc(n) + p = 0 + end + end + for l,r in str:bytepairs() do + if endian then + n = l*256 + r + else + n = r*256 + l + end + if m > 0 then + n = (m-0xD800)*0x400 + (n-0xDC00) + 0x10000 + m = 0 + doit() + elseif n >= 0xD800 and n <= 0xDBFF then + m = n + else + doit() + end + end + if #tmp > 0 then + result[#result+1] = tc(tmp,"") + end + garbagecollector.pop() + return result +end + +function unicode.utf32_to_utf8(str, endian) + garbagecollector.push() + local result = { } + local tc, uc = table.concat, unicode.utf8.char + local tmp, n, m, p = { }, 0, -1, 0 + -- lf | cr | crlf / (cr:13, lf:10) + local function doit() + if n == 10 then + if p ~= 13 then + result[#result+1] = tc(tmp,"") + tmp = { } + p = 0 + end + elseif n == 13 then + result[#result+1] = tc(tmp,"") + tmp = { } + p = n + else + tmp[#tmp+1] = uc(n) + p = 0 + end + end + for a,b in str:bytepairs() do + if a and b then + if m < 0 then + if endian then + m = a*256*256*256 + b*256*256 + else + m = b*256 + a + end + else + if endian then + n = m + a*256 + b + else + n = m + b*256*256*256 + a*256*256 + end + m = -1 + doit() + end + else + break + end + end + if #tmp > 0 then + result[#result+1] = tc(tmp,"") + end + garbagecollector.pop() + return result +end + + +-- filename : l-utils.lua +-- comment : split off from luat-lib +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-utils'] = 1.001 + +if not utils then utils = { } end +if not utils.merger then utils.merger = { } end +if not utils.lua then utils.lua = { } end + +utils.merger.m_begin = "begin library merge" +utils.merger.m_end = "end library merge" +utils.merger.pattern = + "%c+" .. + "%-%-%s+" .. utils.merger.m_begin .. + "%c+(.-)%c+" .. + "%-%-%s+" .. utils.merger.m_end .. + "%c+" + +function utils.merger._self_fake_() + return + "-- " .. "created merged file" .. "\n\n" .. + "-- " .. utils.merger.m_begin .. "\n\n" .. + "-- " .. utils.merger.m_end .. "\n\n" +end + +function utils.report(...) + print(...) +end + +function utils.merger._self_load_(name) + local f, data = io.open(name), "" + if f then + data = f:read("*all") + f:close() + end + return data or "" +end + +function utils.merger._self_save_(name, data) + if data ~= "" then + local f = io.open(name,'w') + if f then + f:write(data) + f:close() + end + end +end + +function utils.merger._self_swap_(data,code) + if data ~= "" then + return (data:gsub(utils.merger.pattern, function(s) + return "\n\n" .. "-- "..utils.merger.m_begin .. "\n" .. code .. "\n" .. "-- "..utils.merger.m_end .. "\n\n" + end, 1)) + else + return "" + end +end + +function utils.merger._self_libs_(libs,list) + local result, f = { }, nil + if type(libs) == 'string' then libs = { libs } end + if type(list) == 'string' then list = { list } end + for _, lib in ipairs(libs) do + for _, pth in ipairs(list) do + local name = string.gsub(pth .. "/" .. lib,"\\","/") + f = io.open(name) + if f then + -- utils.report("merging library",name) + result[#result+1] = f:read("*all") + f:close() + list = { pth } -- speed up the search + break + else + -- utils.report("no library",name) + end + end + end + return table.concat(result, "\n\n") +end + +function utils.merger.selfcreate(libs,list,target) + if target then + utils.merger._self_save_( + target, + utils.merger._self_swap_( + utils.merger._self_fake_(), + utils.merger._self_libs_(libs,list) + ) + ) + end +end + +function utils.merger.selfmerge(name,libs,list,target) + utils.merger._self_save_( + target or name, + utils.merger._self_swap_( + utils.merger._self_load_(name), + utils.merger._self_libs_(libs,list) + ) + ) +end + +function utils.merger.selfclean(name) + utils.merger._self_save_( + name, + utils.merger._self_swap_( + utils.merger._self_load_(name), + "" + ) + ) +end + +utils.lua.compile_strip = true + +function utils.lua.compile(luafile, lucfile) + -- utils.report("compiling",luafile,"into",lucfile) + os.remove(lucfile) + local command = "-o " .. string.quote(lucfile) .. " " .. string.quote(luafile) + if utils.lua.compile_strip then + command = "-s " .. command + end + if os.spawn("texluac " .. command) == 0 then + return true + elseif os.spawn("luac " .. command) == 0 then + return true + else + return false + end +end + + + +-- filename : luat-lib.lua +-- comment : companion to luat-lib.tex +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['luat-lib'] = 1.001 + +-- mostcode moved to the l-*.lua and other luat-*.lua files + +-- os / io + +os.setlocale(nil,nil) -- useless feature and even dangerous in luatex + +-- os.platform + +-- mswin|bccwin|mingw|cygwin windows +-- darwin|rhapsody|nextstep macosx +-- netbsd|unix unix +-- linux linux + +if not io.fileseparator then + if string.find(os.getenv("PATH"),";") then + io.fileseparator, io.pathseparator, os.platform = "\\", ";", os.type or "windows" + else + io.fileseparator, io.pathseparator, os.platform = "/" , ":", os.type or "unix" + end +end + +os.platform = os.platform or os.type or (io.pathseparator == ";" and "windows") or "unix" + +-- arg normalization +-- +-- for k,v in pairs(arg) do print(k,v) end + +-- environment + +if not environment then environment = { } end + +environment.ownbin = environment.ownbin or arg[-2] or arg[-1] or arg[0] or "luatex" + +local ownpath = nil -- we could use a metatable here + +function environment.ownpath() + if not ownpath then + for p in string.gmatch(os.getenv("PATH"),"[^"..io.pathseparator.."]+") do + local b = file.join(p,environment.ownbin) + if lfs.isfile(b..".exe") or lfs.isfile(b) then + ownpath = p + break + end + end + if not ownpath then ownpath = '.' end + end + return ownpath +end + +if arg and (arg[0] == 'luatex' or arg[0] == 'luatex.exe') and arg[1] == "--luaonly" then + arg[-1]=arg[0] arg[0]=arg[2] for k=3,#arg do arg[k-2]=arg[k] end arg[#arg]=nil arg[#arg]=nil +end + +environment.arguments = { } +environment.files = { } +environment.sorted_argument_keys = nil + +environment.platform = os.platform + +function environment.initialize_arguments(arg) + environment.arguments = { } + environment.files = { } + environment.sorted_argument_keys = nil + for index, argument in pairs(arg) do + if index > 0 then + local flag, value = argument:match("^%-+(.+)=(.-)$") + if flag then + environment.arguments[flag] = string.unquote(value or "") + else + flag = argument:match("^%-+(.+)") + if flag then + environment.arguments[flag] = true + else + environment.files[#environment.files+1] = argument + end + end + end + end + environment.ownname = environment.ownname or arg[0] or 'unknown.lua' +end + +function environment.showarguments() + for k,v in pairs(environment.arguments) do + print(k .. " : " .. tostring(v)) + end + if #environment.files > 0 then + print("files : " .. table.concat(environment.files, " ")) + end +end + +function environment.setargument(name,value) + environment.arguments[name] = value +end + +function environment.argument(name) + if environment.arguments[name] then + return environment.arguments[name] + else + if not environment.sorted_argument_keys then + environment.sorted_argument_keys = { } + for _,v in pairs(table.sortedkeys(environment.arguments)) do + table.insert(environment.sorted_argument_keys, "^" .. v) + end + end + for _,v in pairs(environment.sorted_argument_keys) do + if name:find(v) then + return environment.arguments[v:sub(2,#v)] + end + end + end + return nil +end + +function environment.split_arguments(separator) -- rather special, cut-off before separator + local done, before, after = false, { }, { } + for _,v in ipairs(environment.original_arguments) do + if not done and v == separator then + done = true + elseif done then + after[#after+1] = v + else + before[#before+1] = v + end + end + return before, after +end + +function environment.reconstruct_commandline(arg) + if not arg then arg = environment.original_arguments end + local result = { } + for _,a in ipairs(arg) do -- ipairs 1 .. #n + local kk, vv = a:match("^(%-+.-)=(.+)$") + if kk and vv then + if vv:find(" ") then + result[#result+1] = kk .. "=" .. string.quote(vv) + else + result[#result+1] = a + end + elseif a:find(" ") then + result[#result+1] = string.quote(a) + else + result[#result+1] = a + end + end + return table.join(result," ") +end + +if arg then + environment.initialize_arguments(arg) + environment.original_arguments = arg + arg = { } -- prevent duplicate handling +end + + +-- filename : luat-inp.lua +-- comment : companion to luat-lib.tex +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +-- This lib is multi-purpose and can be loaded again later on so that +-- additional functionality becomes available. We will split this +-- module in components when we're done with prototyping. + +-- TODO: os.getenv -> os.env[] +-- TODO: instances.[hashes,cnffiles,configurations,522] -> ipairs (alles check, sneller) +-- TODO: check escaping in find etc, too much, too slow + +-- This is the first code I wrote for LuaTeX, so it needs some cleanup. + +-- To be considered: hash key lowercase, first entry in table filename +-- (any case), rest paths (so no need for optimization). Or maybe a +-- separate table that matches lowercase names to mixed case when +-- present. In that case the lower() cases can go away. I will do that +-- only when we run into problems with names ... well ... Iwona-Regular. + +-- Beware, loading and saving is overloaded in luat-tmp! + +if not versions then versions = { } end versions['luat-inp'] = 1.001 +if not environment then environment = { } end +if not file then file = { } end + +if environment.aleph_mode == nil then environment.aleph_mode = true end -- temp hack + +if not input then input = { } end +if not input.suffixes then input.suffixes = { } end +if not input.formats then input.formats = { } end +if not input.aux then input.aux = { } end + +if not input.suffixmap then input.suffixmap = { } end + +if not input.locators then input.locators = { } end -- locate databases +if not input.hashers then input.hashers = { } end -- load databases +if not input.generators then input.generators = { } end -- generate databases +if not input.filters then input.filters = { } end -- conversion filters + +local format = string.format + +input.locators.notfound = { nil } +input.hashers.notfound = { nil } +input.generators.notfound = { nil } + +input.cacheversion = '1.0.1' +input.banner = nil +input.verbose = false +input.debug = false +input.cnfname = 'texmf.cnf' +input.luaname = 'texmfcnf.lua' +input.lsrname = 'ls-R' +input.luasuffix = '.tma' +input.lucsuffix = '.tmc' + +-- we use a cleaned up list / format=any is a wildcard, as is *name + +input.formats['afm'] = 'AFMFONTS' input.suffixes['afm'] = { 'afm' } +input.formats['enc'] = 'ENCFONTS' input.suffixes['enc'] = { 'enc' } +input.formats['fmt'] = 'TEXFORMATS' input.suffixes['fmt'] = { 'fmt' } +input.formats['map'] = 'TEXFONTMAPS' input.suffixes['map'] = { 'map' } +input.formats['mp'] = 'MPINPUTS' input.suffixes['mp'] = { 'mp' } +input.formats['ocp'] = 'OCPINPUTS' input.suffixes['ocp'] = { 'ocp' } +input.formats['ofm'] = 'OFMFONTS' input.suffixes['ofm'] = { 'ofm', 'tfm' } +input.formats['otf'] = 'OPENTYPEFONTS' input.suffixes['otf'] = { 'otf' } -- 'ttf' +input.formats['opl'] = 'OPLFONTS' input.suffixes['opl'] = { 'opl' } +input.formats['otp'] = 'OTPINPUTS' input.suffixes['otp'] = { 'otp' } +input.formats['ovf'] = 'OVFFONTS' input.suffixes['ovf'] = { 'ovf', 'vf' } +input.formats['ovp'] = 'OVPFONTS' input.suffixes['ovp'] = { 'ovp' } +input.formats['tex'] = 'TEXINPUTS' input.suffixes['tex'] = { 'tex' } +input.formats['tfm'] = 'TFMFONTS' input.suffixes['tfm'] = { 'tfm' } +input.formats['ttf'] = 'TTFONTS' input.suffixes['ttf'] = { 'ttf', 'ttc' } +input.formats['pfb'] = 'T1FONTS' input.suffixes['pfb'] = { 'pfb', 'pfa' } +input.formats['vf'] = 'VFFONTS' input.suffixes['vf'] = { 'vf' } + +input.formats['fea'] = 'FONTFEATURES' input.suffixes['fea'] = { 'fea' } +input.formats['cid'] = 'FONTCIDMAPS' input.suffixes['cid'] = { 'cid', 'cidmap' } + +input.formats ['texmfscripts'] = 'TEXMFSCRIPTS' -- new +input.suffixes['texmfscripts'] = { 'rb', 'pl', 'py' } -- 'lua' + +input.formats ['lua'] = 'LUAINPUTS' -- new +input.suffixes['lua'] = { 'lua', 'luc', 'tma', 'tmc' } + +-- here we catch a few new thingies (todo: add these paths to context.tmf) +-- +-- FONTFEATURES = .;$TEXMF/fonts/fea// +-- FONTCIDMAPS = .;$TEXMF/fonts/cid// + +function input.checkconfigdata(instance) -- not yet ok, no time for debugging now + local function fix(varname,default) + local proname = varname .. "." .. instance.progname or "crap" + local p = instance.environment[proname] + local v = instance.environment[varname] + if not ((p and p ~= "") or (v and v ~= "")) then + instance.variables[varname] = default -- or environment? + end + end + fix("LUAINPUTS" , ".;$TEXINPUTS;$TEXMFSCRIPTS") + fix("FONTFEATURES", ".;$TEXMF/fonts/fea//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") + fix("FONTCIDMAPS" , ".;$TEXMF/fonts/cid//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") +end + +-- backward compatible ones + +input.alternatives = { } + +input.alternatives['map files'] = 'map' +input.alternatives['enc files'] = 'enc' +input.alternatives['cid files'] = 'cid' +input.alternatives['fea files'] = 'fea' +input.alternatives['opentype fonts'] = 'otf' +input.alternatives['truetype fonts'] = 'ttf' +input.alternatives['truetype collections'] = 'ttc' +input.alternatives['type1 fonts'] = 'pfb' + +-- obscure ones + +input.formats ['misc fonts'] = '' +input.suffixes['misc fonts'] = { } + +input.formats ['sfd'] = 'SFDFONTS' +input.suffixes ['sfd'] = { 'sfd' } +input.alternatives['subfont definition files'] = 'sfd' + +function input.reset() + + local instance = { } + + instance.rootpath = '' + instance.treepath = '' + instance.progname = environment.progname or 'context' + instance.engine = environment.engine or 'luatex' + instance.format = '' + instance.environment = { } + instance.variables = { } + instance.expansions = { } + instance.files = { } + instance.remap = { } + instance.configuration = { } + instance.setup = { } + instance.order = { } + instance.found = { } + instance.foundintrees = { } + instance.kpsevars = { } + instance.hashes = { } + instance.cnffiles = { } + instance.luafiles = { } + instance.lists = { } + instance.remember = true + instance.diskcache = true + instance.renewcache = false + instance.scandisk = true + instance.cachepath = nil + instance.loaderror = false + instance.smallcache = false + instance.savelists = true + instance.cleanuppaths = true + instance.allresults = false + instance.pattern = nil -- lists + instance.kpseonly = false -- lists + instance.cachefile = 'tmftools' + instance.loadtime = 0 + instance.starttime = 0 + instance.stoptime = 0 + instance.validfile = function(path,name) return true end + instance.data = { } -- only for loading + instance.force_suffixes = true + instance.dummy_path_expr = "^!*unset/*$" + instance.fakepaths = { } + instance.lsrmode = false + + if os.env then + -- store once, freeze and faster + for k,v in pairs(os.env) do + instance.environment[k] = input.bare_variable(v) + end + else + -- we will access os.env frequently + for k,v in pairs({'HOME','TEXMF','TEXMFCNF'}) do + local e = os.getenv(v) + if e then + -- input.report("setting",v,"to",input.bare_variable(e)) + instance.environment[v] = input.bare_variable(e) + end + end + end + + -- cross referencing + + for k, v in pairs(input.suffixes) do + for _, vv in pairs(v) do + if vv then + input.suffixmap[vv] = k + end + end + end + + return instance + +end + +function input.reset_hashes(instance) + instance.lists = { } + instance.found = { } +end + +function input.bare_variable(str) -- assumes str is a string + -- return string.gsub(string.gsub(string.gsub(str,"%s+$",""),'^"(.+)"$',"%1"),"^'(.+)'$","%1") + return (str:gsub("\s*([\"\']?)(.+)%1\s*", "%2")) +end + +if texio then + input.log = texio.write_nl +else + input.log = print +end + +function input.simple_logger(kind, name) + if name and name ~= "" then + if input.banner then + input.log(input.banner..kind..": "..name) + else + input.log("<<"..kind..": "..name..">>") + end + else + if input.banner then + input.log(input.banner..kind..": no name") + else + input.log("<<"..kind..": no name>>") + end + end +end + +function input.dummy_logger() +end + +function input.settrace(n) + input.trace = tonumber(n or 0) + if input.trace > 0 then + input.logger = input.simple_logger + input.verbose = true + else + input.logger = function() end + end +end + +function input.report(...) -- inefficient + if input.verbose then + if input.banner then + input.log(input.banner .. table.concat({...},' ')) + elseif input.logmode() == 'xml' then + input.log("<t>"..table.concat({...},' ').."</t>") + else + input.log("<<"..table.concat({...},' ')..">>") + end + end +end + +function input.reportlines(str) + if type(str) == "string" then + str = str:split("\n") + end + for _,v in pairs(str) do input.report(v) end +end + +input.settrace(tonumber(os.getenv("MTX.INPUT.TRACE") or os.getenv("MTX_INPUT_TRACE") or input.trace or 0)) + +-- These functions can be used to test the performance, especially +-- loading the database files. + +do + local clock = os.gettimeofday or os.clock + + function input.starttiming(instance) + if instance then + instance.starttime = clock() + if not instance.loadtime then + instance.loadtime = 0 + end + end + end + + function input.stoptiming(instance, report) + if instance then + local starttime = instance.starttime + if starttime then + local stoptime = clock() + local loadtime = stoptime - starttime + instance.stoptime = stoptime + instance.loadtime = instance.loadtime + loadtime + if report then + input.report('load time', format("%0.3f",loadtime)) + end + return loadtime + end + end + return 0 + end + +end + +function input.elapsedtime(instance) + return format("%0.3f",(instance and instance.loadtime) or 0) +end + +function input.report_loadtime(instance) + if instance then + input.report('total load time', input.elapsedtime(instance)) + end +end + +input.loadtime = input.elapsedtime + +function input.env(instance,key) + return instance.environment[key] or input.osenv(instance,key) +end + +function input.osenv(instance,key) + local ie = instance.environment + local value = ie[key] + if value == nil then + -- local e = os.getenv(key) + local e = os.env[key] + if e == nil then + -- value = "" -- false + else + value = input.bare_variable(e) + end + ie[key] = value + end + return value or "" +end + +-- we follow a rather traditional approach: +-- +-- (1) texmf.cnf given in TEXMFCNF +-- (2) texmf.cnf searched in TEXMF/web2c +-- +-- for the moment we don't expect a configuration file in a zip + +function input.identify_cnf(instance) + -- we no longer support treepath and rootpath (was handy for testing); + -- also we now follow the stupid route: if not set then just assume *one* + -- cnf file under texmf (i.e. distribution) + if #instance.cnffiles == 0 then + if input.env(instance,'TEXMFCNF') == "" then + local ownpath = environment.ownpath() or "." + if ownpath then + -- beware, this is tricky on my own system because at that location I do have + -- the raw tree that ends up in the zip; i.e. I cannot test this kind of mess + local function locate(filename,list) + local ownroot = input.normalize_name(file.join(ownpath,"../..")) + if not lfs.isdir(file.join(ownroot,"texmf")) then + ownroot = input.normalize_name(file.join(ownpath,"..")) + if not lfs.isdir(file.join(ownroot,"texmf")) then + input.verbose = true + input.report("error", "unable to identify cnf file") + return + end + end + local texmfcnf = file.join(ownroot,"texmf-local/web2c",filename) -- for minimals and myself + if not lfs.isfile(texmfcnf) then + texmfcnf = file.join(ownroot,"texmf/web2c",filename) + if not lfs.isfile(texmfcnf) then + input.verbose = true + input.report("error", "unable to locate",filename) + return + end + end + table.insert(list,texmfcnf) + local ie = instance.environment + if not ie['SELFAUTOPARENT'] then ie['SELFAUTOPARENT'] = ownroot end + if not ie['TEXMFCNF'] then ie['TEXMFCNF'] = file.dirname(texmfcnf) end + end + locate(input.luaname,instance.luafiles) + locate(input.cnfname,instance.cnffiles) + if #instance.luafiles == 0 and instance.cnffiles == 0 then + input.verbose = true + input.report("error", "unable to locate",filename) + os.exit() + end + -- here we also assume then TEXMF is set in the distribution, if this trickery is + -- used in the minimals, then users who don't use setuptex are on their own with + -- regards to extra trees + else + input.verbose = true + input.report("error", "unable to identify own path") + os.exit() + end + else + local t = input.split_path(input.env(instance,'TEXMFCNF')) + t = input.aux.expanded_path(instance,t) + input.aux.expand_vars(instance,t) + local function locate(filename,list) + for _,v in ipairs(t) do + local texmfcnf = input.normalize_name(file.join(v,filename)) + if lfs.isfile(texmfcnf) then + table.insert(list,texmfcnf) + end + end + end + locate(input.luaname,instance.luafiles) + locate(input.cnfname,instance.cnffiles) + end + end +end + +function input.load_cnf(instance) + local function loadoldconfigdata() + for _, fname in ipairs(instance.cnffiles) do + input.aux.load_cnf(instance,fname) + end + end + -- instance.cnffiles contain complete names now ! + if #instance.cnffiles == 0 then + input.report("no cnf files found (TEXMFCNF may not be set/known)") + else + instance.rootpath = instance.cnffiles[1] + for k,fname in ipairs(instance.cnffiles) do + instance.cnffiles[k] = input.normalize_name(fname:gsub("\\",'/')) + end + for i=1,3 do + instance.rootpath = file.dirname(instance.rootpath) + end + instance.rootpath = input.normalize_name(instance.rootpath) + instance.environment['SELFAUTOPARENT'] = instance.rootpath -- just to be sure + if instance.lsrmode then + loadoldconfigdata() + elseif instance.diskcache and not instance.renewcache then + input.loadoldconfig(instance,instance.cnffiles) + if instance.loaderror then + loadoldconfigdata() + input.saveoldconfig(instance) + end + else + loadoldconfigdata() + if instance.renewcache then + input.saveoldconfig(instance) + end + end + input.aux.collapse_cnf_data(instance) + end + input.checkconfigdata(instance) +end + +function input.load_lua(instance) + if #instance.luafiles == 0 then + -- yet harmless + else + instance.rootpath = instance.luafiles[1] + for k,fname in ipairs(instance.luafiles) do + instance.luafiles[k] = input.normalize_name(fname:gsub("\\",'/')) + end + for i=1,3 do + instance.rootpath = file.dirname(instance.rootpath) + end + instance.rootpath = input.normalize_name(instance.rootpath) + instance.environment['SELFAUTOPARENT'] = instance.rootpath -- just to be sure + input.loadnewconfig(instance) + input.aux.collapse_cnf_data(instance) + end + input.checkconfigdata(instance) +end + +function input.aux.collapse_cnf_data(instance) -- potential optmization: pass start index (setup and configuration are shared) + for _,c in ipairs(instance.order) do + for k,v in pairs(c) do + if not instance.variables[k] then + if instance.environment[k] then + instance.variables[k] = instance.environment[k] + else + instance.kpsevars[k] = true + instance.variables[k] = input.bare_variable(v) + end + end + end + end +end + +function input.aux.load_cnf(instance,fname) + fname = input.clean_path(fname) + local lname = fname:gsub("%.%a+$",input.luasuffix) + local f = io.open(lname) + if f then -- this will go + f:close() + local dname = file.dirname(fname) + if not instance.configuration[dname] then + input.aux.load_configuration(instance,dname,lname) + instance.order[#instance.order+1] = instance.configuration[dname] + end + else + f = io.open(fname) + if f then + input.report("loading", fname) + local line, data, n, k, v + local dname = file.dirname(fname) + if not instance.configuration[dname] then + instance.configuration[dname] = { } + instance.order[#instance.order+1] = instance.configuration[dname] + end + local data = instance.configuration[dname] + while true do + local line, n = f:read(), 0 + if line then + while true do -- join lines + line, n = line:gsub("\\%s*$", "") + if n > 0 then + line = line .. f:read() + else + break + end + end + if not line:find("^[%%#]") then + local k, v = (line:gsub("%s*%%.*$","")):match("%s*(.-)%s*=%s*(.-)%s*$") + if k and v and not data[k] then + data[k] = (v:gsub("[%%#].*",'')):gsub("~", "$HOME") + instance.kpsevars[k] = true + end + end + else + break + end + end + f:close() + else + input.report("skipping", fname) + end + end +end + +-- database loading + +function input.load_hash(instance) + input.locatelists(instance) + if instance.lsrmode then + input.loadlists(instance) + elseif instance.diskcache and not instance.renewcache then + input.loadfiles(instance) + if instance.loaderror then + input.loadlists(instance) + input.savefiles(instance) + end + else + input.loadlists(instance) + if instance.renewcache then + input.savefiles(instance) + end + end +end + +function input.aux.append_hash(instance,type,tag,name) + input.logger("= hash append",tag) + table.insert(instance.hashes, { ['type']=type, ['tag']=tag, ['name']=name } ) +end + +function input.aux.prepend_hash(instance,type,tag,name) + input.logger("= hash prepend",tag) + table.insert(instance.hashes, 1, { ['type']=type, ['tag']=tag, ['name']=name } ) +end + +function input.aux.extend_texmf_var(instance,specification) -- crap + if instance.environment['TEXMF'] then + input.report("extending environment variable TEXMF with", specification) + instance.environment['TEXMF'] = instance.environment['TEXMF']:gsub("^%{", function() + return "{" .. specification .. "," + end) + elseif instance.variables['TEXMF'] then + input.report("extending configuration variable TEXMF with", specification) + instance.variables['TEXMF'] = instance.variables['TEXMF']:gsub("^%{", function() + return "{" .. specification .. "," + end) + else + input.report("setting configuration variable TEXMF to", specification) + instance.variables['TEXMF'] = "{" .. specification .. "}" + end + if instance.variables['TEXMF']:find("%,") and not instance.variables['TEXMF']:find("^%{") then + input.report("adding {} to complex TEXMF variable, best do that yourself") + instance.variables['TEXMF'] = "{" .. instance.variables['TEXMF'] .. "}" + end + input.expand_variables(instance) + input.reset_hashes(instance) +end + +-- locators + +function input.locatelists(instance) + for _, path in pairs(input.simplified_list(input.expansion(instance,'TEXMF'))) do + path = file.collapse_path(path) + input.report("locating list of",path) + input.locatedatabase(instance,input.normalize_name(path)) + end +end + +function input.locatedatabase(instance,specification) + return input.methodhandler('locators', instance, specification) +end + +function input.locators.tex(instance,specification) + if specification and specification ~= '' and lfs.isdir(specification) then + input.logger('! tex locator', specification..' found') + input.aux.append_hash(instance,'file',specification,filename) + else + input.logger('? tex locator', specification..' not found') + end +end + +-- hashers + +function input.hashdatabase(instance,tag,name) + return input.methodhandler('hashers',instance,tag,name) +end + +function input.loadfiles(instance) + instance.loaderror = false + instance.files = { } + if not instance.renewcache then + for _, hash in ipairs(instance.hashes) do + input.hashdatabase(instance,hash.tag,hash.name) + if instance.loaderror then break end + end + end +end + +function input.hashers.tex(instance,tag,name) + input.aux.load_files(instance,tag) +end + +-- generators: + +function input.loadlists(instance) + for _, hash in ipairs(instance.hashes) do + input.generatedatabase(instance,hash.tag) + end +end + +function input.generatedatabase(instance,specification) + return input.methodhandler('generators', instance, specification) +end + +do + + local weird = lpeg.anywhere(lpeg.S("~`!#$%^&*()={}[]:;\"\'||<>,?\n\r\t")) + + function input.generators.tex(instance,specification) + local tag = specification + if not instance.lsrmode and lfs and lfs.dir then + input.report("scanning path",specification) + instance.files[tag] = { } + local files = instance.files[tag] + local n, m, r = 0, 0, 0 + local spec = specification .. '/' + local attributes = lfs.attributes + local directory = lfs.dir + local small = instance.smallcache + local function action(path) + local mode, full + if path then + full = spec .. path .. '/' + else + full = spec + end + for name in directory(full) do + if name:find("^%.") then + -- skip + -- elseif name:find("[%~%`%!%#%$%%%^%&%*%(%)%=%{%}%[%]%:%;\"\'%|%<%>%,%?\n\r\t]") then -- too much escaped + elseif weird:match(name) then + -- texio.write_nl("skipping " .. name) + -- skip + else + mode = attributes(full..name,'mode') + if mode == "directory" then + m = m + 1 + if path then + action(path..'/'..name) + else + action(name) + end + elseif path and mode == 'file' then + n = n + 1 + local f = files[name] + if f then + if not small then + if type(f) == 'string' then + files[name] = { f, path } + else + f[#f+1] = path + end + end + else + files[name] = path + local lower = name:lower() + if name ~= lower then + files["remap:"..lower] = name + r = r + 1 + end + end + end + end + end + end + action() + input.report(format("%s files found on %s directories with %s uppercase remappings",n,m,r)) + else + local fullname = file.join(specification,input.lsrname) + local path = '.' + local f = io.open(fullname) + if f then + instance.files[tag] = { } + local files = instance.files[tag] + local small = instance.smallcache + input.report("loading lsr file",fullname) + -- for line in f:lines() do -- much slower then the next one + for line in (f:read("*a")):gmatch("(.-)\n") do + if line:find("^[%a%d]") then + local fl = files[line] + if fl then + if not small then + if type(fl) == 'string' then + files[line] = { fl, path } -- table + else + fl[#fl+1] = path + end + end + else + files[line] = path -- string + local lower = line:lower() + if line ~= lower then + files["remap:"..lower] = line + end + end + else + path = line:match("%.%/(.-)%:$") or path -- match could be nil due to empty line + end + end + f:close() + end + end + end + +end + +-- savers, todo + +function input.savefiles(instance) + input.aux.save_data(instance, 'files', function(k,v) + return instance.validfile(k,v) -- path, name + end) +end + +-- A config (optionally) has the paths split in tables. Internally +-- we join them and split them after the expansion has taken place. This +-- is more convenient. + +function input.splitconfig(instance) + for i,c in ipairs(instance) do + for k,v in pairs(c) do + if type(v) == 'string' then + local t = file.split_path(v) + if #t > 1 then + c[k] = t + end + end + end + end +end +function input.joinconfig(instance) + for i,c in ipairs(instance.order) do + for k,v in pairs(c) do + if type(v) == 'table' then + c[k] = file.join_path(v) + end + end + end +end +function input.split_path(str) + if type(str) == 'table' then + return str + else + return file.split_path(str) + end +end +function input.join_path(str) + if type(str) == 'table' then + return file.join_path(str) + else + return str + end +end + +function input.splitexpansions(instance) + for k,v in pairs(instance.expansions) do + local t, h = { }, { } + for _,vv in pairs(file.split_path(v)) do + if vv ~= "" and not h[vv] then + t[#t+1] = vv + h[vv] = true + end + end + if #t > 1 then + instance.expansions[k] = t + else + instance.expansions[k] = t[1] + end + end +end + +-- end of split/join code + +function input.saveoldconfig(instance) + input.splitconfig(instance) + input.aux.save_data(instance, 'configuration', nil) + input.joinconfig(instance) +end + +input.configbanner = [[ +-- This is a Luatex configuration file created by 'luatools.lua' or +-- 'luatex.exe' directly. For comment, suggestions and questions you can +-- contact the ConTeXt Development Team. This configuration file is +-- not copyrighted. [HH & TH] +]] + +function input.serialize(files) + -- This version is somewhat optimized for the kind of + -- tables that we deal with, so it's much faster than + -- the generic serializer. This makes sense because + -- luatools and mtxtools are called frequently. Okay, + -- we pay a small price for properly tabbed tables. + local t = { } + local concat = table.concat + local sorted = table.sortedkeys + local function dump(k,v,m) + if type(v) == 'string' then + return m .. "['" .. k .. "']='" .. v .. "'," + elseif #v == 1 then + return m .. "['" .. k .. "']='" .. v[1] .. "'," + else + return m .. "['" .. k .. "']={'" .. concat(v,"','").. "'}," + end + end + t[#t+1] = "return {" + if instance.sortdata then + for _, k in pairs(sorted(files)) do + local fk = files[k] + if type(fk) == 'table' then + t[#t+1] = "\t['" .. k .. "']={" + for _, kk in pairs(sorted(fk)) do + t[#t+1] = dump(kk,fk[kk],"\t\t") + end + t[#t+1] = "\t}," + else + t[#t+1] = dump(k,fk,"\t") + end + end + else + for k, v in pairs(files) do + if type(v) == 'table' then + t[#t+1] = "\t['" .. k .. "']={" + for kk,vv in pairs(v) do + t[#t+1] = dump(kk,vv,"\t\t") + end + t[#t+1] = "\t}," + else + t[#t+1] = dump(k,v,"\t") + end + end + end + t[#t+1] = "}" + return concat(t,"\n") +end + +if not texmf then texmf = {} end -- no longer needed, at least not here + +function input.aux.save_data(instance, dataname, check, makename) -- untested without cache overload + for cachename, files in pairs(instance[dataname]) do + local name = (makename or file.join)(cachename,dataname) + local luaname, lucname = name .. input.luasuffix, name .. input.lucsuffix + input.report("preparing " .. dataname .. " for", luaname) + for k, v in pairs(files) do + if not check or check(v,k) then -- path, name + if type(v) == "table" and #v == 1 then + files[k] = v[1] + end + else + files[k] = nil -- false + end + end + local data = { + type = dataname, + root = cachename, + version = input.cacheversion, + date = os.date("%Y-%m-%d"), + time = os.date("%H:%M:%S"), + content = files, + } + local f = io.open(luaname,'w') + if f then + input.report("saving " .. dataname .. " in", luaname) + f:write(input.serialize(data)) + f:close() + input.report("compiling " .. dataname .. " to", lucname) + if not utils.lua.compile(luaname,lucname) then + input.report("compiling failed for " .. dataname .. ", deleting file " .. lucname) + os.remove(lucname) + end + else + input.report("unable to save " .. dataname .. " in " .. name..input.luasuffix) + end + end +end + +function input.aux.load_data(instance,pathname,dataname,filename,makename) -- untested without cache overload + filename = ((not filename or (filename == "")) and dataname) or filename + filename = (makename and makename(dataname,filename)) or file.join(pathname,filename) + local blob = loadfile(filename .. input.lucsuffix) or loadfile(filename .. input.luasuffix) + if blob then + local data = blob() + if data and data.content and data.type == dataname and data.version == input.cacheversion then + input.report("loading",dataname,"for",pathname,"from",filename) + instance[dataname][pathname] = data.content + else + input.report("skipping",dataname,"for",pathname,"from",filename) + instance[dataname][pathname] = { } + instance.loaderror = true + end + else + input.report("skipping",dataname,"for",pathname,"from",filename) + end +end + +-- some day i'll use the nested approach, but not yet (actually we even drop +-- engine/progname support since we have only luatex now) +-- +-- first texmfcnf.lua files are located, next the cached texmf.cnf files +-- +-- return { +-- TEXMFBOGUS = 'effe checken of dit werkt', +-- } + +function input.aux.load_texmfcnf(instance,dataname,pathname) + local filename = file.join(pathname,input.luaname) + local blob = loadfile(filename) + if blob then + local data = blob() + if data then + input.report("loading","configuration file",filename) + if true then + -- flatten to variable.progname + local t = { } + for k, v in pairs(data) do -- v = progname + if type(v) == "string" then + t[k] = v + else + for kk, vv in pairs(v) do -- vv = variable + if type(vv) == "string" then + t[vv.."."..v] = kk + end + end + end + end + instance[dataname][pathname] = t + else + instance[dataname][pathname] = data + end + else + input.report("skipping","configuration file",filename) + instance[dataname][pathname] = { } + instance.loaderror = true + end + else + input.report("skipping","configuration file",filename) + end +end + +function input.aux.load_configuration(instance,dname,lname) + input.aux.load_data(instance,dname,'configuration',lname and file.basename(lname)) +end +function input.aux.load_files(instance,tag) + input.aux.load_data(instance,tag,'files') +end + +function input.resetconfig(instance) + instance.configuration, instance.setup, instance.order, instance.loaderror = { }, { }, { }, false +end + +function input.loadnewconfig(instance) + for _, cnf in ipairs(instance.luafiles) do + local dname = file.dirname(cnf) + input.aux.load_texmfcnf(instance,'setup',dname) + instance.order[#instance.order+1] = instance.setup[dname] + if instance.loaderror then break end + end +end + +function input.loadoldconfig(instance) + if not instance.renewcache then + for _, cnf in ipairs(instance.cnffiles) do + local dname = file.dirname(cnf) + input.aux.load_configuration(instance,dname) + instance.order[#instance.order+1] = instance.configuration[dname] + if instance.loaderror then break end + end + end + input.joinconfig(instance) +end + +function input.expand_variables(instance) + instance.expansions = { } +--~ instance.environment['SELFAUTOPARENT'] = instance.environment['SELFAUTOPARENT'] or instance.rootpath + if instance.engine ~= "" then instance.environment['engine'] = instance.engine end + if instance.progname ~= "" then instance.environment['progname'] = instance.progname end + for k,v in pairs(instance.environment) do + local a, b = k:match("^(%a+)%_(.*)%s*$") + if a and b then + instance.expansions[a..'.'..b] = v + else + instance.expansions[k] = v + end + end + for k,v in pairs(instance.environment) do -- move environment to expansions + if not instance.expansions[k] then instance.expansions[k] = v end + end + for k,v in pairs(instance.variables) do -- move variables to expansions + if not instance.expansions[k] then instance.expansions[k] = v end + end + while true do + local busy = false + for k,v in pairs(instance.expansions) do + local s, n = v:gsub("%$([%a%d%_%-]+)", function(a) + busy = true + return instance.expansions[a] or input.env(instance,a) + end) + local s, m = s:gsub("%$%{([%a%d%_%-]+)%}", function(a) + busy = true + return instance.expansions[a] or input.env(instance,a) + end) + if n > 0 or m > 0 then + instance.expansions[k]= s + end + end + if not busy then break end + end + for k,v in pairs(instance.expansions) do + instance.expansions[k] = v:gsub("\\", '/') + end +end + +function input.aux.expand_vars(instance,lst) -- simple vars + for k,v in pairs(lst) do + lst[k] = v:gsub("%$([%a%d%_%-]+)", function(a) + return instance.variables[a] or input.env(instance,a) + end) + end +end + +function input.aux.expanded_var(instance,var) -- simple vars + return var:gsub("%$([%a%d%_%-]+)", function(a) + return instance.variables[a] or input.env(instance,a) + end) +end + +function input.aux.entry(instance,entries,name) + if name and (name ~= "") then + name = name:gsub('%$','') + local result = entries[name..'.'..instance.progname] or entries[name] + if result then + return result + else + result = input.env(instance,name) + if result then + instance.variables[name] = result + input.expand_variables(instance) + return instance.expansions[name] or "" + end + end + end + return "" +end +function input.variable(instance,name) + return input.aux.entry(instance,instance.variables,name) +end +function input.expansion(instance,name) + return input.aux.entry(instance,instance.expansions,name) +end + +function input.aux.is_entry(instance,entries,name) + if name and name ~= "" then + name = name:gsub('%$','') + return (entries[name..'.'..instance.progname] or entries[name]) ~= nil + else + return false + end +end + +function input.is_variable(instance,name) + return input.aux.is_entry(instance,instance.variables,name) +end +function input.is_expansion(instance,name) + return input.aux.is_entry(instance,instance.expansions,name) +end + +function input.simplified_list(str) + if type(str) == 'table' then + return str -- troubles ; ipv , in texmf + elseif str == '' then + return { } + else + local t = { } + for _,v in ipairs(string.splitchr(str:gsub("^\{(.+)\}$","%1"),",")) do + t[#t+1] = (v:gsub("^[%!]*(.+)[%/\\]*$","%1")) + end + return t + end +end + +function input.unexpanded_path_list(instance,str) + local pth = input.variable(instance,str) + local lst = input.split_path(pth) + return input.aux.expanded_path(instance,lst) +end +function input.unexpanded_path(instance,str) + return file.join_path(input.unexpanded_path_list(instance,str)) +end + +do + local done = { } + + function input.reset_extra_path(instance) + local ep = instance.extra_paths + if not ep then + ep, done = { }, { } + instance.extra_paths = ep + elseif #ep > 0 then + instance.lists, done = { }, { } + end + end + + function input.register_extra_path(instance,paths,subpaths) + local ep = instance.extra_paths or { } + local n = #ep + if paths and paths ~= "" then + if subpaths and subpaths ~= "" then + for p in paths:gmatch("[^,]+") do + -- we gmatch each step again, not that fast, but used seldom + for s in subpaths:gmatch("[^,]+") do + local ps = p .. "/" .. s + if not done[ps] then + ep[#ep+1] = input.clean_path(ps) + done[ps] = true + end + end + end + else + for p in paths:gmatch("[^,]+") do + if not done[p] then + ep[#ep+1] = input.clean_path(p) + done[p] = true + end + end + end + elseif subpaths and subpaths ~= "" then + for i=1,n do + -- we gmatch each step again, not that fast, but used seldom + for s in subpaths:gmatch("[^,]+") do + local ps = ep[i] .. "/" .. s + if not done[ps] then + ep[#ep+1] = input.clean_path(ps) + done[ps] = true + end + end + end + end + if #ep > 0 then + instance.extra_paths = ep -- register paths + end + if #ep > n then + instance.lists = { } -- erase the cache + end + end + +end + +function input.expanded_path_list(instance,str) + local function made_list(list) + local ep = instance.extra_paths + if not ep or #ep == 0 then + return list + else + local done, new = { }, { } + -- honour . .. ../.. but only when at the start + for k, v in ipairs(list) do + if not done[v] then + if v:find("^[%.%/]$") then + done[v] = true + new[#new+1] = v + else + break + end + end + end + -- first the extra paths + for k, v in ipairs(ep) do + if not done[v] then + done[v] = true + new[#new+1] = v + end + end + -- next the formal paths + for k, v in ipairs(list) do + if not done[v] then + done[v] = true + new[#new+1] = v + end + end + return new + end + end + if not str then + return ep or { } + elseif instance.savelists then + -- engine+progname hash + str = str:gsub("%$","") + if not instance.lists[str] then -- cached + local lst = made_list(input.split_path(input.expansion(instance,str))) + instance.lists[str] = input.aux.expanded_path(instance,lst) + end + return instance.lists[str] + else + local lst = input.split_path(input.expansion(instance,str)) + return made_list(input.aux.expanded_path(instance,lst)) + end +end + +function input.expand_path(instance,str) + return file.join_path(input.expanded_path_list(instance,str)) +end + +--~ function input.first_writable_path(instance,name) +--~ for _,v in pairs(input.expanded_path_list(instance,name)) do +--~ if file.is_writable(file.join(v,'luatex-cache.tmp')) then +--~ return v +--~ end +--~ end +--~ return "." +--~ end + +function input.expanded_path_list_from_var(instance,str) -- brrr + local tmp = input.var_of_format_or_suffix(str:gsub("%$","")) + if tmp ~= "" then + return input.expanded_path_list(instance,str) + else + return input.expanded_path_list(instance,tmp) + end +end +function input.expand_path_from_var(instance,str) + return file.join_path(input.expanded_path_list_from_var(instance,str)) +end + +function input.format_of_var(str) + return input.formats[str] or input.formats[input.alternatives[str]] or '' +end +function input.format_of_suffix(str) + return input.suffixmap[file.extname(str)] or 'tex' +end + +function input.variable_of_format(str) + return input.formats[str] or input.formats[input.alternatives[str]] or '' +end + +function input.var_of_format_or_suffix(str) + local v = input.formats[str] + if v then + return v + end + v = input.formats[input.alternatives[str]] + if v then + return v + end + v = input.suffixmap[file.extname(str)] + if v then + return input.formats[isf] + end + return '' +end + +function input.expand_braces(instance,str) -- output variable and brace expansion of STRING + local ori = input.variable(instance,str) + local pth = input.aux.expanded_path(instance,input.split_path(ori)) + return file.join_path(pth) +end + +-- {a,b,c,d} +-- a,b,c/{p,q,r},d +-- a,b,c/{p,q,r}/d/{x,y,z}// +-- a,b,c/{p,q/{x,y,z},r},d/{p,q,r} +-- a,b,c/{p,q/{x,y,z},r},d/{p,q,r} +-- a{b,c}{d,e}f +-- {a,b,c,d} +-- {a,b,c/{p,q,r},d} +-- {a,b,c/{p,q,r}/d/{x,y,z}//} +-- {a,b,c/{p,q/{x,y,z}},d/{p,q,r}} +-- {a,b,c/{p,q/{x,y,z},w}v,d/{p,q,r}} + +-- this one is better and faster, but it took me a while to realize +-- that this kind of replacement is cleaner than messy parsing and +-- fuzzy concatenating we can probably gain a bit with selectively +-- applying lpeg, but experiments with lpeg parsing this proved not to +-- work that well; the parsing is ok, but dealing with the resulting +-- table is a pain because we need to work inside-out recursively + +-- get rid of piecewise here, just a gmatch is ok + +function input.aux.splitpathexpr(str, t, validate) + -- no need for optimization, only called a few times, we can use lpeg for the sub + t = t or { } + local concat = table.concat + while true do + local done = false + while true do + local ok = false + str = str:gsub("([^{},]+){([^{}]-)}", function(a,b) + local t = { } + b:piecewise(",", function(s) t[#t+1] = a .. s end) + ok, done = true, true + return "{" .. concat(t,",") .. "}" + end) + if not ok then break end + end + while true do + local ok = false + str = str:gsub("{([^{}]-)}([^{},]+)", function(a,b) + local t = { } + a:piecewise(",", function(s) t[#t+1] = s .. b end) + ok, done = true, true + return "{" .. concat(t,",") .. "}" + end) + if not ok then break end + end + while true do + local ok = false + str = str:gsub("([,{]){([^{}]+)}([,}])", function(a,b,c) + ok, done = true, true + return a .. b .. c + end) + if not ok then break end + end + if not done then break end + end + while true do + local ok = false + str = str:gsub("{([^{}]-)}{([^{}]-)}", function(a,b) + local t = { } + a:piecewise(",", function(sa) + b:piecewise(",", function(sb) + t[#t+1] = sa .. sb + end) + end) + ok = true + return "{" .. concat(t,",") .. "}" + end) + if not ok then break end + end + while true do + local ok = false + str = str:gsub("{([^{}]-)}", function(a) + ok = true + return a + end) + if not ok then break end + end + if validate then + str:piecewise(",", function(s) + s = validate(s) + if s then t[#t+1] = s end + end) + else + str:piecewise(",", function(s) + t[#t+1] = s + end) + end + return t +end + +function input.aux.expanded_path(instance,pathlist) -- maybe not a list, just a path + -- a previous version fed back into pathlist + local newlist, ok = { }, false + for _,v in ipairs(pathlist) do + if v:find("[{}]") then + ok = true + break + end + end + if ok then + for _, v in ipairs(pathlist) do + input.aux.splitpathexpr(v, newlist, function(s) + s = file.collapse_path(s) + return s ~= "" and not s:find(instance.dummy_path_expr) and s + end) + end + else + for _,v in ipairs(pathlist) do + for vv in string.gmatch(v..',',"(.-),") do + vv = file.collapse_path(v) + if vv ~= "" then newlist[#newlist+1] = vv end + end + end + end + return newlist +end + +input.is_readable = { } + +function input.aux.is_readable(readable, name) + if input.trace > 2 then + if readable then + input.logger("+ readable", name) + else + input.logger("- readable", name) + end + end + return readable +end + +function input.is_readable.file(name) + -- return input.aux.is_readable(file.is_readable(name), name) + return input.aux.is_readable(input.aux.is_file(name), name) +end + +input.is_readable.tex = input.is_readable.file + +-- name +-- name/name + +function input.aux.collect_files(instance,names) + local filelist = { } + for _, fname in pairs(names) do + if fname then + if input.trace > 2 then + input.logger("? blobpath asked",fname) + end + local bname = file.basename(fname) + local dname = file.dirname(fname) + if dname == "" or dname:find("^%.") then + dname = false + else + dname = "/" .. dname .. "$" + end + for _, hash in ipairs(instance.hashes) do + local blobpath = hash.tag + local files = blobpath and instance.files[blobpath] + if files then + if input.trace > 2 then + input.logger('? blobpath do',blobpath .. " (" .. bname ..")") + end + local blobfile = files[bname] + if not blobfile then + local rname = "remap:"..bname + blobfile = files[rname] + if blobfile then + bname = files[rname] + blobfile = files[bname] + end + end + if blobfile then + if type(blobfile) == 'string' then + if not dname or blobfile:find(dname) then + filelist[#filelist+1] = { + hash.type, + file.join(blobpath,blobfile,bname), -- search + input.concatinators[hash.type](blobpath,blobfile,bname) -- result + } + end + else + for _, vv in pairs(blobfile) do + if not dname or vv:find(dname) then + filelist[#filelist+1] = { + hash.type, + file.join(blobpath,vv,bname), -- search + input.concatinators[hash.type](blobpath,vv,bname) -- result + } + end + end + end + end + elseif input.trace > 1 then + input.logger('! blobpath no',blobpath .. " (" .. bname ..")" ) + end + end + end + end + if #filelist > 0 then + return filelist + else + return nil + end +end + +function input.suffix_of_format(str) + if input.suffixes[str] then + return input.suffixes[str][1] + else + return "" + end +end + +function input.suffixes_of_format(str) + if input.suffixes[str] then + return input.suffixes[str] + else + return {} + end +end + +do + + -- called about 700 times for an empty doc (font initializations etc) + -- i need to weed the font files for redundant calls + + local letter = lpeg.R("az","AZ") + local separator = lpeg.P("://") + + local qualified = lpeg.P(".")^0 * lpeg.P("/") + letter*lpeg.P(":") + letter^1*separator + local rootbased = lpeg.P("/") + letter*lpeg.P(":") + + -- ./name ../name /name c: :// + function input.aux.qualified_path(filename) + return qualified:match(filename) + end + function input.aux.rootbased_path(filename) + return rootbased:match(filename) + end + + function input.normalize_name(original) + return original + end + + input.normalize_name = file.collapse_path + +end + +function input.aux.register_in_trees(instance,name) + if not name:find("^%.") then + instance.foundintrees[name] = (instance.foundintrees[name] or 0) + 1 -- maybe only one + end +end + +-- split the next one up, better for jit + +function input.aux.find_file(instance,filename) -- todo : plugin (scanners, checkers etc) + local result = { } + local stamp = nil + filename = input.normalize_name(filename) -- elsewhere + filename = file.collapse_path(filename:gsub("\\","/")) -- elsewhere + -- speed up / beware: format problem + if instance.remember then + stamp = filename .. "--" .. instance.engine .. "--" .. instance.progname .. "--" .. instance.format + if instance.found[stamp] then + input.logger('! remembered', filename) + return instance.found[stamp] + end + end + if filename:find('%*') then + input.logger('! wildcard', filename) + result = input.find_wildcard_files(instance,filename) + elseif input.aux.qualified_path(filename) then + if input.is_readable.file(filename) then + input.logger('! qualified', filename) + result = { filename } + else + local forcedname, ok = "", false + if file.extname(filename) == "" then + if instance.format == "" then + forcedname = filename .. ".tex" + if input.is_readable.file(forcedname) then + input.logger('! no suffix, forcing standard filetype tex') + result, ok = { forcedname }, true + end + else + for _, s in pairs(input.suffixes_of_format(instance.format)) do + forcedname = filename .. "." .. s + if input.is_readable.file(forcedname) then + input.logger('! no suffix, forcing format filetype', s) + result, ok = { forcedname }, true + break + end + end + end + end + if not ok then + input.logger('? qualified', filename) + end + end + else + -- search spec + local filetype, extra, done, wantedfiles, ext = '', nil, false, { }, file.extname(filename) + if ext == "" then + if not instance.force_suffixes then + wantedfiles[#wantedfiles+1] = filename + end + else + wantedfiles[#wantedfiles+1] = filename + end + if instance.format == "" then + if ext == "" then + local forcedname = filename .. '.tex' + wantedfiles[#wantedfiles+1] = forcedname + filetype = input.format_of_suffix(forcedname) + input.logger('! forcing filetype',filetype) + else + filetype = input.format_of_suffix(filename) + input.logger('! using suffix based filetype',filetype) + end + else + if ext == "" then + for _, s in pairs(input.suffixes_of_format(instance.format)) do + wantedfiles[#wantedfiles+1] = filename .. "." .. s + end + end + filetype = instance.format + input.logger('! using given filetype',filetype) + end + local typespec = input.variable_of_format(filetype) + local pathlist = input.expanded_path_list(instance,typespec) + if not pathlist or #pathlist == 0 then + -- no pathlist, access check only / todo == wildcard + if input.trace > 2 then + input.logger('? filename',filename) + input.logger('? filetype',filetype or '?') + input.logger('? wanted files',table.concat(wantedfiles," | ")) + end + for _, fname in pairs(wantedfiles) do + if fname and input.is_readable.file(fname) then + filename, done = fname, true + result[#result+1] = file.join('.',fname) + break + end + end + -- this is actually 'other text files' or 'any' or 'whatever' + local filelist = input.aux.collect_files(instance,wantedfiles) + local fl = filelist and filelist[1] + if fl then + filename = fl[3] + result[#result+1] = filename + done = true + end + else + -- list search + local filelist = input.aux.collect_files(instance,wantedfiles) + local doscan, recurse + if input.trace > 2 then + input.logger('? filename',filename) + -- if pathlist then input.logger('? path list',table.concat(pathlist," | ")) end + -- if filelist then input.logger('? file list',table.concat(filelist," | ")) end + end + -- a bit messy ... esp the doscan setting here + for _, path in pairs(pathlist) do + if path:find("^!!") then doscan = false else doscan = true end + if path:find("//$") then recurse = true else recurse = false end + local pathname = path:gsub("^!+", '') + done = false + -- using file list + if filelist and not (done and not instance.allresults) and recurse then + -- compare list entries with permitted pattern + pathname = pathname:gsub("([%-%.])","%%%1") -- this also influences + pathname = pathname:gsub("/+$", '/.*') -- later usage of pathname + pathname = pathname:gsub("//", '/.-/') -- not ok for /// but harmless + local expr = "^" .. pathname + -- input.debug('?',expr) + for _, fl in ipairs(filelist) do + local f = fl[2] + if f:find(expr) then + -- input.debug('T',' '..f) + if input.trace > 2 then + input.logger('= found in hash',f) + end + --- todo, test for readable + result[#result+1] = fl[3] + input.aux.register_in_trees(instance,f) -- for tracing used files + done = true + if not instance.allresults then break end + else + -- input.debug('F',' '..f) + end + end + end + if not done and doscan then + -- check if on disk / unchecked / does not work at all / also zips + if input.method_is_file(pathname) then -- ? + local pname = pathname:gsub("%.%*$",'') + if not pname:find("%*") then + local ppname = pname:gsub("/+$","") + if input.aux.can_be_dir(instance,ppname) then + for _, w in pairs(wantedfiles) do + local fname = file.join(ppname,w) + if input.is_readable.file(fname) then + if input.trace > 2 then + input.logger('= found by scanning',fname) + end + result[#result+1] = fname + done = true + if not instance.allresults then break end + end + end + else + -- no access needed for non existing path, speedup (esp in large tree with lots of fake) + end + end + end + end + if not done and doscan then + -- todo: slow path scanning + end + if done and not instance.allresults then break end + end + end + end + for k,v in pairs(result) do + result[k] = file.collapse_path(v) + end + if instance.remember then + instance.found[stamp] = result + end + return result +end + +input.aux._find_file_ = input.aux.find_file + +function input.aux.find_file(instance,filename) -- maybe make a lowres cache too + local result = input.aux._find_file_(instance,filename) + if #result == 0 then + local lowered = filename:lower() + if filename ~= lowered then + return input.aux._find_file_(instance,lowered) + end + end + return result +end + +if lfs and lfs.isfile then + input.aux.is_file = lfs.isfile -- to be done: use this +else + input.aux.is_file = file.is_readable +end + +if lfs and lfs.isdir then + function input.aux.can_be_dir(instance,name) + if not instance.fakepaths[name] then + if lfs.isdir(name) then + instance.fakepaths[name] = 1 -- directory + else + instance.fakepaths[name] = 2 -- no directory + end + end + return (instance.fakepaths[name] == 1) + end +else + function input.aux.can_be_dir() + return true + end +end + +if not input.concatinators then input.concatinators = { } end + +input.concatinators.tex = file.join +input.concatinators.file = input.concatinators.tex + +function input.find_files(instance,filename,filetype,mustexist) + if type(mustexist) == boolean then + -- all set + elseif type(filetype) == 'boolean' then + filetype, mustexist = nil, false + elseif type(filetype) ~= 'string' then + filetype, mustexist = nil, false + end + instance.format = filetype or '' + local t = input.aux.find_file(instance,filename,true) + instance.format = '' + return t +end + +function input.find_file(instance,filename,filetype,mustexist) + return (input.find_files(instance,filename,filetype,mustexist)[1] or "") +end + +function input.find_given_files(instance,filename) + local bname, result = file.basename(filename), { } + for k, hash in ipairs(instance.hashes) do + local files = instance.files[hash.tag] + local blist = files[bname] + if not blist then + local rname = "remap:"..bname + blist = files[rname] + if blist then + bname = files[rname] + blist = files[bname] + end + end + if blist then + if type(blist) == 'string' then + result[#result+1] = input.concatinators[hash.type](hash.tag,blist,bname) or "" + if not instance.allresults then break end + else + for kk,vv in pairs(blist) do + result[#result+1] = input.concatinators[hash.type](hash.tag,vv,bname) or "" + if not instance.allresults then break end + end + end + end + end + return result +end + +function input.find_given_file(instance,filename) + return (input.find_given_files(instance,filename)[1] or "") +end + +function input.find_wildcard_files(instance,filename) -- todo: remap: + local result = { } + local bname, dname = file.basename(filename), file.dirname(filename) + local path = dname:gsub("^*/","") + path = path:gsub("*",".*") + path = path:gsub("-","%%-") + if dname == "" then + path = ".*" + end + local name = bname + name = name:gsub("*",".*") + name = name:gsub("-","%%-") + path = path:lower() + name = name:lower() + local function doit(blist,bname,hash,allresults) + local done = false + if blist then + if type(blist) == 'string' then + -- make function and share code + if (blist:lower()):find(path) then + result[#result+1] = input.concatinators[hash.type](hash.tag,blist,bname) or "" + done = true + end + else + for kk,vv in pairs(blist) do + if (vv:lower()):find(path) then + result[#result+1] = input.concatinators[hash.type](hash.tag,vv,bname) or "" + done = true + if not allresults then break end + end + end + end + end + return done + end + local files, allresults, done = instance.files, instance.allresults, false + if name:find("%*") then + for k, hash in ipairs(instance.hashes) do + for kk, hh in pairs(files[hash.tag]) do + if not kk:find("^remap:") then + if (kk:lower()):find(name) then + if doit(hh,kk,hash,allresults) then done = true end + if done and not allresults then break end + end + end + end + end + else + for k, hash in ipairs(instance.hashes) do + if doit(files[hash.tag][bname],bname,hash,allresults) then done = true end + if done and not allresults then break end + end + end + return result +end + +function input.find_wildcard_file(instance,filename) + return (input.find_wildcard_files(instance,filename)[1] or "") +end + +-- main user functions + +function input.save_used_files_in_trees(instance, filename,jobname) + if not filename then filename = 'luatex.jlg' end + local f = io.open(filename,'w') + if f then + f:write("<?xml version='1.0' standalone='yes'?>\n") + f:write("<rl:job>\n") + if jobname then + f:write("\t<rl:name>" .. jobname .. "</rl:name>\n") + end + f:write("\t<rl:files>\n") + for _,v in pairs(table.sortedkeys(instance.foundintrees)) do + f:write("\t\t<rl:file n='" .. instance.foundintrees[v] .. "'>" .. v .. "</rl:file>\n") + end + f:write("\t</rl:files>\n") + f:write("</rl:usedfiles>\n") + f:close() + end +end + +function input.automount(instance) + -- implemented later +end + +function input.load(instance) + input.starttiming(instance) + input.resetconfig(instance) + input.identify_cnf(instance) + input.load_lua(instance) + input.expand_variables(instance) + input.load_cnf(instance) + input.expand_variables(instance) + input.load_hash(instance) + input.automount(instance) + input.stoptiming(instance) +end + +function input.for_files(instance, command, files, filetype, mustexist) + if files and #files > 0 then + local function report(str) + if input.verbose then + input.report(str) -- has already verbose + else + print(str) + end + end + if input.verbose then + report('') + end + for _, file in pairs(files) do + local result = command(instance,file,filetype,mustexist) + if type(result) == 'string' then + report(result) + else + for _,v in pairs(result) do + report(v) + end + end + end + end +end + +-- strtab + +function input.var_value(instance,str) -- output the value of variable $STRING. + return input.variable(instance,str) +end +function input.expand_var(instance,str) -- output variable expansion of STRING. + return input.expansion(instance,str) +end +function input.show_path(instance,str) -- output search path for file type NAME + return file.join_path(input.expanded_path_list(instance,input.format_of_var(str))) +end + +-- input.find_file(filename) +-- input.find_file(filename, filetype, mustexist) +-- input.find_file(filename, mustexist) +-- input.find_file(filename, filetype) + +function input.aux.register_file(files, name, path) + if files[name] then + if type(files[name]) == 'string' then + files[name] = { files[name], path } + else + files[name] = path + end + else + files[name] = path + end +end + +if not input.finders then input.finders = { } end +if not input.openers then input.openers = { } end +if not input.loaders then input.loaders = { } end + +input.finders.notfound = { nil } +input.openers.notfound = { nil } +input.loaders.notfound = { false, nil, 0 } + +function input.splitmethod(filename) + if not filename then + return { } -- safeguard + elseif type(filename) == "table" then + return filename -- already split + elseif not filename:find("://") then + return { scheme="file", path = filename, original=filename } -- quick hack + else + return url.hashed(filename) + end +end + +function input.method_is_file(filename) + return input.splitmethod(filename).scheme == 'file' +end + +function table.sequenced(t,sep) -- temp here + local s = { } + for k, v in pairs(t) do + s[#s+1] = k .. "=" .. v + end + return table.concat(s, sep or " | ") +end + +function input.methodhandler(what, instance, filename, filetype) -- ... + local specification = (type(filename) == "string" and input.splitmethod(filename)) or filename -- no or { }, let it bomb + local scheme = specification.scheme + if input[what][scheme] then + input.logger('= handler',specification.original .." -> " .. what .. " -> " .. table.sequenced(specification)) + return input[what][scheme](instance,filename,filetype) -- todo: specification + else + return input[what].tex(instance,filename,filetype) -- todo: specification + end +end + +-- also inside next test? + +function input.findtexfile(instance, filename, filetype) + return input.methodhandler('finders',instance, input.normalize_name(filename), filetype) +end +function input.opentexfile(instance,filename) + return input.methodhandler('openers',instance, input.normalize_name(filename)) +end + +function input.findbinfile(instance, filename, filetype) + return input.methodhandler('finders',instance, input.normalize_name(filename), filetype) +end +function input.openbinfile(instance,filename) + return input.methodhandler('loaders',instance, input.normalize_name(filename)) +end + +function input.loadbinfile(instance, filename, filetype) + local fname = input.findbinfile(instance, input.normalize_name(filename), filetype) + if fname and fname ~= "" then + return input.openbinfile(instance,fname) + else + return unpack(input.loaders.notfound) + end +end + +function input.texdatablob(instance, filename, filetype) + local ok, data, size = input.loadbinfile(instance, filename, filetype) + return data or "" +end + +input.loadtexfile = input.texdatablob + +function input.openfile(filename) -- brrr texmf.instance here / todo ! ! ! ! ! + local fullname = input.findtexfile(texmf.instance, filename) + if fullname and (fullname ~= "") then + return input.opentexfile(texmf.instance, fullname) + else + return nil + end +end + +function input.logmode() + return (os.getenv("MTX.LOG.MODE") or os.getenv("MTX_LOG_MODE") or "tex"):lower() +end + +-- this is a prelude to engine/progname specific configuration files +-- in which case we can omit files meant for other programs and +-- packages + +--- ctx + +-- maybe texinputs + font paths +-- maybe positive selection tex/context fonts/tfm|afm|vf|opentype|type1|map|enc + +input.validators = { } +input.validators.visibility = { } + +function input.validators.visibility.default(path, name) + return true +end + +function input.validators.visibility.context(path, name) + path = path[1] or path -- some day a loop + return not ( + path:find("latex") or +-- path:find("doc") or + path:find("tex4ht") or + path:find("source") or +-- path:find("config") or +-- path:find("metafont") or + path:find("lists$") or + name:find("%.tpm$") or + name:find("%.bak$") + ) +end + +-- todo: describe which functions are public (maybe input.private. ... ) + +-- beware: i need to check where we still need a / on windows: + +function input.clean_path(str) +--~ return (((str:gsub("\\","/")):gsub("^!+","")):gsub("//+","//")) + if str then + return ((str:gsub("\\","/")):gsub("^!+","")) + else + return nil + end +end + +function input.do_with_path(name,func) + for _, v in pairs(input.expanded_path_list(instance,name)) do + func("^"..input.clean_path(v)) + end +end + +function input.do_with_var(name,func) + func(input.aux.expanded_var(name)) +end + +function input.with_files(instance,pattern,handle) + for _, hash in ipairs(instance.hashes) do + local blobpath = hash.tag + local blobtype = hash.type + if blobpath then + local files = instance.files[blobpath] + if files then + for k,v in pairs(files) do + if k:find("^remap:") then + k = files[k] + v = files[k] -- chained + end + if k:find(pattern) then + if type(v) == "string" then + handle(blobtype,blobpath,v,k) + else + for _,vv in pairs(v) do + handle(blobtype,blobpath,vv,k) + end + end + end + end + end + end + end +end + +--~ function input.update_script(oldname,newname) -- oldname -> own.name, not per se a suffix +--~ newname = file.addsuffix(newname,"lua") +--~ local newscript = input.clean_path(input.find_file(instance, newname)) +--~ local oldscript = input.clean_path(oldname) +--~ input.report("old script", oldscript) +--~ input.report("new script", newscript) +--~ if oldscript ~= newscript and (oldscript:find(file.removesuffix(newname).."$") or oldscript:find(newname.."$")) then +--~ local newdata = io.loaddata(newscript) +--~ if newdata then +--~ input.report("old script content replaced by new content") +--~ io.savedata(oldscript,newdata) +--~ end +--~ end +--~ end + +function input.update_script(instance,oldname,newname) -- oldname -> own.name, not per se a suffix + local scriptpath = "scripts/context/lua" + newname = file.addsuffix(newname,"lua") + local oldscript = input.clean_path(oldname) + input.report("to be replaced old script", oldscript) + local newscripts = input.find_files(instance, newname) or { } + if #newscripts == 0 then + input.report("unable to locate new script") + else + for _, newscript in ipairs(newscripts) do + newscript = input.clean_path(newscript) + input.report("checking new script", newscript) + if oldscript == newscript then + input.report("old and new script are the same") + elseif not newscript:find(scriptpath) then + input.report("new script should come from",scriptpath) + elseif not (oldscript:find(file.removesuffix(newname).."$") or oldscript:find(newname.."$")) then + input.report("invalid new script name") + else + local newdata = io.loaddata(newscript) + if newdata then + input.report("old script content replaced by new content") + io.savedata(oldscript,newdata) + break + else + input.report("unable to load new script") + end + end + end + end +end + + +--~ print(table.serialize(input.aux.splitpathexpr("/usr/share/texmf-{texlive,tetex}", {}))) + +-- command line resolver: + +--~ print(input.resolve("abc env:tmp file:cont-en.tex path:cont-en.tex full:cont-en.tex rel:zapf/one/p-chars.tex")) + +do + + local resolvers = { } + + resolvers.environment = function(instance,str) + return input.clean_path(os.getenv(str) or os.getenv(str:upper()) or os.getenv(str:lower()) or "") + end + resolvers.relative = function(instance,str,n) + if io.exists(str) then + -- nothing + elseif io.exists("./" .. str) then + str = "./" .. str + else + local p = "../" + for i=1,n or 2 do + if io.exists(p .. str) then + str = p .. str + break + else + p = p .. "../" + end + end + end + return input.clean_path(str) + end + resolvers.locate = function(instance,str) + local fullname = input.find_given_file(instance,str) or "" + return input.clean_path((fullname ~= "" and fullname) or str) + end + resolvers.filename = function(instance,str) + local fullname = input.find_given_file(instance,str) or "" + return input.clean_path(file.basename((fullname ~= "" and fullname) or str)) + end + resolvers.pathname = function(instance,str) + local fullname = input.find_given_file(instance,str) or "" + return input.clean_path(file.dirname((fullname ~= "" and fullname) or str)) + end + + resolvers.env = resolvers.environment + resolvers.rel = resolvers.relative + resolvers.loc = resolvers.locate + resolvers.kpse = resolvers.locate + resolvers.full = resolvers.locate + resolvers.file = resolvers.filename + resolvers.path = resolvers.pathname + + local function resolve(instance,str) + if type(str) == "table" then + for k, v in pairs(str) do + str[k] = resolve(instance,v) or v + end + elseif str and str ~= "" then + str = str:gsub("([a-z]+):([^ ]+)", function(method,target) + if resolvers[method] then + return resolvers[method](instance,target) + else + return method .. ":" .. target + end + end) + end + return str + end + + input.resolve = resolve + +end + + +if not modules then modules = { } end modules ['luat-tmp'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +--[[ldx-- +<p>This module deals with caching data. It sets up the paths and +implements loaders and savers for tables. Best is to set the +following variable. When not set, the usual paths will be +checked. Personally I prefer the (users) temporary path.</p> + +</code> +TEXMFCACHE=$TMP;$TEMP;$TMPDIR;$TEMPDIR;$HOME;$TEXMFVAR;$VARTEXMF;. +</code> + +<p>Currently we do no locking when we write files. This is no real +problem because most caching involves fonts and the chance of them +being written at the same time is small. We also need to extend +luatools with a recache feature.</p> +--ldx]]-- + +caches = caches or { } +dir = dir or { } +texmf = texmf or { } + +caches.path = caches.path or nil +caches.base = caches.base or "luatex-cache" +caches.more = caches.more or "context" +caches.direct = false -- true is faster but may need huge amounts of memory +caches.trace = false +caches.tree = false +caches.paths = caches.paths or nil +caches.force = false + +input.usecache = not toboolean(os.getenv("TEXMFSHARECACHE") or "false",true) -- true + +function caches.temp(instance) + local function checkpath(cachepath) + if not cachepath or cachepath == "" then + return nil + elseif lfs.attributes(cachepath,"mode") == "directory" then -- lfs.isdir(cachepath) then + return cachepath + elseif caches.force or io.ask(string.format("Should I create the cache path %s?",cachepath), "no", { "yes", "no" }) == "yes" then + dir.mkdirs(cachepath) + return (lfs.attributes(cachepath,"mode") == "directory") and cachepath + else + return nil + end + end + local cachepath = input.expanded_path_list(instance,"TEXMFCACHE") + cachepath = cachepath and #cachepath > 0 and checkpath(cachepath[1]) + if not cachepath then + cachepath = os.getenv("TEXMFCACHE") or os.getenv("HOME") or os.getenv("HOMEPATH") or os.getenv("TMP") or os.getenv("TEMP") or os.getenv("TMPDIR") or nil + cachepath = checkpath(cachepath) + end + if not cachepath then + print("\nfatal error: there is no valid cache path defined\n") + os.exit() + elseif lfs.attributes(cachepath,"mode") ~= "directory" then + print(string.format("\nfatal error: cache path %s is not a directory\n",cachepath)) + os.exit() + end + function caches.temp(instance) + return cachepath + end + return cachepath +end + +function caches.configpath(instance) + return table.concat(instance.cnffiles,";") +end + +function caches.hashed(tree) + return md5.hex((tree:lower()):gsub("[\\\/]+","/")) +end + +function caches.treehash(instance) + local tree = caches.configpath(instance) + if not tree or tree == "" then + return false + else + return caches.hashed(tree) + end +end + +function caches.setpath(instance,...) + if not caches.path then + if not caches.path then + caches.path = caches.temp(instance) + end + caches.path = input.clean_path(caches.path) -- to be sure + if lfs then + caches.tree = caches.tree or caches.treehash(instance) + if caches.tree then + caches.path = dir.mkdirs(caches.path,caches.base,caches.more,caches.tree) + else + caches.path = dir.mkdirs(caches.path,caches.base,caches.more) + end + end + end + if not caches.path then + caches.path = '.' + end + caches.path = input.clean_path(caches.path) + if lfs and not table.is_empty({...}) then + local pth = dir.mkdirs(caches.path,...) + return pth + end + caches.path = dir.expand_name(caches.path) + return caches.path +end + +function caches.definepath(instance,category,subcategory) + return function() + return caches.setpath(instance,category,subcategory) + end +end + +function caches.setluanames(path,name) + return path .. "/" .. name .. ".tma", path .. "/" .. name .. ".tmc" +end + +function caches.loaddata(path,name) + local tmaname, tmcname = caches.setluanames(path,name) + local loader = loadfile(tmcname) or loadfile(tmaname) + if loader then + return loader() + else + return false + end +end + +function caches.is_writable(filepath,filename) + local tmaname, tmcname = caches.setluanames(filepath,filename) + return file.is_writable(tmaname) +end + +function caches.savedata(filepath,filename,data,raw) -- raw needed for file cache + local tmaname, tmcname = caches.setluanames(filepath,filename) + local reduce, simplify = true, true + if raw then + reduce, simplify = false, false + end + if caches.direct then + file.savedata(tmaname, table.serialize(data,'return',true,true)) + else + table.tofile (tmaname, data,'return',true,true) -- maybe not the last true + end + utils.lua.compile(tmaname, tmcname) +end + +-- here we use the cache for format loading (texconfig.[formatname|jobname]) + +--~ if tex and texconfig and texconfig.formatname and texconfig.formatname == "" then +if tex and texconfig and (not texconfig.formatname or texconfig.formatname == "") and texmf.instance then + if not texconfig.luaname then texconfig.luaname = "cont-en.lua" end -- or luc + texconfig.formatname = caches.setpath(texmf.instance,"formats") .. "/" .. texconfig.luaname:gsub("%.lu.$",".fmt") +end + +--[[ldx-- +<p>Once we found ourselves defining similar cache constructs +several times, containers were introduced. Containers are used +to collect tables in memory and reuse them when possible based +on (unique) hashes (to be provided by the calling function).</p> + +<p>Caching to disk is disabled by default. Version numbers are +stored in the saved table which makes it possible to change the +table structures without bothering about the disk cache.</p> + +<p>Examples of usage can be found in the font related code.</p> +--ldx]]-- + +containers = { } +containers.trace = false + +do -- local report + + local function report(container,tag,name) + if caches.trace or containers.trace or container.trace then + logs.report(string.format("%s cache",container.subcategory),string.format("%s: %s",tag,name or 'invalid')) + end + end + + local allocated = { } + + -- tracing + + function containers.define(category, subcategory, version, enabled) + return function() + if category and subcategory then + local c = allocated[category] + if not c then + c = { } + allocated[category] = c + end + local s = c[subcategory] + if not s then + s = { + category = category, + subcategory = subcategory, + storage = { }, + enabled = enabled, + version = version or 1.000, + trace = false, + path = caches.setpath(texmf.instance,category,subcategory), + } + c[subcategory] = s + end + return s + else + return nil + end + end + end + + function containers.is_usable(container, name) + return container.enabled and caches.is_writable(container.path, name) + end + + function containers.is_valid(container, name) + if name and name ~= "" then + local storage = container.storage[name] + return storage and not table.is_empty(storage) and storage.cache_version == container.version + else + return false + end + end + + function containers.read(container,name) + if container.enabled and not container.storage[name] then + container.storage[name] = caches.loaddata(container.path,name) + if containers.is_valid(container,name) then + report(container,"loaded",name) + else + container.storage[name] = nil + end + end + if container.storage[name] then + report(container,"reusing",name) + end + return container.storage[name] + end + + function containers.write(container, name, data) + if data then + data.cache_version = container.version + if container.enabled then + local unique, shared = data.unique, data.shared + data.unique, data.shared = nil, nil + caches.savedata(container.path, name, data) + report(container,"saved",name) + data.unique, data.shared = unique, shared + end + report(container,"stored",name) + container.storage[name] = data + end + return data + end + + function containers.content(container,name) + return container.storage[name] + end + +end + +-- since we want to use the cache instead of the tree, we will now +-- reimplement the saver. + +local save_data = input.aux.save_data + +input.cachepath = nil + +function input.aux.save_data(instance, dataname, check) + input.cachepath = input.cachepath or caches.definepath(instance,"trees") + save_data(instance, dataname, check, function(cachename,dataname) + if input.usecache then + return file.join(input.cachepath(),caches.hashed(cachename)) + else + return file.join(cachename,dataname) + end + end) +end + +local load_data = input.aux.load_data + +function input.aux.load_data(instance,pathname,dataname,filename) + input.cachepath = input.cachepath or caches.definepath(instance,"trees") + load_data(instance,pathname,dataname,filename,function(dataname,filename) + if input.usecache then + return file.join(input.cachepath(),caches.hashed(pathname)) + else + if not filename or (filename == "") then + filename = dataname + end + return file.join(pathname,filename) + end + end) +end + +-- we will make a better format, maybe something xml or just text or lua + +input.automounted = input.automounted or { } + +function input.automount(instance,usecache) + local mountpaths = input.simplified_list(input.expansion(instance,'TEXMFMOUNT')) + if table.is_empty(mountpaths) and usecache then + mountpaths = { caches.setpath(instance,"mount") } + end + if not table.is_empty(mountpaths) then + input.starttiming(instance) + for k, root in pairs(mountpaths) do + local f = io.open(root.."/url.tmi") + if f then + for line in f:lines() do + if line then + if line:find("^[%%#%-]") then -- or %W + -- skip + elseif line:find("^zip://") then + input.report("mounting",line) + table.insert(input.automounted,line) + input.usezipfile(instance,line) + end + end + end + f:close() + end + end + input.stoptiming(instance) + end +end + +-- store info in format + +input.storage = { } +input.storage.data = { } +input.storage.min = 0 -- 500 +input.storage.max = input.storage.min - 1 +input.storage.trace = false -- true +input.storage.done = 0 +input.storage.evaluators = { } +-- (evaluate,message,names) + +function input.storage.register(...) + input.storage.data[#input.storage.data+1] = { ... } +end + +function input.storage.evaluate(name) + input.storage.evaluators[#input.storage.evaluators+1] = name +end + +function input.storage.finalize() -- we can prepend the string with "evaluate:" + for _, t in ipairs(input.storage.evaluators) do + for i, v in pairs(t) do + if type(v) == "string" then + t[i] = loadstring(v)() + elseif type(v) == "table" then + for _, vv in pairs(v) do + if type(vv) == "string" then + t[i] = loadstring(vv)() + end + end + end + end + end +end + +function input.storage.dump() + for name, data in ipairs(input.storage.data) do + local evaluate, message, original, target = data[1], data[2], data[3] ,data[4] + local name, initialize, finalize, code = nil, "", "", "" + for str in target:gmatch("([^%.]+)") do + if name then + name = name .. "." .. str + else + name = str + end + initialize = string.format("%s %s = %s or {} ", initialize, name, name) + end + if evaluate then + finalize = "input.storage.evaluate(" .. name .. ")" + end + input.storage.max = input.storage.max + 1 + if input.storage.trace then + logs.report('storage',string.format('saving %s in slot %s',message,input.storage.max)) + code = + initialize .. + string.format("logs.report('storage','restoring %s from slot %s') ",message,input.storage.max) .. + table.serialize(original,name) .. + finalize + else + code = initialize .. table.serialize(original,name) .. finalize + end + lua.bytecode[input.storage.max] = loadstring(code) + end +end + +if lua.bytecode then -- from 0 upwards + local i = input.storage.min + while lua.bytecode[i] do + lua.bytecode[i]() + lua.bytecode[i] = nil + i = i + 1 + end + input.storage.done = i +end + + +-- filename : luat-zip.lua +-- comment : companion to luat-lib.tex +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['luat-zip'] = 1.001 + +if zip and input then + zip.supported = true +else + zip = { } + zip.supported = false +end + +if not zip.supported then + + if not input then input = { } end -- will go away + + function zip.openarchive (...) return nil end -- needed ? + function zip.closenarchive (...) end -- needed ? + function input.usezipfile (...) end -- needed ? + +else + + -- zip:///oeps.zip?name=bla/bla.tex + -- zip:///oeps.zip?tree=tex/texmf-local + + local function validzip(str) + if not str:find("^zip://") then + return "zip:///" .. str + else + return str + end + end + + zip.archives = { } + zip.registeredfiles = { } + + function zip.openarchive(instance,name) + if not name or name == "" then + return nil + else + local arch = zip.archives[name] + if arch then + return arch + else + local full = input.find_file(instance,name) or "" + local arch = (full ~= "" and zip.open(full)) or false + zip.archives[name] = arch + return arch + end + end + end + + function zip.closearchive(instance,name) + if not name or name == "" and zip.archives[name] then + zip.close(zip.archives[name]) + zip.archives[name] = nil + end + end + + -- zip:///texmf.zip?tree=/tex/texmf + -- zip:///texmf.zip?tree=/tex/texmf-local + -- zip:///texmf-mine.zip?tree=/tex/texmf-projects + + function input.locators.zip(instance,specification) -- where is this used? startup zips (untested) + specification = input.splitmethod(specification) + local zipfile = specification.path + local zfile = zip.openarchive(instance,name) -- tricky, could be in to be initialized tree + if zfile then + input.logger('! zip locator', specification.original ..' found') + else + input.logger('? zip locator', specification.original ..' not found') + end + end + + function input.hashers.zip(instance,tag,name) + input.report("loading zip file",name,"as",tag) + input.usezipfile(instance,tag .."?tree=" .. name) + end + + function input.concatinators.zip(tag,path,name) + if not path or path == "" then + return tag .. '?name=' .. name + else + return tag .. '?name=' .. path .. "/" .. name + end + end + + function input.is_readable.zip(name) + return true + end + + function input.finders.zip(instance,specification,filetype) + specification = input.splitmethod(specification) + if specification.path then + local q = url.query(specification.query) + if q.name then + local zfile = zip.openarchive(instance,specification.path) + if zfile then + input.logger('! zip finder',specification.path) + local dfile = zfile:open(q.name) + if dfile then + dfile = zfile:close() + input.logger('+ zip finder',q.name) + return specification.original + end + else + input.logger('? zip finder',specification.path) + end + end + end + input.logger('- zip finder',filename) + return unpack(input.finders.notfound) + end + + function input.openers.zip(instance,specification) + local zipspecification = input.splitmethod(specification) + if zipspecification.path then + local q = url.query(zipspecification.query) + if q.name then + local zfile = zip.openarchive(instance,zipspecification.path) + if zfile then + input.logger('+ zip starter',zipspecification.path) + local dfile = zfile:open(q.name) + if dfile then + input.show_open(specification) + return input.openers.text_opener(specification,dfile,'zip') + end + else + input.logger('- zip starter',zipspecification.path) + end + end + end + input.logger('- zip opener',filename) + return unpack(input.openers.notfound) + end + + function input.loaders.zip(instance,specification) + specification = input.splitmethod(specification) + if specification.path then + local q = url.query(specification.query) + if q.name then + local zfile = zip.openarchive(instance,specification.path) + if zfile then + input.logger('+ zip starter',specification.path) + local dfile = zfile:open(q.name) + if dfile then + input.show_load(filename) + input.logger('+ zip loader',filename) + local s = dfile:read("*all") + dfile:close() + return true, s, #s + end + else + input.logger('- zip starter',specification.path) + end + end + end + input.logger('- zip loader',filename) + return unpack(input.openers.notfound) + end + + -- zip:///somefile.zip + -- zip:///somefile.zip?tree=texmf-local -> mount + + function input.usezipfile(instance,zipname) + zipname = validzip(zipname) + input.logger('! zip use','file '..zipname) + local specification = input.splitmethod(zipname) + local zipfile = specification.path + if zipfile and not zip.registeredfiles[zipname] then + local tree = url.query(specification.query).tree or "" + input.logger('! zip register','file '..zipname) + local z = zip.openarchive(instance,zipfile) + if z then + input.logger("= zipfile","registering "..zipname) + input.starttiming(instance) + input.aux.prepend_hash(instance,'zip',zipname,zipfile) + input.aux.extend_texmf_var(instance,zipname) -- resets hashes too + zip.registeredfiles[zipname] = z + instance.files[zipname] = input.aux.register_zip_file(z,tree or "") + input.stoptiming(instance) + else + input.logger("? zipfile","unknown "..zipname) + end + else + input.logger('! zip register','no file '..zipname) + end + end + + function input.aux.register_zip_file(z,tree) + local files, filter = { }, "" + if tree == "" then + filter = "^(.+)/(.-)$" + else + filter = "^"..tree.."/(.+)/(.-)$" + end + input.logger('= zip filter',filter) + local register, n = input.aux.register_file, 0 + for i in z:files() do + local path, name = i.filename:match(filter) + if path then + if name and name ~= '' then + register(files, name, path) + n = n + 1 + else + -- directory + end + else + register(files, i.filename, '') + n = n + 1 + end + end + input.report('= zip entries',n) + return files + end + +end + + +-- filename : luat-zip.lua +-- comment : companion to luat-lib.tex +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['luat-tex'] = 1.001 + +-- special functions that deal with io + +if texconfig and not texlua then + + input.level = input.level or 0 + + if input.logmode() == 'xml' then + function input.show_open(name) + input.level = input.level + 1 + texio.write_nl("<f l='"..input.level.."' n='"..name.."'>") + end + function input.show_close(name) + texio.write("</f> ") + input.level = input.level - 1 + end + function input.show_load(name) + texio.write_nl("<f l='"..(input.level+1).."' n='"..name.."'/>") -- level? + end + else + function input.show_open () end + function input.show_close() end + function input.show_load () end + end + + function input.finders.generic(instance,tag,filename,filetype) + local foundname = input.find_file(instance,filename,filetype) + if foundname and foundname ~= "" then + input.logger('+ ' .. tag .. ' finder',filename,'filetype') + return foundname + else + input.logger('- ' .. tag .. ' finder',filename,'filetype') + return unpack(input.finders.notfound) + end + end + + input.filters.dynamic_translator = nil + input.filters.frozen_translator = nil + input.filters.utf_translator = nil + + function input.openers.text_opener(filename,file_handle,tag) + local u = unicode.utftype(file_handle) + local t = { } + if u > 0 then + input.logger('+ ' .. tag .. ' opener (' .. unicode.utfname[u] .. ')',filename) + local l + if u > 2 then + l = unicode.utf32_to_utf8(file_handle:read("*a"),u==4) + else + l = unicode.utf16_to_utf8(file_handle:read("*a"),u==2) + end + file_handle:close() + t = { + utftype = u, -- may go away + lines = l, + current = 0, -- line number, not really needed + handle = nil, + noflines = #l, + close = function() + input.logger('= ' .. tag .. ' closer (' .. unicode.utfname[u] .. ')',filename) + input.show_close(filename) + end, +--~ getline = function(n) +--~ local line = t.lines[n] +--~ if not line or line == "" then +--~ return "" +--~ else +--~ local translator = input.filters.utf_translator +--~ return (translator and translator(line)) or line +--~ end +--~ end, + reader = function(self) + self = self or t + local current, lines = self.current, self.lines + if current >= #lines then + return nil + else + current = current + 1 + self.current = current + local line = lines[current] + if line == "" then + return "" + else + local translator = input.filters.utf_translator + -- return (translator and translator(line)) or line + if translator then + return translator(line) + else + return line + end + end + end + end + } + else + input.logger('+ ' .. tag .. ' opener',filename) + -- todo: file;name -> freeze / eerste regel scannen -> freeze + local filters = input.filters + t = { + reader = function(self) + local line = file_handle:read() + if line == "" then + return "" + end + local translator = filters.utf_translator + if translator then + return translator(line) + end + translator = filters.dynamic_translator + if translator then + return translator(line) + end + return line + end, + close = function() + input.logger('= ' .. tag .. ' closer',filename) + input.show_close(filename) + file_handle:close() + end, + handle = function() + return file_handle + end, + noflines = function() + t.noflines = io.noflines(file_handle) + return t.noflines + end + } + end + return t + end + + function input.openers.generic(instance,tag,filename) + if filename and filename ~= "" then + local f = io.open(filename,"r") + if f then + input.show_open(filename) + return input.openers.text_opener(filename,f,tag) + end + end + input.logger('- ' .. tag .. ' opener',filename) + return unpack(input.openers.notfound) + end + + function input.loaders.generic(instance,tag,filename) + if filename and filename ~= "" then + local f = io.open(filename,"rb") + if f then + input.show_load(filename) + input.logger('+ ' .. tag .. ' loader',filename) + local s = f:read("*a") + f:close() + if s then + return true, s, #s + end + end + end + input.logger('- ' .. tag .. ' loader',filename) + return unpack(input.loaders.notfound) + end + + function input.finders.tex(instance,filename,filetype) + return input.finders.generic(instance,'tex',filename,filetype) + end + function input.openers.tex(instance,filename) + return input.openers.generic(instance,'tex',filename) + end + function input.loaders.tex(instance,filename) + return input.loaders.generic(instance,'tex',filename) + end + +end + +-- callback into the file io and related things; disabling kpse + + +if texconfig and not texlua then do + + -- this is not the right place, because we refer to quite some not yet defined tables, but who cares ... + + ctx = ctx or { } + + local ss = { } + + function ctx.writestatus(a,b) + local s = ss[a] + if not ss[a] then + s = a:rpadd(15) .. ": " + ss[a] = s + end + texio.write_nl(s .. b .. "\n") + end + + -- this will become: ctx.install_statistics(fnc() return ..,.. end) etc + + local statusinfo, n = { }, 0 + + function ctx.register_statistics(tag,pattern,fnc) + statusinfo[#statusinfo+1] = { tag, pattern, fnc } + if #tag > n then n = #tag end + end + + function ctx.show_statistics() -- todo: move calls + if caches then + ctx.register_statistics("used config path", "%s", function() return caches.configpath(texmf.instance) end) + ctx.register_statistics("used cache path", "%s", function() return caches.path end) + end + if status.luabytecodes > 0 and input.storage and input.storage.done then + ctx.register_statistics("modules/dumps/instances", "%s/%s/%s", function() return status.luabytecodes-500, input.storage.done, status.luastates end) + end + if texmf.instance then + ctx.register_statistics("input load time", "%s seconds", function() return input.loadtime(texmf.instance) end) + end + if fonts then + ctx.register_statistics("fonts load time","%s seconds", function() return input.loadtime(fonts) end) + end + if xml then + ctx.register_statistics("xml load time", "%s seconds, backreferences: %i, outer filtering time: %s", function() return input.loadtime(xml), #lxml.self, input.loadtime(lxml) end) + end + if mptopdf then + ctx.register_statistics("mps conversion time", "%s seconds", function() return input.loadtime(mptopdf) end) + end + if nodes then + ctx.register_statistics("node processing time", "%s seconds (including kernel)", function() return input.loadtime(nodes) end) + end + if kernel then + ctx.register_statistics("kernel processing time", "%s seconds", function() return input.loadtime(kernel) end) + end + if attributes then + ctx.register_statistics("attribute processing time", "%s seconds", function() return input.loadtime(attributes) end) + end + if languages then + ctx.register_statistics("language load time", "%s seconds, n=%s", function() return input.loadtime(languages), languages.hyphenation.n() end) + end + if figures then + ctx.register_statistics("graphics processing time", "%s seconds, n=%s (including tex)", function() return input.loadtime(figures), figures.n or "?" end) + end + if metapost then + ctx.register_statistics("metapost processing time", "%s seconds, loading: %s seconds, execution: %s seconds, n: %s", function() return input.loadtime(metapost), input.loadtime(mplib), input.loadtime(metapost.exectime), metapost.n end) + end + if status.luastate_bytes then + ctx.register_statistics("current memory usage", "%s bytes", function() return status.luastate_bytes end) + end + if nodes then + ctx.register_statistics("cleaned up reserved nodes", "%s nodes, %s lists of %s", function() return nodes.cleanup_reserved(tex.count[24]) end) -- \topofboxstack + end + if status.node_mem_usage then + ctx.register_statistics("node memory usage", "%s", function() return status.node_mem_usage end) + end + if languages then + ctx.register_statistics("loaded patterns", "%s", function() return languages.logger.report() end) + end + if fonts then + ctx.register_statistics("loaded fonts", "%s", function() return fonts.logger.report() end) + end + if xml then -- so we are in mkiv, we need a different check + ctx.register_statistics("runtime", "%s seconds, %i processed pages, %i shipped pages, %.3f pages/second", function() + input.stoptiming(texmf) + local runtime = input.loadtime(texmf) + local shipped = tex.count['nofshipouts'] + local pages = tex.count['realpageno'] - 1 + local persecond = shipped / runtime + return runtime, pages, shipped, persecond + end) + end + for _, t in ipairs(statusinfo) do + local tag, pattern, fnc = t[1], t[2], t[3] + ctx.writestatus("mkiv lua stats", string.format("%s - %s", tag:rpadd(n," "), pattern:format(fnc()))) + end + end + +end end + +if texconfig and not texlua then + + texconfig.kpse_init = false + texconfig.trace_file_names = input.logmode() == 'tex' + texconfig.max_print_line = 100000 + + -- if still present, we overload kpse (put it off-line so to say) + + if not texmf then texmf = { } end + + input.starttiming(texmf) + + if not texmf.instance then + + if not texmf.instance then -- prevent a second loading + + texmf.instance = input.reset() + texmf.instance.progname = environment.progname or 'context' + texmf.instance.engine = environment.engine or 'luatex' + texmf.instance.validfile = input.validctxfile + + input.load(texmf.instance) + + end + + if callback then + callback.register('find_read_file' , function(id,name) return input.findtexfile(texmf.instance,name) end) + callback.register('open_read_file' , function( name) return input.opentexfile(texmf.instance,name) end) + end + + if callback then + callback.register('find_data_file' , function(name) return input.findbinfile(texmf.instance,name,"tex") end) + callback.register('find_enc_file' , function(name) return input.findbinfile(texmf.instance,name,"enc") end) + callback.register('find_font_file' , function(name) return input.findbinfile(texmf.instance,name,"tfm") end) + callback.register('find_format_file' , function(name) return input.findbinfile(texmf.instance,name,"fmt") end) + callback.register('find_image_file' , function(name) return input.findbinfile(texmf.instance,name,"tex") end) + callback.register('find_map_file' , function(name) return input.findbinfile(texmf.instance,name,"map") end) + callback.register('find_ocp_file' , function(name) return input.findbinfile(texmf.instance,name,"ocp") end) + callback.register('find_opentype_file' , function(name) return input.findbinfile(texmf.instance,name,"otf") end) + callback.register('find_output_file' , function(name) return name end) + callback.register('find_pk_file' , function(name) return input.findbinfile(texmf.instance,name,"pk") end) + callback.register('find_sfd_file' , function(name) return input.findbinfile(texmf.instance,name,"sfd") end) + callback.register('find_truetype_file' , function(name) return input.findbinfile(texmf.instance,name,"ttf") end) + callback.register('find_type1_file' , function(name) return input.findbinfile(texmf.instance,name,"pfb") end) + callback.register('find_vf_file' , function(name) return input.findbinfile(texmf.instance,name,"vf") end) + + callback.register('read_data_file' , function(file) return input.loadbinfile(texmf.instance,file,"tex") end) + callback.register('read_enc_file' , function(file) return input.loadbinfile(texmf.instance,file,"enc") end) + callback.register('read_font_file' , function(file) return input.loadbinfile(texmf.instance,file,"tfm") end) + -- format + -- image + callback.register('read_map_file' , function(file) return input.loadbinfile(texmf.instance,file,"map") end) + callback.register('read_ocp_file' , function(file) return input.loadbinfile(texmf.instance,file,"ocp") end) + callback.register('read_opentype_file' , function(file) return input.loadbinfile(texmf.instance,file,"otf") end) + -- output + callback.register('read_pk_file' , function(file) return input.loadbinfile(texmf.instance,file,"pk") end) + callback.register('read_sfd_file' , function(file) return input.loadbinfile(texmf.instance,file,"sfd") end) + callback.register('read_truetype_file' , function(file) return input.loadbinfile(texmf.instance,file,"ttf") end) + callback.register('read_type1_file' , function(file) return input.loadbinfile(texmf.instance,file,"pfb") end) + callback.register('read_vf_file' , function(file) return input.loadbinfile(texmf.instance,file,"vf" ) end) + end + + if callback and environment.aleph_mode then + callback.register('find_font_file' , function(name) return input.findbinfile(texmf.instance,name,"ofm") end) + callback.register('read_font_file' , function(file) return input.loadbinfile(texmf.instance,file,"ofm") end) + callback.register('find_vf_file' , function(name) return input.findbinfile(texmf.instance,name,"ovf") end) + callback.register('read_vf_file' , function(file) return input.loadbinfile(texmf.instance,file,"ovf") end) + end + + if callback then + callback.register('find_write_file' , function(id,name) return name end) + end + + if callback and (not config or (#config == 0)) then + callback.register('find_format_file' , function(name) return name end) + end + + if callback and false then + for k, v in pairs(callback.list()) do + if not v then texio.write_nl("<w>callback "..k.." is not set</w>") end + end + end + + if callback then + + input.start_actions = { } + input.stop_actions = { } + + function input.register_start_actions(f) table.insert(input.start_actions, f) end + function input.register_stop_actions (f) table.insert(input.stop_actions, f) end + + --~ callback.register('start_run', function() for _, a in pairs(input.start_actions) do a() end end) + --~ callback.register('stop_run' , function() for _, a in pairs(input.stop_actions ) do a() end end) + + end + + if callback then + + if input.logmode() == 'xml' then + + function input.start_page_number() + texio.write_nl("<p real='" .. tex.count[0] .. "' page='"..tex.count[1].."' sub='"..tex.count[2].."'") + end + function input.stop_page_number() + texio.write("/>") + texio.write_nl("") + end + + callback.register('start_page_number' , input.start_page_number) + callback.register('stop_page_number' , input.stop_page_number ) + + function input.report_output_pages(p,b) + texio.write_nl("<v k='pages'>"..p.."</v>") + texio.write_nl("<v k='bytes'>"..b.."</v>") + texio.write_nl("") + end + function input.report_output_log() + end + + callback.register('report_output_pages', input.report_output_pages) + callback.register('report_output_log' , input.report_output_log ) + + function input.start_run() + texio.write_nl("<?xml version='1.0' standalone='yes'?>") + texio.write_nl("<job xmlns='www.tug.org/luatex/schemas/context-job.rng'>") + texio.write_nl("") + end + function input.stop_run() + texio.write_nl("</job>") + end + function input.show_statistics() + for k,v in pairs(status.list()) do + texio.write_nl("log","<v k='"..k.."'>"..tostring(v).."</v>") + end + end + + table.insert(input.start_actions, input.start_run) + table.insert(input.stop_actions , input.show_statistics) + table.insert(input.stop_actions , input.stop_run) + + else + table.insert(input.stop_actions , input.show_statistics) + end + + callback.register('start_run', function() for _, a in pairs(input.start_actions) do a() end end) + callback.register('stop_run' , function() for _, a in pairs(input.stop_actions ) do a() end ctx.show_statistics() end) + + end + + end + + if kpse then + + function kpse.find_file(filename,filetype,mustexist) + return input.find_file(texmf.instance,filename,filetype,mustexist) + end + function kpse.expand_path(variable) + return input.expand_path(texmf.instance,variable) + end + function kpse.expand_var(variable) + return input.expand_var(texmf.instance,variable) + end + function kpse.expand_braces(variable) + return input.expand_braces(texmf.instance,variable) + end + + end + +end + +-- program specific configuration (memory settings and alike) + +if texconfig and not texlua then + + luatex = luatex or { } + + luatex.variablenames = { + 'main_memory', 'extra_mem_bot', 'extra_mem_top', + 'buf_size','expand_depth', + 'font_max', 'font_mem_size', + 'hash_extra', 'max_strings', 'pool_free', 'pool_size', 'string_vacancies', + 'obj_tab_size', 'pdf_mem_size', 'dest_names_size', + 'nest_size', 'param_size', 'save_size', 'stack_size', + 'trie_size', 'hyph_size', 'max_in_open', + 'ocp_stack_size', 'ocp_list_size', 'ocp_buf_size' + } + + function luatex.variables() + local t, x = { }, nil + for _,v in pairs(luatex.variablenames) do + x = input.var_value(texmf.instance,v) + if x and x:find("^%d+$") then + t[v] = tonumber(x) + end + end + return t + end + + function luatex.setvariables(tab) + for k,v in pairs(luatex.variables()) do + tab[k] = v + end + end + + if not luatex.variables_set then + luatex.setvariables(texconfig) + luatex.variables_set = true + end + + texconfig.max_print_line = 100000 + texconfig.max_in_open = 127 + +end + +-- some tex basics + +if not cs then cs = { } end + +function cs.def(k,v) + tex.sprint(tex.texcatcodes, "\\def\\" .. k .. "{" .. v .. "}") +end + +function cs.chardef(k,v) + tex.sprint(tex.texcatcodes, "\\chardef\\" .. k .. "=" .. v .. "\\relax") +end + +function cs.boolcase(b) + if b then tex.write(1) else tex.write(0) end +end + +function cs.testcase(b) + if b then + tex.sprint(tex.texcatcodes, "\\firstoftwoarguments") + else + tex.sprint(tex.texcatcodes, "\\secondoftwoarguments") + end +end + + +if not modules then modules = { } end modules ['luat-kps'] = { + version = 1.001, + comment = "companion to luatools.lua", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +--[[ldx-- +<p>This file is used when we want the input handlers to behave like +<type>kpsewhich</type>. What to do with the following:</p> + +<typing> +{$SELFAUTOLOC,$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,}/web2c} +$SELFAUTOLOC : /usr/tex/bin/platform +$SELFAUTODIR : /usr/tex/bin +$SELFAUTOPARENT : /usr/tex +</typing> + +<p>How about just forgetting abou them?</p> +--ldx]]-- + +input = input or { } +input.suffixes = input.suffixes or { } +input.formats = input.formats or { } + +input.suffixes['gf'] = { '<resolution>gf' } +input.suffixes['pk'] = { '<resolution>pk' } +input.suffixes['base'] = { 'base' } +input.suffixes['bib'] = { 'bib' } +input.suffixes['bst'] = { 'bst' } +input.suffixes['cnf'] = { 'cnf' } +input.suffixes['mem'] = { 'mem' } +input.suffixes['mf'] = { 'mf' } +input.suffixes['mfpool'] = { 'pool' } +input.suffixes['mft'] = { 'mft' } +input.suffixes['mppool'] = { 'pool' } +input.suffixes['graphic/figure'] = { 'eps', 'epsi' } +input.suffixes['texpool'] = { 'pool' } +input.suffixes['PostScript header'] = { 'pro' } +input.suffixes['ist'] = { 'ist' } +input.suffixes['web'] = { 'web', 'ch' } +input.suffixes['cweb'] = { 'w', 'web', 'ch' } +input.suffixes['cmap files'] = { 'cmap' } +input.suffixes['lig files'] = { 'lig' } +input.suffixes['bitmap font'] = { } +input.suffixes['MetaPost support'] = { } +input.suffixes['TeX system documentation'] = { } +input.suffixes['TeX system sources'] = { } +input.suffixes['dvips config'] = { } +input.suffixes['type42 fonts'] = { } +input.suffixes['web2c files'] = { } +input.suffixes['other text files'] = { } +input.suffixes['other binary files'] = { } +input.suffixes['opentype fonts'] = { 'otf' } + +input.suffixes['fmt'] = { 'fmt' } +input.suffixes['texmfscripts'] = { 'rb','lua','py','pl' } + +input.suffixes['pdftex config'] = { } +input.suffixes['Troff fonts'] = { } + +input.suffixes['ls-R'] = { } + +--[[ldx-- +<p>If you wondered abou tsome of the previous mappings, how about +the next bunch:</p> +--ldx]]-- + +input.formats['bib'] = '' +input.formats['bst'] = '' +input.formats['mft'] = '' +input.formats['ist'] = '' +input.formats['web'] = '' +input.formats['cweb'] = '' +input.formats['MetaPost support'] = '' +input.formats['TeX system documentation'] = '' +input.formats['TeX system sources'] = '' +input.formats['Troff fonts'] = '' +input.formats['dvips config'] = '' +input.formats['graphic/figure'] = '' +input.formats['ls-R'] = '' +input.formats['other text files'] = '' +input.formats['other binary files'] = '' + +input.formats['gf'] = '' +input.formats['pk'] = '' +input.formats['base'] = 'MFBASES' +input.formats['cnf'] = '' +input.formats['mem'] = 'MPMEMS' +input.formats['mf'] = 'MFINPUTS' +input.formats['mfpool'] = 'MFPOOL' +input.formats['mppool'] = 'MPPOOL' +input.formats['texpool'] = 'TEXPOOL' +input.formats['PostScript header'] = 'TEXPSHEADERS' +input.formats['cmap files'] = 'CMAPFONTS' +input.formats['type42 fonts'] = 'T42FONTS' +input.formats['web2c files'] = 'WEB2C' +input.formats['pdftex config'] = 'PDFTEXCONFIG' +input.formats['texmfscripts'] = 'TEXMFSCRIPTS' +input.formats['bitmap font'] = '' +input.formats['lig files'] = 'LIGFONTS' + +-- end library merge + +-- We initialize some characteristics of this program. We need to +-- do this before we load the libraries, else own.name will not be +-- properly set (handy for selfcleaning the file). It's an ugly +-- looking piece of code. + +own = { } + +own.libs = { -- todo: check which ones are really needed + 'l-string.lua', + 'l-lpeg.lua', + 'l-table.lua', + 'l-io.lua', + 'l-number.lua', + 'l-set.lua', + 'l-os.lua', + 'l-md5.lua', + 'l-file.lua', + 'l-url.lua', + 'l-dir.lua', + 'l-boolean.lua', + 'l-unicode.lua', + 'l-utils.lua', + 'luat-lib.lua', + 'luat-inp.lua', + 'luat-tmp.lua', + 'luat-zip.lua', + 'luat-tex.lua', + 'luat-kps.lua', +} + +-- We need this hack till luatex is fixed. + +if arg and arg[0] == 'luatex' and arg[1] == "--luaonly" then + arg[-1]=arg[0] arg[0]=arg[2] for k=3,#arg do arg[k-2]=arg[k] end arg[#arg]=nil arg[#arg]=nil +end + +-- End of hack. + +own.name = (environment and environment.ownname) or arg[0] or 'luatools.lua' +own.path = string.match(own.name,"^(.+)[\\/].-$") or "." +own.list = { '.' } + +if own.path ~= '.' then + table.insert(own.list,own.path) +end + +table.insert(own.list,own.path.."/../../../tex/context/base") +table.insert(own.list,own.path.."/mtx") +table.insert(own.list,own.path.."/../sources") + +function locate_libs() + for _, lib in pairs(own.libs) do + for _, pth in pairs(own.list) do + local filename = string.gsub(pth .. "/" .. lib,"\\","/") + local codeblob = loadfile(filename) + if codeblob then + codeblob() + own.list = { pth } -- speed up te search + break + end + end + end +end + +if not input then + locate_libs() +end + +if not input then + print("") + print("Luatools is unable to start up due to lack of libraries. You may") + print("try to run 'lua luatools.lua --selfmerge' in the path where this") + print("script is located (normally under ..../scripts/context/lua) which") + print("will make luatools library independent.") + os.exit() +end + +instance = input.reset() +input.verbose = environment.arguments["verbose"] or false +input.banner = 'LuaTools | ' +utils.report = input.report + +input.defaultlibs = { -- not all are needed + 'l-string.lua', 'l-lpeg.lua', 'l-table.lua', 'l-boolean.lua', 'l-number.lua', 'l-set.lua', 'l-unicode.lua', + 'l-md5.lua', 'l-os.lua', 'l-io.lua', 'l-file.lua', 'l-url.lua', 'l-dir.lua', 'l-utils.lua', 'l-tex.lua', + 'luat-env.lua', 'luat-lib.lua', 'luat-inp.lua', 'luat-tmp.lua', 'luat-zip.lua', 'luat-tex.lua' +} + +-- todo: use environment.argument() instead of environment.arguments[] + +instance.engine = environment.arguments["engine"] or 'luatex' +instance.progname = environment.arguments["progname"] or 'context' +instance.luaname = environment.arguments["luafile"] or "" -- environment.ownname or "" +instance.lualibs = environment.arguments["lualibs"] or table.concat(input.defaultlibs,",") +instance.allresults = environment.arguments["all"] or false +instance.pattern = environment.arguments["pattern"] or nil +instance.sortdata = environment.arguments["sort"] or false +instance.kpseonly = not environment.arguments["all"] or false +instance.my_format = environment.arguments["format"] or instance.format +instance.lsrmode = environment.arguments["lsr"] or false + +if type(instance.pattern) == 'boolean' then + input.report("invalid pattern specification") -- toto, force verbose for one message + instance.pattern = nil +end + +if environment.arguments["trace"] then input.settrace(environment.arguments["trace"]) end + +if environment.arguments["minimize"] then + if input.validators.visibility[instance.progname] then + instance.validfile = input.validators.visibility[instance.progname] + end +end + +function input.my_prepare_a(instance) + input.resetconfig(instance) + input.identify_cnf(instance) + input.load_lua(instance) + input.expand_variables(instance) + input.load_cnf(instance) + input.expand_variables(instance) +end + +function input.my_prepare_b(instance) + input.my_prepare_a(instance) + input.load_hash(instance) + input.automount(instance) +end + +-- barename + +if not messages then messages = { } end + +messages.no_ini_file = [[ +There is no lua initialization file found. This file can be forced by the +"--progname" directive, or specified with "--luaname", or it is derived +automatically from the formatname (aka jobname). It may be that you have +to regenerate the file database using "luatools --generate". +]] + +messages.help = [[ +--generate generate file database +--variables show configuration variables +--expansions show expanded variables +--configurations show configuration order +--expand-braces expand complex variable +--expand-path expand variable (resolve paths) +--expand-var expand variable (resolve references) +--show-path show path expansion of ... +--var-value report value of variable +--find-file report file location +--find-path report path of file +--make or --ini make luatex format +--run or --fmt= run luatex format +--luafile=str lua inifile (default is <progname>.lua) +--lualibs=list libraries to assemble (optional when --compile) +--compile assemble and compile lua inifile +--mkii force context mkii mode (only for testing, not usable!) +--verbose give a bit more info +--minimize optimize lists for format +--all show all found files +--sort sort cached data +--engine=str target engine +--progname=str format or backend +--pattern=str filter variables +--lsr use lsr and cnf directly +]] + +function input.my_make_format(instance,texname) + if texname and texname ~= "" then + if input.usecache then + local path = file.join(caches.setpath(instance,"formats")) -- maybe platform + if path and lfs then + lfs.chdir(path) + end + end + local barename = texname:gsub("%.%a+$","") + if barename == texname then + texname = texname .. ".tex" + end + local fullname = input.find_files(instance,texname)[1] or "" + if fullname == "" then + input.report("no tex file with name",texname) + else + local luaname, lucname, luapath, lualibs = "", "", "", { } + -- the following is optional, since context.lua can also + -- handle this collect and compile business + if environment.arguments["compile"] then + if luaname == "" then luaname = barename end + input.report("creating initialization file " .. luaname) + luapath = file.dirname(luaname) + if luapath == "" then + luapath = file.dirname(texname) + end + if luapath == "" then + luapath = file.dirname(input.find_files(instance,texname)[1] or "") + end + lualibs = string.split(instance.lualibs,",") + luaname = file.basename(barename .. ".lua") + lucname = file.basename(barename .. ".luc") + -- todo: when this fails, we can just copy the merged libraries from + -- luatools since they are normally the same, at least for context + if lualibs[1] then + local firstlib = file.join(luapath,lualibs[1]) + if not lfs.isfile(firstlib) then + local foundname = input.find_files(instance,lualibs[1])[1] + if foundname then + input.report("located library path : " .. luapath) + luapath = file.dirname(foundname) + end + end + end + input.report("using library path : " .. luapath) + input.report("using lua libraries: " .. table.join(lualibs," ")) + utils.merger.selfcreate(lualibs,luapath,luaname) + if utils.lua.compile(luaname, lucname) and io.exists(lucname) then + luaname = lucname + input.report("using compiled initialization file " .. lucname) + else + input.report("using uncompiled initialization file " .. luaname) + end + else + for _, v in pairs({instance.luaname, instance.progname, barename}) do + v = string.gsub(v..".lua","%.lua%.lua$",".lua") + if v and (v ~= "") then + luaname = input.find_files(instance,v)[1] or "" + if luaname ~= "" then + break + end + end + end + end + if luaname == "" then + input.reportlines(messages.no_ini_file) + input.report("texname : " .. texname) + input.report("luaname : " .. instance.luaname) + input.report("progname : " .. instance.progname) + input.report("barename : " .. barename) + else + input.report("using lua initialization file " .. luaname) + local flags = { "--ini" } + if environment.arguments["mkii"] then + flags[#flags+1] = "--progname=" .. instance.progname + else + flags[#flags+1] = "--lua=" .. string.quote(luaname) + end + local bs = (environment.platform == "unix" and "\\\\") or "\\" -- todo: make a function + local command = "luatex ".. table.concat(flags," ") .. " " .. string.quote(fullname) .. " " .. bs .. "dump" + input.report("running command: " .. command .. "\n") + os.spawn(command) + end + end + else + input.report("no tex file given") + end +end + +function input.my_run_format(instance,name,data,more) + -- hm, rather old code here; we can now use the file.whatever functions + if name and (name ~= "") then + local barename = name:gsub("%.%a+$","") + local fmtname = "" + if input.usecache then + local path = file.join(caches.setpath(instance,"formats")) -- maybe platform + fmtname = file.join(path,barename..".fmt") or "" + end + if fmtname == "" then + fmtname = input.find_files(instance,barename..".fmt")[1] or "" + end + fmtname = input.clean_path(fmtname) + barename = fmtname:gsub("%.%a+$","") + if fmtname == "" then + input.report("no format with name",name) + else + local luaname = barename .. ".luc" + local f = io.open(luaname) + if not f then + luaname = barename .. ".lua" + f = io.open(luaname) + end + if f then + f:close() + local command = "luatex --fmt=" .. string.quote(barename) .. " --lua=" .. string.quote(luaname) .. " " .. string.quote(data) .. " " .. string.quote(more) + input.report("running command: " .. command) + os.spawn(command) + else + input.report("using format name",fmtname) + input.report("no luc/lua with name",barename) + end + end + end +end + +-- helpers for verbose lists + +input.listers = input.listers or { } + +local function tabstr(str) + if type(str) == 'table' then + return table.concat(str," | ") + else + return str + end +end + +local function list(instance,list) + local pat = string.upper(instance.pattern or "","") + for _,key in pairs(table.sortedkeys(list)) do + if instance.pattern == "" or string.find(key:upper(),pat) then + if instance.kpseonly then + if instance.kpsevars[key] then + print(format("%s=%s",key,tabstr(list[key]))) + end + else + print(format('%s %s=%s',(instance.kpsevars[key] and 'K') or 'E',key,tabstr(list[key]))) + end + end + end +end + +function input.listers.variables (instance) list(instance,instance.variables ) end +function input.listers.expansions(instance) list(instance,instance.expansions) end + +function input.listers.configurations(instance) + for _,key in pairs(table.sortedkeys(instance.kpsevars)) do + if not instance.pattern or (instance.pattern=="") or key:find(instance.pattern) then + print(key.."\n") + for i,c in ipairs(instance.order) do + local str = c[key] + if str then + print(format("\t%s\t\t%s",i,input.aux.tabstr(str))) + end + end + print() + end + end +end + +input.report(banner,"\n") + +local ok = true + +if environment.arguments["find-file"] then + input.my_prepare_b(instance) + instance.format = environment.arguments["format"] or instance.format + if instance.pattern then + instance.allresults = true + input.for_files(instance, input.find_files, { instance.pattern }, instance.my_format) + else + input.for_files(instance, input.find_files, environment.files, instance.my_format) + end +elseif environment.arguments["find-path"] then + input.my_prepare_b(instance) + local path = input.find_file(instance, environment.files[1], instance.my_format) + if input.verbose then + input.report(file.dirname(path)) + else + print(file.dirname(path)) + end +--~ elseif environment.arguments["first-writable-path"] then +--~ input.my_prepare_b(instance) +--~ input.report(input.first_writable_path(instance,environment.files[1] or ".")) +elseif environment.arguments["run"] then + input.my_prepare_a(instance) -- ! no need for loading databases + input.verbose = true + input.my_run_format(instance,environment.files[1] or "",environment.files[2] or "",environment.files[3] or "") +elseif environment.arguments["fmt"] then + input.my_prepare_a(instance) -- ! no need for loading databases + input.verbose = true + input.my_run_format(instance,environment.arguments["fmt"], environment.files[1] or "",environment.files[2] or "") +elseif environment.arguments["expand-braces"] then + input.my_prepare_a(instance) + input.for_files(instance, input.expand_braces, environment.files) +elseif environment.arguments["expand-path"] then + input.my_prepare_a(instance) + input.for_files(instance, input.expand_path, environment.files) +elseif environment.arguments["expand-var"] or environment.arguments["expand-variable"] then + input.my_prepare_a(instance) + input.for_files(instance, input.expand_var, environment.files) +elseif environment.arguments["show-path"] or environment.arguments["path-value"] then + input.my_prepare_a(instance) + input.for_files(instance, input.show_path, environment.files) +elseif environment.arguments["var-value"] or environment.arguments["show-value"] then + input.my_prepare_a(instance) + input.for_files(instance, input.var_value, environment.files) +elseif environment.arguments["format-path"] then + input.my_prepare_b(instance) + input.report(caches.setpath(instance,"format")) +elseif instance.pattern then -- brrr + input.my_prepare_b(instance) + instance.format = environment.arguments["format"] or instance.format + instance.allresults = true + input.for_files(instance, input.find_files, { instance.pattern }, instance.my_format) +elseif environment.arguments["generate"] then + instance.renewcache = true + input.verbose = true + input.my_prepare_b(instance) +elseif environment.arguments["make"] or environment.arguments["ini"] or environment.arguments["compile"] then + input.my_prepare_b(instance) + input.verbose = true + input.my_make_format(instance,environment.files[1] or "") +elseif environment.arguments["selfmerge"] then + utils.merger.selfmerge(own.name,own.libs,own.list) +elseif environment.arguments["selfclean"] then + utils.merger.selfclean(own.name) +elseif environment.arguments["selfupdate"] then + input.my_prepare_b(instance) + input.verbose = true + input.update_script(instance,own.name,"luatools") +elseif environment.arguments["variables"] or environment.arguments["show-variables"] then + input.my_prepare_a(instance) + input.listers.variables(instance) +elseif environment.arguments["expansions"] or environment.arguments["show-expansions"] then + input.my_prepare_a(instance) + input.listers.expansions(instance) +elseif environment.arguments["configurations"] or environment.arguments["show-configurations"] then + input.my_prepare_a(instance) + input.listers.configurations(instance) +elseif environment.arguments["help"] or (environment.files[1]=='help') or (#environment.files==0) then + if not input.verbose then + input.verbose = true + input.report(banner,"\n") + end + input.reportlines(messages.help) +else + input.my_prepare_b(instance) + input.for_files(instance, input.find_files, environment.files, instance.my_format) +end + +if input.verbose then + input.report("") + input.report(string.format("runtime: %0.3f seconds",os.runtime())) +end + +--~ if ok then +--~ input.report("exit code: 0") os.exit(0) +--~ else +--~ input.report("exit code: 1") os.exit(1) +--~ end + +if environment.platform == "unix" then + io.write("\n") +end diff --git a/Master/texmf-dist/scripts/context/lua/luatools.rme b/Master/texmf-dist/scripts/context/lua/luatools.rme new file mode 100644 index 00000000000..b320e1184b5 --- /dev/null +++ b/Master/texmf-dist/scripts/context/lua/luatools.rme @@ -0,0 +1,3 @@ +On MSWindows the luatools.lua script is called +with luatools.cmd. On Unix you can either rename +luatools.lua to luatools, or use a symlink. diff --git a/Master/texmf-dist/scripts/context/lua/mtx-babel.lua b/Master/texmf-dist/scripts/context/lua/mtx-babel.lua new file mode 100644 index 00000000000..c9855b86a40 --- /dev/null +++ b/Master/texmf-dist/scripts/context/lua/mtx-babel.lua @@ -0,0 +1,430 @@ +if not modules then modules = { } end modules ['mtx-babel'] = { + version = 1.001, + comment = "companion to mtxrun.lua", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- data tables by Thomas A. Schmitz + +texmf.instance = instance -- we need to get rid of this / maybe current instance in global table + +scripts = scripts or { } +scripts.babel = scripts.babel or { } + +do + + local converters = { } + + -- greek + + local replace_01 = { -- <' * | + a = "ᾅ", + h = "ᾕ", + w = "ᾥ", + } + + local replace_02 = { -- >' * | + a = "ᾄ", + h = "ᾔ", + w = "ᾤ", + } + + local replace_03 = { -- <` * | + a = "ᾃ", + h = "ᾓ", + w = "ᾣ", + } + + local replace_04 = { -- >` * | + a = "ᾂ", + h = "ᾒ", + w = "ᾢ", + } + + local replace_05 = { -- <~ * | + a = "ᾇ", + h = "ᾗ", + w = "ᾧ", + } + + local replace_06 = { -- >~ * | + a = "ᾆ", + h = "ᾖ", + w = "ᾦ" + } + + local replace_07 = { -- "' * + i = "ΐ", + u = "ΰ", + } + + local replace_08 = { -- "` * + i = "ῒ", + u = "ῢ", + } + + local replace_09 = { -- "~ * + i = "ῗ", + u = "ῧ", + } + + local replace_10 = { -- <' * + a = "ἅ", + e = "ἕ", + h = "ἥ", + i = "ἵ", + o = "ὅ", + u = "ὕ", + w = "ὥ", + A = "Ἅ", + E = "Ἕ", + H = "Ἥ", + I = "Ἵ", + O = "Ὅ", + U = "Ὕ", + W = "Ὥ", + } + + local replace_11 = { -- >' * + a = "ἄ", + e = "ἔ", + h = "ἤ", + i = "ἴ", + o = "ὄ", + u = "ὔ", + w = "ὤ", + A = "Ἄ", + E = "Ἔ", + H = "Ἤ", + I = "Ἴ", + O = "Ὄ", + U = "῎Υ", + W = "Ὤ", + } + + local replace_12 = { -- <` * + a = "ἃ", + e = "ἓ", + h = "ἣ", + i = "ἳ", + o = "ὃ", + u = "ὓ", + w = "ὣ", + A = "Ἃ", + E = "Ἒ", + H = "Ἣ", + I = "Ἳ", + O = "Ὃ", + U = "Ὓ", + W = "Ὣ", + } + + local replace_13 = { -- >` * + a = "ἂ", + e = "ἒ", + h = "ἢ", + i = "ἲ", + o = "ὂ", + u = "ὒ", + w = "ὢ", + A = "Ἂ", + E = "Ἒ", + H = "Ἢ", + I = "Ἲ", + O = "Ὂ", + U = "῍Υ", + W = "Ὢ", + } + + local replace_14 = { -- <~ * + a = "ἇ", + h = "ἧ", + i = "ἷ", + u = "ὗ", + w = "ὧ", + A = "Ἇ", + H = "Ἧ", + I = "Ἷ", + U = "Ὗ", + W = "Ὧ", + } + + local replace_15 = { -- >~ * + a = "ἆ", + h = "ἦ", + i = "ἶ", + u = "ὖ", + w = "ὦ", + A = "Ἆ", + H = "Ἦ", + I = "Ἶ", + U = "῏Υ", + W = "Ὦ", + } + + local replace_16 = { -- ' * | + a = "ᾴ", + h = "ῄ", + w = "ῴ", + } + + local replace_17 = { -- ` * | + a = "ᾲ", + h = "ῂ", + w = "ῲ", + } + + local replace_18 = { -- ~ * | + a = "ᾷ", + h = "ῇ", + w = "ῷ" + } + + local replace_19 = { -- ' * + a = "ά", + e = "έ", + h = "ή", + i = "ί", + o = "ό", + u = "ύ", + w = "ώ", + } + + local replace_20 = { -- ` * + a = "ὰ", + e = "ὲ", + h = "ὴ", + i = "ὶ", + o = "ὸ", + u = "ὺ", + w = "ὼ", + } + + local replace_21 = { -- ~ * + a = "ᾶ", + h = "ῆ", + i = "ῖ", + u = "ῦ", + w = "ῶ", + } + + local replace_22 = { -- < * + a = "ἁ", + e = "ἑ", + h = "ἡ", + i = "ἱ", + o = "ὁ", + u = "ὑ", + w = "ὡ", + r = "ῥ", + A = "Ἁ", + E = "Ἑ", + H = "Ἡ", + I = "Ἱ", + O = "Ὁ", + U = "Ὑ", + W = "Ὡ", + R = "Ῥ", + } + + local replace_23 = { -- > * + a = "ἀ", + e = "ἐ", + h = "ἠ", + i = "ἰ", + o = "ὀ", + u = "ὐ", + w = "ὠ", + A = "Ἀ", + E = "Ἐ", + H = "Ἠ", + I = "Ἰ", + O = "Ὀ", + U = "᾿Υ", + W = "Ὠ", + } + + local replace_24 = { -- * | + a = "ᾳ", + h = "ῃ", + w = "ῳ", + } + + local replace_25 = { -- " * + i = "ϊ", + u = "ϋ", + } + + local replace_26 = { -- * + a = "α", + b = "β", + g = "γ", + d = "δ", + e = "ε", + z = "ζ", + h = "η", + j = "θ", + i = "ι", + k = "κ", + l = "λ", + m = "μ", + n = "ν", + x = "ξ", + o = "ο", + p = "π", + r = "ρ", + s = "σ", + c = "ς", + t = "τ", + u = "υ", + f = "φ", + q = "χ", + y = "ψ", + w = "ω", + A = "Α", + B = "Β", + G = "Γ", + D = "Δ", + E = "Ε", + Z = "Ζ", + H = "Η", + J = "Θ", + I = "Ι", + K = "Κ", + L = "Λ", + M = "Μ", + N = "Ν", + X = "Ξ", + O = "Ο", + P = "Π", + R = "Ρ", + S = "Σ", + T = "Τ", + U = "Υ", + F = "Φ", + Q = "Χ", + Y = "Ψ", + W = "Ω" + } + + local skips_01 = lpeg.P("\\") * lpeg.R("az", "AZ")^1 + local skips_02 = lpeg.P("[") * (1- lpeg.S("[]"))^1 * lpeg.P("]") + + local greek_01 = (lpeg.P("<'") * lpeg.Cs(1) * lpeg.P('|')) / replace_01 + local greek_02 = (lpeg.P(">'") * lpeg.Cs(1) * lpeg.P('|')) / replace_02 + local greek_03 = (lpeg.P("<`") * lpeg.Cs(1) * lpeg.P('|')) / replace_03 + local greek_04 = (lpeg.P(">`") * lpeg.Cs(1) * lpeg.P('|')) / replace_04 + local greek_05 = (lpeg.P("<~") * lpeg.Cs(1) * lpeg.P('|')) / replace_05 + local greek_06 = (lpeg.P(">~") * lpeg.Cs(1) * lpeg.P('|')) / replace_06 + local greek_07 = (lpeg.P('"\'') * lpeg.Cs(1) ) / replace_07 + local greek_08 = (lpeg.P('"`') * lpeg.Cs(1) ) / replace_08 + local greek_09 = (lpeg.P('"~') * lpeg.Cs(1) ) / replace_09 + local greek_10 = (lpeg.P("<'") * lpeg.Cs(1) ) / replace_10 + local greek_11 = (lpeg.P(">'") * lpeg.Cs(1) ) / replace_11 + local greek_12 = (lpeg.P("<`") * lpeg.Cs(1) ) / replace_12 + local greek_13 = (lpeg.P(">`") * lpeg.Cs(1) ) / replace_13 + local greek_14 = (lpeg.P("<~") * lpeg.Cs(1) ) / replace_14 + local greek_15 = (lpeg.P(">~") * lpeg.Cs(1) ) / replace_15 + local greek_16 = (lpeg.P("'") * lpeg.Cs(1) * lpeg.P('|')) / replace_16 + local greek_17 = (lpeg.P("`") * lpeg.Cs(1) * lpeg.P('|')) / replace_17 + local greek_18 = (lpeg.P("~") * lpeg.Cs(1) * lpeg.P('|')) / replace_18 + local greek_19 = (lpeg.P("'") * lpeg.Cs(1) ) / replace_19 + local greek_20 = (lpeg.P("`") * lpeg.Cs(1) ) / replace_20 + local greek_21 = (lpeg.P("~") * lpeg.Cs(1) ) / replace_21 + local greek_22 = (lpeg.P("<") * lpeg.Cs(1) ) / replace_22 + local greek_23 = (lpeg.P(">") * lpeg.Cs(1) ) / replace_23 + local greek_24 = (lpeg.Cs(1) * lpeg.P('|') ) / replace_24 + local greek_25 = (lpeg.P('"') * lpeg.Cs(1) ) / replace_25 + local greek_26 = (lpeg.Cs(1) ) / replace_26 + + local skips = + skips_01 + skips_02 + + local greek = + greek_01 + greek_02 + greek_03 + greek_04 + greek_05 + + greek_06 + greek_07 + greek_08 + greek_09 + greek_10 + + greek_11 + greek_12 + greek_13 + greek_14 + greek_15 + + greek_16 + greek_17 + greek_18 + greek_19 + greek_20 + + greek_21 + greek_22 + greek_23 + greek_24 + greek_25 + + greek_26 + + local spacing = lpeg.S(" \n\r\t") + local startgreek = lpeg.P("\\startgreek") + local stopgreek = lpeg.P("\\stopgreek") + local localgreek = lpeg.P("\\localgreek") + local lbrace = lpeg.P("{") + local rbrace = lpeg.P("}") + + local documentparser = lpeg.Cs((skips + greek + 1)^0) + + local contextgrammar = lpeg.Cs ( lpeg.P { "scan", + ["scan"] = (lpeg.V("global") + lpeg.V("local") + skips + 1)^0, + ["global"] = startgreek * ((skips + greek + 1)-stopgreek )^0 , + ["local"] = localgreek * lpeg.V("grouped"), + ["grouped"] = spacing^0 * lbrace * (lpeg.V("grouped") + skips + (greek - rbrace))^0 * rbrace, + } ) + + converters['greek'] = { + document = documentparser, + context = contextgrammar, + } + + -- lpeg.print(parser): 254 lines + + function scripts.babel.convert(filename) + if filename and filename ~= empty then + local data = io.loaddata(filename) or "" + if data ~= "" then + local language = environment.argument("language") or "" + if language ~= "" then + local converter = converters[language] + if converter then + local structure = environment.argument("structure") or "document" + converter = converter[structure] + if converter then + input.report(string.format("converting '%s' using language '%s' with structure '%s'", filename, language, structure)) + data = converter:match(data) + local newfilename = filename .. ".utf" + io.savedata(newfilename, data) + input.report(string.format("converted data saved in '%s'", newfilename)) + else + input.report(string.format("unknown structure '%s' language '%s'", structure, language)) + end + else + input.report(string.format("no converter for language '%s'", language)) + end + else + input.report(string.format("provide language")) + end + else + input.report(string.format("no data in '%s'",filename)) + end + end + end + + --~ print(contextgrammar:match [[ + --~ oeps abg \localgreek{a} + --~ \startgreek abg \stopgreek \oeps + --~ oeps abg \localgreek{a{b}\oeps g} + --~ ]]) + +end + +banner = banner .. " | babel conversion tools " + +messages.help = [[ +--language=string conversion language (e.g. greek) +--structure=string obey given structure (e.g. 'document', default: 'context') +--convert convert babel codes into utf +]] + +input.verbose = true + +if environment.argument("convert") then + scripts.babel.convert(environment.files[1] or "") +else + input.help(banner,messages.help) +end + diff --git a/Master/texmf-dist/scripts/context/lua/mtx-cache.lua b/Master/texmf-dist/scripts/context/lua/mtx-cache.lua new file mode 100644 index 00000000000..8091eaa6515 --- /dev/null +++ b/Master/texmf-dist/scripts/context/lua/mtx-cache.lua @@ -0,0 +1,99 @@ +if not modules then modules = { } end modules ['mtx-cache'] = { + version = 1.001, + comment = "companion to mtxrun.lua", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +texmf.instance = instance -- we need to get rid of this / maybe current instance in global table + +scripts = scripts or { } +scripts.cache = scripts.cache or { } + +function scripts.cache.collect_one(...) + local path = caches.setpath(instance,...) + local tmas = dir.glob(path .. "/*.tma") + local tmcs = dir.glob(path .. "/*.tmc") + return path, tmas, tmcs +end + +function scripts.cache.collect_two(...) + local path = caches.setpath(instance,...) + local rest = dir.glob(path .. "/**/*") + return path, rest +end + +function scripts.cache.process_one(action) + for k, v in ipairs({ "afm", "tfm", "def", "enc", "otf", "mp", "data" }) do + action("fonts", v) + end +end + +function scripts.cache.process_two(action) + action("curl") +end + +-- todo: recursive delete of paths + +function scripts.cache.remove(list,keep) + local keepsuffixes = { } + for _, v in ipairs(keep or {}) do + keepsuffixes[v] = true + end + local n = 0 + for _,filename in ipairs(list) do + if filename:find("luatex%-cache") then -- safeguard + if not keepsuffixes[file.extname(filename) or ""] then + os.remove(filename) + n = n + 1 + end + end + end + return n +end + +function scripts.cache.delete(all,keep) + scripts.cache.process_one(function(...) + local path, rest = scripts.cache.collect_one(...) + local n = scripts.cache.remove(rest,keep) + logs.report("cache path",string.format("%4i files out of %4i deleted on %s",n,#rest,path)) + end) + scripts.cache.process_two(function(...) + local path, rest = scripts.cache.collect_two(...) + local n = scripts.cache.remove(rest,keep) + logs.report("cache path",string.format("%4i files out of %4i deleted on %s",n,#rest,path)) + end) +end + +function scripts.cache.list(all) + scripts.cache.process_one(function(...) + local path, tmas, tmcs = scripts.cache.collect_one(...) + logs.report("cache path",string.format("%4i (tma:%4i, tmc:%4i) %s",#tmas+#tmcs,#tmas,#tmcs,path)) + logs.report("cache path",string.format("%4i (tma:%4i, tmc:%4i) %s",#tmas+#tmcs,#tmas,#tmcs,path)) + end) + scripts.cache.process_two(function(...) + local path, rest = scripts.cache.collect_two("curl") + logs.report("cache path",string.format("%4i %s",#rest,path)) + end) +end + +banner = banner .. " | cache tools " + +messages.help = [[ +--purge remove not used files +--erase completely remove cache +--list show cache + +--all all (not yet implemented) +]] + +if environment.argument("purge") then + scripts.cache.delete(environment.argument("all"),{"tmc"}) +elseif environment.argument("erase") then + scripts.cache.delete(environment.argument("all")) +elseif environment.argument("list") then + scripts.cache.list(environment.argument("all")) +else + input.help(banner,messages.help) +end diff --git a/Master/texmf-dist/scripts/context/lua/mtx-chars.lua b/Master/texmf-dist/scripts/context/lua/mtx-chars.lua new file mode 100644 index 00000000000..a05ab9f0860 --- /dev/null +++ b/Master/texmf-dist/scripts/context/lua/mtx-chars.lua @@ -0,0 +1,182 @@ +if not modules then modules = { } end modules ['mtx-chars'] = { + version = 1.001, + comment = "companion to mtxrun.lua", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +texmf.instance = instance -- we need to get rid of this / maybe current instance in global table + +scripts = scripts or { } +scripts.chars = scripts.chars or { } + +function scripts.chars.stixtomkiv(inname,outname) + if inname == "" then + logs.report("aquiring math data","invalid datafilename") + end + local f = io.open(inname) + if not f then + logs.report("aquiring math data","invalid datafile") + else + logs.report("aquiring math data","processing " .. inname) + if not outname or outname == "" then + outname = "char-mth.lua" + end + local classes = { + N = "normal", + A = "alphabetic", + D = "diacritic", + P = "punctuation", + B = "binary", + R = "relation", + L = "large", + O = "opening", + C = "closing", + F = "fence" + } + local format, concat = string.format, table.concat + local valid, done = false, { } + local g = io.open(outname,'w') + g:write([[ +-- filename : char-mth.lua +-- comment : companion to char-mth.tex (in ConTeXt) +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- license : see context related readme files +-- comment : generated from data file downloaded from STIX website + +if not versions then versions = { } end versions['char-mth'] = 1.001 +if not characters then characters = { } end + ]]) + g:write(format("\ncharacters.math = {\n")) + for l in f:lines() do + if not valid then + valid = l:find("AMS/TeX name") + end + if valid then + local unicode = l:sub(2,6) + if unicode:sub(1,1) ~= " " and unicode ~= "" and not done[unicode] then + local mathclass, adobename, texname = l:sub(57,57) or "", l:sub(13,36) or "", l:sub(84,109) or "" + texname, adobename = texname:gsub("[\\ ]",""), adobename:gsub("[\\ ]","") + local t = { } + if mathclass ~= "" then t[#t+1] = format("mathclass='%s'", classes[mathclass] or "unknown") end + if adobename ~= "" then t[#t+1] = format("adobename='%s'", adobename ) end + if texname ~= "" then t[#t+1] = format("texname='%s'" , texname ) end + if #t > 0 then + g:write(format("\t[0x%s] = { %s },\n",unicode, concat(t,", "))) + end + done[unicode] = true + end + end + end + if not valid then + g:write("\t-- The data file is corrupt, invalid or maybe the format has changed.\n") + logs.report("aquiring math data","problems with data table") + else + logs.report("aquiring math data","table saved in " .. outname) + end + g:write("}\n") + g:close() + f:close() + end +end + +scripts.chars.banner_utf_1 = [[ +% filename : enco-utf.tex +% comment : generated by mtxrun --script chars --utf +% author : Hans Hagen, PRAGMA-ADE, Hasselt NL +% copyright: PRAGMA ADE / ConTeXt Development Team +% license : see context related readme files + +\ifx\setcclcucx\undefined + + \def\setcclcucx #1 #2 #3 % + {\global\catcode"#1=11 + \global\lccode "#1="#2 + \global\uccode "#1="#3 } + +\fi +]] + +scripts.chars.banner_utf_2 = [[ + +% lc/uc/catcode mappings + +]] + +scripts.chars.banner_utf_3 = [[ + +% named characters mapped onto utf (\\char is needed for accents) + +]] + +scripts.chars.banner_utf_4 = [[ + +\endinput +]] + +function scripts.chars.makeencoutf() + local chartable = input.find_file(instance,"char-def.lua") or "" + if chartable ~= "" then + dofile(chartable) + if characters and characters.data then + local f = io.open("enco-utf.tex", 'w') + if f then + local char, format = unicode.utf8.char, string.format + f:write(scripts.chars.banner_utf_1) + f:write(scripts.chars.banner_utf_2) + local list = table.sortedkeys(characters.data) + local length = 0 + for i=1,#list do + local code = list[i] + if code <= 0xFFFF then + local chr = characters.data[code] + local cc = chr.category + if cc == 'll' or cc == 'lu' or cc == 'lt' then + if not chr.lccode then chr.lccode = code end + if not chr.uccode then chr.uccode = code end + f:write(format("\\setcclcucx %04X %04X %04X %% %s\n",code,chr.lccode,chr.uccode,chr.description)) + end + if #(chr.contextname or "") > length then + length = #chr.contextname + end + end + end + f:write(scripts.chars.banner_utf_3) + for i=1,#list do + local code = list[i] + if code > 0x5B and code <= 0xFFFF then + local chr = characters.data[code] + if chr.contextname then + local ch = char(code) +--~ if ch:find("[~#$%%^&{}\\]") then + f:write(format("\\def\\%s{\\char\"%04X } %% %s: %s\n", chr.contextname:rpadd(length," "), code, chr.description, ch)) +--~ else +--~ f:write(format("\\def\\%s{%s} %% %s\n", chr.contextname:rpadd(length," "), ch,chr.description)) +--~ end + end + end + end + f:write(scripts.chars.banner_utf_4) + f:close() + end + end + end +end + +banner = banner .. " | character tools " + +messages.help = [[ +--stix convert stix table to math table +--utf generate enco-utf.tex (used by xetex) +]] + +if environment.argument("stix") then + local inname = environment.files[1] or "" + local outname = environment.files[2] or "" + scripts.chars.stixtomkiv(inname,outname) +elseif environment.argument("utf") then + scripts.chars.makeencoutf() +else + input.help(banner,messages.help) +end diff --git a/Master/texmf-dist/scripts/context/lua/mtx-check.lua b/Master/texmf-dist/scripts/context/lua/mtx-check.lua new file mode 100644 index 00000000000..dd8a71264ac --- /dev/null +++ b/Master/texmf-dist/scripts/context/lua/mtx-check.lua @@ -0,0 +1,138 @@ +if not modules then modules = { } end modules ['mtx-check'] = { + version = 1.001, + comment = "companion to mtxrun.lua", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +texmf.instance = instance -- we need to get rid of this / maybe current instance in global table + +scripts = scripts or { } +scripts.checker = scripts.checker or { } + +local validator = { } + +do + + validator.n = 1 + validator.errors = { } + validator.trace = false + validator.direct = false + + validator.printer = print + validator.tracer = print + + local message = function(position, kind) + local ve = validator.errors + ve[#ve+1] = { kind, position, validator.n } + if validator.direct then + validator.printer(string.format("%s error at position %s (line %s)", kind, position, validator.n)) + end + end + local progress = function(position, data, kind) + if validator.trace then + validator.tracer(string.format("%s at position %s: %s", kind, position, data or "")) + end + end + + local P, S, V, C, CP, CC = lpeg.P, lpeg.S, lpeg.V, lpeg.C, lpeg.Cp, lpeg.Cc + + local i_m, d_m = P("$"), P("$$") + local l_s, r_s = P("["), P("]") + local l_g, r_g = P("{"), P("}") + + local esc = P("\\") + local cr = P("\r") + local lf = P("\n") + local crlf = P("\r\n") + local space = S(" \t\f\v") + local newline = crlf + cr + lf + + local line = newline / function() validator.n = validator.n + 1 end + + local grammar = P { "tokens", + ["tokens"] = (V("whatever") + V("grouped") + V("setup") + V("display") + V("inline") + V("errors") + 1)^0, + ["whatever"] = line + esc * 1 + C(P("%") * (1-line)^0), + ["grouped"] = CP() * C(l_g * (V("whatever") + V("grouped") + V("setup") + V("display") + V("inline") + (1 - l_g - r_g))^0 * r_g) * CC("group") / progress, + ["setup"] = CP() * C(l_s * (V("whatever") + V("grouped") + V("setup") + V("display") + V("inline") + (1 - l_s - r_s))^0 * r_s) * CC("setup") / progress, + ["display"] = CP() * C(d_m * (V("whatever") + V("grouped") + (1 - d_m))^0 * d_m) * CC("display") / progress, + ["inline"] = CP() * C(i_m * (V("whatever") + V("grouped") + (1 - i_m))^0 * i_m) * CC("inline") / progress, + ["errors"] = (V("gerror") + V("serror") + V("derror") + V("ierror")) * true, + ["gerror"] = CP() * (l_g + r_g) * CC("grouping") / message, + ["serror"] = CP() * (l_s + r_g) * CC("setup error") / message, + ["derror"] = CP() * d_m * CC("display math error") / message, + ["ierror"] = CP() * i_m * CC("inline math error") / message, + } + + local grammar = P { "tokens", + ["tokens"] = (V("whatever") + V("grouped") + V("setup") + V("display") + V("inline") + V("errors") + 1)^0, + ["whatever"] = line + esc * 1 + C(P("%") * (1-line)^0), + ["grouped"] = l_g * (V("whatever") + V("grouped") + V("setup") + V("display") + V("inline") + (1 - l_g - r_g))^0 * r_g, + ["setup"] = l_s * (V("whatever") + V("grouped") + V("setup") + V("display") + V("inline") + (1 - l_s - r_s))^0 * r_s, + ["display"] = d_m * (V("whatever") + V("grouped") + (1 - d_m))^0 * d_m, + ["inline"] = i_m * (V("whatever") + V("grouped") + (1 - i_m))^0 * i_m, + ["errors"] = (V("gerror")+ V("serror") + V("derror") + V("ierror")), + ["gerror"] = CP() * (l_g + r_g) * CC("grouping") / message, + ["serror"] = CP() * (l_s + r_g) * CC("setup error") / message, + ["derror"] = CP() * d_m * CC("display math error") / message, + ["ierror"] = CP() * i_m * CC("inline math error") / message, + } + + function validator.check(str) + validator.n = 1 + validator.errors = { } + grammar:match(str) + end + +end + +--~ str = [[ +--~ a{oeps {oe\{\}ps} } +--~ test { oeps \} \[\] oeps \setupxxx[oeps=bla]} +--~ test $$ \hbox{$ oeps \} \[\] oeps $} $$ +--~ {$x\$xx$ $ +--~ ]] +--~ str = string.rep(str,10) + +function scripts.checker.check(filename) + local str = io.loaddata(filename) + if str then + validator.check(str) + if #validator.errors > 0 then + for k, v in ipairs(validator.errors) do + local kind, position, line = v[1], v[2], v[3] + local data = str:sub(position-30,position+30) + data = data:gsub("(.)", { + ["\n"] = " <lf> ", + ["\r"] = " <cr> ", + ["\t"] = " <tab> ", + }) + data = data:gsub("^ *","") + print(string.format("% 5i %s %s", line,string.rpadd(kind,10," "),data)) + end + else + print("no error") + end + else + print("no file") + end +end + + +banner = banner .. " | tex check tools " + +messages.help = [[ +--convert check tex file for errors +]] + +input.verbose = true + +if environment.argument("check") then + scripts.checker.check(environment.files[1]) +elseif environment.argument("help") then + input.help(banner,messages.help) +elseif environment.files[1] then + scripts.checker.check(environment.files[1]) +end + diff --git a/Master/texmf-dist/scripts/context/lua/mtx-context.lua b/Master/texmf-dist/scripts/context/lua/mtx-context.lua new file mode 100644 index 00000000000..6c444d53162 --- /dev/null +++ b/Master/texmf-dist/scripts/context/lua/mtx-context.lua @@ -0,0 +1,721 @@ +if not modules then modules = { } end modules ['mtx-context'] = { + version = 1.001, + comment = "companion to mtxrun.lua", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +texmf.instance = instance -- we need to get rid of this / maybe current instance in global table + +scripts = scripts or { } +scripts.context = scripts.context or { } + +-- l-file / todo + +function file.needsupdate(oldfile,newfile) + return true +end +function file.syncmtimes(oldfile,newfile) +end + +-- l-io + +function io.copydata(fromfile,tofile) + io.savedata(tofile,io.loaddata(fromfile) or "") +end + +-- luat-inp + +function input.locate_format(name) -- move this to core / luat-xxx + local barename, fmtname = name:gsub("%.%a+$",""), "" + if input.usecache then + local path = file.join(caches.setpath(instance,"formats")) -- maybe platform + fmtname = file.join(path,barename..".fmt") or "" + end + if fmtname == "" then + fmtname = input.find_files(instance,barename..".fmt")[1] or "" + end + fmtname = input.clean_path(fmtname) + if fmtname ~= "" then + barename = fmtname:gsub("%.%a+$","") + local luaname, lucname = barename .. ".lua", barename .. ".luc" + if lfs.isfile(lucname) then + return barename, luaname + elseif lfs.isfile(luaname) then + return barename, luaname + end + end + return nil, nil +end + +-- ctx + +ctxrunner = { } + +do + + function ctxrunner.filtered(str,method) + str = tostring(str) + if method == 'name' then str = file.removesuffix(file.basename(str)) + elseif method == 'path' then str = file.dirname(str) + elseif method == 'suffix' then str = file.extname(str) + elseif method == 'nosuffix' then str = file.removesuffix(str) + elseif method == 'nopath' then str = file.basename(str) + elseif method == 'base' then str = file.basename(str) + -- elseif method == 'full' then + -- elseif method == 'complete' then + -- elseif method == 'expand' then -- str = file.expand_path(str) + end + return str:gsub("\\","/") + end + + function ctxrunner.substitute(e,str) + local attributes = e.at + if str and attributes then + if attributes['method'] then + str = ctxrunner.filtered(str,attributes['method']) + end + if str == "" and attributes['default'] then + str = attributes['default'] + end + end + return str + end + + function ctxrunner.reflag(flags) + local t = { } + for _, flag in pairs(flags) do + local key, value = flag:match("^(.-)=(.+)$") + if key and value then + t[key] = value + else + t[flag] = true + end + end + return t + end + + function ctxrunner.substitute(str) + return str + end + + function ctxrunner.justtext(str) + str = xml.unescaped(tostring(str)) + str = xml.cleansed(str) + str = str:gsub("\\+",'/') + str = str:gsub("%s+",' ') + return str + end + + function ctxrunner.new() + return { + ctxname = "", + jobname = "", + xmldata = nil, + suffix = "prep", + locations = { '..', '../..' }, + variables = { }, + messages = { }, + environments = { }, + modules = { }, + filters = { }, + flags = { }, + modes = { }, + prepfiles = { }, + paths = { }, + } + end + + function ctxrunner.savelog(ctxdata,ctlname) + local function yn(b) + if b then return 'yes' else return 'no' end + end + if not ctlname or ctlname == "" or ctlname == ctxdata.jobname then + if ctxdata.jobname then + ctlname = file.replacesuffix(ctxdata.jobname,'ctl') + elseif ctxdata.ctxname then + ctlname = file.replacesuffix(ctxdata.ctxname,'ctl') + else + input.report(string.format("invalid ctl name %s",ctlname or "?")) + return + end + end + if table.is_empty(ctxdata.prepfiles) then + input.report("nothing prepared, no ctl file saved") + os.remove(ctlname) + else + input.report(string.format("saving logdata in %s",ctlname)) + f = io.open(ctlname,'w') + if f then + f:write("<?xml version='1.0' standalone='yes'?>\n\n") + f:write(string.format("<ctx:preplist local='%s'>\n",yn(ctxdata.runlocal))) +--~ for name, value in pairs(ctxdata.prepfiles) do + for _, name in ipairs(table.sortedkeys(ctxdata.prepfiles)) do + f:write(string.format("\t<ctx:prepfile done='%s'>%s</ctx:prepfile>\n",yn(ctxdata.prepfiles[name]),name)) + end + f:write("</ctx:preplist>\n") + f:close() + end + end + end + + function ctxrunner.register_path(ctxdata,path) + -- test if exists + ctxdata.paths[ctxdata.paths+1] = path + end + + function ctxrunner.trace(ctxdata) + print(table.serialize(ctxdata.messages)) + print(table.serialize(ctxdata.flags)) + print(table.serialize(ctxdata.environments)) + print(table.serialize(ctxdata.modules)) + print(table.serialize(ctxdata.filters)) + print(table.serialize(ctxdata.modes)) + print(xml.serialize(ctxdata.xmldata)) + end + + function ctxrunner.manipulate(ctxdata,ctxname,defaultname) + + if not ctxdata.jobname or ctxdata.jobname == "" then + return + end + + ctxdata.ctxname = ctxname or file.removesuffix(ctxdata.jobname) or "" + + if ctxdata.ctxname == "" then + return + end + + ctxdata.jobname = file.addsuffix(ctxdata.jobname,'tex') + ctxdata.ctxname = file.addsuffix(ctxdata.ctxname,'ctx') + + input.report("jobname:",ctxdata.jobname) + input.report("ctxname:",ctxdata.ctxname) + + -- mtxrun should resolve kpse: and file: + + local usedname = ctxdata.ctxname + local found = lfs.isfile(usedname) + + if not found then + for _, path in pairs(ctxdata.locations) do + local fullname = file.join(path,ctxdata.ctxname) + if lfs.isfile(fullname) then + usedname, found = fullname, true + break + end + end + end + + if not found and defaultname and defaultname ~= "" and file.exists(defaultname) then + usedname, found = defaultname, true + end + + if not found then + return + end + + ctxdata.xmldata = xml.load(usedname) + + if not ctxdata.xmldata then + return + else + -- test for valid, can be text file + end + + xml.include(ctxdata.xmldata,'ctx:include','name', table.append({'.', file.dirname(ctxdata.ctxname)},ctxdata.locations)) + + ctxdata.variables['job'] = ctxdata.jobname + + ctxdata.flags = xml.collect_texts(ctxdata.xmldata,"/ctx:job/ctx:flags/ctx:flag",true) + ctxdata.environments = xml.collect_texts(ctxdata.xmldata,"/ctx:job/ctx:process/ctx:resources/ctx:environment",true) + ctxdata.modules = xml.collect_texts(ctxdata.xmldata,"/ctx:job/ctx:process/ctx:resources/ctx:module",true) + ctxdata.filters = xml.collect_texts(ctxdata.xmldata,"/ctx:job/ctx:process/ctx:resources/ctx:filter",true) + ctxdata.modes = xml.collect_texts(ctxdata.xmldata,"/ctx:job/ctx:process/ctx:resources/ctx:mode",true) + ctxdata.messages = xml.collect_texts(ctxdata.xmldata,"ctx:message",true) + + ctxdata.flags = ctxrunner.reflag(ctxdata.flags) + + for _, message in ipairs(ctxdata.messages) do + -- message ctxdata.justtext(xml.tostring(message)) + end + + --~ REXML::XPath.each(root,"//ctx:block") do |blk| + --~ if @jobname && blk.attributes['pattern'] then + --~ root.delete(blk) unless @jobname =~ /#{blk.attributes['pattern']}/ + --~ else + --~ root.delete(blk) + --~ end + --~ end + + xml.each(ctxdata.xmldata,"ctx:value[@name='job']", function(ek,e,k) + e[k] = ctxdata.variables['job'] or "" + end) + + local commands = { } + xml.each(ctxdata.xmldata,"/ctx:job/ctx:preprocess/ctx:processors/ctx:processor", function(r,d,k) + local ek = d[k] + commands[ek.at and ek.at['name'] or "unknown"] = ek + end) + + local suffix = xml.first(ctxdata.xmldata,"/ctx:job/ctx:preprocess/@suffix") or ctxdata.suffix + local runlocal = xml.first(ctxdata.xmldata,"/ctx:job/ctx:preprocess/ctx:processors/@local") + + runlocal = toboolean(runlocal) + + for _, files in ipairs(xml.filters.elements(ctxdata.xmldata,"/ctx:job/ctx:preprocess/ctx:files")) do + for _, pattern in ipairs(xml.filters.elements(files,"ctx:file")) do + + preprocessor = pattern.at['processor'] or "" + + if preprocessor ~= "" then + + ctxdata.variables['old'] = ctxdata.jobname + xml.each(ctxdata.xmldata,"ctx:value", function(r,d,k) + local ek = d[k] + local ekat = ek.at['name'] + if ekat == 'old' then + d[k] = ctxrunner.substitute(ctxdata.variables[ekat] or "") + end + end) + + pattern = ctxrunner.justtext(xml.tostring(pattern)) + + local oldfiles = dir.glob(pattern) + local pluspath = false + if #oldfiles == 0 then + -- message: no files match pattern + for _, p in ipairs(ctxdata.paths) do + local oldfiles = dir.glob(path.join(p,pattern)) + if #oldfiles > 0 then + pluspath = true + break + end + end + end + if #oldfiles == 0 then + -- message: no old files + else + for _, oldfile in ipairs(oldfiles) do + newfile = oldfile .. "." .. suffix -- addsuffix will add one only + if ctxdata.runlocal then + newfile = file.basename(newfile) + end + if oldfile ~= newfile and file.needsupdate(oldfile,newfile) then + -- message: oldfile needs preprocessing + -- os.remove(newfile) + for _, pp in ipairs(preprocessor:split(',')) do + local command = commands[pp] + if command then + command = xml.copy(command) + local suf = (command.at and command.at['suffix']) or ctxdata.suffix + if suf then + newfile = oldfile .. "." .. suf + end + if ctxdata.runlocal then + newfile = file.basename(newfile) + end + xml.each(command,"ctx:old", function(r,d,k) + d[k] = ctxrunner.substitute(oldfile) + end) + xml.each(command,"ctx:new", function(r,d,k) + d[k] = ctxrunner.substitute(newfile) + end) + -- message: preprocessing #{oldfile} into #{newfile} using #{pp} + ctxdata.variables['old'] = oldfile + ctxdata.variables['new'] = newfile + xml.each(command,"ctx:value", function(r,d,k) + local ek = d[k] + local ekat = ek.at and ek.at['name'] + if ekat then + d[k] = ctxrunner.substitute(ctxdata.variables[ekat] or "") + end + end) + -- potential optimization: when mtxrun run internal + command = ctxrunner.justtext(command) -- command is still xml element here + input.report("command",command) + local result = os.spawn(command) + if result > 0 then + input.report("error, return code",result) + end + if ctxdata.runlocal then + oldfile = file.basename(oldfile) + end + end + end + if lfs.isfile(newfile) then + file.syncmtimes(oldfile,newfile) + ctxdata.prepfiles[oldfile] = true + else + input.report("error, check target location of new file", newfile) + ctxdata.prepfiles[oldfile] = false + end + else + input.report("old file needs no preprocessing") + ctxdata.prepfiles[oldfile] = lfs.isfile(newfile) + end + end + end + end + end + end + + ctxrunner.savelog(ctxdata) + + end + +end + +-- rest + +scripts.context.multipass = { + suffixes = { ".tuo", ".tuc" }, + nofruns = 8, +} + +function scripts.context.multipass.hashfiles(jobname) + local hash = { } + for _, suffix in ipairs(scripts.context.multipass.suffixes) do + local full = jobname .. suffix + hash[full] = md5.hex(io.loaddata(full) or "unknown") + end + return hash +end + +function scripts.context.multipass.changed(oldhash, newhash) + for k,v in pairs(oldhash) do + if v ~= newhash[k] then + return true + end + end + return false +end + +scripts.context.backends = { + pdftex = 'pdftex', + luatex = 'pdftex', + pdf = 'pdftex', + dvi = 'dvipdfmx', + dvips = 'dvips' +} + +function scripts.context.multipass.makeoptionfile(jobname,ctxdata) + -- take jobname from ctx + local f = io.open(jobname..".top","w") + if f then + local finalrun, kindofrun, currentrun = false, 0, 0 + local function someflag(flag) + return (ctxdata and ctxdata.flags[flag]) or environment.argument(flag) + end +--~ local someflag = environment.argument + local function setvalue(flag,format,hash,default) + local a = someflag(flag) or default + if a and a ~= "" then + if hash then + if hash[a] then + f:write(format:format(a),"\n") + end + else + f:write(format:format(a),"\n") + end + end + end + local function setvalues(flag,format) + if type(flag) == "table" then + for k, v in pairs(flag) do + f:write(format:format(v),"\n") + end + else + local a = someflag(flag) + if a and a ~= "" then + for v in a:gmatch("%s*([^,]+)") do + f:write(format:format(v),"\n") + end + end + end + end + local function setfixed(flag,format,...) + if someflag(flag) then + f:write(format:format(...),"\n") + end + end + local function setalways(format,...) + f:write(format:format(...),"\n") + end + setalways("\\unprotect") + setvalue('output' , "\\setupoutput[%s]", scripts.context.backends, 'pdftex') + setalways( "\\setupsystem[\\c!n=%s,\\c!m=%s]", kindofrun, currentrun) + setalways( "\\setupsystem[\\c!type=%s]",os.platform) + setfixed ("batchmode" , "\\batchmode") + setfixed ("nonstopmode" , "\\nonstopmode") + setfixed ("tracefiles" , "\\tracefilestrue") + setfixed ("paranoid" , "\\def\\maxreadlevel{1}") + setvalues("modefile" , "\\readlocfile{%s}{}{}") + setvalue ("result" , "\\setupsystem[file=%s]") + setvalues("path" , "\\usepath[%s]") + setfixed ("color" , "\\setupcolors[\\c!state=\\v!start]") + setfixed ("nompmode" , "\\runMPgraphicsfalse") -- obsolete, we assume runtime mp graphics + setfixed ("nomprun" , "\\runMPgraphicsfalse") -- obsolete, we assume runtime mp graphics + setfixed ("automprun" , "\\runMPgraphicsfalse") -- obsolete, we assume runtime mp graphics + setfixed ("fast" , "\\fastmode\n") + setfixed ("silentmode" , "\\silentmode\n") + setvalue ("separation" , "\\setupcolors[\\c!split=%s]") + setvalue ("setuppath" , "\\setupsystem[\\c!directory={%s}]") + setfixed ("noarrange" , "\\setuparranging[\\v!disable]") + if environment.argument('arrange') and not finalrun then + setalways( "\\setuparranging[\\v!disable]") + end + setvalue ("arguments" , "\\setupenv[%s]") + setvalue ("randomseed" , "\\setupsystem[\\c!random=%s]") + setvalues("modes" , "\\enablemode[%s]") + setvalues("mode" , "\\enablemode[%s]") + setvalues("filters" , "\\useXMLfilter[%s]") + setvalues("usemodules" , "\\usemodule[%s]") + setvalues("environments" , "\\environment %s ") + -- ctx stuff + if ctxdata then + setvalues(ctxdata.modes, "\\enablemode[%s]") + setvalues(ctxdata.modules, "\\usemodule[%s]") + setvalues(ctxdata.environments, "\\environment %s ") + end + -- done + setalways( "\\protect") + setalways( "\\endinput") + f:close() + end +end + +function scripts.context.multipass.copyluafile(jobname) + io.savedata(jobname..".tuc",io.loaddata(jobname..".tua") or "") +end + +function scripts.context.multipass.copytuifile(jobname) + local f, g = io.open(jobname..".tui"), io.open(jobname..".tuo",'w') + if f and g then + g:write("% traditional utility file, only commands written by mtxrun/context\n%\n") + for line in f:lines() do + if line:find("^c ") then + g:write((line:gsub("^c ","")),"%\n") + end + end + g:write("\\endinput\n") + f:close() + g:close() + end +end + +scripts.context.xmlsuffixes = table.tohash { + "xml", +} + +function scripts.context.run(ctxdata) + local function makestub(format,filename) + local stubname = file.replacesuffix(file.basename(filename),'run') + local f = io.open(stubname,'w') + if f then + f:write("\\starttext\n") + f:write(string.format(format,filename),"\n") + f:write("\\stoptext\n") + f:close() + filename = stubname + end + return filename + end + if ctxdata then + -- todo: interface + for k,v in pairs(ctxdata.flags) do + environment.setargument(k,v) + end + end + local files = environment.files + if #files > 0 then + input.identify_cnf(instance) + input.load_cnf(instance) + input.expand_variables(instance) + local formatname = "cont-en" + local formatfile, scriptfile = input.locate_format(formatname) + if formatfile and scriptfile then + for _, filename in ipairs(files) do + local basename, pathname = file.basename(filename), file.dirname(filename) + local jobname = file.removesuffix(basename) + if pathname == "" then + filename = "./" .. filename + end + -- also other stubs + if environment.argument("forcexml") or scripts.context.xmlsuffixes[file.extname(filename) or "?"] then -- mkii + filename = makestub("\\processXMLfilegrouped{%s}",filename) + elseif environment.argument("processxml") then -- mkiv + filename = makestub("\\xmlprocess{%s}",filename) + end + -- + if environment.argument("autopdf") then + os.spawn(string.format('pdfclose --file "%s" 2>&1', file.replacesuffix(filename,"pdf"))) + end + -- + local command = "luatex --fmt=" .. string.quote(formatfile) .. " --lua=" .. string.quote(scriptfile) .. " " .. string.quote(filename) + local oldhash, newhash = scripts.context.multipass.hashfiles(jobname), { } + scripts.context.multipass.makeoptionfile(jobname,ctxdata) + for i=1, scripts.context.multipass.nofruns do + input.report(string.format("run %s: %s",i,command)) + local returncode = os.spawn(command) + input.report("return code: " .. returncode) + if returncode > 0 then + input.report("fatal error, run aborted") + break + else + scripts.context.multipass.copyluafile(jobname) + scripts.context.multipass.copytuifile(jobname) + newhash = scripts.context.multipass.hashfiles(jobname) + if scripts.context.multipass.changed(oldhash,newhash) then + oldhash = newhash + else + break + end + end + end + -- + -- todo: result + -- + if environment.argument("autopdf") then + os.spawn(string.format('pdfopen --file "%s" 2>&1', file.replacesuffix(filename,"pdf"))) + end + -- + end + else + input.verbose = true + input.report("error", "no format found with name " .. formatname) + end + end +end + +function scripts.context.make() + local list = (environment.files[1] and environment.files) or { "cont-en", "cont-nl", "mptopdf" } + for _, name in ipairs(list) do + local command = "luatools --make --compile " .. name + input.report("running command: " .. command) + os.spawn(command) + end +end + +function scripts.context.generate() + -- hack, should also be a shared function + local command = "luatools --generate " + input.report("running command: " .. command) + os.spawn(command) +end + +function scripts.context.ctx() + local ctxdata = ctxrunner.new() + ctxdata.jobname = environment.files[1] + ctxrunner.manipulate(ctxdata,environment.argument("ctx")) + scripts.context.run(ctxdata) +end + +function scripts.context.version() + local name = input.find_file(instance,"context.tex") + if name ~= "" then + input.report(string.format("main context file: %s",name)) + local data = io.loaddata(name) + if data then + local version = data:match("\\edef\\contextversion{(.-)}") + if version then + input.report(string.format("current version : %s",version)) + else + input.report("context version: unknown, no timestamp found") + end + else + input.report("context version: unknown, load error") + end + else + input.report("main context file: unknown, 'context.tex' not found") + end +end + +function scripts.context.touch() + if environment.argument("expert") then + local function touch(name,pattern) + local name = input.find_file(instance,name) + local olddata = io.loaddata(name) + if olddata then + local oldversion, newversion = "", os.date("%Y.%M.%d %H:%m") + local newdata, ok = olddata:gsub(pattern,function(pre,mid,post) + oldversion = mid + return pre .. newversion .. post + end) + if ok > 0 then + local backup = file.replacesuffix(name,"tmp") + os.remove(backup) + os.rename(name,backup) + io.savedata(name,newdata) + return true, oldversion, newversion, name + else + return false + end + end + end + local done, oldversion, newversion, foundname = touch("context.tex", "(\\edef\\contextversion{)(.-)(})") + if done then + input.report(string.format("old version : %s", oldversion)) + input.report(string.format("new version : %s", newversion)) + input.report(string.format("touched file: %s", foundname)) + local ok, _, _, foundname = touch("cont-new.tex", "(\\newcontextversion{)(.-)(})") + if ok then + input.report(string.format("touched file: %s", foundname)) + end + end + end +end + +function scripts.context.timed(action) + input.starttiming(scripts.context) + action() + input.stoptiming(scripts.context) + input.report("total runtime: " .. input.elapsedtime(scripts.context)) +end + +banner = banner .. " | context tools " + +messages.help = [[ +--run process (one or more) files (default action) +--make create context formats formats +--generate generate file database etc. +--ctx=name use ctx file +--version report installed context version +--autopdf open pdf file afterwards + +--expert expert options +]] + +messages.expert = [[ +expert options: also provide --expert + +--touch update context version (remake needed afterwards) +]] + +input.verbose = true + +if environment.argument("once") then + scripts.context.multipass.nofruns = 1 +end + +if environment.argument("run") then + scripts.context.timed(scripts.context.run) +elseif environment.argument("make") then + scripts.context.timed(scripts.context.make) +elseif environment.argument("ctx") then + scripts.context.timed(scripts.context.ctx) +elseif environment.argument("version") then + scripts.context.version() +elseif environment.argument("touch") then + scripts.context.touch() +elseif environment.argument("help") then + input.help(banner,messages.help) +elseif environment.argument("expert") then + input.help(banner,messages.expert) +elseif environment.files[1] then + scripts.context.timed(scripts.context.run) +else + input.help(banner,messages.help) +end + diff --git a/Master/texmf-dist/scripts/context/lua/mtx-convert.lua b/Master/texmf-dist/scripts/context/lua/mtx-convert.lua new file mode 100644 index 00000000000..1bfc10c0fa8 --- /dev/null +++ b/Master/texmf-dist/scripts/context/lua/mtx-convert.lua @@ -0,0 +1,86 @@ +if not modules then modules = { } end modules ['mtx-convert'] = { + version = 1.001, + comment = "companion to mtxrun.lua", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +do + + graphics = graphics or { } + graphics.converters = graphics.converters or { } + + local gsprogram = (os.platform == "windows" and "gswin32c") or "gs" + local gstemplate = "%s -q -sDEVICE=pdfwrite -dEPSCrop -dNOPAUSE -dNOCACHE -dBATCH -dAutoRotatePages=/None -dProcessColorModel=/DeviceCMYK -sOutputFile=%s %s -c quit" + + function graphics.converters.epstopdf(inputpath,outputpath,epsname) + inputpath = inputpath or "." + outputpath = outputpath or "." + local oldname = file.join(inputpath,epsname) + local newname = file.join(outputpath,file.replacesuffix(epsname,"pdf")) + local et = lfs.attributes(oldname,"modification") + local pt = lfs.attributes(newname,"modification") + if not pt or et > pt then + dir.mkdirs(outputpath) + local tmpname = file.replacesuffix(newname,"tmp") + local command = string.format(gstemplate,gsprogram,tmpname,oldname) + os.spawn(command) + os.remove(newname) + os.rename(tmpname,newname) + end + end + + function graphics.converters.convertpath(inputpath,outputpath) + for name in lfs.dir(inputpath or ".") do + if name:find("%.$") then + -- skip . and .. + elseif name:find("%.eps$") then + graphics.converters.epstopdf(inputpath,outputpath, name) + elseif lfs.attributes(inputpath .. "/".. name,"mode") == "directory" then + graphics.converters.convertpath(inputpath .. "/".. name,outputpath .. "/".. name) + end + end + end + +end + +texmf.instance = instance -- we need to get rid of this / maybe current instance in global table + +scripts = scripts or { } +scripts.convert = scripts.convert or { } + +scripts.convert.delay = 5 * 60 -- 5 minutes + +function scripts.convert.convertall() + local watch = environment.arguments.watch or false + local delay = environment.arguments.delay or scripts.convert.delay + local input = environment.arguments.inputpath or "." + local output = environment.arguments.outputpath or "." + while true do + graphics.converters.convertpath(input, output) + if watch then + os.sleep(delay) + else + break + end + end +end + +banner = banner .. " | graphic conversion tools " + +messages.help = [[ +--convertall convert all graphics on path +--inputpath=string original graphics path +--outputpath=string converted graphics path +--watch watch folders +--delay time between sweeps +]] + +input.verbose = true + +if environment.argument("convertall") then + scripts.convert.convertall() +else + input.help(banner,messages.help) +end diff --git a/Master/texmf-dist/scripts/context/lua/mtx-fonts.lua b/Master/texmf-dist/scripts/context/lua/mtx-fonts.lua new file mode 100644 index 00000000000..395e9764ed4 --- /dev/null +++ b/Master/texmf-dist/scripts/context/lua/mtx-fonts.lua @@ -0,0 +1,103 @@ +if not modules then modules = { } end modules ['mtx-fonts'] = { + version = 1.001, + comment = "companion to mtxrun.lua", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +dofile(input.find_file(instance,"font-syn.lua")) + +texmf.instance = instance -- we need to get rid of this / maybe current instance in global table + +scripts = scripts or { } +scripts.fonts = scripts.fonts or { } + +function scripts.fonts.reload() + fonts.names.load(true) +end + +function scripts.fonts.list(pattern,reload,all) + if reload then + logs.report("fontnames","reloading font database") + end + local t = fonts.names.list(pattern,reload) + if reload then + logs.report("fontnames","done\n\n") + end + if t then + local s, w = table.sortedkeys(t), { 0, 0, 0 } + local function action(f) + for k,v in pairs(s) do + if all or v == t[v][2]:lower() then + local type, name, file, sub = unpack(t[v]) + f(v,name,file,sub) + end + end + end + action(function(v,n,f,s) + if #v > w[1] then w[1] = #v end + if #n > w[2] then w[2] = #n end + if #f > w[3] then w[3] = #f end + end) + action(function(v,n,f,s) + if s then s = "(sub)" else s = "" end + print(string.format("%s %s %s %s",v:padd(w[1]," "),n:padd(w[2]," "),f:padd(w[3]," "), s)) + end) + end +end + +function scripts.fonts.save(name,sub) + local function save(savename,fontblob) + if fontblob then + savename = savename:lower() .. ".lua" + logs.report("fontsave","saving data in " .. savename) + table.tofile(savename,fontforge.to_table(fontblob),"return") + fontforge.close(fontblob) + end + end + if name and name ~= "" then + local filename = input.find_file(texmf.instance,name) -- maybe also search for opentype + if filename and filename ~= "" then + local suffix = file.extname(filename) + if suffix == 'ttf' or suffix == 'otf' or suffix == 'ttc' then + local fontinfo = fontforge.info(filename) + if fontinfo then + if fontinfo[1] then + for _, v in ipairs(fontinfo) do + save(v.fontname,fontforge.open(filename,v.fullname)) + end + else + save(fontinfo.fullname,fontforge.open(filename)) + end + end + end + end + end +end + +banner = banner .. " | font tools " + +messages.help = [[ +--reload generate new font database +--list list installed fonts +--save save open type font in raw table + +--pattern=str filter files +--all provide alternatives +]] + +if environment.argument("reload") then + scripts.fonts.reload() +elseif environment.argument("list") then + local pattern = environment.argument("pattern") or environment.files[1] or "" + local all = environment.argument("all") + local reload = environment.argument("reload") + scripts.fonts.list(pattern,reload,all) +elseif environment.argument("save") then + local name = environment.files[1] or "" + local sub = environment.files[2] or "" + scripts.fonts.save(name,sub) +else + input.help(banner,messages.help) +end diff --git a/Master/texmf-dist/scripts/context/lua/mtx-server.lua b/Master/texmf-dist/scripts/context/lua/mtx-server.lua new file mode 100644 index 00000000000..293bc0c1c37 --- /dev/null +++ b/Master/texmf-dist/scripts/context/lua/mtx-server.lua @@ -0,0 +1,83 @@ +if not modules then modules = { } end modules ['mtx-server'] = { + version = 1.001, + comment = "companion to mtxrun.lua", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +texmf.instance = instance -- we need to get rid of this / maybe current instance in global table + +-- The starting point was stripped down webserver.lua by Samuel +-- Saint-Pettersen (as downloaded on 21-5-2008) which only served +-- html and was not configureable. In due time I will extend the +-- next code. Eventually we may move code to l-server. + +scripts = scripts or { } +scripts.webserver = scripts.webserver or {} + +local socket = require("socket") + +local function message(str) + return string.format("<h1>%s</h1>",str) +end + +function scripts.webserver.run(configuration) + local server = assert(socket.bind("*", tonumber(configuration.port or 8080))) + while true do + local client = server:accept() + client:settimeout(configuration.timeout or 60) + local request, e = client:receive() + if e then + client:send(message("404 Not Found")) + else + -- GET /showcase.pdf HTTP/1.1 + local filename = request:match("GET (.+) HTTP/.*$") -- todo: more clever + -- filename = filename:gsub("%%(%d%d)",function(c) return string.char(tonumber(c,16)) end) + filename = socket.url.unescape(filename) + if filename == nil or filename == "" then + filename = configuration.index or "index.html" + end + -- todo chunked + local fullname = file.join(configuration.root,filename) + local data = io.loaddata(fullname) + if data and data ~= "" then + local result + client:send("HTTP/1.1 200 OK\r\n") + client:send("Connection: close\r\n") + if filename:find(".pdf$") then -- todo: special handler + client:send(string.format("Content-Length: %s\r\n",#data)) + client:send("Content-Type: application/pdf\r\n") + else + client:send("Content-Type: text/html\r\n") + end + client:send("\r\n") + client:send(data) + client:send("\r\n") + else + client:send(message("404 Not Found")) + end + end + client:close() + end +end + + +banner = banner .. " | webserver " + +messages.help = [[ +--start start server +--port port to listen to +--root server root +--index index file +]] + +if environment.argument("start") then + scripts.webserver.run { + port = environment.argument("port") or "8080", + root = environment.argument("root") or ".", -- "e:/websites/www.pragma-ade.com", + index = environment.argument("index") or "index.html", + } +else + input.help(banner,messages.help) +end diff --git a/Master/texmf-dist/scripts/context/lua/mtx-update.lua b/Master/texmf-dist/scripts/context/lua/mtx-update.lua new file mode 100644 index 00000000000..581774f1ea0 --- /dev/null +++ b/Master/texmf-dist/scripts/context/lua/mtx-update.lua @@ -0,0 +1,380 @@ +if not modules then modules = { } end modules ['mtx-update'] = { + version = 1.001, + comment = "companion to mtxrun.lua", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- This script is dedicated to Mojca Miklavec, who is the driving force behind +-- moving minimal generation from our internal machines to the context garden. +-- Together with Arthur Reutenauer she made sure that it worked well on all +-- platforms that matter. + +texmf.instance = instance -- we need to get rid of this / maybe current instance in global table + +scripts = scripts or { } +scripts.update = scripts.update or { } + +minimals = minimals or { } +minimals.config = minimals.config or { } + +os.setenv("CYGWIN","nontsec") + +scripts.update.formats = { + "cont-en", + "cont-nl", + "cont-cz", + "cont-de", + "cont-fa", + "cont-it", + "cont-ro", + "cont-uk", + "metafun", + "mptopdf", + "plain" +} + +scripts.update.repositories = { + "current", + "experimental" +} + +scripts.update.versions = { + "current", + "latest" +} + +scripts.update.engines = { + ["luatex"] = { + { "base/tex/", "texmf" }, + { "base/metapost/", "texmf" }, + { "fonts/new/", "texmf" }, + { "fonts/common/", "texmf" }, + { "fonts/other/", "texmf" }, + { "context/<version>/", "texmf-context" }, + { "context/img/", "texmf-context" }, + { "context/config/", "texmf-context" }, + { "misc/setuptex/", "." }, + { "misc/web2c", "texmf" }, + { "bin/common/<platform>/", "texmf-<platform>" }, + { "bin/context/<platform>/", "texmf-<platform>" }, + { "bin/metapost/<platform>/", "texmf-<platform>" }, + { "bin/luatex/<platform>/", "texmf-<platform>" }, + { "bin/man/", "texmf-<platform>" } + }, + ["xetex"] = { + { "base/tex/", "texmf" }, + { "base/metapost/", "texmf" }, + { "base/xetex/", "texmf" }, + { "fonts/new/", "texmf" }, + { "fonts/common/", "texmf" }, + { "fonts/other/", "texmf" }, + { "context/<version>/", "texmf-context" }, + { "context/img/", "texmf-context" }, + { "context/config/", "texmf-context" }, + { "misc/setuptex/", "." }, + { "misc/web2c", "texmf" }, + { "bin/common/<platform>/", "texmf-<platform>" }, + { "bin/context/<platform>/", "texmf-<platform>" }, + { "bin/metapost/<platform>/", "texmf-<platform>" }, + { "bin/xetex/<platform>/", "texmf-<platform>" }, + { "bin/man/", "texmf-<platform>" } + }, + ["pdftex"] = { + { "base/tex/", "texmf" }, + { "base/metapost/", "texmf" }, + { "fonts/old/", "texmf" }, + { "fonts/common/", "texmf" }, + { "fonts/other/", "texmf" }, + { "context/<version>/", "texmf-context" }, + { "context/img/", "texmf-context" }, + { "context/config/", "texmf-context" }, + { "misc/setuptex/", "." }, + { "misc/web2c", "texmf" }, + { "bin/common/<platform>/", "texmf-<platform>" }, + { "bin/context/<platform>/", "texmf-<platform>" }, + { "bin/metapost/<platform>/", "texmf-<platform>" }, + { "bin/pdftex/<platform>/", "texmf-<platform>" }, + { "bin/man/", "texmf-<platform>" } + }, + ["all"] = { + { "base/tex/", "texmf" }, + { "base/metapost/", "texmf" }, + { "base/xetex/", "texmf" }, + { "fonts/old/", "texmf" }, + { "fonts/new/", "texmf" }, + { "fonts/common/", "texmf" }, + { "fonts/other/", "texmf" }, + { "context/<version>/", "texmf-context" }, + { "context/img/", "texmf-context" }, + { "context/config/", "texmf-context" }, + { "misc/setuptex/", "." }, + { "misc/web2c", "texmf" }, + { "bin/common/<platform>/", "texmf-<platform>" }, + { "bin/context/<platform>/", "texmf-<platform>" }, + { "bin/metapost/<platform>/", "texmf-<platform>" }, + { "bin/luatex/<platform>/", "texmf-<platform>" }, + { "bin/xetex/<platform>/", "texmf-<platform>" }, + { "bin/pdftex/<platform>/", "texmf-<platform>" }, + { "bin/man/", "texmf-<platform>" } + }, +} + +scripts.update.platforms = { + ["mswin"] = "mswin", + ["windows"] = "mswin", + ["win32"] = "mswin", + ["win"] = "mswin", + ["linux"] = "linux", + ["freebsd"] = "freebsd", + ["linux-32"] = "linux", + ["linux-64"] = "linux-64", + ["osx"] = "osx-intel", + ["osx-intel"] = "osx-intel", + ["osx-ppc"] = "osx-ppc", +} + +function scripts.update.run(str) + logs.report("run", str) + if environment.argument("force") then + -- important, otherwise formats fly to a weird place + -- (texlua sets luatex as the engine, we need to reset that or to fix texexec :) + os.setenv("engine",nil) + os.execute(str) + end +end + +function scripts.update.fullpath(path) + if input.aux.rootbased_path(path) then + return path + else + return lfs.currentdir() .. "/" .. path + end +end + +function scripts.update.synchronize() + logs.report("update","start") + local texroot = scripts.update.fullpath(states.get("paths.root")) + local engines = states.get('engines') + local platforms = states.get('platforms') + local repositories = states.get('repositories') + local bin = states.get("rsync.program") + local url = states.get("rsync.server") + local version = states.get("context.version") + local force = environment.argument("force") + if not url:find("::$") then url = url .. "::" end + local ok = lfs.attributes(texroot,"mode") == "directory" + if not ok and force then + dir.mkdirs(texroot) + ok = lfs.attributes(texroot,"mode") == "directory" + end + if ok or not force then + if force then + dir.mkdirs(string.format("%s/%s", texroot, "texmf-cache")) + end + local fetched, individual = { }, { } + for engine, _ in pairs(engines) do + local collections = scripts.update.engines[engine] + if collections then + for _, collection in ipairs(collections) do + for platform, _ in pairs(platforms) do + platform = scripts.update.platforms[platform] + if platform then + local archive = collection[1]:gsub("<platform>", platform) + local destination = string.format("%s/%s", texroot, collection[2]:gsub("<platform>", platform)) + destination = destination:gsub("\\","/") + archive = archive:gsub("<version>",version) + if platform == "windows" or platform == "mswin" then + destination = destination:gsub("([a-zA-Z]):/", "/cygdrive/%1/") + end + individual[#individual+1] = { archive, destination } + end + end + end + end + end + local combined = { } + for _, repository in ipairs(scripts.update.repositories) do + if repositories[repository] then + for _, v in pairs(individual) do + local archive, destination = v[1], v[2] + local cd = combined[destination] + if not cd then + cd = { } + combined[destination] = cd + end + cd[#cd+1] = string.format("%s/%s/%s",states.get('rsync.module'),repository,archive) + end + end + end + if input.verbose then + for k, v in pairs(combined) do + logs.report("update", k) + for k,v in ipairs(v) do + logs.report("update", " <= " .. v) + end + end + end + for destination, archive in pairs(combined) do + local archives, command = table.concat(archive," "), "" + local normalflags, deleteflags = states.get("rsync.flags.normal"), states.get("rsync.flags.delete") + if true then -- environment.argument("keep") or destination:find("%.$") then + command = string.format("%s %s %s'%s' '%s'", bin, normalflags, url, archives, destination) + else + command = string.format("%s %s %s %s'%s' '%s'", bin, normalflags, deleteflags, url, archives, destination) + end + logs.report("mtx update", string.format("running command: %s",command)) + if not fetched[command] then + scripts.update.run(command) + fetched[command] = command + end + end + else + logs.report("mtx update", string.format("no valid texroot: %s",texroot)) + end + if not force then + logs.report("update", "use --force to really update") + end + logs.report("update","done") +end + +function table.fromhash(t) + local h = { } + for k, v in pairs(t) do -- no ipairs here + if v then h[#h+1] = k end + end + return h +end + + +function scripts.update.make() + logs.report("make","start") + local force = environment.argument("force") + local texroot = scripts.update.fullpath(states.get("paths.root")) + local engines = states.get('engines') + local platforms = states.get('platforms') + local formats = states.get('formats') + input.load_tree(texroot) + scripts.update.run("mktexlsr") + scripts.update.run("luatools --generate") + local formatlist = table.concat(table.fromhash(formats), " ") + if formatlist ~= "" then + for engine in pairs(engines) do + -- todo: just handle make here or in mtxrun --script context --make +--~ os.execute("set") + scripts.update.run(string.format("texexec --make --all --fast --%s %s",engine,formatlist)) + end + end + if not force then + logs.report("make", "use --force to really make") + end + logs.report("make","done") +end + +banner = banner .. " | download tools " + +messages.help = [[ +--platform=string platform (windows, linux, linux-64, osx-intel, osx-ppc) +--server=string repository url (rsync://contextgarden.net) +--module=string repository url (minimals) +--repository=string specify version (current, experimental) +--context=string specify version (current, latest, yyyy.mm.dd) +--rsync=string rsync binary (rsync) +--texroot installation directory (not guessed for the moment) +--engine tex engine (luatex, pdftex, xetex) +--force instead of a dryrun, do the real thing +--update update minimal tree +--make also make formats and generate file databases +--keep don't delete unused or obsolete files +--state update tree using saved state +]] + +input.verbose = true + +scripts.savestate = true + +if scripts.savestate then + + states.load("status-of-update.lua") + + -- tag, value, default, persistent + + input.starttiming(states) + + states.set("info.version",0.1) -- ok + states.set("info.count",(states.get("info.count") or 0) + 1,1,false) -- ok + states.set("info.comment","this file contains the settings of the last 'mtxrun --script update ' run",false) -- ok + states.set("info.date",os.date("!%Y-%m-%d %H:%M:%S")) -- ok + + states.set("rsync.program", environment.argument("rsync"), "rsync", true) -- ok + states.set("rsync.server", environment.argument("server"), "contextgarden.net::", true) -- ok + states.set("rsync.module", environment.argument("module"), "minimals", true) -- ok + states.set("rsync.flags.normal", environment.argument("flags"), "-rpztlv --stats", true) -- ok + states.set("rsync.flags.delete", nil, "--delete", true) -- ok + + states.set("paths.root", environment.argument("texroot"), "tex", true) -- ok + + states.set("context.version", environment.argument("context"), "current", true) -- ok + + local valid = table.tohash(scripts.update.repositories) + for r in string.gmatch(environment.argument("repository") or "current","([^, ]+)") do + if valid[r] then states.set("repositories." .. r, true) end + end + local valid = scripts.update.engines + for r in string.gmatch(environment.argument("engine") or "all","([^, ]+)") do + if r == "all" then + for k, v in pairs(valid) do + if k ~= "all" then + states.set("engines." .. k, true) + end + end + elseif valid[r] then + states.set("engines." .. r, true) + end + end + local valid = scripts.update.platforms + for r in string.gmatch(environment.argument("platform") or os.currentplatform(),"([^, ]+)") do + if valid[r] then states.set("platforms." .. r, true) end + end + + local valid = table.tohash(scripts.update.formats) + for r in string.gmatch(environment.argument("formats") or "","([^, ]+)") do + if valid[r] then states.set("formats." .. r, true) end + end + + states.set("formats.cont-en", true) + states.set("formats.cont-nl", true) + states.set("formats.metafun", true) + + -- modules + + logs.report("state","loaded") + +end + +if environment.argument("state") then + environment.setargument("update",true) + environment.setargument("force",true) + environment.setargument("make",true) +end + +if environment.argument("update") then + scripts.update.synchronize() + if environment.argument("make") then + scripts.update.make() + end +elseif environment.argument("make") then + scripts.update.make() +else + input.help(banner,messages.help) +end + +if scripts.savestate then + input.stoptiming(states) + states.set("info.runtime",tonumber(input.elapsedtime(states))) + if environment.argument("force") then + states.save() + logs.report("state","saved") + end +end diff --git a/Master/texmf-dist/scripts/context/lua/mtx-watch.lua b/Master/texmf-dist/scripts/context/lua/mtx-watch.lua new file mode 100644 index 00000000000..f9e81da4257 --- /dev/null +++ b/Master/texmf-dist/scripts/context/lua/mtx-watch.lua @@ -0,0 +1,228 @@ +if not modules then modules = { } end modules ['mtx-watch'] = { + version = 1.001, + comment = "companion to mtxrun.lua", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +texmf.instance = instance -- we need to get rid of this / maybe current instance in global table + +scripts = scripts or { } +scripts.watch = scripts.watch or { } + +do + + function scripts.watch.watch() + local delay = environment.argument("delay") or 5 + local logpath = environment.argument("logpath") or "" + local pipe = environment.argument("pipe") or false + if #environment.files > 0 then + for _, path in ipairs(environment.files) do + logs.report("watch", "watching path ".. path) + end + local function glob(files,path) + for name in lfs.dir(path) do + if name:find("^%.") then + -- skip . and .. + else + name = path .. "/" .. name + local a = lfs.attributes(name) + if not a then + -- weird + elseif a.mode == "directory" then + if name:find("graphics$") or name:find("figures$") or name:find("resources$") then + -- skip these too + else + glob(files,name) + end + elseif name:find(".%luj$") then + files[name] = a.change or a.ctime or a.modification or a.mtime + end + end + end + end + local function process() + local done = false + for _, path in ipairs(environment.files) do + lfs.chdir(path) + local files = { } + glob(files,path) + table.sort(files) -- what gets sorted here + for name, time in pairs(files) do + --~ local ok, joblog = xpcall(function() return dofile(name) end, function() end ) + local ok, joblog = pcall(dofile,name) + if ok and joblog then + if joblog.status == "processing" then + logs.report("watch",string.format("aborted job, %s added to queue",name)) + joblog.status = "queued" + io.savedata(name, table.serialize(joblog,true)) + elseif joblog.status == "queued" then + local command = joblog.command + if command then + local replacements = { + inputpath = (joblog.paths and joblog.paths.input ) or ".", + outputpath = (joblog.paths and joblog.paths.output) or ".", + filename = joblog.filename or "", + } + command = command:gsub("%%(.-)%%", replacements) + if command ~= "" then + joblog.status = "processing" + joblog.runtime = os.time() -- os.clock() + io.savedata(name, table.serialize(joblog,true)) + logs.report("watch",string.format("running: %s", command)) + local newpath = file.dirname(name) + io.flush() + local result = "" + if newpath ~= "" and newpath ~= "." then + local oldpath = lfs.currentdir() + lfs.chdir(newpath) + if pipe then result = os.resultof(command) else result = os.spawn(command) end + lfs.chdir(oldpath) + else + if pipe then result = os.resultof(command) else result = os.spawn(command) end + end + logs.report("watch",string.format("return value: %s", result)) + done = true + local path, base = replacements.outputpath, file.basename(replacements.filename) + joblog.runtime = os.time() - joblog.runtime -- os.clock() - joblog.runtime + joblog.result = file.replacesuffix(file.join(path,base),"pdf") + joblog.size = lfs.attributes(joblog.result,"size") + joblog.status = "finished" + else + joblog.status = "invalid command" + end + else + joblog.status = "no command" + end + -- pcall, when error sleep + again + io.savedata(name, table.serialize(joblog,true)) + if logpath ~= "" then + local name = string.format("%s/%s%04i%09i.lua", logpath, os.time(), math.floor((os.clock()*100)%1000), math.random(99999999)) + io.savedata(name, table.serialize(joblog,true)) + logs.report("watch", "saving joblog ".. name) + end + end + end + end + end + end + local n, start = 0, os.clock() + local function wait() + io.flush() + if not done then + n = n + 1 + if n >= 10 then + logs.report("watch", string.format("run time: %i seconds, memory usage: %0.3g MB", os.clock() - start, (status.luastate_bytes/1024)/1000)) + n = 0 + end + os.sleep(delay) + end + end + while true do + pcall(process) + pcall(wait) + end + else + logs.report("watch", "no paths to watch") + end + end + +end + +function scripts.watch.collect_logs(path) -- clean 'm up too + path = path or environment.argument("logpath") or "" + path = (path == "" and ".") or path + local files = dir.globfiles(path,false,"^%d+%.lua$") + local collection = { } + local valid = table.tohash({"filename","result","runtime","size","status"}) + for _, name in ipairs(files) do + local t = dofile(name) + if t and type(t) == "table" and t.status then + for k, v in pairs(t) do + if not valid[k] then + t[k] = nil + end + end + collection[name:gsub("[^%d]","")] = t + end + end + return collection +end + +function scripts.watch.save_logs(collection,path) -- play safe + if collection and not table.is_empty(collection) then + path = path or environment.argument("logpath") or "" + path = (path == "" and ".") or path + local filename = string.format("%s/collected-%s.lua",path,tostring(os.time())) + io.savedata(filename,table.serialize(collection,true)) + local check = dofile(filename) + for k,v in pairs(check) do + if not collection[k] then + logs.error("watch", "error in saving file") + os.remove(filename) + return false + end + end + for k,v in pairs(check) do + os.remove(string.format("%s.lua",k)) + end + return true + else + return false + end +end + +function scripts.watch.collect_collections(path) -- removes duplicates + path = path or environment.argument("logpath") or "" + path = (path == "" and ".") or path + local files = dir.globfiles(path,false,"^collected%-%d+%.lua$") + local collection = { } + for _, name in ipairs(files) do + local t = dofile(name) + if t and type(t) == "table" then + for k, v in pairs(t) do + collection[k] = v + end + end + end + return collection +end + +function scripts.watch.show_logs(path) -- removes duplicates + local collection = scripts.watch.collect_collections(path) or { } + local max = 0 + for k,v in pairs(collection) do + v = v.filename or "?" + if #v > max then max = #v end + end + print(max) + for k,v in ipairs(table.sortedkeys(collection)) do + local c = collection[v] + local f, s, r, n = c.filename or "?", c.status or "?", c.runtime or 0, c.size or 0 + logs.report("watch", string.format("%s %s %3i %8i %s",string.padd(f,max," "),string.padd(s,10," "),r,n,v)) + end +end + +banner = banner .. " | watchdog" + +messages.help = [[ +--logpath optional path for log files +--watch watch given path +--pipe use pipe instead of execute +--delay delay between sweeps +--collect condense log files +--showlog show log data +]] + +input.verbose = true + +if environment.argument("watch") then + scripts.watch.watch() +elseif environment.argument("collect") then + scripts.watch.save_logs(scripts.watch.collect_logs()) +elseif environment.argument("showlog") then + scripts.watch.show_logs() +else + input.help(banner,messages.help) +end diff --git a/Master/texmf-dist/scripts/context/lua/mtxrun.lua b/Master/texmf-dist/scripts/context/lua/mtxrun.lua new file mode 100644 index 00000000000..040214178b8 --- /dev/null +++ b/Master/texmf-dist/scripts/context/lua/mtxrun.lua @@ -0,0 +1,8549 @@ +#!/usr/bin/env texlua + +if not modules then modules = { } end modules ['mtxrun'] = { + version = 1.001, + comment = "runner, lua replacement for texmfstart.rb", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- one can make a stub: +-- +-- #!/bin/sh +-- env LUATEXDIR=/....../texmf/scripts/context/lua luatex --luaonly mtxrun.lua "$@" + +-- filename : mtxrun.lua +-- comment : companion to context.tex +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +-- This script is based on texmfstart.rb but does not use kpsewhich to +-- locate files. Although kpse is a library it never came to opening up +-- its interface to other programs (esp scripting languages) and so we +-- do it ourselves. The lua variant evolved out of an experimental ruby +-- one. Interesting is that using a scripting language instead of c does +-- not have a speed penalty. Actually the lua variant is more efficient, +-- especially when multiple calls to kpsewhich are involved. The lua +-- library also gives way more ocntrol. + +-- to be done / considered +-- +-- support for --exec or make it default +-- support for jar files (or maybe not, never used, too messy) +-- support for $RUBYINPUTS cum suis (if still needed) +-- remember for subruns: _CTX_K_V_#{original}_ +-- remember for subruns: _CTX_K_S_#{original}_ +-- remember for subruns: TEXMFSTART.#{original} [tex.rb texmfstart.rb] + +banner = "version 1.0.2 - 2007+ - PRAGMA ADE / CONTEXT" +texlua = true + +-- begin library merge +-- filename : l-string.lua +-- comment : split off from luat-lib +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-string'] = 1.001 + +--~ function string.split(str, pat) -- taken from the lua wiki +--~ local t = {n = 0} -- so this table has a length field, traverse with ipairs then! +--~ local fpat = "(.-)"..pat +--~ local last_end = 1 +--~ local s, e, cap = string.find(str, fpat, 1) +--~ while s ~= nil do +--~ if s~=1 or cap~="" then +--~ table.insert(t,cap) +--~ end +--~ last_end = e+1 +--~ s, e, cap = string.find(str, fpat, last_end) +--~ end +--~ if last_end<=string.len(str) then +--~ table.insert(t,(string.sub(str,last_end))) +--~ end +--~ return t +--~ end + +--~ function string:split(pat) -- taken from the lua wiki but adapted +--~ local t = { } -- self and colon usage (faster) +--~ local fpat = "(.-)"..pat +--~ local last_end = 1 +--~ local s, e, cap = self:find(fpat, 1) +--~ while s ~= nil do +--~ if s~=1 or cap~="" then +--~ t[#t+1] = cap +--~ end +--~ last_end = e+1 +--~ s, e, cap = self:find(fpat, last_end) +--~ end +--~ if last_end <= #self then +--~ t[#t+1] = self:sub(last_end) +--~ end +--~ return t +--~ end + +--~ a piece of brilliant code by Rici Lake (posted on lua list) -- only names changed +--~ +--~ function string:splitter(pat) +--~ local st, g = 1, self:gmatch("()"..pat.."()") +--~ local function splitter(self) +--~ if st then +--~ local s, f = g() +--~ local rv = self:sub(st, (s or 0)-1) +--~ st = f +--~ return rv +--~ end +--~ end +--~ return splitter, self +--~ end + +function string:splitter(pat) + -- by Rici Lake (posted on lua list) -- only names changed + -- p 79 ref man: () returns position of match + local st, g = 1, self:gmatch("()("..pat..")") + local function strgetter(self, segs, seps, sep, cap1, ...) + st = sep and seps + #sep + return self:sub(segs, (seps or 0) - 1), cap1 or sep, ... + end + local function strsplitter(self) + if st then return strgetter(self, st, g()) end + end + return strsplitter, self +end + +function string:split(separator) + local t = {} + for k in self:splitter(separator) do t[#t+1] = k end + return t +end + +-- faster than a string:split: + +function string:splitchr(chr) + if #self > 0 then + local t = { } + for s in string.gmatch(self..chr,"(.-)"..chr) do + t[#t+1] = s + end + return t + else + return { } + end +end + +--~ function string.piecewise(str, pat, fnc) -- variant of split +--~ local fpat = "(.-)"..pat +--~ local last_end = 1 +--~ local s, e, cap = string.find(str, fpat, 1) +--~ while s ~= nil do +--~ if s~=1 or cap~="" then +--~ fnc(cap) +--~ end +--~ last_end = e+1 +--~ s, e, cap = string.find(str, fpat, last_end) +--~ end +--~ if last_end <= #str then +--~ fnc((string.sub(str,last_end))) +--~ end +--~ end + +function string.piecewise(str, pat, fnc) -- variant of split + for k in string.splitter(str,pat) do fnc(k) end +end + +--~ function string.piecewise(str, pat, fnc) -- variant of split +--~ for k in str:splitter(pat) do fnc(k) end +--~ end + +--~ do if lpeg then + +--~ -- this alternative is 30% faster esp when we cache them +--~ -- problem: no expressions + +--~ splitters = { } + +--~ function string:split(separator) +--~ if #self > 0 then +--~ local split = splitters[separator] +--~ if not split then +--~ -- based on code by Roberto +--~ local p = lpeg.P(separator) +--~ local c = lpeg.C((1-p)^0) +--~ split = lpeg.Ct(c*(p*c)^0) +--~ splitters[separator] = split +--~ end +--~ return split:match(self) +--~ else +--~ return { } +--~ end +--~ end + +--~ string.splitchr = string.split + +--~ function string:piecewise(separator,fnc) +--~ for _,v in pairs(self:split(separator)) do +--~ fnc(v) +--~ end +--~ end + +--~ end end + +string.chr_to_esc = { + ["%"] = "%%", + ["."] = "%.", + ["+"] = "%+", ["-"] = "%-", ["*"] = "%*", + ["^"] = "%^", ["$"] = "%$", + ["["] = "%[", ["]"] = "%]", + ["("] = "%(", [")"] = "%)", + ["{"] = "%{", ["}"] = "%}" +} + +function string:esc() -- variant 2 + return (self:gsub("(.)",string.chr_to_esc)) +end + +function string.unquote(str) + return (str:gsub("^([\"\'])(.*)%1$","%2")) +end + +function string.quote(str) + return '"' .. str:unquote() .. '"' +end + +function string:count(pattern) -- variant 3 + local n = 0 + for _ in self:gmatch(pattern) do + n = n + 1 + end + return n +end + +function string:limit(n,sentinel) + if #self > n then + sentinel = sentinel or " ..." + return self:sub(1,(n-#sentinel)) .. sentinel + else + return self + end +end + +function string:strip() + return (self:gsub("^%s*(.-)%s*$", "%1")) +end + +--~ function string.strip(str) -- slightly different +--~ return (string.gsub(string.gsub(str,"^%s*(.-)%s*$","%1"),"%s+"," ")) +--~ end + +function string:is_empty() + return not self:find("%S") +end + +function string:enhance(pattern,action) + local ok, n = true, 0 + while ok do + ok = false + self = self:gsub(pattern, function(...) + ok, n = true, n + 1 + return action(...) + end) + end + return self, n +end + +--~ function string:enhance(pattern,action) +--~ local ok, n = 0, 0 +--~ repeat +--~ self, ok = self:gsub(pattern, function(...) +--~ n = n + 1 +--~ return action(...) +--~ end) +--~ until ok == 0 +--~ return self, n +--~ end + +--~ function string:to_hex() +--~ if self then +--~ return (self:gsub("(.)",function(c) +--~ return string.format("%02X",c:byte()) +--~ end)) +--~ else +--~ return "" +--~ end +--~ end + +--~ function string:from_hex() +--~ if self then +--~ return (self:gsub("(..)",function(c) +--~ return string.char(tonumber(c,16)) +--~ end)) +--~ else +--~ return "" +--~ end +--~ end + +string.chr_to_hex = { } +string.hex_to_chr = { } + +for i=0,255 do + local c, h = string.char(i), string.format("%02X",i) + string.chr_to_hex[c], string.hex_to_chr[h] = h, c +end + +--~ function string:to_hex() +--~ if self then return (self:gsub("(.)",string.chr_to_hex)) else return "" end +--~ end + +--~ function string:from_hex() +--~ if self then return (self:gsub("(..)",string.hex_to_chr)) else return "" end +--~ end + +function string:to_hex() + return ((self or ""):gsub("(.)",string.chr_to_hex)) +end + +function string:from_hex() + return ((self or ""):gsub("(..)",string.hex_to_chr)) +end + +if not string.characters then + + local function nextchar(str, index) + index = index + 1 + return (index <= #str) and index or nil, str:sub(index,index) + end + function string:characters() + return nextchar, self, 0 + end + local function nextbyte(str, index) + index = index + 1 + return (index <= #str) and index or nil, string.byte(str:sub(index,index)) + end + function string:bytes() + return nextbyte, self, 0 + end + +end + +--~ function string:padd(n,chr) +--~ return self .. self.rep(chr or " ",n-#self) +--~ end + +function string:rpadd(n,chr) + local m = n-#self + if m > 0 then + return self .. self.rep(chr or " ",m) + else + return self + end +end + +function string:lpadd(n,chr) + local m = n-#self + if m > 0 then + return self.rep(chr or " ",m) .. self + else + return self + end +end + +string.padd = string.rpadd + +function is_number(str) + return str:find("^[%-%+]?[%d]-%.?[%d+]$") == 1 +end + +--~ print(is_number("1")) +--~ print(is_number("1.1")) +--~ print(is_number(".1")) +--~ print(is_number("-0.1")) +--~ print(is_number("+0.1")) +--~ print(is_number("-.1")) +--~ print(is_number("+.1")) + +function string:split_settings() -- no {} handling, see l-aux for lpeg variant + if self:find("=") then + local t = { } + for k,v in self:gmatch("(%a+)=([^%,]*)") do + t[k] = v + end + return t + else + return nil + end +end + +local patterns_escapes = { + ["-"] = "%-", + ["."] = "%.", + ["+"] = "%+", + ["*"] = "%*", + ["%"] = "%%", + ["("] = "%)", + [")"] = "%)", + ["["] = "%[", + ["]"] = "%]", +} + +function string:pattesc() + return (self:gsub(".",patterns_escapes)) +end + +function string:tohash() + local t = { } + for s in self:gmatch("([^, ]+)") do -- lpeg + t[s] = true + end + return t +end + + +-- filename : l-lpeg.lua +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-lpeg'] = 1.001 + +--~ l-lpeg.lua : + +--~ lpeg.digit = lpeg.R('09')^1 +--~ lpeg.sign = lpeg.S('+-')^1 +--~ lpeg.cardinal = lpeg.P(lpeg.sign^0 * lpeg.digit^1) +--~ lpeg.integer = lpeg.P(lpeg.sign^0 * lpeg.digit^1) +--~ lpeg.float = lpeg.P(lpeg.sign^0 * lpeg.digit^0 * lpeg.P('.') * lpeg.digit^1) +--~ lpeg.number = lpeg.float + lpeg.integer +--~ lpeg.oct = lpeg.P("0") * lpeg.R('07')^1 +--~ lpeg.hex = lpeg.P("0x") * (lpeg.R('09') + lpeg.R('AF'))^1 +--~ lpeg.uppercase = lpeg.P("AZ") +--~ lpeg.lowercase = lpeg.P("az") + +--~ lpeg.eol = lpeg.S('\r\n\f')^1 -- includes formfeed +--~ lpeg.space = lpeg.S(' ')^1 +--~ lpeg.nonspace = lpeg.P(1-lpeg.space)^1 +--~ lpeg.whitespace = lpeg.S(' \r\n\f\t')^1 +--~ lpeg.nonwhitespace = lpeg.P(1-lpeg.whitespace)^1 + +local hash = { } + +function lpeg.anywhere(pattern) --slightly adapted from website + return lpeg.P { lpeg.P(pattern) + 1 * lpeg.V(1) } +end + +function lpeg.startswith(pattern) --slightly adapted + return lpeg.P(pattern) +end + +--~ g = lpeg.splitter(" ",function(s) ... end) -- gmatch:lpeg = 3:2 + +function lpeg.splitter(pattern, action) + return (((1-lpeg.P(pattern))^1)/action+1)^0 +end + +local crlf = lpeg.P("\r\n") +local cr = lpeg.P("\r") +local lf = lpeg.P("\n") +local space = lpeg.S(" \t\f\v") +local newline = crlf + cr + lf +local spacing = space^0 * newline + +local empty = spacing * lpeg.Cc("") +local nonempty = lpeg.Cs((1-spacing)^1) * spacing^-1 +local content = (empty + nonempty)^1 + +local capture = lpeg.Ct(content^0) + +function string:splitlines() + return capture:match(self) +end + + +-- filename : l-table.lua +-- comment : split off from luat-lib +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-table'] = 1.001 + +table.join = table.concat + +function table.strip(tab) + local lst = { } + for k, v in ipairs(tab) do + -- s = string.gsub(v, "^%s*(.-)%s*$", "%1") + s = v:gsub("^%s*(.-)%s*$", "%1") + if s == "" then + -- skip this one + else + lst[#lst+1] = s + end + end + return lst +end + +--~ function table.sortedkeys(tab) +--~ local srt = { } +--~ for key,_ in pairs(tab) do +--~ srt[#srt+1] = key +--~ end +--~ table.sort(srt) +--~ return srt +--~ end + +function table.sortedkeys(tab) + local srt, kind = { }, 0 -- 0=unknown 1=string, 2=number 3=mixed + for key,_ in pairs(tab) do + srt[#srt+1] = key + if kind == 3 then + -- no further check + else + local tkey = type(key) + if tkey == "string" then + -- if kind == 2 then kind = 3 else kind = 1 end + kind = (kind == 2 and 3) or 1 + elseif tkey == "number" then + -- if kind == 1 then kind = 3 else kind = 2 end + kind = (kind == 1 and 3) or 2 + else + kind = 3 + end + end + end + if kind == 0 or kind == 3 then + table.sort(srt,function(a,b) return (tostring(a) < tostring(b)) end) + else + table.sort(srt) + end + return srt +end + +function table.append(t, list) + for _,v in pairs(list) do + table.insert(t,v) + end +end + +function table.prepend(t, list) + for k,v in pairs(list) do + table.insert(t,k,v) + end +end + +function table.merge(t, ...) -- first one is target + t = t or {} + local lst = {...} + for i=1,#lst do + for k, v in pairs(lst[i]) do + t[k] = v + end + end + return t +end + +function table.merged(...) + local tmp, lst = { }, {...} + for i=1,#lst do + for k, v in pairs(lst[i]) do + tmp[k] = v + end + end + return tmp +end + +function table.imerge(t, ...) + local lst = {...} + for i=1,#lst do + local nst = lst[i] + for j=1,#nst do + t[#t+1] = nst[j] + end + end + return t +end + +function table.imerged(...) + local tmp, lst = { }, {...} + for i=1,#lst do + local nst = lst[i] + for j=1,#nst do + tmp[#tmp+1] = nst[j] + end + end + return tmp +end + +if not table.fastcopy then do + + local type, pairs, getmetatable, setmetatable = type, pairs, getmetatable, setmetatable + + local function fastcopy(old) -- fast one + if old then + local new = { } + for k,v in pairs(old) do + if type(v) == "table" then + new[k] = fastcopy(v) -- was just table.copy + else + new[k] = v + end + end + local mt = getmetatable(old) + if mt then + setmetatable(new,mt) + end + return new + else + return { } + end + end + + table.fastcopy = fastcopy + +end end + +if not table.copy then do + + local type, pairs, getmetatable, setmetatable = type, pairs, getmetatable, setmetatable + + local function copy(t, tables) -- taken from lua wiki, slightly adapted + tables = tables or { } + local tcopy = {} + if not tables[t] then + tables[t] = tcopy + end + for i,v in pairs(t) do -- brrr, what happens with sparse indexed + if type(i) == "table" then + if tables[i] then + i = tables[i] + else + i = copy(i, tables) + end + end + if type(v) ~= "table" then + tcopy[i] = v + elseif tables[v] then + tcopy[i] = tables[v] + else + tcopy[i] = copy(v, tables) + end + end + local mt = getmetatable(t) + if mt then + setmetatable(tcopy,mt) + end + return tcopy + end + + table.copy = copy + +end end + +-- rougly: copy-loop : unpack : sub == 0.9 : 0.4 : 0.45 (so in critical apps, use unpack) + +function table.sub(t,i,j) + return { unpack(t,i,j) } +end + +function table.replace(a,b) + for k,v in pairs(b) do + a[k] = v + end +end + +-- slower than #t on indexed tables (#t only returns the size of the numerically indexed slice) + +function table.is_empty(t) + return not t or not next(t) +end + +function table.one_entry(t) + local n = next(t) + return n and not next(t,n) +end + +function table.starts_at(t) + return ipairs(t,1)(t,0) +end + +do + + -- one of my first exercises in lua ... + + -- 34.055.092 32.403.326 arabtype.tma + -- 1.620.614 1.513.863 lmroman10-italic.tma + -- 1.325.585 1.233.044 lmroman10-regular.tma + -- 1.248.157 1.158.903 lmsans10-regular.tma + -- 194.646 153.120 lmtypewriter10-regular.tma + -- 1.771.678 1.658.461 palatinosanscom-bold.tma + -- 1.695.251 1.584.491 palatinosanscom-regular.tma + -- 13.736.534 13.409.446 zapfinoextraltpro.tma + + -- 13.679.038 11.774.106 arabtype.tmc + -- 886.248 754.944 lmroman10-italic.tmc + -- 729.828 466.864 lmroman10-regular.tmc + -- 688.482 441.962 lmsans10-regular.tmc + -- 128.685 95.853 lmtypewriter10-regular.tmc + -- 715.929 582.985 palatinosanscom-bold.tmc + -- 669.942 540.126 palatinosanscom-regular.tmc + -- 1.560.588 1.317.000 zapfinoextraltpro.tmc + + table.serialize_functions = true + table.serialize_compact = true + table.serialize_inline = true + + local function key(k) + if type(k) == "number" then -- or k:find("^%d+$") then + return "["..k.."]" + elseif noquotes and k:find("^%a[%a%d%_]*$") then + return k + else + return '["'..k..'"]' + end + end + + local function simple_table(t) + if #t > 0 then + local n = 0 + for _,v in pairs(t) do + n = n + 1 + end + if n == #t then + local tt = { } + for i=1,#t do + local v = t[i] + local tv = type(v) + if tv == "number" or tv == "boolean" then + tt[#tt+1] = tostring(v) + elseif tv == "string" then + tt[#tt+1] = ("%q"):format(v) + else + tt = nil + break + end + end + return tt + end + end + return nil + end + + local function serialize(root,name,handle,depth,level,reduce,noquotes,indexed) + handle = handle or print + reduce = reduce or false + if depth then + depth = depth .. " " + if indexed then + handle(("%s{"):format(depth)) + else + handle(("%s%s={"):format(depth,key(name))) + end + else + depth = "" + local tname = type(name) + if tname == "string" then + if name == "return" then + handle("return {") + else + handle(name .. "={") + end + elseif tname == "number" then + handle("[" .. name .. "]={") + elseif tname == "boolean" then + if name then + handle("return {") + else + handle("{") + end + else + handle("t={") + end + end + if root and next(root) then + local compact = table.serialize_compact + local inline = compact and table.serialize_inline + local first, last = nil, 0 -- #root cannot be trusted here + if compact then + for k,v in ipairs(root) do -- NOT: for k=1,#root do (why) + if not first then first = k end + last = last + 1 + end + end + for _,k in pairs(table.sortedkeys(root)) do + local v = root[k] + local t = type(v) + if compact and first and type(k) == "number" and k >= first and k <= last then + if t == "number" then + handle(("%s %s,"):format(depth,v)) + elseif t == "string" then + if reduce and (v:find("^[%-%+]?[%d]-%.?[%d+]$") == 1) then + handle(("%s %s,"):format(depth,v)) + else + handle(("%s %q,"):format(depth,v)) + end + elseif t == "table" then + if not next(v) then + handle(("%s {},"):format(depth)) + elseif inline then + local st = simple_table(v) + if st then + handle(("%s { %s },"):format(depth,table.concat(st,", "))) + else + serialize(v,k,handle,depth,level+1,reduce,noquotes,true) + end + else + serialize(v,k,handle,depth,level+1,reduce,noquotes,true) + end + elseif t == "boolean" then + handle(("%s %s,"):format(depth,tostring(v))) + elseif t == "function" then + if table.serialize_functions then + handle(('%s loadstring(%q),'):format(depth,string.dump(v))) + else + handle(('%s "function",'):format(depth)) + end + else + handle(("%s %q,"):format(depth,tostring(v))) + end + elseif k == "__p__" then -- parent + if false then + handle(("%s __p__=nil,"):format(depth)) + end + elseif t == "number" then + handle(("%s %s=%s,"):format(depth,key(k),v)) + elseif t == "string" then + if reduce and (v:find("^[%-%+]?[%d]-%.?[%d+]$") == 1) then + handle(("%s %s=%s,"):format(depth,key(k),v)) + else + handle(("%s %s=%q,"):format(depth,key(k),v)) + end + elseif t == "table" then + if not next(v) then + handle(("%s %s={},"):format(depth,key(k))) + elseif inline then + local st = simple_table(v) + if st then + handle(("%s %s={ %s },"):format(depth,key(k),table.concat(st,", "))) + else + serialize(v,k,handle,depth,level+1,reduce,noquotes) + end + else + serialize(v,k,handle,depth,level+1,reduce,noquotes) + end + elseif t == "boolean" then + handle(("%s %s=%s,"):format(depth,key(k),tostring(v))) + elseif t == "function" then + if table.serialize_functions then + handle(('%s %s=loadstring(%q),'):format(depth,key(k),string.dump(v))) + else + handle(('%s %s="function",'):format(depth,key(k))) + end + else + handle(("%s %s=%q,"):format(depth,key(k),tostring(v))) + -- handle(('%s %s=loadstring(%q),'):format(depth,key(k),string.dump(function() return v end))) + end + end + if level > 0 then + handle(("%s},"):format(depth)) + else + handle(("%s}"):format(depth)) + end + else + handle(("%s}"):format(depth)) + end + end + + --~ name: + --~ + --~ true : return { } + --~ false : { } + --~ nil : t = { } + --~ string : string = { } + --~ 'return' : return { } + --~ number : [number] = { } + + function table.serialize(root,name,reduce,noquotes) + local t = { } + local function flush(s) + t[#t+1] = s + end + serialize(root, name, flush, nil, 0, reduce, noquotes) + return table.concat(t,"\n") + end + + function table.tohandle(handle,root,name,reduce,noquotes) + serialize(root, name, handle, nil, 0, reduce, noquotes) + end + + -- sometimes tables are real use (zapfino extra pro is some 85M) in which + -- case a stepwise serialization is nice; actually, we could consider: + -- + -- for line in table.serializer(root,name,reduce,noquotes) do + -- ...(line) + -- end + -- + -- so this is on the todo list + + table.tofile_maxtab = 2*1024 + + function table.tofile(filename,root,name,reduce,noquotes) + local f = io.open(filename,'w') + if f then + local concat = table.concat + local maxtab = table.tofile_maxtab + if maxtab > 1 then + local t = { } + local function flush(s) + t[#t+1] = s + if #t > maxtab then + f:write(concat(t,"\n"),"\n") -- hm, write(sometable) should be nice + t = { } + end + end + serialize(root, name, flush, nil, 0, reduce, noquotes) + f:write(concat(t,"\n"),"\n") + else + local function flush(s) + f:write(s,"\n") + end + serialize(root, name, flush, nil, 0, reduce, noquotes) + end + f:close() + end + end + +end + +--~ t = { +--~ b = "123", +--~ a = "x", +--~ c = 1.23, +--~ d = "1.23", +--~ e = true, +--~ f = { +--~ d = "1.23", +--~ a = "x", +--~ b = "123", +--~ c = 1.23, +--~ e = true, +--~ f = { +--~ e = true, +--~ f = { +--~ e = true +--~ }, +--~ }, +--~ }, +--~ g = function() end +--~ } + +--~ print(table.serialize(t), "\n") +--~ print(table.serialize(t,"name"), "\n") +--~ print(table.serialize(t,false), "\n") +--~ print(table.serialize(t,true), "\n") +--~ print(table.serialize(t,"name",true), "\n") +--~ print(table.serialize(t,"name",true,true), "\n") + +do + + local function flatten(t,f,complete) + for i=1,#t do + local v = t[i] + if type(v) == "table" then + if complete or type(v[1]) == "table" then + flatten(v,f,complete) + else + f[#f+1] = v + end + else + f[#f+1] = v + end + end + end + + function table.flatten(t) + local f = { } + flatten(t,f,true) + return f + end + + function table.unnest(t) -- bad name + local f = { } + flatten(t,f,false) + return f + end + + table.flatten_one_level = table.unnest + +end + +function table.insert_before_value(t,value,str) + for i=1,#t do + if t[i] == value then + table.insert(t,i,str) + return + end + end + table.insert(t,1,str) +end + +function table.insert_after_value(t,value,str) + for i=1,#t do + if t[i] == value then + table.insert(t,i+1,str) + return + end + end + t[#t+1] = str +end + +function table.are_equal(a,b,n,m) + if #a == #b then + n = n or 1 + m = m or #a + for i=n,m do + local ai, bi = a[i], b[i] + if (ai==bi) or (type(ai)=="table" and type(bi)=="table" and table.are_equal(ai,bi)) then + -- continue + else + return false + end + end + return true + else + return false + end +end + +function table.compact(t) + if t then + for k,v in pairs(t) do + if not next(v) then + t[k] = nil + end + end + end +end + +function table.tohash(t) + local h = { } + for _, v in pairs(t) do -- no ipairs here + h[v] = true + end + return h +end + +function table.fromhash(t) + local h = { } + for k, v in pairs(t) do -- no ipairs here + if v then h[#h+1] = k end + end + return h +end + +function table.contains(t, v) + if t then + for i=1, #t do + if t[i] == v then + return true + end + end + end + return false +end + +function table.count(t) + local n, e = 0, next(t) + while e do + n, e = n + 1, next(t,e) + end + return n +end + +function table.swapped(t) + local s = { } + for k, v in pairs(t) do + s[v] = k + end + return s +end + +--~ function table.are_equal(a,b) +--~ return table.serialize(a) == table.serialize(b) +--~ end + +function table.clone(t,p) -- t is optional or nil or table + if not p then + t, p = { }, t or { } + elseif not t then + t = { } + end + setmetatable(t, { __index = function(_,key) return p[key] end }) + return t +end + + +function table.hexed(t,seperator) + local tt = { } + for i=1,#t do tt[i] = string.format("0x%04X",t[i]) end + return table.concat(tt,seperator or " ") +end + +function table.reverse_hash(h) + local r = { } + for k,v in pairs(h) do + r[v] = (k:gsub(" ","")):lower() + end + return r +end + +function table.reverse(t) + local tt = { } + if #t > 0 then + for i=#t,1,-1 do + tt[#tt+1] = t[i] + end + end + return tt +end + + +-- filename : l-io.lua +-- comment : split off from luat-lib +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-io'] = 1.001 + +if string.find(os.getenv("PATH"),";") then + io.fileseparator, io.pathseparator = "\\", ";" +else + io.fileseparator, io.pathseparator = "/" , ":" +end + +function io.loaddata(filename) + local f = io.open(filename,'rb') + if f then + local data = f:read('*all') + f:close() + return data + else + return nil + end +end + +function io.savedata(filename,data,joiner) + local f = io.open(filename, "wb") + if f then + if type(data) == "table" then + f:write(table.join(data,joiner or "")) + elseif type(data) == "function" then + data(f) + else + f:write(data) + end + f:close() + end +end + +function io.exists(filename) + local f = io.open(filename) + if f == nil then + return false + else + assert(f:close()) + return true + end +end + +function io.size(filename) + local f = io.open(filename) + if f == nil then + return 0 + else + local s = f:seek("end") + assert(f:close()) + return s + end +end + +function io.noflines(f) + local n = 0 + for _ in f:lines() do + n = n + 1 + end + f:seek('set',0) + return n +end + +do + + local sb = string.byte + + local nextchar = { + [ 4] = function(f) + return f:read(1,1,1,1) + end, + [ 2] = function(f) + return f:read(1,1) + end, + [ 1] = function(f) + return f:read(1) + end, + [-2] = function(f) + local a, b = f:read(1,1) + return b, a + end, + [-4] = function(f) + local a, b, c, d = f:read(1,1,1,1) + return d, c, b, a + end + } + + function io.characters(f,n) + if f then + return nextchar[n or 1], f + else + return nil, nil + end + end + +end + +do + + local sb = string.byte + +--~ local nextbyte = { +--~ [4] = function(f) +--~ local a = f:read(1) +--~ local b = f:read(1) +--~ local c = f:read(1) +--~ local d = f:read(1) +--~ if d then +--~ return sb(a), sb(b), sb(c), sb(d) +--~ else +--~ return nil, nil, nil, nil +--~ end +--~ end, +--~ [2] = function(f) +--~ local a = f:read(1) +--~ local b = f:read(1) +--~ if b then +--~ return sb(a), sb(b) +--~ else +--~ return nil, nil +--~ end +--~ end, +--~ [1] = function (f) +--~ local a = f:read(1) +--~ if a then +--~ return sb(a) +--~ else +--~ return nil +--~ end +--~ end, +--~ [-2] = function (f) +--~ local a = f:read(1) +--~ local b = f:read(1) +--~ if b then +--~ return sb(b), sb(a) +--~ else +--~ return nil, nil +--~ end +--~ end, +--~ [-4] = function(f) +--~ local a = f:read(1) +--~ local b = f:read(1) +--~ local c = f:read(1) +--~ local d = f:read(1) +--~ if d then +--~ return sb(d), sb(c), sb(b), sb(a) +--~ else +--~ return nil, nil, nil, nil +--~ end +--~ end +--~ } + + local nextbyte = { + [4] = function(f) + local a, b, c, d = f:read(1,1,1,1) + if d then + return sb(a), sb(b), sb(c), sb(d) + else + return nil, nil, nil, nil + end + end, + [2] = function(f) + local a, b = f:read(1,1) + if b then + return sb(a), sb(b) + else + return nil, nil + end + end, + [1] = function (f) + local a = f:read(1) + if a then + return sb(a) + else + return nil + end + end, + [-2] = function (f) + local a, b = f:read(1,1) + if b then + return sb(b), sb(a) + else + return nil, nil + end + end, + [-4] = function(f) + local a, b, c, d = f:read(1,1,1,1) + if d then + return sb(d), sb(c), sb(b), sb(a) + else + return nil, nil, nil, nil + end + end + } + + function io.bytes(f,n) + if f then + return nextbyte[n or 1], f + else + return nil, nil + end + end + +end + +function io.ask(question,default,options) + while true do + io.write(question) + if options then + io.write(string.format(" [%s]",table.concat(options,"|"))) + end + if default then + io.write(string.format(" [%s]",default)) + end + io.write(string.format(" ")) + local answer = io.read() + answer = answer:gsub("^%s*(.*)%s*$","%1") + if answer == "" and default then + return default + elseif not options then + return answer + else + for _,v in pairs(options) do + if v == answer then + return answer + end + end + local pattern = "^" .. answer + for _,v in pairs(options) do + if v:find(pattern) then + return v + end + end + end + end +end + + +-- filename : l-md5.lua +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-md5'] = 1.001 + +if md5 then do + + local function convert(str,fmt) + return (string.gsub(md5.sum(str),".",function(chr) return string.format(fmt,string.byte(chr)) end)) + end + + if not md5.HEX then function md5.HEX(str) return convert(str,"%02X") end end + if not md5.hex then function md5.hex(str) return convert(str,"%02x") end end + if not md5.dec then function md5.dec(str) return convert(str,"%03i") end end + +end end + + +-- filename : l-number.lua +-- comment : split off from luat-lib +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-number'] = 1.001 + +if not number then number = { } end + +-- a,b,c,d,e,f = number.toset(100101) + +function number.toset(n) + return (tostring(n)):match("(.?)(.?)(.?)(.?)(.?)(.?)(.?)(.?)") +end + +local format = string.format + +function number.toevenhex(n) + local s = format("%X",n) + if #s % 2 == 0 then + return s + else + return "0" .. s + end +end + +-- the lpeg way is slower on 8 digits, but faster on 4 digits, some 7.5% +-- on +-- +-- for i=1,1000000 do +-- local a,b,c,d,e,f,g,h = number.toset(12345678) +-- local a,b,c,d = number.toset(1234) +-- local a,b,c = number.toset(123) +-- end +-- +-- of course dedicated "(.)(.)(.)(.)" matches are even faster + +do + local one = lpeg.C(1-lpeg.S(''))^1 + + function number.toset(n) + return one:match(tostring(n)) + end +end + + + +-- filename : l-set.lua +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-set'] = 1.001 + +if not set then set = { } end + +do + + local nums = { } + local tabs = { } + local concat = table.concat + + set.create = table.tohash + + function set.tonumber(t) + if next(t) then + local s = "" + -- we could save mem by sorting, but it slows down + for k, v in pairs(t) do + if v then + -- why bother about the leading space + s = s .. " " .. k + end + end + if not nums[s] then + tabs[#tabs+1] = t + nums[s] = #tabs + end + return nums[s] + else + return 0 + end + end + + function set.totable(n) + if n == 0 then + return { } + else + return tabs[n] or { } + end + end + + function set.contains(n,s) + if type(n) == "table" then + return n[s] + elseif n == 0 then + return false + else + local t = tabs[n] + return t and t[s] + end + end + +end + +--~ local c = set.create{'aap','noot','mies'} +--~ local s = set.tonumber(c) +--~ local t = set.totable(s) +--~ print(t['aap']) +--~ local c = set.create{'zus','wim','jet'} +--~ local s = set.tonumber(c) +--~ local t = set.totable(s) +--~ print(t['aap']) +--~ print(t['jet']) +--~ print(set.contains(t,'jet')) +--~ print(set.contains(t,'aap')) + + + +-- filename : l-os.lua +-- comment : split off from luat-lib +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-os'] = 1.001 + +function os.resultof(command) + return io.popen(command,"r"):read("*all") +end + +if not os.exec then os.exec = os.execute end +if not os.spawn then os.spawn = os.execute end + +--~ os.type : windows | unix (new, we already guessed os.platform) +--~ os.name : windows | msdos | linux | macosx | solaris | .. | generic (new) + +if not io.fileseparator then + if string.find(os.getenv("PATH"),";") then + io.fileseparator, io.pathseparator, os.platform = "\\", ";", os.type or "windows" + else + io.fileseparator, io.pathseparator, os.platform = "/" , ":", os.type or "unix" + end +end + +os.platform = os.platform or os.type or (io.pathseparator == ";" and "windows") or "unix" + +function os.launch(str) + if os.platform == "windows" then + os.execute("start " .. str) -- os.spawn ? + else + os.execute(str .. " &") -- os.spawn ? + end +end + +if not os.setenv then + function os.setenv() return false end +end + +if not os.times then + -- utime = user time + -- stime = system time + -- cutime = children user time + -- cstime = children system time + function os.times() + return { + utime = os.gettimeofday(), -- user + stime = 0, -- system + cutime = 0, -- children user + cstime = 0, -- children system + } + end +end + +os.gettimeofday = os.gettimeofday or os.clock + +do + local startuptime = os.gettimeofday() + function os.runtime() + return os.gettimeofday() - startuptime + end +end + +--~ print(os.gettimeofday()-os.time()) +--~ os.sleep(1.234) +--~ print (">>",os.runtime()) +--~ print(os.date("%H:%M:%S",os.gettimeofday())) +--~ print(os.date("%H:%M:%S",os.time())) + + +-- filename : l-file.lua +-- comment : split off from luat-lib +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-file'] = 1.001 + +if not file then file = { } end + +function file.removesuffix(filename) + return filename:gsub("%.[%a%d]+$", "") +end + +function file.addsuffix(filename, suffix) + if not filename:find("%.[%a%d]+$") then + return filename .. "." .. suffix + else + return filename + end +end + +function file.replacesuffix(filename, suffix) + if not filename:find("%.[%a%d]+$") then + return filename .. "." .. suffix + else + return (filename:gsub("%.[%a%d]+$","."..suffix)) + end +end + +function file.dirname(name) + return name:match("^(.+)[/\\].-$") or "" +end + +function file.basename(name) + return name:match("^.+[/\\](.-)$") or name +end + +function file.nameonly(name) + return ((name:match("^.+[/\\](.-)$") or name):gsub("%..*$","")) +end + +function file.extname(name) + return name:match("^.+%.([^/\\]-)$") or "" +end + +file.suffix = file.extname + +function file.stripsuffix(name) + return (name:gsub("%.[%a%d]+$","")) +end + +--~ function file.join(...) +--~ local t = { ... } +--~ for i=1,#t do +--~ t[i] = (t[i]:gsub("\\","/")):gsub("/+$","") +--~ end +--~ return table.concat(t,"/") +--~ end + +--~ print(file.join("x/","/y")) +--~ print(file.join("http://","/y")) +--~ print(file.join("http://a","/y")) +--~ print(file.join("http:///a","/y")) +--~ print(file.join("//nas-1","/y")) + +function file.join(...) + local pth = table.concat({...},"/") + pth = pth:gsub("\\","/") + local a, b = pth:match("^(.*://)(.*)$") + if a and b then + return a .. b:gsub("//+","/") + end + a, b = pth:match("^(//)(.*)$") + if a and b then + return a .. b:gsub("//+","/") + end + return (pth:gsub("//+","/")) +end + +function file.is_writable(name) + local f = io.open(name, 'w') + if f then + f:close() + return true + else + return false + end +end + +function file.is_readable(name) + local f = io.open(name,'r') + if f then + f:close() + return true + else + return false + end +end + +--~ function file.split_path(str) +--~ if str:find(';') then +--~ return str:splitchr(";") +--~ else +--~ return str:splitchr(io.pathseparator) +--~ end +--~ end + +-- todo: lpeg + +function file.split_path(str) + local t = { } + str = str:gsub("\\", "/") + str = str:gsub("(%a):([;/])", "%1\001%2") + for name in str:gmatch("([^;:]+)") do + if name ~= "" then + name = name:gsub("\001",":") + t[#t+1] = name + end + end + return t +end + +function file.join_path(tab) + return table.concat(tab,io.pathseparator) -- can have trailing // +end + +--~ print('test' .. " == " .. file.collapse_path("test")) +--~ print("test/test" .. " == " .. file.collapse_path("test/test")) +--~ print("test/test/test" .. " == " .. file.collapse_path("test/test/test")) +--~ print("test/test" .. " == " .. file.collapse_path("test/../test/test")) +--~ print("test" .. " == " .. file.collapse_path("test/../test")) +--~ print("../test" .. " == " .. file.collapse_path("../test")) +--~ print("../test/" .. " == " .. file.collapse_path("../test/")) +--~ print("a/a" .. " == " .. file.collapse_path("a/b/c/../../a")) + +--~ function file.collapse_path(str) +--~ local ok, n = false, 0 +--~ while not ok do +--~ ok = true +--~ str, n = str:gsub("[^%./]+/%.%./", function(s) +--~ ok = false +--~ return "" +--~ end) +--~ end +--~ return (str:gsub("/%./","/")) +--~ end + +function file.collapse_path(str) + local n = 1 + while n > 0 do + str, n = str:gsub("([^/%.]+/%.%./)","") + end + return (str:gsub("/%./","/")) +end + +function file.robustname(str) + return (str:gsub("[^%a%d%/%-%.\\]+","-")) +end + +file.readdata = io.loaddata +file.savedata = io.savedata + +function file.copy(oldname,newname) + file.savedata(newname,io.loaddata(oldname)) +end + + +-- filename : l-dir.lua +-- comment : split off from luat-lib +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-dir'] = 1.001 + +dir = { } + +-- optimizing for no string.find (*) does not save time + +if lfs then do + +--~ local attributes = lfs.attributes +--~ local walkdir = lfs.dir +--~ +--~ local function glob_pattern(path,patt,recurse,action) +--~ local ok, scanner = xpcall(function() return walkdir(path) end, function() end) -- kepler safe +--~ if ok and type(scanner) == "function" then +--~ if not path:find("/$") then path = path .. '/' end +--~ for name in scanner do +--~ local full = path .. name +--~ local mode = attributes(full,'mode') +--~ if mode == 'file' then +--~ if name:find(patt) then +--~ action(full) +--~ end +--~ elseif recurse and (mode == "directory") and (name ~= '.') and (name ~= "..") then +--~ glob_pattern(full,patt,recurse,action) +--~ end +--~ end +--~ end +--~ end +--~ +--~ dir.glob_pattern = glob_pattern +--~ +--~ local function glob(pattern, action) +--~ local t = { } +--~ local action = action or function(name) t[#t+1] = name end +--~ local path, patt = pattern:match("^(.*)/*%*%*/*(.-)$") +--~ local recurse = path and patt +--~ if not recurse then +--~ path, patt = pattern:match("^(.*)/(.-)$") +--~ if not (path and patt) then +--~ path, patt = '.', pattern +--~ end +--~ end +--~ patt = patt:gsub("([%.%-%+])", "%%%1") +--~ patt = patt:gsub("%*", ".*") +--~ patt = patt:gsub("%?", ".") +--~ patt = "^" .. patt .. "$" +--~ -- print('path: ' .. path .. ' | pattern: ' .. patt .. ' | recurse: ' .. tostring(recurse)) +--~ glob_pattern(path,patt,recurse,action) +--~ return t +--~ end +--~ +--~ dir.glob = glob + + local attributes = lfs.attributes + local walkdir = lfs.dir + + local function glob_pattern(path,patt,recurse,action) + local ok, scanner + if path == "/" then + ok, scanner = xpcall(function() return walkdir(path..".") end, function() end) -- kepler safe + else + ok, scanner = xpcall(function() return walkdir(path) end, function() end) -- kepler safe + end + if ok and type(scanner) == "function" then + if not path:find("/$") then path = path .. '/' end + for name in scanner do + local full = path .. name + local mode = attributes(full,'mode') + if mode == 'file' then + if full:find(patt) then + action(full) + end + elseif recurse and (mode == "directory") and (name ~= '.') and (name ~= "..") then + glob_pattern(full,patt,recurse,action) + end + end + end + end + + dir.glob_pattern = glob_pattern + + --~ local function glob(pattern, action) + --~ local t = { } + --~ local path, rest, patt, recurse + --~ local action = action or function(name) t[#t+1] = name end + --~ local pattern = pattern:gsub("^%*%*","./**") + --~ local pattern = pattern:gsub("/%*/","/**/") + --~ path, rest = pattern:match("^(/)(.-)$") + --~ if path then + --~ path = path + --~ else + --~ path, rest = pattern:match("^([^/]*)/(.-)$") + --~ end + --~ if rest then + --~ patt = rest:gsub("([%.%-%+])", "%%%1") + --~ end + --~ patt = patt:gsub("%*", "[^/]*") + --~ patt = patt:gsub("%?", "[^/]") + --~ patt = patt:gsub("%[%^/%]%*%[%^/%]%*", ".*") + --~ if path == "" then path = "." end + --~ recurse = patt:find("%.%*/") ~= nil + --~ glob_pattern(path,patt,recurse,action) + --~ return t + --~ end + + local P, S, R, C, Cc, Cs, Ct, Cv, V = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.Cc, lpeg.Cs, lpeg.Ct, lpeg.Cv, lpeg.V + + local pattern = Ct { + [1] = (C(P(".") + P("/")^1) + C(R("az","AZ") * P(":") * P("/")^0) + Cc("./")) * V(2) * V(3), + [2] = C(((1-S("*?/"))^0 * P("/"))^0), + [3] = C(P(1)^0) + } + + local filter = Cs ( ( + P("**") / ".*" + + P("*") / "[^/]*" + + P("?") / "[^/]" + + P(".") / "%%." + + P("+") / "%%+" + + P("-") / "%%-" + + P(1) + )^0 ) + + local function glob(str,t) + if type(str) == "table" then + local t = t or { } + for _, s in ipairs(str) do + glob(s,t) + end + return t + else + local split = pattern:match(str) + if split then + local t = t or { } + local action = action or function(name) t[#t+1] = name end + local root, path, base = split[1], split[2], split[3] + local recurse = base:find("**") + local start = root .. path + local result = filter:match(start .. base) + glob_pattern(start,result,recurse,action) + return t + else + return { } + end + end + end + + dir.glob = glob + + --~ list = dir.glob("**/*.tif") + --~ list = dir.glob("/**/*.tif") + --~ list = dir.glob("./**/*.tif") + --~ list = dir.glob("oeps/**/*.tif") + --~ list = dir.glob("/oeps/**/*.tif") + + local function globfiles(path,recurse,func,files) -- func == pattern or function + if type(func) == "string" then + local s = func -- alas, we need this indirect way + func = function(name) return name:find(s) end + end + files = files or { } + for name in walkdir(path) do + if name:find("^%.") then + --- skip + elseif attributes(name,'mode') == "directory" then + if recurse then + globfiles(path .. "/" .. name,recurse,func,files) + end + elseif func then + if func(name) then + files[#files+1] = path .. "/" .. name + end + else + files[#files+1] = path .. "/" .. name + end + end + return files + end + + dir.globfiles = globfiles + + -- t = dir.glob("c:/data/develop/context/sources/**/????-*.tex") + -- t = dir.glob("c:/data/develop/tex/texmf/**/*.tex") + -- t = dir.glob("c:/data/develop/context/texmf/**/*.tex") + -- t = dir.glob("f:/minimal/tex/**/*") + -- print(dir.ls("f:/minimal/tex/**/*")) + -- print(dir.ls("*.tex")) + + function dir.ls(pattern) + return table.concat(glob(pattern),"\n") + end + + --~ mkdirs("temp") + --~ mkdirs("a/b/c") + --~ mkdirs(".","/a/b/c") + --~ mkdirs("a","b","c") + + local make_indeed = true -- false + + if string.find(os.getenv("PATH"),";") then + + function dir.mkdirs(...) + local str, pth = "", "" + for _, s in ipairs({...}) do + if s ~= "" then + if str ~= "" then + str = str .. "/" .. s + else + str = s + end + end + end + local first, middle, last + local drive = false + first, middle, last = str:match("^(//)(//*)(.*)$") + if first then + -- empty network path == local path + else + first, last = str:match("^(//)/*(.-)$") + if first then + middle, last = str:match("([^/]+)/+(.-)$") + if middle then + pth = "//" .. middle + else + pth = "//" .. last + last = "" + end + else + first, middle, last = str:match("^([a-zA-Z]:)(/*)(.-)$") + if first then + pth, drive = first .. middle, true + else + middle, last = str:match("^(/*)(.-)$") + if not middle then + last = str + end + end + end + end + for s in last:gmatch("[^/]+") do + if pth == "" then + pth = s + elseif drive then + pth, drive = pth .. s, false + else + pth = pth .. "/" .. s + end + if make_indeed and not lfs.isdir(pth) then + lfs.mkdir(pth) + end + end + return pth, (lfs.isdir(pth) == true) + end + +--~ print(dir.mkdirs("","","a","c")) +--~ print(dir.mkdirs("a")) +--~ print(dir.mkdirs("a:")) +--~ print(dir.mkdirs("a:/b/c")) +--~ print(dir.mkdirs("a:b/c")) +--~ print(dir.mkdirs("a:/bbb/c")) +--~ print(dir.mkdirs("/a/b/c")) +--~ print(dir.mkdirs("/aaa/b/c")) +--~ print(dir.mkdirs("//a/b/c")) +--~ print(dir.mkdirs("///a/b/c")) +--~ print(dir.mkdirs("a/bbb//ccc/")) + + function dir.expand_name(str) + local first, nothing, last = str:match("^(//)(//*)(.*)$") + if first then + first = lfs.currentdir() .. "/" + first = first:gsub("\\","/") + end + if not first then + first, last = str:match("^(//)/*(.*)$") + end + if not first then + first, last = str:match("^([a-zA-Z]:)(.*)$") + if first and not last:find("^/") then + local d = lfs.currentdir() + if lfs.chdir(first) then + first = lfs.currentdir() + first = first:gsub("\\","/") + end + lfs.chdir(d) + end + end + if not first then + first, last = lfs.currentdir(), str + first = first:gsub("\\","/") + end + last = last:gsub("//","/") + last = last:gsub("/%./","/") + last = last:gsub("^/*","") + first = first:gsub("/*$","") + if last == "" then + return first + else + return first .. "/" .. last + end + end + + else + + function dir.mkdirs(...) + local str, pth = "", "" + for _, s in ipairs({...}) do + if s ~= "" then + if str ~= "" then + str = str .. "/" .. s + else + str = s + end + end + end + str = str:gsub("/+","/") + if str:find("^/") then + pth = "/" + for s in str:gmatch("[^/]+") do + local first = (pth == "/") + if first then + pth = pth .. s + else + pth = pth .. "/" .. s + end + if make_indeed and not first and not lfs.isdir(pth) then + lfs.mkdir(pth) + end + end + else + pth = "." + for s in str:gmatch("[^/]+") do + pth = pth .. "/" .. s + if make_indeed and not lfs.isdir(pth) then + lfs.mkdir(pth) + end + end + end + return pth, (lfs.isdir(pth) == true) + end + +--~ print(dir.mkdirs("","","a","c")) +--~ print(dir.mkdirs("a")) +--~ print(dir.mkdirs("/a/b/c")) +--~ print(dir.mkdirs("/aaa/b/c")) +--~ print(dir.mkdirs("//a/b/c")) +--~ print(dir.mkdirs("///a/b/c")) +--~ print(dir.mkdirs("a/bbb//ccc/")) + + function dir.expand_name(str) + if not str:find("^/") then + str = lfs.currentdir() .. "/" .. str + end + str = str:gsub("//","/") + str = str:gsub("/%./","/") + return str + end + + end + + dir.makedirs = dir.mkdirs + +end end + + +-- filename : l-boolean.lua +-- comment : split off from luat-lib +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-boolean'] = 1.001 +if not boolean then boolean = { } end + +function boolean.tonumber(b) + if b then return 1 else return 0 end +end + +function toboolean(str,tolerant) + if tolerant then + local tstr = type(str) + if tstr == "string" then + return str == "true" or str == "yes" or str == "on" or str == "1" + elseif tstr == "number" then + return tonumber(str) ~= 0 + elseif tstr == "nil" then + return false + else + return str + end + elseif str == "true" then + return true + elseif str == "false" then + return false + else + return str + end +end + +function string.is_boolean(str) + if type(str) == "string" then + if str == "true" or str == "yes" or str == "on" then + return true + elseif str == "false" or str == "no" or str == "off" then + return false + end + end + return nil +end + +function boolean.alwaystrue() + return true +end + +function boolean.falsetrue() + return false +end + + +if not modules then modules = { } end modules ['l-xml'] = { + version = 1.001, + comment = "this module is the basis for the lxml-* ones", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- RJ: key=value ... lpeg.Ca(lpeg.Cc({}) * (pattern-producing-key-and-value / rawset)^0) + +-- some code may move to l-xmlext + +--[[ldx-- +<p>The parser used here is inspired by the variant discussed in the lua book, but +handles comment and processing instructions, has a different structure, provides +parent access; a first version used different tricky but was less optimized to we +went this route. First we had a find based parser, now we have an <l n='lpeg'/> based one. +The find based parser can be found in l-xml-edu.lua along with other older code.</p> + +<p>Expecially the lpath code is experimental, we will support some of xpath, but +only things that make sense for us; as compensation it is possible to hook in your +own functions. Apart from preprocessing content for <l n='context'/> we also need +this module for process management, like handling <l n='ctx'/> and <l n='rlx'/> +files.</p> + +<typing> +a/b/c /*/c +a/b/c/first() a/b/c/last() a/b/c/index(n) a/b/c/index(-n) +a/b/c/text() a/b/c/text(1) a/b/c/text(-1) a/b/c/text(n) +</typing> + +<p>Beware, the interface may change. For instance at, ns, tg, dt may get more +verbose names. Once the code is stable we will also remove some tracing and +optimize the code.</p> +--ldx]]-- + +xml = xml or { } +tex = tex or { } + +xml.trace_lpath = false +xml.trace_print = false +xml.trace_remap = false + +local format, concat = string.format, table.concat + +--~ local pairs, next, type = pairs, next, type + +-- todo: some things per xml file, liek namespace remapping + +--[[ldx-- +<p>First a hack to enable namespace resolving. A namespace is characterized by +a <l n='url'/>. The following function associates a namespace prefix with a +pattern. We use <l n='lpeg'/>, which in this case is more than twice as fast as a +find based solution where we loop over an array of patterns. Less code and +much cleaner.</p> +--ldx]]-- + +xml.xmlns = xml.xmlns or { } + +do + + local check = lpeg.P(false) + local parse = check + + --[[ldx-- + <p>The next function associates a namespace prefix with an <l n='url'/>. This + normally happens independent of parsing.</p> + + <typing> + xml.registerns("mml","mathml") + </typing> + --ldx]]-- + + function xml.registerns(namespace, pattern) -- pattern can be an lpeg + check = check + lpeg.C(lpeg.P(pattern:lower())) / namespace + parse = lpeg.P { lpeg.P(check) + 1 * lpeg.V(1) } + end + + --[[ldx-- + <p>The next function also registers a namespace, but this time we map a + given namespace prefix onto a registered one, using the given + <l n='url'/>. This used for attributes like <t>xmlns:m</t>.</p> + + <typing> + xml.checkns("m","http://www.w3.org/mathml") + </typing> + --ldx]]-- + + function xml.checkns(namespace,url) + local ns = parse:match(url:lower()) + if ns and namespace ~= ns then + xml.xmlns[namespace] = ns + end + end + + --[[ldx-- + <p>Next we provide a way to turn an <l n='url'/> into a registered + namespace. This used for the <t>xmlns</t> attribute.</p> + + <typing> + resolvedns = xml.resolvens("http://www.w3.org/mathml") + </typing> + + This returns <t>mml</t>. + --ldx]]-- + + function xml.resolvens(url) + return parse:match(url:lower()) or "" + end + + --[[ldx-- + <p>A namespace in an element can be remapped onto the registered + one efficiently by using the <t>xml.xmlns</t> table.</p> + --ldx]]-- + +end + +--[[ldx-- +<p>This version uses <l n='lpeg'/>. We follow the same approach as before, stack and top and +such. This version is about twice as fast which is mostly due to the fact that +we don't have to prepare the stream for cdata, doctype etc etc. This variant is +is dedicated to Luigi Scarso, who challenged me with 40 megabyte <l n='xml'/> files that +took 12.5 seconds to load (1.5 for file io and the rest for tree building). With +the <l n='lpeg'/> implementation we got that down to less 7.3 seconds. Loading the 14 +<l n='context'/> interface definition files (2.6 meg) went down from 1.05 seconds to 0.55.</p> + +<p>Next comes the parser. The rather messy doctype definition comes in many +disguises so it is no surprice that later on have to dedicate quite some +<l n='lpeg'/> code to it.</p> + +<typing> +<!DOCTYPE Something PUBLIC "... ..." "..." [ ... ] > +<!DOCTYPE Something PUBLIC "... ..." "..." > +<!DOCTYPE Something SYSTEM "... ..." [ ... ] > +<!DOCTYPE Something SYSTEM "... ..." > +<!DOCTYPE Something [ ... ] > +<!DOCTYPE Something > +</typing> + +<p>The code may look a bit complex but this is mostly due to the fact that we +resolve namespaces and attach metatables. There is only one public function:</p> + +<typing> +local x = xml.convert(somestring) +</typing> + +<p>An optional second boolean argument tells this function not to create a root +element.</p> +--ldx]]-- + +xml.strip_cm_and_dt = false -- an extra global flag, in case we have many includes + +do + + -- not just one big nested table capture (lpeg overflow) + + local remove, nsremap, resolvens = table.remove, xml.xmlns, xml.resolvens + + local stack, top, dt, at, xmlns, errorstr, entities = {}, {}, {}, {}, {}, nil, {} + + local mt = { __tostring = xml.text } + + function xml.check_error(top,toclose) + return "" + end + + local strip = false + local cleanup = false + + function xml.set_text_cleanup(fnc) + cleanup = fnc + end + + local function add_attribute(namespace,tag,value) + if tag == "xmlns" then + xmlns[#xmlns+1] = resolvens(value) + at[tag] = value + elseif namespace == "xmlns" then + xml.checkns(tag,value) + at["xmlns:" .. tag] = value + else + at[tag] = value + end + end + local function add_begin(spacing, namespace, tag) + if #spacing > 0 then + dt[#dt+1] = spacing + end + local resolved = (namespace == "" and xmlns[#xmlns]) or nsremap[namespace] or namespace + top = { ns=namespace or "", rn=resolved, tg=tag, at=at, dt={}, __p__ = stack[#stack] } + setmetatable(top, mt) + dt = top.dt + stack[#stack+1] = top + at = { } + end + local function add_end(spacing, namespace, tag) + if #spacing > 0 then + dt[#dt+1] = spacing + end + local toclose = remove(stack) + top = stack[#stack] + if #stack < 1 then + errorstr = format("nothing to close with %s %s", tag, xml.check_error(top,toclose) or "") + elseif toclose.tg ~= tag then -- no namespace check + errorstr = format("unable to close %s with %s %s", toclose.tg, tag, xml.check_error(top,toclose) or "") + end + dt = top.dt + dt[#dt+1] = toclose + if at.xmlns then + remove(xmlns) + end + end + local function add_empty(spacing, namespace, tag) + if #spacing > 0 then + dt[#dt+1] = spacing + end + local resolved = (namespace == "" and xmlns[#xmlns]) or nsremap[namespace] or namespace + top = stack[#stack] + dt = top.dt + local t = { ns=namespace or "", rn=resolved, tg=tag, at=at, dt={}, __p__ = top } + dt[#dt+1] = t + setmetatable(t, mt) + at = { } + if at.xmlns then + remove(xmlns) + end + end + local function add_text(text) + if cleanup and #text > 0 then + dt[#dt+1] = cleanup(text) + else + dt[#dt+1] = text + end + end + local function add_special(what, spacing, text) + if #spacing > 0 then + dt[#dt+1] = spacing + end + if strip and (what == "@cm@" or what == "@dt@") then + -- forget it + else + dt[#dt+1] = { special=true, ns="", tg=what, dt={text} } + end + end + local function set_message(txt) + errorstr = "garbage at the end of the file: " .. txt:gsub("([ \n\r\t]*)","") + end + + local P, S, R, C, V = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V + + local space = S(' \r\n\t') + local open = P('<') + local close = P('>') + local squote = S("'") + local dquote = S('"') + local equal = P('=') + local slash = P('/') + local colon = P(':') + local valid = R('az', 'AZ', '09') + S('_-.') + local name_yes = C(valid^1) * colon * C(valid^1) + local name_nop = C(P(true)) * C(valid^1) + local name = name_yes + name_nop + + local utfbom = P('\000\000\254\255') + P('\255\254\000\000') + + P('\255\254') + P('\254\255') + P('\239\187\191') -- no capture + + local spacing = C(space^0) + local justtext = C((1-open)^1) + local somespace = space^1 + local optionalspace = space^0 + + local value = (squote * C((1 - squote)^0) * squote) + (dquote * C((1 - dquote)^0) * dquote) + local attribute = (somespace * name * optionalspace * equal * optionalspace * value) / add_attribute + local attributes = attribute^0 + + local text = justtext / add_text + local balanced = P { "[" * ((1 - S"[]") + V(1))^0 * "]" } -- taken from lpeg manual, () example + + local emptyelement = (spacing * open * name * attributes * optionalspace * slash * close) / add_empty + local beginelement = (spacing * open * name * attributes * optionalspace * close) / add_begin + local endelement = (spacing * open * slash * name * optionalspace * close) / add_end + + local begincomment = open * P("!--") + local endcomment = P("--") * close + local begininstruction = open * P("?") + local endinstruction = P("?") * close + local begincdata = open * P("![CDATA[") + local endcdata = P("]]") * close + + local someinstruction = C((1 - endinstruction)^0) + local somecomment = C((1 - endcomment )^0) + local somecdata = C((1 - endcdata )^0) + + function entity(k,v) entities[k] = v end + + local begindoctype = open * P("!DOCTYPE") + local enddoctype = close + local beginset = P("[") + local endset = P("]") + local doctypename = C((1-somespace)^0) + local elementdoctype = optionalspace * P("<!ELEMENT") * (1-close)^0 * close + local entitydoctype = optionalspace * P("<!ENTITY") * somespace * (doctypename * somespace * value)/entity * optionalspace * close + local publicdoctype = doctypename * somespace * P("PUBLIC") * somespace * value * somespace * value * somespace + local systemdoctype = doctypename * somespace * P("SYSTEM") * somespace * value * somespace + local definitiondoctype= doctypename * somespace * beginset * P(elementdoctype + entitydoctype)^0 * optionalspace * endset + local simpledoctype = (1-close)^1 -- * balanced^0 + local somedoctype = C((somespace * (publicdoctype + systemdoctype + definitiondoctype + simpledoctype) * optionalspace)^0) + + local instruction = (spacing * begininstruction * someinstruction * endinstruction) / function(...) add_special("@pi@",...) end + local comment = (spacing * begincomment * somecomment * endcomment ) / function(...) add_special("@cm@",...) end + local cdata = (spacing * begincdata * somecdata * endcdata ) / function(...) add_special("@cd@",...) end + local doctype = (spacing * begindoctype * somedoctype * enddoctype ) / function(...) add_special("@dt@",...) end + + -- nicer but slower: + -- + -- local instruction = (lpeg.Cc("@pi@") * spacing * begininstruction * someinstruction * endinstruction) / add_special + -- local comment = (lpeg.Cc("@cm@") * spacing * begincomment * somecomment * endcomment ) / add_special + -- local cdata = (lpeg.Cc("@cd@") * spacing * begincdata * somecdata * endcdata ) / add_special + -- local doctype = (lpeg.Cc("@dt@") * spacing * begindoctype * somedoctype * enddoctype ) / add_special + + local trailer = space^0 * (justtext/set_message)^0 + + -- comment + emptyelement + text + cdata + instruction + V("parent"), -- 6.5 seconds on 40 MB database file + -- text + comment + emptyelement + cdata + instruction + V("parent"), -- 5.8 + -- text + V("parent") + emptyelement + comment + cdata + instruction, -- 5.5 + + local grammar = P { "preamble", + preamble = utfbom^0 * instruction^0 * (doctype + comment + instruction)^0 * V("parent") * trailer, + parent = beginelement * V("children")^0 * endelement, + children = text + V("parent") + emptyelement + comment + cdata + instruction, + } + + -- todo: xml.new + properties like entities and strip and such (store in root) + + function xml.convert(data, no_root, strip_cm_and_dt, given_entities) -- maybe use table met k/v (given_entities may disapear) + strip = strip_cm_and_dt or xml.strip_cm_and_dt + stack, top, at, xmlns, errorstr, result, entities = {}, {}, {}, {}, nil, nil, given_entities or {} + stack[#stack+1] = top + top.dt = { } + dt = top.dt + if not data or data == "" then + errorstr = "empty xml file" + elseif not grammar:match(data) then + errorstr = "invalid xml file" + else + errorstr = "" + end + if errorstr and errorstr ~= "" then + result = { dt = { { ns = "", tg = "error", dt = { errorstr }, at={}, er = true } }, error = true } + setmetatable(stack, mt) + if xml.error_handler then xml.error_handler("load",errorstr) end + else + result = stack[1] + end + if not no_root then + result = { special = true, ns = "", tg = '@rt@', dt = result.dt, at={}, entities = entities } + setmetatable(result, mt) + local rdt = result.dt + for k=1,#rdt do + local v = rdt[k] + if type(v) == "table" and not v.special then -- always table -) + result.ri = k -- rootindex + break + end + end + end + return result + end + + --[[ldx-- + <p>Packaging data in an xml like table is done with the following + function. Maybe it will go away (when not used).</p> + --ldx]]-- + + function xml.is_valid(root) + return root and root.dt and root.dt[1] and type(root.dt[1]) == "table" and not root.dt[1].er + end + + function xml.package(tag,attributes,data) + local ns, tg = tag:match("^(.-):?([^:]+)$") + local t = { ns = ns, tg = tg, dt = data or "", at = attributes or {} } + setmetatable(t, mt) + return t + end + + function xml.is_valid(root) + return root and not root.error + end + + xml.error_handler = (logs and logs.report) or (input and input.report) or print + +end + +--[[ldx-- +<p>We cannot load an <l n='lpeg'/> from a filehandle so we need to load +the whole file first. The function accepts a string representing +a filename or a file handle.</p> +--ldx]]-- + +function xml.load(filename) + if type(filename) == "string" then + local f = io.open(filename,'r') + if f then + local root = xml.convert(f:read("*all")) + f:close() + return root + else + return xml.convert("") + end + elseif filename then -- filehandle + return xml.convert(filename:read("*all")) + else + return xml.convert("") + end +end + +--[[ldx-- +<p>When we inject new elements, we need to convert strings to +valid trees, which is what the next function does.</p> +--ldx]]-- + +function xml.toxml(data) + if type(data) == "string" then + local root = { xml.convert(data,true) } + return (#root > 1 and root) or root[1] + else + return data + end +end + +--[[ldx-- +<p>For copying a tree we use a dedicated function instead of the +generic table copier. Since we know what we're dealing with we +can speed up things a bit. The second argument is not to be used!</p> +--ldx]]-- + +do + + function copy(old,tables) + if old then + tables = tables or { } + local new = { } + if not tables[old] then + tables[old] = new + end + for k,v in pairs(old) do + new[k] = (type(v) == "table" and (tables[v] or copy(v, tables))) or v + end + local mt = getmetatable(old) + if mt then + setmetatable(new,mt) + end + return new + else + return { } + end + end + + xml.copy = copy + +end + +--[[ldx-- +<p>In <l n='context'/> serializing the tree or parts of the tree is a major +actitivity which is why the following function is pretty optimized resulting +in a few more lines of code than needed. The variant that uses the formatting +function for all components is about 15% slower than the concatinating +alternative.</p> +--ldx]]-- + +do + + -- todo: add <?xml version='1.0' standalone='yes'?> when not present + + local fallbackhandle = (tex and tex.sprint) or io.write + + local function serialize(e, handle, textconverter, attributeconverter, specialconverter, nocommands) + if not e then + return + elseif not nocommands then + local ec = e.command + if ec ~= nil then -- we can have all kind of types + +if e.special then -- todo test for true/false + local etg, edt = e.tg, e.dt + local spc = specialconverter and specialconverter[etg] + if spc then +--~ print("SPECIAL",etg,table.serialize(specialconverter), spc) + local result = spc(edt[1]) + if result then + handle(result) + return + else + -- no need to handle any further + end + end +end + + local xc = xml.command + if xc then + xc(e,ec) + return + end + end + end + handle = handle or fallbackhandle + local etg = e.tg + if etg then + if e.special then + local edt = e.dt + local spc = specialconverter and specialconverter[etg] + if spc then + local result = spc(edt[1]) + if result then + handle(result) + else + -- no need to handle any further + end + elseif etg == "@pi@" then + -- handle(format("<?%s?>",edt[1])) + handle("<?" .. edt[1] .. "?>") + elseif etg == "@cm@" then + -- handle(format("<!--%s-->",edt[1])) + handle("<!--" .. edt[1] .. "-->") + elseif etg == "@cd@" then + -- handle(format("<![CDATA[%s]]>",edt[1])) + handle("<![CDATA[" .. edt[1] .. "]]>") + elseif etg == "@dt@" then + -- handle(format("<!DOCTYPE %s>",edt[1])) + handle("<!DOCTYPE " .. edt[1] .. ">") + elseif etg == "@rt@" then + serialize(edt,handle,textconverter,attributeconverter,specialconverter,nocommands) + end + else + local ens, eat, edt, ern = e.ns, e.at, e.dt, e.rn + local ats = eat and next(eat) and { } -- type test maybe faster + if ats then + if attributeconverter then + for k,v in pairs(eat) do + ats[#ats+1] = format('%s=%q',k,attributeconverter(v)) + end + else + for k,v in pairs(eat) do + ats[#ats+1] = format('%s=%q',k,v) + end + end + end + if ern and xml.trace_remap and ern ~= ens then +--~ if ats then +--~ ats[#ats+1] = format("xmlns:remapped='%s'",ern) +--~ else +--~ ats = { format("xmlns:remapped='%s'",ern) } +--~ end +--~ if ats then +--~ ats[#ats+1] = format("remappedns='%s'",ens or '-') +--~ else +--~ ats = { format("remappedns='%s'",ens or '-') } +--~ end +ens = ern + end + if ens ~= "" then + if edt and #edt > 0 then + if ats then + -- handle(format("<%s:%s %s>",ens,etg,concat(ats," "))) + handle("<" .. ens .. ":" .. etg .. " " .. concat(ats," ") .. ">") + else + -- handle(format("<%s:%s>",ens,etg)) + handle("<" .. ens .. ":" .. etg .. ">") + end + for i=1,#edt do + local e = edt[i] + if type(e) == "string" then + if textconverter then + handle(textconverter(e)) + else + handle(e) + end + else + serialize(e,handle,textconverter,attributeconverter,specialconverter,nocommands) + end + end + -- handle(format("</%s:%s>",ens,etg)) + handle("</" .. ens .. ":" .. etg .. ">") + else + if ats then + -- handle(format("<%s:%s %s/>",ens,etg,concat(ats," "))) + handle("<" .. ens .. ":" .. etg .. " " .. concat(ats," ") .. "/>") + else + -- handle(format("<%s:%s/>",ens,etg)) + handle("<" .. ens .. ":" .. etg .. "/>") + end + end + else + if edt and #edt > 0 then + if ats then + -- handle(format("<%s %s>",etg,concat(ats," "))) + handle("<" .. etg .. " " .. concat(ats," ") .. ">") + else + -- handle(format("<%s>",etg)) + handle("<" .. etg .. ">") + end + for i=1,#edt do + serialize(edt[i],handle,textconverter,attributeconverter,specialconverter,nocommands) + end + -- handle(format("</%s>",etg)) + handle("</" .. etg .. ">") + else + if ats then + -- handle(format("<%s %s/>",etg,concat(ats," "))) + handle("<" .. etg .. " " .. concat(ats," ") .. "/>") + else + -- handle(format("<%s/>",etg)) + handle("<" .. etg .. "/>") + end + end + end + end + elseif type(e) == "string" then + if textconverter then + handle(textconverter(e)) + else + handle(e) + end + else + for i=1,#e do + serialize(e[i],handle,textconverter,attributeconverter,specialconverter,nocommands) + end + end + end + + xml.serialize = serialize + + function xml.checkbom(root) -- can be made faster + if root.ri then + local dt, found = root.dt, false + for k,v in ipairs(dt) do + if type(v) == "table" and v.special and v.tg == "@pi" and v.dt:find("xml.*version=") then + found = true + break + end + end + if not found then + table.insert(dt, 1, { special=true, ns="", tg="@pi@", dt = { "xml version='1.0' standalone='yes'"} } ) + table.insert(dt, 2, "\n" ) + end + end + end + + --[[ldx-- + <p>At the cost of some 25% runtime overhead you can first convert the tree to a string + and then handle the lot.</p> + --ldx]]-- + + function xml.tostring(root) -- 25% overhead due to collecting + if root then + if type(root) == 'string' then + return root + elseif next(root) then -- next is faster than type (and >0 test) + local result = { } + serialize(root,function(s) result[#result+1] = s end) + return concat(result,"") + end + end + return "" + end + +end + +--[[ldx-- +<p>The next function operated on the content only and needs a handle function +that accepts a string.</p> +--ldx]]-- + +function xml.string(e,handle) + if not handle or (e.special and e.tg ~= "@rt@") then + -- nothing + elseif e.tg then + local edt = e.dt + if edt then + for i=1,#edt do + xml.string(edt[i],handle) + end + end + else + handle(e) + end +end + +--[[ldx-- +<p>How you deal with saving data depends on your preferences. For a 40 MB database +file the timing on a 2.3 Core Duo are as follows (time in seconds):</p> + +<lines> +1.3 : load data from file to string +6.1 : convert string into tree +5.3 : saving in file using xmlsave +6.8 : converting to string using xml.tostring +3.6 : saving converted string in file +</lines> + +<p>The save function is given below.</p> +--ldx]]-- + +function xml.save(root,name) + local f = io.open(name,"w") + if f then + xml.serialize(root,function(s) f:write(s) end) + f:close() + end +end + +--[[ldx-- +<p>A few helpers:</p> +--ldx]]-- + +function xml.body(root) + return (root.ri and root.dt[root.ri]) or root +end + +function xml.text(root) + return (root and xml.tostring(root)) or "" +end + +function xml.content(root) -- bugged + return (root and root.dt and xml.tostring(root.dt)) or "" +end + +--[[ldx-- +<p>The next helper erases an element but keeps the table as it is, +and since empty strings are not serialized (effectively) it does +not harm. Copying the table would take more time. Usage:</p> + +<typing> +dt[k] = xml.empty() or xml.empty(dt,k) +</typing> +--ldx]]-- + +function xml.empty(dt,k) + if dt and k then + dt[k] = "" + return dt[k] + else + return "" + end +end + +--[[ldx-- +<p>The next helper assigns a tree (or string). Usage:</p> + +<typing> +dt[k] = xml.assign(root) or xml.assign(dt,k,root) +</typing> +--ldx]]-- + +function xml.assign(dt,k,root) + if dt and k then + dt[k] = (type(root) == "table" and xml.body(root)) or root + return dt[k] + else + return xml.body(root) + end +end + +--[[ldx-- +<p>We've now arrived at an intersting part: accessing the tree using a subset +of <l n='xpath'/> and since we're not compatible we call it <l n='lpath'/>. We +will explain more about its usage in other documents.</p> +--ldx]]-- + +do + + xml.functions = xml.functions or { } + + local functions = xml.functions + + local actions = { + [10] = "stay", + [11] = "parent", + [12] = "subtree root", + [13] = "document root", + [14] = "any", + [15] = "many", + [16] = "initial", + [20] = "match", + [21] = "match one of", + [22] = "match and attribute eq", + [23] = "match and attribute ne", + [24] = "match one of and attribute eq", + [25] = "match one of and attribute ne", + [27] = "has attribute", + [28] = "has value", + [29] = "fast match", + [30] = "select", + [31] = "expression", + [40] = "processing instruction", + } + + --~ local function make_expression(str) --could also be an lpeg + --~ str = str:gsub("@([a-zA-Z%-_]+)", "(a['%1'] or '')") + --~ str = str:gsub("position%(%)", "i") + --~ str = str:gsub("text%(%)", "t") + --~ str = str:gsub("!=", "~=") + --~ str = str:gsub("([^=!~<>])=([^=!~<>])", "%1==%2") + --~ str = str:gsub("([a-zA-Z%-_]+)%(", "functions.%1(") + --~ return str, loadstring(format("return function(functions,i,a,t) return %s end", str))() + --~ end + + -- a rather dumb lpeg + + local P, S, R, C, V, Cc = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc + + local lp_position = P("position()") / "id" + local lp_text = P("text()") / "tx" + local lp_name = P("name()") / "((rt.ns~='' and rt.ns..':'..rt.tg) or '')" + local lp_tag = P("tag()") / "(rt.tg or '')" + local lp_ns = P("ns()") / "(rt.ns or '')" + local lp_noequal = P("!=") / "~=" + P("<=") + P(">=") + P("==") + local lp_doequal = P("=") / "==" + local lp_attribute = P("@") / "" * Cc("(at['") * R("az","AZ","--","__")^1 * Cc("'] or '')") + + local lp_function = C(R("az","AZ","--","__")^1) * P("(") / function(t) + if functions[t] then + return "functions." .. t .. "(" + else + return "functions.error(" + end + end + + local lparent = lpeg.P("(") + local rparent = lpeg.P(")") + local noparent = 1 - (lparent+rparent) + local nested = lpeg.P{lparent * (noparent + lpeg.V(1))^0 * rparent} + local value = lpeg.P(lparent * lpeg.C((noparent + nested)^0) * rparent) + +--~ local value = P { "(" * C(((1 - S("()")) + V(1))^0) * ")" } + + local lp_special = (C(P("name")+P("text")+P("tag"))) * value / function(t,s) + if functions[t] then + if s then + return "functions." .. t .. "(rt,k," .. s ..")" + else + return "functions." .. t .. "(rt,k)" + end + else + return "functions.error(" .. t .. ")" + end + end + + local converter = lpeg.Cs ( ( + lp_position + + lp_text + lp_name + -- fast one + lp_special + + lp_noequal + lp_doequal + + lp_attribute + + lp_function + + 1 )^1 ) + + local function make_expression(str) + str = converter:match(str) + return str, loadstring(format("return function(functions,id,at,tx,rt,k) return %s end", str))() + end + + local map = { } + + local space = S(' \r\n\t') + local squote = S("'") + local dquote = S('"') + local lparent = P('(') + local rparent = P(')') + local atsign = P('@') + local lbracket = P('[') + local rbracket = P(']') + local exclam = P('!') + local period = P('.') + local eq = P('==') + P('=') + local ne = P('<>') + P('!=') + local star = P('*') + local slash = P('/') + local colon = P(':') + local bar = P('|') + local hat = P('^') + local valid = R('az', 'AZ', '09') + S('_-') +--~ local name_yes = C(valid^1 + star) * colon * C(valid^1 + star) -- permits ns:* *:tg *:* +--~ local name_nop = C(P(true)) * C(valid^1) + local name_yes = C(valid^1 + star) * colon * C(valid^1 + star) -- permits ns:* *:tg *:* + local name_nop = Cc("*") * C(valid^1) + local name = name_yes + name_nop + local number = C((S('+-')^0 * R('09')^1)) / tonumber + local names = (bar^0 * name)^1 + local morenames = name * (bar^0 * name)^1 + local instructiontag = P('pi::') + local spacing = C(space^0) + local somespace = space^1 + local optionalspace = space^0 + local text = C(valid^0) + local value = (squote * C((1 - squote)^0) * squote) + (dquote * C((1 - dquote)^0) * dquote) + local empty = 1-slash + + local is_eq = lbracket * atsign * name * eq * value * rbracket + local is_ne = lbracket * atsign * name * ne * value * rbracket + local is_attribute = lbracket * atsign * name * rbracket + local is_value = lbracket * value * rbracket + local is_number = lbracket * number * rbracket + + local nobracket = 1-(lbracket+rbracket) -- must be improved + local is_expression = lbracket * C(((C(nobracket^1))/make_expression)) * rbracket + + local is_expression = lbracket * (C(nobracket^1))/make_expression * rbracket + + local is_one = name + local is_none = exclam * name + local is_one_of = ((lparent * names * rparent) + morenames) + local is_none_of = exclam * ((lparent * names * rparent) + morenames) + + local stay = (period ) + local parent = (period * period ) / function( ) map[#map+1] = { 11 } end + local subtreeroot = (slash + hat ) / function( ) map[#map+1] = { 12 } end + local documentroot = (hat * hat ) / function( ) map[#map+1] = { 13 } end + local any = (star ) / function( ) map[#map+1] = { 14 } end + local many = (star * star ) / function( ) map[#map+1] = { 15 } end + local initial = (hat * hat * hat ) / function( ) map[#map+1] = { 16 } end + + local match = (is_one ) / function(...) map[#map+1] = { 20, true , ... } end + local match_one_of = (is_one_of ) / function(...) map[#map+1] = { 21, true , ... } end + local dont_match = (is_none ) / function(...) map[#map+1] = { 20, false, ... } end + local dont_match_one_of = (is_none_of ) / function(...) map[#map+1] = { 21, false, ... } end + + local match_and_eq = (is_one * is_eq ) / function(...) map[#map+1] = { 22, true , ... } end + local match_and_ne = (is_one * is_ne ) / function(...) map[#map+1] = { 23, true , ... } end + local dont_match_and_eq = (is_none * is_eq ) / function(...) map[#map+1] = { 22, false, ... } end + local dont_match_and_ne = (is_none * is_ne ) / function(...) map[#map+1] = { 23, false, ... } end + + local match_one_of_and_eq = (is_one_of * is_eq ) / function(...) map[#map+1] = { 24, true , ... } end + local match_one_of_and_ne = (is_one_of * is_ne ) / function(...) map[#map+1] = { 25, true , ... } end + local dont_match_one_of_and_eq = (is_none_of * is_eq ) / function(...) map[#map+1] = { 24, false, ... } end + local dont_match_one_of_and_ne = (is_none_of * is_ne ) / function(...) map[#map+1] = { 25, false, ... } end + + local has_attribute = (is_one * is_attribute) / function(...) map[#map+1] = { 27, true , ... } end + local has_value = (is_one * is_value ) / function(...) map[#map+1] = { 28, true , ... } end + local dont_has_attribute = (is_none * is_attribute) / function(...) map[#map+1] = { 27, false, ... } end + local dont_has_value = (is_none * is_value ) / function(...) map[#map+1] = { 28, false, ... } end + local position = (is_one * is_number ) / function(...) map[#map+1] = { 30, true, ... } end + local dont_position = (is_none * is_number ) / function(...) map[#map+1] = { 30, false, ... } end + + local expression = (is_one * is_expression)/ function(...) map[#map+1] = { 31, true, ... } end + local dont_expression = (is_none * is_expression)/ function(...) map[#map+1] = { 31, false, ... } end + + local self_expression = ( is_expression) / function(...) if #map == 0 then map[#map+1] = { 11 } end + map[#map+1] = { 31, true, "*", "*", ... } end + local dont_self_expression = (exclam * is_expression) / function(...) if #map == 0 then map[#map+1] = { 11 } end + map[#map+1] = { 31, false, "*", "*", ... } end + + local instruction = (instructiontag * text ) / function(...) map[#map+1] = { 40, ... } end + local nothing = (empty ) / function( ) map[#map+1] = { 15 } end -- 15 ? + local crap = (1-slash)^1 + + -- a few ugly goodies: + + local docroottag = P('^^') / function( ) map[#map+1] = { 12 } end + local subroottag = P('^') / function( ) map[#map+1] = { 13 } end + local roottag = P('root::') / function( ) map[#map+1] = { 12 } end + local parenttag = P('parent::') / function( ) map[#map+1] = { 11 } end + local childtag = P('child::') + local selftag = P('self::') + + -- there will be more and order will be optimized + + local selector = ( + instruction + + many + any + + parent + stay + + dont_position + position + + dont_match_one_of_and_eq + dont_match_one_of_and_ne + + match_one_of_and_eq + match_one_of_and_ne + + dont_match_and_eq + dont_match_and_ne + + match_and_eq + match_and_ne + + dont_expression + expression + + dont_self_expression + self_expression + + has_attribute + has_value + + dont_match_one_of + match_one_of + + dont_match + match + + crap + empty + ) + + local grammar = P { "startup", + startup = (initial + documentroot + subtreeroot + roottag + docroottag + subroottag)^0 * V("followup"), + followup = ((slash + parenttag + childtag + selftag)^0 * selector)^1, + } + + local function compose(str) + if not str or str == "" then + -- wildcard + return true + elseif str == '/' then + -- root + return false + else + map = { } + grammar:match(str) + if #map == 0 then + return true + else + local m = map[1][1] + if #map == 1 then + if m == 14 or m == 15 then + -- wildcard + return true + elseif m == 12 then + -- root + return false + end + elseif #map == 2 and m == 12 and map[2][1] == 20 then + -- return { { 29, map[2][2], map[2][3], map[2][4], map[2][5] } } + map[2][1] = 29 + return { map[2] } + end + if m ~= 11 and m ~= 12 and m ~= 13 and m ~= 14 and m ~= 15 and m ~= 16 then + table.insert(map, 1, { 16 }) + end + -- print((table.serialize(map)):gsub("[ \n]+"," ")) + return map + end + end + end + + local cache = { } + + function xml.lpath(pattern,trace) + if type(pattern) == "string" then + local result = cache[pattern] + if not result then + result = compose(pattern) + cache[pattern] = result + end + if trace or xml.trace_lpath then + xml.lshow(result) + end + return result + else + return pattern + end + end + + local fallbackreport = (texio and texio.write) or io.write + + function xml.lshow(pattern,report) + report = report or fallbackreport + local lp = xml.lpath(pattern) + if lp == false then + report(" -: root\n") + elseif lp == true then + report(" -: wildcard\n") + else + if type(pattern) == "string" then + report(format("pattern: %s\n",pattern)) + end + for k,v in ipairs(lp) do + if #v > 1 then + local t = { } + for i=2,#v do + local vv = v[i] + if type(vv) == "string" then + t[#t+1] = (vv ~= "" and vv) or "#" + elseif type(vv) == "boolean" then + t[#t+1] = (vv and "==") or "<>" + end + end + report(format("%2i: %s %s -> %s\n", k,v[1],actions[v[1]],concat(t," "))) + else + report(format("%2i: %s %s\n", k,v[1],actions[v[1]])) + end + end + end + end + + function xml.xshow(e,...) -- also handy when report is given, use () to isolate first e + local t = { ... } + local report = (type(t[#t]) == "function" and t[#t]) or fallbackreport + if e == nil then + report("<!-- no element -->\n") + elseif type(e) ~= "table" then + report(tostring(e)) + elseif e.tg then + report(tostring(e) .. "\n") + else + for i=1,#e do + report(tostring(e[i]) .. "\n") + end + end + end + +end + +--[[ldx-- +<p>An <l n='lpath'/> is converted to a table with instructions for traversing the +tree. Hoever, simple cases are signaled by booleans. Because we don't know in +advance what we want to do with the found element the handle gets three arguments:</p> + +<lines> +<t>r</t> : the root element of the data table +<t>d</t> : the data table of the result +<t>t</t> : the index in the data table of the result +</lines> + +<p> Access to the root and data table makes it possible to construct insert and delete +functions.</p> +--ldx]]-- + +do + + local functions = xml.functions + + functions.contains = string.find + functions.find = string.find + functions.upper = string.upper + functions.lower = string.lower + functions.number = tonumber + functions.boolean = toboolean + + functions.oneof = function(s,...) -- slow + local t = {...} for i=1,#t do if s == t[i] then return true end end return false + end + functions.error = function(str) + xml.error_handler("unknown function in lpath expression",str) + return false + end + functions.text = function(root,k,n) -- unchecked, maybe one deeper + local t = type(t) + if t == "string" then + return t + else -- todo n + local rdt = root.dt + return (rdt and rdt[k]) or root[k] or "" + end + end + functions.name = function(root,k,n) + -- way too fuzzy + local found + if not k or not n then + local ns, tg = root.rn or root.ns or "", root.tg + if not tg then + for i=1,#root do + local e = root[i] + if type(e) == "table" then + found = e + break + end + end + elseif ns ~= "" then + return ns .. ":" .. tg + else + return tg + end + elseif n == 0 then + local e = root[k] + if type(e) ~= "table" then + found = e + end + elseif n < 0 then + for i=k-1,1,-1 do + local e = root[i] + if type(e) == "table" then + if n == -1 then + found = e + break + else + n = n + 1 + end + end + end + else +--~ print(k,n,#root) + for i=k+1,#root,1 do + local e = root[i] + if type(e) == "table" then + if n == 1 then + found = e + break + else + n = n - 1 + end + end + end + end + if found then + local ns, tg = found.rn or found.ns or "", found.tg + if ns ~= "" then + return ns .. ":" .. tg + else + return tg + end + else + return "" + end + end + + local function traverse(root,pattern,handle,reverse,index,parent,wildcard) -- multiple only for tags, not for namespaces + if not root then -- error + return false + elseif pattern == false then -- root + handle(root,root.dt,root.ri) + return false + elseif pattern == true then -- wildcard + local rootdt = root.dt + if rootdt then + local start, stop, step = 1, #rootdt, 1 + if reverse then + start, stop, step = stop, start, -1 + end + for k=start,stop,step do + if handle(root,rootdt,root.ri or k) then return false end + if not traverse(rootdt[k],true,handle,reverse) then return false end + end + end + return false + elseif root.dt then + index = index or 1 + local action = pattern[index] + local command = action[1] + if command == 29 then -- fast case /oeps + local rootdt = root.dt + for k=1,#rootdt do + local e = rootdt[k] + local tg = e.tg + if e.tg then + local ns = e.rn or e.ns + local ns_a, tg_a = action[3], action[4] + local matched = (ns_a == "*" or ns == ns_a) and (tg_a == "*" or tg == tg_a) + if not action[2] then matched = not matched end + if matched then + if handle(root,rootdt,k) then return false end + end + end + end + elseif command == 11 then -- parent + local ep = root.__p__ or parent + if index < #pattern then + if not traverse(ep,pattern,handle,reverse,index+1,root) then return false end + elseif handle(root,rootdt,k) then + return false + end + else + if (command == 16 or command == 12) and index == 1 then -- initial + -- wildcard = true + wildcard = command == 16 -- ok? + index = index + 1 + action = pattern[index] + command = action and action[1] or 0 -- something is wrong + end + if command == 11 then -- parent + local ep = root.__p__ or parent + if index < #pattern then + if not traverse(ep,pattern,handle,reverse,index+1,root) then return false end + elseif handle(root,rootdt,k) then + return false + end + else + local rootdt = root.dt + local start, stop, step, n, dn = 1, #rootdt, 1, 0, 1 + if command == 30 then + if action[5] < 0 then + start, stop, step = stop, start, -1 + dn = -1 + end + elseif reverse and index == #pattern then + start, stop, step = stop, start, -1 + end + local idx = 0 + for k=start,stop,step do -- we used to have functions for all but a case is faster + local e = rootdt[k] + local ns, tg = e.rn or e.ns, e.tg + if tg then + idx = idx + 1 + if command == 30 then + local ns_a, tg_a = action[3], action[4] + if tg == tg_a then + matched = ns_a == "*" or ns == ns_a + elseif tg_a == '*' then + matched, multiple = ns_a == "*" or ns == ns_a, true + else + matched = false + end + if not action[2] then matched = not matched end + if matched then + n = n + dn + if n == action[5] then + if index == #pattern then + if handle(root,rootdt,root.ri or k) then return false end + else + if not traverse(e,pattern,handle,reverse,index+1,root) then return false end + end + break + end + elseif wildcard then + if not traverse(e,pattern,handle,reverse,index,root,true) then return false end + end + else + local matched, multiple = false, false + if command == 20 then -- match + local ns_a, tg_a = action[3], action[4] + if tg == tg_a then + matched = ns_a == "*" or ns == ns_a + elseif tg_a == '*' then + matched, multiple = ns_a == "*" or ns == ns_a, true + else + matched = false + end + if not action[2] then matched = not matched end + elseif command == 21 then -- match one of + multiple = true + for i=3,#action,2 do + local ns_a, tg_a = action[i], action[i+1] + if (ns_a == "*" or ns == ns_a) and (tg == "*" or tg == tg_a) then + matched = true + break + end + end + if not action[2] then matched = not matched end + elseif command == 22 then -- eq + local ns_a, tg_a = action[3], action[4] + if tg == tg_a then + matched = ns_a == "*" or ns == ns_a + elseif tg_a == '*' then + matched, multiple = ns_a == "*" or ns == ns_a, true + else + matched = false + end + matched = matched and e.at[action[6]] == action[7] + elseif command == 23 then -- ne + local ns_a, tg_a = action[3], action[4] + if tg == tg_a then + matched = ns_a == "*" or ns == ns_a + elseif tg_a == '*' then + matched, multiple = ns_a == "*" or ns == ns_a, true + else + matched = false + end + if not action[2] then matched = not matched end + matched = mached and e.at[action[6]] ~= action[7] + elseif command == 24 then -- one of eq + multiple = true + for i=3,#action-2,2 do + local ns_a, tg_a = action[i], action[i+1] + if (ns_a == "*" or ns == ns_a) and (tg == "*" or tg == tg_a) then + matched = true + break + end + end + if not action[2] then matched = not matched end + matched = matched and e.at[action[#action-1]] == action[#action] + elseif command == 25 then -- one of ne + multiple = true + for i=3,#action-2,2 do + local ns_a, tg_a = action[i], action[i+1] + if (ns_a == "*" or ns == ns_a) and (tg == "*" or tg == tg_a) then + matched = true + break + end + end + if not action[2] then matched = not matched end + matched = matched and e.at[action[#action-1]] ~= action[#action] + elseif command == 27 then -- has attribute + local ns_a, tg_a = action[3], action[4] + if tg == tg_a then + matched = ns_a == "*" or ns == ns_a + elseif tg_a == '*' then + matched, multiple = ns_a == "*" or ns == ns_a, true + else + matched = false + end + if not action[2] then matched = not matched end + matched = matched and e.at[action[5]] + elseif command == 28 then -- has value + local edt, ns_a, tg_a = e.dt, action[3], action[4] + if tg == tg_a then + matched = ns_a == "*" or ns == ns_a + elseif tg_a == '*' then + matched, multiple = ns_a == "*" or ns == ns_a, true + else + matched = false + end + if not action[2] then matched = not matched end + matched = matched and edt and edt[1] == action[5] + elseif command == 31 then + local edt, ns_a, tg_a = e.dt, action[3], action[4] + if tg == tg_a then + matched = ns_a == "*" or ns == ns_a + elseif tg_a == '*' then + matched, multiple = ns_a == "*" or ns == ns_a, true + else + matched = false + end + if not action[2] then matched = not matched end + if matched then + matched = action[6](functions,idx,e.at or { },edt[1],rootdt,k) + end + end + if matched then -- combine tg test and at test + if index == #pattern then + if handle(root,rootdt,root.ri or k) then return false end + if wildcard then + if multiple then + if not traverse(e,pattern,handle,reverse,index,root,true) then return false end + else + -- maybe or multiple; anyhow, check on (section|title) vs just section and title in example in lxml + if not traverse(e,pattern,handle,reverse,index,root) then return false end + end + end + else + if not traverse(e,pattern,handle,reverse,index+1,root) then return false end + end + elseif command == 14 then -- any + if index == #pattern then + if handle(root,rootdt,root.ri or k) then return false end + else + if not traverse(e,pattern,handle,reverse,index+1,root) then return false end + end + elseif command == 15 then -- many + if index == #pattern then + if handle(root,rootdt,root.ri or k) then return false end + else + if not traverse(e,pattern,handle,reverse,index+1,root,true) then return false end + end + -- not here : 11 + elseif command == 11 then -- parent + local ep = e.__p__ or parent + if index < #pattern then + if not traverse(ep,pattern,handle,reverse,root,index+1) then return false end + elseif handle(root,rootdt,k) then + return false + end + elseif command == 40 and e.special and tg == "@pi@" then -- pi + local pi = action[2] + if pi ~= "" then + local pt = e.dt[1] + if pt and pt:find(pi) then + if handle(root,rootdt,k) then + return false + end + end + elseif handle(root,rootdt,k) then + return false + end + elseif wildcard then + if not traverse(e,pattern,handle,reverse,index,root,true) then return false end + end + end + else + -- not here : 11 + if command == 11 then -- parent + local ep = e.__p__ or parent + if index < #pattern then + if not traverse(ep,pattern,handle,reverse,index+1,root) then return false end + elseif handle(root,rootdt,k) then + return false + end + break -- else loop + end + end + end + end + end + end + return true + end + + xml.traverse = traverse + +end + +--[[ldx-- +<p>Next come all kind of locators and manipulators. The most generic function here +is <t>xml.filter(root,pattern)</t>. All registers functions in the filters namespace +can be path of a search path, as in:</p> + +<typing> +local r, d, k = xml.filter(root,"/a/b/c/position(4)" +</typing> +--ldx]]-- + +do + + local traverse, lpath, convert = xml.traverse, xml.lpath, xml.convert + + xml.filters = { } + + function xml.filters.default(root,pattern) + local rt, dt, dk + traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end) + return dt and dt[dk], rt, dt, dk + end + function xml.filters.attributes(root,pattern,arguments) + local rt, dt, dk + traverse(root, lpath(pattern), function(r,d,k) rt, dt, dk = r, d, k return true end) + local ekat = (dt and dt[dk] and dt[dk].at) or (rt and rt.at) + if ekat then + if arguments then + return ekat[arguments] or "", rt, dt, dk + else + return ekat, rt, dt, dk + end + else + return { }, rt, dt, dk + end + end + function xml.filters.reverse(root,pattern) + local rt, dt, dk + traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end, 'reverse') + return dt and dt[dk], rt, dt, dk + end + function xml.filters.count(root,pattern,everything) + local n = 0 + traverse(root, lpath(pattern), function(r,d,t) + if everything or type(d[t]) == "table" then + n = n + 1 + end + end) + return n + end + function xml.filters.elements(root, pattern) -- == all + local t = { } + traverse(root, lpath(pattern), function(r,d,k) + local e = d[k] + if e then + t[#t+1] = e + end + end) + return t + end + function xml.filters.texts(root, pattern) + local t = { } + traverse(root, lpath(pattern), function(r,d,k) + local e = d[k] + if e and e.dt then + t[#t+1] = e.dt + end + end) + return t + end + function xml.filters.first(root,pattern) + local rt, dt, dk + traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end) + return dt and dt[dk], rt, dt, dk + end + function xml.filters.last(root,pattern) + local rt, dt, dk + traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end, 'reverse') + return dt and dt[dk], rt, dt, dk + end + function xml.filters.index(root,pattern,arguments) + local rt, dt, dk, reverse, i = nil, nil, nil, false, tonumber(arguments or '1') or 1 + if i and i ~= 0 then + if i < 0 then + reverse, i = true, -i + end + traverse(root, lpath(pattern), function(r,d,k) rt, dt, dk, i = r, d, k, i-1 return i == 0 end, reverse) + if i == 0 then + return dt and dt[dk], rt, dt, dk + end + end + return nil, nil, nil, nil + end + function xml.filters.attribute(root,pattern,arguments) + local rt, dt, dk + traverse(root, lpath(pattern), function(r,d,k) rt, dt, dk = r, d, k return true end) + local ekat = (dt and dt[dk] and dt[dk].at) or (rt and rt.at) + return (ekat and (ekat[arguments] or ekat[arguments:gsub("^([\"\'])(.*)%1$","%2")])) or "" + end + function xml.filters.text(root,pattern,arguments) -- ?? why index, tostring slow + local dtk, rt, dt, dk = xml.filters.index(root,pattern,arguments) + if dtk then -- n + local dtkdt = dtk.dt + if not dtkdt then + return "", rt, dt, dk + elseif #dtkdt == 1 and type(dtkdt[1]) == "string" then + return dtkdt[1], rt, dt, dk + else + return xml.tostring(dtkdt), rt, dt, dk + end + else + return "", rt, dt, dk + end + end + function xml.filters.tag(root,pattern,n) + local tag = "" + traverse(root, lpath(pattern), function(r,d,k) + tag = xml.functions.tag(d,k,n and tonumber(n)) + return true + end) + return tag + end + function xml.filters.name(root,pattern,n) + local tag = "" + traverse(root, lpath(pattern), function(r,d,k) + tag = xml.functions.name(d,k,n and tonumber(n)) + return true + end) + return tag + end + + --[[ldx-- + <p>For splitting the filter function from the path specification, we can + use string matching or lpeg matching. Here the difference in speed is + neglectable but the lpeg variant is more robust.</p> + --ldx]]-- + + -- not faster but hipper ... although ... i can't get rid of the trailing / in the path + + local P, S, R, C, V, Cc = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc + + local slash = P('/') + local name = (R("az","AZ","--","__"))^1 + local path = C(((1-slash)^0 * slash)^1) + local argument = P { "(" * C(((1 - S("()")) + V(1))^0) * ")" } + local action = Cc(1) * path * C(name) * argument + local attribute = Cc(2) * path * P('@') * C(name) + local direct = Cc(3) * Cc("../*") * slash^0 * C(name) * argument + + local parser = direct + action + attribute + + local filters = xml.filters + local attribute_filter = xml.filters.attributes + local default_filter = xml.filters.default + + -- todo: also hash, could be gc'd + + function xml.filter(root,pattern) + local kind, a, b, c = parser:match(pattern) +--~ if xml.trace_lpath then +--~ print(pattern,kind,a,b,c) +--~ end + if kind == 1 or kind == 3 then + return (filters[b] or default_filter)(root,a,c) + elseif kind == 2 then + return attribute_filter(root,a,b) + else + return default_filter(root,pattern) + end + end + + --~ slightly faster, but first we need a proper test file + --~ + --~ local hash = { } + --~ + --~ function xml.filter(root,pattern) + --~ local h = hash[pattern] + --~ if not h then + --~ local kind, a, b, c = parser:match(pattern) + --~ if kind == 1 then + --~ h = { kind, filters[b] or default_filter, a, b, c } + --~ elseif kind == 2 then + --~ h = { kind, attribute_filter, a, b, c } + --~ else + --~ h = { kind, default_filter, a, b, c } + --~ end + --~ hash[pattern] = h + --~ end + --~ local kind = h[1] + --~ if kind == 1 then + --~ return h[2](root,h[2],h[4]) + --~ elseif kind == 2 then + --~ return h[2](root,h[2],h[3]) + --~ else + --~ return h[2](root,pattern) + --~ end + --~ end + + --[[ldx-- + <p>The following functions collect elements and texts.</p> + --ldx]]-- + + -- still somewhat bugged + + function xml.collect_elements(root, pattern, ignorespaces) + local rr, dd = { }, { } + traverse(root, lpath(pattern), function(r,d,k) + local dk = d and d[k] + if dk then + if ignorespaces and type(dk) == "string" and dk:find("[^%S]") then + -- ignore + else + local n = #rr+1 + rr[n], dd[n] = r, dk + end + end + end) + return dd, rr + end + + function xml.collect_texts(root, pattern, flatten) + local t = { } -- no r collector + traverse(root, lpath(pattern), function(r,d,k) + if d then + local ek = d[k] + local tx = ek and ek.dt + if flatten then + if tx then + t[#t+1] = xml.tostring(tx) or "" + else + t[#t+1] = "" + end + else + t[#t+1] = tx or "" + end + else + t[#t+1] = "" + end + end) + return t + end + + function xml.collect_tags(root, pattern, nonamespace) + local t = { } + xml.traverse(root, xml.lpath(pattern), function(r,d,k) + local dk = d and d[k] + if dk and type(dk) == "table" then + local ns, tg = e.ns, e.tg + if nonamespace then + t[#t+1] = tg -- if needed we can return an extra table + elseif ns == "" then + t[#t+1] = tg + else + t[#t+1] = ns .. ":" .. tg + end + end + end) + return #t > 0 and {} + end + + --[[ldx-- + <p>Often using an iterators looks nicer in the code than passing handler + functions. The <l n='lua'/> book describes how to use coroutines for that + purpose (<url href='http://www.lua.org/pil/9.3.html'/>). This permits + code like:</p> + + <typing> + for r, d, k in xml.elements(xml.load('text.xml'),"title") do + print(d[k]) + end + </typing> + + <p>Which will print all the titles in the document. The iterator variant takes + 1.5 times the runtime of the function variant which is due to the overhead in + creating the wrapper. So, instead of:</p> + + <typing> + function xml.filters.first(root,pattern) + for rt,dt,dk in xml.elements(root,pattern) + return dt and dt[dk], rt, dt, dk + end + return nil, nil, nil, nil + end + </typing> + + <p>We use the function variants in the filters.</p> + --ldx]]-- + + local wrap, yield = coroutine.wrap, coroutine.yield + + function xml.elements(root,pattern,reverse) + return wrap(function() traverse(root, lpath(pattern), yield, reverse) end) + end + + function xml.elements_only(root,pattern,reverse) + return wrap(function() traverse(root, lpath(pattern), function(r,d,k) yield(d[k]) end, reverse) end) + end + + function xml.each_element(root, pattern, handle, reverse) + local ok + traverse(root, lpath(pattern), function(r,d,k) ok = true handle(r,d,k) end, reverse) + return ok + end + + function xml.process_elements(root, pattern, handle) + traverse(root, lpath(pattern), function(r,d,k) + local dkdt = d[k].dt + if dkdt then + for i=1,#dkdt do + local v = dkdt[i] + if v.tg then handle(v) end + end + end + end) + end + + function xml.process_attributes(root, pattern, handle) + traverse(root, lpath(pattern), function(r,d,k) + local ek = d[k] + local a = ek.at or { } + handle(a) + if next(a) then -- next is faster than type (and >0 test) + ek.at = a + else + ek.at = nil + end + end) + end + + --[[ldx-- + <p>We've now arrives at the functions that manipulate the tree.</p> + --ldx]]-- + + function xml.inject_element(root, pattern, element, prepend) + if root and element then + local matches, collect = { }, nil + if type(element) == "string" then + element = convert(element,true) + end + if element then + collect = function(r,d,k) matches[#matches+1] = { r, d, k, element } end + traverse(root, lpath(pattern), collect) + for i=1,#matches do + local m = matches[i] + local r, d, k, element, edt = m[1], m[2], m[3], m[4], nil + if element.ri then + element = element.dt[element.ri].dt + else + element = element.dt + end + if r.ri then + edt = r.dt[r.ri].dt + else + edt = d and d[k] and d[k].dt + end + if edt then + local be, af + if prepend then + be, af = xml.copy(element), edt + else + be, af = edt, xml.copy(element) + end + for i=1,#af do + be[#be+1] = af[i] + end + if r.ri then + r.dt[r.ri].dt = be + else + d[k].dt = be + end + else + -- r.dt = element.dt -- todo + end + end + end + end + end + + -- todo: copy ! + + function xml.insert_element(root, pattern, element, before) -- todo: element als functie + if root and element then + if pattern == "/" then + xml.inject_element(root, pattern, element, before) + else + local matches, collect = { }, nil + if type(element) == "string" then + element = convert(element,true) + end + if element and element.ri then + element = element.dt[element.ri] + end + if element then + collect = function(r,d,k) matches[#matches+1] = { r, d, k, element } end + traverse(root, lpath(pattern), collect) + for i=#matches,1,-1 do + local m = matches[i] + local r, d, k, element = m[1], m[2], m[3], m[4] + if not before then k = k + 1 end + if element.tg then + table.insert(d,k,element) -- untested + elseif element.dt then + for _,v in ipairs(element.dt) do -- i added + table.insert(d,k,v) + k = k + 1 + end + end + end + end + end + end + end + + xml.insert_element_after = xml.insert_element + xml.insert_element_before = function(r,p,e) xml.insert_element(r,p,e,true) end + xml.inject_element_after = xml.inject_element + xml.inject_element_before = function(r,p,e) xml.inject_element(r,p,e,true) end + + function xml.delete_element(root, pattern) + local matches, deleted = { }, { } + local collect = function(r,d,k) matches[#matches+1] = { r, d, k } end + traverse(root, lpath(pattern), collect) + for i=#matches,1,-1 do + local m = matches[i] + deleted[#deleted+1] = table.remove(m[2],m[3]) + end + return deleted + end + + function xml.replace_element(root, pattern, element) + if type(element) == "string" then + element = convert(element,true) + end + if element and element.ri then + element = element.dt[element.ri] + end + if element then + traverse(root, lpath(pattern), function(rm, d, k) + d[k] = element.dt -- maybe not clever enough + end) + end + end + + local function load_data(name) -- == io.loaddata + local f, data = io.open(name), "" + if f then + data = f:read("*all",'b') -- 'b' ? + f:close() + end + return data + end + + function xml.include(xmldata,pattern,attribute,recursive,loaddata) + -- parse="text" (default: xml), encoding="" (todo) + -- attribute = attribute or 'href' + pattern = pattern or 'include' + loaddata = loaddata or load_data + local function include(r,d,k) + local ek, name = d[k], nil + if not attribute or attribute == "" then + local ekdt = ek.dt + name = (type(ekdt) == "table" and ekdt[1]) or ekdt + end + if not name then + if ek.at then + for a in (attribute or "href"):gmatch("([^|]+)") do + name = ek.at[a] + if name then break end + end + end + end + local data = (name and name ~= "" and loaddata(name)) or "" + if data == "" then + xml.empty(d,k) + elseif ek.at["parse"] == "text" then -- for the moment hard coded + d[k] = xml.escaped(data) + else + local xi = xml.convert(data) + if not xi then + xml.empty(d,k) + else + if recursive then + xml.include(xi,pattern,attribute,recursive,loaddata) + end + xml.assign(d,k,xi) + end + end + end + xml.each_element(xmldata, pattern, include) + end + + function xml.strip_whitespace(root, pattern) + traverse(root, lpath(pattern), function(r,d,k) + local dkdt = d[k].dt + if dkdt then -- can be optimized + local t = { } + for i=1,#dkdt do + local str = dkdt[i] + if type(str) == "string" and str:find("^[ \n\r\t]*$") then + -- stripped + else + t[#t+1] = str + end + end + d[k].dt = t + end + end) + end + + function xml.rename_space(root, oldspace, newspace) -- fast variant + local ndt = #root.dt + local rename = xml.rename_space + for i=1,ndt or 0 do + local e = root[i] + if type(e) == "table" then + if e.ns == oldspace then + e.ns = newspace + if e.rn then + e.rn = newspace + end + end + local edt = e.dt + if edt then + rename(edt, oldspace, newspace) + end + end + end + end + + function xml.remap_tag(root, pattern, newtg) + traverse(root, lpath(pattern), function(r,d,k) + d[k].tg = newtg + end) + end + function xml.remap_namespace(root, pattern, newns) + traverse(root, lpath(pattern), function(r,d,k) + d[k].ns = newns + end) + end + function xml.check_namespace(root, pattern, newns) + traverse(root, lpath(pattern), function(r,d,k) + local dk = d[k] + if (not dk.rn or dk.rn == "") and dk.ns == "" then + dk.rn = newns + end + end) + end + function xml.remap_name(root, pattern, newtg, newns, newrn) + traverse(root, lpath(pattern), function(r,d,k) + local dk = d[k] + dk.tg = newtg + dk.ns = newns + dk.rn = newrn + end) + end + + function xml.filters.found(root,pattern,check_content) + local found = false + traverse(root, lpath(pattern), function(r,d,k) + if check_content then + local dk = d and d[k] + found = dk and dk.dt and next(dk.dt) and true + else + found = true + end + return true + end) + return found + end + +end + +--[[ldx-- +<p>Here are a few synonyms.</p> +--ldx]]-- + +xml.filters.position = xml.filters.index + +xml.count = xml.filters.count +xml.index = xml.filters.index +xml.position = xml.filters.index +xml.first = xml.filters.first +xml.last = xml.filters.last +xml.found = xml.filters.found + +xml.each = xml.each_element +xml.process = xml.process_element +xml.strip = xml.strip_whitespace +xml.collect = xml.collect_elements +xml.all = xml.collect_elements + +xml.insert = xml.insert_element_after +xml.inject = xml.inject_element_after +xml.after = xml.insert_element_after +xml.before = xml.insert_element_before +xml.delete = xml.delete_element +xml.replace = xml.replace_element + +--[[ldx-- +<p>The following helper functions best belong to the <t>lmxl-ini</t> +module. Some are here because we need then in the <t>mk</t> +document and other manuals, others came up when playing with +this module. Since this module is also used in <l n='mtxrun'/> we've +put them here instead of loading mode modules there then needed.</p> +--ldx]]-- + +function xml.gsub(t,old,new) + if t.dt then + for k,v in ipairs(t.dt) do + if type(v) == "string" then + t.dt[k] = v:gsub(old,new) + else + xml.gsub(v,old,new) + end + end + end +end + +function xml.strip_leading_spaces(dk,d,k) -- cosmetic, for manual + if d and k and d[k-1] and type(d[k-1]) == "string" then + local s = d[k-1]:match("\n(%s+)") + xml.gsub(dk,"\n"..string.rep(" ",#s),"\n") + end +end + +function xml.serialize_path(root,lpath,handle) + local dk, r, d, k = xml.first(root,lpath) + dk = xml.copy(dk) + xml.strip_leading_spaces(dk,d,k) + xml.serialize(dk,handle) +end + +--~ xml.escapes = { ['&'] = '&', ['<'] = '<', ['>'] = '>', ['"'] = '"' } +--~ xml.unescapes = { } for k,v in pairs(xml.escapes) do xml.unescapes[v] = k end + +--~ function xml.escaped (str) return str:gsub("(.)" , xml.escapes ) end +--~ function xml.unescaped(str) return str:gsub("(&.-;)", xml.unescapes) end +--~ function xml.cleansed (str) return str:gsub("<.->" , '' ) end -- "%b<>" + +do + + local P, S, R, C, V, Cc, Cs = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc, lpeg.Cs + + -- 100 * 2500 * "oeps< oeps> oeps&" : gsub:lpeg|lpeg|lpeg + -- + -- 1021:0335:0287:0247 + + -- 10 * 1000 * "oeps< oeps> oeps& asfjhalskfjh alskfjh alskfjh alskfjh ;al J;LSFDJ" + -- + -- 1559:0257:0288:0190 (last one suggested by roberto) + + -- escaped = Cs((S("<&>") / xml.escapes + 1)^0) + -- escaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0) + local normal = (1 - S("<&>"))^0 + local special = P("<")/"<" + P(">")/">" + P("&")/"&" + local escaped = Cs(normal * (special * normal)^0) + + -- 100 * 1000 * "oeps< oeps> oeps&" : gsub:lpeg == 0153:0280:0151:0080 (last one by roberto) + + -- unescaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0) + -- unescaped = Cs((((P("&")/"") * (P("lt")/"<" + P("gt")/">" + P("amp")/"&") * (P(";")/"")) + 1)^0) + local normal = (1 - S"&")^0 + local special = P("<")/"<" + P(">")/">" + P("&")/"&" + local unescaped = Cs(normal * (special * normal)^0) + + -- 100 * 5000 * "oeps <oeps bla='oeps' foo='bar'> oeps </oeps> oeps " : gsub:lpeg == 623:501 msec (short tags, less difference) + + local cleansed = Cs(((P("<") * (1-P(">"))^0 * P(">"))/"" + 1)^0) + + function xml.escaped (str) return escaped :match(str) end + function xml.unescaped(str) return unescaped:match(str) end + function xml.cleansed (str) return cleansed :match(str) end + +end + +function xml.join(t,separator,lastseparator) + if #t > 0 then + local result = { } + for k,v in pairs(t) do + result[k] = xml.tostring(v) + end + if lastseparator then + return concat(result,separator or "",1,#result-1) .. (lastseparator or "") .. result[#result] + else + return concat(result,separator) + end + else + return "" + end +end + + +--[[ldx-- +<p>We provide (at least here) two entity handlers. The more extensive +resolver consults a hash first, tries to convert to <l n='utf'/> next, +and finaly calls a handler when defines. When this all fails, the +original entity is returned.</p> +--ldx]]-- + +do if unicode and unicode.utf8 then + + xml.entities = xml.entities or { } -- xml.entities.handler == function + + function xml.entities.handler(e) + return format("[%s]",e) + end + + local char = unicode.utf8.char + + local function toutf(s) + return char(tonumber(s,16)) + end + + local entities = xml.entities -- global entities + + function utfize(root) + local d = root.dt + for k=1,#d do + local dk = d[k] + if type(dk) == "string" then + -- test prevents copying if no match + if dk:find("&#x.-;") then + d[k] = dk:gsub("&#x(.-);",toutf) + end + else + utfize(dk) + end + end + end + + xml.utfize = utfize + + local function resolve(e) -- hex encoded always first, just to avoid mkii fallbacks + if e:find("#x") then + return char(tonumber(e:sub(3),16)) + else + local ee = entities[e] + if ee then + return ee + else + local h = xml.entities.handler + return (h and h(e)) or "&" .. e .. ";" + end + end + end + + local function resolve_entities(root) + if not root.special or root.tg == "@rt@" then + local d = root.dt + for k=1,#d do + local dk = d[k] + if type(dk) == "string" then + if dk:find("&.-;") then + d[k] = dk:gsub("&(.-);",resolve) + end + else + resolve_entities(dk) + end + end + end + end + + xml.resolve_entities = resolve_entities + + function xml.utfize_text(str) + if str:find("&#") then + return (str:gsub("&#x(.-);",toutf)) + else + return str + end + end + + function xml.resolve_text_entities(str) -- maybe an lpeg. maybe resolve inline + if str:find("&") then + return (str:gsub("&(.-);",resolve)) + else + return str + end + end + + function xml.show_text_entities(str) + if str:find("&") then + return (str:gsub("&(.-);","[%1]")) + else + return str + end + end + + -- experimental, this will be done differently + + function xml.merge_entities(root) + local documententities = root.entities + local allentities = xml.entities + if documententities then + for k, v in pairs(documententities) do + allentities[k] = v + end + end + end + +end end + +-- xml.set_text_cleanup(xml.show_text_entities) +-- xml.set_text_cleanup(xml.resolve_text_entities) + +--~ xml.lshow("/../../../a/(b|c)[@d='e']/f") +--~ xml.lshow("/../../../a/!(b|c)[@d='e']/f") +--~ xml.lshow("/../../../a/!b[@d!='e']/f") + +--~ x = xml.convert([[ +--~ <a> +--~ <b n='01'>01</b> +--~ <b n='02'>02</b> +--~ <b n='03'>03</b> +--~ <b n='04'>OK</b> +--~ <b n='05'>05</b> +--~ <b n='06'>06</b> +--~ <b n='07'>ALSO OK</b> +--~ </a> +--~ ]]) + +--~ xml.trace_lpath = true + +--~ xml.xshow(xml.first(x,"b[position() > 2 and position() < 5 and text() == 'ok']")) +--~ xml.xshow(xml.first(x,"b[position() > 2 and position() < 5 and text() == upper('ok')]")) +--~ xml.xshow(xml.first(x,"b[@n=='03' or @n=='08']")) +--~ xml.xshow(xml.all (x,"b[number(@n)>2 and number(@n)<6]")) +--~ xml.xshow(xml.first(x,"b[find(text(),'ALSO')]")) + +--~ str = [[ +--~ <?xml version="1.0" encoding="utf-8"?> +--~ <story line='mojca'> +--~ <windows>my secret</mouse> +--~ </story> +--~ ]] + +--~ x = xml.convert([[ +--~ <a><b n='01'>01</b><b n='02'>02</b><x>xx</x><b n='03'>03</b><b n='04'>OK</b></a> +--~ ]]) +--~ xml.xshow(xml.first(x,"b[tag(2) == 'x']")) +--~ xml.xshow(xml.first(x,"b[tag(1) == 'x']")) +--~ xml.xshow(xml.first(x,"b[tag(-1) == 'x']")) +--~ xml.xshow(xml.first(x,"b[tag(-2) == 'x']")) + +--~ print(xml.filter(x,"b/tag(2)")) +--~ print(xml.filter(x,"b/tag(1)")) + + +-- filename : l-utils.lua +-- comment : split off from luat-lib +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-utils'] = 1.001 + +if not utils then utils = { } end +if not utils.merger then utils.merger = { } end +if not utils.lua then utils.lua = { } end + +utils.merger.m_begin = "begin library merge" +utils.merger.m_end = "end library merge" +utils.merger.pattern = + "%c+" .. + "%-%-%s+" .. utils.merger.m_begin .. + "%c+(.-)%c+" .. + "%-%-%s+" .. utils.merger.m_end .. + "%c+" + +function utils.merger._self_fake_() + return + "-- " .. "created merged file" .. "\n\n" .. + "-- " .. utils.merger.m_begin .. "\n\n" .. + "-- " .. utils.merger.m_end .. "\n\n" +end + +function utils.report(...) + print(...) +end + +function utils.merger._self_load_(name) + local f, data = io.open(name), "" + if f then + data = f:read("*all") + f:close() + end + return data or "" +end + +function utils.merger._self_save_(name, data) + if data ~= "" then + local f = io.open(name,'w') + if f then + f:write(data) + f:close() + end + end +end + +function utils.merger._self_swap_(data,code) + if data ~= "" then + return (data:gsub(utils.merger.pattern, function(s) + return "\n\n" .. "-- "..utils.merger.m_begin .. "\n" .. code .. "\n" .. "-- "..utils.merger.m_end .. "\n\n" + end, 1)) + else + return "" + end +end + +function utils.merger._self_libs_(libs,list) + local result, f = { }, nil + if type(libs) == 'string' then libs = { libs } end + if type(list) == 'string' then list = { list } end + for _, lib in ipairs(libs) do + for _, pth in ipairs(list) do + local name = string.gsub(pth .. "/" .. lib,"\\","/") + f = io.open(name) + if f then + -- utils.report("merging library",name) + result[#result+1] = f:read("*all") + f:close() + list = { pth } -- speed up the search + break + else + -- utils.report("no library",name) + end + end + end + return table.concat(result, "\n\n") +end + +function utils.merger.selfcreate(libs,list,target) + if target then + utils.merger._self_save_( + target, + utils.merger._self_swap_( + utils.merger._self_fake_(), + utils.merger._self_libs_(libs,list) + ) + ) + end +end + +function utils.merger.selfmerge(name,libs,list,target) + utils.merger._self_save_( + target or name, + utils.merger._self_swap_( + utils.merger._self_load_(name), + utils.merger._self_libs_(libs,list) + ) + ) +end + +function utils.merger.selfclean(name) + utils.merger._self_save_( + name, + utils.merger._self_swap_( + utils.merger._self_load_(name), + "" + ) + ) +end + +utils.lua.compile_strip = true + +function utils.lua.compile(luafile, lucfile) + -- utils.report("compiling",luafile,"into",lucfile) + os.remove(lucfile) + local command = "-o " .. string.quote(lucfile) .. " " .. string.quote(luafile) + if utils.lua.compile_strip then + command = "-s " .. command + end + if os.spawn("texluac " .. command) == 0 then + return true + elseif os.spawn("luac " .. command) == 0 then + return true + else + return false + end +end + + + +-- filename : luat-lib.lua +-- comment : companion to luat-lib.tex +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['luat-lib'] = 1.001 + +-- mostcode moved to the l-*.lua and other luat-*.lua files + +-- os / io + +os.setlocale(nil,nil) -- useless feature and even dangerous in luatex + +-- os.platform + +-- mswin|bccwin|mingw|cygwin windows +-- darwin|rhapsody|nextstep macosx +-- netbsd|unix unix +-- linux linux + +if not io.fileseparator then + if string.find(os.getenv("PATH"),";") then + io.fileseparator, io.pathseparator, os.platform = "\\", ";", os.type or "windows" + else + io.fileseparator, io.pathseparator, os.platform = "/" , ":", os.type or "unix" + end +end + +os.platform = os.platform or os.type or (io.pathseparator == ";" and "windows") or "unix" + +-- arg normalization +-- +-- for k,v in pairs(arg) do print(k,v) end + +-- environment + +if not environment then environment = { } end + +environment.ownbin = environment.ownbin or arg[-2] or arg[-1] or arg[0] or "luatex" + +local ownpath = nil -- we could use a metatable here + +function environment.ownpath() + if not ownpath then + for p in string.gmatch(os.getenv("PATH"),"[^"..io.pathseparator.."]+") do + local b = file.join(p,environment.ownbin) + if lfs.isfile(b..".exe") or lfs.isfile(b) then + ownpath = p + break + end + end + if not ownpath then ownpath = '.' end + end + return ownpath +end + +if arg and (arg[0] == 'luatex' or arg[0] == 'luatex.exe') and arg[1] == "--luaonly" then + arg[-1]=arg[0] arg[0]=arg[2] for k=3,#arg do arg[k-2]=arg[k] end arg[#arg]=nil arg[#arg]=nil +end + +environment.arguments = { } +environment.files = { } +environment.sorted_argument_keys = nil + +environment.platform = os.platform + +function environment.initialize_arguments(arg) + environment.arguments = { } + environment.files = { } + environment.sorted_argument_keys = nil + for index, argument in pairs(arg) do + if index > 0 then + local flag, value = argument:match("^%-+(.+)=(.-)$") + if flag then + environment.arguments[flag] = string.unquote(value or "") + else + flag = argument:match("^%-+(.+)") + if flag then + environment.arguments[flag] = true + else + environment.files[#environment.files+1] = argument + end + end + end + end + environment.ownname = environment.ownname or arg[0] or 'unknown.lua' +end + +function environment.showarguments() + for k,v in pairs(environment.arguments) do + print(k .. " : " .. tostring(v)) + end + if #environment.files > 0 then + print("files : " .. table.concat(environment.files, " ")) + end +end + +function environment.setargument(name,value) + environment.arguments[name] = value +end + +function environment.argument(name) + if environment.arguments[name] then + return environment.arguments[name] + else + if not environment.sorted_argument_keys then + environment.sorted_argument_keys = { } + for _,v in pairs(table.sortedkeys(environment.arguments)) do + table.insert(environment.sorted_argument_keys, "^" .. v) + end + end + for _,v in pairs(environment.sorted_argument_keys) do + if name:find(v) then + return environment.arguments[v:sub(2,#v)] + end + end + end + return nil +end + +function environment.split_arguments(separator) -- rather special, cut-off before separator + local done, before, after = false, { }, { } + for _,v in ipairs(environment.original_arguments) do + if not done and v == separator then + done = true + elseif done then + after[#after+1] = v + else + before[#before+1] = v + end + end + return before, after +end + +function environment.reconstruct_commandline(arg) + if not arg then arg = environment.original_arguments end + local result = { } + for _,a in ipairs(arg) do -- ipairs 1 .. #n + local kk, vv = a:match("^(%-+.-)=(.+)$") + if kk and vv then + if vv:find(" ") then + result[#result+1] = kk .. "=" .. string.quote(vv) + else + result[#result+1] = a + end + elseif a:find(" ") then + result[#result+1] = string.quote(a) + else + result[#result+1] = a + end + end + return table.join(result," ") +end + +if arg then + environment.initialize_arguments(arg) + environment.original_arguments = arg + arg = { } -- prevent duplicate handling +end + + +-- filename : luat-inp.lua +-- comment : companion to luat-lib.tex +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +-- This lib is multi-purpose and can be loaded again later on so that +-- additional functionality becomes available. We will split this +-- module in components when we're done with prototyping. + +-- TODO: os.getenv -> os.env[] +-- TODO: instances.[hashes,cnffiles,configurations,522] -> ipairs (alles check, sneller) +-- TODO: check escaping in find etc, too much, too slow + +-- This is the first code I wrote for LuaTeX, so it needs some cleanup. + +-- To be considered: hash key lowercase, first entry in table filename +-- (any case), rest paths (so no need for optimization). Or maybe a +-- separate table that matches lowercase names to mixed case when +-- present. In that case the lower() cases can go away. I will do that +-- only when we run into problems with names ... well ... Iwona-Regular. + +-- Beware, loading and saving is overloaded in luat-tmp! + +if not versions then versions = { } end versions['luat-inp'] = 1.001 +if not environment then environment = { } end +if not file then file = { } end + +if environment.aleph_mode == nil then environment.aleph_mode = true end -- temp hack + +if not input then input = { } end +if not input.suffixes then input.suffixes = { } end +if not input.formats then input.formats = { } end +if not input.aux then input.aux = { } end + +if not input.suffixmap then input.suffixmap = { } end + +if not input.locators then input.locators = { } end -- locate databases +if not input.hashers then input.hashers = { } end -- load databases +if not input.generators then input.generators = { } end -- generate databases +if not input.filters then input.filters = { } end -- conversion filters + +local format = string.format + +input.locators.notfound = { nil } +input.hashers.notfound = { nil } +input.generators.notfound = { nil } + +input.cacheversion = '1.0.1' +input.banner = nil +input.verbose = false +input.debug = false +input.cnfname = 'texmf.cnf' +input.luaname = 'texmfcnf.lua' +input.lsrname = 'ls-R' +input.luasuffix = '.tma' +input.lucsuffix = '.tmc' + +-- we use a cleaned up list / format=any is a wildcard, as is *name + +input.formats['afm'] = 'AFMFONTS' input.suffixes['afm'] = { 'afm' } +input.formats['enc'] = 'ENCFONTS' input.suffixes['enc'] = { 'enc' } +input.formats['fmt'] = 'TEXFORMATS' input.suffixes['fmt'] = { 'fmt' } +input.formats['map'] = 'TEXFONTMAPS' input.suffixes['map'] = { 'map' } +input.formats['mp'] = 'MPINPUTS' input.suffixes['mp'] = { 'mp' } +input.formats['ocp'] = 'OCPINPUTS' input.suffixes['ocp'] = { 'ocp' } +input.formats['ofm'] = 'OFMFONTS' input.suffixes['ofm'] = { 'ofm', 'tfm' } +input.formats['otf'] = 'OPENTYPEFONTS' input.suffixes['otf'] = { 'otf' } -- 'ttf' +input.formats['opl'] = 'OPLFONTS' input.suffixes['opl'] = { 'opl' } +input.formats['otp'] = 'OTPINPUTS' input.suffixes['otp'] = { 'otp' } +input.formats['ovf'] = 'OVFFONTS' input.suffixes['ovf'] = { 'ovf', 'vf' } +input.formats['ovp'] = 'OVPFONTS' input.suffixes['ovp'] = { 'ovp' } +input.formats['tex'] = 'TEXINPUTS' input.suffixes['tex'] = { 'tex' } +input.formats['tfm'] = 'TFMFONTS' input.suffixes['tfm'] = { 'tfm' } +input.formats['ttf'] = 'TTFONTS' input.suffixes['ttf'] = { 'ttf', 'ttc' } +input.formats['pfb'] = 'T1FONTS' input.suffixes['pfb'] = { 'pfb', 'pfa' } +input.formats['vf'] = 'VFFONTS' input.suffixes['vf'] = { 'vf' } + +input.formats['fea'] = 'FONTFEATURES' input.suffixes['fea'] = { 'fea' } +input.formats['cid'] = 'FONTCIDMAPS' input.suffixes['cid'] = { 'cid', 'cidmap' } + +input.formats ['texmfscripts'] = 'TEXMFSCRIPTS' -- new +input.suffixes['texmfscripts'] = { 'rb', 'pl', 'py' } -- 'lua' + +input.formats ['lua'] = 'LUAINPUTS' -- new +input.suffixes['lua'] = { 'lua', 'luc', 'tma', 'tmc' } + +-- here we catch a few new thingies (todo: add these paths to context.tmf) +-- +-- FONTFEATURES = .;$TEXMF/fonts/fea// +-- FONTCIDMAPS = .;$TEXMF/fonts/cid// + +function input.checkconfigdata(instance) -- not yet ok, no time for debugging now + local function fix(varname,default) + local proname = varname .. "." .. instance.progname or "crap" + local p = instance.environment[proname] + local v = instance.environment[varname] + if not ((p and p ~= "") or (v and v ~= "")) then + instance.variables[varname] = default -- or environment? + end + end + fix("LUAINPUTS" , ".;$TEXINPUTS;$TEXMFSCRIPTS") + fix("FONTFEATURES", ".;$TEXMF/fonts/fea//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") + fix("FONTCIDMAPS" , ".;$TEXMF/fonts/cid//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") +end + +-- backward compatible ones + +input.alternatives = { } + +input.alternatives['map files'] = 'map' +input.alternatives['enc files'] = 'enc' +input.alternatives['cid files'] = 'cid' +input.alternatives['fea files'] = 'fea' +input.alternatives['opentype fonts'] = 'otf' +input.alternatives['truetype fonts'] = 'ttf' +input.alternatives['truetype collections'] = 'ttc' +input.alternatives['type1 fonts'] = 'pfb' + +-- obscure ones + +input.formats ['misc fonts'] = '' +input.suffixes['misc fonts'] = { } + +input.formats ['sfd'] = 'SFDFONTS' +input.suffixes ['sfd'] = { 'sfd' } +input.alternatives['subfont definition files'] = 'sfd' + +function input.reset() + + local instance = { } + + instance.rootpath = '' + instance.treepath = '' + instance.progname = environment.progname or 'context' + instance.engine = environment.engine or 'luatex' + instance.format = '' + instance.environment = { } + instance.variables = { } + instance.expansions = { } + instance.files = { } + instance.remap = { } + instance.configuration = { } + instance.setup = { } + instance.order = { } + instance.found = { } + instance.foundintrees = { } + instance.kpsevars = { } + instance.hashes = { } + instance.cnffiles = { } + instance.luafiles = { } + instance.lists = { } + instance.remember = true + instance.diskcache = true + instance.renewcache = false + instance.scandisk = true + instance.cachepath = nil + instance.loaderror = false + instance.smallcache = false + instance.savelists = true + instance.cleanuppaths = true + instance.allresults = false + instance.pattern = nil -- lists + instance.kpseonly = false -- lists + instance.cachefile = 'tmftools' + instance.loadtime = 0 + instance.starttime = 0 + instance.stoptime = 0 + instance.validfile = function(path,name) return true end + instance.data = { } -- only for loading + instance.force_suffixes = true + instance.dummy_path_expr = "^!*unset/*$" + instance.fakepaths = { } + instance.lsrmode = false + + if os.env then + -- store once, freeze and faster + for k,v in pairs(os.env) do + instance.environment[k] = input.bare_variable(v) + end + else + -- we will access os.env frequently + for k,v in pairs({'HOME','TEXMF','TEXMFCNF'}) do + local e = os.getenv(v) + if e then + -- input.report("setting",v,"to",input.bare_variable(e)) + instance.environment[v] = input.bare_variable(e) + end + end + end + + -- cross referencing + + for k, v in pairs(input.suffixes) do + for _, vv in pairs(v) do + if vv then + input.suffixmap[vv] = k + end + end + end + + return instance + +end + +function input.reset_hashes(instance) + instance.lists = { } + instance.found = { } +end + +function input.bare_variable(str) -- assumes str is a string + -- return string.gsub(string.gsub(string.gsub(str,"%s+$",""),'^"(.+)"$',"%1"),"^'(.+)'$","%1") + return (str:gsub("\s*([\"\']?)(.+)%1\s*", "%2")) +end + +if texio then + input.log = texio.write_nl +else + input.log = print +end + +function input.simple_logger(kind, name) + if name and name ~= "" then + if input.banner then + input.log(input.banner..kind..": "..name) + else + input.log("<<"..kind..": "..name..">>") + end + else + if input.banner then + input.log(input.banner..kind..": no name") + else + input.log("<<"..kind..": no name>>") + end + end +end + +function input.dummy_logger() +end + +function input.settrace(n) + input.trace = tonumber(n or 0) + if input.trace > 0 then + input.logger = input.simple_logger + input.verbose = true + else + input.logger = function() end + end +end + +function input.report(...) -- inefficient + if input.verbose then + if input.banner then + input.log(input.banner .. table.concat({...},' ')) + elseif input.logmode() == 'xml' then + input.log("<t>"..table.concat({...},' ').."</t>") + else + input.log("<<"..table.concat({...},' ')..">>") + end + end +end + +function input.reportlines(str) + if type(str) == "string" then + str = str:split("\n") + end + for _,v in pairs(str) do input.report(v) end +end + +input.settrace(tonumber(os.getenv("MTX.INPUT.TRACE") or os.getenv("MTX_INPUT_TRACE") or input.trace or 0)) + +-- These functions can be used to test the performance, especially +-- loading the database files. + +do + local clock = os.gettimeofday or os.clock + + function input.starttiming(instance) + if instance then + instance.starttime = clock() + if not instance.loadtime then + instance.loadtime = 0 + end + end + end + + function input.stoptiming(instance, report) + if instance then + local starttime = instance.starttime + if starttime then + local stoptime = clock() + local loadtime = stoptime - starttime + instance.stoptime = stoptime + instance.loadtime = instance.loadtime + loadtime + if report then + input.report('load time', format("%0.3f",loadtime)) + end + return loadtime + end + end + return 0 + end + +end + +function input.elapsedtime(instance) + return format("%0.3f",(instance and instance.loadtime) or 0) +end + +function input.report_loadtime(instance) + if instance then + input.report('total load time', input.elapsedtime(instance)) + end +end + +input.loadtime = input.elapsedtime + +function input.env(instance,key) + return instance.environment[key] or input.osenv(instance,key) +end + +function input.osenv(instance,key) + local ie = instance.environment + local value = ie[key] + if value == nil then + -- local e = os.getenv(key) + local e = os.env[key] + if e == nil then + -- value = "" -- false + else + value = input.bare_variable(e) + end + ie[key] = value + end + return value or "" +end + +-- we follow a rather traditional approach: +-- +-- (1) texmf.cnf given in TEXMFCNF +-- (2) texmf.cnf searched in TEXMF/web2c +-- +-- for the moment we don't expect a configuration file in a zip + +function input.identify_cnf(instance) + -- we no longer support treepath and rootpath (was handy for testing); + -- also we now follow the stupid route: if not set then just assume *one* + -- cnf file under texmf (i.e. distribution) + if #instance.cnffiles == 0 then + if input.env(instance,'TEXMFCNF') == "" then + local ownpath = environment.ownpath() or "." + if ownpath then + -- beware, this is tricky on my own system because at that location I do have + -- the raw tree that ends up in the zip; i.e. I cannot test this kind of mess + local function locate(filename,list) + local ownroot = input.normalize_name(file.join(ownpath,"../..")) + if not lfs.isdir(file.join(ownroot,"texmf")) then + ownroot = input.normalize_name(file.join(ownpath,"..")) + if not lfs.isdir(file.join(ownroot,"texmf")) then + input.verbose = true + input.report("error", "unable to identify cnf file") + return + end + end + local texmfcnf = file.join(ownroot,"texmf-local/web2c",filename) -- for minimals and myself + if not lfs.isfile(texmfcnf) then + texmfcnf = file.join(ownroot,"texmf/web2c",filename) + if not lfs.isfile(texmfcnf) then + input.verbose = true + input.report("error", "unable to locate",filename) + return + end + end + table.insert(list,texmfcnf) + local ie = instance.environment + if not ie['SELFAUTOPARENT'] then ie['SELFAUTOPARENT'] = ownroot end + if not ie['TEXMFCNF'] then ie['TEXMFCNF'] = file.dirname(texmfcnf) end + end + locate(input.luaname,instance.luafiles) + locate(input.cnfname,instance.cnffiles) + if #instance.luafiles == 0 and instance.cnffiles == 0 then + input.verbose = true + input.report("error", "unable to locate",filename) + os.exit() + end + -- here we also assume then TEXMF is set in the distribution, if this trickery is + -- used in the minimals, then users who don't use setuptex are on their own with + -- regards to extra trees + else + input.verbose = true + input.report("error", "unable to identify own path") + os.exit() + end + else + local t = input.split_path(input.env(instance,'TEXMFCNF')) + t = input.aux.expanded_path(instance,t) + input.aux.expand_vars(instance,t) + local function locate(filename,list) + for _,v in ipairs(t) do + local texmfcnf = input.normalize_name(file.join(v,filename)) + if lfs.isfile(texmfcnf) then + table.insert(list,texmfcnf) + end + end + end + locate(input.luaname,instance.luafiles) + locate(input.cnfname,instance.cnffiles) + end + end +end + +function input.load_cnf(instance) + local function loadoldconfigdata() + for _, fname in ipairs(instance.cnffiles) do + input.aux.load_cnf(instance,fname) + end + end + -- instance.cnffiles contain complete names now ! + if #instance.cnffiles == 0 then + input.report("no cnf files found (TEXMFCNF may not be set/known)") + else + instance.rootpath = instance.cnffiles[1] + for k,fname in ipairs(instance.cnffiles) do + instance.cnffiles[k] = input.normalize_name(fname:gsub("\\",'/')) + end + for i=1,3 do + instance.rootpath = file.dirname(instance.rootpath) + end + instance.rootpath = input.normalize_name(instance.rootpath) + instance.environment['SELFAUTOPARENT'] = instance.rootpath -- just to be sure + if instance.lsrmode then + loadoldconfigdata() + elseif instance.diskcache and not instance.renewcache then + input.loadoldconfig(instance,instance.cnffiles) + if instance.loaderror then + loadoldconfigdata() + input.saveoldconfig(instance) + end + else + loadoldconfigdata() + if instance.renewcache then + input.saveoldconfig(instance) + end + end + input.aux.collapse_cnf_data(instance) + end + input.checkconfigdata(instance) +end + +function input.load_lua(instance) + if #instance.luafiles == 0 then + -- yet harmless + else + instance.rootpath = instance.luafiles[1] + for k,fname in ipairs(instance.luafiles) do + instance.luafiles[k] = input.normalize_name(fname:gsub("\\",'/')) + end + for i=1,3 do + instance.rootpath = file.dirname(instance.rootpath) + end + instance.rootpath = input.normalize_name(instance.rootpath) + instance.environment['SELFAUTOPARENT'] = instance.rootpath -- just to be sure + input.loadnewconfig(instance) + input.aux.collapse_cnf_data(instance) + end + input.checkconfigdata(instance) +end + +function input.aux.collapse_cnf_data(instance) -- potential optmization: pass start index (setup and configuration are shared) + for _,c in ipairs(instance.order) do + for k,v in pairs(c) do + if not instance.variables[k] then + if instance.environment[k] then + instance.variables[k] = instance.environment[k] + else + instance.kpsevars[k] = true + instance.variables[k] = input.bare_variable(v) + end + end + end + end +end + +function input.aux.load_cnf(instance,fname) + fname = input.clean_path(fname) + local lname = fname:gsub("%.%a+$",input.luasuffix) + local f = io.open(lname) + if f then -- this will go + f:close() + local dname = file.dirname(fname) + if not instance.configuration[dname] then + input.aux.load_configuration(instance,dname,lname) + instance.order[#instance.order+1] = instance.configuration[dname] + end + else + f = io.open(fname) + if f then + input.report("loading", fname) + local line, data, n, k, v + local dname = file.dirname(fname) + if not instance.configuration[dname] then + instance.configuration[dname] = { } + instance.order[#instance.order+1] = instance.configuration[dname] + end + local data = instance.configuration[dname] + while true do + local line, n = f:read(), 0 + if line then + while true do -- join lines + line, n = line:gsub("\\%s*$", "") + if n > 0 then + line = line .. f:read() + else + break + end + end + if not line:find("^[%%#]") then + local k, v = (line:gsub("%s*%%.*$","")):match("%s*(.-)%s*=%s*(.-)%s*$") + if k and v and not data[k] then + data[k] = (v:gsub("[%%#].*",'')):gsub("~", "$HOME") + instance.kpsevars[k] = true + end + end + else + break + end + end + f:close() + else + input.report("skipping", fname) + end + end +end + +-- database loading + +function input.load_hash(instance) + input.locatelists(instance) + if instance.lsrmode then + input.loadlists(instance) + elseif instance.diskcache and not instance.renewcache then + input.loadfiles(instance) + if instance.loaderror then + input.loadlists(instance) + input.savefiles(instance) + end + else + input.loadlists(instance) + if instance.renewcache then + input.savefiles(instance) + end + end +end + +function input.aux.append_hash(instance,type,tag,name) + input.logger("= hash append",tag) + table.insert(instance.hashes, { ['type']=type, ['tag']=tag, ['name']=name } ) +end + +function input.aux.prepend_hash(instance,type,tag,name) + input.logger("= hash prepend",tag) + table.insert(instance.hashes, 1, { ['type']=type, ['tag']=tag, ['name']=name } ) +end + +function input.aux.extend_texmf_var(instance,specification) -- crap + if instance.environment['TEXMF'] then + input.report("extending environment variable TEXMF with", specification) + instance.environment['TEXMF'] = instance.environment['TEXMF']:gsub("^%{", function() + return "{" .. specification .. "," + end) + elseif instance.variables['TEXMF'] then + input.report("extending configuration variable TEXMF with", specification) + instance.variables['TEXMF'] = instance.variables['TEXMF']:gsub("^%{", function() + return "{" .. specification .. "," + end) + else + input.report("setting configuration variable TEXMF to", specification) + instance.variables['TEXMF'] = "{" .. specification .. "}" + end + if instance.variables['TEXMF']:find("%,") and not instance.variables['TEXMF']:find("^%{") then + input.report("adding {} to complex TEXMF variable, best do that yourself") + instance.variables['TEXMF'] = "{" .. instance.variables['TEXMF'] .. "}" + end + input.expand_variables(instance) + input.reset_hashes(instance) +end + +-- locators + +function input.locatelists(instance) + for _, path in pairs(input.simplified_list(input.expansion(instance,'TEXMF'))) do + path = file.collapse_path(path) + input.report("locating list of",path) + input.locatedatabase(instance,input.normalize_name(path)) + end +end + +function input.locatedatabase(instance,specification) + return input.methodhandler('locators', instance, specification) +end + +function input.locators.tex(instance,specification) + if specification and specification ~= '' and lfs.isdir(specification) then + input.logger('! tex locator', specification..' found') + input.aux.append_hash(instance,'file',specification,filename) + else + input.logger('? tex locator', specification..' not found') + end +end + +-- hashers + +function input.hashdatabase(instance,tag,name) + return input.methodhandler('hashers',instance,tag,name) +end + +function input.loadfiles(instance) + instance.loaderror = false + instance.files = { } + if not instance.renewcache then + for _, hash in ipairs(instance.hashes) do + input.hashdatabase(instance,hash.tag,hash.name) + if instance.loaderror then break end + end + end +end + +function input.hashers.tex(instance,tag,name) + input.aux.load_files(instance,tag) +end + +-- generators: + +function input.loadlists(instance) + for _, hash in ipairs(instance.hashes) do + input.generatedatabase(instance,hash.tag) + end +end + +function input.generatedatabase(instance,specification) + return input.methodhandler('generators', instance, specification) +end + +do + + local weird = lpeg.anywhere(lpeg.S("~`!#$%^&*()={}[]:;\"\'||<>,?\n\r\t")) + + function input.generators.tex(instance,specification) + local tag = specification + if not instance.lsrmode and lfs and lfs.dir then + input.report("scanning path",specification) + instance.files[tag] = { } + local files = instance.files[tag] + local n, m, r = 0, 0, 0 + local spec = specification .. '/' + local attributes = lfs.attributes + local directory = lfs.dir + local small = instance.smallcache + local function action(path) + local mode, full + if path then + full = spec .. path .. '/' + else + full = spec + end + for name in directory(full) do + if name:find("^%.") then + -- skip + -- elseif name:find("[%~%`%!%#%$%%%^%&%*%(%)%=%{%}%[%]%:%;\"\'%|%<%>%,%?\n\r\t]") then -- too much escaped + elseif weird:match(name) then + -- texio.write_nl("skipping " .. name) + -- skip + else + mode = attributes(full..name,'mode') + if mode == "directory" then + m = m + 1 + if path then + action(path..'/'..name) + else + action(name) + end + elseif path and mode == 'file' then + n = n + 1 + local f = files[name] + if f then + if not small then + if type(f) == 'string' then + files[name] = { f, path } + else + f[#f+1] = path + end + end + else + files[name] = path + local lower = name:lower() + if name ~= lower then + files["remap:"..lower] = name + r = r + 1 + end + end + end + end + end + end + action() + input.report(format("%s files found on %s directories with %s uppercase remappings",n,m,r)) + else + local fullname = file.join(specification,input.lsrname) + local path = '.' + local f = io.open(fullname) + if f then + instance.files[tag] = { } + local files = instance.files[tag] + local small = instance.smallcache + input.report("loading lsr file",fullname) + -- for line in f:lines() do -- much slower then the next one + for line in (f:read("*a")):gmatch("(.-)\n") do + if line:find("^[%a%d]") then + local fl = files[line] + if fl then + if not small then + if type(fl) == 'string' then + files[line] = { fl, path } -- table + else + fl[#fl+1] = path + end + end + else + files[line] = path -- string + local lower = line:lower() + if line ~= lower then + files["remap:"..lower] = line + end + end + else + path = line:match("%.%/(.-)%:$") or path -- match could be nil due to empty line + end + end + f:close() + end + end + end + +end + +-- savers, todo + +function input.savefiles(instance) + input.aux.save_data(instance, 'files', function(k,v) + return instance.validfile(k,v) -- path, name + end) +end + +-- A config (optionally) has the paths split in tables. Internally +-- we join them and split them after the expansion has taken place. This +-- is more convenient. + +function input.splitconfig(instance) + for i,c in ipairs(instance) do + for k,v in pairs(c) do + if type(v) == 'string' then + local t = file.split_path(v) + if #t > 1 then + c[k] = t + end + end + end + end +end +function input.joinconfig(instance) + for i,c in ipairs(instance.order) do + for k,v in pairs(c) do + if type(v) == 'table' then + c[k] = file.join_path(v) + end + end + end +end +function input.split_path(str) + if type(str) == 'table' then + return str + else + return file.split_path(str) + end +end +function input.join_path(str) + if type(str) == 'table' then + return file.join_path(str) + else + return str + end +end + +function input.splitexpansions(instance) + for k,v in pairs(instance.expansions) do + local t, h = { }, { } + for _,vv in pairs(file.split_path(v)) do + if vv ~= "" and not h[vv] then + t[#t+1] = vv + h[vv] = true + end + end + if #t > 1 then + instance.expansions[k] = t + else + instance.expansions[k] = t[1] + end + end +end + +-- end of split/join code + +function input.saveoldconfig(instance) + input.splitconfig(instance) + input.aux.save_data(instance, 'configuration', nil) + input.joinconfig(instance) +end + +input.configbanner = [[ +-- This is a Luatex configuration file created by 'luatools.lua' or +-- 'luatex.exe' directly. For comment, suggestions and questions you can +-- contact the ConTeXt Development Team. This configuration file is +-- not copyrighted. [HH & TH] +]] + +function input.serialize(files) + -- This version is somewhat optimized for the kind of + -- tables that we deal with, so it's much faster than + -- the generic serializer. This makes sense because + -- luatools and mtxtools are called frequently. Okay, + -- we pay a small price for properly tabbed tables. + local t = { } + local concat = table.concat + local sorted = table.sortedkeys + local function dump(k,v,m) + if type(v) == 'string' then + return m .. "['" .. k .. "']='" .. v .. "'," + elseif #v == 1 then + return m .. "['" .. k .. "']='" .. v[1] .. "'," + else + return m .. "['" .. k .. "']={'" .. concat(v,"','").. "'}," + end + end + t[#t+1] = "return {" + if instance.sortdata then + for _, k in pairs(sorted(files)) do + local fk = files[k] + if type(fk) == 'table' then + t[#t+1] = "\t['" .. k .. "']={" + for _, kk in pairs(sorted(fk)) do + t[#t+1] = dump(kk,fk[kk],"\t\t") + end + t[#t+1] = "\t}," + else + t[#t+1] = dump(k,fk,"\t") + end + end + else + for k, v in pairs(files) do + if type(v) == 'table' then + t[#t+1] = "\t['" .. k .. "']={" + for kk,vv in pairs(v) do + t[#t+1] = dump(kk,vv,"\t\t") + end + t[#t+1] = "\t}," + else + t[#t+1] = dump(k,v,"\t") + end + end + end + t[#t+1] = "}" + return concat(t,"\n") +end + +if not texmf then texmf = {} end -- no longer needed, at least not here + +function input.aux.save_data(instance, dataname, check, makename) -- untested without cache overload + for cachename, files in pairs(instance[dataname]) do + local name = (makename or file.join)(cachename,dataname) + local luaname, lucname = name .. input.luasuffix, name .. input.lucsuffix + input.report("preparing " .. dataname .. " for", luaname) + for k, v in pairs(files) do + if not check or check(v,k) then -- path, name + if type(v) == "table" and #v == 1 then + files[k] = v[1] + end + else + files[k] = nil -- false + end + end + local data = { + type = dataname, + root = cachename, + version = input.cacheversion, + date = os.date("%Y-%m-%d"), + time = os.date("%H:%M:%S"), + content = files, + } + local f = io.open(luaname,'w') + if f then + input.report("saving " .. dataname .. " in", luaname) + f:write(input.serialize(data)) + f:close() + input.report("compiling " .. dataname .. " to", lucname) + if not utils.lua.compile(luaname,lucname) then + input.report("compiling failed for " .. dataname .. ", deleting file " .. lucname) + os.remove(lucname) + end + else + input.report("unable to save " .. dataname .. " in " .. name..input.luasuffix) + end + end +end + +function input.aux.load_data(instance,pathname,dataname,filename,makename) -- untested without cache overload + filename = ((not filename or (filename == "")) and dataname) or filename + filename = (makename and makename(dataname,filename)) or file.join(pathname,filename) + local blob = loadfile(filename .. input.lucsuffix) or loadfile(filename .. input.luasuffix) + if blob then + local data = blob() + if data and data.content and data.type == dataname and data.version == input.cacheversion then + input.report("loading",dataname,"for",pathname,"from",filename) + instance[dataname][pathname] = data.content + else + input.report("skipping",dataname,"for",pathname,"from",filename) + instance[dataname][pathname] = { } + instance.loaderror = true + end + else + input.report("skipping",dataname,"for",pathname,"from",filename) + end +end + +-- some day i'll use the nested approach, but not yet (actually we even drop +-- engine/progname support since we have only luatex now) +-- +-- first texmfcnf.lua files are located, next the cached texmf.cnf files +-- +-- return { +-- TEXMFBOGUS = 'effe checken of dit werkt', +-- } + +function input.aux.load_texmfcnf(instance,dataname,pathname) + local filename = file.join(pathname,input.luaname) + local blob = loadfile(filename) + if blob then + local data = blob() + if data then + input.report("loading","configuration file",filename) + if true then + -- flatten to variable.progname + local t = { } + for k, v in pairs(data) do -- v = progname + if type(v) == "string" then + t[k] = v + else + for kk, vv in pairs(v) do -- vv = variable + if type(vv) == "string" then + t[vv.."."..v] = kk + end + end + end + end + instance[dataname][pathname] = t + else + instance[dataname][pathname] = data + end + else + input.report("skipping","configuration file",filename) + instance[dataname][pathname] = { } + instance.loaderror = true + end + else + input.report("skipping","configuration file",filename) + end +end + +function input.aux.load_configuration(instance,dname,lname) + input.aux.load_data(instance,dname,'configuration',lname and file.basename(lname)) +end +function input.aux.load_files(instance,tag) + input.aux.load_data(instance,tag,'files') +end + +function input.resetconfig(instance) + instance.configuration, instance.setup, instance.order, instance.loaderror = { }, { }, { }, false +end + +function input.loadnewconfig(instance) + for _, cnf in ipairs(instance.luafiles) do + local dname = file.dirname(cnf) + input.aux.load_texmfcnf(instance,'setup',dname) + instance.order[#instance.order+1] = instance.setup[dname] + if instance.loaderror then break end + end +end + +function input.loadoldconfig(instance) + if not instance.renewcache then + for _, cnf in ipairs(instance.cnffiles) do + local dname = file.dirname(cnf) + input.aux.load_configuration(instance,dname) + instance.order[#instance.order+1] = instance.configuration[dname] + if instance.loaderror then break end + end + end + input.joinconfig(instance) +end + +function input.expand_variables(instance) + instance.expansions = { } +--~ instance.environment['SELFAUTOPARENT'] = instance.environment['SELFAUTOPARENT'] or instance.rootpath + if instance.engine ~= "" then instance.environment['engine'] = instance.engine end + if instance.progname ~= "" then instance.environment['progname'] = instance.progname end + for k,v in pairs(instance.environment) do + local a, b = k:match("^(%a+)%_(.*)%s*$") + if a and b then + instance.expansions[a..'.'..b] = v + else + instance.expansions[k] = v + end + end + for k,v in pairs(instance.environment) do -- move environment to expansions + if not instance.expansions[k] then instance.expansions[k] = v end + end + for k,v in pairs(instance.variables) do -- move variables to expansions + if not instance.expansions[k] then instance.expansions[k] = v end + end + while true do + local busy = false + for k,v in pairs(instance.expansions) do + local s, n = v:gsub("%$([%a%d%_%-]+)", function(a) + busy = true + return instance.expansions[a] or input.env(instance,a) + end) + local s, m = s:gsub("%$%{([%a%d%_%-]+)%}", function(a) + busy = true + return instance.expansions[a] or input.env(instance,a) + end) + if n > 0 or m > 0 then + instance.expansions[k]= s + end + end + if not busy then break end + end + for k,v in pairs(instance.expansions) do + instance.expansions[k] = v:gsub("\\", '/') + end +end + +function input.aux.expand_vars(instance,lst) -- simple vars + for k,v in pairs(lst) do + lst[k] = v:gsub("%$([%a%d%_%-]+)", function(a) + return instance.variables[a] or input.env(instance,a) + end) + end +end + +function input.aux.expanded_var(instance,var) -- simple vars + return var:gsub("%$([%a%d%_%-]+)", function(a) + return instance.variables[a] or input.env(instance,a) + end) +end + +function input.aux.entry(instance,entries,name) + if name and (name ~= "") then + name = name:gsub('%$','') + local result = entries[name..'.'..instance.progname] or entries[name] + if result then + return result + else + result = input.env(instance,name) + if result then + instance.variables[name] = result + input.expand_variables(instance) + return instance.expansions[name] or "" + end + end + end + return "" +end +function input.variable(instance,name) + return input.aux.entry(instance,instance.variables,name) +end +function input.expansion(instance,name) + return input.aux.entry(instance,instance.expansions,name) +end + +function input.aux.is_entry(instance,entries,name) + if name and name ~= "" then + name = name:gsub('%$','') + return (entries[name..'.'..instance.progname] or entries[name]) ~= nil + else + return false + end +end + +function input.is_variable(instance,name) + return input.aux.is_entry(instance,instance.variables,name) +end +function input.is_expansion(instance,name) + return input.aux.is_entry(instance,instance.expansions,name) +end + +function input.simplified_list(str) + if type(str) == 'table' then + return str -- troubles ; ipv , in texmf + elseif str == '' then + return { } + else + local t = { } + for _,v in ipairs(string.splitchr(str:gsub("^\{(.+)\}$","%1"),",")) do + t[#t+1] = (v:gsub("^[%!]*(.+)[%/\\]*$","%1")) + end + return t + end +end + +function input.unexpanded_path_list(instance,str) + local pth = input.variable(instance,str) + local lst = input.split_path(pth) + return input.aux.expanded_path(instance,lst) +end +function input.unexpanded_path(instance,str) + return file.join_path(input.unexpanded_path_list(instance,str)) +end + +do + local done = { } + + function input.reset_extra_path(instance) + local ep = instance.extra_paths + if not ep then + ep, done = { }, { } + instance.extra_paths = ep + elseif #ep > 0 then + instance.lists, done = { }, { } + end + end + + function input.register_extra_path(instance,paths,subpaths) + local ep = instance.extra_paths or { } + local n = #ep + if paths and paths ~= "" then + if subpaths and subpaths ~= "" then + for p in paths:gmatch("[^,]+") do + -- we gmatch each step again, not that fast, but used seldom + for s in subpaths:gmatch("[^,]+") do + local ps = p .. "/" .. s + if not done[ps] then + ep[#ep+1] = input.clean_path(ps) + done[ps] = true + end + end + end + else + for p in paths:gmatch("[^,]+") do + if not done[p] then + ep[#ep+1] = input.clean_path(p) + done[p] = true + end + end + end + elseif subpaths and subpaths ~= "" then + for i=1,n do + -- we gmatch each step again, not that fast, but used seldom + for s in subpaths:gmatch("[^,]+") do + local ps = ep[i] .. "/" .. s + if not done[ps] then + ep[#ep+1] = input.clean_path(ps) + done[ps] = true + end + end + end + end + if #ep > 0 then + instance.extra_paths = ep -- register paths + end + if #ep > n then + instance.lists = { } -- erase the cache + end + end + +end + +function input.expanded_path_list(instance,str) + local function made_list(list) + local ep = instance.extra_paths + if not ep or #ep == 0 then + return list + else + local done, new = { }, { } + -- honour . .. ../.. but only when at the start + for k, v in ipairs(list) do + if not done[v] then + if v:find("^[%.%/]$") then + done[v] = true + new[#new+1] = v + else + break + end + end + end + -- first the extra paths + for k, v in ipairs(ep) do + if not done[v] then + done[v] = true + new[#new+1] = v + end + end + -- next the formal paths + for k, v in ipairs(list) do + if not done[v] then + done[v] = true + new[#new+1] = v + end + end + return new + end + end + if not str then + return ep or { } + elseif instance.savelists then + -- engine+progname hash + str = str:gsub("%$","") + if not instance.lists[str] then -- cached + local lst = made_list(input.split_path(input.expansion(instance,str))) + instance.lists[str] = input.aux.expanded_path(instance,lst) + end + return instance.lists[str] + else + local lst = input.split_path(input.expansion(instance,str)) + return made_list(input.aux.expanded_path(instance,lst)) + end +end + +function input.expand_path(instance,str) + return file.join_path(input.expanded_path_list(instance,str)) +end + +--~ function input.first_writable_path(instance,name) +--~ for _,v in pairs(input.expanded_path_list(instance,name)) do +--~ if file.is_writable(file.join(v,'luatex-cache.tmp')) then +--~ return v +--~ end +--~ end +--~ return "." +--~ end + +function input.expanded_path_list_from_var(instance,str) -- brrr + local tmp = input.var_of_format_or_suffix(str:gsub("%$","")) + if tmp ~= "" then + return input.expanded_path_list(instance,str) + else + return input.expanded_path_list(instance,tmp) + end +end +function input.expand_path_from_var(instance,str) + return file.join_path(input.expanded_path_list_from_var(instance,str)) +end + +function input.format_of_var(str) + return input.formats[str] or input.formats[input.alternatives[str]] or '' +end +function input.format_of_suffix(str) + return input.suffixmap[file.extname(str)] or 'tex' +end + +function input.variable_of_format(str) + return input.formats[str] or input.formats[input.alternatives[str]] or '' +end + +function input.var_of_format_or_suffix(str) + local v = input.formats[str] + if v then + return v + end + v = input.formats[input.alternatives[str]] + if v then + return v + end + v = input.suffixmap[file.extname(str)] + if v then + return input.formats[isf] + end + return '' +end + +function input.expand_braces(instance,str) -- output variable and brace expansion of STRING + local ori = input.variable(instance,str) + local pth = input.aux.expanded_path(instance,input.split_path(ori)) + return file.join_path(pth) +end + +-- {a,b,c,d} +-- a,b,c/{p,q,r},d +-- a,b,c/{p,q,r}/d/{x,y,z}// +-- a,b,c/{p,q/{x,y,z},r},d/{p,q,r} +-- a,b,c/{p,q/{x,y,z},r},d/{p,q,r} +-- a{b,c}{d,e}f +-- {a,b,c,d} +-- {a,b,c/{p,q,r},d} +-- {a,b,c/{p,q,r}/d/{x,y,z}//} +-- {a,b,c/{p,q/{x,y,z}},d/{p,q,r}} +-- {a,b,c/{p,q/{x,y,z},w}v,d/{p,q,r}} + +-- this one is better and faster, but it took me a while to realize +-- that this kind of replacement is cleaner than messy parsing and +-- fuzzy concatenating we can probably gain a bit with selectively +-- applying lpeg, but experiments with lpeg parsing this proved not to +-- work that well; the parsing is ok, but dealing with the resulting +-- table is a pain because we need to work inside-out recursively + +-- get rid of piecewise here, just a gmatch is ok + +function input.aux.splitpathexpr(str, t, validate) + -- no need for optimization, only called a few times, we can use lpeg for the sub + t = t or { } + local concat = table.concat + while true do + local done = false + while true do + local ok = false + str = str:gsub("([^{},]+){([^{}]-)}", function(a,b) + local t = { } + b:piecewise(",", function(s) t[#t+1] = a .. s end) + ok, done = true, true + return "{" .. concat(t,",") .. "}" + end) + if not ok then break end + end + while true do + local ok = false + str = str:gsub("{([^{}]-)}([^{},]+)", function(a,b) + local t = { } + a:piecewise(",", function(s) t[#t+1] = s .. b end) + ok, done = true, true + return "{" .. concat(t,",") .. "}" + end) + if not ok then break end + end + while true do + local ok = false + str = str:gsub("([,{]){([^{}]+)}([,}])", function(a,b,c) + ok, done = true, true + return a .. b .. c + end) + if not ok then break end + end + if not done then break end + end + while true do + local ok = false + str = str:gsub("{([^{}]-)}{([^{}]-)}", function(a,b) + local t = { } + a:piecewise(",", function(sa) + b:piecewise(",", function(sb) + t[#t+1] = sa .. sb + end) + end) + ok = true + return "{" .. concat(t,",") .. "}" + end) + if not ok then break end + end + while true do + local ok = false + str = str:gsub("{([^{}]-)}", function(a) + ok = true + return a + end) + if not ok then break end + end + if validate then + str:piecewise(",", function(s) + s = validate(s) + if s then t[#t+1] = s end + end) + else + str:piecewise(",", function(s) + t[#t+1] = s + end) + end + return t +end + +function input.aux.expanded_path(instance,pathlist) -- maybe not a list, just a path + -- a previous version fed back into pathlist + local newlist, ok = { }, false + for _,v in ipairs(pathlist) do + if v:find("[{}]") then + ok = true + break + end + end + if ok then + for _, v in ipairs(pathlist) do + input.aux.splitpathexpr(v, newlist, function(s) + s = file.collapse_path(s) + return s ~= "" and not s:find(instance.dummy_path_expr) and s + end) + end + else + for _,v in ipairs(pathlist) do + for vv in string.gmatch(v..',',"(.-),") do + vv = file.collapse_path(v) + if vv ~= "" then newlist[#newlist+1] = vv end + end + end + end + return newlist +end + +input.is_readable = { } + +function input.aux.is_readable(readable, name) + if input.trace > 2 then + if readable then + input.logger("+ readable", name) + else + input.logger("- readable", name) + end + end + return readable +end + +function input.is_readable.file(name) + -- return input.aux.is_readable(file.is_readable(name), name) + return input.aux.is_readable(input.aux.is_file(name), name) +end + +input.is_readable.tex = input.is_readable.file + +-- name +-- name/name + +function input.aux.collect_files(instance,names) + local filelist = { } + for _, fname in pairs(names) do + if fname then + if input.trace > 2 then + input.logger("? blobpath asked",fname) + end + local bname = file.basename(fname) + local dname = file.dirname(fname) + if dname == "" or dname:find("^%.") then + dname = false + else + dname = "/" .. dname .. "$" + end + for _, hash in ipairs(instance.hashes) do + local blobpath = hash.tag + local files = blobpath and instance.files[blobpath] + if files then + if input.trace > 2 then + input.logger('? blobpath do',blobpath .. " (" .. bname ..")") + end + local blobfile = files[bname] + if not blobfile then + local rname = "remap:"..bname + blobfile = files[rname] + if blobfile then + bname = files[rname] + blobfile = files[bname] + end + end + if blobfile then + if type(blobfile) == 'string' then + if not dname or blobfile:find(dname) then + filelist[#filelist+1] = { + hash.type, + file.join(blobpath,blobfile,bname), -- search + input.concatinators[hash.type](blobpath,blobfile,bname) -- result + } + end + else + for _, vv in pairs(blobfile) do + if not dname or vv:find(dname) then + filelist[#filelist+1] = { + hash.type, + file.join(blobpath,vv,bname), -- search + input.concatinators[hash.type](blobpath,vv,bname) -- result + } + end + end + end + end + elseif input.trace > 1 then + input.logger('! blobpath no',blobpath .. " (" .. bname ..")" ) + end + end + end + end + if #filelist > 0 then + return filelist + else + return nil + end +end + +function input.suffix_of_format(str) + if input.suffixes[str] then + return input.suffixes[str][1] + else + return "" + end +end + +function input.suffixes_of_format(str) + if input.suffixes[str] then + return input.suffixes[str] + else + return {} + end +end + +do + + -- called about 700 times for an empty doc (font initializations etc) + -- i need to weed the font files for redundant calls + + local letter = lpeg.R("az","AZ") + local separator = lpeg.P("://") + + local qualified = lpeg.P(".")^0 * lpeg.P("/") + letter*lpeg.P(":") + letter^1*separator + local rootbased = lpeg.P("/") + letter*lpeg.P(":") + + -- ./name ../name /name c: :// + function input.aux.qualified_path(filename) + return qualified:match(filename) + end + function input.aux.rootbased_path(filename) + return rootbased:match(filename) + end + + function input.normalize_name(original) + return original + end + + input.normalize_name = file.collapse_path + +end + +function input.aux.register_in_trees(instance,name) + if not name:find("^%.") then + instance.foundintrees[name] = (instance.foundintrees[name] or 0) + 1 -- maybe only one + end +end + +-- split the next one up, better for jit + +function input.aux.find_file(instance,filename) -- todo : plugin (scanners, checkers etc) + local result = { } + local stamp = nil + filename = input.normalize_name(filename) -- elsewhere + filename = file.collapse_path(filename:gsub("\\","/")) -- elsewhere + -- speed up / beware: format problem + if instance.remember then + stamp = filename .. "--" .. instance.engine .. "--" .. instance.progname .. "--" .. instance.format + if instance.found[stamp] then + input.logger('! remembered', filename) + return instance.found[stamp] + end + end + if filename:find('%*') then + input.logger('! wildcard', filename) + result = input.find_wildcard_files(instance,filename) + elseif input.aux.qualified_path(filename) then + if input.is_readable.file(filename) then + input.logger('! qualified', filename) + result = { filename } + else + local forcedname, ok = "", false + if file.extname(filename) == "" then + if instance.format == "" then + forcedname = filename .. ".tex" + if input.is_readable.file(forcedname) then + input.logger('! no suffix, forcing standard filetype tex') + result, ok = { forcedname }, true + end + else + for _, s in pairs(input.suffixes_of_format(instance.format)) do + forcedname = filename .. "." .. s + if input.is_readable.file(forcedname) then + input.logger('! no suffix, forcing format filetype', s) + result, ok = { forcedname }, true + break + end + end + end + end + if not ok then + input.logger('? qualified', filename) + end + end + else + -- search spec + local filetype, extra, done, wantedfiles, ext = '', nil, false, { }, file.extname(filename) + if ext == "" then + if not instance.force_suffixes then + wantedfiles[#wantedfiles+1] = filename + end + else + wantedfiles[#wantedfiles+1] = filename + end + if instance.format == "" then + if ext == "" then + local forcedname = filename .. '.tex' + wantedfiles[#wantedfiles+1] = forcedname + filetype = input.format_of_suffix(forcedname) + input.logger('! forcing filetype',filetype) + else + filetype = input.format_of_suffix(filename) + input.logger('! using suffix based filetype',filetype) + end + else + if ext == "" then + for _, s in pairs(input.suffixes_of_format(instance.format)) do + wantedfiles[#wantedfiles+1] = filename .. "." .. s + end + end + filetype = instance.format + input.logger('! using given filetype',filetype) + end + local typespec = input.variable_of_format(filetype) + local pathlist = input.expanded_path_list(instance,typespec) + if not pathlist or #pathlist == 0 then + -- no pathlist, access check only / todo == wildcard + if input.trace > 2 then + input.logger('? filename',filename) + input.logger('? filetype',filetype or '?') + input.logger('? wanted files',table.concat(wantedfiles," | ")) + end + for _, fname in pairs(wantedfiles) do + if fname and input.is_readable.file(fname) then + filename, done = fname, true + result[#result+1] = file.join('.',fname) + break + end + end + -- this is actually 'other text files' or 'any' or 'whatever' + local filelist = input.aux.collect_files(instance,wantedfiles) + local fl = filelist and filelist[1] + if fl then + filename = fl[3] + result[#result+1] = filename + done = true + end + else + -- list search + local filelist = input.aux.collect_files(instance,wantedfiles) + local doscan, recurse + if input.trace > 2 then + input.logger('? filename',filename) + -- if pathlist then input.logger('? path list',table.concat(pathlist," | ")) end + -- if filelist then input.logger('? file list',table.concat(filelist," | ")) end + end + -- a bit messy ... esp the doscan setting here + for _, path in pairs(pathlist) do + if path:find("^!!") then doscan = false else doscan = true end + if path:find("//$") then recurse = true else recurse = false end + local pathname = path:gsub("^!+", '') + done = false + -- using file list + if filelist and not (done and not instance.allresults) and recurse then + -- compare list entries with permitted pattern + pathname = pathname:gsub("([%-%.])","%%%1") -- this also influences + pathname = pathname:gsub("/+$", '/.*') -- later usage of pathname + pathname = pathname:gsub("//", '/.-/') -- not ok for /// but harmless + local expr = "^" .. pathname + -- input.debug('?',expr) + for _, fl in ipairs(filelist) do + local f = fl[2] + if f:find(expr) then + -- input.debug('T',' '..f) + if input.trace > 2 then + input.logger('= found in hash',f) + end + --- todo, test for readable + result[#result+1] = fl[3] + input.aux.register_in_trees(instance,f) -- for tracing used files + done = true + if not instance.allresults then break end + else + -- input.debug('F',' '..f) + end + end + end + if not done and doscan then + -- check if on disk / unchecked / does not work at all / also zips + if input.method_is_file(pathname) then -- ? + local pname = pathname:gsub("%.%*$",'') + if not pname:find("%*") then + local ppname = pname:gsub("/+$","") + if input.aux.can_be_dir(instance,ppname) then + for _, w in pairs(wantedfiles) do + local fname = file.join(ppname,w) + if input.is_readable.file(fname) then + if input.trace > 2 then + input.logger('= found by scanning',fname) + end + result[#result+1] = fname + done = true + if not instance.allresults then break end + end + end + else + -- no access needed for non existing path, speedup (esp in large tree with lots of fake) + end + end + end + end + if not done and doscan then + -- todo: slow path scanning + end + if done and not instance.allresults then break end + end + end + end + for k,v in pairs(result) do + result[k] = file.collapse_path(v) + end + if instance.remember then + instance.found[stamp] = result + end + return result +end + +input.aux._find_file_ = input.aux.find_file + +function input.aux.find_file(instance,filename) -- maybe make a lowres cache too + local result = input.aux._find_file_(instance,filename) + if #result == 0 then + local lowered = filename:lower() + if filename ~= lowered then + return input.aux._find_file_(instance,lowered) + end + end + return result +end + +if lfs and lfs.isfile then + input.aux.is_file = lfs.isfile -- to be done: use this +else + input.aux.is_file = file.is_readable +end + +if lfs and lfs.isdir then + function input.aux.can_be_dir(instance,name) + if not instance.fakepaths[name] then + if lfs.isdir(name) then + instance.fakepaths[name] = 1 -- directory + else + instance.fakepaths[name] = 2 -- no directory + end + end + return (instance.fakepaths[name] == 1) + end +else + function input.aux.can_be_dir() + return true + end +end + +if not input.concatinators then input.concatinators = { } end + +input.concatinators.tex = file.join +input.concatinators.file = input.concatinators.tex + +function input.find_files(instance,filename,filetype,mustexist) + if type(mustexist) == boolean then + -- all set + elseif type(filetype) == 'boolean' then + filetype, mustexist = nil, false + elseif type(filetype) ~= 'string' then + filetype, mustexist = nil, false + end + instance.format = filetype or '' + local t = input.aux.find_file(instance,filename,true) + instance.format = '' + return t +end + +function input.find_file(instance,filename,filetype,mustexist) + return (input.find_files(instance,filename,filetype,mustexist)[1] or "") +end + +function input.find_given_files(instance,filename) + local bname, result = file.basename(filename), { } + for k, hash in ipairs(instance.hashes) do + local files = instance.files[hash.tag] + local blist = files[bname] + if not blist then + local rname = "remap:"..bname + blist = files[rname] + if blist then + bname = files[rname] + blist = files[bname] + end + end + if blist then + if type(blist) == 'string' then + result[#result+1] = input.concatinators[hash.type](hash.tag,blist,bname) or "" + if not instance.allresults then break end + else + for kk,vv in pairs(blist) do + result[#result+1] = input.concatinators[hash.type](hash.tag,vv,bname) or "" + if not instance.allresults then break end + end + end + end + end + return result +end + +function input.find_given_file(instance,filename) + return (input.find_given_files(instance,filename)[1] or "") +end + +function input.find_wildcard_files(instance,filename) -- todo: remap: + local result = { } + local bname, dname = file.basename(filename), file.dirname(filename) + local path = dname:gsub("^*/","") + path = path:gsub("*",".*") + path = path:gsub("-","%%-") + if dname == "" then + path = ".*" + end + local name = bname + name = name:gsub("*",".*") + name = name:gsub("-","%%-") + path = path:lower() + name = name:lower() + local function doit(blist,bname,hash,allresults) + local done = false + if blist then + if type(blist) == 'string' then + -- make function and share code + if (blist:lower()):find(path) then + result[#result+1] = input.concatinators[hash.type](hash.tag,blist,bname) or "" + done = true + end + else + for kk,vv in pairs(blist) do + if (vv:lower()):find(path) then + result[#result+1] = input.concatinators[hash.type](hash.tag,vv,bname) or "" + done = true + if not allresults then break end + end + end + end + end + return done + end + local files, allresults, done = instance.files, instance.allresults, false + if name:find("%*") then + for k, hash in ipairs(instance.hashes) do + for kk, hh in pairs(files[hash.tag]) do + if not kk:find("^remap:") then + if (kk:lower()):find(name) then + if doit(hh,kk,hash,allresults) then done = true end + if done and not allresults then break end + end + end + end + end + else + for k, hash in ipairs(instance.hashes) do + if doit(files[hash.tag][bname],bname,hash,allresults) then done = true end + if done and not allresults then break end + end + end + return result +end + +function input.find_wildcard_file(instance,filename) + return (input.find_wildcard_files(instance,filename)[1] or "") +end + +-- main user functions + +function input.save_used_files_in_trees(instance, filename,jobname) + if not filename then filename = 'luatex.jlg' end + local f = io.open(filename,'w') + if f then + f:write("<?xml version='1.0' standalone='yes'?>\n") + f:write("<rl:job>\n") + if jobname then + f:write("\t<rl:name>" .. jobname .. "</rl:name>\n") + end + f:write("\t<rl:files>\n") + for _,v in pairs(table.sortedkeys(instance.foundintrees)) do + f:write("\t\t<rl:file n='" .. instance.foundintrees[v] .. "'>" .. v .. "</rl:file>\n") + end + f:write("\t</rl:files>\n") + f:write("</rl:usedfiles>\n") + f:close() + end +end + +function input.automount(instance) + -- implemented later +end + +function input.load(instance) + input.starttiming(instance) + input.resetconfig(instance) + input.identify_cnf(instance) + input.load_lua(instance) + input.expand_variables(instance) + input.load_cnf(instance) + input.expand_variables(instance) + input.load_hash(instance) + input.automount(instance) + input.stoptiming(instance) +end + +function input.for_files(instance, command, files, filetype, mustexist) + if files and #files > 0 then + local function report(str) + if input.verbose then + input.report(str) -- has already verbose + else + print(str) + end + end + if input.verbose then + report('') + end + for _, file in pairs(files) do + local result = command(instance,file,filetype,mustexist) + if type(result) == 'string' then + report(result) + else + for _,v in pairs(result) do + report(v) + end + end + end + end +end + +-- strtab + +function input.var_value(instance,str) -- output the value of variable $STRING. + return input.variable(instance,str) +end +function input.expand_var(instance,str) -- output variable expansion of STRING. + return input.expansion(instance,str) +end +function input.show_path(instance,str) -- output search path for file type NAME + return file.join_path(input.expanded_path_list(instance,input.format_of_var(str))) +end + +-- input.find_file(filename) +-- input.find_file(filename, filetype, mustexist) +-- input.find_file(filename, mustexist) +-- input.find_file(filename, filetype) + +function input.aux.register_file(files, name, path) + if files[name] then + if type(files[name]) == 'string' then + files[name] = { files[name], path } + else + files[name] = path + end + else + files[name] = path + end +end + +if not input.finders then input.finders = { } end +if not input.openers then input.openers = { } end +if not input.loaders then input.loaders = { } end + +input.finders.notfound = { nil } +input.openers.notfound = { nil } +input.loaders.notfound = { false, nil, 0 } + +function input.splitmethod(filename) + if not filename then + return { } -- safeguard + elseif type(filename) == "table" then + return filename -- already split + elseif not filename:find("://") then + return { scheme="file", path = filename, original=filename } -- quick hack + else + return url.hashed(filename) + end +end + +function input.method_is_file(filename) + return input.splitmethod(filename).scheme == 'file' +end + +function table.sequenced(t,sep) -- temp here + local s = { } + for k, v in pairs(t) do + s[#s+1] = k .. "=" .. v + end + return table.concat(s, sep or " | ") +end + +function input.methodhandler(what, instance, filename, filetype) -- ... + local specification = (type(filename) == "string" and input.splitmethod(filename)) or filename -- no or { }, let it bomb + local scheme = specification.scheme + if input[what][scheme] then + input.logger('= handler',specification.original .." -> " .. what .. " -> " .. table.sequenced(specification)) + return input[what][scheme](instance,filename,filetype) -- todo: specification + else + return input[what].tex(instance,filename,filetype) -- todo: specification + end +end + +-- also inside next test? + +function input.findtexfile(instance, filename, filetype) + return input.methodhandler('finders',instance, input.normalize_name(filename), filetype) +end +function input.opentexfile(instance,filename) + return input.methodhandler('openers',instance, input.normalize_name(filename)) +end + +function input.findbinfile(instance, filename, filetype) + return input.methodhandler('finders',instance, input.normalize_name(filename), filetype) +end +function input.openbinfile(instance,filename) + return input.methodhandler('loaders',instance, input.normalize_name(filename)) +end + +function input.loadbinfile(instance, filename, filetype) + local fname = input.findbinfile(instance, input.normalize_name(filename), filetype) + if fname and fname ~= "" then + return input.openbinfile(instance,fname) + else + return unpack(input.loaders.notfound) + end +end + +function input.texdatablob(instance, filename, filetype) + local ok, data, size = input.loadbinfile(instance, filename, filetype) + return data or "" +end + +input.loadtexfile = input.texdatablob + +function input.openfile(filename) -- brrr texmf.instance here / todo ! ! ! ! ! + local fullname = input.findtexfile(texmf.instance, filename) + if fullname and (fullname ~= "") then + return input.opentexfile(texmf.instance, fullname) + else + return nil + end +end + +function input.logmode() + return (os.getenv("MTX.LOG.MODE") or os.getenv("MTX_LOG_MODE") or "tex"):lower() +end + +-- this is a prelude to engine/progname specific configuration files +-- in which case we can omit files meant for other programs and +-- packages + +--- ctx + +-- maybe texinputs + font paths +-- maybe positive selection tex/context fonts/tfm|afm|vf|opentype|type1|map|enc + +input.validators = { } +input.validators.visibility = { } + +function input.validators.visibility.default(path, name) + return true +end + +function input.validators.visibility.context(path, name) + path = path[1] or path -- some day a loop + return not ( + path:find("latex") or +-- path:find("doc") or + path:find("tex4ht") or + path:find("source") or +-- path:find("config") or +-- path:find("metafont") or + path:find("lists$") or + name:find("%.tpm$") or + name:find("%.bak$") + ) +end + +-- todo: describe which functions are public (maybe input.private. ... ) + +-- beware: i need to check where we still need a / on windows: + +function input.clean_path(str) +--~ return (((str:gsub("\\","/")):gsub("^!+","")):gsub("//+","//")) + if str then + return ((str:gsub("\\","/")):gsub("^!+","")) + else + return nil + end +end + +function input.do_with_path(name,func) + for _, v in pairs(input.expanded_path_list(instance,name)) do + func("^"..input.clean_path(v)) + end +end + +function input.do_with_var(name,func) + func(input.aux.expanded_var(name)) +end + +function input.with_files(instance,pattern,handle) + for _, hash in ipairs(instance.hashes) do + local blobpath = hash.tag + local blobtype = hash.type + if blobpath then + local files = instance.files[blobpath] + if files then + for k,v in pairs(files) do + if k:find("^remap:") then + k = files[k] + v = files[k] -- chained + end + if k:find(pattern) then + if type(v) == "string" then + handle(blobtype,blobpath,v,k) + else + for _,vv in pairs(v) do + handle(blobtype,blobpath,vv,k) + end + end + end + end + end + end + end +end + +--~ function input.update_script(oldname,newname) -- oldname -> own.name, not per se a suffix +--~ newname = file.addsuffix(newname,"lua") +--~ local newscript = input.clean_path(input.find_file(instance, newname)) +--~ local oldscript = input.clean_path(oldname) +--~ input.report("old script", oldscript) +--~ input.report("new script", newscript) +--~ if oldscript ~= newscript and (oldscript:find(file.removesuffix(newname).."$") or oldscript:find(newname.."$")) then +--~ local newdata = io.loaddata(newscript) +--~ if newdata then +--~ input.report("old script content replaced by new content") +--~ io.savedata(oldscript,newdata) +--~ end +--~ end +--~ end + +function input.update_script(instance,oldname,newname) -- oldname -> own.name, not per se a suffix + local scriptpath = "scripts/context/lua" + newname = file.addsuffix(newname,"lua") + local oldscript = input.clean_path(oldname) + input.report("to be replaced old script", oldscript) + local newscripts = input.find_files(instance, newname) or { } + if #newscripts == 0 then + input.report("unable to locate new script") + else + for _, newscript in ipairs(newscripts) do + newscript = input.clean_path(newscript) + input.report("checking new script", newscript) + if oldscript == newscript then + input.report("old and new script are the same") + elseif not newscript:find(scriptpath) then + input.report("new script should come from",scriptpath) + elseif not (oldscript:find(file.removesuffix(newname).."$") or oldscript:find(newname.."$")) then + input.report("invalid new script name") + else + local newdata = io.loaddata(newscript) + if newdata then + input.report("old script content replaced by new content") + io.savedata(oldscript,newdata) + break + else + input.report("unable to load new script") + end + end + end + end +end + + +--~ print(table.serialize(input.aux.splitpathexpr("/usr/share/texmf-{texlive,tetex}", {}))) + +-- command line resolver: + +--~ print(input.resolve("abc env:tmp file:cont-en.tex path:cont-en.tex full:cont-en.tex rel:zapf/one/p-chars.tex")) + +do + + local resolvers = { } + + resolvers.environment = function(instance,str) + return input.clean_path(os.getenv(str) or os.getenv(str:upper()) or os.getenv(str:lower()) or "") + end + resolvers.relative = function(instance,str,n) + if io.exists(str) then + -- nothing + elseif io.exists("./" .. str) then + str = "./" .. str + else + local p = "../" + for i=1,n or 2 do + if io.exists(p .. str) then + str = p .. str + break + else + p = p .. "../" + end + end + end + return input.clean_path(str) + end + resolvers.locate = function(instance,str) + local fullname = input.find_given_file(instance,str) or "" + return input.clean_path((fullname ~= "" and fullname) or str) + end + resolvers.filename = function(instance,str) + local fullname = input.find_given_file(instance,str) or "" + return input.clean_path(file.basename((fullname ~= "" and fullname) or str)) + end + resolvers.pathname = function(instance,str) + local fullname = input.find_given_file(instance,str) or "" + return input.clean_path(file.dirname((fullname ~= "" and fullname) or str)) + end + + resolvers.env = resolvers.environment + resolvers.rel = resolvers.relative + resolvers.loc = resolvers.locate + resolvers.kpse = resolvers.locate + resolvers.full = resolvers.locate + resolvers.file = resolvers.filename + resolvers.path = resolvers.pathname + + local function resolve(instance,str) + if type(str) == "table" then + for k, v in pairs(str) do + str[k] = resolve(instance,v) or v + end + elseif str and str ~= "" then + str = str:gsub("([a-z]+):([^ ]+)", function(method,target) + if resolvers[method] then + return resolvers[method](instance,target) + else + return method .. ":" .. target + end + end) + end + return str + end + + input.resolve = resolve + +end + + +if not modules then modules = { } end modules ['luat-tmp'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +--[[ldx-- +<p>This module deals with caching data. It sets up the paths and +implements loaders and savers for tables. Best is to set the +following variable. When not set, the usual paths will be +checked. Personally I prefer the (users) temporary path.</p> + +</code> +TEXMFCACHE=$TMP;$TEMP;$TMPDIR;$TEMPDIR;$HOME;$TEXMFVAR;$VARTEXMF;. +</code> + +<p>Currently we do no locking when we write files. This is no real +problem because most caching involves fonts and the chance of them +being written at the same time is small. We also need to extend +luatools with a recache feature.</p> +--ldx]]-- + +caches = caches or { } +dir = dir or { } +texmf = texmf or { } + +caches.path = caches.path or nil +caches.base = caches.base or "luatex-cache" +caches.more = caches.more or "context" +caches.direct = false -- true is faster but may need huge amounts of memory +caches.trace = false +caches.tree = false +caches.paths = caches.paths or nil +caches.force = false + +input.usecache = not toboolean(os.getenv("TEXMFSHARECACHE") or "false",true) -- true + +function caches.temp(instance) + local function checkpath(cachepath) + if not cachepath or cachepath == "" then + return nil + elseif lfs.attributes(cachepath,"mode") == "directory" then -- lfs.isdir(cachepath) then + return cachepath + elseif caches.force or io.ask(string.format("Should I create the cache path %s?",cachepath), "no", { "yes", "no" }) == "yes" then + dir.mkdirs(cachepath) + return (lfs.attributes(cachepath,"mode") == "directory") and cachepath + else + return nil + end + end + local cachepath = input.expanded_path_list(instance,"TEXMFCACHE") + cachepath = cachepath and #cachepath > 0 and checkpath(cachepath[1]) + if not cachepath then + cachepath = os.getenv("TEXMFCACHE") or os.getenv("HOME") or os.getenv("HOMEPATH") or os.getenv("TMP") or os.getenv("TEMP") or os.getenv("TMPDIR") or nil + cachepath = checkpath(cachepath) + end + if not cachepath then + print("\nfatal error: there is no valid cache path defined\n") + os.exit() + elseif lfs.attributes(cachepath,"mode") ~= "directory" then + print(string.format("\nfatal error: cache path %s is not a directory\n",cachepath)) + os.exit() + end + function caches.temp(instance) + return cachepath + end + return cachepath +end + +function caches.configpath(instance) + return table.concat(instance.cnffiles,";") +end + +function caches.hashed(tree) + return md5.hex((tree:lower()):gsub("[\\\/]+","/")) +end + +function caches.treehash(instance) + local tree = caches.configpath(instance) + if not tree or tree == "" then + return false + else + return caches.hashed(tree) + end +end + +function caches.setpath(instance,...) + if not caches.path then + if not caches.path then + caches.path = caches.temp(instance) + end + caches.path = input.clean_path(caches.path) -- to be sure + if lfs then + caches.tree = caches.tree or caches.treehash(instance) + if caches.tree then + caches.path = dir.mkdirs(caches.path,caches.base,caches.more,caches.tree) + else + caches.path = dir.mkdirs(caches.path,caches.base,caches.more) + end + end + end + if not caches.path then + caches.path = '.' + end + caches.path = input.clean_path(caches.path) + if lfs and not table.is_empty({...}) then + local pth = dir.mkdirs(caches.path,...) + return pth + end + caches.path = dir.expand_name(caches.path) + return caches.path +end + +function caches.definepath(instance,category,subcategory) + return function() + return caches.setpath(instance,category,subcategory) + end +end + +function caches.setluanames(path,name) + return path .. "/" .. name .. ".tma", path .. "/" .. name .. ".tmc" +end + +function caches.loaddata(path,name) + local tmaname, tmcname = caches.setluanames(path,name) + local loader = loadfile(tmcname) or loadfile(tmaname) + if loader then + return loader() + else + return false + end +end + +function caches.is_writable(filepath,filename) + local tmaname, tmcname = caches.setluanames(filepath,filename) + return file.is_writable(tmaname) +end + +function caches.savedata(filepath,filename,data,raw) -- raw needed for file cache + local tmaname, tmcname = caches.setluanames(filepath,filename) + local reduce, simplify = true, true + if raw then + reduce, simplify = false, false + end + if caches.direct then + file.savedata(tmaname, table.serialize(data,'return',true,true)) + else + table.tofile (tmaname, data,'return',true,true) -- maybe not the last true + end + utils.lua.compile(tmaname, tmcname) +end + +-- here we use the cache for format loading (texconfig.[formatname|jobname]) + +--~ if tex and texconfig and texconfig.formatname and texconfig.formatname == "" then +if tex and texconfig and (not texconfig.formatname or texconfig.formatname == "") and texmf.instance then + if not texconfig.luaname then texconfig.luaname = "cont-en.lua" end -- or luc + texconfig.formatname = caches.setpath(texmf.instance,"formats") .. "/" .. texconfig.luaname:gsub("%.lu.$",".fmt") +end + +--[[ldx-- +<p>Once we found ourselves defining similar cache constructs +several times, containers were introduced. Containers are used +to collect tables in memory and reuse them when possible based +on (unique) hashes (to be provided by the calling function).</p> + +<p>Caching to disk is disabled by default. Version numbers are +stored in the saved table which makes it possible to change the +table structures without bothering about the disk cache.</p> + +<p>Examples of usage can be found in the font related code.</p> +--ldx]]-- + +containers = { } +containers.trace = false + +do -- local report + + local function report(container,tag,name) + if caches.trace or containers.trace or container.trace then + logs.report(string.format("%s cache",container.subcategory),string.format("%s: %s",tag,name or 'invalid')) + end + end + + local allocated = { } + + -- tracing + + function containers.define(category, subcategory, version, enabled) + return function() + if category and subcategory then + local c = allocated[category] + if not c then + c = { } + allocated[category] = c + end + local s = c[subcategory] + if not s then + s = { + category = category, + subcategory = subcategory, + storage = { }, + enabled = enabled, + version = version or 1.000, + trace = false, + path = caches.setpath(texmf.instance,category,subcategory), + } + c[subcategory] = s + end + return s + else + return nil + end + end + end + + function containers.is_usable(container, name) + return container.enabled and caches.is_writable(container.path, name) + end + + function containers.is_valid(container, name) + if name and name ~= "" then + local storage = container.storage[name] + return storage and not table.is_empty(storage) and storage.cache_version == container.version + else + return false + end + end + + function containers.read(container,name) + if container.enabled and not container.storage[name] then + container.storage[name] = caches.loaddata(container.path,name) + if containers.is_valid(container,name) then + report(container,"loaded",name) + else + container.storage[name] = nil + end + end + if container.storage[name] then + report(container,"reusing",name) + end + return container.storage[name] + end + + function containers.write(container, name, data) + if data then + data.cache_version = container.version + if container.enabled then + local unique, shared = data.unique, data.shared + data.unique, data.shared = nil, nil + caches.savedata(container.path, name, data) + report(container,"saved",name) + data.unique, data.shared = unique, shared + end + report(container,"stored",name) + container.storage[name] = data + end + return data + end + + function containers.content(container,name) + return container.storage[name] + end + +end + +-- since we want to use the cache instead of the tree, we will now +-- reimplement the saver. + +local save_data = input.aux.save_data + +input.cachepath = nil + +function input.aux.save_data(instance, dataname, check) + input.cachepath = input.cachepath or caches.definepath(instance,"trees") + save_data(instance, dataname, check, function(cachename,dataname) + if input.usecache then + return file.join(input.cachepath(),caches.hashed(cachename)) + else + return file.join(cachename,dataname) + end + end) +end + +local load_data = input.aux.load_data + +function input.aux.load_data(instance,pathname,dataname,filename) + input.cachepath = input.cachepath or caches.definepath(instance,"trees") + load_data(instance,pathname,dataname,filename,function(dataname,filename) + if input.usecache then + return file.join(input.cachepath(),caches.hashed(pathname)) + else + if not filename or (filename == "") then + filename = dataname + end + return file.join(pathname,filename) + end + end) +end + +-- we will make a better format, maybe something xml or just text or lua + +input.automounted = input.automounted or { } + +function input.automount(instance,usecache) + local mountpaths = input.simplified_list(input.expansion(instance,'TEXMFMOUNT')) + if table.is_empty(mountpaths) and usecache then + mountpaths = { caches.setpath(instance,"mount") } + end + if not table.is_empty(mountpaths) then + input.starttiming(instance) + for k, root in pairs(mountpaths) do + local f = io.open(root.."/url.tmi") + if f then + for line in f:lines() do + if line then + if line:find("^[%%#%-]") then -- or %W + -- skip + elseif line:find("^zip://") then + input.report("mounting",line) + table.insert(input.automounted,line) + input.usezipfile(instance,line) + end + end + end + f:close() + end + end + input.stoptiming(instance) + end +end + +-- store info in format + +input.storage = { } +input.storage.data = { } +input.storage.min = 0 -- 500 +input.storage.max = input.storage.min - 1 +input.storage.trace = false -- true +input.storage.done = 0 +input.storage.evaluators = { } +-- (evaluate,message,names) + +function input.storage.register(...) + input.storage.data[#input.storage.data+1] = { ... } +end + +function input.storage.evaluate(name) + input.storage.evaluators[#input.storage.evaluators+1] = name +end + +function input.storage.finalize() -- we can prepend the string with "evaluate:" + for _, t in ipairs(input.storage.evaluators) do + for i, v in pairs(t) do + if type(v) == "string" then + t[i] = loadstring(v)() + elseif type(v) == "table" then + for _, vv in pairs(v) do + if type(vv) == "string" then + t[i] = loadstring(vv)() + end + end + end + end + end +end + +function input.storage.dump() + for name, data in ipairs(input.storage.data) do + local evaluate, message, original, target = data[1], data[2], data[3] ,data[4] + local name, initialize, finalize, code = nil, "", "", "" + for str in target:gmatch("([^%.]+)") do + if name then + name = name .. "." .. str + else + name = str + end + initialize = string.format("%s %s = %s or {} ", initialize, name, name) + end + if evaluate then + finalize = "input.storage.evaluate(" .. name .. ")" + end + input.storage.max = input.storage.max + 1 + if input.storage.trace then + logs.report('storage',string.format('saving %s in slot %s',message,input.storage.max)) + code = + initialize .. + string.format("logs.report('storage','restoring %s from slot %s') ",message,input.storage.max) .. + table.serialize(original,name) .. + finalize + else + code = initialize .. table.serialize(original,name) .. finalize + end + lua.bytecode[input.storage.max] = loadstring(code) + end +end + +if lua.bytecode then -- from 0 upwards + local i = input.storage.min + while lua.bytecode[i] do + lua.bytecode[i]() + lua.bytecode[i] = nil + i = i + 1 + end + input.storage.done = i +end + + +if not modules then modules = { } end modules ['luat-log'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +--[[ldx-- +<p>This is a prelude to a more extensive logging module. For the sake +of parsing log files, in addition to the standard logging we will +provide an <l n='xml'/> structured file. Actually, any logging that +is hooked into callbacks will be \XML\ by default.</p> +--ldx]]-- + +input = input or { } +logs = logs or { } + +--[[ldx-- +<p>This looks pretty ugly but we need to speed things up a bit.</p> +--ldx]]-- + +logs.levels = { + ['error'] = 1, + ['warning'] = 2, + ['info'] = 3, + ['debug'] = 4 +} + +logs.functions = { + 'error', 'warning', 'info', 'debug', 'report', + 'start', 'stop', 'push', 'pop' +} + +logs.callbacks = { + 'start_page_number', + 'stop_page_number', + 'report_output_pages', + 'report_output_log' +} + +logs.xml = logs.xml or { } +logs.tex = logs.tex or { } + +logs.level = 0 + +do + local write_nl, write, format = texio.write_nl or print, texio.write or io.write, string.format + + if texlua then + write_nl = print + write = io.write + end + + function logs.xml.debug(category,str) + if logs.level > 3 then write_nl(format("<d category='%s'>%s</d>",category,str)) end + end + function logs.xml.info(category,str) + if logs.level > 2 then write_nl(format("<i category='%s'>%s</i>",category,str)) end + end + function logs.xml.warning(category,str) + if logs.level > 1 then write_nl(format("<w category='%s'>%s</w>",category,str)) end + end + function logs.xml.error(category,str) + if logs.level > 0 then write_nl(format("<e category='%s'>%s</e>",category,str)) end + end + function logs.xml.report(category,str) + write_nl(format("<r category='%s'>%s</r>",category,str)) + end + + function logs.xml.start() if logs.level > 0 then tw("<%s>" ) end end + function logs.xml.stop () if logs.level > 0 then tw("</%s>") end end + function logs.xml.push () if logs.level > 0 then tw("<!-- ") end end + function logs.xml.pop () if logs.level > 0 then tw(" -->" ) end end + + function logs.tex.debug(category,str) + if logs.level > 3 then write_nl(format("debug >> %s: %s" ,category,str)) end + end + function logs.tex.info(category,str) + if logs.level > 2 then write_nl(format("info >> %s: %s" ,category,str)) end + end + function logs.tex.warning(category,str) + if logs.level > 1 then write_nl(format("warning >> %s: %s",category,str)) end + end + function logs.tex.error(category,str) + if logs.level > 0 then write_nl(format("error >> %s: %s" ,category,str)) end + end + function logs.tex.report(category,str) + write_nl(format("report >> %s: %s" ,category,str)) + end + + function logs.set_level(level) + logs.level = logs.levels[level] or level + end + + function logs.set_method(method) + for _, v in pairs(logs.functions) do + logs[v] = logs[method][v] or function() end + end + if callback and input[method] then + for _, cb in pairs(logs.callbacks) do + callback.register(cb, input[method][cb]) + end + end + end + + function logs.xml.start_page_number() + write_nl(format("<p real='%s' page='%s' sub='%s'", tex.count[0], tex.count[1], tex.count[2])) + end + + function logs.xml.stop_page_number() + write("/>") + write_nl("") + end + + function logs.xml.report_output_pages(p,b) + write_nl(format("<v k='pages' v='%s'/>", p)) + write_nl(format("<v k='bytes' v='%s'/>", b)) + write_nl("") + end + + function logs.xml.report_output_log() + end + +end + +logs.set_level('error') +logs.set_method('tex') + + +if not modules then modules = { } end modules ['luat-sta'] = { + version = 1.001, + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +states = states or { } +states.data = states.data or { } +states.hash = states.hash or { } +states.tag = states.tag or "" +states.filename = states.filename or "" + +function states.save(filename,tag) + tag = tag or states.tag + filename = file.addsuffix(filename or states.filename,'lus') + io.savedata(filename, + "-- generator : luat-sta.lua\n" .. + "-- state tag : " .. tag .. "\n\n" .. + table.serialize(states.data[tag or states.tag] or {},true) + ) +end + +function states.load(filename,tag) + states.filename = filename + states.tag = tag or "whatever" + states.filename = file.addsuffix(states.filename,'lus') + states.data[states.tag], states.hash[states.tag] = (io.exists(filename) and dofile(filename)) or { }, { } +end + +function states.set_by_tag(tag,key,value,default,persistent) + local d, h = states.data[tag], states.hash[tag] + if d then + local dkey, hkey = key, key + local pre, post = key:match("(.+)%.([^%.]+)$") + if pre and post then + for k in pre:gmatch("[^%.]+") do + local dk = d[k] + if not dk then + dk = { } + d[k] = dk + end + d = dk + end + dkey, hkey = post, key + end + if type(value) == nil then + value = value or default + elseif persistent then + value = value or d[dkey] or default + else + value = value or default + end + d[dkey], h[hkey] = value, value + end +end + +function states.get_by_tag(tag,key,default) + local h = states.hash[tag] + if h and h[key] then + return h[key] + else + local d = states.data[tag] + if d then + for k in key:gmatch("[^%.]+") do + local dk = d[k] + if dk then + d = dk + else + return default + end + end + return d or default + end + end +end + +function states.set(key,value,default,persistent) + states.set_by_tag(states.tag,key,value,default,persistent) +end + +function states.get(key,default) + return states.get_by_tag(states.tag,key,default) +end + +--~ states.data.update = { +--~ ["version"] = { +--~ ["major"] = 0, +--~ ["minor"] = 1, +--~ }, +--~ ["rsync"] = { +--~ ["server"] = "contextgarden.net", +--~ ["module"] = "minimals", +--~ ["repository"] = "current", +--~ ["flags"] = "-rpztlv --stats", +--~ }, +--~ ["tasks"] = { +--~ ["update"] = true, +--~ ["make"] = true, +--~ ["delete"] = false, +--~ }, +--~ ["platform"] = { +--~ ["host"] = true, +--~ ["other"] = { +--~ ["mswin"] = false, +--~ ["linux"] = false, +--~ ["linux-64"] = false, +--~ ["osx-intel"] = false, +--~ ["osx-ppc"] = false, +--~ ["sun"] = false, +--~ }, +--~ }, +--~ ["context"] = { +--~ ["available"] = {"current", "beta", "alpha", "experimental"}, +--~ ["selected"] = "current", +--~ }, +--~ ["formats"] = { +--~ ["cont-en"] = true, +--~ ["cont-nl"] = true, +--~ ["cont-de"] = false, +--~ ["cont-cz"] = false, +--~ ["cont-fr"] = false, +--~ ["cont-ro"] = false, +--~ }, +--~ ["engine"] = { +--~ ["pdftex"] = { +--~ ["install"] = true, +--~ ["formats"] = { +--~ ["pdftex"] = true, +--~ }, +--~ }, +--~ ["luatex"] = { +--~ ["install"] = true, +--~ ["formats"] = { +--~ }, +--~ }, +--~ ["xetex"] = { +--~ ["install"] = true, +--~ ["formats"] = { +--~ ["xetex"] = false, +--~ }, +--~ }, +--~ ["metapost"] = { +--~ ["install"] = true, +--~ ["formats"] = { +--~ ["mpost"] = true, +--~ ["metafun"] = true, +--~ }, +--~ }, +--~ }, +--~ ["fonts"] = { +--~ }, +--~ ["doc"] = { +--~ }, +--~ ["modules"] = { +--~ ["f-urwgaramond"] = false, +--~ ["f-urwgothic"] = false, +--~ ["t-bnf"] = false, +--~ ["t-chromato"] = false, +--~ ["t-cmscbf"] = false, +--~ ["t-cmttbf"] = false, +--~ ["t-construction-plan"] = false, +--~ ["t-degrade"] = false, +--~ ["t-french"] = false, +--~ ["t-lettrine"] = false, +--~ ["t-lilypond"] = false, +--~ ["t-mathsets"] = false, +--~ ["t-tikz"] = false, +--~ ["t-typearea"] = false, +--~ ["t-vim"] = false, +--~ }, +--~ } + + +--~ states.save("teststate", "update") +--~ states.load("teststate", "update") + +--~ print(states.get_by_tag("update","rsync.server","unknown")) +--~ states.set_by_tag("update","rsync.server","oeps") +--~ print(states.get_by_tag("update","rsync.server","unknown")) +--~ states.save("teststate", "update") +--~ states.load("teststate", "update") +--~ print(states.get_by_tag("update","rsync.server","unknown")) + +-- end library merge + +own = { } + +own.libs = { -- todo: check which ones are really needed + 'l-string.lua', + 'l-lpeg.lua', + 'l-table.lua', + 'l-io.lua', + 'l-md5.lua', + 'l-number.lua', + 'l-set.lua', + 'l-os.lua', + 'l-file.lua', + 'l-dir.lua', + 'l-boolean.lua', + 'l-xml.lua', +-- 'l-unicode.lua', + 'l-utils.lua', +-- 'l-tex.lua', + 'luat-lib.lua', + 'luat-inp.lua', +-- 'luat-zip.lua', +-- 'luat-tex.lua', +-- 'luat-kps.lua', + 'luat-tmp.lua', + 'luat-log.lua', + 'luat-sta.lua', +} + +-- We need this hack till luatex is fixed. +-- +-- for k,v in pairs(arg) do print(k,v) end + +if arg and (arg[0] == 'luatex' or arg[0] == 'luatex.exe') and arg[1] == "--luaonly" then + arg[-1]=arg[0] arg[0]=arg[2] for k=3,#arg do arg[k-2]=arg[k] end arg[#arg]=nil arg[#arg]=nil +end + +-- End of hack. + +own.name = (environment and environment.ownname) or arg[0] or 'luatools.lua' + +own.path = string.match(own.name,"^(.+)[\\/].-$") or "." +own.list = { '.' } +if own.path ~= '.' then + table.insert(own.list,own.path) +end +table.insert(own.list,own.path.."/../../../tex/context/base") +table.insert(own.list,own.path.."/mtx") +table.insert(own.list,own.path.."/../sources") + +function locate_libs() + for _, lib in pairs(own.libs) do + for _, pth in pairs(own.list) do + local filename = string.gsub(pth .. "/" .. lib,"\\","/") + local codeblob = loadfile(filename) + if codeblob then + codeblob() + own.list = { pth } -- speed up te search + break + end + end + end +end + +if not input then + locate_libs() +end + +if not input then + print("") + print("Mtxrun is unable to start up due to lack of libraries. You may") + print("try to run 'lua mtxrun.lua --selfmerge' in the path where this") + print("script is located (normally under ..../scripts/context/lua) which") + print("will make this script library independent.") + os.exit() +end + +instance = input.reset() +input.verbose = environment.argument("verbose") or false +input.banner = 'MtxRun | ' +utils.report = input.report + +instance.engine = environment.argument("engine") or 'luatex' +instance.progname = environment.argument("progname") or 'context' +instance.lsrmode = environment.argument("lsr") or false + +-- use os.env or environment when available + +--~ function input.check_environment(tree) +--~ input.report('') +--~ os.setenv('TMP', os.getenv('TMP') or os.getenv('TEMP') or os.getenv('TMPDIR') or os.getenv('HOME')) +--~ if os.platform == 'linux' then +--~ os.setenv('TEXOS', os.getenv('TEXOS') or 'texmf-linux') +--~ elseif os.platform == 'windows' then +--~ os.setenv('TEXOS', os.getenv('TEXOS') or 'texmf-windows') +--~ elseif os.platform == 'macosx' then +--~ os.setenv('TEXOS', os.getenv('TEXOS') or 'texmf-macosx') +--~ end +--~ os.setenv('TEXOS', string.gsub(string.gsub(os.getenv('TEXOS'),"^[\\\/]*", ''),"[\\\/]*$", '')) +--~ os.setenv('TEXPATH', string.gsub(tree,"\/+$",'')) +--~ os.setenv('TEXMFOS', os.getenv('TEXPATH') .. "/" .. os.getenv('TEXOS')) +--~ input.report('') +--~ input.report("preset : TEXPATH => " .. os.getenv('TEXPATH')) +--~ input.report("preset : TEXOS => " .. os.getenv('TEXOS')) +--~ input.report("preset : TEXMFOS => " .. os.getenv('TEXMFOS')) +--~ input.report("preset : TMP => " .. os.getenv('TMP')) +--~ input.report('') +--~ end + +function input.check_environment(tree) + input.report('') + os.setenv('TMP', os.getenv('TMP') or os.getenv('TEMP') or os.getenv('TMPDIR') or os.getenv('HOME')) + os.setenv('TEXOS', os.getenv('TEXOS') or ("texmf-" .. os.currentplatform())) + os.setenv('TEXPATH', (tree or "tex"):gsub("\/+$",'')) + os.setenv('TEXMFOS', os.getenv('TEXPATH') .. "/" .. os.getenv('TEXOS')) + input.report('') + input.report("preset : TEXPATH => " .. os.getenv('TEXPATH')) + input.report("preset : TEXOS => " .. os.getenv('TEXOS')) + input.report("preset : TEXMFOS => " .. os.getenv('TEXMFOS')) + input.report("preset : TMP => " .. os.getenv('TMP')) + input.report('') +end + +function input.load_environment(name) -- todo: key=value as well as lua + local f = io.open(name) + if f then + for line in f:lines() do + if line:find("^[%%%#]") then + -- skip comment + else + local key, how, value = line:match("^(.-)%s*([<=>%?]+)%s*(.*)%s*$") + if how then + value = value:gsub("%%(.-)%%", function(v) return os.getenv(v) or "" end) + if how == "=" or how == "<<" then + os.setenv(key,value) + elseif how == "?" or how == "??" then + os.setenv(key,os.getenv(key) or value) + elseif how == "<" or how == "+=" then + if os.getenv(key) then + os.setenv(key,os.getenv(key) .. io.fileseparator .. value) + else + os.setenv(key,value) + end + elseif how == ">" or how == "=+" then + if os.getenv(key) then + os.setenv(key,value .. io.pathseparator .. os.getenv(key)) + else + os.setenv(key,value) + end + end + end + end + end + f:close() + end +end + +function input.load_tree(tree) + if tree and tree ~= "" then + local setuptex = 'setuptex.tmf' + if lfs.attributes(tree, "mode") == "directory" then -- check if not nil + setuptex = tree .. "/" .. setuptex + else + setuptex = tree + end + if io.exists(setuptex) then + input.check_environment(tree) + input.load_environment(setuptex) + end + end +end + +-- md5 extensions + +-- maybe md.md5 md.md5hex md.md5HEX + +if not md5 then md5 = { } end + +if not md5.sum then + function md5.sum(k) + return string.rep("x",16) + end +end + +function md5.hexsum(k) + return (string.gsub(md5.sum(k), ".", function(c) return string.format("%02x", string.byte(c)) end)) +end + +function md5.HEXsum(k) + return (string.gsub(md5.sum(k), ".", function(c) return string.format("%02X", string.byte(c)) end)) +end + +-- file extensions + +file.needs_updating_threshold = 1 + +function file.needs_updating(oldname,newname) -- size modification access change + local oldtime = lfs.attributes(oldname, modification) + local newtime = lfs.attributes(newname, modification) + if newtime >= oldtime then + return false + elseif oldtime - newtime < file.needs_updating_threshold then + return false + else + return true + end +end + +function file.mdchecksum(name) + if md5 then + local data = io.loadall(name) + if data then + return md5.HEXsum(data) + end + end + return nil +end + +function file.loadchecksum(name) + if md then + local data = io.loadall(name .. ".md5") + if data then + return string.gsub(md5.HEXsum(data),"%s$","") + end + end + return nil +end + +function file.savechecksum(name, checksum) + if not checksum then checksum = file.mdchecksum(name) end + if checksum then + local f = io.open(name .. ".md5","w") + if f then + f:write(checksum) + f:close() + return checksum + end + end + return nil +end + +os.arch = os.arch or function() + return os.resultof("uname -m") or "linux" +end + +function os.currentplatform(name, default) + local name = os.name or os.platform or name -- os.name is built in, os.platform is mine + if name then + if name == "windows" or name == "mswin" or name == "win32" or name == "msdos" then + return "mswin" + elseif name == "linux" then + local architecture = os.arch() + if architecture:find("x86_64") then + return "linux-64" + else + return "linux" + end + elseif name == "macosx" then + local architecture = os.arch() + if architecture:find("i386") then + return "osx-intel" + else + return "osx-ppc" + end + elseif name == "freebsd" then + return "freebsd" + end + end + return default or name +end + +-- it starts here + +input.runners = { } +input.runners.applications = { } + +input.runners.applications.lua = "luatex --luaonly" +input.runners.applications.pl = "perl" +input.runners.applications.py = "python" +input.runners.applications.rb = "ruby" + +input.runners.suffixes = { + 'rb', 'lua', 'py', 'pl' +} + +input.runners.registered = { + texexec = { 'texexec.rb', true }, + texutil = { 'texutil.rb', true }, + texfont = { 'texfont.pl', true }, + texshow = { 'texshow.pl', false }, + + makempy = { 'makempy.pl', true }, + mptopdf = { 'mptopdf.pl', true }, + pstopdf = { 'pstopdf.rb', true }, + + examplex = { 'examplex.rb', false }, + concheck = { 'concheck.rb', false }, + + runtools = { 'runtools.rb', true }, + textools = { 'textools.rb', true }, + tmftools = { 'tmftools.rb', true }, + ctxtools = { 'ctxtools.rb', true }, + rlxtools = { 'rlxtools.rb', true }, + pdftools = { 'pdftools.rb', true }, + mpstools = { 'mpstools.rb', true }, + exatools = { 'exatools.rb', true }, + xmltools = { 'xmltools.rb', true }, + luatools = { 'luatools.lua', true }, + mtxtools = { 'mtxtools.rb', true }, + + pdftrimwhite = { 'pdftrimwhite.pl', false } +} + +if not messages then messages = { } end + +messages.help = [[ +--script run an mtx script +--execute run a script or program +--resolve resolve prefixed arguments +--ctxlua run internally (using preloaded libs) +--locate locate given filename + +--autotree use texmf tree cf. env 'texmfstart_tree' or 'texmfstarttree' +--tree=pathtotree use given texmf tree (default file: 'setuptex.tmf') +--environment=name use given (tmf) environment file +--path=runpath go to given path before execution +--ifchanged=filename only execute when given file has changed (md checksum) +--iftouched=old,new only execute when given file has changed (time stamp) + +--make create stubs for (context related) scripts +--remove remove stubs (context related) scripts +--stubpath=binpath paths where stubs wil be written +--windows create windows (mswin) stubs +--unix create unix (linux) stubs + +--verbose give a bit more info +--engine=str target engine +--progname=str format or backend + +--edit launch editor with found file +--launch (--all) launch files (assume os support) + +--intern run script using built in libraries +]] + +function input.runners.my_prepare_a(instance) + input.resetconfig(instance) + input.identify_cnf(instance) + input.load_lua(instance) + input.expand_variables(instance) + input.load_cnf(instance) + input.expand_variables(instance) +end + +function input.runners.my_prepare_b(instance) + input.runners.my_prepare_a(instance) + input.load_hash(instance) + input.automount(instance) +end + +function input.runners.prepare(instance) + local checkname = environment.argument("ifchanged") + if checkname and checkname ~= "" then + local oldchecksum = file.loadchecksum(checkname) + local newchecksum = file.checksum(checkname) + if oldchecksum == newchecksum then + report("file '" .. checkname .. "' is unchanged") + return "skip" + else + report("file '" .. checkname .. "' is changed, processing started") + end + file.savechecksum(checkname) + end + local oldname, newname = string.split(environment.argument("iftouched") or "", ",") + if oldname and newname and oldname ~= "" and newname ~= "" then + if not file.needs_updating(oldname,newname) then + report("file '" .. oldname .. "' and '" .. newname .. "'have same age") + return "skip" + else + report("file '" .. newname .. "' is older than '" .. oldname .. "'") + end + end + local tree = environment.argument('tree') or "" + if environment.argument('autotree') then + tree = os.getenv('TEXMFSTART_TREE') or os.getenv('TEXMFSTARTTREE') or tree + end + if tree and tree ~= "" then + input.load_tree(tree) + end + local env = environment.argument('environment') or "" + if env and env ~= "" then + for _,e in pairs(string.split(env)) do + -- maybe force suffix when not given + input.load_tree(e) + end + end + local runpath = environment.argument("path") + if runpath and not dir.chdir(runpath) then + input.report("unable to change to path '" .. runpath .. "'") + return "error" + end + return "run" +end + +function input.runners.execute_script(instance,fullname,internal) + if fullname and fullname ~= "" then + local state = input.runners.prepare(instance) + if state == 'error' then + return false + elseif state == 'skip' then + return true + elseif state == "run" then + instance.progname = environment.argument("progname") or instance.progname + instance.format = environment.argument("format") or instance.format + local path, name, suffix, result = file.dirname(fullname), file.basename(fullname), file.extname(fullname), "" + if path ~= "" then + result = fullname + elseif name then + name = name:gsub("^int[%a]*:",function() + internal = true + return "" + end ) + name = name:gsub("^script:","") + if suffix == "" and input.runners.registered[name] and input.runners.registered[name][1] then + name = input.runners.registered[name][1] + suffix = file.extname(name) + end + if suffix == "" then + -- loop over known suffixes + for _,s in pairs(input.runners.suffixes) do + result = input.find_file(instance, name .. "." .. s, 'texmfscripts') + if result ~= "" then + break + end + end + elseif input.runners.applications[suffix] then + result = input.find_file(instance, name, 'texmfscripts') + else + -- maybe look on path + result = input.find_file(instance, name, 'other text files') + end + end + if result and result ~= "" then + if internal then + local before, after = environment.split_arguments(fullname) + arg = { } for _,v in pairs(after) do arg[#arg+1] = v end + dofile(result) + else + local binary = input.runners.applications[file.extname(result)] + if binary and binary ~= "" then + result = binary .. " " .. result + end + local before, after = environment.split_arguments(fullname) + local command = result .. " " .. environment.reconstruct_commandline(after) + input.report("") + input.report("executing: " .. command) + input.report("\n \n") + io.flush() + local code = os.exec(command) -- maybe spawn + return code == 0 + end + end + end + end + return false +end + +function input.runners.execute_program(instance,fullname) + if fullname and fullname ~= "" then + local state = input.runners.prepare(instance) + if state == 'error' then + return false + elseif state == 'skip' then + return true + elseif state == "run" then + local before, after = environment.split_arguments(fullname) + environment.initialize_arguments(after) + fullname = fullname:gsub("^bin:","") + local command = fullname .. " " .. environment.reconstruct_commandline(after) + input.report("") + input.report("executing: " .. command) + input.report("\n \n") + io.flush() + local code = os.exec(command) -- (fullname,unpack(after)) does not work / maybe spawn + return code == 0 + end + end + return false +end + +function input.runners.handle_stubs(instance,create) + local stubpath = environment.argument('stubpath') or '.' -- 'auto' no longer supported + local windows = environment.argument('windows') or environment.argument('mswin') or false + local unix = environment.argument('unix') or environment.argument('linux') or false + if not windows and not unix then + if environment.platform == "unix" then + unix = true + else + windows = true + end + end + for _,v in pairs(input.runners.registered) do + local name, doit = v[1], v[2] + if doit then + local base = string.gsub(file.basename(name), "%.(.-)$", "") + if create then + -- direct local command = input.runners.applications[file.extname(name)] .. " " .. name + local command = "luatex --luaonly mtxrun.lua " .. name + if windows then + io.savedata(base..".bat", {"@echo off", command.." %*"}, "\013\010") + input.report("windows stub for '" .. base .. "' created") + end + if unix then + io.savedata(base, {"#!/bin/sh", command..' "$@"'}, "\010") + input.report("unix stub for '" .. base .. "' created") + end + else + if windows and (os.remove(base..'.bat') or os.remove(base..'.cmd')) then + input.report("windows stub for '" .. base .. "' removed") + end + if unix and (os.remove(base) or os.remove(base..'.sh')) then + input.report("unix stub for '" .. base .. "' removed") + end + end + end + end +end + +function input.runners.resolve_string(instance,filename) + if filename and filename ~= "" then + input.runners.report_location(instance,input.resolve(instance,filename)) + end +end + +function input.runners.locate_file(instance,filename) + if filename and filename ~= "" then + input.runners.report_location(instance,input.find_given_file(instance,filename)) + end +end + +function input.runners.locate_platform(instance) + input.runners.report_location(instance,os.currentplatform()) +end + +function input.runners.report_location(instance,result) + if input.verbose then + input.report("") + if result and result ~= "" then + input.report(result) + else + input.report("not found") + end + else + io.write(result) + end +end + +function input.runners.edit_script(instance,filename) + local editor = os.getenv("MTXRUN_EDITOR") or os.getenv("TEXMFSTART_EDITOR") or os.getenv("EDITOR") or 'scite' + local rest = input.resolve(instance,filename) + if rest ~= "" then + os.launch(editor .. " " .. rest) + end +end + +function input.runners.save_script_session(filename, list) + local t = { } + for _, key in ipairs(list) do + t[key] = environment.arguments[key] + end + io.savedata(filename,table.serialize(t,true)) +end + +function input.runners.load_script_session(filename) + if lfs.isfile(filename) then + local t = io.loaddata(filename) + if t then + t = loadstring(t) + if t then t = t() end + for key, value in pairs(t) do + environment.arguments[key] = value + end + end + end +end + +input.runners.launchers = { + windows = { }, + unix = { } +} + +function input.launch(str) + -- maybe we also need to test on mtxrun.launcher.suffix environment + -- variable or on windows consult the assoc and ftype vars and such + local launchers = input.runners.launchers[os.platform] if launchers then + local suffix = file.extname(str) if suffix then + local runner = launchers[suffix] if runner then + str = runner .. " " .. str + end + end + end + os.launch(str) +end + +function input.runners.launch_file(instance,filename) + instance.allresults = true + input.verbose = true + local pattern = environment.arguments["pattern"] + if not pattern or pattern == "" then + pattern = filename + end + if not pattern or pattern == "" then + input.report("provide name or --pattern=") + else + local t = input.find_files(instance,pattern) + -- local t = input.aux.find_file(instance,"*/" .. pattern,true) + if t and #t > 0 then + if environment.arguments["all"] then + for _, v in pairs(t) do + input.report("launching", v) + input.launch(v) + end + else + input.report("launching", t[1]) + input.launch(t[1]) + end + else + input.report("no match for", pattern) + end + end +end + +function input.runners.execute_ctx_script(instance,filename,arguments) + local function found(name) + local path = file.dirname(name) + if path and path ~= "" then + return false + else + local fullname = own and own.path and file.join(own.path,name) + return io.exists(fullname) and fullname + end + end + local suffix = "" + if not filename:find("%.lua$") then suffix = ".lua" end + local fullname = filename + -- just <filename> + fullname = filename .. suffix + fullname = input.find_file(instance,fullname) + -- mtx-<filename> + if not fullname or fullname == "" then + fullname = "mtx-" .. filename .. suffix + fullname = found(fullname) or input.find_file(instance,fullname) + end + -- mtx-<filename>s + if not fullname or fullname == "" then + fullname = "mtx-" .. filename .. "s" .. suffix + fullname = found(fullname) or input.find_file(instance,fullname) + end + -- mtx-<filename minus trailing s> + if not fullname or fullname == "" then + fullname = "mtx-" .. filename:gsub("s$","") .. suffix + fullname = found(fullname) or input.find_file(instance,fullname) + end + -- that should do it + if fullname and fullname ~= "" then + local state = input.runners.prepare(instance) + if state == 'error' then + return false + elseif state == 'skip' then + return true + elseif state == "run" then + -- load and save ... kind of undocumented + arg = { } for _,v in pairs(arguments) do arg[#arg+1] = v end + environment.initialize_arguments(arg) + local loadname = environment.arguments['load'] + if loadname then + if type(loadname) ~= "string" then loadname = file.basename(fullname) end + loadname = file.replacesuffix(loadname,"cfg") + input.runners.load_script_session(loadname) + end + filename = environment.files[1] + if input.verbose then + input.report("using script: " .. fullname) + end + dofile(fullname) + local savename = environment.arguments['save'] + if savename and input.runners.save_list and not table.is_empty(input.runners.save_list or { }) then + if type(savename) ~= "string" then savename = file.basename(fullname) end + savename = file.replacesuffix(savename,"cfg") + input.runners.save_script_session(savename, input.runners.save_list) + end + return true + end + else + input.verbose = true + input.report("unknown script: " .. filename) + return false + end +end + +input.report(banner,"\n") + +function input.help(banner,message) + if not input.verbose then + input.verbose = true + input.report(banner,"\n") + end + input.reportlines(message) +end + +-- this is a bit dirty ... first we store the first filename and next we +-- split the arguments so that we only see the ones meant for this script +-- ... later we will use the second half + +local filename = environment.files[1] or "" +local ok = true + +local before, after = environment.split_arguments(filename) + +input.runners.my_prepare_b(instance) +before = input.resolve(instance,before) -- experimental here +after = input.resolve(instance,after) -- experimental here + +environment.initialize_arguments(before) + +if environment.argument("selfmerge") then + -- embed used libraries + utils.merger.selfmerge(own.name,own.libs,own.list) +elseif environment.argument("selfclean") then + -- remove embedded libraries + utils.merger.selfclean(own.name) +elseif environment.argument("selfupdate") then + input.verbose = true + input.update_script(instance,own.name,"mtxrun") +elseif environment.argument("ctxlua") or environment.argument("internal") then + -- run a script by loading it (using libs) + ok = input.runners.execute_script(instance,filename,true) +elseif environment.argument("script") then + -- run a script by loading it (using libs), pass args + ok = input.runners.execute_ctx_script(instance,filename,after) +elseif environment.argument("execute") then + -- execute script + ok = input.runners.execute_script(instance,filename) +elseif environment.argument("direct") then + -- equals bin: + ok = input.runners.execute_program(instance,filename) +elseif environment.argument("edit") then + -- edit file + input.runners.edit_script(instance,filename) +elseif environment.argument("launch") then + input.runners.launch_file(instance,filename) +elseif environment.argument("make") then + -- make stubs + input.runners.handle_stubs(instance,true) +elseif environment.argument("remove") then + -- remove stub + input.runners.handle_stubs(instance,false) +elseif environment.argument("resolve") then + -- resolve string + input.runners.resolve_string(instance,filename) +elseif environment.argument("locate") then + -- locate file + input.runners.locate_file(instance,filename) +elseif environment.argument("platform")then + -- locate platform + input.runners.locate_platform(instance) +elseif environment.argument("help") or filename=='help' or filename == "" then + input.help(banner,messages.help) + -- execute script + if filename:find("^bin:") then + ok = input.runners.execute_program(instance,filename) + else + ok = input.runners.execute_script(instance,filename) + end +end + +--~ if input.verbose then +--~ input.report("") +--~ input.report(string.format("runtime: %0.3f seconds",os.runtime())) +--~ end + +--~ if ok then +--~ input.report("exit code: 0") os.exit(0) +--~ else +--~ input.report("exit code: 1") os.exit(1) +--~ end + +if environment.platform == "unix" then + io.write("\n") +end diff --git a/Master/texmf-dist/scripts/context/lua/mtxrun.rme b/Master/texmf-dist/scripts/context/lua/mtxrun.rme new file mode 100644 index 00000000000..9cb56486aba --- /dev/null +++ b/Master/texmf-dist/scripts/context/lua/mtxrun.rme @@ -0,0 +1,3 @@ +On MSWindows the mtxrun.lua script is called +with mtxrun.cmd. On Unix you can either rename +mtxrun.lua to mtxrun, or use a symlink. diff --git a/Master/texmf-dist/scripts/context/lua/scite-ctx.lua b/Master/texmf-dist/scripts/context/lua/scite-ctx.lua index 5836bcbfb71..1b832928909 100644 --- a/Master/texmf-dist/scripts/context/lua/scite-ctx.lua +++ b/Master/texmf-dist/scripts/context/lua/scite-ctx.lua @@ -1,7 +1,7 @@ --- version : 1.0.0 - 07/2005 +-- version : 1.0.0 - 07/2005 (2008: lua 5.1) -- author : Hans Hagen - PRAGMA ADE - www.pragma-ade.com -- copyright : public domain or whatever suits --- remark : part of the context distribution +-- remark : part of the context distribution, my first lua code -- todo: name space for local functions @@ -75,15 +75,12 @@ function traceln(str) io.flush() end -table.len = table.getn -table.join = table.concat - function table.found(tab, str) local l, r, p - if string.len(str) == 0 then + if #str == 0 then return false else - l, r = 1, table.len(tab) + l, r = 1, #tab while l <= r do p = math.floor((l+r)/2) if str < tab[p] then @@ -98,43 +95,44 @@ function table.found(tab, str) end end -function string.grab(str, delimiter) +function string:grab(delimiter) local list = {} - for snippet in string.gfind(str,delimiter) do - table.insert(list, snippet) + for snippet in self:gmatch(delimiter) do + list[#list+1] = snippet end return list end -function string.join(list, delimiter) - local size, str = table.len(list), '' - if size > 0 then - str = list[1] - for i = 2, size, 1 do - str = str .. delimiter .. list[i] - end - end - return str +function string:is_empty() + return not self:find("%S") end -function string.spacy(str) - if string.find(str,"^%s*$") then - return true - else - return false - end +function string:expand() + return (self:gsub("ENV%((%w+)%)", os.envvar)) end -function string.alphacmp(a,b,i) -- slow but ok - if i and i > 0 then - return string.lower(string.gsub(string.sub(a,i),'0',' ')) < string.lower(string.gsub(string.sub(b,i),'0',' ')) - else - return string.lower(a) < string.lower(b) - end +function string:strip() + return (self:gsub("^%s*(.-)%s*$", "%1")) end -function table.alphasort(list,i) - table.sort(list, function(a,b) return string.alphacmp(a,b,i) end) +do + + local lower, gsub, sub = string.lower, string.gsub, string.sub + + function table.alphasort(list,i) + if i and i > 0 then + local function alphacmp(a,b) + return lower(gsub(sub(a,i),'0',' ')) < lower(gsub(sub(b,i),'0',' ')) + end + table.sort(list,alphacmp) + else + local function alphacmp(a,b) + return a:lower() < b:lower() + end + table.sort(list,alphacmp) + end + end + end function io.exists(filename) @@ -148,29 +146,18 @@ function io.exists(filename) end function os.envvar(str) - if os.getenv(str) ~= '' then - return os.getenv(str) - elseif os.getenv(string.upper(str)) ~= '' then - return os.getenv(string.upper(str)) - elseif os.getenv(string.lower(str)) ~= '' then - return os.getenv(string.lower(str)) - else - return '' + local s = os.getenv(str) + if s ~= '' then + return s + end + s = os.getenv(str:upper()) + if s ~= '' then + return s + end + s = os.getenv(str:lower()) + if s ~= '' then + return s end -end - -function string.expand(str) - return string.gsub(str, "ENV%((%w+)%)", os.envvar) -end - -function string.strip(str) - return string.gsub(string.gsub(str,"^%s+",''),"%s+$",'') -end - -function string.replace(original,pattern,replacement) - local str = string.gsub(original,pattern,replacement) --- print(str) -- indirect, since else str + nofsubs - return str -- indirect, since else str + nofsubs end -- support functions, maybe editor namespace @@ -230,9 +217,9 @@ function getfiletype() return 'tex' elseif editor.Lexer == SCLEX_XML then return 'xml' - elseif string.find(firstline,"^%%") then + elseif firstline:find("^%%") then return 'tex' - elseif string.find(firstline,"^<%?xml") then + elseif firstline:find("^<%?xml") then return 'xml' else return 'unknown' @@ -246,7 +233,7 @@ function get_dir_list(mask) if props['PLAT_GTK'] and props['PLAT_GTK'] ~= "" then f = io.popen('ls -1 ' .. mask) else - mask = string.gsub(mask, '/','\\') + mask = mask:gsub('/','\\') local tmpfile = 'scite-ctx.tmp' local cmd = 'dir /b "' .. mask .. '" > ' .. tmpfile os.execute(cmd) @@ -257,7 +244,7 @@ function get_dir_list(mask) return files end for line in f:lines() do - table.insert(files, line) + files[#files+1] = line end f:close() return files @@ -265,32 +252,29 @@ end -- banner -print("loading scite-ctx.lua definition file") -print("") -print("- see scite-ctx.properties for configuring info") -print("") -print("- ctx.spellcheck.wordpath set to " .. props['ctx.spellcheck.wordpath']) -if string.find(string.lower(props['ctx.spellcheck.wordpath']), "ctxspellpath") then - if os.getenv('ctxspellpath') then - print("- ctxspellpath set to " .. os.getenv('CTXSPELLPATH')) - else - print("- 'ctxspellpath is not set") +do + + print("loading scite-ctx.lua definition file\n") + print("- see scite-ctx.properties for configuring info\n") + print("- ctx.spellcheck.wordpath set to " .. props['ctx.spellcheck.wordpath']) + if (props['ctx.spellcheck.wordpath']:lower()):find("ctxspellpath") then + if os.getenv('ctxspellpath') then + print("- ctxspellpath set to " .. os.getenv('CTXSPELLPATH')) + else + print("- 'ctxspellpath is not set") + end + print("- ctx.spellcheck.wordpath expands to " .. string.expand(props['ctx.spellcheck.wordpath'])) end - print("- ctx.spellcheck.wordpath expands to " .. string.expand(props['ctx.spellcheck.wordpath'])) -end -print("") -print("- ctx.wraptext.length is set to " .. props['ctx.wraptext.length']) -if props['ctx.helpinfo'] ~= '' then - print("- key bindings:") - print("") - print(string.replace(string.strip(props['ctx.helpinfo']),"%s*\|%s*","\n")) -- indirect, since else str + nofsubs -end -print("") -print("- recognized first lines:") -print("") -print("xml <?xml version='1.0' language='nl'") -print("tex % language=nl") + print("\n- ctx.wraptext.length is set to " .. props['ctx.wraptext.length']) + if props['ctx.helpinfo'] ~= '' then + print("\n- key bindings:\n") + print(((string.strip(props['ctx.helpinfo'])):gsub("%s*\|%s*","\n"))) + end + print("\n- recognized first lines:\n") + print("xml <?xml version='1.0' language='nl'") + print("tex % language=nl") +end -- text functions @@ -307,15 +291,15 @@ function wrap_text() if length == '' then length = 80 else length = tonumber(length) end - local startposition = editor.SelectionStart - local endposition = editor.SelectionEnd + local startposition = editor.SelectionStart + local endposition = editor.SelectionEnd if startposition == endposition then return end editor:LineEndExtend() - startposition = editor.SelectionStart - endposition = editor.SelectionEnd + startposition = editor.SelectionStart + endposition = editor.SelectionEnd -- local startline = line_of_position(startposition) -- local endline = line_of_position(endposition) @@ -327,46 +311,47 @@ function wrap_text() local startline = props['SelectionStartLine'] local endline = props['SelectionEndLine'] local startcolumn = props['SelectionStartColumn'] - 1 - local endcolumn = props['SelectionEndColumn'] - 1 + local endcolumn = props['SelectionEndColumn'] - 1 - local indentation = string.rep(' ', startcolumn) --- local selection = string.gsub(editor:GetSelText(),"[\n\r][\n\r]+", ' ' .. magicstring .. ' ') - local selection = string.gsub(editor:GetSelText(),"[\n\r][\n\r]", "\n") - local selection = string.gsub(selection,"\n\n+", ' ' .. magicstring .. ' ') - local replacement = '' + local replacement = { } local templine = '' + local indentation = string.rep(' ',startcolumn) + local selection = editor:GetSelText() - selection = string.gsub(selection,"^%s", '') + selection = selection:gsub("[\n\r][\n\r]","\n") + selection = selection:gsub("\n\n+",' ' .. magicstring .. ' ') + selection = selection:gsub("^%s",'') - for snippet in string.gfind(selection, "%S+") do + for snippet in selection:gmatch("%S+") do if snippet == magicstring then - replacement = replacement .. templine .. "\n\n" + replacement[#replacement+1] = templine + replacement[#replacement+1] = "" templine = '' - elseif string.len(templine) + string.len(snippet) > length then - replacement = replacement .. templine .. "\n" + elseif #templine + #snippet > length then + replacement[#replacement+1] = templine templine = indentation .. snippet - elseif string.len(templine) == 0 then + elseif #templine == 0 then templine = indentation .. snippet else templine = templine .. ' ' .. snippet end end - replacement = replacement .. templine - replacement = string.gsub(replacement, "^%s+", '') + replacement[#replacement+1] = templine + replacement[1] = replacement[1]:gsub("^%s+",'') if endcolumn == 0 then - replacement = replacement .. "\n" + replacement[#replacement+1] = "" end - editor:ReplaceSel(replacement) + editor:ReplaceSel(table.concat(replacement,"\n")) end function unwrap_text() - local startposition = editor.SelectionStart - local endposition = editor.SelectionEnd + local startposition = editor.SelectionStart + local endposition = editor.SelectionEnd if startposition == endposition then return end @@ -380,7 +365,7 @@ function unwrap_text() local selection = string.gsub(editor:GetSelText(),"[\n\r][\n\r]+", ' ' .. magicstring .. ' ') local replacement = '' - for snippet in string.gfind(selection, "%S+") do + for snippet in selection:gmatch("%S+") do if snippet == magicstring then replacement = replacement .. "\n" else @@ -447,25 +432,25 @@ function document_text() for i = editor:LineFromPosition(startposition), editor:LineFromPosition(endposition) do local str = editor:GetLine(i) if filetype == 'xml' then - if string.find(str,"^<%!%-%- .* %-%->%s*$") then - replacement = replacement .. string.gsub(str,"^<%!%-%- (.*) %-%->(%s*)$", "%1\n") - elseif not string.spacy(str) then - replacement = replacement .. '<!-- ' .. string.gsub(str,"(%s*)$", '') .. " -->\n" + if str:find("^<%!%-%- .* %-%->%s*$") then + replacement = replacement .. str:gsub("^<%!%-%- (.*) %-%->(%s*)$","%1\n") + elseif not str:is_empty() then + replacement = replacement .. '<!-- ' .. str:gsub("(%s*)$",'') .. " -->\n" else replacement = replacement .. str end else - if string.find(str,"^%%D%s+$") then + if str:find("^%%D%s+$") then replacement = replacement .. "\n" - elseif string.find(str,"^%%D ") then - replacement = replacement .. string.gsub(str,"^%%D ", '') + elseif str:find("^%%D ") then + replacement = replacement .. str:gsub("^%%D ",'') else replacement = replacement .. '%D ' .. str end end end - editor:ReplaceSel(string.gsub(replacement, "[\n\r]$", '')) + editor:ReplaceSel(replacement:gsub("[\n\r]$",'')) end @@ -482,10 +467,10 @@ function quote_text() end local replacement = editor:GetSelText() - replacement = string.gsub(replacement, "\`\`(.-)\'\'", leftquotation .. "%1" .. rightquotation) - replacement = string.gsub(replacement, "\"(.-)\"", leftquotation .. "%1" .. rightquotation) - replacement = string.gsub(replacement, "\`(.-)\'", leftquote .. "%1" .. rightquote ) - replacement = string.gsub(replacement, "\'(.-)\'", leftquote .. "%1" .. rightquote ) + replacement = replacement.gsub("\`\`(.-)\'\'", leftquotation .. "%1" .. rightquotation) + replacement = replacement.gsub("\"(.-)\"", leftquotation .. "%1" .. rightquotation) + replacement = replacement.gsub("\`(.-)\'", leftquote .. "%1" .. rightquote ) + replacement = replacement.gsub("\'(.-)\'", leftquote .. "%1" .. rightquote ) editor:ReplaceSel(replacement) end @@ -558,9 +543,9 @@ function check_text() if props["ctx.spellcheck.language"] == 'auto' then if filetype == 'tex' then -- % version =1.0 language=uk - firstline = string.gsub(firstline, "^%%%s*", '') - firstline = string.gsub(firstline, "%s*$", '') - for key, val in string.gfind(firstline,"(%w+)=(%w+)") do + firstline = firstline:gsub("^%%%s*",'') + firstline = firstline:gsub("%s*$",'') + for key, val in firstline:gmatch("(%w+)=(%w+)") do if key == "language" then language = val traceln("auto document language " .. "'" .. language .. "' (tex)") @@ -569,9 +554,9 @@ function check_text() skipfirst = true elseif filetype == 'xml' then -- <?xml version='1.0' language='uk' ?> - firstline = string.gsub(firstline, "^%<%?xml%s*", '') - firstline = string.gsub(firstline, "%s*%?%>%s*$", '') - for key, val in string.gfind(firstline,"(%w+)=[\"\'](.-)[\"\']") do + firstline = firstline:gsub("^%<%?xml%s*", '') + firstline = firstline:gsub("%s*%?%>%s*$", '') + for key, val in firstline:gmatch("(%w+)=[\"\'](.-)[\"\']") do if key == "language" then language = val traceln("auto document language " .. "'" .. language .. "' (xml)") @@ -588,16 +573,16 @@ function check_text() if fname ~= '' and fname ~= wordfile then wordfile, worddone, wordlist = fname, 0, {} - for filename in string.gfind(wordfile,"[^%,]+") do + for filename in wordfile:gmatch("[^%,]+") do if wordpath ~= '' then filename = string.expand(wordpath) .. '/' .. filename end if io.exists(filename) then traceln("loading " .. filename) for line in io.lines(filename) do - if not string.find(line,"^[\%\#\-]") then - str = string.gsub(line,"%s*$", '') - rawset(wordlist,str,true) -- table.insert(wordlist,str) + if not line:find("^[\%\#\-]") then + str = line:gsub("%s*$", '') + rawset(wordlist,str,true) worddone = worddone + 1 end end @@ -619,7 +604,7 @@ function check_text() end local i, j, lastpos, startpos, endpos, snippet, len, first = 0, 0, -1, 0, 0, '', 0, 0 local ok, skip, ch = false, false, '' - if skipfirst then first = string.len(firstline) end + if skipfirst then first = #firstline end for k = first, editor.TextLength do ch = editor:textrange(k,k+1) if wordgood ~= '' and ch == wordgood then @@ -627,7 +612,7 @@ function check_text() elseif ch == wordskip then skip = true end - if string.find(ch,"%w") and not string.find(ch,"%d") then + if ch:find("%w") and not ch:find("%d") then if not skip then if ok then endpos = k @@ -642,7 +627,7 @@ function check_text() if len >= wordsize then snippet = editor:textrange(startpos,endpos+1) i = i + 1 - if wordlist[snippet] or wordlist[string.lower(snippet)] then -- table.found(wordlist,snippet) + if wordlist[snippet] or wordlist[snippet:lower()] then -- table.found(wordlist,snippet) j = j + 1 else editor:StartStyling(startpos,INDICS_MASK) @@ -673,15 +658,15 @@ function UserListShow(menutrigger, menulist) local menuentries = {} local list = string.grab(menulist,"[^%|]+") menuactions = {} - for i=1, table.len(list) do + for i=1, #list do if list[i] ~= '' then - for key, val in string.gfind(list[i],"%s*(.+)=(.+)%s*") do - table.insert(menuentries,key) - rawset(menuactions,key,val) + for key, val in list[i]:gmatch("%s*(.+)=(.+)%s*") do + menuentries[#menuentries+1] = key + menuactions[key] = val end end end - local menustring = table.join(menuentries,'|') + local menustring = table.concat(menuentries,'|') if menustring == "" then traceln("There are no templates defined for this file type.") else @@ -708,7 +693,7 @@ function show_menu(menulist) end function process_menu(action) - if not string.find(action,"%(%)$") then + if not action:find("%(%)$") then assert(loadstring(action .. "()"))() else assert(loadstring(action))() @@ -721,73 +706,6 @@ menufunctions[12] = process_menu local templatetrigger = 13 --- local ctx_template_paths = { "./ctx-templates", "../ctx-templates", "../../ctx-templates" } --- local ctx_auto_templates = false --- local ctx_template_list = "" --- local ctx_dir_list = { } --- local ctx_dir_name = "./ctx-templates" - --- local ctx_path_list = {} --- local ctx_path_done = {} - --- function ctx_list_loaded() --- return ctx_dir_list and table.getn(ctx_dir_list) > 0 --- end - --- function insert_template(templatelist) --- if props["ctx.template.scan"] == "yes" then --- local current = props["FileDir"] .. "+" .. props["FileExt"] -- no name --- local rescan = props["ctx.template.rescan"] == "yes" --- local suffix = props["ctx.template.suffix."..props["FileExt"]] -- alas, no suffix expansion here --- if rescan then --- print("re-scanning enabled") --- end --- if current ~= ctx_file_path then --- rescan = true --- ctx_file_path = current --- ctx_file_done = false --- ctx_template_list = "" --- end --- if not ctx_file_done or rescan then --- local pattern = "*.*" --- for i, pathname in ipairs(ctx_template_paths) do --- print("scanning " .. pathname .. " for " .. pattern) --- ctx_dir_name = pathname --- ctx_dir_list = get_dir_list(pathname .. "/" .. pattern) --- if ctx_list_loaded() then --- break --- end --- end --- ctx_file_done = true --- end --- if ctx_list_loaded() then --- ctx_template_list = "" --- local pattern = "%." .. suffix .. "$" --- for j, filename in ipairs(ctx_dir_list) do --- if string.find(filename,pattern) then --- local menuname = string.gsub(filename,"%..-$","") --- if ctx_template_list ~= "" then --- ctx_template_list = ctx_template_list .. "|" --- end --- ctx_template_list = ctx_template_list .. menuname .. "=" .. ctx_dir_name .. "/" .. filename --- end --- end --- else --- print("no template files found") --- end --- if ctx_template_list == "" then --- ctx_auto_templates = false --- print("no file related templates found") --- else --- ctx_auto_templates = true --- templatelist = ctx_template_list --- end --- end --- if templatelist ~= "" then --- UserListShow(templatetrigger, templatelist) --- end --- end - local ctx_template_paths = { "./ctx-templates", "../ctx-templates", "../../ctx-templates" } local ctx_auto_templates = false local ctx_template_list = "" @@ -797,7 +715,7 @@ local ctx_path_done = {} local ctx_path_name = {} function ctx_list_loaded(path) - return ctx_path_list[path] and table.getn(ctx_path_list[path]) > 0 + return ctx_path_list[path] and #ctx_path_list[path] > 0 end function insert_template(templatelist) @@ -813,7 +731,7 @@ function insert_template(templatelist) if not ctx_path_done[path] or rescan then local pattern = "*.*" for i, pathname in ipairs(ctx_template_paths) do - print("scanning " .. string.gsub(path,"\\","/") .. "/" .. pathname) + print("scanning " .. path:gsub("\\","/") .. "/" .. pathname) ctx_path_name[path] = pathname ctx_path_list[path] = get_dir_list(pathname .. "/" .. pattern) if ctx_list_loaded(path) then @@ -822,7 +740,7 @@ function insert_template(templatelist) end end if ctx_list_loaded(path) then - print(table.getn(ctx_path_list[path]) .. " template files found") + print(#ctx_path_list[path] .. " template files found") else print("no template files found") end @@ -832,9 +750,9 @@ function insert_template(templatelist) local pattern = "%." .. suffix .. "$" local n = 0 for j, filename in ipairs(ctx_path_list[path]) do - if string.find(filename,pattern) then + if filename:find(pattern) then n = n + 1 - local menuname = string.gsub(filename,"%..-$","") + local menuname = filename:gsub("%..-$","") if ctx_template_list ~= "" then ctx_template_list = ctx_template_list .. "|" end @@ -897,9 +815,9 @@ function process_template_one(action) end end if text then - text = string.replace(text,"\\n","\n") - local pos = string.find(text,"%?") - text = string.replace(text,"%?","") + text = text:gsub("\\n","\n") + local pos = text:find("%?") + text = text:gsub("%?","") editor:insert(editor.CurrentPos,text) if pos then editor.CurrentPos = editor.CurrentPos + pos - 1 diff --git a/Master/texmf-dist/scripts/context/lua/x-ldx.lua b/Master/texmf-dist/scripts/context/lua/x-ldx.lua new file mode 100644 index 00000000000..d9d4e6a72aa --- /dev/null +++ b/Master/texmf-dist/scripts/context/lua/x-ldx.lua @@ -0,0 +1,382 @@ +--[[ldx-- +<source>Lua Documentation Module</source> + +This file is part of the <logo label='context'/> documentation suite and +itself serves as an example of using <logo label='lua'/> in combination +with <logo label='tex'/>. + +I will rewrite this using lpeg once I have the time to study that nice new +subsystem. +--ldx]]-- + +banner = "version 1.0.1 - 2007+ - PRAGMA ADE / CONTEXT" + +--[[ +This script needs a few libraries. Instead of merging the code here +we can use + +<typing> +mtxrun --internal x-ldx.lua +</typing> + +That way, the libraries included in the runner will be used. +]]-- + +-- libraries l-string.lua l-table.lua l-io.lua l-file.lua + +-- begin library merge +-- end library merge + +--[[ +Just a demo comment line. We will handle such multiline comments but +only when they start and end at the beginning of a line. More rich +comments are tagged differently. +]]-- + +--[[ldx-- +First we define a proper namespace for this module. The <q>l</q> stands for +<logo label='lua'/>, the <q>d</q> for documentation and the <q>x</q> for +<logo label='xml'/>. +--ldx]]-- + +if not ldx then ldx = { } end + +--[[ldx-- +We load the lua file into a table. The entries in this table themselves are +tables and have keys like <t>code</t> and <t>comment</t>. +--ldx]]-- + +function ldx.load(filename) + local data = file.readdata(filename) + local expr = "%s*%-%-%[%[ldx%-*%s*(.-)%s*%-%-ldx%]%]%-*%s*" + local i, j, t = 0, 0, { } + while true do + local comment, ni + ni, j, comment = data:find(expr, j) + if not ni then break end + t[#t+1] = { code = data:sub(i, ni-1) } + t[#t+1] = { comment = comment } + i = j + 1 + end + local str = data:sub(i, #data) + str = str:gsub("^%s*(.-)%s*$", "%1") + if #str > 0 then + t[#t+1] = { code = str } + end + return t +end + +--[[ldx-- +We will tag keywords so that we can higlight them using a special font +or color. Users can extend this list when needed. +--ldx]]-- + +ldx.keywords = { } + +--[[ldx-- +Here come the reserved words: +--ldx]]-- + +ldx.keywords.reserved = { + ["and"] = 1, + ["break"] = 1, + ["do"] = 1, + ["else"] = 1, + ["elseif"] = 1, + ["end"] = 1, + ["false"] = 1, + ["for"] = 1, + ["function"] = 1, + ["if"] = 1, + ["in"] = 1, + ["local"] = 1, + ["nil"] = 1, + ["not"] = 1, + ["or"] = 1, + ["repeat"] = 1, + ["return"] = 1, + ["then"] = 1, + ["true"] = 1, + ["until"] = 1, + ["while"] = 1 +} + +--[[ldx-- +We need to escape a few tokens. We keep the hash local to the +definition but set it up only once, hence the <key>do</key> +construction. +--ldx]]-- + +do + local e = { [">"] = ">", ["<"] = "<", ["&"] = "&" } + function ldx.escape(str) + return (str:gsub("([><&])",e)) + end +end + +--[[ldx-- +Enhancing the code is a bit tricky due to the fact that we have to +deal with strings and escaped quotes within these strings. Before we +mess around with the code, we hide the strings, and after that we +insert them again. Single and double quoted strings are tagged so +that we can use a different font to highlight them. +--ldx]]-- + +ldx.make_index = true + +function ldx.enhance(data) -- i need to use lpeg and then we can properly autoindent -) + local e = ldx.escape + for _,v in pairs(data) do + if v.code then + local dqs, sqs, com, cmt, cod = { }, { }, { }, { }, e(v.code) + cod = cod:gsub('\\"', "##d##") + cod = cod:gsub("\\'", "##s##") + cod = cod:gsub("%-%-%[%[.-%]%]%-%-", function(s) + cmt[#cmt+1] = s + return "[[[[".. #cmt .."]]]]" + end) + cod = cod:gsub("%-%-([^\n]*)", function(s) + com[#com+1] = s + return "[[".. #com .."]]" + end) + cod = cod:gsub("(%b\"\")", function(s) + dqs[#dqs+1] = s:sub(2,-2) or "" + return "<<<<".. #dqs ..">>>>" + end) + cod = cod:gsub("(%b\'\')", function(s) + sqs[#sqs+1] = s:sub(2,-2) or "" + return "<<".. #sqs ..">>" + end) + cod = cod:gsub("(%a+)",function(key) + local class = ldx.keywords.reserved[key] + if class then + return "<ldx:key class='" .. class .. "'>" .. key .. "</ldx:key>" + else + return key + end + end) + cod = cod:gsub("<<<<(%d+)>>>>", function(s) + return "<ldx:dqs>" .. dqs[tonumber(s)] .. "</ldx:dqs>" + end) + cod = cod:gsub("<<(%d+)>>", function(s) + return "<ldx:sqs>" .. sqs[tonumber(s)] .. "</ldx:sqs>" + end) + cod = cod:gsub("%[%[%[%[(%d+)%]%]%]%]", function(s) + return cmt[tonumber(s)] + end) + cod = cod:gsub("%[%[(%d+)%]%]", function(s) + return "<ldx:com>" .. com[tonumber(s)] .. "</ldx:com>" + end) + cod = cod:gsub("##d##", "\\\"") + cod = cod:gsub("##s##", "\\\'") + if ldx.make_index then + local lines = cod:split("\n") + local f = "(<ldx:key class='1'>function</ldx:key>)%s+([%w%.]+)%s*%(" + for k,v in pairs(lines) do + -- functies + v = v:gsub(f,function(key, str) + return "<ldx:function>" .. str .. "</ldx:function>(" + end) + -- variables + v = v:gsub("^([%w][%w%,%s]-)(=[^=])",function(str, rest) + local t = string.split(str, ",%s*") + for k,v in pairs(t) do + t[k] = "<ldx:variable>" .. v .. "</ldx:variable>" + end + return table.join(t,", ") .. rest + end) + -- so far + lines[k] = v + end + v.code = table.concat(lines,"\n") + else + v.code = cod + end + end + end +end + +function ldx.enhance(data) -- i need to use lpeg and then we can properly autoindent -) + local e = ldx.escape + for _,v in pairs(data) do + if v.code then + local dqs, sqs, com, cmt, cod = { }, { }, { }, { }, e(v.code) + cod = cod:gsub('\\"', "##d##") + cod = cod:gsub("\\'", "##s##") + cod = cod:gsub("%-%-%[%[.-%]%]%-%-", function(s) + cmt[#cmt+1] = s + return "<l<<<".. #cmt ..">>>l>" + end) + cod = cod:gsub("%-%-([^\n]*)", function(s) + com[#com+1] = s + return "<c<<<".. #com ..">>>c>" + end) + cod = cod:gsub("(%b\"\")", function(s) + dqs[#dqs+1] = s:sub(2,-2) or "" + return "<d<<<".. #dqs ..">>>d>" + end) + cod = cod:gsub("(%b\'\')", function(s) + sqs[#sqs+1] = s:sub(2,-2) or "" + return "<s<<<".. #sqs ..">>>s>" + end) + cod = cod:gsub("(%a+)",function(key) + local class = ldx.keywords.reserved[key] + if class then + return "<ldx:key class='" .. class .. "'>" .. key .. "</ldx:key>" + else + return key + end + end) + cod = cod:gsub("<s<<<(%d+)>>>s>", function(s) + return "<ldx:sqs>" .. sqs[tonumber(s)] .. "</ldx:sqs>" + end) + cod = cod:gsub("<d<<<(%d+)>>>d>", function(s) + return "<ldx:dqs>" .. dqs[tonumber(s)] .. "</ldx:dqs>" + end) + cod = cod:gsub("<c<<<(%d+)>>>c>", function(s) + return "<ldx:com>" .. com[tonumber(s)] .. "</ldx:com>" + end) + cod = cod:gsub("<l<<<(%d+)>>>l>", function(s) + return cmt[tonumber(s)] + end) + cod = cod:gsub("##d##", "\\\"") + cod = cod:gsub("##s##", "\\\'") + if ldx.make_index then + local lines = cod:split("\n") + local f = "(<ldx:key class='1'>function</ldx:key>)%s+([%w%.]+)%s*%(" + for k,v in pairs(lines) do + -- functies + v = v:gsub(f,function(key, str) + return "<ldx:function>" .. str .. "</ldx:function>(" + end) + -- variables + v = v:gsub("^([%w][%w%,%s]-)(=[^=])",function(str, rest) + local t = string.split(str, ",%s*") + for k,v in pairs(t) do + t[k] = "<ldx:variable>" .. v .. "</ldx:variable>" + end + return table.join(t,", ") .. rest + end) + -- so far + lines[k] = v + end + v.code = table.concat(lines,"\n") + else + v.code = cod + end + end + end +end + +--[[ldx-- +We're now ready to save the file in <logo label='xml'/> format. This boils +down to wrapping the code and comment as well as the whole document. We tag +lines in the code as such so that we don't need messy <t>CDATA</t> constructs +and by calculating the indentation we also avoid space troubles. It also makes +it possible to change the indentation afterwards. +--ldx]]-- + +function ldx.as_xml(data) + local t, cmode = { }, false + t[#t+1] = "<?xml version='1.0' standalone='yes'?>\n" + t[#t+1] = "\n<ldx:document xmlns:ldx='http://www.pragma-ade.com/schemas/ldx.rng'>\n" + for _,v in pairs(data) do + if v.code and not v.code:is_empty() then + t[#t+1] = "\n<ldx:code>\n" + for k,v in pairs(v.code:split("\n")) do -- make this faster + local a, b = v:find("^(%s+)") + if a and b then + if cmode then + t[#t+1] = "<ldx:line comment='yes' n='" .. b .. "'>" .. v:sub(b+1,#v) .. "</ldx:line>\n" + else + t[#t+1] = "<ldx:line n='" .. b .. "'>" .. v:sub(b+1,#v) .. "</ldx:line>\n" + end + elseif v:is_empty() then + if cmode then + t[#t+1] = "<ldx:line comment='yes'/>\n" + else + t[#t+1] = "<ldx:line/>\n" + end + elseif v:find("^%-%-%[%[") then + t[#t+1] = "<ldx:line comment='yes'>" .. v .. "</ldx:line>\n" + cmode= true + elseif v:find("^%]%]%-%-") then + t[#t+1] = "<ldx:line comment='yes'>" .. v .. "</ldx:line>\n" + cmode= false + elseif cmode then + t[#t+1] = "<ldx:line comment='yes'>" .. v .. "</ldx:line>\n" + else + t[#t+1] = "<ldx:line>" .. v .. "</ldx:line>\n" + end + end + t[#t+1] = "</ldx:code>\n" + elseif v.comment then + t[#t+1] = "\n<ldx:comment>\n" .. v.comment .. "\n</ldx:comment>\n" + else + -- cannot happen + end + end + t[#t+1] = "\n</ldx:document>\n" + return table.concat(t,"") +end + +--[[ldx-- +Saving the result is a trivial effort. +--ldx]]-- + +function ldx.save(filename,data) + file.savedata(filename,ldx.as_xml(data)) +end + +--[[ldx-- +The next function wraps it all in one call: +--ldx]]-- + +function ldx.convert(luaname,ldxname) + if not file.is_readable(luaname) then + luaname = luaname .. ".lua" + end + if file.is_readable(luaname) then + if not ldxname then + ldxname = file.replacesuffix(luaname,"ldx") + end + local data = ldx.load(luaname) + if data then + ldx.enhance(data) + if ldxname ~= luaname then + ldx.save(ldxname,data) + end + end + end +end + +--[[ldx-- +This module can be used directly: + +<typing> +mtxrun --internal x-ldx somefile.lua +</typing> + +will produce an ldx file that can be processed with <logo label='context'/> +by running: + +<typing> +texexec --use=x-ldx --forcexml somefile.ldx +</typing> + +You can do this in one step by saying: + +<typing> +texmfstart texexec --ctx=x-ldx somefile.lua +</typing> + +This will trigger <logo label='texexec'/> into loading the mentioned +<logo label='ctx'/> file. That file describes the conversion as well +as the module to be used. + +The main conversion call is: +--ldx]]-- + +if arg and arg[1] then + ldx.convert(arg[1],arg[2]) +end diff --git a/Master/texmf-dist/scripts/context/perl/makempy.pl b/Master/texmf-dist/scripts/context/perl/makempy.pl index f263dd4259d..7cba2e1a698 100644 --- a/Master/texmf-dist/scripts/context/perl/makempy.pl +++ b/Master/texmf-dist/scripts/context/perl/makempy.pl @@ -289,7 +289,7 @@ sub construct_mpy_file { error("unable to open $mpyfile file") } print MPY "% mpochecksum : $mpochecksum\n" ; my $copying = my $n = 0 ; - while (<TMP>) + while (<TMP>) # a simple sub is faster { if (s/beginfig/begingraphictextfig/o) { print MPY $_ ; $copying = 1 ; ++$n } elsif (s/endfig/endgraphictextfig/o) diff --git a/Master/texmf-dist/scripts/context/ruby/base/ctx.rb b/Master/texmf-dist/scripts/context/ruby/base/ctx.rb index f86e92d18c0..a077297f243 100644 --- a/Master/texmf-dist/scripts/context/ruby/base/ctx.rb +++ b/Master/texmf-dist/scripts/context/ruby/base/ctx.rb @@ -21,7 +21,7 @@ require 'rexml/document' class CtxRunner - attr_reader :environments, :modules, :filters, :flags + attr_reader :environments, :modules, :filters, :flags, :modes @@suffix = 'prep' @@ -43,7 +43,13 @@ class CtxRunner @modules = Array.new @filters = Array.new @flags = Array.new + @modes = Array.new @local = false + @paths = Array.new + end + + def register_path(str) + @paths << str end def manipulate(ctxname=nil,defaultname=nil) @@ -60,6 +66,10 @@ class CtxRunner return end + if @ctxname !~ /\.[a-z]+$/ then + @ctxname += ".ctx" + end + # name can be kpse:res-make.ctx if not FileTest.file?(@ctxname) then fullname, done = '', false @@ -83,6 +93,12 @@ class CtxRunner end break if done end + if ! done then + fullname = Kpse.found(@ctxname) + if FileTest.file?(fullname) then + @ctxname, done = fullname, true + end + end end if ! done && defaultname && FileTest.file?(defaultname) then report("using default ctxfile #{defaultname}") @@ -94,7 +110,12 @@ class CtxRunner end end - @xmldata = IO.read(@ctxname) + if FileTest.file?(@ctxname) then + @xmldata = IO.read(@ctxname) + else + report('no ctx file found') + return false + end unless @xmldata =~ /^.*<\?xml.*?\?>/moi then report("ctx file #{@ctxname} is no xml file, skipping") @@ -130,6 +151,9 @@ class CtxRunner REXML::XPath.each(root,"/ctx:job//ctx:resources/ctx:filter") do |fil| @filters << justtext(fil) end + REXML::XPath.each(root,"/ctx:job//ctx:resources/ctx:mode") do |fil| + @modes << justtext(fil) + end begin REXML::XPath.each(root,"//ctx:block") do |blk| if @jobname && blk.attributes['pattern'] then @@ -155,6 +179,9 @@ class CtxRunner REXML::XPath.each(root,"/ctx:job//ctx:process/ctx:resources/ctx:filter") do |fil| @filters << justtext(fil) end + REXML::XPath.each(root,"/ctx:job//ctx:process/ctx:resources/ctx:mode") do |fil| + @modes << justtext(fil) + end REXML::XPath.each(root,"/ctx:job//ctx:process/ctx:flags/ctx:flag") do |flg| @flags << justtext(flg) end @@ -180,6 +207,14 @@ class CtxRunner end REXML::XPath.each(root,"/ctx:job//ctx:preprocess/ctx:files") do |files| REXML::XPath.each(files,"ctx:file") do |pattern| + suffix = @@suffix + begin + suffix = REXML::XPath.match(root,"/ctx:job//ctx:preprocess/@suffix").to_s + rescue + suffix = @@suffix + else + if suffix && suffix.empty? then suffix = @@suffix end + end preprocessor = pattern.attributes['processor'] if preprocessor and not preprocessor.empty? then begin @@ -196,12 +231,25 @@ class CtxRunner end pattern = justtext(pattern) oldfiles = Dir.glob(pattern) + pluspath = false if oldfiles.length == 0 then report("no files match #{pattern}") + if @paths.length > 0 then + @paths.each do |p| + oldfiles = Dir.glob("#{p}/#{pattern}") + if oldfiles.length > 0 then + pluspath = true + break + end + end + if oldfiles.length == 0 then + report("no files match #{pattern} on path") + end + end end oldfiles.each do |oldfile| newfile = "#{oldfile}.#{suffix}" - newfile = File.basename(newfile) if @local + newfile = File.basename(newfile) if @local # or pluspath if File.expand_path(oldfile) != File.expand_path(newfile) && File.needsupdate(oldfile,newfile) then report("#{oldfile} needs preprocessing") begin @@ -214,10 +262,10 @@ class CtxRunner if command = commands[pp] then # a lie: no <?xml ...?> command = REXML::Document.new(command.to_s) # don't infect original + # command = command.deep_clone() # don't infect original command = command.elements["ctx:processor"] - begin - newfile = "#{oldfile}.#{suf}" if suf = command.attributes['suffix'] - rescue + if suf = command.attributes['suffix'] then + newfile = "#{oldfile}.#{suf}" end begin newfile = File.basename(newfile) if @local @@ -398,6 +446,7 @@ class CtxRunner when 'suffix' then if str =~ /^.*\.(.*?)$/o then $1 else '' end when 'nosuffix' then if str =~ /^(.*)\..*?$/o then $1 else str end when 'nopath' then if str =~ /^.*[\\\/](.*?)$/o then $1 else str end + when 'base' then if str =~ /^.*[\\\/](.*?)$/o then $1 else str end when 'full' then str when 'complete' then str when 'expand' then File.expand_path(str).gsub(/\\/,"/") diff --git a/Master/texmf-dist/scripts/context/ruby/base/exa.rb b/Master/texmf-dist/scripts/context/ruby/base/exa.rb index fdc5b5093e1..5a094351ec5 100644 --- a/Master/texmf-dist/scripts/context/ruby/base/exa.rb +++ b/Master/texmf-dist/scripts/context/ruby/base/exa.rb @@ -45,7 +45,7 @@ module ExaEncrypt "#{pre}#{password}#{post}" end else - data.gsub!(/<exa:password([^>]*?)>(.*?)<\/exa:password>/mois) do + data.gsub!(/<exa:password([^>]*?)>(.*?)<\/exa:password>/moi) do attributes, password = $1, $2 unless password =~ /^([0-9A-F][0-9A-F])+$/ then done = true @@ -238,11 +238,11 @@ module ExaModes binln, einln = "<inlinetexmath>" , "</inlinetexmath>" egraf = "<p/>" end - str.gsub!(/\n\s*\n+/mois, "\\ENDGRAF ") + str.gsub!(/\n\s*\n+/moi, "\\ENDGRAF ") str.gsub!(/(\[\[)\s*(.*?)\s*(\]\])/mos) do $1 + docalculatortexmath($2) + $3 end - str.gsub!(/(\\ENDGRAF)+\s*(\[\[)\s*(.*?)\s*(\]\])/mois) do + str.gsub!(/(\\ENDGRAF)+\s*(\[\[)\s*(.*?)\s*(\]\])/moi) do $1 + bdisp + $3 + edisp end str.gsub!(/(\[\[)\s*(.*?)\s*(\]\])/o) do diff --git a/Master/texmf-dist/scripts/context/ruby/base/kpse.rb b/Master/texmf-dist/scripts/context/ruby/base/kpse.rb index a4babae5505..0e185b5b8d7 100644 --- a/Master/texmf-dist/scripts/context/ruby/base/kpse.rb +++ b/Master/texmf-dist/scripts/context/ruby/base/kpse.rb @@ -64,8 +64,12 @@ module Kpse # @@distribution = 'miktex' if ENV['PATH'] =~ /miktex[\\\/]bin/o - if ENV['PATH'] =~ /(.*?)miktex[\\\/]bin/i then - @@distribution = 'miktex' unless $1 =~ /(texmf\-mswin[\/\\]bin|bin[\/\\]win32)/i + # if ENV['PATH'] =~ /(.*?)miktex[\\\/]bin/i then + # @@distribution = 'miktex' unless $1 =~ /(texmf\-mswin[\/\\]bin|bin[\/\\]win32)/i + # end + + if @@mswindows && (ENV['PATH'] =~ /(.*?)miktex[\\\/]bin/i) then + @@distribution = 'miktex' unless $1 =~ /(texmf\-mswin[\/\\]bin|bin[\/\\]win32)/i end @@re_true = /yes|on|true|1/i diff --git a/Master/texmf-dist/scripts/context/ruby/base/merge.rb b/Master/texmf-dist/scripts/context/ruby/base/merge.rb index c6ea6c09e66..a66b97e916b 100644 --- a/Master/texmf-dist/scripts/context/ruby/base/merge.rb +++ b/Master/texmf-dist/scripts/context/ruby/base/merge.rb @@ -38,7 +38,7 @@ module SelfMerge @@modules = $".collect do |file| File.expand_path(file) end @@modules.delete_if do |file| - file !~ /^#{@@ownpath}\/#{@@modroot}.*$/ + file !~ /^#{@@ownpath}\/#{@@modroot}.*$/i end def SelfMerge::ok? @@ -65,7 +65,7 @@ module SelfMerge end inserts << "#{@@kpsemergestop}\n\n" # no gsub! else we end up in SelfMerge - rbfile.sub!(/#{@@kpsemergestart}\s*#{@@kpsemergestop}/mois) do + rbfile.sub!(/#{@@kpsemergestart}\s*#{@@kpsemergestop}/moi) do inserts end rbfile.gsub!(/^(.*)(require [\"\'].*?#{@@modroot}.*)$/) do @@ -99,7 +99,7 @@ module SelfMerge begin if rbfile = IO.read(@@filename) then begin - rbfile.sub!(/#{@@kpsemergestart}(.*)#{@@kpsemergestop}\s*/mois) do + rbfile.sub!(/#{@@kpsemergestart}(.*)#{@@kpsemergestop}\s*/moi) do "#{@@kpsemergestart}\n\n#{@@kpsemergestop}\n\n" end rbfile.gsub!(/^(.*#{@@kpsemergedone}.*)$/) do diff --git a/Master/texmf-dist/scripts/context/ruby/base/mp.rb b/Master/texmf-dist/scripts/context/ruby/base/mp.rb index 5dd2948cffc..d168bde1d43 100644 --- a/Master/texmf-dist/scripts/context/ruby/base/mp.rb +++ b/Master/texmf-dist/scripts/context/ruby/base/mp.rb @@ -144,7 +144,7 @@ EOT result.gsub!(/(.{80,})(\-\-\-|\-\-|\.\.\.|\.\.)/) do "#{$1}#{$2}\n" end - result.gsub!(/\n[\s\n]+/mois) do + result.gsub!(/\n[\s\n]+/moi) do "\n" end result.gsub!(/btex\((\d+)\)/) do diff --git a/Master/texmf-dist/scripts/context/ruby/base/pdf.rb b/Master/texmf-dist/scripts/context/ruby/base/pdf.rb index 9f4e9a6c3f6..5aec06fc50c 100644 --- a/Master/texmf-dist/scripts/context/ruby/base/pdf.rb +++ b/Master/texmf-dist/scripts/context/ruby/base/pdf.rb @@ -7,7 +7,7 @@ module PDFview @method = 'default' # 'xpdf' - @opencalls['default'] = "pdfopen --file" + @opencalls['default'] = "pdfopen --file" # "pdfopen --back --file" @opencalls['xpdf'] = "xpdfopen" @closecalls['default'] = "pdfclose --file" diff --git a/Master/texmf-dist/scripts/context/ruby/base/system.rb b/Master/texmf-dist/scripts/context/ruby/base/system.rb index 267a22cb98f..c3fb08645a7 100644 --- a/Master/texmf-dist/scripts/context/ruby/base/system.rb +++ b/Master/texmf-dist/scripts/context/ruby/base/system.rb @@ -30,7 +30,7 @@ module System def System.null - if @@mswindows then 'nul' else '/dev/null/' end + if @@mswindows then 'nul' else '/dev/null' end end def System.unix? diff --git a/Master/texmf-dist/scripts/context/ruby/base/tex.rb b/Master/texmf-dist/scripts/context/ruby/base/tex.rb index 8c5cc63ef53..582ff640c6e 100644 --- a/Master/texmf-dist/scripts/context/ruby/base/tex.rb +++ b/Master/texmf-dist/scripts/context/ruby/base/tex.rb @@ -72,6 +72,7 @@ class TEX @@backends = Hash.new @@mappaths = Hash.new @@runoptions = Hash.new + @@tcxflag = Hash.new @@draftoptions = Hash.new @@texformats = Hash.new @@mpsformats = Hash.new @@ -89,29 +90,12 @@ class TEX @@luafiles = "luafiles.tmp" @@luatarget = "lua/context" - # we now drop pdfetex definitely - - # ENV['PATH'].split(File::PATH_SEPARATOR).each do |p| - # if System.unix? then - # pp, pe = "#{p}/pdftex" , "#{p}/pdfetex" - # else - # pp, pe = "#{p}/pdftex.exe", "#{p}/pdfetex.exe" - # end - # if FileTest.file?(pe) then # we assume no update - # @@pdftex = 'pdfetex' - # break - # elsif FileTest.file?(pp) then # we assume an update - # @@pdftex = 'pdftex' - # break - # end - # end - - # ['etex','pdfetex','standard'] .each do |e| @@texengines[e] = @@pdftex end - # ['tex','pdftex'] .each do |e| @@texengines[e] = 'pdftex' end + @@platformslash = if System.unix? then "\\\\" else "\\" end ['tex','etex','pdftex','pdfetex','standard'] .each do |e| @@texengines[e] = 'pdftex' end ['aleph','omega'] .each do |e| @@texengines[e] = 'aleph' end ['xetex'] .each do |e| @@texengines[e] = 'xetex' end + ['petex'] .each do |e| @@texengines[e] = 'petex' end ['luatex'] .each do |e| @@texengines[e] = 'luatex' end ['metapost','mpost', 'standard'] .each do |e| @@mpsengines[e] = 'mpost' end @@ -119,6 +103,8 @@ class TEX ['pdfetex','pdftex','pdf','pdftex','standard'] .each do |b| @@backends[b] = 'pdftex' end ['dvipdfmx','dvipdfm','dpx','dpm'] .each do |b| @@backends[b] = 'dvipdfmx' end ['xetex','xtx'] .each do |b| @@backends[b] = 'xetex' end + ['petex'] .each do |b| @@backends[b] = 'dvipdfmx' end + ['aleph'] .each do |b| @@backends[b] = 'dvipdfmx' end ['dvips','ps','dvi'] .each do |b| @@backends[b] = 'dvips' end ['dvipsone'] .each do |b| @@backends[b] = 'dvipsone' end ['acrobat','adobe','distiller'] .each do |b| @@backends[b] = 'acrobat' end @@ -126,7 +112,7 @@ class TEX ['tex','standard'] .each do |b| @@mappaths[b] = 'dvips' end ['pdftex','pdfetex'] .each do |b| @@mappaths[b] = 'pdftex' end - ['aleph','omega','xetex'] .each do |b| @@mappaths[b] = 'dvipdfm' end + ['aleph','omega','xetex','petex'] .each do |b| @@mappaths[b] = 'dvipdfm' end ['dvipdfm', 'dvipdfmx', 'xdvipdfmx'] .each do |b| @@mappaths[b] = 'dvipdfm' end ['xdv','xdv2pdf'] .each do |b| @@mappaths[b] = 'dvips' end @@ -148,34 +134,40 @@ class TEX ['plain','mpost'] .each do |f| @@mpsformats[f] = 'plain' end ['metafun','context','standard'] .each do |f| @@mpsformats[f] = 'metafun' end - # no 'standard' progname ! / beware, when using texexec we always use the context/metafun values - - ['pdftex','pdfetex','aleph','omega', + ['pdftex','pdfetex','aleph','omega','petex', 'xetex','luatex'] .each do |p| @@prognames[p] = 'context' end ['mpost'] .each do |p| @@prognames[p] = 'metafun' end + ['latex','pdflatex'] .each do |p| @@prognames[p] = 'latex' end ['plain','default','standard','mptopdf'] .each do |f| @@texmethods[f] = 'plain' end ['cont-en','cont-nl','cont-de','cont-it', 'cont-fr','cont-cz','cont-ro','cont-uk'] .each do |f| @@texmethods[f] = 'context' end - ['latex'] .each do |f| @@texmethods[f] = 'latex' end + ['latex','pdflatex'] .each do |f| @@texmethods[f] = 'latex' end ['plain','default','standard'] .each do |f| @@mpsmethods[f] = 'plain' end ['metafun'] .each do |f| @@mpsmethods[f] = 'metafun' end - @@texmakestr['plain'] = "\\dump" - @@mpsmakestr['plain'] = "\\dump" + @@texmakestr['plain'] = @@platformslash + "dump" + @@mpsmakestr['plain'] = @@platformslash + "dump" ['cont-en','cont-nl','cont-de','cont-it', - 'cont-fr','cont-cz','cont-ro','cont-uk'] .each do |f| @@texprocstr[f] = "\\emergencyend" end + 'cont-fr','cont-cz','cont-ro','cont-uk'] .each do |f| @@texprocstr[f] = @@platformslash + "emergencyend" end - # @@runoptions['xetex'] = ['--output-driver \\\"-d 4 -V 5\\\"'] # we need the pos pass - # @@runoptions['xetex'] = ['--8bit','-no-pdf'] # from now on we assume (x)dvipdfmx to be used - @@runoptions['xetex'] = ['--8bit','-output-driver="xdvipdfmx -E -d 4 -V 5"'] - @@runoptions['pdfetex'] = ['--8bit'] # obsolete - @@runoptions['pdftex'] = ['--8bit'] # pdftex is now pdfetex - @@runoptions['luatex'] = ['--file-line-error'] @@runoptions['aleph'] = ['--8bit'] + @@runoptions['luatex'] = ['--file-line-error'] @@runoptions['mpost'] = ['--8bit'] + @@runoptions['pdfetex'] = ['--8bit'] # obsolete + @@runoptions['pdftex'] = ['--8bit'] # pdftex is now pdfetex + # @@runoptions['petex'] = [] + @@runoptions['xetex'] = ['--8bit','-output-driver="xdvipdfmx -E -d 4 -V 5"'] + + @@tcxflag['aleph'] = true + @@tcxflag['luatex'] = false + @@tcxflag['mpost'] = true + @@tcxflag['pdfetex'] = true + @@tcxflag['pdftex'] = true + @@tcxflag['petex'] = false + @@tcxflag['xetex'] = false @@draftoptions['pdftex'] = ['--draftmode'] @@ -480,8 +472,10 @@ class TEX def validprogname(str) if str then [str].flatten.each do |s| + s = s.sub(/\.\S*/,"") return @@prognames[s] if @@prognames.key?(s) end + return str[0].sub(/\.\S*/,"") else return nil end @@ -497,11 +491,7 @@ class TEX else return "" end if format then - # if engine then - # "#{prefix}=#{engine}/#{format}" - # else - "#{prefix}=#{format}" - # end + "#{prefix}=#{format.sub(/\.\S+$/,"")}" else prefix end @@ -542,11 +532,16 @@ class TEX "--ini" end end - def tcxflag(file="natural.tcx") - if Kpse.miktex? then - "-tcx=#{file}" + def tcxflag(engine) + if @@tcxflag[engine] then + file = "natural.tcx" + if Kpse.miktex? then + "-tcx=#{file}" + else + "-translate-file=#{file}" + end else - "-translate-file=#{file}" + "" end end @@ -582,7 +577,7 @@ class TEX if data = (IO.readlines(@@luafiles) rescue nil) then report("compiling lua files (using #{File.expand_path(@@luafiles)})") begin - Dir.makedirs(@@luatarget) rescue false + File.makedirs(@@luatarget) rescue false data.each do |line| luafile = line.chomp lucfile = File.basename(luafile).gsub(/\..*?$/,'') + ".luc" @@ -612,15 +607,12 @@ class TEX report('using existing database') else report('updating file database') - Kpse.update + Kpse.update # obsolete here if getvariable('luatex') then begin - luatools = `texmfstart luatools --format=texmfscripts luatools.lua`.chomp.strip - unless luatools.empty? then - runcommand(["luatex","--luaonly #{luatools}","--generate","--verbose"]) - end + runcommand(["luatools","--generate","--verbose"]) rescue - report("run 'luatex --luaonly pathto/luatools.lua --generate' manually") + report("run 'luatools --generate' manualy") exit end end @@ -666,12 +658,13 @@ class TEX texformats.each do |texformat| report("generating tex format #{texformat}") progname = validprogname([getvariable('progname'),texformat,texengine]) - runcommand([quoted(texengine),prognameflag(progname),iniflag,tcxflag,prefixed(texformat,texengine),texmakeextras(texformat)]) + runcommand([quoted(texengine),prognameflag(progname),iniflag,tcxflag(texengine),prefixed(texformat,texengine),texmakeextras(texformat)]) end end else report("unable to make format due to lack of permissions") texformatpath = '' + setvariable('error','no permissions to write') end else texformatpath = '' @@ -686,7 +679,7 @@ class TEX mpsformats.each do |mpsformat| report("generating mps format #{mpsformat}") progname = validprogname([getvariable('progname'),mpsformat,mpsengine]) - if not runcommand([quoted(mpsengine),prognameflag(progname),iniflag,tcxflag,runoptions(mpsengine),mpsformat,mpsmakeextras(mpsformat)]) then + if not runcommand([quoted(mpsengine),prognameflag(progname),iniflag,tcxflag(mpsengine),runoptions(mpsengine),mpsformat,mpsmakeextras(mpsformat)]) then setvariable('error','no format made') end end @@ -783,12 +776,12 @@ class TEX result = runtexexec([tempfilename], flags, 1) if FileTest.file?("#{@@temprunfile}.log") then logdata = IO.read("#{@@temprunfile}.log") - if logdata =~ /^\s*This is (.*?)[\s\,]+(.*?)$/mois then + if logdata =~ /^\s*This is (.*?)[\s\,]+(.*?)$/moi then if validtexengine($1.downcase) then results.push("#{$1} #{$2.gsub(/\(format.*$/,'')}".strip) end end - if logdata =~ /^\s*(ConTeXt)\s+(.*int:\s+[a-z]+.*?)\s*$/mois then + if logdata =~ /^\s*(ConTeXt)\s+(.*int:\s+[a-z]+.*?)\s*$/moi then results.push("#{$1} #{$2}".gsub(/\s+/,' ').strip) end else @@ -817,12 +810,12 @@ class TEX private - def makeuserfile + def makeuserfile # not used in luatex (yet) language = getvariable('language') mainlanguage = getvariable('mainlanguage') bodyfont = getvariable('bodyfont') if f = openedfile("cont-fmt.tex") then - f << "\\unprotect" + f << "\\unprotect\n" case language when 'all' then f << "\\preloadallpatterns\n" @@ -910,9 +903,14 @@ class TEX end if ok then # we have a valid line + @@preamblekeys.each do |v| setvariable(v[1],vars[v[0]]) if vars.key?(v[0]) && vars[v[0]] end + +if getvariable('given.backend') == "standard" or getvariable('given.backend') == "" then + setvariable('backend',@@backends[getvariable('texengine')] || 'standard') +end break end else @@ -932,7 +930,8 @@ class TEX case str.chomp when /^\%/o then # next - when /\\(starttekst|stoptekst|startonderdeel|startdocument|startoverzicht)/o then + # when /\\(starttekst|stoptekst|startonderdeel|startdocument|startoverzicht)/o then + when /\\(starttekst|stoptekst|startonderdeel|startoverzicht)/o then setvariable('texformats','nl') ; break when /\\(stelle|verwende|umgebung|benutze)/o then setvariable('texformats','de') ; break @@ -1104,8 +1103,10 @@ class TEX end rescue TimeoutError report("job aborted due to timeout '#{delay}'") + setvariable('error','timeout') rescue report("job aborted due to error") + setvariable('error','fatal error') else report("job finished within timeout '#{delay}'") end @@ -1153,18 +1154,22 @@ class TEX public - def run_luatools(args) + # def run_luatools(args) # dirty trick: we know that the lua path is relative to the ruby path; of course this # will not work well when stubs are used - [(ENV["_CTX_K_S_texexec_"] or ENV["_CTX_K_S_THREAD_"] or ENV["TEXMFSTART.THREAD"]), File.dirname($0)].each do |path| - if path then - script = "#{path}/../lua/luatools.lua" - if FileTest.file?(script) then - return runcommand("luatex --luaonly #{script} #{args}") - end - end - end - return runcommand("texmfstart luatools #{args}") + # [(ENV["_CTX_K_S_texexec_"] or ENV["_CTX_K_S_THREAD_"] or ENV["TEXMFSTART.THREAD"]), File.dirname($0)].each do |path| + # if path then + # script = "#{path}/../lua/luatools.lua" + # if FileTest.file?(script) then + # return runcommand("luatex --luaonly #{script} #{args}") + # end + # end + # end + # return runcommand("texmfstart luatools #{args}") + # end + + def run_luatools(args) + return runcommand("luatools #{args}") end def processmpgraphic @@ -1182,7 +1187,7 @@ class TEX begin data = IO.read(File.suffixed(filename,'log')) basename = filename.sub(/\.mp$/, '') - if data =~ /output files* written\:\s*(.*)$/mois then + if data =~ /output files* written\:\s*(.*)$/moi then files, number, range, list = $1.split(/\s+/), 0, false, [] files.each do |fname| if fname =~ /^.*\.(\d+)$/ then @@ -1268,7 +1273,7 @@ class TEX File.open("texexec.tex",'w') do |f| f << "\\setupoutput[pdftex]\n" f << "\\setupcolors[state=start]\n" - data.sub!(/^%mpenvironment\:\s*(.*?)$/mois) do + data.sub!(/^%mpenvironment\:\s*(.*?)$/moi) do f << $1 "\n" end @@ -1448,9 +1453,10 @@ class TEX end end opt << "\\protect\n"; - begin getvariable('filters' ).split(',').uniq.each do |f| opt << "\\useXMLfilter[#{f}]\n" end ; rescue ; end - begin getvariable('usemodules' ).split(',').uniq.each do |m| opt << "\\usemodule[#{m}]\n" end ; rescue ; end - begin getvariable('environments').split(',').uniq.each do |e| opt << "\\environment #{e}\n" end ; rescue ; end + # begin getvariable('modes' ).split(',').uniq.each do |e| opt << "\\enablemode [#{e}]\n" end ; rescue ; end + begin getvariable('filters' ).split(',').uniq.each do |f| opt << "\\useXMLfilter[#{f}]\n" end ; rescue ; end + begin getvariable('usemodules' ).split(',').uniq.each do |m| opt << "\\usemodule [#{m}]\n" end ; rescue ; end + begin getvariable('environments').split(',').uniq.each do |e| opt << "\\environment #{e} \n" end ; rescue ; end # this will become: # begin getvariable('environments').split(',').uniq.each do |e| opt << "\\useenvironment[#{e}]\n" end ; rescue ; end opt << "\\endinput\n" @@ -1485,7 +1491,14 @@ class TEX [getvariable('runpath'),getvariable('path')].each do |pat| unless pat.empty? then if ENV.key?(res) then - ENV[res] = if ENV[res].empty? then pat else pat + ":" + ENV[res] end + # ENV[res] = if ENV[res].empty? then pat else pat + ":" + ENV[res] end +if ENV[res].empty? then + ENV[res] = pat +elsif ENV[res] == pat || ENV[res] =~ /^#{pat}\:/ || ENV[res] =~ /\:#{pat}\:/ then + # skip +else + ENV[res] = pat + ":" + ENV[res] +end else ENV[res] = pat end @@ -1544,7 +1557,7 @@ class TEX run_luatools("--fmt=#{texformat} #{filename}") else progname = validprogname([getvariable('progname'),texformat,texengine]) - runcommand([quoted(texengine),prognameflag(progname),formatflag(texengine,texformat),tcxflag,runoptions(texengine),filename,texprocextras(texformat)]) + runcommand([quoted(texengine),prognameflag(progname),formatflag(texengine,texformat),tcxflag(texengine),runoptions(texengine),filename,texprocextras(texformat)]) end # true else @@ -1559,8 +1572,7 @@ class TEX if mpsengine && mpsformat then ENV["MPXCOMMAND"] = "0" unless mpx progname = validprogname([getvariable('progname'),mpsformat,mpsengine]) - runcommand([quoted(mpsengine),prognameflag(progname),formatflag(mpsengine,mpsformat),tcxflag,runoptions(mpsengine),mpname,mpsprocextras(mpsformat)]) - # runcommand([quoted(mpsengine),formatflag(mpsengine,mpsformat),tcxflag,runoptions(mpsengine),mpname,mpsprocextras(mpsformat)]) + runcommand([quoted(mpsengine),prognameflag(progname),formatflag(mpsengine,mpsformat),tcxflag(mpsengine),runoptions(mpsengine),mpname,mpsprocextras(mpsformat)]) true else false @@ -1728,14 +1740,9 @@ class TEX takeprecautions report("using search method '#{Kpse.searchmethod}'") if getvariable('verbose') + rawname = getvariable('filename') jobname = getvariable('filename') - suffix = getvariable('suffix') - result = getvariable('result') - forcexml = getvariable('forcexml') - runonce = getvariable('once') - finalrun = getvariable('final') || (getvariable('arrange') && ! getvariable('noarrange')) - globalfile = getvariable('globalfile') if getvariable('autopath') then jobname = File.basename(jobname) @@ -1748,18 +1755,11 @@ class TEX jobname = File.unixfied(jobname) inppath = File.unixfied(inppath) - result = File.unixfied(result) orisuffix = jobsuffix # still needed ? - setvariable('nomprun',true) if orisuffix == 'mpx' # else cylic run - - PDFview.setmethod('xpdf') if getvariable('xpdf') - - PDFview.closeall if getvariable('autopdf') - if jobsuffix =~ /^(htm|html|xhtml|xml|fo|fox|rlg|exa)$/io then - forcexml = true + setvariable('forcexml',true) end dummyfile = false @@ -1776,7 +1776,6 @@ class TEX # we can have funny names, like 2005.10.10 (given without suffix) rawname = jobname + '.' + jobsuffix - rawpath = File.dirname(rawname) rawbase = File.basename(rawname) @@ -1786,16 +1785,22 @@ class TEX end end - if dummyfile || forcexml then + forcexml = getvariable('forcexml') + + if dummyfile || forcexml then # after ctx? jobsuffix = makestubfile(rawname,rawbase,forcexml) checkxmlfile(rawname) end - # preprocess files unless getvariable('noctx') then ctx = CtxRunner.new(rawname,@logger) + if pth = getvariable('path') then + pth.split(',').each do |p| + ctx.register_path(p) + end + end if getvariable('ctxfile').empty? then if rawname == rawbase then ctx.manipulate(File.suffixed(rawname,'ctx'),'jobname.ctx') @@ -1810,6 +1815,7 @@ class TEX envs = ctx.environments mods = ctx.modules flags = ctx.flags + mdes = ctx.modes flags.each do |f| f.sub!(/^\-+/,'') @@ -1825,24 +1831,47 @@ class TEX # merge environment and module specs envs << getvariable('environments') unless getvariable('environments').empty? - mods << getvariable('modules') unless getvariable('modules') .empty? + mods << getvariable('usemodules') unless getvariable('usemodules') .empty? + mdes << getvariable('modes') unless getvariable('modes') .empty? envs = envs.uniq.join(',') mods = mods.uniq.join(',') + mdes = mdes.uniq.join(',') report("using search method '#{Kpse.searchmethod}'") if getvariable('verbose') + report("using environments #{envs}") if envs.length > 0 report("using modules #{mods}") if mods.length > 0 + report("using modes #{mdes}") if mdes.length > 0 setvariable('environments', envs) - setvariable('modules', mods) + setvariable('usemodules', mods) + setvariable('modes', mdes) end # end of preprocessing and merging + setvariable('nomprun',true) if orisuffix == 'mpx' # else cylic run + PDFview.setmethod('xpdf') if getvariable('xpdf') + PDFview.closeall if getvariable('autopdf') + + runonce = getvariable('once') + finalrun = getvariable('final') || (getvariable('arrange') && ! getvariable('noarrange')) + suffix = getvariable('suffix') + result = getvariable('result') + globalfile = getvariable('globalfile') + forcexml = getvariable('forcexml') # can be set in ctx file + +if dummyfile || forcexml then # after ctx? + jobsuffix = makestubfile(rawname,rawbase,forcexml) + checkxmlfile(rawname) +end + + result = File.unixfied(result) + if globalfile || FileTest.file?(rawname) then - if not dummyfile and not globalfile then + if not dummyfile and not globalfile and not forcexml then scantexpreamble(rawname) scantexcontent(rawname) if getvariable('texformats').standard? end @@ -1897,7 +1926,15 @@ class TEX end end # goto . + ok = runtex(File.suffixed(if dummyfile || forcexml then rawbase else rawname end,jobsuffix)) + +if getvariable('texengine') == "xetex" then + ok = true +end + +############################ + # goto tmp/jobname when present if ok && (nofruns > 1) then unless getvariable('nompmode') then @@ -1951,7 +1988,7 @@ class TEX # next line is empty ok = 2 when 2 then - if line =~ /^\%\s+\>\s+(.*?)\s+(\d+)/mois then + if line =~ /^\%\s+\>\s+(.*?)\s+(\d+)/moi then filename, n = $1, $2 done = File.delete(filename) rescue false if done && getvariable('verbose') then @@ -1961,7 +1998,7 @@ class TEX break end else - if line =~ /^\%\s+temporary files\:\s+(\d+)/mois then + if line =~ /^\%\s+temporary files\:\s+(\d+)/moi then if $1.to_i == 0 then break else @@ -2021,7 +2058,7 @@ class TEX setvariable('mp.line','') setvariable('mp.error','') if mpdata = File.silentread(mpname) then - mpdata.gsub!(/^\#.*\n/o,'') + mpdata.gsub!(/^\%.*\n/o,'') File.silentrename(mpname,mpcopy) texfound = mergebe || (mpdata =~ /btex .*? etex/mo) if mp = openedfile(mpname) then diff --git a/Master/texmf-dist/scripts/context/ruby/base/texutil.rb b/Master/texmf-dist/scripts/context/ruby/base/texutil.rb index 7c402b98f85..063f67f2d38 100644 --- a/Master/texmf-dist/scripts/context/ruby/base/texutil.rb +++ b/Master/texmf-dist/scripts/context/ruby/base/texutil.rb @@ -187,7 +187,14 @@ class TeXUtil # shortcut("\\\"e", 'ediaeresis') # shortcut("\\\'o", 'oacute') + def hextoutf(str) + str.gsub(/^(0x[A-F\d]+)$/) do + [$1.hex()].pack("U") + end + end + def shortcut(from,to) + from = hextoutf(from) replacer(from,to) expander(to) end @@ -400,7 +407,7 @@ class TeXUtil def MyCommands::writer(logger,handle) handle << logger.banner("commands: #{@@commands.size}") @@commands.each do |c| - handle << "#{c}\n" + handle << "#{c}%\n" end end @@ -433,8 +440,8 @@ class TeXUtil def MyExtras::writer(logger,handle) handle << logger.banner("programs: #{@@programs.size}") - @@programs.each do |p| - handle << "% #{p} (#{@@programs[p.to_i]})\n" + @@programs.each_with_index do |cmd, p| + handle << "% #{p+1} (#{cmd})\n" end end @@ -447,10 +454,11 @@ class TeXUtil end def MyExtras::finalizer(logger) - @@programs.each do |p| - cmd = @@programs[p.to_i] - logger.report("running #{cmd}") - system(cmd) + unless (ENV["CTX.TEXUTIL.EXTRAS"] =~ /^(no|off|false|0)$/io) || (ENV["CTX_TEXUTIL_EXTRAS"] =~ /^(no|off|false|0)$/io) then + @@programs.each do |cmd| + logger.report("running #{cmd}") + system(cmd) + end end end @@ -493,7 +501,7 @@ class TeXUtil end end list.each do |entry| - handle << "\\synonymentry{#{entry.type}}{#{entry.command}}{#{entry.key}}{#{entry.data}}\n" + handle << "\\synonymentry{#{entry.type}}{#{entry.command}}{#{entry.key}}{#{entry.data}}%\n" end end @@ -601,7 +609,7 @@ class TeXUtil end else # @entry, @key = cleanupsplit(@entry), cleanupsplit(@key) -@entry, @key = cleanupsplit(@entry), xcleanupsplit(@key) + @entry, @key = cleanupsplit(@entry), xcleanupsplit(@key) end @sortkey = sorter.simplify(@key) # special = @sortkey =~ /^([^a-zA-Z\\])/o @@ -631,23 +639,23 @@ class TeXUtil end end -def xcleanupsplit(target) # +a+b+c &a&b&c a+b+c a&b&c - t = Array.new - case target[0,1] - when '&' then - t = target.sub(/^./o,'').split(/([^\\])\&/o) - when '+' then - t = target.sub(/^./o,'').split(/([^\\])\+/o) - else - # t = target.split(/([^\\])[\&\+]/o) - # t = target.split(/[\&\+]/o) - t = target.split(/(?!\\)[\&\+]/o) # lookahead - end - if not t[1] then t[1] = " " end # we need some entry else we get subentries first - if not t[2] then t[2] = " " end # we need some entry else we get subentries first - return t.join(@@split) -end - + def xcleanupsplit(target) # +a+b+c &a&b&c a+b+c a&b&c + t = Array.new + case target[0,1] + when '&' then + t = target.sub(/^./o,'').split(/([^\\])\&/o) + when '+' then + t = target.sub(/^./o,'').split(/([^\\])\+/o) + else + # t = target.split(/([^\\])[\&\+]/o) + # t = target.split(/[\&\+]/o) + t = target.split(/(?!\\)[\&\+]/o) # lookahead + end + if not t[1] then t[1] = " " end # we need some entry else we get subentries first + if not t[2] then t[2] = " " end # we need some entry else we get subentries first + if not t[3] then t[3] = " " end # we need some entry else we get subentries first + return t.join(@@split) + end def <=> (other) @sortkey <=> other.sortkey end @@ -660,10 +668,10 @@ end def Register.flushsavedline(handle) if @@collapse && ! @@savedfrom.empty? then if ! @@savedto.empty? then - handle << "\\registerfrom#{@@savedfrom}" - handle << "\\registerto#{@@savedto}" + handle << "\\registerfrom#{@@savedfrom}%" + handle << "\\registerto#{@@savedto}%" else - handle << "\\registerpage#{@@savedfrom}" + handle << "\\registerpage#{@@savedfrom}%" end end @@savedhowto, @@savedfrom, @@savedto, @@savedentry = '', '', '', '' @@ -710,10 +718,10 @@ end else character = "\\unknown" end - handle << "\\registerentry{#{entry.type}}{#{character}}\n" + handle << "\\registerentry{#{entry.type}}{#{character}}%\n" end end - current = [entry.entry.split(@@split),'','',''].flatten + current = [entry.entry.split(@@split),'','','',''].flatten howto = current.collect do |e| e + '::' + entry.texthowto end @@ -723,38 +731,51 @@ end previous[0] = howto[0].dup previous[1] = '' previous[2] = '' + previous[3] = '' end if howto[1] == previous[1] then current[1] = '' else previous[1] = howto[1].dup previous[2] = '' + previous[3] = '' end if howto[2] == previous[2] then current[2] = '' else previous[2] = howto[2].dup + previous[3] = '' + end + if howto[3] == previous[3] then + current[3] = '' + else + previous[3] = howto[3].dup end copied = false unless current[0].empty? then Register.flushsavedline(handle) - handle << "\\registerentrya{#{entry.type}}{#{current[0]}}\n" + handle << "\\registerentrya{#{entry.type}}{#{current[0]}}%\n" copied = true end unless current[1].empty? then Register.flushsavedline(handle) - handle << "\\registerentryb{#{entry.type}}{#{current[1]}}\n" + handle << "\\registerentryb{#{entry.type}}{#{current[1]}}%\n" copied = true end unless current[2].empty? then Register.flushsavedline(handle) - handle << "\\registerentryc{#{entry.type}}{#{current[2]}}\n" + handle << "\\registerentryc{#{entry.type}}{#{current[2]}}%\n" + copied = true + end + unless current[3].empty? then + Register.flushsavedline(handle) + handle << "\\registerentryd{#{entry.type}}{#{current[3]}}%\n" copied = true end @nofentries += 1 if copied if entry.realpage.to_i == 0 then Register.flushsavedline(handle) - handle << "\\registersee{#{entry.type}}{#{entry.pagehowto},#{entry.texthowto}}{#{entry.seetoo}}{#{entry.page}}\n" ; + handle << "\\registersee{#{entry.type}}{#{entry.pagehowto},#{entry.texthowto}}{#{entry.seetoo}}{#{entry.page}}%\n" ; lastpage, lastrealpage = entry.page, entry.realpage copied = false # no page ! elsif @@savedhowto != entry.pagehowto and ! entry.pagehowto.empty? then @@ -762,14 +783,14 @@ end end # beware, we keep multiple page entries per realpage because of possible prefix usage if copied || ! ((lastpage == entry.page) && (lastrealpage == entry.realpage)) then - nextentry = "{#{entry.type}}{#{previous[0]}}{#{previous[1]}}{#{previous[2]}}{#{entry.pagehowto},#{entry.texthowto}}" + nextentry = "{#{entry.type}}{#{previous[0]}}{#{previous[1]}}{#{previous[2]}}{#{previous[3]}}{#{entry.pagehowto},#{entry.texthowto}}" savedline = "{#{entry.type}}{#{@@savedhowto},#{entry.texthowto}}{#{entry.location}}{#{entry.page}}{#{entry.realpage}}" if entry.state == 1 then # from Register.flushsavedline(handle) - handle << "\\registerfrom#{savedline}\n" + handle << "\\registerfrom#{savedline}%\n" elsif entry.state == 3 then # to Register.flushsavedline(handle) - handle << "\\registerto#{savedline}\n" + handle << "\\registerto#{savedline}%\n" @@savedhowto = '' # test elsif @@collapse then if savedentry != nextentry then @@ -778,7 +799,7 @@ end savedTo, savedentry = savedline, nextentry end else - handle << "\\registerpage#{savedline}\n" + handle << "\\registerpage#{savedline}%\n" @@savedhowto = '' # test end @nofpages += 1 @@ -1026,6 +1047,7 @@ end begin if f = File.open(File.suffixed(filename,'tuo'),'w') then @plugins.writers(f) + f << "\\endinput\n" f.close end rescue diff --git a/Master/texmf-dist/scripts/context/ruby/base/tool.rb b/Master/texmf-dist/scripts/context/ruby/base/tool.rb index 77ad947fe1e..5ccedfec1c1 100644 --- a/Master/texmf-dist/scripts/context/ruby/base/tool.rb +++ b/Master/texmf-dist/scripts/context/ruby/base/tool.rb @@ -129,6 +129,8 @@ module Tool def Tool.simplefilename(old) + return old # too fragile + return old if not FileTest.file?(old) new = old.downcase @@ -258,7 +260,7 @@ module Tool new = checksuffix(simplefilename(old)) unless new == old - begin + begin # bugged, should only be name, not path File.rename(old,new) logging.report("renaming fuzzy name #{old} to #{new}") unless logging return old diff --git a/Master/texmf-dist/scripts/context/ruby/concheck.rb b/Master/texmf-dist/scripts/context/ruby/concheck.rb index 6c7512bff8f..3db3e5148bf 100644 --- a/Master/texmf-dist/scripts/context/ruby/concheck.rb +++ b/Master/texmf-dist/scripts/context/ruby/concheck.rb @@ -308,7 +308,7 @@ def some_wrd_error(data, filename, start, stop, ignore) end end -def some_sym_error (data, filename, symbol,template=false) +def some_sym_error (data, filename, symbol, template=false) saved = Array.new inside = false level = 0 diff --git a/Master/texmf-dist/scripts/context/ruby/ctxtools.rb b/Master/texmf-dist/scripts/context/ruby/ctxtools.rb index b306ee5dc43..00d2b494bb0 100644 --- a/Master/texmf-dist/scripts/context/ruby/ctxtools.rb +++ b/Master/texmf-dist/scripts/context/ruby/ctxtools.rb @@ -44,7 +44,7 @@ # it cannot do that (it tries to hyphenate as if the "ffi" was a # character), and the result is wrong hyphenation. -banner = ['CtxTools', 'version 1.3.3', '2004/2006', 'PRAGMA ADE/POD'] +banner = ['CtxTools', 'version 1.3.5', '2004/2008', 'PRAGMA ADE'] $: << File.expand_path(File.dirname($0)) ; $: << File.join($:.last,'lib') ; $:.uniq! @@ -334,7 +334,7 @@ class Commands report("generating #{keyfile}") begin - one = "texexec --make --alone --all #{interface}" + one = "texexec --make --all #{interface}" two = "texexec --batch --silent --interface=#{interface} x-set-01" if @commandline.option("force") then system(one) @@ -534,7 +534,7 @@ class Commands ] $dontasksuffixes = [ "mp(graph|run)\\.mp", "mp(graph|run)\\.mpd", "mp(graph|run)\\.mpo", "mp(graph|run)\\.mpy", - "mp(graph|run)\\.\\d+", + "mp(graph|run)\\.\\d+", "mp(graph|run)\\.mp.keep", "xlscript\\.xsl" ] $forsuresuffixes = [ @@ -866,6 +866,7 @@ class Language @filenames = filenames @remapping = Array.new @demapping = Array.new + @cloning = Array.new @unicode = Hash.new @encoding = encoding @data = '' @@ -895,6 +896,9 @@ class Language def demap(from, to) @demapping.push([from,to]) end + def clone(from, to) + @cloning.push([from,to]) + end def load(filenames=@filenames) found = false @@ -908,7 +912,7 @@ class Language @data += data.gsub(/\%.*$/, '').gsub(/\\message\{.*?\}/, '') data.gsub!(/(\\patterns|\\hyphenation)\s*\{.*/mo) do '' end @read += "\n% preamble of file #{fname}\n\n#{data}\n" - @data.gsub!(/^[\s\n]+$/mois, '') + @data.gsub!(/^[\s\n]+$/moi, '') report("file #{fname} is loaded") found = true break # next fileset @@ -957,6 +961,22 @@ class Language @remapping[$1.to_i][1] end report(" nothing remapped") unless done + @cloning.each_index do |i| + c = 0 + f, s = @cloning[i][0], @cloning[i][1] + str = "#{f}|#{s}" + str.gsub!(/([\[\]])/) do "\\" + "#{$1}" end + reg = /(#{str})/ + content.gsub!(/(\S*(#{str})\S*)/) do + a, b = $1, $1 + a.gsub!(reg, f) + b.gsub!(reg, s) + c = c + 1 + "#{a} #{b}" + end + report("#{c.to_s.rjust(5)} times #{f} cloned to #{s}") + n += c + end report("") content.to_s end @@ -1300,6 +1320,14 @@ class Language remap(/\\a\s*/, "[aeligature]") remap(/\\o\s*/, "[oeligature]") when 'agr' then + # bug fix + remap("a2|", "[greekalphaiotasub]") + remap("h2|", "[greeketaiotasub]") + remap("w2|", "[greekomegaiotasub]") + remap(">2r1<2r", "[2ῤ1ῥ]") + remap(">a2n1wdu'", "[ἀ2ν1ωδύ]") + remap(">e3s2ou'", "[ἐ3σ2ού]") + # main conversion remap(/\<\'a\|/, "[greekalphaiotasubdasiatonos]") # remap(/\<\'a\|/, "[greekdasiatonos][greekAlpha][greekiota]") remap(/\>\'a\|/, "[greekalphaiotasubpsilitonos]") @@ -1399,7 +1427,7 @@ class Language remap(/\<\'u/, "[greekupsilondasiatonos]") remap(/\>\'u/, "[greekupsilonpsilitonos]") remap(/\<\`u/, "[greekupsilondasiavaria]") - remap(/\>\'u/, "[greekupsilonpsilivaria]") + remap(/\>\`u/, "[greekupsilonpsilivaria]") remap(/\<\~u/, "[greekupsilondasiaperispomeni]") remap(/\>\~u/, "[greekupsilonpsiliperispomeni]") remap(/\"\'u/, "[greekupsilondialytikatonos]") @@ -1433,14 +1461,15 @@ class Language remap(/\"\'/, "[greekdialytikatonos]") remap(/\"\`/, "[greekdialytikavaria]") remap(/\"\~/, "[greekdialytikaperispomeni]") - remap(/\</, "[dasia]") - remap(/\>/, "[psili]") - remap(/\'/, "[oxia]") + remap(/\</, "[greekdasia]") + remap(/\>/, "[greekpsili]") + remap(/\d.{0,2}''/, "") + remap(/\'/, "[greekoxia]") remap(/\`/, "[greekvaria]") remap(/\~/, "[perispomeni]") - remap(/\"/, "[dialytika]") + remap(/\"/, "[greekdialytika]") # unknown - remap(/\|/, "[greekIotadialytika]") + # remap(/\|/, "[greekIotadialytika]") # next remap(/A/, "[greekAlpha]") remap(/B/, "[greekBeta]") @@ -1491,6 +1520,13 @@ class Language remap(/x/, "[greekxi]") remap(/y/, "[greekpsi]") remap(/z/, "[greekzeta]") + clone("[greekalphatonos]", "[greekalphaoxia]") + clone("[greekepsilontonos]", "[greekepsilonoxia]") + clone("[greeketatonos]", "[greeketaoxia]") + clone("[greekiotatonos]", "[greekiotaoxia]") + clone("[greekomicrontonos]", "[greekomicronoxia]") + clone("[greekupsilontonos]", "[greekupsilonoxia]") + clone("[greekomegatonos]", "[greekomegaoxia]") when 'ru' then remap(/\xC1/, "[cyrillica]") remap(/\xC2/, "[cyrillicb]") @@ -1602,14 +1638,14 @@ class Commands @@languagedata['da' ] = [ 'ec' , ['dkspecial.tex','dkcommon.tex'] ] # elhyph.tex @@languagedata['es' ] = [ 'ec' , ['eshyph.tex'] ] - @@languagedata['fi' ] = [ 'ec' , ['ethyph.tex'] ] + @@languagedata['et' ] = [ 'ec' , ['ethyph.tex'] ] @@languagedata['fi' ] = [ 'ec' , ['fihyph.tex'] ] @@languagedata['fr' ] = [ 'ec' , ['frhyph.tex'] ] # ghyphen.readme ghyph31.readme grphyph @@languagedata['hr' ] = [ 'ec' , ['hrhyph.tex'] ] @@languagedata['hu' ] = [ 'ec' , ['huhyphn.tex'] ] - @@languagedata['en' ] = [ 'default' , ['ushyphmax.tex'],['ushyph.tex'],['hyphen.tex'] ] - @@languagedata['us' ] = [ 'default' , ['ushyphmax.tex'],['ushyph.tex'],['hyphen.tex'] ] + @@languagedata['en' ] = [ 'default' , [['ushyphmax.tex'],['ushyph.tex'],['hyphen.tex']] ] + @@languagedata['us' ] = [ 'default' , [['ushyphmax.tex'],['ushyph.tex'],['hyphen.tex']] ] # inhyph.tex @@languagedata['is' ] = [ 'ec' , ['ishyph.tex'] ] @@languagedata['it' ] = [ 'ec' , ['ithyph.tex'] ] @@ -1617,12 +1653,12 @@ class Commands # mnhyph @@languagedata['nl' ] = [ 'ec' , ['nehyph96.tex'] ] # @@languagedata['no' ] = [ 'ec' , ['nohyphbx.tex'],['nohyphb.tex'],['nohyph2.tex'],['nohyph1.tex'],['nohyph.tex'] ] - @@languagedata['no' ] = [ 'ec' , ['asxsx.tex','nohyphbx.tex'],['nohyphb.tex'],['nohyph2.tex'],['nohyph1.tex'],['nohyph.tex'] ] - @@languagedata['agr'] = [ 'agr' , [['grahyph4.tex','oldgrhyph.tex']] ] # new, todo + @@languagedata['no' ] = [ 'ec' , [['asxsx.tex','nohyphbx.tex'],['nohyphb.tex'],['nohyph2.tex'],['nohyph1.tex'],['nohyph.tex']] ] + @@languagedata['agr'] = [ 'agr' , [['grahyph4.tex'], ['oldgrhyph.tex']] ] # new, todo @@languagedata['pl' ] = [ 'ec' , ['plhyph.tex'] ] @@languagedata['pt' ] = [ 'ec' , ['pthyph.tex'] ] @@languagedata['ro' ] = [ 'ec' , ['rohyph.tex'] ] - @@languagedata['sl' ] = [ 'ec' , ['sihyph.tex'] ] + @@languagedata['sl' ] = [ 'ec' , [['slhyph.tex'], ['sihyph.tex']] ] @@languagedata['sk' ] = [ 'ec' , ['skhyphen.tex','skhyphen.ex'] ] # sorhyph.tex / upper sorbian # srhyphc.tex / cyrillic @@ -1639,19 +1675,22 @@ class Commands def dpxmapfiles - force = @commandline.option("force") + force = @commandline.option("force") texmfroot = @commandline.argument('first') texmfroot = '.' if texmfroot.empty? - maproot = "#{texmfroot.gsub(/\\/,'/')}/fonts/map/pdftex/context" - + if @commandline.option('maproot') != "" then + maproot = @commandline.option('maproot') + else + maproot = "#{texmfroot.gsub(/\\/,'/')}/fonts/map/pdftex/context" + end if File.directory?(maproot) then files = Dir.glob("#{maproot}/*.map") if files.size > 0 then files.each do |pdffile| next if File.basename(pdffile) == 'pdftex.map' pdffile = File.expand_path(pdffile) - dpxfile = File.expand_path(pdffile.sub(/pdftex/i,'dvipdfm')) + dpxfile = File.expand_path(pdffile.sub(/(dvips|pdftex)/i,'dvipdfm')) unless pdffile == dpxfile then begin if data = File.read(pdffile) then @@ -1737,7 +1776,7 @@ class Array def add_shebang(filename,program) unless self[0] =~ /^\#/ then - self.insert(0,"\#!/usr/env #{program}") + self.insert(0,"\#!/usr/bin/env #{program}") end unless self[2] =~ /^\#.*?copyright\=/ then self.insert(1,"\#") @@ -2238,7 +2277,7 @@ class TexDeps report('') n = 0 @files.each do |filename| - if f = File.open(filename) then + if File.file?(filename) and f = File.open(filename) then defs, uses, l = 0, 0, 0 n += 1 report("#{n.to_s.rjust(5,' ')} #{filename}") @@ -2592,7 +2631,17 @@ class Commands def fetchfile(site, name, target=nil) begin - http = Net::HTTP.new(site) + proxy = @commandline.option('proxy') + if proxy && ! proxy.empty? then + address, port = proxy.split(":") + if address && port then + http = Net::HTTP::Proxy(address, port).new(site) + else + http = Net::HTTP::Proxy(proxy, 80).new(site) + end + else + http = Net::HTTP.new(site) + end resp, data = http.get(name.gsub(/^\/*/, '/')) rescue return false @@ -2640,7 +2689,14 @@ class Commands end def remakeformats - return system("texmfstart texexec --make --all") + system("mktexlsr") + system("luatools --selfupdate") + system("mtxrun --selfupdate") + system("luatools --generate") + system("texmfstart texexec --make --all --fast --pdftex") + system("texmfstart texexec --make --all --fast --luatex") + system("texmfstart texexec --make --all --fast --xetex") + return true end if localtree = locatedlocaltree then @@ -2693,11 +2749,13 @@ commandline.registeraction('listentities' , 'create doctype entity definiti commandline.registeraction('brandfiles' , 'add context copyright notice [--force]') commandline.registeraction('platformize' , 'replace line-endings [--recurse --force] [pattern]') commandline.registeraction('dependencies' , 'analyze depedencies within context [--save --compact --filter=[macros|filenames]] [filename]') -commandline.registeraction('updatecontext' , 'download latest version and remake formats') +commandline.registeraction('updatecontext' , 'download latest version and remake formats [--proxy]') commandline.registeraction('disarmutfbom' , 'remove utf bom [--force]') commandline.registervalue('type','') commandline.registervalue('filter','') +commandline.registervalue('maproot','') +commandline.registervalue('proxy','') commandline.registerflag('recurse') commandline.registerflag('force') diff --git a/Master/texmf-dist/scripts/context/ruby/graphics/gs.rb b/Master/texmf-dist/scripts/context/ruby/graphics/gs.rb index 070fd1be85a..cb3d016f4cb 100644 --- a/Master/texmf-dist/scripts/context/ruby/graphics/gs.rb +++ b/Master/texmf-dist/scripts/context/ruby/graphics/gs.rb @@ -296,9 +296,9 @@ class GhostScript def gscolorswitch case getvariable('colormodel') - when 'cmyk' then '-dProcessColorModel=/DeviceCMYK ' - when 'rgb' then '-dProcessColorModel=/DeviceRGB ' - when 'gray' then '-dProcessColorModel=/DeviceGRAY ' + when 'cmyk' then '-dProcessColorModel=/DeviceCMYK -dColorConversionStrategy=/CMYK ' + when 'rgb' then '-dProcessColorModel=/DeviceRGB -dColorConversionStrategy=/RGB ' + when 'gray' then '-dProcessColorModel=/DeviceGRAY -dColorConversionStrategy=/GRAY ' else '' end @@ -452,10 +452,10 @@ class GhostScript debug('bbox spec', bbox) - if bbox =~ /(Exact|HiRes)BoundingBox:#{@@bboxspec}/mois then + if bbox =~ /(Exact|HiRes)BoundingBox:#{@@bboxspec}/moi then debug("high res bbox #{$2} #{$3} #{$4} #{$5}") setdimensions($2,$3,$4,$5) - elsif bbox =~ /BoundingBox:#{@@bboxspec}/mois + elsif bbox =~ /BoundingBox:#{@@bboxspec}/moi debug("low res bbox #{$1} #{$2} #{$3} #{$4}") setdimensions($1,$2,$3,$4) end @@ -570,7 +570,12 @@ end if ! epsbbox && str =~ /^%%(Page:|EndProlog)/io then out.puts(str) if $1 == "EndProlog" debug('faking papersize') - out.puts("<< /PageSize [#{@width} #{@height}] >> setpagedevice\n") + # out.puts("<< /PageSize [#{@width} #{@height}] >> setpagedevice\n") + if ! dimensions? then + out.puts("<< /PageSize [1 1] >> setpagedevice\n") + else + out.puts("<< /PageSize [#{@width} #{@height}] >> setpagedevice\n") + end out.puts("gsave #{@xoffset} #{@yoffset} translate\n") epsbbox = true elsif str =~ /^%%BeginBinary\:\s*\d+\s*$/o then diff --git a/Master/texmf-dist/scripts/context/ruby/graphics/inkscape.rb b/Master/texmf-dist/scripts/context/ruby/graphics/inkscape.rb index 4495c3070f4..8d3b2646826 100644 --- a/Master/texmf-dist/scripts/context/ruby/graphics/inkscape.rb +++ b/Master/texmf-dist/scripts/context/ruby/graphics/inkscape.rb @@ -43,6 +43,10 @@ class InkScape def convert(logfile=System.null) + directpdf = false + + logfile = logfile.gsub(/\/+$/,"") + inpfilename = getvariable('inputfile').dup outfilename = getvariable('outputfile').dup outfilename = inpfilename.dup if outfilename.empty? @@ -62,11 +66,15 @@ class InkScape return false end - report("converting #{inpfilename} to #{tmpfilename}") - # we need to redirect the error info else we get a pop up console - resultpipe = "--without-gui --print=\">#{tmpfilename}\" 2>#{logfile}" + if directpdf then + report("converting #{inpfilename} to #{outfilename}") + resultpipe = "--without-gui --export-pdf=\"#{outfilename}\" 2>#{logfile}" + else + report("converting #{inpfilename} to #{tmpfilename}") + resultpipe = "--without-gui --print=\">#{tmpfilename}\" 2>#{logfile}" + end arguments = [resultpipe,inpfilename].join(' ').gsub(/\s+/,' ') @@ -76,8 +84,11 @@ class InkScape # should work # ok = System.run('inkscape',arguments) # does not work here # but 0.40 only works with this: - ok = system("inkscape #{arguments}") + command = "inkscape #{arguments}" + report(command) + ok = system(command) # and 0.41 fails with everything + # and 0.45 is better rescue report("aborted due to error") return false @@ -85,18 +96,16 @@ class InkScape return false unless ok end - ghostscript = GhostScript.new(@logger) - - ghostscript.setvariable('inputfile',tmpfilename) - ghostscript.setvariable('outputfile',outfilename) - - report("converting #{tmpfilename} to #{outfilename}") - - ghostscript.convert - - begin - File.delete(tmpfilename) - rescue + if not directpdf then + ghostscript = GhostScript.new(@logger) + ghostscript.setvariable('inputfile',tmpfilename) + ghostscript.setvariable('outputfile',outfilename) + report("converting #{tmpfilename} to #{outfilename}") + ghostscript.convert + begin + File.delete(tmpfilename) + rescue + end end end diff --git a/Master/texmf-dist/scripts/context/ruby/mtxtools.rb b/Master/texmf-dist/scripts/context/ruby/mtxtools.rb index 7cbe68c0d4c..41d0f7085b1 100644 --- a/Master/texmf-dist/scripts/context/ruby/mtxtools.rb +++ b/Master/texmf-dist/scripts/context/ruby/mtxtools.rb @@ -1,426 +1,475 @@ -#!/usr/bin/env ruby
-
-# program : mtxtools
-# copyright : PRAGMA Advanced Document Engineering
-# version : 2004-2005
-# author : Hans Hagen
-#
-# info : j.hagen@xs4all.nl
-# www : www.pragma-ade.com
-
-# This script hosts MetaTeX related features.
-
-banner = ['MtxTools', 'version 1.0.0', '2006', 'PRAGMA ADE/POD']
-
-$: << File.expand_path(File.dirname($0)) ; $: << File.join($:.last,'lib') ; $:.uniq!
-
-require 'base/switch'
-require 'base/logger'
-require 'base/system'
-require 'base/kpse'
-
-class Reporter
- def report(str)
- puts(str)
- end
-end
-
-module ConTeXt
-
- def ConTeXt::banner(filename,companionname,compact=false)
- "-- filename : #{File.basename(filename)}\n" +
- "-- comment : companion to #{File.basename(companionname)} (in ConTeXt)\n" +
- "-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL\n" +
- "-- copyright: PRAGMA ADE / ConTeXt Development Team\n" +
- "-- license : see context related readme files\n" +
- if compact then "\n-- remark : compact version\n" else "" end
- end
-
-end
-
-class UnicodeTables
-
- @@version = "1.001"
-
- @@shape_a = /^((GREEK|LATIN|HEBREW)\s*(SMALL|CAPITAL|)\s*LETTER\s*[A-Z]+)$/
- @@shape_b = /^((GREEK|LATIN|HEBREW)\s*(SMALL|CAPITAL|)\s*LETTER\s*[A-Z]+)\s*(.+)$/
-
- @@shape_a = /^(.*\s*LETTER\s*[A-Z]+)$/
- @@shape_b = /^(.*\s*LETTER\s*[A-Z]+)\s+WITH\s+(.+)$/
-
- attr_accessor :context, :comment
-
- def initialize(logger=Reporter.new)
- @data = Array.new
- @logger = logger
- @error = false
- @context = true
- @comment = true
- @shapes = Hash.new
- end
-
- def load_unicode_data(filename='unicodedata.txt')
- # beware, the unicodedata table is bugged, sometimes ending
- @logger.report("reading base data from #{filename}") if @logger
- begin
- IO.readlines(filename).each do |line|
- if line =~ /^[0-9A-F]{4,4}/ then
- d = line.chomp.sub(/\;$/, '').split(';')
- if d then
- while d.size < 15 do d << '' end
- n = d[0].hex
- @data[n] = d
- if d[1] =~ @@shape_a then
- @shapes[$1] = d[0]
- end
- end
- end
- end
- rescue
- @error = true
- @logger.report("error while reading base data from #{filename}") if @logger
- end
- end
-
- def load_context_data(filename='contextnames.txt')
- @logger.report("reading data from #{filename}") if @logger
- begin
- IO.readlines(filename).each do |line|
- if line =~ /^[0-9A-F]{4,4}/ then
- d = line.chomp.split(';')
- if d then
- n = d[0].hex
- if @data[n] then
- @data[d[0].hex] << d[1] # adobename == 15
- @data[d[0].hex] << d[2] # contextname == 16
- else
- @logger.report("missing information about #{d} in #{filename}") if @logger
- end
- end
- end
- end
- rescue
- @error = true
- @logger.report("error while reading context data from #{filename}") if @logger
- end
- end
-
- def save_metatex_data(filename='char-def.lua',compact=false)
- if not @error then
- begin
- File.open(filename,'w') do |f|
- @logger.report("saving data in #{filename}") if @logger
- f << ConTeXt::banner(filename,'char-def.tex',compact)
- f << "\n"
- f << "\nif not versions then versions = { } end versions['#{filename.gsub(/\..*?$/,'')}'] = #{@@version}\n"
- f << "\n"
- f << "if not characters then characters = { } end\n"
- f << "if not characters.data then characters.data = { } end\n"
- f << "\n"
- f << "characters.data = {\n" if compact
- @data.each do |d|
- if d then
- r = metatex_data(d)
- if compact then
- f << "\t" << "[0x#{d[0]}]".rjust(8,' ') << " = { #{r.join(", ").gsub(/\t/,'')} }, \n"
- else
- f << "characters.define { -- #{d[0].hex}" << "\n"
- f << r.join(",\n") << "\n"
- f << "}" << "\n"
- end
- end
- end
- f << "}\n" if compact
- end
- rescue
- @logger.report("error while saving data in #{filename}") if @logger
- else
- @logger.report("#{@data.size} (#{sprintf('%X',@data.size)}) entries saved in #{filename}") if @logger
- end
- else
- @logger.report("not saving data in #{filename} due to previous error") if @logger
- end
- end
-
- def metatex_data(d)
- r = Array.new
- r << "\tunicodeslot=0x#{d[0]}"
- if d[2] && ! d[2].empty? then
- r << "\tcategory='#{d[2].downcase}'"
- end
- if @context then
- if d[15] && ! d[15].empty? then
- r << "\tadobename='#{d[15]}'"
- end
- if d[16] && ! d[16].empty? then
- r << "\tcontextname='#{d[16]}'"
- end
- end
- if @comment then
- if d[1] == "<control>" then
- r << "\tdescription='#{d[10]}'" unless d[10].empty?
- else
- r << "\tdescription='#{d[1]}'" unless d[1].empty?
- end
- end
- if d[1] =~ @@shape_b then
- r << "\tshcode=0x#{@shapes[$1]}" if @shapes[$1]
- end
- if d[12] && ! d[12].empty? then
- r << "\tuccode=0x#{d[12]}"
- elsif d[14] && ! d[14].empty? then
- r << "\tuccode=0x#{d[14]}"
- end
- if d[13] && ! d[13].empty? then
- r << "\tlccode=0x#{d[13]}"
- end
- if d[5] && ! d[5].empty? then
- special, specials = '', Array.new
- c = d[5].split(/\s+/).collect do |cc|
- if cc =~ /^\<(.*)\>$/io then
- special = $1.downcase
- else
- specials << "0x#{cc}"
- end
- end
- if specials.size > 0 then
- special = 'char' if special.empty?
- r << "\tspecials={'#{special}',#{specials.join(',')}}"
- end
- end
- return r
- end
-
- def save_xetex_data(filename='enco-xtx.tex')
- if not @error then
- begin
- minnumber, maxnumber, n = 0x001F, 0xFFFF, 0
- File.open(filename,'w') do |f|
- @logger.report("saving data in #{filename}") if @logger
- f << "% filename : #{filename}\n"
- f << "% comment : poor man's alternative for a proper enco file\n"
- f << "% author : Hans Hagen, PRAGMA-ADE, Hasselt NL\n"
- f << "% copyright: PRAGMA ADE / ConTeXt Development Team\n"
- f << "% license : see context related readme files\n"
- f << "\n"
- f << "\\ifx\\setcclcuc\\undefined\n"
- f << "\n"
- f << " \\def\\setcclcuc #1 #2 #3 %\n"
- f << " {\\global\\catcode\"#1=11 \n"
- f << " \\global\\lccode \"#1=\"#2 \n"
- f << " \\global\\uccode \"#1=\"#3 }\n"
- f << "\n"
- f << "\\fi\n"
- f << "\n"
- @data.each do |d|
- if d then
- number, type = d[0], d[2].downcase
- if number.hex >= minnumber && number.hex <= maxnumber && type =~ /^l(l|u|t)$/o then
- if d[13] && ! d[13].empty? then
- lc = d[13]
- else
- lc = number
- end
- if d[12] && ! d[12].empty? then
- uc = d[12]
- elsif d[14] && ! d[14].empty? then
- uc = d[14]
- else
- uc = number
- end
- if @comment then
- f << "\\setcclcuc #{number} #{lc} #{uc} % #{d[1]}\n"
- else
- f << "\\setcclcuc #{number} #{lc} #{uc} \n"
- end
- n += 1
- end
- end
- end
- f << "\n"
- f << "\\endinput\n"
- end
- rescue
- @logger.report("error while saving data in #{filename}") if @logger
- else
- @logger.report("#{n} entries saved in #{filename}") if @logger
- end
- else
- @logger.report("not saving data in #{filename} due to previous error") if @logger
- end
- end
-
-end
-
-class RegimeTables
-
- @@version = "1.001"
-
- def initialize(logger=Reporter.new)
- @logger = logger
- reset
- end
-
- def reset
- @code, @regime, @filename, @loaded = Array.new(256), '', '', false
- (32..127).each do |i|
- @code[i] = [sprintf('%04X',i), i.chr]
- end
- end
-
- def load(filename)
- begin
- reset
- if filename =~ /regi\-(ini|run|uni|utf|syn)/ then
- report("skipping #{filename}")
- else
- report("loading file #{filename}")
- @regime, unicodeset = File.basename(filename).sub(/\..*?$/,''), false
- IO.readlines(filename).each do |line|
- case line
- when /^\#/ then
- # skip
- when /^(0x[0-9A-F]+)\s+(0x[0-9A-F]+)\s+\#\s+(.*)$/ then
- @code[$1.hex], unicodeset = [$2, $3], true
- when /^(0x[0-9A-F]+)\s+(0x[0-9A-F]+)\s+/ then
- @code[$1.hex], unicodeset = [$2, ''], true
- end
- end
- reset if not unicodeset
- end
- rescue
- report("problem in loading file #{filename}")
- reset
- else
- if ! @regime.empty? then
- @loaded = true
- else
- reset
- end
- end
- end
-
- def save(filename,compact=false)
- begin
- if @loaded && ! @regime.empty? then
- if File.expand_path(filename) == File.expand_path(@filename) then
- report("saving in #{filename} is blocked")
- else
- report("saving file #{filename}")
- File.open(filename,'w') do |f|
- f << ConTeXt::banner(filename,'regi-ini.tex',compact)
- f << "\n"
- f << "\nif not versions then versions = { } end versions['#{filename.gsub(/\..*?$/,'')}'] = #{@@version}\n"
- f << "\n"
- f << "if not regimes then regimes = { } end\n"
- f << "if not regimes.data then regimes.data = { } end\n"
- f << "\n"
- if compact then
- f << "regimes.data[\"#{@regime}\"] = { [0] = \n\t"
- i = 17
- @code.each_index do |c|
- if (i-=1) == 0 then
- i = 16
- f << "\n\t"
- end
- if @code[c] then
- f << @code[c][0].rjust(6,' ')
- else
- f << "0x0000".rjust(6,' ')
- end
- f << ', ' if c<@code.length-1
- end
- f << "\n}\n"
- else
- @code.each_index do |c|
- if @code[c] then
- f << someregimeslot(@regime,c,@code[c][0],@code[c][1])
- else
- f << someregimeslot(@regime,c,'','')
- end
- end
- end
- end
- end
- end
- rescue
- report("problem in saving file #{filename} #{$!}")
- end
- end
-
- def report(str)
- @logger.report(str)
- end
-
- private
-
- def someregimeslot(regime,slot,unicodeslot,comment)
- "regimes.define { #{if comment.empty? then '' else '-- ' end} #{comment}\n" +
- "\tregime='#{regime}',\n" +
- "\tslot='#{sprintf('0x%02X',slot)}',\n" +
- "\tunicodeslot='#{if unicodeslot.empty? then '0x0000' else unicodeslot end}'\n" +
- "}\n"
- end
-
- public
-
- def RegimeTables::convert(filenames,compact=false)
- filenames.each do |filename|
- txtfile = File.expand_path(filename)
- luafile = File.join(File.dirname(txtfile),'regi-'+File.basename(txtfile.sub(/\..*?$/, '.lua')))
- unless txtfile == luafile then
- regime = RegimeTables.new
- regime.load(txtfile)
- regime.save(luafile,compact)
- end
- end
- end
-
-end
-
-class Commands
-
- include CommandBase
-
- def unicodetable
- unicode = UnicodeTables.new(logger)
- unicode.load_unicode_data
- unicode.load_context_data
- unicode.save_metatex_data('char-def.lua',@commandline.option('compact'))
- end
-
- def xetextable
- unicode = UnicodeTables.new(logger)
- unicode.load_unicode_data
- unicode.load_context_data
- # unicode.comment = false
- unicode.save_xetex_data
- end
-
- def regimetable
- if @commandline.arguments.length > 0 then
- RegimeTables::convert(@commandline.arguments, @commandline.option('compact'))
- else
- RegimeTables::convert(Dir.glob("cp*.txt") , @commandline.option('compact'))
- RegimeTables::convert(Dir.glob("8859*.txt") , @commandline.option('compact'))
- end
- end
-
-end
-
-logger = Logger.new(banner.shift)
-commandline = CommandLine.new
-
-commandline.registeraction('unicodetable', 'create unicode table for metatex/luatex')
-commandline.registeraction('regimetable', 'create regime table(s) for metatex/luatex [--compact]')
-commandline.registeraction('xetextable' , 'create unicode table for xetex')
-
-# general
-
-commandline.registeraction('help')
-commandline.registeraction('version')
-commandline.registerflag('compact')
-
-commandline.expand
-
-Commands.new(commandline,logger,banner).send(commandline.action || 'help')
+#!/usr/bin/env ruby + +# program : mtxtools +# copyright : PRAGMA Advanced Document Engineering +# version : 2004-2005 +# author : Hans Hagen +# +# info : j.hagen@xs4all.nl +# www : www.pragma-ade.com + +# This script hosts MetaTeX related features. + +banner = ['MtxTools', 'version 1.0.0', '2006', 'PRAGMA ADE/POD'] + +$: << File.expand_path(File.dirname($0)) ; $: << File.join($:.last,'lib') ; $:.uniq! + +require 'base/switch' +require 'base/logger' +require 'base/system' +require 'base/kpse' + +class Reporter + def report(str) + puts(str) + end +end + +module ConTeXt + + def ConTeXt::banner(filename,companionname,compact=false) + "-- filename : #{File.basename(filename)}\n" + + "-- comment : companion to #{File.basename(companionname)} (in ConTeXt)\n" + + "-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL\n" + + "-- copyright: PRAGMA ADE / ConTeXt Development Team\n" + + "-- license : see context related readme files\n" + + if compact then "\n-- remark : compact version\n" else "" end + end + +end + +class UnicodeTables + + @@version = "1.001" + + @@shape_a = /^((GREEK|LATIN|HEBREW)\s*(SMALL|CAPITAL|)\s*LETTER\s*[A-Z]+)$/ + @@shape_b = /^((GREEK|LATIN|HEBREW)\s*(SMALL|CAPITAL|)\s*LETTER\s*[A-Z]+)\s*(.+)$/ + + @@shape_a = /^(.*\s*LETTER\s*[A-Z]+)$/ + @@shape_b = /^(.*\s*LETTER\s*[A-Z]+)\s+WITH\s+(.+)$/ + + attr_accessor :context, :comment + + def initialize(logger=Reporter.new) + @data = Array.new + @logger = logger + @error = false + @context = true + @comment = true + @shapes = Hash.new + end + + def load_unicode_data(filename='unicodedata.txt') + # beware, the unicodedata table is bugged, sometimes ending + @logger.report("reading base data from #{filename}") if @logger + begin + IO.readlines(filename).each do |line| + if line =~ /^[0-9A-F]{4,4}/ then + d = line.chomp.sub(/\;$/, '').split(';') + if d then + while d.size < 15 do d << '' end + n = d[0].hex + @data[n] = d + if d[1] =~ @@shape_a then + @shapes[$1] = d[0] + end + end + end + end + rescue + @error = true + @logger.report("error while reading base data from #{filename}") if @logger + end + end + + def load_context_data(filename='contextnames.txt') + @logger.report("reading data from #{filename}") if @logger + begin + IO.readlines(filename).each do |line| + if line =~ /^[0-9A-F]{4,4}/ then + d = line.chomp.split(';') + if d then + n = d[0].hex + if @data[n] then + @data[d[0].hex] << d[1] # adobename == 15 + @data[d[0].hex] << d[2] # contextname == 16 + else + @logger.report("missing information about #{d} in #{filename}") if @logger + end + end + end + end + rescue + @error = true + @logger.report("error while reading context data from #{filename}") if @logger + end + end + + def save_metatex_data(filename='char-def.lua',compact=false) + if not @error then + begin + File.open(filename,'w') do |f| + @logger.report("saving data in #{filename}") if @logger + f << ConTeXt::banner(filename,'char-def.tex',compact) + f << "\n" + f << "\nif not versions then versions = { } end versions['#{filename.gsub(/\..*?$/,'')}'] = #{@@version}\n" + f << "\n" + f << "if not characters then characters = { } end\n" + f << "if not characters.data then characters.data = { } end\n" + f << "\n" + f << "characters.data = {\n" if compact + @data.each do |d| + if d then + r = metatex_data(d) + if compact then + f << "\t" << "[0x#{d[0]}]".rjust(8,' ') << " = { #{r.join(", ").gsub(/\t/,'')} }, \n" + else + f << "characters.define { -- #{d[0].hex}" << "\n" + f << r.join(",\n") << "\n" + f << "}" << "\n" + end + end + end + f << "}\n" if compact + end + rescue + @logger.report("error while saving data in #{filename}") if @logger + else + @logger.report("#{@data.size} (#{sprintf('%X',@data.size)}) entries saved in #{filename}") if @logger + end + else + @logger.report("not saving data in #{filename} due to previous error") if @logger + end + end + + def metatex_data(d) + r = Array.new + r << "\tunicodeslot=0x#{d[0]}" + if d[2] && ! d[2].empty? then + r << "\tcategory='#{d[2].downcase}'" + end + if @context then + if d[15] && ! d[15].empty? then + r << "\tadobename='#{d[15]}'" + end + if d[16] && ! d[16].empty? then + r << "\tcontextname='#{d[16]}'" + end + end + if @comment then + if d[1] == "<control>" then + r << "\tdescription='#{d[10]}'" unless d[10].empty? + else + r << "\tdescription='#{d[1]}'" unless d[1].empty? + end + end + if d[1] =~ @@shape_b then + r << "\tshcode=0x#{@shapes[$1]}" if @shapes[$1] + end + if d[12] && ! d[12].empty? then + r << "\tuccode=0x#{d[12]}" + elsif d[14] && ! d[14].empty? then + r << "\tuccode=0x#{d[14]}" + end + if d[13] && ! d[13].empty? then + r << "\tlccode=0x#{d[13]}" + end + if d[5] && ! d[5].empty? then + special, specials = '', Array.new + c = d[5].split(/\s+/).collect do |cc| + if cc =~ /^\<(.*)\>$/io then + special = $1.downcase + else + specials << "0x#{cc}" + end + end + if specials.size > 0 then + special = 'char' if special.empty? + r << "\tspecials={'#{special}',#{specials.join(',')}}" + end + end + return r + end + + def save_xetex_data(filename='enco-utf.tex') + if not @error then + begin + minnumber, maxnumber, n = 0x001F, 0xFFFF, 0 + File.open(filename,'w') do |f| + @logger.report("saving data in #{filename}") if @logger + f << "% filename : #{filename}\n" + f << "% comment : poor man's alternative for a proper enco file\n" + f << "% this file is generated by mtxtools and can be\n" + f << "% used in xetex and luatex mkii mode\n" + f << "% author : Hans Hagen, PRAGMA-ADE, Hasselt NL\n" + f << "% copyright: PRAGMA ADE / ConTeXt Development Team\n" + f << "% license : see context related readme files\n" + f << "\n" + f << "\\ifx\\setcclcucx\\undefined\n" + f << "\n" + f << " \\def\\setcclcucx #1 #2 #3 %\n" + f << " {\\global\\catcode\"#1=11 \n" + f << " \\global\\lccode \"#1=\"#2 \n" + f << " \\global\\uccode \"#1=\"#3 }\n" + f << "\n" + f << "\\fi\n" + f << "\n" + @data.each do |d| + if d then + number, type = d[0], d[2].downcase + if number.hex >= minnumber && number.hex <= maxnumber && type =~ /^l(l|u|t)$/o then + if d[13] && ! d[13].empty? then + lc = d[13] + else + lc = number + end + if d[12] && ! d[12].empty? then + uc = d[12] + elsif d[14] && ! d[14].empty? then + uc = d[14] + else + uc = number + end + if @comment then + f << "\\setcclcuc #{number} #{lc} #{uc} % #{d[1]}\n" + else + f << "\\setcclcuc #{number} #{lc} #{uc} \n" + end + n += 1 + end + end + end + f << "\n" + f << "\\endinput\n" + end + rescue + @logger.report("error while saving data in #{filename}") if @logger + else + @logger.report("#{n} entries saved in #{filename}") if @logger + end + else + @logger.report("not saving data in #{filename} due to previous error") if @logger + end + end + +end + +class RegimeTables + + @@version = "1.001" + + def initialize(logger=Reporter.new) + @logger = logger + reset + end + + def reset + @code, @regime, @filename, @loaded = Array.new(256), '', '', false + (32..127).each do |i| + @code[i] = [sprintf('%04X',i), i.chr] + end + end + + def load(filename) + begin + reset + if filename =~ /regi\-(ini|run|uni|utf|syn)/ then + report("skipping #{filename}") + else + report("loading file #{filename}") + @regime, unicodeset = File.basename(filename).sub(/\..*?$/,''), false + IO.readlines(filename).each do |line| + case line + when /^\#/ then + # skip + when /^(0x[0-9A-F]+)\s+(0x[0-9A-F]+)\s+\#\s+(.*)$/ then + @code[$1.hex], unicodeset = [$2, $3], true + when /^(0x[0-9A-F]+)\s+(0x[0-9A-F]+)\s+/ then + @code[$1.hex], unicodeset = [$2, ''], true + end + end + reset if not unicodeset + end + rescue + report("problem in loading file #{filename}") + reset + else + if ! @regime.empty? then + @loaded = true + else + reset + end + end + end + + def save(filename,compact=false) + begin + if @loaded && ! @regime.empty? then + if File.expand_path(filename) == File.expand_path(@filename) then + report("saving in #{filename} is blocked") + else + report("saving file #{filename}") + File.open(filename,'w') do |f| + f << ConTeXt::banner(filename,'regi-ini.tex',compact) + f << "\n" + f << "\nif not versions then versions = { } end versions['#{filename.gsub(/\..*?$/,'')}'] = #{@@version}\n" + f << "\n" + f << "if not regimes then regimes = { } end\n" + f << "if not regimes.data then regimes.data = { } end\n" + f << "\n" + if compact then + f << "regimes.data[\"#{@regime}\"] = { [0] = \n\t" + i = 17 + @code.each_index do |c| + if (i-=1) == 0 then + i = 16 + f << "\n\t" + end + if @code[c] then + f << @code[c][0].rjust(6,' ') + else + f << "0x0000".rjust(6,' ') + end + f << ', ' if c<@code.length-1 + end + f << "\n}\n" + else + @code.each_index do |c| + if @code[c] then + f << someregimeslot(@regime,c,@code[c][0],@code[c][1]) + else + f << someregimeslot(@regime,c,'','') + end + end + end + end + end + end + rescue + report("problem in saving file #{filename} #{$!}") + end + end + + def report(str) + @logger.report(str) + end + + private + + def someregimeslot(regime,slot,unicodeslot,comment) + "regimes.define { #{if comment.empty? then '' else '-- ' end} #{comment}\n" + + "\tregime='#{regime}',\n" + + "\tslot='#{sprintf('0x%02X',slot)}',\n" + + "\tunicodeslot='#{if unicodeslot.empty? then '0x0000' else unicodeslot end}'\n" + + "}\n" + end + + public + + def RegimeTables::convert(filenames,compact=false) + filenames.each do |filename| + txtfile = File.expand_path(filename) + luafile = File.join(File.dirname(txtfile),'regi-'+File.basename(txtfile.sub(/\..*?$/, '.lua'))) + unless txtfile == luafile then + regime = RegimeTables.new + regime.load(txtfile) + regime.save(luafile,compact) + end + end + end + +end + +class Commands + + include CommandBase + + def unicodetable + unicode = UnicodeTables.new(logger) + unicode.load_unicode_data + unicode.load_context_data + unicode.save_metatex_data('char-def.lua',@commandline.option('compact')) + end + + def xetextable + unicode = UnicodeTables.new(logger) + unicode.load_unicode_data + unicode.load_context_data + # unicode.comment = false + unicode.save_xetex_data + end + + def regimetable + if @commandline.arguments.length > 0 then + RegimeTables::convert(@commandline.arguments, @commandline.option('compact')) + else + RegimeTables::convert(Dir.glob("cp*.txt") , @commandline.option('compact')) + RegimeTables::convert(Dir.glob("8859*.txt") , @commandline.option('compact')) + end + end + + def pdftextable + # instead of directly saving the data, we use luatex (kind of test) + pdfrdef = 'pdfr-def.tex' + tmpfile = 'mtxtools.tmp' + File.delete(pdfrdef) rescue false + if f = File.open(tmpfile,'w') then + f << "\\starttext\n" + f << "\\ctxlua{characters.pdftex.make_pdf_to_unicodetable('#{pdfrdef}')}\n" + f << "\\stoptext\n" + f.close() + system("texmfstart texexec --luatex --once --purge mtxtools.tmp") + report("vecor saved in #{pdfrdef}") + end + File.delete(tmpfile) rescue false + end + + def xmlmapfile + # instead of directly saving the data, we use luatex (kind of test) + tmpfile = 'mtxtools.tmp' + xmlsuffix = 'frx' + @commandline.arguments.each do |mapname| + if f = File.open(tmpfile,'w') then + xmlname = mapname.gsub(/\.map$/,".#{xmlsuffix}") + File.delete(xmlname) rescue false + f << "\\starttext\n" + f << "\\ctxlua{\n" + f << " mapname = input.find_file(texmf.instance,'#{mapname}') or ''\n" + f << " xmlname = '#{xmlname}'\n" + f << " if mapname and not mapname:is_empty() then\n" + f << " ctx.fonts.map.convert_file(mapname,xmlname)\n" + f << " end\n" + f << "}\n" + f << "\\stoptext\n" + f.close() + system("texmfstart texexec --luatex --once --purge mtxtools.tmp") + if FileTest.file?(xmlname) then + report("map file #{mapname} converted to #{xmlname}") + else + report("no valid map file #{mapname}") + end + end + end + File.delete(tmpfile) rescue false + end + +end + +logger = Logger.new(banner.shift) +commandline = CommandLine.new + +commandline.registeraction('unicodetable', 'create unicode table for metatex/luatex') +commandline.registeraction('regimetable' , 'create regime table(s) for metatex/luatex [--compact]') +commandline.registeraction('xetextable' , 'create unicode table for xetex') +commandline.registeraction('pdftextable' , 'create unicode table for xetex') +commandline.registeraction('xmlmapfile' , 'convert traditional mapfile to xml font resourse') + +# general + +commandline.registeraction('help') +commandline.registeraction('version') +commandline.registerflag('compact') + +commandline.expand + +Commands.new(commandline,logger,banner).send(commandline.action || 'help') diff --git a/Master/texmf-dist/scripts/context/ruby/pdftools.rb b/Master/texmf-dist/scripts/context/ruby/pdftools.rb index 25e7250b67f..23edfeca226 100644 --- a/Master/texmf-dist/scripts/context/ruby/pdftools.rb +++ b/Master/texmf-dist/scripts/context/ruby/pdftools.rb @@ -397,20 +397,20 @@ class SpotColorImage if data =~ /(\d+)\s+0\s+obj\s+\[\/Separation\s+\/#{@colorname}/mos then @command.report("replacing separation color") object = $1 - data.gsub!(/(\/Type\s+\/XObject.*?)(\/ColorSpace\s*(\/DeviceGray|\/DeviceCMYK|\/DeviceRGB|\d+\s+\d+\s+R))/mois) do + data.gsub!(/(\/Type\s+\/XObject.*?)(\/ColorSpace\s*(\/DeviceGray|\/DeviceCMYK|\/DeviceRGB|\d+\s+\d+\s+R))/moi) do $1 + "/ColorSpace #{object} 0 R".ljust($2.length) end elsif data =~ /(\d+)\s+0\s+obj\s+\[\/Indexed\s*\[/mos then @command.report("replacing indexed color") # todo: more precise check on color object = $1 - data.gsub!(/(\/Type\s+\/XObject.*?)(\/ColorSpace\s*(\/DeviceGray|\/DeviceCMYK|\/DeviceRGB|\d+\s+\d+\s+R))/mois) do + data.gsub!(/(\/Type\s+\/XObject.*?)(\/ColorSpace\s*(\/DeviceGray|\/DeviceCMYK|\/DeviceRGB|\d+\s+\d+\s+R))/moi) do $1 + "/ColorSpace #{object} 0 R".ljust($2.length) end elsif data =~ /(\d+)\s+0\s+obj\s+\[\/Separation/mos then @command.report("replacing separation color") object = $1 - data.gsub!(/(\/Type\s+\/XObject.*?)(\/ColorSpace\s*(\/DeviceGray|\/DeviceCMYK|\/DeviceRGB|\d+\s+\d+\s+R))/mois) do + data.gsub!(/(\/Type\s+\/XObject.*?)(\/ColorSpace\s*(\/DeviceGray|\/DeviceCMYK|\/DeviceRGB|\d+\s+\d+\s+R))/moi) do $1 + "/ColorSpace #{object} 0 R".ljust($2.length) end end @@ -663,7 +663,7 @@ class Commands pairs = Hash.new data.each do |d| - if (d =~ /^\s*(.*?)\s*\:\s*(.*?)\s*$/mois) then + if (d =~ /^\s*(.*?)\s*\:\s*(.*?)\s*$/moi) then key, val = $1, $2 pairs[key.downcase.sub(/ /,'')] = val end @@ -780,7 +780,7 @@ class Commands end threshold = @commandline.option('threshold').to_i rescue 0 filenames.each do |filename| - if `pdfinfo #{filename}`.chomp =~ /^pages\s*\:\s*(\d+)/mois then + if `pdfinfo #{filename}`.chomp =~ /^pages\s*\:\s*(\d+)/moi then p = $1 m = p.to_i rescue 0 if threshold == 0 or m > threshold then diff --git a/Master/texmf-dist/scripts/context/ruby/pstopdf.rb b/Master/texmf-dist/scripts/context/ruby/pstopdf.rb index 3625f4d83dc..73e628df23b 100644 --- a/Master/texmf-dist/scripts/context/ruby/pstopdf.rb +++ b/Master/texmf-dist/scripts/context/ruby/pstopdf.rb @@ -76,7 +76,7 @@ class Commands @commandline.arguments.each do |filename| - filename = Tool.cleanfilename(filename,@commandline) + filename = Tool.cleanfilename(filename,@commandline) # brrrr inppath = @commandline.option('inputpath') if inppath.empty? then inppath = '.' diff --git a/Master/texmf-dist/scripts/context/ruby/rlxtools.rb b/Master/texmf-dist/scripts/context/ruby/rlxtools.rb index ea065612f0a..1617fcad4ae 100644 --- a/Master/texmf-dist/scripts/context/ruby/rlxtools.rb +++ b/Master/texmf-dist/scripts/context/ruby/rlxtools.rb @@ -300,6 +300,7 @@ class Commands begin str = nil if FileTest.file?(filename) then + # todo: use pdfinto for pdf files, identify is bugged if centimeters then result = `identify -units PixelsPerCentimeter -format \"x=%x,y=%y,w=%w,h=%h,b=%b\" #{filename}`.chomp.split(',') else diff --git a/Master/texmf-dist/scripts/context/ruby/texexec.rb b/Master/texmf-dist/scripts/context/ruby/texexec.rb index f37cfd8c2b0..a09572c6caa 100644 --- a/Master/texmf-dist/scripts/context/ruby/texexec.rb +++ b/Master/texmf-dist/scripts/context/ruby/texexec.rb @@ -111,7 +111,7 @@ class Commands if job = TEX.new(logger) then prepare(job) job.cleanuptemprunfiles - files = @commandline.arguments.sort + files = if @commandline.option('sort') then @commandline.arguments.sort else @commandline.arguments end if files.length > 0 then if f = File.open(job.tempfilename('tex'),'w') then backspace = @commandline.checkedoption('backspace', '1.5cm') @@ -156,7 +156,7 @@ class Commands prepare(job) job.cleanuptemprunfiles fast = @commandline.option('fast') - files = @commandline.arguments.sort + files = if @commandline.option('sort') then @commandline.arguments.sort else @commandline.arguments end if fast or (files.length > 0) then if f = File.open(job.tempfilename('tex'),'w') then files.delete("texexec.pdf") @@ -202,7 +202,7 @@ class Commands if job = TEX.new(logger) then prepare(job) job.cleanuptemprunfiles - files = @commandline.arguments.sort + files = if @commandline.option('sort') then @commandline.arguments.sort else @commandline.arguments end msuffixes = ['tex','mkii','mkiv','mp','pl','pm','rb'] if files.length > 0 then files.each do |fname| @@ -242,7 +242,7 @@ class Commands mod << "\\stoptext\n" mod.close job.setvariable('interface','english') # redundant - job.setvariable('simplerun',true) + # job.setvariable('simplerun',true) # job.setvariable('nooptionfile',true) job.setvariable('files',[job.tempfilename]) result = File.unsuffixed(File.basename(ffname)) @@ -302,7 +302,7 @@ class Commands if job = TEX.new(logger) then prepare(job) job.cleanuptemprunfiles - files = @commandline.arguments.sort + files = if @commandline.option('sort') then @commandline.arguments.sort else @commandline.arguments end if files.length > 0 then if f = File.open(job.tempfilename('tex'),'w') then emptypages = @commandline.checkedoption('addempty', '') @@ -355,7 +355,7 @@ class Commands if job = TEX.new(logger) then prepare(job) job.cleanuptemprunfiles - files = @commandline.arguments.sort + files = if @commandline.option('sort') then @commandline.arguments.sort else @commandline.arguments end if files.length > 0 then if f = File.open(job.tempfilename('tex'),'w') then selection = @commandline.checkedoption('selection', '') @@ -425,15 +425,16 @@ class Commands if job = TEX.new(logger) then prepare(job) job.cleanuptemprunfiles - files = @commandline.arguments.sort + files = if @commandline.option('sort') then @commandline.arguments.sort else @commandline.arguments end if files.length > 0 then if f = File.open(job.tempfilename('tex'),'w') then scale = @commandline.checkedoption('scale') begin - scale = (scale.to_i * 1000).to_i if scale.to_i < 10 + scale = (scale.to_f * 1000.0).to_i if scale.to_i < 10 rescue scale = 1000 end + scale = scale.to_i paperoffset = @commandline.checkedoption('paperoffset', '0cm') f << "\\starttext\n" files.each do |filename| @@ -491,7 +492,7 @@ class Commands if job = TEX.new(logger) then prepare(job) job.cleanuptemprunfiles - files = @commandline.arguments.sort + files = if @commandline.option('sort') then @commandline.arguments.sort else @commandline.arguments end if files.length > 0 then if f = File.open(job.tempfilename('tex'),'w') then paperoffset = @commandline.checkedoption('paperoffset', '0cm') @@ -570,16 +571,20 @@ class Commands job.setvariable(k,@commandline.option(k)) unless @commandline.option(k).empty? end +job.setvariable('given.backend',job.getvariable('backend')) + if (str = @commandline.option('engine')) && ! str.standard? && ! str.empty? then job.setvariable('texengine',str) + elsif @commandline.oneof('luatex') then + job.setvariable('texengine','luatex') elsif @commandline.oneof('pdfetex','pdftex','pdf') then job.setvariable('texengine','pdftex') elsif @commandline.oneof('xetex','xtx') then job.setvariable('texengine','xetex') elsif @commandline.oneof('aleph') then job.setvariable('texengine','aleph') - elsif @commandline.oneof('luatex') then - job.setvariable('texengine','luatex') + elsif @commandline.oneof('petex') then + job.setvariable('texengine','petex') else job.setvariable('texengine','standard') end @@ -594,6 +599,8 @@ class Commands job.setvariable('backend','xetex') elsif @commandline.oneof('aleph') then job.setvariable('backend','dvipdfmx') + elsif @commandline.oneof('petex') then + job.setvariable('backend','dvipdfmx') elsif @commandline.oneof('dvips','ps') then job.setvariable('backend','dvips') elsif @commandline.oneof('xdv') then @@ -604,6 +611,7 @@ class Commands when 'pdftex' then job.setvariable('backend','pdftex') when 'luatex' then job.setvariable('backend','pdftex') when 'xetex' then job.setvariable('backend','xetex') + when 'petex' then job.setvariable('backend','dvipdfmx') when 'aleph' then job.setvariable('backend','dvipdfmx') else job.setvariable('backend','standard') @@ -757,8 +765,11 @@ commandline.registerflag('xdv') commandline.registerflag('aleph') +commandline.registerflag('petex') + commandline.registerflag('all') commandline.registerflag('fast') +commandline.registerflag('sort') # generic diff --git a/Master/texmf-dist/scripts/context/ruby/texmfstart.rb b/Master/texmf-dist/scripts/context/ruby/texmfstart.rb index 4d1e2976145..e43929213f5 100644 --- a/Master/texmf-dist/scripts/context/ruby/texmfstart.rb +++ b/Master/texmf-dist/scripts/context/ruby/texmfstart.rb @@ -36,6 +36,8 @@ $: << File.expand_path(File.dirname($0)) ; $: << File.join($:.last,'lib') ; $:.u require "rbconfig" require "md5" +# funny, selfmergs was suddenly broken to case problems + # kpse_merge_done: require 'base/kpseremote' # kpse_merge_done: require 'base/kpsedirect' # kpse_merge_done: require 'base/kpsefast' @@ -43,127 +45,7 @@ require "md5" # kpse_merge_start -# kpse_merge_file: 'C:/data/develop/context/ruby/base/kpseremote.rb' - -# kpse_merge_done: require 'base/kpsefast' - -case ENV['KPSEMETHOD'] - when /soap/o then # kpse_merge_done: require 'base/kpse/soap' - when /drb/o then # kpse_merge_done: require 'base/kpse/drb' - else # kpse_merge_done: require 'base/kpse/drb' -end - -class KpseRemote - - @@port = ENV['KPSEPORT'] || 7000 - @@method = ENV['KPSEMETHOD'] || 'drb' - - def KpseRemote::available? - @@method && @@port - end - - def KpseRemote::start_server(port=nil) - kpse = KpseServer.new(port || @@port) - kpse.start - end - - def KpseRemote::start_client(port=nil) # keeps object in server - kpseclient = KpseClient.new(port || @@port) - kpseclient.start - kpse = kpseclient.object - tree = kpse.choose(KpseUtil::identify, KpseUtil::environment) - [kpse, tree] - end - - def KpseRemote::fetch(port=nil) # no need for defining methods but slower, send whole object - kpseclient = KpseClient.new(port || @@port) - kpseclient.start - kpseclient.object.fetch(KpseUtil::identify, KpseUtil::environment) rescue nil - end - - def initialize(port=nil) - if KpseRemote::available? then - begin - @kpse, @tree = KpseRemote::start_client(port) - rescue - @kpse, @tree = nil, nil - end - else - @kpse, @tree = nil, nil - end - end - - def progname=(value) - @kpse.set(@tree,'progname',value) - end - def format=(value) - @kpse.set(@tree,'format',value) - end - def engine=(value) - @kpse.set(@tree,'engine',value) - end - - def progname - @kpse.get(@tree,'progname') - end - def format - @kpse.get(@tree,'format') - end - def engine - @kpse.get(@tree,'engine') - end - - def load - @kpse.load(KpseUtil::identify, KpseUtil::environment) - end - def okay? - @kpse && @tree - end - def set(key,value) - @kpse.set(@tree,key,value) - end - def load_cnf - @kpse.load_cnf(@tree) - end - def load_lsr - @kpse.load_lsr(@tree) - end - def expand_variables - @kpse.expand_variables(@tree) - end - def expand_braces(str) - clean_name(@kpse.expand_braces(@tree,str)) - end - def expand_path(str) - clean_name(@kpse.expand_path(@tree,str)) - end - def expand_var(str) - clean_name(@kpse.expand_var(@tree,str)) - end - def show_path(str) - clean_name(@kpse.show_path(@tree,str)) - end - def var_value(str) - clean_name(@kpse.var_value(@tree,str)) - end - def find_file(filename) - clean_name(@kpse.find_file(@tree,filename)) - end - def find_files(filename,first=false) - # dodo: each filename - @kpse.find_files(@tree,filename,first) - end - - private - - def clean_name(str) - str.gsub(/\\/,'/') - end - -end - - -# kpse_merge_file: 'C:/data/develop/context/ruby/base/kpsefast.rb' +# kpse_merge_file: 't:/ruby/base/kpsefast.rb' # module : base/kpsefast # copyright : PRAGMA Advanced Document Engineering @@ -1097,67 +979,7 @@ end -# kpse_merge_file: 'C:/data/develop/context/ruby/base/kpse/drb.rb' - -require 'drb' -# kpse_merge_done: require 'base/kpse/trees' - -class KpseServer - - attr_accessor :port - - def initialize(port=7000) - @port = port - end - - def start - puts "starting drb service at port #{@port}" - DRb.start_service("druby://localhost:#{@port}", KpseTrees.new) - trap(:INT) do - DRb.stop_service - end - DRb.thread.join - end - - def stop - # todo - end - -end - -class KpseClient - - attr_accessor :port - - def initialize(port=7000) - @port = port - @kpse = nil - end - - def start - # only needed when callbacks are used / slow, due to Socket::getaddrinfo - # DRb.start_service - end - - def object - @kpse = DRbObject.new(nil,"druby://localhost:#{@port}") - end - -end - - -# SERVER_URI="druby://localhost:8787" -# -# # Start a local DRbServer to handle callbacks. -# # -# # Not necessary for this small example, but will be required -# # as soon as we pass a non-marshallable object as an argument -# # to a dRuby call. -# DRb.start_service -# - - -# kpse_merge_file: 'C:/data/develop/context/ruby/base/kpse/trees.rb' +# kpse_merge_file: 't:/ruby/base/kpse/trees.rb' require 'monitor' # kpse_merge_done: require 'base/kpsefast' @@ -1245,179 +1067,219 @@ class KpseTrees < Monitor end -# kpse_merge_file: 'C:/data/develop/context/ruby/base/kpsedirect.rb' +# kpse_merge_file: 't:/ruby/base/kpse/drb.rb' -class KpseDirect +require 'drb' +# kpse_merge_done: require 'base/kpse/trees' - attr_accessor :progname, :format, :engine +class KpseServer - def initialize - @progname, @format, @engine = '', '', '' - end + attr_accessor :port - def expand_path(str) - clean_name(`kpsewhich -expand-path=#{str}`.chomp) + def initialize(port=7000) + @port = port end - def expand_var(str) - clean_name(`kpsewhich -expand-var=#{str}`.chomp) + def start + puts "starting drb service at port #{@port}" + DRb.start_service("druby://localhost:#{@port}", KpseTrees.new) + trap(:INT) do + DRb.stop_service + end + DRb.thread.join end - def find_file(str) - clean_name(`kpsewhich #{_progname_} #{_format_} #{str}`.chomp) + def stop + # todo end - def _progname_ - if @progname.empty? then '' else "-progname=#{@progname}" end - end - def _format_ - if @format.empty? then '' else "-format=\"#{@format}\"" end +end + +class KpseClient + + attr_accessor :port + + def initialize(port=7000) + @port = port + @kpse = nil end - private + def start + # only needed when callbacks are used / slow, due to Socket::getaddrinfo + # DRb.start_service + end - def clean_name(str) - str.gsub(/\\/,'/') + def object + @kpse = DRbObject.new(nil,"druby://localhost:#{@port}") end end -# kpse_merge_file: 'C:/data/develop/context/ruby/base/merge.rb' - -# module : base/merge -# copyright : PRAGMA Advanced Document Engineering -# version : 2006 -# author : Hans Hagen +# SERVER_URI="druby://localhost:8787" +# +# # Start a local DRbServer to handle callbacks. +# # +# # Not necessary for this small example, but will be required +# # as soon as we pass a non-marshallable object as an argument +# # to a dRuby call. +# DRb.start_service # -# project : ConTeXt / eXaMpLe -# concept : Hans Hagen -# info : j.hagen@xs4all.nl -# this module will package all the used modules in the file itself -# so that we can relocate the file at wish, usage: -# -# merge: -# -# unless SelfMerge::ok? && SelfMerge::merge then -# puts("merging should happen on the path were the base inserts reside") -# end -# -# cleanup: -# -# unless SelfMerge::cleanup then -# puts("merging should happen on the path were the base inserts reside") +# kpse_merge_file: 't:/ruby/base/kpseremote.rb' -module SelfMerge +# kpse_merge_done: require 'base/kpsefast' - @@kpsemergestart = "\# kpse_merge_start" - @@kpsemergestop = "\# kpse_merge_stop" - @@kpsemergefile = "\# kpse_merge_file: " - @@kpsemergedone = "\# kpse_merge_done: " +case ENV['KPSEMETHOD'] + when /soap/o then # kpse_merge_done: require 'base/kpse/soap' + when /drb/o then # kpse_merge_done: require 'base/kpse/drb' + else # kpse_merge_done: require 'base/kpse/drb' +end - @@filename = File.basename($0) - @@ownpath = File.expand_path(File.dirname($0)) - @@modroot = '(base|graphics|rslb|www)' # needed in regex in order not to mess up SelfMerge - @@modules = $".collect do |file| File.expand_path(file) end +class KpseRemote - @@modules.delete_if do |file| - file !~ /^#{@@ownpath}\/#{@@modroot}.*$/ + @@port = ENV['KPSEPORT'] || 7000 + @@method = ENV['KPSEMETHOD'] || 'drb' + + def KpseRemote::available? + @@method && @@port end - def SelfMerge::ok? - begin - @@modules.each do |file| - return false unless FileTest.file?(file) - end - rescue - return false - else - return true - end + def KpseRemote::start_server(port=nil) + kpse = KpseServer.new(port || @@port) + kpse.start end - def SelfMerge::merge - begin - if SelfMerge::ok? && rbfile = IO.read(@@filename) then - begin - inserts = "#{@@kpsemergestart}\n\n" - @@modules.each do |file| - inserts << "#{@@kpsemergefile}'#{file}'\n\n" - inserts << IO.read(file).gsub(/^#.*?\n$/,'') - inserts << "\n\n" - end - inserts << "#{@@kpsemergestop}\n\n" - # no gsub! else we end up in SelfMerge - rbfile.sub!(/#{@@kpsemergestart}\s*#{@@kpsemergestop}/mois) do - inserts - end - rbfile.gsub!(/^(.*)(require [\"\'].*?#{@@modroot}.*)$/) do - pre, post = $1, $2 - if pre =~ /#{@@kpsemergedone}/ then - "#{pre}#{post}" - else - "#{pre}#{@@kpsemergedone}#{post}" - end - end - rescue - return false - else - begin - File.open(@@filename,'w') do |f| - f << rbfile - end - rescue - return false - end - end - end - rescue - return false - else - return true - end + def KpseRemote::start_client(port=nil) # keeps object in server + kpseclient = KpseClient.new(port || @@port) + kpseclient.start + kpse = kpseclient.object + tree = kpse.choose(KpseUtil::identify, KpseUtil::environment) + [kpse, tree] end - def SelfMerge::cleanup - begin - if rbfile = IO.read(@@filename) then - begin - rbfile.sub!(/#{@@kpsemergestart}(.*)#{@@kpsemergestop}\s*/mois) do - "#{@@kpsemergestart}\n\n#{@@kpsemergestop}\n\n" - end - rbfile.gsub!(/^(.*#{@@kpsemergedone}.*)$/) do - str = $1 - if str =~ /require [\"\']/ then - str.gsub(/#{@@kpsemergedone}/, '') - else - str - end - end - rescue - return false - else - begin - File.open(@@filename,'w') do |f| - f << rbfile - end - rescue - return false - end - end + def KpseRemote::fetch(port=nil) # no need for defining methods but slower, send whole object + kpseclient = KpseClient.new(port || @@port) + kpseclient.start + kpseclient.object.fetch(KpseUtil::identify, KpseUtil::environment) rescue nil + end + + def initialize(port=nil) + if KpseRemote::available? then + begin + @kpse, @tree = KpseRemote::start_client(port) + rescue + @kpse, @tree = nil, nil end - rescue - return false else - return true + @kpse, @tree = nil, nil end end - def SelfMerge::replace - if SelfMerge::ok? then - SelfMerge::cleanup - SelfMerge::merge - end + def progname=(value) + @kpse.set(@tree,'progname',value) + end + def format=(value) + @kpse.set(@tree,'format',value) + end + def engine=(value) + @kpse.set(@tree,'engine',value) + end + + def progname + @kpse.get(@tree,'progname') + end + def format + @kpse.get(@tree,'format') + end + def engine + @kpse.get(@tree,'engine') + end + + def load + @kpse.load(KpseUtil::identify, KpseUtil::environment) + end + def okay? + @kpse && @tree + end + def set(key,value) + @kpse.set(@tree,key,value) + end + def load_cnf + @kpse.load_cnf(@tree) + end + def load_lsr + @kpse.load_lsr(@tree) + end + def expand_variables + @kpse.expand_variables(@tree) + end + def expand_braces(str) + clean_name(@kpse.expand_braces(@tree,str)) + end + def expand_path(str) + clean_name(@kpse.expand_path(@tree,str)) + end + def expand_var(str) + clean_name(@kpse.expand_var(@tree,str)) + end + def show_path(str) + clean_name(@kpse.show_path(@tree,str)) + end + def var_value(str) + clean_name(@kpse.var_value(@tree,str)) + end + def find_file(filename) + clean_name(@kpse.find_file(@tree,filename)) + end + def find_files(filename,first=false) + # dodo: each filename + @kpse.find_files(@tree,filename,first) + end + + private + + def clean_name(str) + str.gsub(/\\/,'/') + end + +end + + +# kpse_merge_file: 't:/ruby/base/kpsedirect.rb' + +class KpseDirect + + attr_accessor :progname, :format, :engine + + def initialize + @progname, @format, @engine = '', '', '' + end + + def expand_path(str) + clean_name(`kpsewhich -expand-path=#{str}`.chomp) + end + + def expand_var(str) + clean_name(`kpsewhich -expand-var=#{str}`.chomp) + end + + def find_file(str) + clean_name(`kpsewhich #{_progname_} #{_format_} #{str}`.chomp) + end + + def _progname_ + if @progname.empty? then '' else "-progname=#{@progname}" end + end + def _format_ + if @format.empty? then '' else "-format=\"#{@format}\"" end + end + + private + + def clean_name(str) + str.gsub(/\\/,'/') end end @@ -1445,6 +1307,7 @@ $stderr.sync = true $applications = Hash.new $suffixinputs = Hash.new $predefined = Hash.new +$runners = Hash.new $suffixinputs['pl'] = 'PERLINPUTS' $suffixinputs['rb'] = 'RUBYINPUTS' @@ -1474,8 +1337,8 @@ $predefined['pdftools'] = 'pdftools.rb' $predefined['mpstools'] = 'mpstools.rb' $predefined['exatools'] = 'exatools.rb' $predefined['xmltools'] = 'xmltools.rb' -$predefined['luatools'] = 'luatools.lua' -$predefined['mtxtools'] = 'mtxtools.rb' +# $predefined['luatools'] = 'luatools.lua' +# $predefined['mtxtools'] = 'mtxtools.rb' $predefined['newpstopdf'] = 'pstopdf.rb' $predefined['newtexexec'] = 'texexec.rb' @@ -1500,9 +1363,12 @@ $makelist = [ 'exatools', 'runtools', 'rlxtools', - 'luatools', - 'mtxtools', + 'pdftrimwhite', + 'texfind', + 'texshow' # + # no 'luatools', + # no 'mtxtools', # no, 'texmfstart' ] @@ -1517,10 +1383,10 @@ $kpse = nil def set_applications(page=1) $applications['unknown'] = '' - $applications['perl'] = $applications['pl'] = 'perl' $applications['ruby'] = $applications['rb'] = 'ruby' + $applications['lua'] = $applications['lua'] = 'lua' + $applications['perl'] = $applications['pl'] = 'perl' $applications['python'] = $applications['py'] = 'python' - $applications['lua'] = $applications['lua'] = 'luatex --luaonly' $applications['java'] = $applications['jar'] = 'java' if $mswindows then @@ -1536,6 +1402,8 @@ def set_applications(page=1) $applications['htm'] = $applications['html'] $applications['eps'] = $applications['ps'] + $runners['lua'] = "texlua" + end set_applications() @@ -1675,6 +1543,33 @@ class File end +# def hashed (arr=[]) + # arg = if arr.class == String then arr.split(' ') else arr.dup end + # hsh = Hash.new + # if arg.length > 0 + # hsh['arguments'] = '' + # done = false + # arg.each do |s| + # if done then + # if s =~ / / then + # hsh['arguments'] += " \"#{s}\"" # maybe split on = + # else + # hsh['arguments'] += " #{s}" + # end + # else + # kvl = s.split('=') + # if kvl[0].sub!(/^\-+/,'') then + # hsh[kvl[0]] = if kvl.length > 1 then kvl[1] else true end + # else + # hsh['file'] = s + # done = true + # end + # end + # end + # end + # return hsh +# end + def hashed (arr=[]) arg = if arr.class == String then arr.split(' ') else arr.dup end hsh = Hash.new @@ -1683,7 +1578,18 @@ def hashed (arr=[]) done = false arg.each do |s| if done then - hsh['arguments'] += ' ' + s + if s =~ /\s/ then + kvl = s.split('=') + if kvl[1] and kvl[1] !~ /^[\"\']/ then + hsh['arguments'] += ' ' + kvl[0] + "=" + '"' + kvl[1] + '"' + elsif s =~ /\s/ then + hsh['arguments'] += ' "' + s + '"' + else + hsh['arguments'] += ' ' + s + end + else + hsh['arguments'] += ' ' + s + end else kvl = s.split('=') if kvl[0].sub!(/^\-+/,'') then @@ -1698,6 +1604,7 @@ def hashed (arr=[]) return hsh end + def launch(filename) if $browser && $mswindows then filename = filename.gsub(/\.[\/\\]/) do @@ -1717,15 +1624,25 @@ end # rel|relative # loc|locate|kpse|path|file +def quoted(str) + if str =~ /^\"/ then + return str + elsif str =~ / / then + return "\"#{str}\"" + else + return str + end +end + def expanded(arg) # no "other text files", too restricted arg.gsub(/(env|environment)\:([a-zA-Z\-\_\.0-9]+)/o) do method, original, resolved = $1, $2, '' if resolved = ENV[original] then report("environment variable #{original} expands to #{resolved}") unless $report - resolved + quoted(resolved) else report("environment variable #{original} cannot be resolved") unless $report - original + quoted(original) end end . gsub(/(rel|relative)\:([a-zA-Z\-\_\.0-9]+)/o) do method, original, resolved = $1, $2, '' @@ -1736,9 +1653,9 @@ def expanded(arg) # no "other text files", too restricted end end if resolved.empty? then - original + quoted(original) else - resolved + quoted(resolved) end end . gsub(/(kpse|loc|locate|file|path)\:([a-zA-Z\-\_\.0-9]+)/o) do method, original, resolved = $1, $2, '' @@ -1753,7 +1670,7 @@ def expanded(arg) # no "other text files", too restricted if ENV["_CTX_K_V_#{original}_"] then resolved = ENV["_CTX_K_V_#{original}_"] report("environment provides #{original} as #{resolved}") unless $report - resolved + quoted(resolved) else check_kpse pstrings.each do |pstr| @@ -1789,12 +1706,12 @@ def expanded(arg) # no "other text files", too restricted original = File.dirname(original) if method =~ /path/ report("#{original} is not resolved") unless $report ENV["_CTX_K_V_#{original}_"] = original if $crossover - original + quoted(original) else resolved = File.dirname(resolved) if method =~ /path/ report("#{original} is resolved to #{resolved}") unless $report ENV["_CTX_K_V_#{original}_"] = resolved if $crossover - resolved + quoted(resolved) end end end @@ -1839,10 +1756,16 @@ def runcommand(command) end end +def join_command(args) + args[0] = $runners[args[0]] || args[0] + [args].join(' ') +end + def runoneof(application,fullname,browserpermitted) if browserpermitted && launch(fullname) then return true else + fullname = quoted(fullname) # added because MM ran into problems report("starting #{$filename}") unless $report output("\n") if $report && $verbose applications = $applications[application.downcase] @@ -1851,26 +1774,26 @@ def runoneof(application,fullname,browserpermitted) return true elsif applications.class == Array then if $report then - output([fullname,expanded($arguments)].join(' ')) + output(join_command([fullname,expanded($arguments)])) return true else applications.each do |a| - return true if runcommand([a,fullname,expanded($arguments)].join(' ')) + return true if runcommand(join_command([a,fullname,expanded($arguments)])) end end elsif applications.empty? then if $report then - output([fullname,expanded($arguments)].join(' ')) + output(join_command([fullname,expanded($arguments)])) return true else - return runcommand([fullname,expanded($arguments)].join(' ')) + return runcommand(join_command([fullname,expanded($arguments)])) end else if $report then - output([applications,fullname,expanded($arguments)].join(' ')) + output(join_command([applications,fullname,expanded($arguments)])) return true else - return runcommand([applications,fullname,expanded($arguments)].join(' ')) + return runcommand(join_command([applications,fullname,expanded($arguments)])) end end return false @@ -1920,6 +1843,7 @@ def usage end # somehow registration does not work out (at least not under windows) +# the . is also not accepted by unix as seperator def tag(name) if $crossover then "_CTX_K_S_#{name}_" else "TEXMFSTART.#{name}" end @@ -2410,10 +2334,9 @@ def show_environment end end -def execute(arguments) +def execute(arguments) # br global arguments = arguments.split(/\s+/) if arguments.class == String - $directives = hashed(arguments) $help = $directives['help'] || false @@ -2576,5 +2499,3 @@ else report("\nexecution failed") if $verbose exit(1) end - -# exit (if ($?.to_i rescue 0) > 0 then 1 else 0 end) diff --git a/Master/texmf-dist/scripts/context/ruby/texsync.rb b/Master/texmf-dist/scripts/context/ruby/texsync.rb index ade307d8ef2..00461039982 100644 --- a/Master/texmf-dist/scripts/context/ruby/texsync.rb +++ b/Master/texmf-dist/scripts/context/ruby/texsync.rb @@ -14,6 +14,8 @@ # distribution. In due time I will add a few more options, like # synchronization of the iso image. +# taco's sync: rsync -au -v rsync://www.pragma-ade.com/all ./htdocs + banner = ['TeXSync', 'version 1.1.1', '2002/2004', 'PRAGMA ADE/POD'] $: << File.expand_path(File.dirname($0)) ; $: << File.join($:.last,'lib') ; $:.uniq! @@ -28,7 +30,7 @@ class Commands include CommandBase - @@formats = ['en','nl','de','cz','it','ro'] + @@formats = ['en','nl','de','cz','it','ro', 'fr'] @@always = ['metafun','mptopdf','en','nl'] @@rsync = 'rsync -r -z -c --progress --stats "--exclude=*.fmt" "--exclude=*.efmt" "--exclude=*.mem"' diff --git a/Master/texmf-dist/scripts/context/ruby/textools.rb b/Master/texmf-dist/scripts/context/ruby/textools.rb index b6300215e78..442dc19240e 100644 --- a/Master/texmf-dist/scripts/context/ruby/textools.rb +++ b/Master/texmf-dist/scripts/context/ruby/textools.rb @@ -713,7 +713,7 @@ class Commands f << "\n" f << "/#{encoding.gsub(/[^a-zA-Z]/,'')}encoding [\n" 256.times do |i| - f << " /#{chars[i] || '.notdef'}\n" + f << " /#{chars[i] || '.notdef'} % #{i}\n" end f << "] def\n" f.close @@ -974,6 +974,8 @@ class Commands donepaths.keys.sort.each do |d| pathfiles = Dir.glob("#{path}/#{d}/**/*") pathfiles.each do |p| +# puts(File.dirname(p)) +# if donepaths[File.dirname(p)] then r = File.join(root,d,File.basename(p)) if FileTest.file?(p) and not FileTest.file?(r) and not copiedfiles.key?(File.expand_path(p)) then if delete then @@ -988,6 +990,7 @@ class Commands end end end +# end end end diff --git a/Master/texmf-dist/scripts/context/ruby/www/dir.rb b/Master/texmf-dist/scripts/context/ruby/www/dir.rb index 09e088d772d..115fd8911d9 100644 --- a/Master/texmf-dist/scripts/context/ruby/www/dir.rb +++ b/Master/texmf-dist/scripts/context/ruby/www/dir.rb @@ -62,9 +62,9 @@ class WWW end u = dir_uri(@variables.get('path') || '.') str << "<div class='dir-view'>\n<pre>\n" - str << "<a href=\"#{u}&n=#{d1}\">name</A>".ljust(49+u.length) - str << "<a href=\"#{u}&m=#{d1}\">last modified</A>".ljust(41+u.length) - str << "<a href=\"#{u}&s=#{d1}\">size</A>".rjust(31+u.length) << "\n" << "\n" + str << "<a href=\"#{u}&n=#{d1}\">name</a>".ljust(49+u.length) + str << "<a href=\"#{u}&m=#{d1}\">last modified</a>".ljust(41+u.length) + str << "<a href=\"#{u}&s=#{d1}\">size</a>".rjust(31+u.length) << "\n" << "\n" # parent path if showdirs && ! hidden.include?('..') then dname = "parent directory" diff --git a/Master/texmf-dist/scripts/context/ruby/www/exa.rb b/Master/texmf-dist/scripts/context/ruby/www/exa.rb index 5e9b3fd82e3..20a40fc7b66 100644 --- a/Master/texmf-dist/scripts/context/ruby/www/exa.rb +++ b/Master/texmf-dist/scripts/context/ruby/www/exa.rb @@ -156,32 +156,32 @@ class WWW req << "</exa:request>\n" else # better use rexml but slower - if req =~ /<exa:request[^>]*>.*?\s*<exa:threshold>\s*(.*?)\s*<\/exa:threshold>\s*.*?<\/exa:request>/mois then + if req =~ /<exa:request[^>]*>.*?\s*<exa:threshold>\s*(.*?)\s*<\/exa:threshold>\s*.*?<\/exa:request>/moi then threshold = $1 unless threshold.empty? then @interface.set('process:threshold', threshold) @session.set('threshold', threshold) end end - req.sub!(/(<exa:request[^>]*>.*?)\s*<exa:option>\s*\-\-action\=(.*?)\s*<\/exa:option>\s*(.*?<\/exa:request>)/mois) do + req.sub!(/(<exa:request[^>]*>.*?)\s*<exa:option>\s*\-\-action\=(.*?)\s*<\/exa:option>\s*(.*?<\/exa:request>)/moi) do pre, act, pos = $1, $2, $3 action = act.sub(/\.exa$/,'') if action.empty? str = "#{pre}<exa:action>#{action}</exa:action>#{pos}" - str.sub(/\s*<exa:command>.*?<\/exa:command>\s*/mois ,'') + str.sub(/\s*<exa:command>.*?<\/exa:command>\s*/moi ,'') end - req.sub!(/(<exa:request[^>]*>.*?)<exa:action>\s*(.*?)\s*<\/exa:action>(.*?<\/exa:request>)/mois) do + req.sub!(/(<exa:request[^>]*>.*?)<exa:action>\s*(.*?)\s*<\/exa:action>(.*?<\/exa:request>)/moi) do pre, act, pos = $1, $2, $3 action = act.sub(/\.exa$/,'') if action.empty? str = "#{pre}<exa:action>#{action}</exa:action>#{pos}" - str.sub(/\s*<exa:command>.*?<\/exa:command>\s*/mois ,'') + str.sub(/\s*<exa:command>.*?<\/exa:command>\s*/moi ,'') end - unless req =~ /<exa:data>(.*?)<\/exa:data>/mois then + unless req =~ /<exa:data>(.*?)<\/exa:data>/moi then req.sub!(/(<\/exa:request>)/) do dat + $1 end end end - req.sub!(/<exa:filename>.*?<\/exa:filename>/mois, '') + req.sub!(/<exa:filename>.*?<\/exa:filename>/moi, '') unless @variables.empty?('exa:filename') then - req.sub!(/(<\/exa:application>)/mois) do + req.sub!(/(<\/exa:application>)/moi) do "<exa:filename>#{@variables.get('exa:filename')}<\/exa:filename>" + $1 end end @@ -312,13 +312,13 @@ class WWW end unless req.empty? then # better use rexml but slower / reuse these : command = filter_from_request('exa:command') - if req =~ /<exa:request[^>]*>.*?\s*<exa:command>\s*(.*?)\s*<\/exa:command>\s*.*?<\/exa:request>/mois then + if req =~ /<exa:request[^>]*>.*?\s*<exa:command>\s*(.*?)\s*<\/exa:command>\s*.*?<\/exa:request>/moi then command = $1 end - if req =~ /<exa:request[^>]*>.*?\s*<exa:url>\s*(.*?)\s*<\/exa:url>\s*.*?<\/exa:request>/mois then + if req =~ /<exa:request[^>]*>.*?\s*<exa:url>\s*(.*?)\s*<\/exa:url>\s*.*?<\/exa:request>/moi then url = $1 end - if req =~ /<exa:request[^>]*>.*?\s*<exa:threshold>\s*(.*?)\s*<\/exa:threshold>\s*.*?<\/exa:request>/mois then + if req =~ /<exa:request[^>]*>.*?\s*<exa:threshold>\s*(.*?)\s*<\/exa:threshold>\s*.*?<\/exa:request>/moi then threshold = $1 unless threshold.empty? then @interface.set('process:threshold', threshold) @@ -368,6 +368,7 @@ class WWW end def handle_exastatus + get_cfg() # weird, needed for apache, but not for wwwserver if request_variable('id').empty? then if id = valid_session() then send_result() diff --git a/Master/texmf-dist/scripts/context/ruby/www/lib.rb b/Master/texmf-dist/scripts/context/ruby/www/lib.rb index f5f362b12d0..b9a44c9f61c 100644 --- a/Master/texmf-dist/scripts/context/ruby/www/lib.rb +++ b/Master/texmf-dist/scripts/context/ruby/www/lib.rb @@ -163,7 +163,7 @@ class WWW @interface.set('template:login' , 'exalogin.htm') @interface.set('process:timeout' , @@session_max_age) @interface.set('process:threshold' , @@send_threshold) - @interface.set('process:background', 'yes') # this demands a watchdog being active + @interface.set('process:background', 'yes') # this demands a watchdog being active @interface.set('process:indirect' , 'no') # indirect download, no direct feed @interface.set('process:autologin' , 'yes') # provide default interface when applicable @interface.set('process:exaurl' , '') # this one will be used as replacement in templates @@ -1226,6 +1226,12 @@ class WWW return ! (@session.nothing?('gui') && @session.nothing?('path') && @session.nothing?('process')) end + def get_cfg() + if data = load_interface_file() then + fetch_session_interface_variables(data) + end + end + end class WWW diff --git a/Master/texmf-dist/scripts/context/ruby/wwwserver.rb b/Master/texmf-dist/scripts/context/ruby/wwwserver.rb index 53e1cdc518e..13d5d13125c 100644 --- a/Master/texmf-dist/scripts/context/ruby/wwwserver.rb +++ b/Master/texmf-dist/scripts/context/ruby/wwwserver.rb @@ -1,4 +1,4 @@ -#!/usr/env ruby +#!/usr/bin/env ruby banner = ['WWWServer', 'version 1.0.0', '2003-2006', 'PRAGMA ADE/POD'] diff --git a/Master/texmf-dist/scripts/context/ruby/xmltools.rb b/Master/texmf-dist/scripts/context/ruby/xmltools.rb index 5b4a112b8b5..c28df200dd6 100644 --- a/Master/texmf-dist/scripts/context/ruby/xmltools.rb +++ b/Master/texmf-dist/scripts/context/ruby/xmltools.rb @@ -15,7 +15,7 @@ # This script will harbor some handy manipulations on tex # related files. -banner = ['XMLTools', 'version 1.2.1', '2002/2006', 'PRAGMA ADE/POD'] +banner = ['XMLTools', 'version 1.2.2', '2002/2007', 'PRAGMA ADE/POD'] $: << File.expand_path(File.dirname($0)) ; $: << File.join($:.last,'lib') ; $:.uniq! @@ -466,35 +466,35 @@ class Commands elements = Array.new preamble = "" done = false - data.sub!(/^(.*?)\s*(<[a-z])/mois) do + data.sub!(/^(.*?)\s*(<[a-z])/moi) do preamble = $1 $2 end # hide elements - data.gsub!(/<(.*?)>/mois) do + data.gsub!(/<([^>]*?)>/moi) do elements << $1 "<#{elements.length}>" end # abc[-/]def - data.gsub!(/([a-z]{3,})([\/\-])([a-z]{3,})/mois) do + data.gsub!(/([a-z]{3,})([\/\-\(\)]+)([a-z]{3,})/moi) do done = true report("compound: #{$1}#{$2}#{$3}") if verbose "#{$1}<compound token='#{$2}'/>#{$3}" end # (abcd - data.gsub!(/(\()([a-z]{4,})/mois) do - done = true - report("compound: #{$1}#{$2}") if verbose - "<compound token='#{$1}'/>#{$2}" - end + # data.gsub!(/(\()([a-z]{4,})/moi) do + # done = true + # report("compound: #{$1}#{$2}") if verbose + # "<compound token='#{$1}'/>#{$2}" + # end # abcd) - data.gsub!(/(\()([a-z]{4,})/mois) do - done = true - report("compound: #{$1}#{$2}") if verbose - "#{$2}<compound token='#{$2}'/>" - end + # data.gsub!(/(\()([a-z]{4,})/moi) do + # done = true + # report("compound: #{$1}#{$2}") if verbose + # "#{$2}<compound token='#{$2}'/>" + # end # roll back elements - data.gsub!(/<(\d+)>/mois) do + data.gsub!(/<(\d+)>/moi) do "<#{elements.shift}>" end File.open(newname,'wb') do |f| diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/context.cmd b/Master/texmf-dist/scripts/context/stubs/mswin/context.cmd new file mode 100644 index 00000000000..64ac1ea9147 --- /dev/null +++ b/Master/texmf-dist/scripts/context/stubs/mswin/context.cmd @@ -0,0 +1,5 @@ +@echo off +setlocal +set ownpath=%~dp0% +texlua "%ownpath%mtxrun.lua" --script context %* +endlocal diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/ctxtools.bat b/Master/texmf-dist/scripts/context/stubs/mswin/ctxtools.bat index 13614a1d98d..f1f5e019e38 100755 --- a/Master/texmf-dist/scripts/context/stubs/mswin/ctxtools.bat +++ b/Master/texmf-dist/scripts/context/stubs/mswin/ctxtools.bat @@ -1,2 +1,2 @@ -@echo off
-texmfstart ctxtools.rb %*
+@echo off +texmfstart ctxtools.rb %* diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/exatools.bat b/Master/texmf-dist/scripts/context/stubs/mswin/exatools.bat index 36230a528f4..57f798e8249 100755 --- a/Master/texmf-dist/scripts/context/stubs/mswin/exatools.bat +++ b/Master/texmf-dist/scripts/context/stubs/mswin/exatools.bat @@ -1,2 +1,2 @@ -@echo off
-texmfstart exatools.rb %*
+@echo off +texmfstart exatools.rb %* diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/luatools.cmd b/Master/texmf-dist/scripts/context/stubs/mswin/luatools.cmd new file mode 100644 index 00000000000..4bc998d65e0 --- /dev/null +++ b/Master/texmf-dist/scripts/context/stubs/mswin/luatools.cmd @@ -0,0 +1,5 @@ +@echo off +setlocal +set ownpath=%~dp0% +texlua "%ownpath%luatools.lua" %* +endlocal diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/makempy.bat b/Master/texmf-dist/scripts/context/stubs/mswin/makempy.bat index 9622d96a5f3..e339058c69b 100755 --- a/Master/texmf-dist/scripts/context/stubs/mswin/makempy.bat +++ b/Master/texmf-dist/scripts/context/stubs/mswin/makempy.bat @@ -1,2 +1,2 @@ -@echo off
-texmfstart makempy.pl %*
+@echo off +texmfstart makempy.pl %* diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/mpstools.bat b/Master/texmf-dist/scripts/context/stubs/mswin/mpstools.bat index 10ec3f1d687..df1732e17de 100755 --- a/Master/texmf-dist/scripts/context/stubs/mswin/mpstools.bat +++ b/Master/texmf-dist/scripts/context/stubs/mswin/mpstools.bat @@ -1,2 +1,2 @@ -@echo off
-texmfstart mpstools.rb %*
+@echo off +texmfstart mpstools.rb %* diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/mptopdf.bat b/Master/texmf-dist/scripts/context/stubs/mswin/mptopdf.bat index 757ea58f9ce..242854337ba 100755 --- a/Master/texmf-dist/scripts/context/stubs/mswin/mptopdf.bat +++ b/Master/texmf-dist/scripts/context/stubs/mswin/mptopdf.bat @@ -1,2 +1,2 @@ -@echo off
-texmfstart mptopdf.pl %*
+@echo off +texmfstart mptopdf.pl %* diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/mtxrun.cmd b/Master/texmf-dist/scripts/context/stubs/mswin/mtxrun.cmd new file mode 100644 index 00000000000..f30148ddbd0 --- /dev/null +++ b/Master/texmf-dist/scripts/context/stubs/mswin/mtxrun.cmd @@ -0,0 +1,5 @@ +@echo off +setlocal +set ownpath=%~dp0% +texlua "%ownpath%mtxrun.lua" %* +endlocal diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/pdftools.bat b/Master/texmf-dist/scripts/context/stubs/mswin/pdftools.bat index 71662d3f766..adc48eacf4b 100755 --- a/Master/texmf-dist/scripts/context/stubs/mswin/pdftools.bat +++ b/Master/texmf-dist/scripts/context/stubs/mswin/pdftools.bat @@ -1,2 +1,2 @@ -@echo off
-texmfstart pdftools.rb %*
+@echo off +texmfstart pdftools.rb %* diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/pdftrimwhite.bat b/Master/texmf-dist/scripts/context/stubs/mswin/pdftrimwhite.bat new file mode 100644 index 00000000000..a7034b4000b --- /dev/null +++ b/Master/texmf-dist/scripts/context/stubs/mswin/pdftrimwhite.bat @@ -0,0 +1,2 @@ +@echo off +texmfstart pdftrimwhite.pl %* diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/pstopdf.bat b/Master/texmf-dist/scripts/context/stubs/mswin/pstopdf.bat index 091b0d4d7bd..248e34caf5d 100755 --- a/Master/texmf-dist/scripts/context/stubs/mswin/pstopdf.bat +++ b/Master/texmf-dist/scripts/context/stubs/mswin/pstopdf.bat @@ -1,2 +1,2 @@ -@echo off
-texmfstart pstopdf.rb %*
+@echo off +texmfstart pstopdf.rb %* diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/rlxtools.bat b/Master/texmf-dist/scripts/context/stubs/mswin/rlxtools.bat index e5cdca25009..b78dec13b16 100755 --- a/Master/texmf-dist/scripts/context/stubs/mswin/rlxtools.bat +++ b/Master/texmf-dist/scripts/context/stubs/mswin/rlxtools.bat @@ -1,2 +1,2 @@ -@echo off
-texmfstart rlxtools.rb %*
+@echo off +texmfstart rlxtools.rb %* diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/runtools.bat b/Master/texmf-dist/scripts/context/stubs/mswin/runtools.bat index 37df398ab37..68a7b4f9745 100755 --- a/Master/texmf-dist/scripts/context/stubs/mswin/runtools.bat +++ b/Master/texmf-dist/scripts/context/stubs/mswin/runtools.bat @@ -1,2 +1,2 @@ -@echo off
-texmfstart runtools.rb %*
+@echo off +texmfstart runtools.rb %* diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/texexec.bat b/Master/texmf-dist/scripts/context/stubs/mswin/texexec.bat index 44615dbd918..02b297b9cdb 100755 --- a/Master/texmf-dist/scripts/context/stubs/mswin/texexec.bat +++ b/Master/texmf-dist/scripts/context/stubs/mswin/texexec.bat @@ -1,2 +1,2 @@ -@echo off
-texmfstart texexec.rb %*
+@echo off +texmfstart texexec.rb %* diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/texfind.bat b/Master/texmf-dist/scripts/context/stubs/mswin/texfind.bat new file mode 100644 index 00000000000..b7c11cbcaad --- /dev/null +++ b/Master/texmf-dist/scripts/context/stubs/mswin/texfind.bat @@ -0,0 +1,2 @@ +@echo off +texmfstart texfind %* diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/texfont.bat b/Master/texmf-dist/scripts/context/stubs/mswin/texfont.bat index 9dd50309ff3..3134bf14c2f 100755 --- a/Master/texmf-dist/scripts/context/stubs/mswin/texfont.bat +++ b/Master/texmf-dist/scripts/context/stubs/mswin/texfont.bat @@ -1,2 +1,2 @@ -@echo off
-texmfstart texfont.pl %*
+@echo off +texmfstart texfont.pl %* diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/texshow.bat b/Master/texmf-dist/scripts/context/stubs/mswin/texshow.bat new file mode 100644 index 00000000000..2060846ada0 --- /dev/null +++ b/Master/texmf-dist/scripts/context/stubs/mswin/texshow.bat @@ -0,0 +1,2 @@ +@echo off +texmfstart texshow.pl %* diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/textools.bat b/Master/texmf-dist/scripts/context/stubs/mswin/textools.bat index 4fc5c549255..727b4a36da4 100755 --- a/Master/texmf-dist/scripts/context/stubs/mswin/textools.bat +++ b/Master/texmf-dist/scripts/context/stubs/mswin/textools.bat @@ -1,2 +1,2 @@ -@echo off
-texmfstart textools.rb %*
+@echo off +texmfstart textools.rb %* diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/texutil.bat b/Master/texmf-dist/scripts/context/stubs/mswin/texutil.bat index 61e90ca6509..1e63639bb08 100755 --- a/Master/texmf-dist/scripts/context/stubs/mswin/texutil.bat +++ b/Master/texmf-dist/scripts/context/stubs/mswin/texutil.bat @@ -1,2 +1,2 @@ -@echo off
-texmfstart texutil.rb %*
+@echo off +texmfstart texutil.rb %* diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/tmftools.bat b/Master/texmf-dist/scripts/context/stubs/mswin/tmftools.bat index a7053043d22..c9c0c08bd90 100755 --- a/Master/texmf-dist/scripts/context/stubs/mswin/tmftools.bat +++ b/Master/texmf-dist/scripts/context/stubs/mswin/tmftools.bat @@ -1,2 +1,2 @@ -@echo off
-texmfstart tmftools.rb %*
+@echo off +texmfstart tmftools.rb %* diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/xmltools.bat b/Master/texmf-dist/scripts/context/stubs/mswin/xmltools.bat index 9721095b298..2de0e445709 100755 --- a/Master/texmf-dist/scripts/context/stubs/mswin/xmltools.bat +++ b/Master/texmf-dist/scripts/context/stubs/mswin/xmltools.bat @@ -1,2 +1,2 @@ -@echo off
-texmfstart xmltools.rb %*
+@echo off +texmfstart xmltools.rb %* diff --git a/Master/texmf-dist/scripts/context/stubs/unix/context b/Master/texmf-dist/scripts/context/stubs/unix/context new file mode 100755 index 00000000000..c7341904f10 --- /dev/null +++ b/Master/texmf-dist/scripts/context/stubs/unix/context @@ -0,0 +1,3 @@ +#!/bin/sh + +mtxrun --script context "$@" diff --git a/Master/texmf-dist/scripts/context/stubs/unix/luatools b/Master/texmf-dist/scripts/context/stubs/unix/luatools index cb3ec1add4a..89e5e0eb4db 100644 --- a/Master/texmf-dist/scripts/context/stubs/unix/luatools +++ b/Master/texmf-dist/scripts/context/stubs/unix/luatools @@ -1,2 +1,6656 @@ -#!/bin/sh -texmfstart luatools.lua "$@" +#!/usr/bin/env texlua + +-- one can make a stub: +-- +-- #!/bin/sh +-- env LUATEXDIR=/....../texmf/scripts/context/lua luatex --luaonly=luatools.lua "$@" +-- filename : luatools.lua +-- comment : companion to context.tex +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files +-- Although this script is part of the ConTeXt distribution it is +-- relatively indepent of ConTeXt. The same is true for some of +-- the luat files. We may may make them even less dependent in +-- the future. As long as Luatex is under development the +-- interfaces and names of functions may change. + +banner = "version 1.2.0 - 2006+ - PRAGMA ADE / CONTEXT" +texlua = true + +-- For the sake of independence we optionally can merge the library +-- code here. It's too much code, but that does not harm. Much of the +-- library code is used elsewhere. We don't want dependencies on +-- Lua library paths simply because these scripts are located in the +-- texmf tree and not in some Lua path. Normally this merge is not +-- needed when texmfstart is used, or when the proper stub is used or +-- when (windows) suffix binding is active. + +-- begin library merge +-- filename : l-string.lua +-- comment : split off from luat-lib +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-string'] = 1.001 + +--~ function string.split(str, pat) -- taken from the lua wiki +--~ local t = {n = 0} -- so this table has a length field, traverse with ipairs then! +--~ local fpat = "(.-)"..pat +--~ local last_end = 1 +--~ local s, e, cap = string.find(str, fpat, 1) +--~ while s ~= nil do +--~ if s~=1 or cap~="" then +--~ table.insert(t,cap) +--~ end +--~ last_end = e+1 +--~ s, e, cap = string.find(str, fpat, last_end) +--~ end +--~ if last_end<=string.len(str) then +--~ table.insert(t,(string.sub(str,last_end))) +--~ end +--~ return t +--~ end + +--~ function string:split(pat) -- taken from the lua wiki but adapted +--~ local t = { } -- self and colon usage (faster) +--~ local fpat = "(.-)"..pat +--~ local last_end = 1 +--~ local s, e, cap = self:find(fpat, 1) +--~ while s ~= nil do +--~ if s~=1 or cap~="" then +--~ t[#t+1] = cap +--~ end +--~ last_end = e+1 +--~ s, e, cap = self:find(fpat, last_end) +--~ end +--~ if last_end <= #self then +--~ t[#t+1] = self:sub(last_end) +--~ end +--~ return t +--~ end + +--~ a piece of brilliant code by Rici Lake (posted on lua list) -- only names changed +--~ +--~ function string:splitter(pat) +--~ local st, g = 1, self:gmatch("()"..pat.."()") +--~ local function splitter(self) +--~ if st then +--~ local s, f = g() +--~ local rv = self:sub(st, (s or 0)-1) +--~ st = f +--~ return rv +--~ end +--~ end +--~ return splitter, self +--~ end + +function string:splitter(pat) + -- by Rici Lake (posted on lua list) -- only names changed + -- p 79 ref man: () returns position of match + local st, g = 1, self:gmatch("()("..pat..")") + local function strgetter(self, segs, seps, sep, cap1, ...) + st = sep and seps + #sep + return self:sub(segs, (seps or 0) - 1), cap1 or sep, ... + end + local function strsplitter(self) + if st then return strgetter(self, st, g()) end + end + return strsplitter, self +end + +function string:split(separator) + local t = {} + for k in self:splitter(separator) do t[#t+1] = k end + return t +end + +-- faster than a string:split: + +function string:splitchr(chr) + if #self > 0 then + local t = { } + for s in string.gmatch(self..chr,"(.-)"..chr) do + t[#t+1] = s + end + return t + else + return { } + end +end + +--~ function string.piecewise(str, pat, fnc) -- variant of split +--~ local fpat = "(.-)"..pat +--~ local last_end = 1 +--~ local s, e, cap = string.find(str, fpat, 1) +--~ while s ~= nil do +--~ if s~=1 or cap~="" then +--~ fnc(cap) +--~ end +--~ last_end = e+1 +--~ s, e, cap = string.find(str, fpat, last_end) +--~ end +--~ if last_end <= #str then +--~ fnc((string.sub(str,last_end))) +--~ end +--~ end + +function string.piecewise(str, pat, fnc) -- variant of split + for k in string.splitter(str,pat) do fnc(k) end +end + +--~ function string.piecewise(str, pat, fnc) -- variant of split +--~ for k in str:splitter(pat) do fnc(k) end +--~ end + +--~ do if lpeg then + +--~ -- this alternative is 30% faster esp when we cache them +--~ -- problem: no expressions + +--~ splitters = { } + +--~ function string:split(separator) +--~ if #self > 0 then +--~ local split = splitters[separator] +--~ if not split then +--~ -- based on code by Roberto +--~ local p = lpeg.P(separator) +--~ local c = lpeg.C((1-p)^0) +--~ split = lpeg.Ct(c*(p*c)^0) +--~ splitters[separator] = split +--~ end +--~ return split:match(self) +--~ else +--~ return { } +--~ end +--~ end + +--~ string.splitchr = string.split + +--~ function string:piecewise(separator,fnc) +--~ for _,v in pairs(self:split(separator)) do +--~ fnc(v) +--~ end +--~ end + +--~ end end + +string.chr_to_esc = { + ["%"] = "%%", + ["."] = "%.", + ["+"] = "%+", ["-"] = "%-", ["*"] = "%*", + ["^"] = "%^", ["$"] = "%$", + ["["] = "%[", ["]"] = "%]", + ["("] = "%(", [")"] = "%)", + ["{"] = "%{", ["}"] = "%}" +} + +function string:esc() -- variant 2 + return (self:gsub("(.)",string.chr_to_esc)) +end + +function string.unquote(str) + return (str:gsub("^([\"\'])(.*)%1$","%2")) +end + +function string.quote(str) + return '"' .. str:unquote() .. '"' +end + +function string:count(pattern) -- variant 3 + local n = 0 + for _ in self:gmatch(pattern) do + n = n + 1 + end + return n +end + +function string:limit(n,sentinel) + if #self > n then + sentinel = sentinel or " ..." + return self:sub(1,(n-#sentinel)) .. sentinel + else + return self + end +end + +function string:strip() + return (self:gsub("^%s*(.-)%s*$", "%1")) +end + +--~ function string.strip(str) -- slightly different +--~ return (string.gsub(string.gsub(str,"^%s*(.-)%s*$","%1"),"%s+"," ")) +--~ end + +function string:is_empty() + return not self:find("%S") +end + +function string:enhance(pattern,action) + local ok, n = true, 0 + while ok do + ok = false + self = self:gsub(pattern, function(...) + ok, n = true, n + 1 + return action(...) + end) + end + return self, n +end + +--~ function string:enhance(pattern,action) +--~ local ok, n = 0, 0 +--~ repeat +--~ self, ok = self:gsub(pattern, function(...) +--~ n = n + 1 +--~ return action(...) +--~ end) +--~ until ok == 0 +--~ return self, n +--~ end + +--~ function string:to_hex() +--~ if self then +--~ return (self:gsub("(.)",function(c) +--~ return string.format("%02X",c:byte()) +--~ end)) +--~ else +--~ return "" +--~ end +--~ end + +--~ function string:from_hex() +--~ if self then +--~ return (self:gsub("(..)",function(c) +--~ return string.char(tonumber(c,16)) +--~ end)) +--~ else +--~ return "" +--~ end +--~ end + +string.chr_to_hex = { } +string.hex_to_chr = { } + +for i=0,255 do + local c, h = string.char(i), string.format("%02X",i) + string.chr_to_hex[c], string.hex_to_chr[h] = h, c +end + +--~ function string:to_hex() +--~ if self then return (self:gsub("(.)",string.chr_to_hex)) else return "" end +--~ end + +--~ function string:from_hex() +--~ if self then return (self:gsub("(..)",string.hex_to_chr)) else return "" end +--~ end + +function string:to_hex() + return ((self or ""):gsub("(.)",string.chr_to_hex)) +end + +function string:from_hex() + return ((self or ""):gsub("(..)",string.hex_to_chr)) +end + +if not string.characters then + + local function nextchar(str, index) + index = index + 1 + return (index <= #str) and index or nil, str:sub(index,index) + end + function string:characters() + return nextchar, self, 0 + end + local function nextbyte(str, index) + index = index + 1 + return (index <= #str) and index or nil, string.byte(str:sub(index,index)) + end + function string:bytes() + return nextbyte, self, 0 + end + +end + +--~ function string:padd(n,chr) +--~ return self .. self.rep(chr or " ",n-#self) +--~ end + +function string:rpadd(n,chr) + local m = n-#self + if m > 0 then + return self .. self.rep(chr or " ",m) + else + return self + end +end + +function string:lpadd(n,chr) + local m = n-#self + if m > 0 then + return self.rep(chr or " ",m) .. self + else + return self + end +end + +string.padd = string.rpadd + +function is_number(str) + return str:find("^[%-%+]?[%d]-%.?[%d+]$") == 1 +end + +--~ print(is_number("1")) +--~ print(is_number("1.1")) +--~ print(is_number(".1")) +--~ print(is_number("-0.1")) +--~ print(is_number("+0.1")) +--~ print(is_number("-.1")) +--~ print(is_number("+.1")) + +function string:split_settings() -- no {} handling, see l-aux for lpeg variant + if self:find("=") then + local t = { } + for k,v in self:gmatch("(%a+)=([^%,]*)") do + t[k] = v + end + return t + else + return nil + end +end + +local patterns_escapes = { + ["-"] = "%-", + ["."] = "%.", + ["+"] = "%+", + ["*"] = "%*", + ["%"] = "%%", + ["("] = "%)", + [")"] = "%)", + ["["] = "%[", + ["]"] = "%]", +} + +function string:pattesc() + return (self:gsub(".",patterns_escapes)) +end + +function string:tohash() + local t = { } + for s in self:gmatch("([^, ]+)") do -- lpeg + t[s] = true + end + return t +end + + +-- filename : l-lpeg.lua +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-lpeg'] = 1.001 + +--~ l-lpeg.lua : + +--~ lpeg.digit = lpeg.R('09')^1 +--~ lpeg.sign = lpeg.S('+-')^1 +--~ lpeg.cardinal = lpeg.P(lpeg.sign^0 * lpeg.digit^1) +--~ lpeg.integer = lpeg.P(lpeg.sign^0 * lpeg.digit^1) +--~ lpeg.float = lpeg.P(lpeg.sign^0 * lpeg.digit^0 * lpeg.P('.') * lpeg.digit^1) +--~ lpeg.number = lpeg.float + lpeg.integer +--~ lpeg.oct = lpeg.P("0") * lpeg.R('07')^1 +--~ lpeg.hex = lpeg.P("0x") * (lpeg.R('09') + lpeg.R('AF'))^1 +--~ lpeg.uppercase = lpeg.P("AZ") +--~ lpeg.lowercase = lpeg.P("az") + +--~ lpeg.eol = lpeg.S('\r\n\f')^1 -- includes formfeed +--~ lpeg.space = lpeg.S(' ')^1 +--~ lpeg.nonspace = lpeg.P(1-lpeg.space)^1 +--~ lpeg.whitespace = lpeg.S(' \r\n\f\t')^1 +--~ lpeg.nonwhitespace = lpeg.P(1-lpeg.whitespace)^1 + +local hash = { } + +function lpeg.anywhere(pattern) --slightly adapted from website + return lpeg.P { lpeg.P(pattern) + 1 * lpeg.V(1) } +end + +function lpeg.startswith(pattern) --slightly adapted + return lpeg.P(pattern) +end + +--~ g = lpeg.splitter(" ",function(s) ... end) -- gmatch:lpeg = 3:2 + +function lpeg.splitter(pattern, action) + return (((1-lpeg.P(pattern))^1)/action+1)^0 +end + +local crlf = lpeg.P("\r\n") +local cr = lpeg.P("\r") +local lf = lpeg.P("\n") +local space = lpeg.S(" \t\f\v") +local newline = crlf + cr + lf +local spacing = space^0 * newline + +local empty = spacing * lpeg.Cc("") +local nonempty = lpeg.Cs((1-spacing)^1) * spacing^-1 +local content = (empty + nonempty)^1 + +local capture = lpeg.Ct(content^0) + +function string:splitlines() + return capture:match(self) +end + + +-- filename : l-table.lua +-- comment : split off from luat-lib +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-table'] = 1.001 + +table.join = table.concat + +function table.strip(tab) + local lst = { } + for k, v in ipairs(tab) do + -- s = string.gsub(v, "^%s*(.-)%s*$", "%1") + s = v:gsub("^%s*(.-)%s*$", "%1") + if s == "" then + -- skip this one + else + lst[#lst+1] = s + end + end + return lst +end + +--~ function table.sortedkeys(tab) +--~ local srt = { } +--~ for key,_ in pairs(tab) do +--~ srt[#srt+1] = key +--~ end +--~ table.sort(srt) +--~ return srt +--~ end + +function table.sortedkeys(tab) + local srt, kind = { }, 0 -- 0=unknown 1=string, 2=number 3=mixed + for key,_ in pairs(tab) do + srt[#srt+1] = key + if kind == 3 then + -- no further check + else + local tkey = type(key) + if tkey == "string" then + -- if kind == 2 then kind = 3 else kind = 1 end + kind = (kind == 2 and 3) or 1 + elseif tkey == "number" then + -- if kind == 1 then kind = 3 else kind = 2 end + kind = (kind == 1 and 3) or 2 + else + kind = 3 + end + end + end + if kind == 0 or kind == 3 then + table.sort(srt,function(a,b) return (tostring(a) < tostring(b)) end) + else + table.sort(srt) + end + return srt +end + +function table.append(t, list) + for _,v in pairs(list) do + table.insert(t,v) + end +end + +function table.prepend(t, list) + for k,v in pairs(list) do + table.insert(t,k,v) + end +end + +function table.merge(t, ...) -- first one is target + t = t or {} + local lst = {...} + for i=1,#lst do + for k, v in pairs(lst[i]) do + t[k] = v + end + end + return t +end + +function table.merged(...) + local tmp, lst = { }, {...} + for i=1,#lst do + for k, v in pairs(lst[i]) do + tmp[k] = v + end + end + return tmp +end + +function table.imerge(t, ...) + local lst = {...} + for i=1,#lst do + local nst = lst[i] + for j=1,#nst do + t[#t+1] = nst[j] + end + end + return t +end + +function table.imerged(...) + local tmp, lst = { }, {...} + for i=1,#lst do + local nst = lst[i] + for j=1,#nst do + tmp[#tmp+1] = nst[j] + end + end + return tmp +end + +if not table.fastcopy then do + + local type, pairs, getmetatable, setmetatable = type, pairs, getmetatable, setmetatable + + local function fastcopy(old) -- fast one + if old then + local new = { } + for k,v in pairs(old) do + if type(v) == "table" then + new[k] = fastcopy(v) -- was just table.copy + else + new[k] = v + end + end + local mt = getmetatable(old) + if mt then + setmetatable(new,mt) + end + return new + else + return { } + end + end + + table.fastcopy = fastcopy + +end end + +if not table.copy then do + + local type, pairs, getmetatable, setmetatable = type, pairs, getmetatable, setmetatable + + local function copy(t, tables) -- taken from lua wiki, slightly adapted + tables = tables or { } + local tcopy = {} + if not tables[t] then + tables[t] = tcopy + end + for i,v in pairs(t) do -- brrr, what happens with sparse indexed + if type(i) == "table" then + if tables[i] then + i = tables[i] + else + i = copy(i, tables) + end + end + if type(v) ~= "table" then + tcopy[i] = v + elseif tables[v] then + tcopy[i] = tables[v] + else + tcopy[i] = copy(v, tables) + end + end + local mt = getmetatable(t) + if mt then + setmetatable(tcopy,mt) + end + return tcopy + end + + table.copy = copy + +end end + +-- rougly: copy-loop : unpack : sub == 0.9 : 0.4 : 0.45 (so in critical apps, use unpack) + +function table.sub(t,i,j) + return { unpack(t,i,j) } +end + +function table.replace(a,b) + for k,v in pairs(b) do + a[k] = v + end +end + +-- slower than #t on indexed tables (#t only returns the size of the numerically indexed slice) + +function table.is_empty(t) + return not t or not next(t) +end + +function table.one_entry(t) + local n = next(t) + return n and not next(t,n) +end + +function table.starts_at(t) + return ipairs(t,1)(t,0) +end + +do + + -- one of my first exercises in lua ... + + -- 34.055.092 32.403.326 arabtype.tma + -- 1.620.614 1.513.863 lmroman10-italic.tma + -- 1.325.585 1.233.044 lmroman10-regular.tma + -- 1.248.157 1.158.903 lmsans10-regular.tma + -- 194.646 153.120 lmtypewriter10-regular.tma + -- 1.771.678 1.658.461 palatinosanscom-bold.tma + -- 1.695.251 1.584.491 palatinosanscom-regular.tma + -- 13.736.534 13.409.446 zapfinoextraltpro.tma + + -- 13.679.038 11.774.106 arabtype.tmc + -- 886.248 754.944 lmroman10-italic.tmc + -- 729.828 466.864 lmroman10-regular.tmc + -- 688.482 441.962 lmsans10-regular.tmc + -- 128.685 95.853 lmtypewriter10-regular.tmc + -- 715.929 582.985 palatinosanscom-bold.tmc + -- 669.942 540.126 palatinosanscom-regular.tmc + -- 1.560.588 1.317.000 zapfinoextraltpro.tmc + + table.serialize_functions = true + table.serialize_compact = true + table.serialize_inline = true + + local function key(k) + if type(k) == "number" then -- or k:find("^%d+$") then + return "["..k.."]" + elseif noquotes and k:find("^%a[%a%d%_]*$") then + return k + else + return '["'..k..'"]' + end + end + + local function simple_table(t) + if #t > 0 then + local n = 0 + for _,v in pairs(t) do + n = n + 1 + end + if n == #t then + local tt = { } + for i=1,#t do + local v = t[i] + local tv = type(v) + if tv == "number" or tv == "boolean" then + tt[#tt+1] = tostring(v) + elseif tv == "string" then + tt[#tt+1] = ("%q"):format(v) + else + tt = nil + break + end + end + return tt + end + end + return nil + end + + local function serialize(root,name,handle,depth,level,reduce,noquotes,indexed) + handle = handle or print + reduce = reduce or false + if depth then + depth = depth .. " " + if indexed then + handle(("%s{"):format(depth)) + else + handle(("%s%s={"):format(depth,key(name))) + end + else + depth = "" + local tname = type(name) + if tname == "string" then + if name == "return" then + handle("return {") + else + handle(name .. "={") + end + elseif tname == "number" then + handle("[" .. name .. "]={") + elseif tname == "boolean" then + if name then + handle("return {") + else + handle("{") + end + else + handle("t={") + end + end + if root and next(root) then + local compact = table.serialize_compact + local inline = compact and table.serialize_inline + local first, last = nil, 0 -- #root cannot be trusted here + if compact then + for k,v in ipairs(root) do -- NOT: for k=1,#root do (why) + if not first then first = k end + last = last + 1 + end + end + for _,k in pairs(table.sortedkeys(root)) do + local v = root[k] + local t = type(v) + if compact and first and type(k) == "number" and k >= first and k <= last then + if t == "number" then + handle(("%s %s,"):format(depth,v)) + elseif t == "string" then + if reduce and (v:find("^[%-%+]?[%d]-%.?[%d+]$") == 1) then + handle(("%s %s,"):format(depth,v)) + else + handle(("%s %q,"):format(depth,v)) + end + elseif t == "table" then + if not next(v) then + handle(("%s {},"):format(depth)) + elseif inline then + local st = simple_table(v) + if st then + handle(("%s { %s },"):format(depth,table.concat(st,", "))) + else + serialize(v,k,handle,depth,level+1,reduce,noquotes,true) + end + else + serialize(v,k,handle,depth,level+1,reduce,noquotes,true) + end + elseif t == "boolean" then + handle(("%s %s,"):format(depth,tostring(v))) + elseif t == "function" then + if table.serialize_functions then + handle(('%s loadstring(%q),'):format(depth,string.dump(v))) + else + handle(('%s "function",'):format(depth)) + end + else + handle(("%s %q,"):format(depth,tostring(v))) + end + elseif k == "__p__" then -- parent + if false then + handle(("%s __p__=nil,"):format(depth)) + end + elseif t == "number" then + handle(("%s %s=%s,"):format(depth,key(k),v)) + elseif t == "string" then + if reduce and (v:find("^[%-%+]?[%d]-%.?[%d+]$") == 1) then + handle(("%s %s=%s,"):format(depth,key(k),v)) + else + handle(("%s %s=%q,"):format(depth,key(k),v)) + end + elseif t == "table" then + if not next(v) then + handle(("%s %s={},"):format(depth,key(k))) + elseif inline then + local st = simple_table(v) + if st then + handle(("%s %s={ %s },"):format(depth,key(k),table.concat(st,", "))) + else + serialize(v,k,handle,depth,level+1,reduce,noquotes) + end + else + serialize(v,k,handle,depth,level+1,reduce,noquotes) + end + elseif t == "boolean" then + handle(("%s %s=%s,"):format(depth,key(k),tostring(v))) + elseif t == "function" then + if table.serialize_functions then + handle(('%s %s=loadstring(%q),'):format(depth,key(k),string.dump(v))) + else + handle(('%s %s="function",'):format(depth,key(k))) + end + else + handle(("%s %s=%q,"):format(depth,key(k),tostring(v))) + -- handle(('%s %s=loadstring(%q),'):format(depth,key(k),string.dump(function() return v end))) + end + end + if level > 0 then + handle(("%s},"):format(depth)) + else + handle(("%s}"):format(depth)) + end + else + handle(("%s}"):format(depth)) + end + end + + --~ name: + --~ + --~ true : return { } + --~ false : { } + --~ nil : t = { } + --~ string : string = { } + --~ 'return' : return { } + --~ number : [number] = { } + + function table.serialize(root,name,reduce,noquotes) + local t = { } + local function flush(s) + t[#t+1] = s + end + serialize(root, name, flush, nil, 0, reduce, noquotes) + return table.concat(t,"\n") + end + + function table.tohandle(handle,root,name,reduce,noquotes) + serialize(root, name, handle, nil, 0, reduce, noquotes) + end + + -- sometimes tables are real use (zapfino extra pro is some 85M) in which + -- case a stepwise serialization is nice; actually, we could consider: + -- + -- for line in table.serializer(root,name,reduce,noquotes) do + -- ...(line) + -- end + -- + -- so this is on the todo list + + table.tofile_maxtab = 2*1024 + + function table.tofile(filename,root,name,reduce,noquotes) + local f = io.open(filename,'w') + if f then + local concat = table.concat + local maxtab = table.tofile_maxtab + if maxtab > 1 then + local t = { } + local function flush(s) + t[#t+1] = s + if #t > maxtab then + f:write(concat(t,"\n"),"\n") -- hm, write(sometable) should be nice + t = { } + end + end + serialize(root, name, flush, nil, 0, reduce, noquotes) + f:write(concat(t,"\n"),"\n") + else + local function flush(s) + f:write(s,"\n") + end + serialize(root, name, flush, nil, 0, reduce, noquotes) + end + f:close() + end + end + +end + +--~ t = { +--~ b = "123", +--~ a = "x", +--~ c = 1.23, +--~ d = "1.23", +--~ e = true, +--~ f = { +--~ d = "1.23", +--~ a = "x", +--~ b = "123", +--~ c = 1.23, +--~ e = true, +--~ f = { +--~ e = true, +--~ f = { +--~ e = true +--~ }, +--~ }, +--~ }, +--~ g = function() end +--~ } + +--~ print(table.serialize(t), "\n") +--~ print(table.serialize(t,"name"), "\n") +--~ print(table.serialize(t,false), "\n") +--~ print(table.serialize(t,true), "\n") +--~ print(table.serialize(t,"name",true), "\n") +--~ print(table.serialize(t,"name",true,true), "\n") + +do + + local function flatten(t,f,complete) + for i=1,#t do + local v = t[i] + if type(v) == "table" then + if complete or type(v[1]) == "table" then + flatten(v,f,complete) + else + f[#f+1] = v + end + else + f[#f+1] = v + end + end + end + + function table.flatten(t) + local f = { } + flatten(t,f,true) + return f + end + + function table.unnest(t) -- bad name + local f = { } + flatten(t,f,false) + return f + end + + table.flatten_one_level = table.unnest + +end + +function table.insert_before_value(t,value,str) + for i=1,#t do + if t[i] == value then + table.insert(t,i,str) + return + end + end + table.insert(t,1,str) +end + +function table.insert_after_value(t,value,str) + for i=1,#t do + if t[i] == value then + table.insert(t,i+1,str) + return + end + end + t[#t+1] = str +end + +function table.are_equal(a,b,n,m) + if #a == #b then + n = n or 1 + m = m or #a + for i=n,m do + local ai, bi = a[i], b[i] + if (ai==bi) or (type(ai)=="table" and type(bi)=="table" and table.are_equal(ai,bi)) then + -- continue + else + return false + end + end + return true + else + return false + end +end + +function table.compact(t) + if t then + for k,v in pairs(t) do + if not next(v) then + t[k] = nil + end + end + end +end + +function table.tohash(t) + local h = { } + for _, v in pairs(t) do -- no ipairs here + h[v] = true + end + return h +end + +function table.fromhash(t) + local h = { } + for k, v in pairs(t) do -- no ipairs here + if v then h[#h+1] = k end + end + return h +end + +function table.contains(t, v) + if t then + for i=1, #t do + if t[i] == v then + return true + end + end + end + return false +end + +function table.count(t) + local n, e = 0, next(t) + while e do + n, e = n + 1, next(t,e) + end + return n +end + +function table.swapped(t) + local s = { } + for k, v in pairs(t) do + s[v] = k + end + return s +end + +--~ function table.are_equal(a,b) +--~ return table.serialize(a) == table.serialize(b) +--~ end + +function table.clone(t,p) -- t is optional or nil or table + if not p then + t, p = { }, t or { } + elseif not t then + t = { } + end + setmetatable(t, { __index = function(_,key) return p[key] end }) + return t +end + + +function table.hexed(t,seperator) + local tt = { } + for i=1,#t do tt[i] = string.format("0x%04X",t[i]) end + return table.concat(tt,seperator or " ") +end + +function table.reverse_hash(h) + local r = { } + for k,v in pairs(h) do + r[v] = (k:gsub(" ","")):lower() + end + return r +end + +function table.reverse(t) + local tt = { } + if #t > 0 then + for i=#t,1,-1 do + tt[#tt+1] = t[i] + end + end + return tt +end + + +-- filename : l-io.lua +-- comment : split off from luat-lib +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-io'] = 1.001 + +if string.find(os.getenv("PATH"),";") then + io.fileseparator, io.pathseparator = "\\", ";" +else + io.fileseparator, io.pathseparator = "/" , ":" +end + +function io.loaddata(filename) + local f = io.open(filename,'rb') + if f then + local data = f:read('*all') + f:close() + return data + else + return nil + end +end + +function io.savedata(filename,data,joiner) + local f = io.open(filename, "wb") + if f then + if type(data) == "table" then + f:write(table.join(data,joiner or "")) + elseif type(data) == "function" then + data(f) + else + f:write(data) + end + f:close() + end +end + +function io.exists(filename) + local f = io.open(filename) + if f == nil then + return false + else + assert(f:close()) + return true + end +end + +function io.size(filename) + local f = io.open(filename) + if f == nil then + return 0 + else + local s = f:seek("end") + assert(f:close()) + return s + end +end + +function io.noflines(f) + local n = 0 + for _ in f:lines() do + n = n + 1 + end + f:seek('set',0) + return n +end + +do + + local sb = string.byte + + local nextchar = { + [ 4] = function(f) + return f:read(1,1,1,1) + end, + [ 2] = function(f) + return f:read(1,1) + end, + [ 1] = function(f) + return f:read(1) + end, + [-2] = function(f) + local a, b = f:read(1,1) + return b, a + end, + [-4] = function(f) + local a, b, c, d = f:read(1,1,1,1) + return d, c, b, a + end + } + + function io.characters(f,n) + if f then + return nextchar[n or 1], f + else + return nil, nil + end + end + +end + +do + + local sb = string.byte + +--~ local nextbyte = { +--~ [4] = function(f) +--~ local a = f:read(1) +--~ local b = f:read(1) +--~ local c = f:read(1) +--~ local d = f:read(1) +--~ if d then +--~ return sb(a), sb(b), sb(c), sb(d) +--~ else +--~ return nil, nil, nil, nil +--~ end +--~ end, +--~ [2] = function(f) +--~ local a = f:read(1) +--~ local b = f:read(1) +--~ if b then +--~ return sb(a), sb(b) +--~ else +--~ return nil, nil +--~ end +--~ end, +--~ [1] = function (f) +--~ local a = f:read(1) +--~ if a then +--~ return sb(a) +--~ else +--~ return nil +--~ end +--~ end, +--~ [-2] = function (f) +--~ local a = f:read(1) +--~ local b = f:read(1) +--~ if b then +--~ return sb(b), sb(a) +--~ else +--~ return nil, nil +--~ end +--~ end, +--~ [-4] = function(f) +--~ local a = f:read(1) +--~ local b = f:read(1) +--~ local c = f:read(1) +--~ local d = f:read(1) +--~ if d then +--~ return sb(d), sb(c), sb(b), sb(a) +--~ else +--~ return nil, nil, nil, nil +--~ end +--~ end +--~ } + + local nextbyte = { + [4] = function(f) + local a, b, c, d = f:read(1,1,1,1) + if d then + return sb(a), sb(b), sb(c), sb(d) + else + return nil, nil, nil, nil + end + end, + [2] = function(f) + local a, b = f:read(1,1) + if b then + return sb(a), sb(b) + else + return nil, nil + end + end, + [1] = function (f) + local a = f:read(1) + if a then + return sb(a) + else + return nil + end + end, + [-2] = function (f) + local a, b = f:read(1,1) + if b then + return sb(b), sb(a) + else + return nil, nil + end + end, + [-4] = function(f) + local a, b, c, d = f:read(1,1,1,1) + if d then + return sb(d), sb(c), sb(b), sb(a) + else + return nil, nil, nil, nil + end + end + } + + function io.bytes(f,n) + if f then + return nextbyte[n or 1], f + else + return nil, nil + end + end + +end + +function io.ask(question,default,options) + while true do + io.write(question) + if options then + io.write(string.format(" [%s]",table.concat(options,"|"))) + end + if default then + io.write(string.format(" [%s]",default)) + end + io.write(string.format(" ")) + local answer = io.read() + answer = answer:gsub("^%s*(.*)%s*$","%1") + if answer == "" and default then + return default + elseif not options then + return answer + else + for _,v in pairs(options) do + if v == answer then + return answer + end + end + local pattern = "^" .. answer + for _,v in pairs(options) do + if v:find(pattern) then + return v + end + end + end + end +end + + +-- filename : l-number.lua +-- comment : split off from luat-lib +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-number'] = 1.001 + +if not number then number = { } end + +-- a,b,c,d,e,f = number.toset(100101) + +function number.toset(n) + return (tostring(n)):match("(.?)(.?)(.?)(.?)(.?)(.?)(.?)(.?)") +end + +local format = string.format + +function number.toevenhex(n) + local s = format("%X",n) + if #s % 2 == 0 then + return s + else + return "0" .. s + end +end + +-- the lpeg way is slower on 8 digits, but faster on 4 digits, some 7.5% +-- on +-- +-- for i=1,1000000 do +-- local a,b,c,d,e,f,g,h = number.toset(12345678) +-- local a,b,c,d = number.toset(1234) +-- local a,b,c = number.toset(123) +-- end +-- +-- of course dedicated "(.)(.)(.)(.)" matches are even faster + +do + local one = lpeg.C(1-lpeg.S(''))^1 + + function number.toset(n) + return one:match(tostring(n)) + end +end + + + +-- filename : l-set.lua +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-set'] = 1.001 + +if not set then set = { } end + +do + + local nums = { } + local tabs = { } + local concat = table.concat + + set.create = table.tohash + + function set.tonumber(t) + if next(t) then + local s = "" + -- we could save mem by sorting, but it slows down + for k, v in pairs(t) do + if v then + -- why bother about the leading space + s = s .. " " .. k + end + end + if not nums[s] then + tabs[#tabs+1] = t + nums[s] = #tabs + end + return nums[s] + else + return 0 + end + end + + function set.totable(n) + if n == 0 then + return { } + else + return tabs[n] or { } + end + end + + function set.contains(n,s) + if type(n) == "table" then + return n[s] + elseif n == 0 then + return false + else + local t = tabs[n] + return t and t[s] + end + end + +end + +--~ local c = set.create{'aap','noot','mies'} +--~ local s = set.tonumber(c) +--~ local t = set.totable(s) +--~ print(t['aap']) +--~ local c = set.create{'zus','wim','jet'} +--~ local s = set.tonumber(c) +--~ local t = set.totable(s) +--~ print(t['aap']) +--~ print(t['jet']) +--~ print(set.contains(t,'jet')) +--~ print(set.contains(t,'aap')) + + + +-- filename : l-os.lua +-- comment : split off from luat-lib +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-os'] = 1.001 + +function os.resultof(command) + return io.popen(command,"r"):read("*all") +end + +if not os.exec then os.exec = os.execute end +if not os.spawn then os.spawn = os.execute end + +--~ os.type : windows | unix (new, we already guessed os.platform) +--~ os.name : windows | msdos | linux | macosx | solaris | .. | generic (new) + +if not io.fileseparator then + if string.find(os.getenv("PATH"),";") then + io.fileseparator, io.pathseparator, os.platform = "\\", ";", os.type or "windows" + else + io.fileseparator, io.pathseparator, os.platform = "/" , ":", os.type or "unix" + end +end + +os.platform = os.platform or os.type or (io.pathseparator == ";" and "windows") or "unix" + +function os.launch(str) + if os.platform == "windows" then + os.execute("start " .. str) -- os.spawn ? + else + os.execute(str .. " &") -- os.spawn ? + end +end + +if not os.setenv then + function os.setenv() return false end +end + +if not os.times then + -- utime = user time + -- stime = system time + -- cutime = children user time + -- cstime = children system time + function os.times() + return { + utime = os.gettimeofday(), -- user + stime = 0, -- system + cutime = 0, -- children user + cstime = 0, -- children system + } + end +end + +os.gettimeofday = os.gettimeofday or os.clock + +do + local startuptime = os.gettimeofday() + function os.runtime() + return os.gettimeofday() - startuptime + end +end + +--~ print(os.gettimeofday()-os.time()) +--~ os.sleep(1.234) +--~ print (">>",os.runtime()) +--~ print(os.date("%H:%M:%S",os.gettimeofday())) +--~ print(os.date("%H:%M:%S",os.time())) + + +-- filename : l-md5.lua +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-md5'] = 1.001 + +if md5 then do + + local function convert(str,fmt) + return (string.gsub(md5.sum(str),".",function(chr) return string.format(fmt,string.byte(chr)) end)) + end + + if not md5.HEX then function md5.HEX(str) return convert(str,"%02X") end end + if not md5.hex then function md5.hex(str) return convert(str,"%02x") end end + if not md5.dec then function md5.dec(str) return convert(str,"%03i") end end + +end end + + +-- filename : l-file.lua +-- comment : split off from luat-lib +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-file'] = 1.001 + +if not file then file = { } end + +function file.removesuffix(filename) + return filename:gsub("%.[%a%d]+$", "") +end + +function file.addsuffix(filename, suffix) + if not filename:find("%.[%a%d]+$") then + return filename .. "." .. suffix + else + return filename + end +end + +function file.replacesuffix(filename, suffix) + if not filename:find("%.[%a%d]+$") then + return filename .. "." .. suffix + else + return (filename:gsub("%.[%a%d]+$","."..suffix)) + end +end + +function file.dirname(name) + return name:match("^(.+)[/\\].-$") or "" +end + +function file.basename(name) + return name:match("^.+[/\\](.-)$") or name +end + +function file.nameonly(name) + return ((name:match("^.+[/\\](.-)$") or name):gsub("%..*$","")) +end + +function file.extname(name) + return name:match("^.+%.([^/\\]-)$") or "" +end + +file.suffix = file.extname + +function file.stripsuffix(name) + return (name:gsub("%.[%a%d]+$","")) +end + +--~ function file.join(...) +--~ local t = { ... } +--~ for i=1,#t do +--~ t[i] = (t[i]:gsub("\\","/")):gsub("/+$","") +--~ end +--~ return table.concat(t,"/") +--~ end + +--~ print(file.join("x/","/y")) +--~ print(file.join("http://","/y")) +--~ print(file.join("http://a","/y")) +--~ print(file.join("http:///a","/y")) +--~ print(file.join("//nas-1","/y")) + +function file.join(...) + local pth = table.concat({...},"/") + pth = pth:gsub("\\","/") + local a, b = pth:match("^(.*://)(.*)$") + if a and b then + return a .. b:gsub("//+","/") + end + a, b = pth:match("^(//)(.*)$") + if a and b then + return a .. b:gsub("//+","/") + end + return (pth:gsub("//+","/")) +end + +function file.is_writable(name) + local f = io.open(name, 'w') + if f then + f:close() + return true + else + return false + end +end + +function file.is_readable(name) + local f = io.open(name,'r') + if f then + f:close() + return true + else + return false + end +end + +--~ function file.split_path(str) +--~ if str:find(';') then +--~ return str:splitchr(";") +--~ else +--~ return str:splitchr(io.pathseparator) +--~ end +--~ end + +-- todo: lpeg + +function file.split_path(str) + local t = { } + str = str:gsub("\\", "/") + str = str:gsub("(%a):([;/])", "%1\001%2") + for name in str:gmatch("([^;:]+)") do + if name ~= "" then + name = name:gsub("\001",":") + t[#t+1] = name + end + end + return t +end + +function file.join_path(tab) + return table.concat(tab,io.pathseparator) -- can have trailing // +end + +--~ print('test' .. " == " .. file.collapse_path("test")) +--~ print("test/test" .. " == " .. file.collapse_path("test/test")) +--~ print("test/test/test" .. " == " .. file.collapse_path("test/test/test")) +--~ print("test/test" .. " == " .. file.collapse_path("test/../test/test")) +--~ print("test" .. " == " .. file.collapse_path("test/../test")) +--~ print("../test" .. " == " .. file.collapse_path("../test")) +--~ print("../test/" .. " == " .. file.collapse_path("../test/")) +--~ print("a/a" .. " == " .. file.collapse_path("a/b/c/../../a")) + +--~ function file.collapse_path(str) +--~ local ok, n = false, 0 +--~ while not ok do +--~ ok = true +--~ str, n = str:gsub("[^%./]+/%.%./", function(s) +--~ ok = false +--~ return "" +--~ end) +--~ end +--~ return (str:gsub("/%./","/")) +--~ end + +function file.collapse_path(str) + local n = 1 + while n > 0 do + str, n = str:gsub("([^/%.]+/%.%./)","") + end + return (str:gsub("/%./","/")) +end + +function file.robustname(str) + return (str:gsub("[^%a%d%/%-%.\\]+","-")) +end + +file.readdata = io.loaddata +file.savedata = io.savedata + +function file.copy(oldname,newname) + file.savedata(newname,io.loaddata(oldname)) +end + + +-- filename : l-url.lua +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-url'] = 1.001 +if not url then url = { } end + +-- from the spec (on the web): +-- +-- foo://example.com:8042/over/there?name=ferret#nose +-- \_/ \______________/\_________/ \_________/ \__/ +-- | | | | | +-- scheme authority path query fragment +-- | _____________________|__ +-- / \ / \ +-- urn:example:animal:ferret:nose + +do + + local function tochar(s) + return string.char(tonumber(s,16)) + end + + local colon, qmark, hash, slash, percent, endofstring = lpeg.P(":"), lpeg.P("?"), lpeg.P("#"), lpeg.P("/"), lpeg.P("%"), lpeg.P(-1) + + local hexdigit = lpeg.R("09","AF","af") + local escaped = percent * lpeg.C(hexdigit * hexdigit) / tochar + + local scheme = lpeg.Cs((escaped+(1-colon-slash-qmark-hash))^0) * colon + lpeg.Cc("") + local authority = slash * slash * lpeg.Cs((escaped+(1- slash-qmark-hash))^0) + lpeg.Cc("") + local path = slash * lpeg.Cs((escaped+(1- qmark-hash))^0) + lpeg.Cc("") + local query = qmark * lpeg.Cs((escaped+(1- hash))^0) + lpeg.Cc("") + local fragment = hash * lpeg.Cs((escaped+(1- endofstring))^0) + lpeg.Cc("") + + local parser = lpeg.Ct(scheme * authority * path * query * fragment) + + function url.split(str) + return (type(str) == "string" and parser:match(str)) or str + end + +end + +function url.hashed(str) + local s = url.split(str) + return { + scheme = (s[1] ~= "" and s[1]) or "file", + authority = s[2], + path = s[3], + query = s[4], + fragment = s[5], + original = str + } +end + +function url.filename(filename) + local t = url.hashed(filename) + return (t.scheme == "file" and t.path:gsub("^/([a-zA-Z])([:|])/)","%1:")) or filename +end + +function url.query(str) + if type(str) == "string" then + local t = { } + for k, v in str:gmatch("([^&=]*)=([^&=]*)") do + t[k] = v + end + return t + else + return str + end +end + +--~ print(url.filename("file:///c:/oeps.txt")) +--~ print(url.filename("c:/oeps.txt")) +--~ print(url.filename("file:///oeps.txt")) +--~ print(url.filename("file:///etc/test.txt")) +--~ print(url.filename("/oeps.txt")) + +-- from the spec on the web (sort of): +--~ +--~ function test(str) +--~ print(table.serialize(url.hashed(str))) +--~ end +---~ +--~ test("%56pass%20words") +--~ test("file:///c:/oeps.txt") +--~ test("file:///c|/oeps.txt") +--~ test("file:///etc/oeps.txt") +--~ test("file://./etc/oeps.txt") +--~ test("file:////etc/oeps.txt") +--~ test("ftp://ftp.is.co.za/rfc/rfc1808.txt") +--~ test("http://www.ietf.org/rfc/rfc2396.txt") +--~ test("ldap://[2001:db8::7]/c=GB?objectClass?one#what") +--~ test("mailto:John.Doe@example.com") +--~ test("news:comp.infosystems.www.servers.unix") +--~ test("tel:+1-816-555-1212") +--~ test("telnet://192.0.2.16:80/") +--~ test("urn:oasis:names:specification:docbook:dtd:xml:4.1.2") +--~ test("/etc/passwords") +--~ test("http://www.pragma-ade.com/spaced%20name") + +--~ test("zip:///oeps/oeps.zip#bla/bla.tex") +--~ test("zip:///oeps/oeps.zip?bla/bla.tex") + + +-- filename : l-dir.lua +-- comment : split off from luat-lib +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-dir'] = 1.001 + +dir = { } + +-- optimizing for no string.find (*) does not save time + +if lfs then do + +--~ local attributes = lfs.attributes +--~ local walkdir = lfs.dir +--~ +--~ local function glob_pattern(path,patt,recurse,action) +--~ local ok, scanner = xpcall(function() return walkdir(path) end, function() end) -- kepler safe +--~ if ok and type(scanner) == "function" then +--~ if not path:find("/$") then path = path .. '/' end +--~ for name in scanner do +--~ local full = path .. name +--~ local mode = attributes(full,'mode') +--~ if mode == 'file' then +--~ if name:find(patt) then +--~ action(full) +--~ end +--~ elseif recurse and (mode == "directory") and (name ~= '.') and (name ~= "..") then +--~ glob_pattern(full,patt,recurse,action) +--~ end +--~ end +--~ end +--~ end +--~ +--~ dir.glob_pattern = glob_pattern +--~ +--~ local function glob(pattern, action) +--~ local t = { } +--~ local action = action or function(name) t[#t+1] = name end +--~ local path, patt = pattern:match("^(.*)/*%*%*/*(.-)$") +--~ local recurse = path and patt +--~ if not recurse then +--~ path, patt = pattern:match("^(.*)/(.-)$") +--~ if not (path and patt) then +--~ path, patt = '.', pattern +--~ end +--~ end +--~ patt = patt:gsub("([%.%-%+])", "%%%1") +--~ patt = patt:gsub("%*", ".*") +--~ patt = patt:gsub("%?", ".") +--~ patt = "^" .. patt .. "$" +--~ -- print('path: ' .. path .. ' | pattern: ' .. patt .. ' | recurse: ' .. tostring(recurse)) +--~ glob_pattern(path,patt,recurse,action) +--~ return t +--~ end +--~ +--~ dir.glob = glob + + local attributes = lfs.attributes + local walkdir = lfs.dir + + local function glob_pattern(path,patt,recurse,action) + local ok, scanner + if path == "/" then + ok, scanner = xpcall(function() return walkdir(path..".") end, function() end) -- kepler safe + else + ok, scanner = xpcall(function() return walkdir(path) end, function() end) -- kepler safe + end + if ok and type(scanner) == "function" then + if not path:find("/$") then path = path .. '/' end + for name in scanner do + local full = path .. name + local mode = attributes(full,'mode') + if mode == 'file' then + if full:find(patt) then + action(full) + end + elseif recurse and (mode == "directory") and (name ~= '.') and (name ~= "..") then + glob_pattern(full,patt,recurse,action) + end + end + end + end + + dir.glob_pattern = glob_pattern + + --~ local function glob(pattern, action) + --~ local t = { } + --~ local path, rest, patt, recurse + --~ local action = action or function(name) t[#t+1] = name end + --~ local pattern = pattern:gsub("^%*%*","./**") + --~ local pattern = pattern:gsub("/%*/","/**/") + --~ path, rest = pattern:match("^(/)(.-)$") + --~ if path then + --~ path = path + --~ else + --~ path, rest = pattern:match("^([^/]*)/(.-)$") + --~ end + --~ if rest then + --~ patt = rest:gsub("([%.%-%+])", "%%%1") + --~ end + --~ patt = patt:gsub("%*", "[^/]*") + --~ patt = patt:gsub("%?", "[^/]") + --~ patt = patt:gsub("%[%^/%]%*%[%^/%]%*", ".*") + --~ if path == "" then path = "." end + --~ recurse = patt:find("%.%*/") ~= nil + --~ glob_pattern(path,patt,recurse,action) + --~ return t + --~ end + + local P, S, R, C, Cc, Cs, Ct, Cv, V = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.Cc, lpeg.Cs, lpeg.Ct, lpeg.Cv, lpeg.V + + local pattern = Ct { + [1] = (C(P(".") + P("/")^1) + C(R("az","AZ") * P(":") * P("/")^0) + Cc("./")) * V(2) * V(3), + [2] = C(((1-S("*?/"))^0 * P("/"))^0), + [3] = C(P(1)^0) + } + + local filter = Cs ( ( + P("**") / ".*" + + P("*") / "[^/]*" + + P("?") / "[^/]" + + P(".") / "%%." + + P("+") / "%%+" + + P("-") / "%%-" + + P(1) + )^0 ) + + local function glob(str,t) + if type(str) == "table" then + local t = t or { } + for _, s in ipairs(str) do + glob(s,t) + end + return t + else + local split = pattern:match(str) + if split then + local t = t or { } + local action = action or function(name) t[#t+1] = name end + local root, path, base = split[1], split[2], split[3] + local recurse = base:find("**") + local start = root .. path + local result = filter:match(start .. base) + glob_pattern(start,result,recurse,action) + return t + else + return { } + end + end + end + + dir.glob = glob + + --~ list = dir.glob("**/*.tif") + --~ list = dir.glob("/**/*.tif") + --~ list = dir.glob("./**/*.tif") + --~ list = dir.glob("oeps/**/*.tif") + --~ list = dir.glob("/oeps/**/*.tif") + + local function globfiles(path,recurse,func,files) -- func == pattern or function + if type(func) == "string" then + local s = func -- alas, we need this indirect way + func = function(name) return name:find(s) end + end + files = files or { } + for name in walkdir(path) do + if name:find("^%.") then + --- skip + elseif attributes(name,'mode') == "directory" then + if recurse then + globfiles(path .. "/" .. name,recurse,func,files) + end + elseif func then + if func(name) then + files[#files+1] = path .. "/" .. name + end + else + files[#files+1] = path .. "/" .. name + end + end + return files + end + + dir.globfiles = globfiles + + -- t = dir.glob("c:/data/develop/context/sources/**/????-*.tex") + -- t = dir.glob("c:/data/develop/tex/texmf/**/*.tex") + -- t = dir.glob("c:/data/develop/context/texmf/**/*.tex") + -- t = dir.glob("f:/minimal/tex/**/*") + -- print(dir.ls("f:/minimal/tex/**/*")) + -- print(dir.ls("*.tex")) + + function dir.ls(pattern) + return table.concat(glob(pattern),"\n") + end + + --~ mkdirs("temp") + --~ mkdirs("a/b/c") + --~ mkdirs(".","/a/b/c") + --~ mkdirs("a","b","c") + + local make_indeed = true -- false + + if string.find(os.getenv("PATH"),";") then + + function dir.mkdirs(...) + local str, pth = "", "" + for _, s in ipairs({...}) do + if s ~= "" then + if str ~= "" then + str = str .. "/" .. s + else + str = s + end + end + end + local first, middle, last + local drive = false + first, middle, last = str:match("^(//)(//*)(.*)$") + if first then + -- empty network path == local path + else + first, last = str:match("^(//)/*(.-)$") + if first then + middle, last = str:match("([^/]+)/+(.-)$") + if middle then + pth = "//" .. middle + else + pth = "//" .. last + last = "" + end + else + first, middle, last = str:match("^([a-zA-Z]:)(/*)(.-)$") + if first then + pth, drive = first .. middle, true + else + middle, last = str:match("^(/*)(.-)$") + if not middle then + last = str + end + end + end + end + for s in last:gmatch("[^/]+") do + if pth == "" then + pth = s + elseif drive then + pth, drive = pth .. s, false + else + pth = pth .. "/" .. s + end + if make_indeed and not lfs.isdir(pth) then + lfs.mkdir(pth) + end + end + return pth, (lfs.isdir(pth) == true) + end + +--~ print(dir.mkdirs("","","a","c")) +--~ print(dir.mkdirs("a")) +--~ print(dir.mkdirs("a:")) +--~ print(dir.mkdirs("a:/b/c")) +--~ print(dir.mkdirs("a:b/c")) +--~ print(dir.mkdirs("a:/bbb/c")) +--~ print(dir.mkdirs("/a/b/c")) +--~ print(dir.mkdirs("/aaa/b/c")) +--~ print(dir.mkdirs("//a/b/c")) +--~ print(dir.mkdirs("///a/b/c")) +--~ print(dir.mkdirs("a/bbb//ccc/")) + + function dir.expand_name(str) + local first, nothing, last = str:match("^(//)(//*)(.*)$") + if first then + first = lfs.currentdir() .. "/" + first = first:gsub("\\","/") + end + if not first then + first, last = str:match("^(//)/*(.*)$") + end + if not first then + first, last = str:match("^([a-zA-Z]:)(.*)$") + if first and not last:find("^/") then + local d = lfs.currentdir() + if lfs.chdir(first) then + first = lfs.currentdir() + first = first:gsub("\\","/") + end + lfs.chdir(d) + end + end + if not first then + first, last = lfs.currentdir(), str + first = first:gsub("\\","/") + end + last = last:gsub("//","/") + last = last:gsub("/%./","/") + last = last:gsub("^/*","") + first = first:gsub("/*$","") + if last == "" then + return first + else + return first .. "/" .. last + end + end + + else + + function dir.mkdirs(...) + local str, pth = "", "" + for _, s in ipairs({...}) do + if s ~= "" then + if str ~= "" then + str = str .. "/" .. s + else + str = s + end + end + end + str = str:gsub("/+","/") + if str:find("^/") then + pth = "/" + for s in str:gmatch("[^/]+") do + local first = (pth == "/") + if first then + pth = pth .. s + else + pth = pth .. "/" .. s + end + if make_indeed and not first and not lfs.isdir(pth) then + lfs.mkdir(pth) + end + end + else + pth = "." + for s in str:gmatch("[^/]+") do + pth = pth .. "/" .. s + if make_indeed and not lfs.isdir(pth) then + lfs.mkdir(pth) + end + end + end + return pth, (lfs.isdir(pth) == true) + end + +--~ print(dir.mkdirs("","","a","c")) +--~ print(dir.mkdirs("a")) +--~ print(dir.mkdirs("/a/b/c")) +--~ print(dir.mkdirs("/aaa/b/c")) +--~ print(dir.mkdirs("//a/b/c")) +--~ print(dir.mkdirs("///a/b/c")) +--~ print(dir.mkdirs("a/bbb//ccc/")) + + function dir.expand_name(str) + if not str:find("^/") then + str = lfs.currentdir() .. "/" .. str + end + str = str:gsub("//","/") + str = str:gsub("/%./","/") + return str + end + + end + + dir.makedirs = dir.mkdirs + +end end + + +-- filename : l-boolean.lua +-- comment : split off from luat-lib +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-boolean'] = 1.001 +if not boolean then boolean = { } end + +function boolean.tonumber(b) + if b then return 1 else return 0 end +end + +function toboolean(str,tolerant) + if tolerant then + local tstr = type(str) + if tstr == "string" then + return str == "true" or str == "yes" or str == "on" or str == "1" + elseif tstr == "number" then + return tonumber(str) ~= 0 + elseif tstr == "nil" then + return false + else + return str + end + elseif str == "true" then + return true + elseif str == "false" then + return false + else + return str + end +end + +function string.is_boolean(str) + if type(str) == "string" then + if str == "true" or str == "yes" or str == "on" then + return true + elseif str == "false" or str == "no" or str == "off" then + return false + end + end + return nil +end + +function boolean.alwaystrue() + return true +end + +function boolean.falsetrue() + return false +end + + +-- filename : l-unicode.lua +-- comment : split off from luat-inp +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-unicode'] = 1.001 +if not unicode then unicode = { } end + +if not garbagecollector then + garbagecollector = { + push = function() collectgarbage("stop") end, + pop = function() collectgarbage("restart") end, + } +end + +-- 0 EF BB BF UTF-8 +-- 1 FF FE UTF-16-little-endian +-- 2 FE FF UTF-16-big-endian +-- 3 FF FE 00 00 UTF-32-little-endian +-- 4 00 00 FE FF UTF-32-big-endian + +unicode.utfname = { + [0] = 'utf-8', + [1] = 'utf-16-le', + [2] = 'utf-16-be', + [3] = 'utf-32-le', + [4] = 'utf-32-be' +} + +function unicode.utftype(f) -- \000 fails ! + local str = f:read(4) + if not str then + f:seek('set') + return 0 + elseif str:find("^%z%z\254\255") then + return 4 + elseif str:find("^\255\254%z%z") then + return 3 + elseif str:find("^\254\255") then + f:seek('set',2) + return 2 + elseif str:find("^\255\254") then + f:seek('set',2) + return 1 + elseif str:find("^\239\187\191") then + f:seek('set',3) + return 0 + else + f:seek('set') + return 0 + end +end + +function unicode.utf16_to_utf8(str, endian) + garbagecollector.push() + local result = { } + local tc, uc = table.concat, unicode.utf8.char + local tmp, n, m, p = { }, 0, 0, 0 + -- lf | cr | crlf / (cr:13, lf:10) + local function doit() + if n == 10 then + if p ~= 13 then + result[#result+1] = tc(tmp,"") + tmp = { } + p = 0 + end + elseif n == 13 then + result[#result+1] = tc(tmp,"") + tmp = { } + p = n + else + tmp[#tmp+1] = uc(n) + p = 0 + end + end + for l,r in str:bytepairs() do + if endian then + n = l*256 + r + else + n = r*256 + l + end + if m > 0 then + n = (m-0xD800)*0x400 + (n-0xDC00) + 0x10000 + m = 0 + doit() + elseif n >= 0xD800 and n <= 0xDBFF then + m = n + else + doit() + end + end + if #tmp > 0 then + result[#result+1] = tc(tmp,"") + end + garbagecollector.pop() + return result +end + +function unicode.utf32_to_utf8(str, endian) + garbagecollector.push() + local result = { } + local tc, uc = table.concat, unicode.utf8.char + local tmp, n, m, p = { }, 0, -1, 0 + -- lf | cr | crlf / (cr:13, lf:10) + local function doit() + if n == 10 then + if p ~= 13 then + result[#result+1] = tc(tmp,"") + tmp = { } + p = 0 + end + elseif n == 13 then + result[#result+1] = tc(tmp,"") + tmp = { } + p = n + else + tmp[#tmp+1] = uc(n) + p = 0 + end + end + for a,b in str:bytepairs() do + if a and b then + if m < 0 then + if endian then + m = a*256*256*256 + b*256*256 + else + m = b*256 + a + end + else + if endian then + n = m + a*256 + b + else + n = m + b*256*256*256 + a*256*256 + end + m = -1 + doit() + end + else + break + end + end + if #tmp > 0 then + result[#result+1] = tc(tmp,"") + end + garbagecollector.pop() + return result +end + + +-- filename : l-utils.lua +-- comment : split off from luat-lib +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-utils'] = 1.001 + +if not utils then utils = { } end +if not utils.merger then utils.merger = { } end +if not utils.lua then utils.lua = { } end + +utils.merger.m_begin = "begin library merge" +utils.merger.m_end = "end library merge" +utils.merger.pattern = + "%c+" .. + "%-%-%s+" .. utils.merger.m_begin .. + "%c+(.-)%c+" .. + "%-%-%s+" .. utils.merger.m_end .. + "%c+" + +function utils.merger._self_fake_() + return + "-- " .. "created merged file" .. "\n\n" .. + "-- " .. utils.merger.m_begin .. "\n\n" .. + "-- " .. utils.merger.m_end .. "\n\n" +end + +function utils.report(...) + print(...) +end + +function utils.merger._self_load_(name) + local f, data = io.open(name), "" + if f then + data = f:read("*all") + f:close() + end + return data or "" +end + +function utils.merger._self_save_(name, data) + if data ~= "" then + local f = io.open(name,'w') + if f then + f:write(data) + f:close() + end + end +end + +function utils.merger._self_swap_(data,code) + if data ~= "" then + return (data:gsub(utils.merger.pattern, function(s) + return "\n\n" .. "-- "..utils.merger.m_begin .. "\n" .. code .. "\n" .. "-- "..utils.merger.m_end .. "\n\n" + end, 1)) + else + return "" + end +end + +function utils.merger._self_libs_(libs,list) + local result, f = { }, nil + if type(libs) == 'string' then libs = { libs } end + if type(list) == 'string' then list = { list } end + for _, lib in ipairs(libs) do + for _, pth in ipairs(list) do + local name = string.gsub(pth .. "/" .. lib,"\\","/") + f = io.open(name) + if f then + -- utils.report("merging library",name) + result[#result+1] = f:read("*all") + f:close() + list = { pth } -- speed up the search + break + else + -- utils.report("no library",name) + end + end + end + return table.concat(result, "\n\n") +end + +function utils.merger.selfcreate(libs,list,target) + if target then + utils.merger._self_save_( + target, + utils.merger._self_swap_( + utils.merger._self_fake_(), + utils.merger._self_libs_(libs,list) + ) + ) + end +end + +function utils.merger.selfmerge(name,libs,list,target) + utils.merger._self_save_( + target or name, + utils.merger._self_swap_( + utils.merger._self_load_(name), + utils.merger._self_libs_(libs,list) + ) + ) +end + +function utils.merger.selfclean(name) + utils.merger._self_save_( + name, + utils.merger._self_swap_( + utils.merger._self_load_(name), + "" + ) + ) +end + +utils.lua.compile_strip = true + +function utils.lua.compile(luafile, lucfile) + -- utils.report("compiling",luafile,"into",lucfile) + os.remove(lucfile) + local command = "-o " .. string.quote(lucfile) .. " " .. string.quote(luafile) + if utils.lua.compile_strip then + command = "-s " .. command + end + if os.spawn("texluac " .. command) == 0 then + return true + elseif os.spawn("luac " .. command) == 0 then + return true + else + return false + end +end + + + +-- filename : luat-lib.lua +-- comment : companion to luat-lib.tex +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['luat-lib'] = 1.001 + +-- mostcode moved to the l-*.lua and other luat-*.lua files + +-- os / io + +os.setlocale(nil,nil) -- useless feature and even dangerous in luatex + +-- os.platform + +-- mswin|bccwin|mingw|cygwin windows +-- darwin|rhapsody|nextstep macosx +-- netbsd|unix unix +-- linux linux + +if not io.fileseparator then + if string.find(os.getenv("PATH"),";") then + io.fileseparator, io.pathseparator, os.platform = "\\", ";", os.type or "windows" + else + io.fileseparator, io.pathseparator, os.platform = "/" , ":", os.type or "unix" + end +end + +os.platform = os.platform or os.type or (io.pathseparator == ";" and "windows") or "unix" + +-- arg normalization +-- +-- for k,v in pairs(arg) do print(k,v) end + +-- environment + +if not environment then environment = { } end + +environment.ownbin = environment.ownbin or arg[-2] or arg[-1] or arg[0] or "luatex" + +local ownpath = nil -- we could use a metatable here + +function environment.ownpath() + if not ownpath then + for p in string.gmatch(os.getenv("PATH"),"[^"..io.pathseparator.."]+") do + local b = file.join(p,environment.ownbin) + if lfs.isfile(b..".exe") or lfs.isfile(b) then + ownpath = p + break + end + end + if not ownpath then ownpath = '.' end + end + return ownpath +end + +if arg and (arg[0] == 'luatex' or arg[0] == 'luatex.exe') and arg[1] == "--luaonly" then + arg[-1]=arg[0] arg[0]=arg[2] for k=3,#arg do arg[k-2]=arg[k] end arg[#arg]=nil arg[#arg]=nil +end + +environment.arguments = { } +environment.files = { } +environment.sorted_argument_keys = nil + +environment.platform = os.platform + +function environment.initialize_arguments(arg) + environment.arguments = { } + environment.files = { } + environment.sorted_argument_keys = nil + for index, argument in pairs(arg) do + if index > 0 then + local flag, value = argument:match("^%-+(.+)=(.-)$") + if flag then + environment.arguments[flag] = string.unquote(value or "") + else + flag = argument:match("^%-+(.+)") + if flag then + environment.arguments[flag] = true + else + environment.files[#environment.files+1] = argument + end + end + end + end + environment.ownname = environment.ownname or arg[0] or 'unknown.lua' +end + +function environment.showarguments() + for k,v in pairs(environment.arguments) do + print(k .. " : " .. tostring(v)) + end + if #environment.files > 0 then + print("files : " .. table.concat(environment.files, " ")) + end +end + +function environment.setargument(name,value) + environment.arguments[name] = value +end + +function environment.argument(name) + if environment.arguments[name] then + return environment.arguments[name] + else + if not environment.sorted_argument_keys then + environment.sorted_argument_keys = { } + for _,v in pairs(table.sortedkeys(environment.arguments)) do + table.insert(environment.sorted_argument_keys, "^" .. v) + end + end + for _,v in pairs(environment.sorted_argument_keys) do + if name:find(v) then + return environment.arguments[v:sub(2,#v)] + end + end + end + return nil +end + +function environment.split_arguments(separator) -- rather special, cut-off before separator + local done, before, after = false, { }, { } + for _,v in ipairs(environment.original_arguments) do + if not done and v == separator then + done = true + elseif done then + after[#after+1] = v + else + before[#before+1] = v + end + end + return before, after +end + +function environment.reconstruct_commandline(arg) + if not arg then arg = environment.original_arguments end + local result = { } + for _,a in ipairs(arg) do -- ipairs 1 .. #n + local kk, vv = a:match("^(%-+.-)=(.+)$") + if kk and vv then + if vv:find(" ") then + result[#result+1] = kk .. "=" .. string.quote(vv) + else + result[#result+1] = a + end + elseif a:find(" ") then + result[#result+1] = string.quote(a) + else + result[#result+1] = a + end + end + return table.join(result," ") +end + +if arg then + environment.initialize_arguments(arg) + environment.original_arguments = arg + arg = { } -- prevent duplicate handling +end + + +-- filename : luat-inp.lua +-- comment : companion to luat-lib.tex +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +-- This lib is multi-purpose and can be loaded again later on so that +-- additional functionality becomes available. We will split this +-- module in components when we're done with prototyping. + +-- TODO: os.getenv -> os.env[] +-- TODO: instances.[hashes,cnffiles,configurations,522] -> ipairs (alles check, sneller) +-- TODO: check escaping in find etc, too much, too slow + +-- This is the first code I wrote for LuaTeX, so it needs some cleanup. + +-- To be considered: hash key lowercase, first entry in table filename +-- (any case), rest paths (so no need for optimization). Or maybe a +-- separate table that matches lowercase names to mixed case when +-- present. In that case the lower() cases can go away. I will do that +-- only when we run into problems with names ... well ... Iwona-Regular. + +-- Beware, loading and saving is overloaded in luat-tmp! + +if not versions then versions = { } end versions['luat-inp'] = 1.001 +if not environment then environment = { } end +if not file then file = { } end + +if environment.aleph_mode == nil then environment.aleph_mode = true end -- temp hack + +if not input then input = { } end +if not input.suffixes then input.suffixes = { } end +if not input.formats then input.formats = { } end +if not input.aux then input.aux = { } end + +if not input.suffixmap then input.suffixmap = { } end + +if not input.locators then input.locators = { } end -- locate databases +if not input.hashers then input.hashers = { } end -- load databases +if not input.generators then input.generators = { } end -- generate databases +if not input.filters then input.filters = { } end -- conversion filters + +local format = string.format + +input.locators.notfound = { nil } +input.hashers.notfound = { nil } +input.generators.notfound = { nil } + +input.cacheversion = '1.0.1' +input.banner = nil +input.verbose = false +input.debug = false +input.cnfname = 'texmf.cnf' +input.luaname = 'texmfcnf.lua' +input.lsrname = 'ls-R' +input.luasuffix = '.tma' +input.lucsuffix = '.tmc' + +-- we use a cleaned up list / format=any is a wildcard, as is *name + +input.formats['afm'] = 'AFMFONTS' input.suffixes['afm'] = { 'afm' } +input.formats['enc'] = 'ENCFONTS' input.suffixes['enc'] = { 'enc' } +input.formats['fmt'] = 'TEXFORMATS' input.suffixes['fmt'] = { 'fmt' } +input.formats['map'] = 'TEXFONTMAPS' input.suffixes['map'] = { 'map' } +input.formats['mp'] = 'MPINPUTS' input.suffixes['mp'] = { 'mp' } +input.formats['ocp'] = 'OCPINPUTS' input.suffixes['ocp'] = { 'ocp' } +input.formats['ofm'] = 'OFMFONTS' input.suffixes['ofm'] = { 'ofm', 'tfm' } +input.formats['otf'] = 'OPENTYPEFONTS' input.suffixes['otf'] = { 'otf' } -- 'ttf' +input.formats['opl'] = 'OPLFONTS' input.suffixes['opl'] = { 'opl' } +input.formats['otp'] = 'OTPINPUTS' input.suffixes['otp'] = { 'otp' } +input.formats['ovf'] = 'OVFFONTS' input.suffixes['ovf'] = { 'ovf', 'vf' } +input.formats['ovp'] = 'OVPFONTS' input.suffixes['ovp'] = { 'ovp' } +input.formats['tex'] = 'TEXINPUTS' input.suffixes['tex'] = { 'tex' } +input.formats['tfm'] = 'TFMFONTS' input.suffixes['tfm'] = { 'tfm' } +input.formats['ttf'] = 'TTFONTS' input.suffixes['ttf'] = { 'ttf', 'ttc' } +input.formats['pfb'] = 'T1FONTS' input.suffixes['pfb'] = { 'pfb', 'pfa' } +input.formats['vf'] = 'VFFONTS' input.suffixes['vf'] = { 'vf' } + +input.formats['fea'] = 'FONTFEATURES' input.suffixes['fea'] = { 'fea' } +input.formats['cid'] = 'FONTCIDMAPS' input.suffixes['cid'] = { 'cid', 'cidmap' } + +input.formats ['texmfscripts'] = 'TEXMFSCRIPTS' -- new +input.suffixes['texmfscripts'] = { 'rb', 'pl', 'py' } -- 'lua' + +input.formats ['lua'] = 'LUAINPUTS' -- new +input.suffixes['lua'] = { 'lua', 'luc', 'tma', 'tmc' } + +-- here we catch a few new thingies (todo: add these paths to context.tmf) +-- +-- FONTFEATURES = .;$TEXMF/fonts/fea// +-- FONTCIDMAPS = .;$TEXMF/fonts/cid// + +function input.checkconfigdata(instance) -- not yet ok, no time for debugging now + local function fix(varname,default) + local proname = varname .. "." .. instance.progname or "crap" + local p = instance.environment[proname] + local v = instance.environment[varname] + if not ((p and p ~= "") or (v and v ~= "")) then + instance.variables[varname] = default -- or environment? + end + end + fix("LUAINPUTS" , ".;$TEXINPUTS;$TEXMFSCRIPTS") + fix("FONTFEATURES", ".;$TEXMF/fonts/fea//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") + fix("FONTCIDMAPS" , ".;$TEXMF/fonts/cid//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") +end + +-- backward compatible ones + +input.alternatives = { } + +input.alternatives['map files'] = 'map' +input.alternatives['enc files'] = 'enc' +input.alternatives['cid files'] = 'cid' +input.alternatives['fea files'] = 'fea' +input.alternatives['opentype fonts'] = 'otf' +input.alternatives['truetype fonts'] = 'ttf' +input.alternatives['truetype collections'] = 'ttc' +input.alternatives['type1 fonts'] = 'pfb' + +-- obscure ones + +input.formats ['misc fonts'] = '' +input.suffixes['misc fonts'] = { } + +input.formats ['sfd'] = 'SFDFONTS' +input.suffixes ['sfd'] = { 'sfd' } +input.alternatives['subfont definition files'] = 'sfd' + +function input.reset() + + local instance = { } + + instance.rootpath = '' + instance.treepath = '' + instance.progname = environment.progname or 'context' + instance.engine = environment.engine or 'luatex' + instance.format = '' + instance.environment = { } + instance.variables = { } + instance.expansions = { } + instance.files = { } + instance.remap = { } + instance.configuration = { } + instance.setup = { } + instance.order = { } + instance.found = { } + instance.foundintrees = { } + instance.kpsevars = { } + instance.hashes = { } + instance.cnffiles = { } + instance.luafiles = { } + instance.lists = { } + instance.remember = true + instance.diskcache = true + instance.renewcache = false + instance.scandisk = true + instance.cachepath = nil + instance.loaderror = false + instance.smallcache = false + instance.savelists = true + instance.cleanuppaths = true + instance.allresults = false + instance.pattern = nil -- lists + instance.kpseonly = false -- lists + instance.cachefile = 'tmftools' + instance.loadtime = 0 + instance.starttime = 0 + instance.stoptime = 0 + instance.validfile = function(path,name) return true end + instance.data = { } -- only for loading + instance.force_suffixes = true + instance.dummy_path_expr = "^!*unset/*$" + instance.fakepaths = { } + instance.lsrmode = false + + if os.env then + -- store once, freeze and faster + for k,v in pairs(os.env) do + instance.environment[k] = input.bare_variable(v) + end + else + -- we will access os.env frequently + for k,v in pairs({'HOME','TEXMF','TEXMFCNF'}) do + local e = os.getenv(v) + if e then + -- input.report("setting",v,"to",input.bare_variable(e)) + instance.environment[v] = input.bare_variable(e) + end + end + end + + -- cross referencing + + for k, v in pairs(input.suffixes) do + for _, vv in pairs(v) do + if vv then + input.suffixmap[vv] = k + end + end + end + + return instance + +end + +function input.reset_hashes(instance) + instance.lists = { } + instance.found = { } +end + +function input.bare_variable(str) -- assumes str is a string + -- return string.gsub(string.gsub(string.gsub(str,"%s+$",""),'^"(.+)"$',"%1"),"^'(.+)'$","%1") + return (str:gsub("\s*([\"\']?)(.+)%1\s*", "%2")) +end + +if texio then + input.log = texio.write_nl +else + input.log = print +end + +function input.simple_logger(kind, name) + if name and name ~= "" then + if input.banner then + input.log(input.banner..kind..": "..name) + else + input.log("<<"..kind..": "..name..">>") + end + else + if input.banner then + input.log(input.banner..kind..": no name") + else + input.log("<<"..kind..": no name>>") + end + end +end + +function input.dummy_logger() +end + +function input.settrace(n) + input.trace = tonumber(n or 0) + if input.trace > 0 then + input.logger = input.simple_logger + input.verbose = true + else + input.logger = function() end + end +end + +function input.report(...) -- inefficient + if input.verbose then + if input.banner then + input.log(input.banner .. table.concat({...},' ')) + elseif input.logmode() == 'xml' then + input.log("<t>"..table.concat({...},' ').."</t>") + else + input.log("<<"..table.concat({...},' ')..">>") + end + end +end + +function input.reportlines(str) + if type(str) == "string" then + str = str:split("\n") + end + for _,v in pairs(str) do input.report(v) end +end + +input.settrace(tonumber(os.getenv("MTX.INPUT.TRACE") or os.getenv("MTX_INPUT_TRACE") or input.trace or 0)) + +-- These functions can be used to test the performance, especially +-- loading the database files. + +do + local clock = os.gettimeofday or os.clock + + function input.starttiming(instance) + if instance then + instance.starttime = clock() + if not instance.loadtime then + instance.loadtime = 0 + end + end + end + + function input.stoptiming(instance, report) + if instance then + local starttime = instance.starttime + if starttime then + local stoptime = clock() + local loadtime = stoptime - starttime + instance.stoptime = stoptime + instance.loadtime = instance.loadtime + loadtime + if report then + input.report('load time', format("%0.3f",loadtime)) + end + return loadtime + end + end + return 0 + end + +end + +function input.elapsedtime(instance) + return format("%0.3f",(instance and instance.loadtime) or 0) +end + +function input.report_loadtime(instance) + if instance then + input.report('total load time', input.elapsedtime(instance)) + end +end + +input.loadtime = input.elapsedtime + +function input.env(instance,key) + return instance.environment[key] or input.osenv(instance,key) +end + +function input.osenv(instance,key) + local ie = instance.environment + local value = ie[key] + if value == nil then + -- local e = os.getenv(key) + local e = os.env[key] + if e == nil then + -- value = "" -- false + else + value = input.bare_variable(e) + end + ie[key] = value + end + return value or "" +end + +-- we follow a rather traditional approach: +-- +-- (1) texmf.cnf given in TEXMFCNF +-- (2) texmf.cnf searched in TEXMF/web2c +-- +-- for the moment we don't expect a configuration file in a zip + +function input.identify_cnf(instance) + -- we no longer support treepath and rootpath (was handy for testing); + -- also we now follow the stupid route: if not set then just assume *one* + -- cnf file under texmf (i.e. distribution) + if #instance.cnffiles == 0 then + if input.env(instance,'TEXMFCNF') == "" then + local ownpath = environment.ownpath() or "." + if ownpath then + -- beware, this is tricky on my own system because at that location I do have + -- the raw tree that ends up in the zip; i.e. I cannot test this kind of mess + local function locate(filename,list) + local ownroot = input.normalize_name(file.join(ownpath,"../..")) + if not lfs.isdir(file.join(ownroot,"texmf")) then + ownroot = input.normalize_name(file.join(ownpath,"..")) + if not lfs.isdir(file.join(ownroot,"texmf")) then + input.verbose = true + input.report("error", "unable to identify cnf file") + return + end + end + local texmfcnf = file.join(ownroot,"texmf-local/web2c",filename) -- for minimals and myself + if not lfs.isfile(texmfcnf) then + texmfcnf = file.join(ownroot,"texmf/web2c",filename) + if not lfs.isfile(texmfcnf) then + input.verbose = true + input.report("error", "unable to locate",filename) + return + end + end + table.insert(list,texmfcnf) + local ie = instance.environment + if not ie['SELFAUTOPARENT'] then ie['SELFAUTOPARENT'] = ownroot end + if not ie['TEXMFCNF'] then ie['TEXMFCNF'] = file.dirname(texmfcnf) end + end + locate(input.luaname,instance.luafiles) + locate(input.cnfname,instance.cnffiles) + if #instance.luafiles == 0 and instance.cnffiles == 0 then + input.verbose = true + input.report("error", "unable to locate",filename) + os.exit() + end + -- here we also assume then TEXMF is set in the distribution, if this trickery is + -- used in the minimals, then users who don't use setuptex are on their own with + -- regards to extra trees + else + input.verbose = true + input.report("error", "unable to identify own path") + os.exit() + end + else + local t = input.split_path(input.env(instance,'TEXMFCNF')) + t = input.aux.expanded_path(instance,t) + input.aux.expand_vars(instance,t) + local function locate(filename,list) + for _,v in ipairs(t) do + local texmfcnf = input.normalize_name(file.join(v,filename)) + if lfs.isfile(texmfcnf) then + table.insert(list,texmfcnf) + end + end + end + locate(input.luaname,instance.luafiles) + locate(input.cnfname,instance.cnffiles) + end + end +end + +function input.load_cnf(instance) + local function loadoldconfigdata() + for _, fname in ipairs(instance.cnffiles) do + input.aux.load_cnf(instance,fname) + end + end + -- instance.cnffiles contain complete names now ! + if #instance.cnffiles == 0 then + input.report("no cnf files found (TEXMFCNF may not be set/known)") + else + instance.rootpath = instance.cnffiles[1] + for k,fname in ipairs(instance.cnffiles) do + instance.cnffiles[k] = input.normalize_name(fname:gsub("\\",'/')) + end + for i=1,3 do + instance.rootpath = file.dirname(instance.rootpath) + end + instance.rootpath = input.normalize_name(instance.rootpath) + instance.environment['SELFAUTOPARENT'] = instance.rootpath -- just to be sure + if instance.lsrmode then + loadoldconfigdata() + elseif instance.diskcache and not instance.renewcache then + input.loadoldconfig(instance,instance.cnffiles) + if instance.loaderror then + loadoldconfigdata() + input.saveoldconfig(instance) + end + else + loadoldconfigdata() + if instance.renewcache then + input.saveoldconfig(instance) + end + end + input.aux.collapse_cnf_data(instance) + end + input.checkconfigdata(instance) +end + +function input.load_lua(instance) + if #instance.luafiles == 0 then + -- yet harmless + else + instance.rootpath = instance.luafiles[1] + for k,fname in ipairs(instance.luafiles) do + instance.luafiles[k] = input.normalize_name(fname:gsub("\\",'/')) + end + for i=1,3 do + instance.rootpath = file.dirname(instance.rootpath) + end + instance.rootpath = input.normalize_name(instance.rootpath) + instance.environment['SELFAUTOPARENT'] = instance.rootpath -- just to be sure + input.loadnewconfig(instance) + input.aux.collapse_cnf_data(instance) + end + input.checkconfigdata(instance) +end + +function input.aux.collapse_cnf_data(instance) -- potential optmization: pass start index (setup and configuration are shared) + for _,c in ipairs(instance.order) do + for k,v in pairs(c) do + if not instance.variables[k] then + if instance.environment[k] then + instance.variables[k] = instance.environment[k] + else + instance.kpsevars[k] = true + instance.variables[k] = input.bare_variable(v) + end + end + end + end +end + +function input.aux.load_cnf(instance,fname) + fname = input.clean_path(fname) + local lname = fname:gsub("%.%a+$",input.luasuffix) + local f = io.open(lname) + if f then -- this will go + f:close() + local dname = file.dirname(fname) + if not instance.configuration[dname] then + input.aux.load_configuration(instance,dname,lname) + instance.order[#instance.order+1] = instance.configuration[dname] + end + else + f = io.open(fname) + if f then + input.report("loading", fname) + local line, data, n, k, v + local dname = file.dirname(fname) + if not instance.configuration[dname] then + instance.configuration[dname] = { } + instance.order[#instance.order+1] = instance.configuration[dname] + end + local data = instance.configuration[dname] + while true do + local line, n = f:read(), 0 + if line then + while true do -- join lines + line, n = line:gsub("\\%s*$", "") + if n > 0 then + line = line .. f:read() + else + break + end + end + if not line:find("^[%%#]") then + local k, v = (line:gsub("%s*%%.*$","")):match("%s*(.-)%s*=%s*(.-)%s*$") + if k and v and not data[k] then + data[k] = (v:gsub("[%%#].*",'')):gsub("~", "$HOME") + instance.kpsevars[k] = true + end + end + else + break + end + end + f:close() + else + input.report("skipping", fname) + end + end +end + +-- database loading + +function input.load_hash(instance) + input.locatelists(instance) + if instance.lsrmode then + input.loadlists(instance) + elseif instance.diskcache and not instance.renewcache then + input.loadfiles(instance) + if instance.loaderror then + input.loadlists(instance) + input.savefiles(instance) + end + else + input.loadlists(instance) + if instance.renewcache then + input.savefiles(instance) + end + end +end + +function input.aux.append_hash(instance,type,tag,name) + input.logger("= hash append",tag) + table.insert(instance.hashes, { ['type']=type, ['tag']=tag, ['name']=name } ) +end + +function input.aux.prepend_hash(instance,type,tag,name) + input.logger("= hash prepend",tag) + table.insert(instance.hashes, 1, { ['type']=type, ['tag']=tag, ['name']=name } ) +end + +function input.aux.extend_texmf_var(instance,specification) -- crap + if instance.environment['TEXMF'] then + input.report("extending environment variable TEXMF with", specification) + instance.environment['TEXMF'] = instance.environment['TEXMF']:gsub("^%{", function() + return "{" .. specification .. "," + end) + elseif instance.variables['TEXMF'] then + input.report("extending configuration variable TEXMF with", specification) + instance.variables['TEXMF'] = instance.variables['TEXMF']:gsub("^%{", function() + return "{" .. specification .. "," + end) + else + input.report("setting configuration variable TEXMF to", specification) + instance.variables['TEXMF'] = "{" .. specification .. "}" + end + if instance.variables['TEXMF']:find("%,") and not instance.variables['TEXMF']:find("^%{") then + input.report("adding {} to complex TEXMF variable, best do that yourself") + instance.variables['TEXMF'] = "{" .. instance.variables['TEXMF'] .. "}" + end + input.expand_variables(instance) + input.reset_hashes(instance) +end + +-- locators + +function input.locatelists(instance) + for _, path in pairs(input.simplified_list(input.expansion(instance,'TEXMF'))) do + path = file.collapse_path(path) + input.report("locating list of",path) + input.locatedatabase(instance,input.normalize_name(path)) + end +end + +function input.locatedatabase(instance,specification) + return input.methodhandler('locators', instance, specification) +end + +function input.locators.tex(instance,specification) + if specification and specification ~= '' and lfs.isdir(specification) then + input.logger('! tex locator', specification..' found') + input.aux.append_hash(instance,'file',specification,filename) + else + input.logger('? tex locator', specification..' not found') + end +end + +-- hashers + +function input.hashdatabase(instance,tag,name) + return input.methodhandler('hashers',instance,tag,name) +end + +function input.loadfiles(instance) + instance.loaderror = false + instance.files = { } + if not instance.renewcache then + for _, hash in ipairs(instance.hashes) do + input.hashdatabase(instance,hash.tag,hash.name) + if instance.loaderror then break end + end + end +end + +function input.hashers.tex(instance,tag,name) + input.aux.load_files(instance,tag) +end + +-- generators: + +function input.loadlists(instance) + for _, hash in ipairs(instance.hashes) do + input.generatedatabase(instance,hash.tag) + end +end + +function input.generatedatabase(instance,specification) + return input.methodhandler('generators', instance, specification) +end + +do + + local weird = lpeg.anywhere(lpeg.S("~`!#$%^&*()={}[]:;\"\'||<>,?\n\r\t")) + + function input.generators.tex(instance,specification) + local tag = specification + if not instance.lsrmode and lfs and lfs.dir then + input.report("scanning path",specification) + instance.files[tag] = { } + local files = instance.files[tag] + local n, m, r = 0, 0, 0 + local spec = specification .. '/' + local attributes = lfs.attributes + local directory = lfs.dir + local small = instance.smallcache + local function action(path) + local mode, full + if path then + full = spec .. path .. '/' + else + full = spec + end + for name in directory(full) do + if name:find("^%.") then + -- skip + -- elseif name:find("[%~%`%!%#%$%%%^%&%*%(%)%=%{%}%[%]%:%;\"\'%|%<%>%,%?\n\r\t]") then -- too much escaped + elseif weird:match(name) then + -- texio.write_nl("skipping " .. name) + -- skip + else + mode = attributes(full..name,'mode') + if mode == "directory" then + m = m + 1 + if path then + action(path..'/'..name) + else + action(name) + end + elseif path and mode == 'file' then + n = n + 1 + local f = files[name] + if f then + if not small then + if type(f) == 'string' then + files[name] = { f, path } + else + f[#f+1] = path + end + end + else + files[name] = path + local lower = name:lower() + if name ~= lower then + files["remap:"..lower] = name + r = r + 1 + end + end + end + end + end + end + action() + input.report(format("%s files found on %s directories with %s uppercase remappings",n,m,r)) + else + local fullname = file.join(specification,input.lsrname) + local path = '.' + local f = io.open(fullname) + if f then + instance.files[tag] = { } + local files = instance.files[tag] + local small = instance.smallcache + input.report("loading lsr file",fullname) + -- for line in f:lines() do -- much slower then the next one + for line in (f:read("*a")):gmatch("(.-)\n") do + if line:find("^[%a%d]") then + local fl = files[line] + if fl then + if not small then + if type(fl) == 'string' then + files[line] = { fl, path } -- table + else + fl[#fl+1] = path + end + end + else + files[line] = path -- string + local lower = line:lower() + if line ~= lower then + files["remap:"..lower] = line + end + end + else + path = line:match("%.%/(.-)%:$") or path -- match could be nil due to empty line + end + end + f:close() + end + end + end + +end + +-- savers, todo + +function input.savefiles(instance) + input.aux.save_data(instance, 'files', function(k,v) + return instance.validfile(k,v) -- path, name + end) +end + +-- A config (optionally) has the paths split in tables. Internally +-- we join them and split them after the expansion has taken place. This +-- is more convenient. + +function input.splitconfig(instance) + for i,c in ipairs(instance) do + for k,v in pairs(c) do + if type(v) == 'string' then + local t = file.split_path(v) + if #t > 1 then + c[k] = t + end + end + end + end +end +function input.joinconfig(instance) + for i,c in ipairs(instance.order) do + for k,v in pairs(c) do + if type(v) == 'table' then + c[k] = file.join_path(v) + end + end + end +end +function input.split_path(str) + if type(str) == 'table' then + return str + else + return file.split_path(str) + end +end +function input.join_path(str) + if type(str) == 'table' then + return file.join_path(str) + else + return str + end +end + +function input.splitexpansions(instance) + for k,v in pairs(instance.expansions) do + local t, h = { }, { } + for _,vv in pairs(file.split_path(v)) do + if vv ~= "" and not h[vv] then + t[#t+1] = vv + h[vv] = true + end + end + if #t > 1 then + instance.expansions[k] = t + else + instance.expansions[k] = t[1] + end + end +end + +-- end of split/join code + +function input.saveoldconfig(instance) + input.splitconfig(instance) + input.aux.save_data(instance, 'configuration', nil) + input.joinconfig(instance) +end + +input.configbanner = [[ +-- This is a Luatex configuration file created by 'luatools.lua' or +-- 'luatex.exe' directly. For comment, suggestions and questions you can +-- contact the ConTeXt Development Team. This configuration file is +-- not copyrighted. [HH & TH] +]] + +function input.serialize(files) + -- This version is somewhat optimized for the kind of + -- tables that we deal with, so it's much faster than + -- the generic serializer. This makes sense because + -- luatools and mtxtools are called frequently. Okay, + -- we pay a small price for properly tabbed tables. + local t = { } + local concat = table.concat + local sorted = table.sortedkeys + local function dump(k,v,m) + if type(v) == 'string' then + return m .. "['" .. k .. "']='" .. v .. "'," + elseif #v == 1 then + return m .. "['" .. k .. "']='" .. v[1] .. "'," + else + return m .. "['" .. k .. "']={'" .. concat(v,"','").. "'}," + end + end + t[#t+1] = "return {" + if instance.sortdata then + for _, k in pairs(sorted(files)) do + local fk = files[k] + if type(fk) == 'table' then + t[#t+1] = "\t['" .. k .. "']={" + for _, kk in pairs(sorted(fk)) do + t[#t+1] = dump(kk,fk[kk],"\t\t") + end + t[#t+1] = "\t}," + else + t[#t+1] = dump(k,fk,"\t") + end + end + else + for k, v in pairs(files) do + if type(v) == 'table' then + t[#t+1] = "\t['" .. k .. "']={" + for kk,vv in pairs(v) do + t[#t+1] = dump(kk,vv,"\t\t") + end + t[#t+1] = "\t}," + else + t[#t+1] = dump(k,v,"\t") + end + end + end + t[#t+1] = "}" + return concat(t,"\n") +end + +if not texmf then texmf = {} end -- no longer needed, at least not here + +function input.aux.save_data(instance, dataname, check, makename) -- untested without cache overload + for cachename, files in pairs(instance[dataname]) do + local name = (makename or file.join)(cachename,dataname) + local luaname, lucname = name .. input.luasuffix, name .. input.lucsuffix + input.report("preparing " .. dataname .. " for", luaname) + for k, v in pairs(files) do + if not check or check(v,k) then -- path, name + if type(v) == "table" and #v == 1 then + files[k] = v[1] + end + else + files[k] = nil -- false + end + end + local data = { + type = dataname, + root = cachename, + version = input.cacheversion, + date = os.date("%Y-%m-%d"), + time = os.date("%H:%M:%S"), + content = files, + } + local f = io.open(luaname,'w') + if f then + input.report("saving " .. dataname .. " in", luaname) + f:write(input.serialize(data)) + f:close() + input.report("compiling " .. dataname .. " to", lucname) + if not utils.lua.compile(luaname,lucname) then + input.report("compiling failed for " .. dataname .. ", deleting file " .. lucname) + os.remove(lucname) + end + else + input.report("unable to save " .. dataname .. " in " .. name..input.luasuffix) + end + end +end + +function input.aux.load_data(instance,pathname,dataname,filename,makename) -- untested without cache overload + filename = ((not filename or (filename == "")) and dataname) or filename + filename = (makename and makename(dataname,filename)) or file.join(pathname,filename) + local blob = loadfile(filename .. input.lucsuffix) or loadfile(filename .. input.luasuffix) + if blob then + local data = blob() + if data and data.content and data.type == dataname and data.version == input.cacheversion then + input.report("loading",dataname,"for",pathname,"from",filename) + instance[dataname][pathname] = data.content + else + input.report("skipping",dataname,"for",pathname,"from",filename) + instance[dataname][pathname] = { } + instance.loaderror = true + end + else + input.report("skipping",dataname,"for",pathname,"from",filename) + end +end + +-- some day i'll use the nested approach, but not yet (actually we even drop +-- engine/progname support since we have only luatex now) +-- +-- first texmfcnf.lua files are located, next the cached texmf.cnf files +-- +-- return { +-- TEXMFBOGUS = 'effe checken of dit werkt', +-- } + +function input.aux.load_texmfcnf(instance,dataname,pathname) + local filename = file.join(pathname,input.luaname) + local blob = loadfile(filename) + if blob then + local data = blob() + if data then + input.report("loading","configuration file",filename) + if true then + -- flatten to variable.progname + local t = { } + for k, v in pairs(data) do -- v = progname + if type(v) == "string" then + t[k] = v + else + for kk, vv in pairs(v) do -- vv = variable + if type(vv) == "string" then + t[vv.."."..v] = kk + end + end + end + end + instance[dataname][pathname] = t + else + instance[dataname][pathname] = data + end + else + input.report("skipping","configuration file",filename) + instance[dataname][pathname] = { } + instance.loaderror = true + end + else + input.report("skipping","configuration file",filename) + end +end + +function input.aux.load_configuration(instance,dname,lname) + input.aux.load_data(instance,dname,'configuration',lname and file.basename(lname)) +end +function input.aux.load_files(instance,tag) + input.aux.load_data(instance,tag,'files') +end + +function input.resetconfig(instance) + instance.configuration, instance.setup, instance.order, instance.loaderror = { }, { }, { }, false +end + +function input.loadnewconfig(instance) + for _, cnf in ipairs(instance.luafiles) do + local dname = file.dirname(cnf) + input.aux.load_texmfcnf(instance,'setup',dname) + instance.order[#instance.order+1] = instance.setup[dname] + if instance.loaderror then break end + end +end + +function input.loadoldconfig(instance) + if not instance.renewcache then + for _, cnf in ipairs(instance.cnffiles) do + local dname = file.dirname(cnf) + input.aux.load_configuration(instance,dname) + instance.order[#instance.order+1] = instance.configuration[dname] + if instance.loaderror then break end + end + end + input.joinconfig(instance) +end + +function input.expand_variables(instance) + instance.expansions = { } +--~ instance.environment['SELFAUTOPARENT'] = instance.environment['SELFAUTOPARENT'] or instance.rootpath + if instance.engine ~= "" then instance.environment['engine'] = instance.engine end + if instance.progname ~= "" then instance.environment['progname'] = instance.progname end + for k,v in pairs(instance.environment) do + local a, b = k:match("^(%a+)%_(.*)%s*$") + if a and b then + instance.expansions[a..'.'..b] = v + else + instance.expansions[k] = v + end + end + for k,v in pairs(instance.environment) do -- move environment to expansions + if not instance.expansions[k] then instance.expansions[k] = v end + end + for k,v in pairs(instance.variables) do -- move variables to expansions + if not instance.expansions[k] then instance.expansions[k] = v end + end + while true do + local busy = false + for k,v in pairs(instance.expansions) do + local s, n = v:gsub("%$([%a%d%_%-]+)", function(a) + busy = true + return instance.expansions[a] or input.env(instance,a) + end) + local s, m = s:gsub("%$%{([%a%d%_%-]+)%}", function(a) + busy = true + return instance.expansions[a] or input.env(instance,a) + end) + if n > 0 or m > 0 then + instance.expansions[k]= s + end + end + if not busy then break end + end + for k,v in pairs(instance.expansions) do + instance.expansions[k] = v:gsub("\\", '/') + end +end + +function input.aux.expand_vars(instance,lst) -- simple vars + for k,v in pairs(lst) do + lst[k] = v:gsub("%$([%a%d%_%-]+)", function(a) + return instance.variables[a] or input.env(instance,a) + end) + end +end + +function input.aux.expanded_var(instance,var) -- simple vars + return var:gsub("%$([%a%d%_%-]+)", function(a) + return instance.variables[a] or input.env(instance,a) + end) +end + +function input.aux.entry(instance,entries,name) + if name and (name ~= "") then + name = name:gsub('%$','') + local result = entries[name..'.'..instance.progname] or entries[name] + if result then + return result + else + result = input.env(instance,name) + if result then + instance.variables[name] = result + input.expand_variables(instance) + return instance.expansions[name] or "" + end + end + end + return "" +end +function input.variable(instance,name) + return input.aux.entry(instance,instance.variables,name) +end +function input.expansion(instance,name) + return input.aux.entry(instance,instance.expansions,name) +end + +function input.aux.is_entry(instance,entries,name) + if name and name ~= "" then + name = name:gsub('%$','') + return (entries[name..'.'..instance.progname] or entries[name]) ~= nil + else + return false + end +end + +function input.is_variable(instance,name) + return input.aux.is_entry(instance,instance.variables,name) +end +function input.is_expansion(instance,name) + return input.aux.is_entry(instance,instance.expansions,name) +end + +function input.simplified_list(str) + if type(str) == 'table' then + return str -- troubles ; ipv , in texmf + elseif str == '' then + return { } + else + local t = { } + for _,v in ipairs(string.splitchr(str:gsub("^\{(.+)\}$","%1"),",")) do + t[#t+1] = (v:gsub("^[%!]*(.+)[%/\\]*$","%1")) + end + return t + end +end + +function input.unexpanded_path_list(instance,str) + local pth = input.variable(instance,str) + local lst = input.split_path(pth) + return input.aux.expanded_path(instance,lst) +end +function input.unexpanded_path(instance,str) + return file.join_path(input.unexpanded_path_list(instance,str)) +end + +do + local done = { } + + function input.reset_extra_path(instance) + local ep = instance.extra_paths + if not ep then + ep, done = { }, { } + instance.extra_paths = ep + elseif #ep > 0 then + instance.lists, done = { }, { } + end + end + + function input.register_extra_path(instance,paths,subpaths) + local ep = instance.extra_paths or { } + local n = #ep + if paths and paths ~= "" then + if subpaths and subpaths ~= "" then + for p in paths:gmatch("[^,]+") do + -- we gmatch each step again, not that fast, but used seldom + for s in subpaths:gmatch("[^,]+") do + local ps = p .. "/" .. s + if not done[ps] then + ep[#ep+1] = input.clean_path(ps) + done[ps] = true + end + end + end + else + for p in paths:gmatch("[^,]+") do + if not done[p] then + ep[#ep+1] = input.clean_path(p) + done[p] = true + end + end + end + elseif subpaths and subpaths ~= "" then + for i=1,n do + -- we gmatch each step again, not that fast, but used seldom + for s in subpaths:gmatch("[^,]+") do + local ps = ep[i] .. "/" .. s + if not done[ps] then + ep[#ep+1] = input.clean_path(ps) + done[ps] = true + end + end + end + end + if #ep > 0 then + instance.extra_paths = ep -- register paths + end + if #ep > n then + instance.lists = { } -- erase the cache + end + end + +end + +function input.expanded_path_list(instance,str) + local function made_list(list) + local ep = instance.extra_paths + if not ep or #ep == 0 then + return list + else + local done, new = { }, { } + -- honour . .. ../.. but only when at the start + for k, v in ipairs(list) do + if not done[v] then + if v:find("^[%.%/]$") then + done[v] = true + new[#new+1] = v + else + break + end + end + end + -- first the extra paths + for k, v in ipairs(ep) do + if not done[v] then + done[v] = true + new[#new+1] = v + end + end + -- next the formal paths + for k, v in ipairs(list) do + if not done[v] then + done[v] = true + new[#new+1] = v + end + end + return new + end + end + if not str then + return ep or { } + elseif instance.savelists then + -- engine+progname hash + str = str:gsub("%$","") + if not instance.lists[str] then -- cached + local lst = made_list(input.split_path(input.expansion(instance,str))) + instance.lists[str] = input.aux.expanded_path(instance,lst) + end + return instance.lists[str] + else + local lst = input.split_path(input.expansion(instance,str)) + return made_list(input.aux.expanded_path(instance,lst)) + end +end + +function input.expand_path(instance,str) + return file.join_path(input.expanded_path_list(instance,str)) +end + +--~ function input.first_writable_path(instance,name) +--~ for _,v in pairs(input.expanded_path_list(instance,name)) do +--~ if file.is_writable(file.join(v,'luatex-cache.tmp')) then +--~ return v +--~ end +--~ end +--~ return "." +--~ end + +function input.expanded_path_list_from_var(instance,str) -- brrr + local tmp = input.var_of_format_or_suffix(str:gsub("%$","")) + if tmp ~= "" then + return input.expanded_path_list(instance,str) + else + return input.expanded_path_list(instance,tmp) + end +end +function input.expand_path_from_var(instance,str) + return file.join_path(input.expanded_path_list_from_var(instance,str)) +end + +function input.format_of_var(str) + return input.formats[str] or input.formats[input.alternatives[str]] or '' +end +function input.format_of_suffix(str) + return input.suffixmap[file.extname(str)] or 'tex' +end + +function input.variable_of_format(str) + return input.formats[str] or input.formats[input.alternatives[str]] or '' +end + +function input.var_of_format_or_suffix(str) + local v = input.formats[str] + if v then + return v + end + v = input.formats[input.alternatives[str]] + if v then + return v + end + v = input.suffixmap[file.extname(str)] + if v then + return input.formats[isf] + end + return '' +end + +function input.expand_braces(instance,str) -- output variable and brace expansion of STRING + local ori = input.variable(instance,str) + local pth = input.aux.expanded_path(instance,input.split_path(ori)) + return file.join_path(pth) +end + +-- {a,b,c,d} +-- a,b,c/{p,q,r},d +-- a,b,c/{p,q,r}/d/{x,y,z}// +-- a,b,c/{p,q/{x,y,z},r},d/{p,q,r} +-- a,b,c/{p,q/{x,y,z},r},d/{p,q,r} +-- a{b,c}{d,e}f +-- {a,b,c,d} +-- {a,b,c/{p,q,r},d} +-- {a,b,c/{p,q,r}/d/{x,y,z}//} +-- {a,b,c/{p,q/{x,y,z}},d/{p,q,r}} +-- {a,b,c/{p,q/{x,y,z},w}v,d/{p,q,r}} + +-- this one is better and faster, but it took me a while to realize +-- that this kind of replacement is cleaner than messy parsing and +-- fuzzy concatenating we can probably gain a bit with selectively +-- applying lpeg, but experiments with lpeg parsing this proved not to +-- work that well; the parsing is ok, but dealing with the resulting +-- table is a pain because we need to work inside-out recursively + +-- get rid of piecewise here, just a gmatch is ok + +function input.aux.splitpathexpr(str, t, validate) + -- no need for optimization, only called a few times, we can use lpeg for the sub + t = t or { } + local concat = table.concat + while true do + local done = false + while true do + local ok = false + str = str:gsub("([^{},]+){([^{}]-)}", function(a,b) + local t = { } + b:piecewise(",", function(s) t[#t+1] = a .. s end) + ok, done = true, true + return "{" .. concat(t,",") .. "}" + end) + if not ok then break end + end + while true do + local ok = false + str = str:gsub("{([^{}]-)}([^{},]+)", function(a,b) + local t = { } + a:piecewise(",", function(s) t[#t+1] = s .. b end) + ok, done = true, true + return "{" .. concat(t,",") .. "}" + end) + if not ok then break end + end + while true do + local ok = false + str = str:gsub("([,{]){([^{}]+)}([,}])", function(a,b,c) + ok, done = true, true + return a .. b .. c + end) + if not ok then break end + end + if not done then break end + end + while true do + local ok = false + str = str:gsub("{([^{}]-)}{([^{}]-)}", function(a,b) + local t = { } + a:piecewise(",", function(sa) + b:piecewise(",", function(sb) + t[#t+1] = sa .. sb + end) + end) + ok = true + return "{" .. concat(t,",") .. "}" + end) + if not ok then break end + end + while true do + local ok = false + str = str:gsub("{([^{}]-)}", function(a) + ok = true + return a + end) + if not ok then break end + end + if validate then + str:piecewise(",", function(s) + s = validate(s) + if s then t[#t+1] = s end + end) + else + str:piecewise(",", function(s) + t[#t+1] = s + end) + end + return t +end + +function input.aux.expanded_path(instance,pathlist) -- maybe not a list, just a path + -- a previous version fed back into pathlist + local newlist, ok = { }, false + for _,v in ipairs(pathlist) do + if v:find("[{}]") then + ok = true + break + end + end + if ok then + for _, v in ipairs(pathlist) do + input.aux.splitpathexpr(v, newlist, function(s) + s = file.collapse_path(s) + return s ~= "" and not s:find(instance.dummy_path_expr) and s + end) + end + else + for _,v in ipairs(pathlist) do + for vv in string.gmatch(v..',',"(.-),") do + vv = file.collapse_path(v) + if vv ~= "" then newlist[#newlist+1] = vv end + end + end + end + return newlist +end + +input.is_readable = { } + +function input.aux.is_readable(readable, name) + if input.trace > 2 then + if readable then + input.logger("+ readable", name) + else + input.logger("- readable", name) + end + end + return readable +end + +function input.is_readable.file(name) + -- return input.aux.is_readable(file.is_readable(name), name) + return input.aux.is_readable(input.aux.is_file(name), name) +end + +input.is_readable.tex = input.is_readable.file + +-- name +-- name/name + +function input.aux.collect_files(instance,names) + local filelist = { } + for _, fname in pairs(names) do + if fname then + if input.trace > 2 then + input.logger("? blobpath asked",fname) + end + local bname = file.basename(fname) + local dname = file.dirname(fname) + if dname == "" or dname:find("^%.") then + dname = false + else + dname = "/" .. dname .. "$" + end + for _, hash in ipairs(instance.hashes) do + local blobpath = hash.tag + local files = blobpath and instance.files[blobpath] + if files then + if input.trace > 2 then + input.logger('? blobpath do',blobpath .. " (" .. bname ..")") + end + local blobfile = files[bname] + if not blobfile then + local rname = "remap:"..bname + blobfile = files[rname] + if blobfile then + bname = files[rname] + blobfile = files[bname] + end + end + if blobfile then + if type(blobfile) == 'string' then + if not dname or blobfile:find(dname) then + filelist[#filelist+1] = { + hash.type, + file.join(blobpath,blobfile,bname), -- search + input.concatinators[hash.type](blobpath,blobfile,bname) -- result + } + end + else + for _, vv in pairs(blobfile) do + if not dname or vv:find(dname) then + filelist[#filelist+1] = { + hash.type, + file.join(blobpath,vv,bname), -- search + input.concatinators[hash.type](blobpath,vv,bname) -- result + } + end + end + end + end + elseif input.trace > 1 then + input.logger('! blobpath no',blobpath .. " (" .. bname ..")" ) + end + end + end + end + if #filelist > 0 then + return filelist + else + return nil + end +end + +function input.suffix_of_format(str) + if input.suffixes[str] then + return input.suffixes[str][1] + else + return "" + end +end + +function input.suffixes_of_format(str) + if input.suffixes[str] then + return input.suffixes[str] + else + return {} + end +end + +do + + -- called about 700 times for an empty doc (font initializations etc) + -- i need to weed the font files for redundant calls + + local letter = lpeg.R("az","AZ") + local separator = lpeg.P("://") + + local qualified = lpeg.P(".")^0 * lpeg.P("/") + letter*lpeg.P(":") + letter^1*separator + local rootbased = lpeg.P("/") + letter*lpeg.P(":") + + -- ./name ../name /name c: :// + function input.aux.qualified_path(filename) + return qualified:match(filename) + end + function input.aux.rootbased_path(filename) + return rootbased:match(filename) + end + + function input.normalize_name(original) + return original + end + + input.normalize_name = file.collapse_path + +end + +function input.aux.register_in_trees(instance,name) + if not name:find("^%.") then + instance.foundintrees[name] = (instance.foundintrees[name] or 0) + 1 -- maybe only one + end +end + +-- split the next one up, better for jit + +function input.aux.find_file(instance,filename) -- todo : plugin (scanners, checkers etc) + local result = { } + local stamp = nil + filename = input.normalize_name(filename) -- elsewhere + filename = file.collapse_path(filename:gsub("\\","/")) -- elsewhere + -- speed up / beware: format problem + if instance.remember then + stamp = filename .. "--" .. instance.engine .. "--" .. instance.progname .. "--" .. instance.format + if instance.found[stamp] then + input.logger('! remembered', filename) + return instance.found[stamp] + end + end + if filename:find('%*') then + input.logger('! wildcard', filename) + result = input.find_wildcard_files(instance,filename) + elseif input.aux.qualified_path(filename) then + if input.is_readable.file(filename) then + input.logger('! qualified', filename) + result = { filename } + else + local forcedname, ok = "", false + if file.extname(filename) == "" then + if instance.format == "" then + forcedname = filename .. ".tex" + if input.is_readable.file(forcedname) then + input.logger('! no suffix, forcing standard filetype tex') + result, ok = { forcedname }, true + end + else + for _, s in pairs(input.suffixes_of_format(instance.format)) do + forcedname = filename .. "." .. s + if input.is_readable.file(forcedname) then + input.logger('! no suffix, forcing format filetype', s) + result, ok = { forcedname }, true + break + end + end + end + end + if not ok then + input.logger('? qualified', filename) + end + end + else + -- search spec + local filetype, extra, done, wantedfiles, ext = '', nil, false, { }, file.extname(filename) + if ext == "" then + if not instance.force_suffixes then + wantedfiles[#wantedfiles+1] = filename + end + else + wantedfiles[#wantedfiles+1] = filename + end + if instance.format == "" then + if ext == "" then + local forcedname = filename .. '.tex' + wantedfiles[#wantedfiles+1] = forcedname + filetype = input.format_of_suffix(forcedname) + input.logger('! forcing filetype',filetype) + else + filetype = input.format_of_suffix(filename) + input.logger('! using suffix based filetype',filetype) + end + else + if ext == "" then + for _, s in pairs(input.suffixes_of_format(instance.format)) do + wantedfiles[#wantedfiles+1] = filename .. "." .. s + end + end + filetype = instance.format + input.logger('! using given filetype',filetype) + end + local typespec = input.variable_of_format(filetype) + local pathlist = input.expanded_path_list(instance,typespec) + if not pathlist or #pathlist == 0 then + -- no pathlist, access check only / todo == wildcard + if input.trace > 2 then + input.logger('? filename',filename) + input.logger('? filetype',filetype or '?') + input.logger('? wanted files',table.concat(wantedfiles," | ")) + end + for _, fname in pairs(wantedfiles) do + if fname and input.is_readable.file(fname) then + filename, done = fname, true + result[#result+1] = file.join('.',fname) + break + end + end + -- this is actually 'other text files' or 'any' or 'whatever' + local filelist = input.aux.collect_files(instance,wantedfiles) + local fl = filelist and filelist[1] + if fl then + filename = fl[3] + result[#result+1] = filename + done = true + end + else + -- list search + local filelist = input.aux.collect_files(instance,wantedfiles) + local doscan, recurse + if input.trace > 2 then + input.logger('? filename',filename) + -- if pathlist then input.logger('? path list',table.concat(pathlist," | ")) end + -- if filelist then input.logger('? file list',table.concat(filelist," | ")) end + end + -- a bit messy ... esp the doscan setting here + for _, path in pairs(pathlist) do + if path:find("^!!") then doscan = false else doscan = true end + if path:find("//$") then recurse = true else recurse = false end + local pathname = path:gsub("^!+", '') + done = false + -- using file list + if filelist and not (done and not instance.allresults) and recurse then + -- compare list entries with permitted pattern + pathname = pathname:gsub("([%-%.])","%%%1") -- this also influences + pathname = pathname:gsub("/+$", '/.*') -- later usage of pathname + pathname = pathname:gsub("//", '/.-/') -- not ok for /// but harmless + local expr = "^" .. pathname + -- input.debug('?',expr) + for _, fl in ipairs(filelist) do + local f = fl[2] + if f:find(expr) then + -- input.debug('T',' '..f) + if input.trace > 2 then + input.logger('= found in hash',f) + end + --- todo, test for readable + result[#result+1] = fl[3] + input.aux.register_in_trees(instance,f) -- for tracing used files + done = true + if not instance.allresults then break end + else + -- input.debug('F',' '..f) + end + end + end + if not done and doscan then + -- check if on disk / unchecked / does not work at all / also zips + if input.method_is_file(pathname) then -- ? + local pname = pathname:gsub("%.%*$",'') + if not pname:find("%*") then + local ppname = pname:gsub("/+$","") + if input.aux.can_be_dir(instance,ppname) then + for _, w in pairs(wantedfiles) do + local fname = file.join(ppname,w) + if input.is_readable.file(fname) then + if input.trace > 2 then + input.logger('= found by scanning',fname) + end + result[#result+1] = fname + done = true + if not instance.allresults then break end + end + end + else + -- no access needed for non existing path, speedup (esp in large tree with lots of fake) + end + end + end + end + if not done and doscan then + -- todo: slow path scanning + end + if done and not instance.allresults then break end + end + end + end + for k,v in pairs(result) do + result[k] = file.collapse_path(v) + end + if instance.remember then + instance.found[stamp] = result + end + return result +end + +input.aux._find_file_ = input.aux.find_file + +function input.aux.find_file(instance,filename) -- maybe make a lowres cache too + local result = input.aux._find_file_(instance,filename) + if #result == 0 then + local lowered = filename:lower() + if filename ~= lowered then + return input.aux._find_file_(instance,lowered) + end + end + return result +end + +if lfs and lfs.isfile then + input.aux.is_file = lfs.isfile -- to be done: use this +else + input.aux.is_file = file.is_readable +end + +if lfs and lfs.isdir then + function input.aux.can_be_dir(instance,name) + if not instance.fakepaths[name] then + if lfs.isdir(name) then + instance.fakepaths[name] = 1 -- directory + else + instance.fakepaths[name] = 2 -- no directory + end + end + return (instance.fakepaths[name] == 1) + end +else + function input.aux.can_be_dir() + return true + end +end + +if not input.concatinators then input.concatinators = { } end + +input.concatinators.tex = file.join +input.concatinators.file = input.concatinators.tex + +function input.find_files(instance,filename,filetype,mustexist) + if type(mustexist) == boolean then + -- all set + elseif type(filetype) == 'boolean' then + filetype, mustexist = nil, false + elseif type(filetype) ~= 'string' then + filetype, mustexist = nil, false + end + instance.format = filetype or '' + local t = input.aux.find_file(instance,filename,true) + instance.format = '' + return t +end + +function input.find_file(instance,filename,filetype,mustexist) + return (input.find_files(instance,filename,filetype,mustexist)[1] or "") +end + +function input.find_given_files(instance,filename) + local bname, result = file.basename(filename), { } + for k, hash in ipairs(instance.hashes) do + local files = instance.files[hash.tag] + local blist = files[bname] + if not blist then + local rname = "remap:"..bname + blist = files[rname] + if blist then + bname = files[rname] + blist = files[bname] + end + end + if blist then + if type(blist) == 'string' then + result[#result+1] = input.concatinators[hash.type](hash.tag,blist,bname) or "" + if not instance.allresults then break end + else + for kk,vv in pairs(blist) do + result[#result+1] = input.concatinators[hash.type](hash.tag,vv,bname) or "" + if not instance.allresults then break end + end + end + end + end + return result +end + +function input.find_given_file(instance,filename) + return (input.find_given_files(instance,filename)[1] or "") +end + +function input.find_wildcard_files(instance,filename) -- todo: remap: + local result = { } + local bname, dname = file.basename(filename), file.dirname(filename) + local path = dname:gsub("^*/","") + path = path:gsub("*",".*") + path = path:gsub("-","%%-") + if dname == "" then + path = ".*" + end + local name = bname + name = name:gsub("*",".*") + name = name:gsub("-","%%-") + path = path:lower() + name = name:lower() + local function doit(blist,bname,hash,allresults) + local done = false + if blist then + if type(blist) == 'string' then + -- make function and share code + if (blist:lower()):find(path) then + result[#result+1] = input.concatinators[hash.type](hash.tag,blist,bname) or "" + done = true + end + else + for kk,vv in pairs(blist) do + if (vv:lower()):find(path) then + result[#result+1] = input.concatinators[hash.type](hash.tag,vv,bname) or "" + done = true + if not allresults then break end + end + end + end + end + return done + end + local files, allresults, done = instance.files, instance.allresults, false + if name:find("%*") then + for k, hash in ipairs(instance.hashes) do + for kk, hh in pairs(files[hash.tag]) do + if not kk:find("^remap:") then + if (kk:lower()):find(name) then + if doit(hh,kk,hash,allresults) then done = true end + if done and not allresults then break end + end + end + end + end + else + for k, hash in ipairs(instance.hashes) do + if doit(files[hash.tag][bname],bname,hash,allresults) then done = true end + if done and not allresults then break end + end + end + return result +end + +function input.find_wildcard_file(instance,filename) + return (input.find_wildcard_files(instance,filename)[1] or "") +end + +-- main user functions + +function input.save_used_files_in_trees(instance, filename,jobname) + if not filename then filename = 'luatex.jlg' end + local f = io.open(filename,'w') + if f then + f:write("<?xml version='1.0' standalone='yes'?>\n") + f:write("<rl:job>\n") + if jobname then + f:write("\t<rl:name>" .. jobname .. "</rl:name>\n") + end + f:write("\t<rl:files>\n") + for _,v in pairs(table.sortedkeys(instance.foundintrees)) do + f:write("\t\t<rl:file n='" .. instance.foundintrees[v] .. "'>" .. v .. "</rl:file>\n") + end + f:write("\t</rl:files>\n") + f:write("</rl:usedfiles>\n") + f:close() + end +end + +function input.automount(instance) + -- implemented later +end + +function input.load(instance) + input.starttiming(instance) + input.resetconfig(instance) + input.identify_cnf(instance) + input.load_lua(instance) + input.expand_variables(instance) + input.load_cnf(instance) + input.expand_variables(instance) + input.load_hash(instance) + input.automount(instance) + input.stoptiming(instance) +end + +function input.for_files(instance, command, files, filetype, mustexist) + if files and #files > 0 then + local function report(str) + if input.verbose then + input.report(str) -- has already verbose + else + print(str) + end + end + if input.verbose then + report('') + end + for _, file in pairs(files) do + local result = command(instance,file,filetype,mustexist) + if type(result) == 'string' then + report(result) + else + for _,v in pairs(result) do + report(v) + end + end + end + end +end + +-- strtab + +function input.var_value(instance,str) -- output the value of variable $STRING. + return input.variable(instance,str) +end +function input.expand_var(instance,str) -- output variable expansion of STRING. + return input.expansion(instance,str) +end +function input.show_path(instance,str) -- output search path for file type NAME + return file.join_path(input.expanded_path_list(instance,input.format_of_var(str))) +end + +-- input.find_file(filename) +-- input.find_file(filename, filetype, mustexist) +-- input.find_file(filename, mustexist) +-- input.find_file(filename, filetype) + +function input.aux.register_file(files, name, path) + if files[name] then + if type(files[name]) == 'string' then + files[name] = { files[name], path } + else + files[name] = path + end + else + files[name] = path + end +end + +if not input.finders then input.finders = { } end +if not input.openers then input.openers = { } end +if not input.loaders then input.loaders = { } end + +input.finders.notfound = { nil } +input.openers.notfound = { nil } +input.loaders.notfound = { false, nil, 0 } + +function input.splitmethod(filename) + if not filename then + return { } -- safeguard + elseif type(filename) == "table" then + return filename -- already split + elseif not filename:find("://") then + return { scheme="file", path = filename, original=filename } -- quick hack + else + return url.hashed(filename) + end +end + +function input.method_is_file(filename) + return input.splitmethod(filename).scheme == 'file' +end + +function table.sequenced(t,sep) -- temp here + local s = { } + for k, v in pairs(t) do + s[#s+1] = k .. "=" .. v + end + return table.concat(s, sep or " | ") +end + +function input.methodhandler(what, instance, filename, filetype) -- ... + local specification = (type(filename) == "string" and input.splitmethod(filename)) or filename -- no or { }, let it bomb + local scheme = specification.scheme + if input[what][scheme] then + input.logger('= handler',specification.original .." -> " .. what .. " -> " .. table.sequenced(specification)) + return input[what][scheme](instance,filename,filetype) -- todo: specification + else + return input[what].tex(instance,filename,filetype) -- todo: specification + end +end + +-- also inside next test? + +function input.findtexfile(instance, filename, filetype) + return input.methodhandler('finders',instance, input.normalize_name(filename), filetype) +end +function input.opentexfile(instance,filename) + return input.methodhandler('openers',instance, input.normalize_name(filename)) +end + +function input.findbinfile(instance, filename, filetype) + return input.methodhandler('finders',instance, input.normalize_name(filename), filetype) +end +function input.openbinfile(instance,filename) + return input.methodhandler('loaders',instance, input.normalize_name(filename)) +end + +function input.loadbinfile(instance, filename, filetype) + local fname = input.findbinfile(instance, input.normalize_name(filename), filetype) + if fname and fname ~= "" then + return input.openbinfile(instance,fname) + else + return unpack(input.loaders.notfound) + end +end + +function input.texdatablob(instance, filename, filetype) + local ok, data, size = input.loadbinfile(instance, filename, filetype) + return data or "" +end + +input.loadtexfile = input.texdatablob + +function input.openfile(filename) -- brrr texmf.instance here / todo ! ! ! ! ! + local fullname = input.findtexfile(texmf.instance, filename) + if fullname and (fullname ~= "") then + return input.opentexfile(texmf.instance, fullname) + else + return nil + end +end + +function input.logmode() + return (os.getenv("MTX.LOG.MODE") or os.getenv("MTX_LOG_MODE") or "tex"):lower() +end + +-- this is a prelude to engine/progname specific configuration files +-- in which case we can omit files meant for other programs and +-- packages + +--- ctx + +-- maybe texinputs + font paths +-- maybe positive selection tex/context fonts/tfm|afm|vf|opentype|type1|map|enc + +input.validators = { } +input.validators.visibility = { } + +function input.validators.visibility.default(path, name) + return true +end + +function input.validators.visibility.context(path, name) + path = path[1] or path -- some day a loop + return not ( + path:find("latex") or +-- path:find("doc") or + path:find("tex4ht") or + path:find("source") or +-- path:find("config") or +-- path:find("metafont") or + path:find("lists$") or + name:find("%.tpm$") or + name:find("%.bak$") + ) +end + +-- todo: describe which functions are public (maybe input.private. ... ) + +-- beware: i need to check where we still need a / on windows: + +function input.clean_path(str) +--~ return (((str:gsub("\\","/")):gsub("^!+","")):gsub("//+","//")) + if str then + return ((str:gsub("\\","/")):gsub("^!+","")) + else + return nil + end +end + +function input.do_with_path(name,func) + for _, v in pairs(input.expanded_path_list(instance,name)) do + func("^"..input.clean_path(v)) + end +end + +function input.do_with_var(name,func) + func(input.aux.expanded_var(name)) +end + +function input.with_files(instance,pattern,handle) + for _, hash in ipairs(instance.hashes) do + local blobpath = hash.tag + local blobtype = hash.type + if blobpath then + local files = instance.files[blobpath] + if files then + for k,v in pairs(files) do + if k:find("^remap:") then + k = files[k] + v = files[k] -- chained + end + if k:find(pattern) then + if type(v) == "string" then + handle(blobtype,blobpath,v,k) + else + for _,vv in pairs(v) do + handle(blobtype,blobpath,vv,k) + end + end + end + end + end + end + end +end + +--~ function input.update_script(oldname,newname) -- oldname -> own.name, not per se a suffix +--~ newname = file.addsuffix(newname,"lua") +--~ local newscript = input.clean_path(input.find_file(instance, newname)) +--~ local oldscript = input.clean_path(oldname) +--~ input.report("old script", oldscript) +--~ input.report("new script", newscript) +--~ if oldscript ~= newscript and (oldscript:find(file.removesuffix(newname).."$") or oldscript:find(newname.."$")) then +--~ local newdata = io.loaddata(newscript) +--~ if newdata then +--~ input.report("old script content replaced by new content") +--~ io.savedata(oldscript,newdata) +--~ end +--~ end +--~ end + +function input.update_script(instance,oldname,newname) -- oldname -> own.name, not per se a suffix + local scriptpath = "scripts/context/lua" + newname = file.addsuffix(newname,"lua") + local oldscript = input.clean_path(oldname) + input.report("to be replaced old script", oldscript) + local newscripts = input.find_files(instance, newname) or { } + if #newscripts == 0 then + input.report("unable to locate new script") + else + for _, newscript in ipairs(newscripts) do + newscript = input.clean_path(newscript) + input.report("checking new script", newscript) + if oldscript == newscript then + input.report("old and new script are the same") + elseif not newscript:find(scriptpath) then + input.report("new script should come from",scriptpath) + elseif not (oldscript:find(file.removesuffix(newname).."$") or oldscript:find(newname.."$")) then + input.report("invalid new script name") + else + local newdata = io.loaddata(newscript) + if newdata then + input.report("old script content replaced by new content") + io.savedata(oldscript,newdata) + break + else + input.report("unable to load new script") + end + end + end + end +end + + +--~ print(table.serialize(input.aux.splitpathexpr("/usr/share/texmf-{texlive,tetex}", {}))) + +-- command line resolver: + +--~ print(input.resolve("abc env:tmp file:cont-en.tex path:cont-en.tex full:cont-en.tex rel:zapf/one/p-chars.tex")) + +do + + local resolvers = { } + + resolvers.environment = function(instance,str) + return input.clean_path(os.getenv(str) or os.getenv(str:upper()) or os.getenv(str:lower()) or "") + end + resolvers.relative = function(instance,str,n) + if io.exists(str) then + -- nothing + elseif io.exists("./" .. str) then + str = "./" .. str + else + local p = "../" + for i=1,n or 2 do + if io.exists(p .. str) then + str = p .. str + break + else + p = p .. "../" + end + end + end + return input.clean_path(str) + end + resolvers.locate = function(instance,str) + local fullname = input.find_given_file(instance,str) or "" + return input.clean_path((fullname ~= "" and fullname) or str) + end + resolvers.filename = function(instance,str) + local fullname = input.find_given_file(instance,str) or "" + return input.clean_path(file.basename((fullname ~= "" and fullname) or str)) + end + resolvers.pathname = function(instance,str) + local fullname = input.find_given_file(instance,str) or "" + return input.clean_path(file.dirname((fullname ~= "" and fullname) or str)) + end + + resolvers.env = resolvers.environment + resolvers.rel = resolvers.relative + resolvers.loc = resolvers.locate + resolvers.kpse = resolvers.locate + resolvers.full = resolvers.locate + resolvers.file = resolvers.filename + resolvers.path = resolvers.pathname + + local function resolve(instance,str) + if type(str) == "table" then + for k, v in pairs(str) do + str[k] = resolve(instance,v) or v + end + elseif str and str ~= "" then + str = str:gsub("([a-z]+):([^ ]+)", function(method,target) + if resolvers[method] then + return resolvers[method](instance,target) + else + return method .. ":" .. target + end + end) + end + return str + end + + input.resolve = resolve + +end + + +if not modules then modules = { } end modules ['luat-tmp'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +--[[ldx-- +<p>This module deals with caching data. It sets up the paths and +implements loaders and savers for tables. Best is to set the +following variable. When not set, the usual paths will be +checked. Personally I prefer the (users) temporary path.</p> + +</code> +TEXMFCACHE=$TMP;$TEMP;$TMPDIR;$TEMPDIR;$HOME;$TEXMFVAR;$VARTEXMF;. +</code> + +<p>Currently we do no locking when we write files. This is no real +problem because most caching involves fonts and the chance of them +being written at the same time is small. We also need to extend +luatools with a recache feature.</p> +--ldx]]-- + +caches = caches or { } +dir = dir or { } +texmf = texmf or { } + +caches.path = caches.path or nil +caches.base = caches.base or "luatex-cache" +caches.more = caches.more or "context" +caches.direct = false -- true is faster but may need huge amounts of memory +caches.trace = false +caches.tree = false +caches.paths = caches.paths or nil +caches.force = false + +input.usecache = not toboolean(os.getenv("TEXMFSHARECACHE") or "false",true) -- true + +function caches.temp(instance) + local function checkpath(cachepath) + if not cachepath or cachepath == "" then + return nil + elseif lfs.attributes(cachepath,"mode") == "directory" then -- lfs.isdir(cachepath) then + return cachepath + elseif caches.force or io.ask(string.format("Should I create the cache path %s?",cachepath), "no", { "yes", "no" }) == "yes" then + dir.mkdirs(cachepath) + return (lfs.attributes(cachepath,"mode") == "directory") and cachepath + else + return nil + end + end + local cachepath = input.expanded_path_list(instance,"TEXMFCACHE") + cachepath = cachepath and #cachepath > 0 and checkpath(cachepath[1]) + if not cachepath then + cachepath = os.getenv("TEXMFCACHE") or os.getenv("HOME") or os.getenv("HOMEPATH") or os.getenv("TMP") or os.getenv("TEMP") or os.getenv("TMPDIR") or nil + cachepath = checkpath(cachepath) + end + if not cachepath then + print("\nfatal error: there is no valid cache path defined\n") + os.exit() + elseif lfs.attributes(cachepath,"mode") ~= "directory" then + print(string.format("\nfatal error: cache path %s is not a directory\n",cachepath)) + os.exit() + end + function caches.temp(instance) + return cachepath + end + return cachepath +end + +function caches.configpath(instance) + return table.concat(instance.cnffiles,";") +end + +function caches.hashed(tree) + return md5.hex((tree:lower()):gsub("[\\\/]+","/")) +end + +function caches.treehash(instance) + local tree = caches.configpath(instance) + if not tree or tree == "" then + return false + else + return caches.hashed(tree) + end +end + +function caches.setpath(instance,...) + if not caches.path then + if not caches.path then + caches.path = caches.temp(instance) + end + caches.path = input.clean_path(caches.path) -- to be sure + if lfs then + caches.tree = caches.tree or caches.treehash(instance) + if caches.tree then + caches.path = dir.mkdirs(caches.path,caches.base,caches.more,caches.tree) + else + caches.path = dir.mkdirs(caches.path,caches.base,caches.more) + end + end + end + if not caches.path then + caches.path = '.' + end + caches.path = input.clean_path(caches.path) + if lfs and not table.is_empty({...}) then + local pth = dir.mkdirs(caches.path,...) + return pth + end + caches.path = dir.expand_name(caches.path) + return caches.path +end + +function caches.definepath(instance,category,subcategory) + return function() + return caches.setpath(instance,category,subcategory) + end +end + +function caches.setluanames(path,name) + return path .. "/" .. name .. ".tma", path .. "/" .. name .. ".tmc" +end + +function caches.loaddata(path,name) + local tmaname, tmcname = caches.setluanames(path,name) + local loader = loadfile(tmcname) or loadfile(tmaname) + if loader then + return loader() + else + return false + end +end + +function caches.is_writable(filepath,filename) + local tmaname, tmcname = caches.setluanames(filepath,filename) + return file.is_writable(tmaname) +end + +function caches.savedata(filepath,filename,data,raw) -- raw needed for file cache + local tmaname, tmcname = caches.setluanames(filepath,filename) + local reduce, simplify = true, true + if raw then + reduce, simplify = false, false + end + if caches.direct then + file.savedata(tmaname, table.serialize(data,'return',true,true)) + else + table.tofile (tmaname, data,'return',true,true) -- maybe not the last true + end + utils.lua.compile(tmaname, tmcname) +end + +-- here we use the cache for format loading (texconfig.[formatname|jobname]) + +--~ if tex and texconfig and texconfig.formatname and texconfig.formatname == "" then +if tex and texconfig and (not texconfig.formatname or texconfig.formatname == "") and texmf.instance then + if not texconfig.luaname then texconfig.luaname = "cont-en.lua" end -- or luc + texconfig.formatname = caches.setpath(texmf.instance,"formats") .. "/" .. texconfig.luaname:gsub("%.lu.$",".fmt") +end + +--[[ldx-- +<p>Once we found ourselves defining similar cache constructs +several times, containers were introduced. Containers are used +to collect tables in memory and reuse them when possible based +on (unique) hashes (to be provided by the calling function).</p> + +<p>Caching to disk is disabled by default. Version numbers are +stored in the saved table which makes it possible to change the +table structures without bothering about the disk cache.</p> + +<p>Examples of usage can be found in the font related code.</p> +--ldx]]-- + +containers = { } +containers.trace = false + +do -- local report + + local function report(container,tag,name) + if caches.trace or containers.trace or container.trace then + logs.report(string.format("%s cache",container.subcategory),string.format("%s: %s",tag,name or 'invalid')) + end + end + + local allocated = { } + + -- tracing + + function containers.define(category, subcategory, version, enabled) + return function() + if category and subcategory then + local c = allocated[category] + if not c then + c = { } + allocated[category] = c + end + local s = c[subcategory] + if not s then + s = { + category = category, + subcategory = subcategory, + storage = { }, + enabled = enabled, + version = version or 1.000, + trace = false, + path = caches.setpath(texmf.instance,category,subcategory), + } + c[subcategory] = s + end + return s + else + return nil + end + end + end + + function containers.is_usable(container, name) + return container.enabled and caches.is_writable(container.path, name) + end + + function containers.is_valid(container, name) + if name and name ~= "" then + local storage = container.storage[name] + return storage and not table.is_empty(storage) and storage.cache_version == container.version + else + return false + end + end + + function containers.read(container,name) + if container.enabled and not container.storage[name] then + container.storage[name] = caches.loaddata(container.path,name) + if containers.is_valid(container,name) then + report(container,"loaded",name) + else + container.storage[name] = nil + end + end + if container.storage[name] then + report(container,"reusing",name) + end + return container.storage[name] + end + + function containers.write(container, name, data) + if data then + data.cache_version = container.version + if container.enabled then + local unique, shared = data.unique, data.shared + data.unique, data.shared = nil, nil + caches.savedata(container.path, name, data) + report(container,"saved",name) + data.unique, data.shared = unique, shared + end + report(container,"stored",name) + container.storage[name] = data + end + return data + end + + function containers.content(container,name) + return container.storage[name] + end + +end + +-- since we want to use the cache instead of the tree, we will now +-- reimplement the saver. + +local save_data = input.aux.save_data + +input.cachepath = nil + +function input.aux.save_data(instance, dataname, check) + input.cachepath = input.cachepath or caches.definepath(instance,"trees") + save_data(instance, dataname, check, function(cachename,dataname) + if input.usecache then + return file.join(input.cachepath(),caches.hashed(cachename)) + else + return file.join(cachename,dataname) + end + end) +end + +local load_data = input.aux.load_data + +function input.aux.load_data(instance,pathname,dataname,filename) + input.cachepath = input.cachepath or caches.definepath(instance,"trees") + load_data(instance,pathname,dataname,filename,function(dataname,filename) + if input.usecache then + return file.join(input.cachepath(),caches.hashed(pathname)) + else + if not filename or (filename == "") then + filename = dataname + end + return file.join(pathname,filename) + end + end) +end + +-- we will make a better format, maybe something xml or just text or lua + +input.automounted = input.automounted or { } + +function input.automount(instance,usecache) + local mountpaths = input.simplified_list(input.expansion(instance,'TEXMFMOUNT')) + if table.is_empty(mountpaths) and usecache then + mountpaths = { caches.setpath(instance,"mount") } + end + if not table.is_empty(mountpaths) then + input.starttiming(instance) + for k, root in pairs(mountpaths) do + local f = io.open(root.."/url.tmi") + if f then + for line in f:lines() do + if line then + if line:find("^[%%#%-]") then -- or %W + -- skip + elseif line:find("^zip://") then + input.report("mounting",line) + table.insert(input.automounted,line) + input.usezipfile(instance,line) + end + end + end + f:close() + end + end + input.stoptiming(instance) + end +end + +-- store info in format + +input.storage = { } +input.storage.data = { } +input.storage.min = 0 -- 500 +input.storage.max = input.storage.min - 1 +input.storage.trace = false -- true +input.storage.done = 0 +input.storage.evaluators = { } +-- (evaluate,message,names) + +function input.storage.register(...) + input.storage.data[#input.storage.data+1] = { ... } +end + +function input.storage.evaluate(name) + input.storage.evaluators[#input.storage.evaluators+1] = name +end + +function input.storage.finalize() -- we can prepend the string with "evaluate:" + for _, t in ipairs(input.storage.evaluators) do + for i, v in pairs(t) do + if type(v) == "string" then + t[i] = loadstring(v)() + elseif type(v) == "table" then + for _, vv in pairs(v) do + if type(vv) == "string" then + t[i] = loadstring(vv)() + end + end + end + end + end +end + +function input.storage.dump() + for name, data in ipairs(input.storage.data) do + local evaluate, message, original, target = data[1], data[2], data[3] ,data[4] + local name, initialize, finalize, code = nil, "", "", "" + for str in target:gmatch("([^%.]+)") do + if name then + name = name .. "." .. str + else + name = str + end + initialize = string.format("%s %s = %s or {} ", initialize, name, name) + end + if evaluate then + finalize = "input.storage.evaluate(" .. name .. ")" + end + input.storage.max = input.storage.max + 1 + if input.storage.trace then + logs.report('storage',string.format('saving %s in slot %s',message,input.storage.max)) + code = + initialize .. + string.format("logs.report('storage','restoring %s from slot %s') ",message,input.storage.max) .. + table.serialize(original,name) .. + finalize + else + code = initialize .. table.serialize(original,name) .. finalize + end + lua.bytecode[input.storage.max] = loadstring(code) + end +end + +if lua.bytecode then -- from 0 upwards + local i = input.storage.min + while lua.bytecode[i] do + lua.bytecode[i]() + lua.bytecode[i] = nil + i = i + 1 + end + input.storage.done = i +end + + +-- filename : luat-zip.lua +-- comment : companion to luat-lib.tex +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['luat-zip'] = 1.001 + +if zip and input then + zip.supported = true +else + zip = { } + zip.supported = false +end + +if not zip.supported then + + if not input then input = { } end -- will go away + + function zip.openarchive (...) return nil end -- needed ? + function zip.closenarchive (...) end -- needed ? + function input.usezipfile (...) end -- needed ? + +else + + -- zip:///oeps.zip?name=bla/bla.tex + -- zip:///oeps.zip?tree=tex/texmf-local + + local function validzip(str) + if not str:find("^zip://") then + return "zip:///" .. str + else + return str + end + end + + zip.archives = { } + zip.registeredfiles = { } + + function zip.openarchive(instance,name) + if not name or name == "" then + return nil + else + local arch = zip.archives[name] + if arch then + return arch + else + local full = input.find_file(instance,name) or "" + local arch = (full ~= "" and zip.open(full)) or false + zip.archives[name] = arch + return arch + end + end + end + + function zip.closearchive(instance,name) + if not name or name == "" and zip.archives[name] then + zip.close(zip.archives[name]) + zip.archives[name] = nil + end + end + + -- zip:///texmf.zip?tree=/tex/texmf + -- zip:///texmf.zip?tree=/tex/texmf-local + -- zip:///texmf-mine.zip?tree=/tex/texmf-projects + + function input.locators.zip(instance,specification) -- where is this used? startup zips (untested) + specification = input.splitmethod(specification) + local zipfile = specification.path + local zfile = zip.openarchive(instance,name) -- tricky, could be in to be initialized tree + if zfile then + input.logger('! zip locator', specification.original ..' found') + else + input.logger('? zip locator', specification.original ..' not found') + end + end + + function input.hashers.zip(instance,tag,name) + input.report("loading zip file",name,"as",tag) + input.usezipfile(instance,tag .."?tree=" .. name) + end + + function input.concatinators.zip(tag,path,name) + if not path or path == "" then + return tag .. '?name=' .. name + else + return tag .. '?name=' .. path .. "/" .. name + end + end + + function input.is_readable.zip(name) + return true + end + + function input.finders.zip(instance,specification,filetype) + specification = input.splitmethod(specification) + if specification.path then + local q = url.query(specification.query) + if q.name then + local zfile = zip.openarchive(instance,specification.path) + if zfile then + input.logger('! zip finder',specification.path) + local dfile = zfile:open(q.name) + if dfile then + dfile = zfile:close() + input.logger('+ zip finder',q.name) + return specification.original + end + else + input.logger('? zip finder',specification.path) + end + end + end + input.logger('- zip finder',filename) + return unpack(input.finders.notfound) + end + + function input.openers.zip(instance,specification) + local zipspecification = input.splitmethod(specification) + if zipspecification.path then + local q = url.query(zipspecification.query) + if q.name then + local zfile = zip.openarchive(instance,zipspecification.path) + if zfile then + input.logger('+ zip starter',zipspecification.path) + local dfile = zfile:open(q.name) + if dfile then + input.show_open(specification) + return input.openers.text_opener(specification,dfile,'zip') + end + else + input.logger('- zip starter',zipspecification.path) + end + end + end + input.logger('- zip opener',filename) + return unpack(input.openers.notfound) + end + + function input.loaders.zip(instance,specification) + specification = input.splitmethod(specification) + if specification.path then + local q = url.query(specification.query) + if q.name then + local zfile = zip.openarchive(instance,specification.path) + if zfile then + input.logger('+ zip starter',specification.path) + local dfile = zfile:open(q.name) + if dfile then + input.show_load(filename) + input.logger('+ zip loader',filename) + local s = dfile:read("*all") + dfile:close() + return true, s, #s + end + else + input.logger('- zip starter',specification.path) + end + end + end + input.logger('- zip loader',filename) + return unpack(input.openers.notfound) + end + + -- zip:///somefile.zip + -- zip:///somefile.zip?tree=texmf-local -> mount + + function input.usezipfile(instance,zipname) + zipname = validzip(zipname) + input.logger('! zip use','file '..zipname) + local specification = input.splitmethod(zipname) + local zipfile = specification.path + if zipfile and not zip.registeredfiles[zipname] then + local tree = url.query(specification.query).tree or "" + input.logger('! zip register','file '..zipname) + local z = zip.openarchive(instance,zipfile) + if z then + input.logger("= zipfile","registering "..zipname) + input.starttiming(instance) + input.aux.prepend_hash(instance,'zip',zipname,zipfile) + input.aux.extend_texmf_var(instance,zipname) -- resets hashes too + zip.registeredfiles[zipname] = z + instance.files[zipname] = input.aux.register_zip_file(z,tree or "") + input.stoptiming(instance) + else + input.logger("? zipfile","unknown "..zipname) + end + else + input.logger('! zip register','no file '..zipname) + end + end + + function input.aux.register_zip_file(z,tree) + local files, filter = { }, "" + if tree == "" then + filter = "^(.+)/(.-)$" + else + filter = "^"..tree.."/(.+)/(.-)$" + end + input.logger('= zip filter',filter) + local register, n = input.aux.register_file, 0 + for i in z:files() do + local path, name = i.filename:match(filter) + if path then + if name and name ~= '' then + register(files, name, path) + n = n + 1 + else + -- directory + end + else + register(files, i.filename, '') + n = n + 1 + end + end + input.report('= zip entries',n) + return files + end + +end + + +-- filename : luat-zip.lua +-- comment : companion to luat-lib.tex +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['luat-tex'] = 1.001 + +-- special functions that deal with io + +if texconfig and not texlua then + + input.level = input.level or 0 + + if input.logmode() == 'xml' then + function input.show_open(name) + input.level = input.level + 1 + texio.write_nl("<f l='"..input.level.."' n='"..name.."'>") + end + function input.show_close(name) + texio.write("</f> ") + input.level = input.level - 1 + end + function input.show_load(name) + texio.write_nl("<f l='"..(input.level+1).."' n='"..name.."'/>") -- level? + end + else + function input.show_open () end + function input.show_close() end + function input.show_load () end + end + + function input.finders.generic(instance,tag,filename,filetype) + local foundname = input.find_file(instance,filename,filetype) + if foundname and foundname ~= "" then + input.logger('+ ' .. tag .. ' finder',filename,'filetype') + return foundname + else + input.logger('- ' .. tag .. ' finder',filename,'filetype') + return unpack(input.finders.notfound) + end + end + + input.filters.dynamic_translator = nil + input.filters.frozen_translator = nil + input.filters.utf_translator = nil + + function input.openers.text_opener(filename,file_handle,tag) + local u = unicode.utftype(file_handle) + local t = { } + if u > 0 then + input.logger('+ ' .. tag .. ' opener (' .. unicode.utfname[u] .. ')',filename) + local l + if u > 2 then + l = unicode.utf32_to_utf8(file_handle:read("*a"),u==4) + else + l = unicode.utf16_to_utf8(file_handle:read("*a"),u==2) + end + file_handle:close() + t = { + utftype = u, -- may go away + lines = l, + current = 0, -- line number, not really needed + handle = nil, + noflines = #l, + close = function() + input.logger('= ' .. tag .. ' closer (' .. unicode.utfname[u] .. ')',filename) + input.show_close(filename) + end, +--~ getline = function(n) +--~ local line = t.lines[n] +--~ if not line or line == "" then +--~ return "" +--~ else +--~ local translator = input.filters.utf_translator +--~ return (translator and translator(line)) or line +--~ end +--~ end, + reader = function(self) + self = self or t + local current, lines = self.current, self.lines + if current >= #lines then + return nil + else + current = current + 1 + self.current = current + local line = lines[current] + if line == "" then + return "" + else + local translator = input.filters.utf_translator + -- return (translator and translator(line)) or line + if translator then + return translator(line) + else + return line + end + end + end + end + } + else + input.logger('+ ' .. tag .. ' opener',filename) + -- todo: file;name -> freeze / eerste regel scannen -> freeze + local filters = input.filters + t = { + reader = function(self) + local line = file_handle:read() + if line == "" then + return "" + end + local translator = filters.utf_translator + if translator then + return translator(line) + end + translator = filters.dynamic_translator + if translator then + return translator(line) + end + return line + end, + close = function() + input.logger('= ' .. tag .. ' closer',filename) + input.show_close(filename) + file_handle:close() + end, + handle = function() + return file_handle + end, + noflines = function() + t.noflines = io.noflines(file_handle) + return t.noflines + end + } + end + return t + end + + function input.openers.generic(instance,tag,filename) + if filename and filename ~= "" then + local f = io.open(filename,"r") + if f then + input.show_open(filename) + return input.openers.text_opener(filename,f,tag) + end + end + input.logger('- ' .. tag .. ' opener',filename) + return unpack(input.openers.notfound) + end + + function input.loaders.generic(instance,tag,filename) + if filename and filename ~= "" then + local f = io.open(filename,"rb") + if f then + input.show_load(filename) + input.logger('+ ' .. tag .. ' loader',filename) + local s = f:read("*a") + f:close() + if s then + return true, s, #s + end + end + end + input.logger('- ' .. tag .. ' loader',filename) + return unpack(input.loaders.notfound) + end + + function input.finders.tex(instance,filename,filetype) + return input.finders.generic(instance,'tex',filename,filetype) + end + function input.openers.tex(instance,filename) + return input.openers.generic(instance,'tex',filename) + end + function input.loaders.tex(instance,filename) + return input.loaders.generic(instance,'tex',filename) + end + +end + +-- callback into the file io and related things; disabling kpse + + +if texconfig and not texlua then do + + -- this is not the right place, because we refer to quite some not yet defined tables, but who cares ... + + ctx = ctx or { } + + local ss = { } + + function ctx.writestatus(a,b) + local s = ss[a] + if not ss[a] then + s = a:rpadd(15) .. ": " + ss[a] = s + end + texio.write_nl(s .. b .. "\n") + end + + -- this will become: ctx.install_statistics(fnc() return ..,.. end) etc + + local statusinfo, n = { }, 0 + + function ctx.register_statistics(tag,pattern,fnc) + statusinfo[#statusinfo+1] = { tag, pattern, fnc } + if #tag > n then n = #tag end + end + + function ctx.show_statistics() -- todo: move calls + if caches then + ctx.register_statistics("used config path", "%s", function() return caches.configpath(texmf.instance) end) + ctx.register_statistics("used cache path", "%s", function() return caches.path end) + end + if status.luabytecodes > 0 and input.storage and input.storage.done then + ctx.register_statistics("modules/dumps/instances", "%s/%s/%s", function() return status.luabytecodes-500, input.storage.done, status.luastates end) + end + if texmf.instance then + ctx.register_statistics("input load time", "%s seconds", function() return input.loadtime(texmf.instance) end) + end + if fonts then + ctx.register_statistics("fonts load time","%s seconds", function() return input.loadtime(fonts) end) + end + if xml then + ctx.register_statistics("xml load time", "%s seconds, backreferences: %i, outer filtering time: %s", function() return input.loadtime(xml), #lxml.self, input.loadtime(lxml) end) + end + if mptopdf then + ctx.register_statistics("mps conversion time", "%s seconds", function() return input.loadtime(mptopdf) end) + end + if nodes then + ctx.register_statistics("node processing time", "%s seconds (including kernel)", function() return input.loadtime(nodes) end) + end + if kernel then + ctx.register_statistics("kernel processing time", "%s seconds", function() return input.loadtime(kernel) end) + end + if attributes then + ctx.register_statistics("attribute processing time", "%s seconds", function() return input.loadtime(attributes) end) + end + if languages then + ctx.register_statistics("language load time", "%s seconds, n=%s", function() return input.loadtime(languages), languages.hyphenation.n() end) + end + if figures then + ctx.register_statistics("graphics processing time", "%s seconds, n=%s (including tex)", function() return input.loadtime(figures), figures.n or "?" end) + end + if metapost then + ctx.register_statistics("metapost processing time", "%s seconds, loading: %s seconds, execution: %s seconds, n: %s", function() return input.loadtime(metapost), input.loadtime(mplib), input.loadtime(metapost.exectime), metapost.n end) + end + if status.luastate_bytes then + ctx.register_statistics("current memory usage", "%s bytes", function() return status.luastate_bytes end) + end + if nodes then + ctx.register_statistics("cleaned up reserved nodes", "%s nodes, %s lists of %s", function() return nodes.cleanup_reserved(tex.count[24]) end) -- \topofboxstack + end + if status.node_mem_usage then + ctx.register_statistics("node memory usage", "%s", function() return status.node_mem_usage end) + end + if languages then + ctx.register_statistics("loaded patterns", "%s", function() return languages.logger.report() end) + end + if fonts then + ctx.register_statistics("loaded fonts", "%s", function() return fonts.logger.report() end) + end + if xml then -- so we are in mkiv, we need a different check + ctx.register_statistics("runtime", "%s seconds, %i processed pages, %i shipped pages, %.3f pages/second", function() + input.stoptiming(texmf) + local runtime = input.loadtime(texmf) + local shipped = tex.count['nofshipouts'] + local pages = tex.count['realpageno'] - 1 + local persecond = shipped / runtime + return runtime, pages, shipped, persecond + end) + end + for _, t in ipairs(statusinfo) do + local tag, pattern, fnc = t[1], t[2], t[3] + ctx.writestatus("mkiv lua stats", string.format("%s - %s", tag:rpadd(n," "), pattern:format(fnc()))) + end + end + +end end + +if texconfig and not texlua then + + texconfig.kpse_init = false + texconfig.trace_file_names = input.logmode() == 'tex' + texconfig.max_print_line = 100000 + + -- if still present, we overload kpse (put it off-line so to say) + + if not texmf then texmf = { } end + + input.starttiming(texmf) + + if not texmf.instance then + + if not texmf.instance then -- prevent a second loading + + texmf.instance = input.reset() + texmf.instance.progname = environment.progname or 'context' + texmf.instance.engine = environment.engine or 'luatex' + texmf.instance.validfile = input.validctxfile + + input.load(texmf.instance) + + end + + if callback then + callback.register('find_read_file' , function(id,name) return input.findtexfile(texmf.instance,name) end) + callback.register('open_read_file' , function( name) return input.opentexfile(texmf.instance,name) end) + end + + if callback then + callback.register('find_data_file' , function(name) return input.findbinfile(texmf.instance,name,"tex") end) + callback.register('find_enc_file' , function(name) return input.findbinfile(texmf.instance,name,"enc") end) + callback.register('find_font_file' , function(name) return input.findbinfile(texmf.instance,name,"tfm") end) + callback.register('find_format_file' , function(name) return input.findbinfile(texmf.instance,name,"fmt") end) + callback.register('find_image_file' , function(name) return input.findbinfile(texmf.instance,name,"tex") end) + callback.register('find_map_file' , function(name) return input.findbinfile(texmf.instance,name,"map") end) + callback.register('find_ocp_file' , function(name) return input.findbinfile(texmf.instance,name,"ocp") end) + callback.register('find_opentype_file' , function(name) return input.findbinfile(texmf.instance,name,"otf") end) + callback.register('find_output_file' , function(name) return name end) + callback.register('find_pk_file' , function(name) return input.findbinfile(texmf.instance,name,"pk") end) + callback.register('find_sfd_file' , function(name) return input.findbinfile(texmf.instance,name,"sfd") end) + callback.register('find_truetype_file' , function(name) return input.findbinfile(texmf.instance,name,"ttf") end) + callback.register('find_type1_file' , function(name) return input.findbinfile(texmf.instance,name,"pfb") end) + callback.register('find_vf_file' , function(name) return input.findbinfile(texmf.instance,name,"vf") end) + + callback.register('read_data_file' , function(file) return input.loadbinfile(texmf.instance,file,"tex") end) + callback.register('read_enc_file' , function(file) return input.loadbinfile(texmf.instance,file,"enc") end) + callback.register('read_font_file' , function(file) return input.loadbinfile(texmf.instance,file,"tfm") end) + -- format + -- image + callback.register('read_map_file' , function(file) return input.loadbinfile(texmf.instance,file,"map") end) + callback.register('read_ocp_file' , function(file) return input.loadbinfile(texmf.instance,file,"ocp") end) + callback.register('read_opentype_file' , function(file) return input.loadbinfile(texmf.instance,file,"otf") end) + -- output + callback.register('read_pk_file' , function(file) return input.loadbinfile(texmf.instance,file,"pk") end) + callback.register('read_sfd_file' , function(file) return input.loadbinfile(texmf.instance,file,"sfd") end) + callback.register('read_truetype_file' , function(file) return input.loadbinfile(texmf.instance,file,"ttf") end) + callback.register('read_type1_file' , function(file) return input.loadbinfile(texmf.instance,file,"pfb") end) + callback.register('read_vf_file' , function(file) return input.loadbinfile(texmf.instance,file,"vf" ) end) + end + + if callback and environment.aleph_mode then + callback.register('find_font_file' , function(name) return input.findbinfile(texmf.instance,name,"ofm") end) + callback.register('read_font_file' , function(file) return input.loadbinfile(texmf.instance,file,"ofm") end) + callback.register('find_vf_file' , function(name) return input.findbinfile(texmf.instance,name,"ovf") end) + callback.register('read_vf_file' , function(file) return input.loadbinfile(texmf.instance,file,"ovf") end) + end + + if callback then + callback.register('find_write_file' , function(id,name) return name end) + end + + if callback and (not config or (#config == 0)) then + callback.register('find_format_file' , function(name) return name end) + end + + if callback and false then + for k, v in pairs(callback.list()) do + if not v then texio.write_nl("<w>callback "..k.." is not set</w>") end + end + end + + if callback then + + input.start_actions = { } + input.stop_actions = { } + + function input.register_start_actions(f) table.insert(input.start_actions, f) end + function input.register_stop_actions (f) table.insert(input.stop_actions, f) end + + --~ callback.register('start_run', function() for _, a in pairs(input.start_actions) do a() end end) + --~ callback.register('stop_run' , function() for _, a in pairs(input.stop_actions ) do a() end end) + + end + + if callback then + + if input.logmode() == 'xml' then + + function input.start_page_number() + texio.write_nl("<p real='" .. tex.count[0] .. "' page='"..tex.count[1].."' sub='"..tex.count[2].."'") + end + function input.stop_page_number() + texio.write("/>") + texio.write_nl("") + end + + callback.register('start_page_number' , input.start_page_number) + callback.register('stop_page_number' , input.stop_page_number ) + + function input.report_output_pages(p,b) + texio.write_nl("<v k='pages'>"..p.."</v>") + texio.write_nl("<v k='bytes'>"..b.."</v>") + texio.write_nl("") + end + function input.report_output_log() + end + + callback.register('report_output_pages', input.report_output_pages) + callback.register('report_output_log' , input.report_output_log ) + + function input.start_run() + texio.write_nl("<?xml version='1.0' standalone='yes'?>") + texio.write_nl("<job xmlns='www.tug.org/luatex/schemas/context-job.rng'>") + texio.write_nl("") + end + function input.stop_run() + texio.write_nl("</job>") + end + function input.show_statistics() + for k,v in pairs(status.list()) do + texio.write_nl("log","<v k='"..k.."'>"..tostring(v).."</v>") + end + end + + table.insert(input.start_actions, input.start_run) + table.insert(input.stop_actions , input.show_statistics) + table.insert(input.stop_actions , input.stop_run) + + else + table.insert(input.stop_actions , input.show_statistics) + end + + callback.register('start_run', function() for _, a in pairs(input.start_actions) do a() end end) + callback.register('stop_run' , function() for _, a in pairs(input.stop_actions ) do a() end ctx.show_statistics() end) + + end + + end + + if kpse then + + function kpse.find_file(filename,filetype,mustexist) + return input.find_file(texmf.instance,filename,filetype,mustexist) + end + function kpse.expand_path(variable) + return input.expand_path(texmf.instance,variable) + end + function kpse.expand_var(variable) + return input.expand_var(texmf.instance,variable) + end + function kpse.expand_braces(variable) + return input.expand_braces(texmf.instance,variable) + end + + end + +end + +-- program specific configuration (memory settings and alike) + +if texconfig and not texlua then + + luatex = luatex or { } + + luatex.variablenames = { + 'main_memory', 'extra_mem_bot', 'extra_mem_top', + 'buf_size','expand_depth', + 'font_max', 'font_mem_size', + 'hash_extra', 'max_strings', 'pool_free', 'pool_size', 'string_vacancies', + 'obj_tab_size', 'pdf_mem_size', 'dest_names_size', + 'nest_size', 'param_size', 'save_size', 'stack_size', + 'trie_size', 'hyph_size', 'max_in_open', + 'ocp_stack_size', 'ocp_list_size', 'ocp_buf_size' + } + + function luatex.variables() + local t, x = { }, nil + for _,v in pairs(luatex.variablenames) do + x = input.var_value(texmf.instance,v) + if x and x:find("^%d+$") then + t[v] = tonumber(x) + end + end + return t + end + + function luatex.setvariables(tab) + for k,v in pairs(luatex.variables()) do + tab[k] = v + end + end + + if not luatex.variables_set then + luatex.setvariables(texconfig) + luatex.variables_set = true + end + + texconfig.max_print_line = 100000 + texconfig.max_in_open = 127 + +end + +-- some tex basics + +if not cs then cs = { } end + +function cs.def(k,v) + tex.sprint(tex.texcatcodes, "\\def\\" .. k .. "{" .. v .. "}") +end + +function cs.chardef(k,v) + tex.sprint(tex.texcatcodes, "\\chardef\\" .. k .. "=" .. v .. "\\relax") +end + +function cs.boolcase(b) + if b then tex.write(1) else tex.write(0) end +end + +function cs.testcase(b) + if b then + tex.sprint(tex.texcatcodes, "\\firstoftwoarguments") + else + tex.sprint(tex.texcatcodes, "\\secondoftwoarguments") + end +end + + +if not modules then modules = { } end modules ['luat-kps'] = { + version = 1.001, + comment = "companion to luatools.lua", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +--[[ldx-- +<p>This file is used when we want the input handlers to behave like +<type>kpsewhich</type>. What to do with the following:</p> + +<typing> +{$SELFAUTOLOC,$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,}/web2c} +$SELFAUTOLOC : /usr/tex/bin/platform +$SELFAUTODIR : /usr/tex/bin +$SELFAUTOPARENT : /usr/tex +</typing> + +<p>How about just forgetting abou them?</p> +--ldx]]-- + +input = input or { } +input.suffixes = input.suffixes or { } +input.formats = input.formats or { } + +input.suffixes['gf'] = { '<resolution>gf' } +input.suffixes['pk'] = { '<resolution>pk' } +input.suffixes['base'] = { 'base' } +input.suffixes['bib'] = { 'bib' } +input.suffixes['bst'] = { 'bst' } +input.suffixes['cnf'] = { 'cnf' } +input.suffixes['mem'] = { 'mem' } +input.suffixes['mf'] = { 'mf' } +input.suffixes['mfpool'] = { 'pool' } +input.suffixes['mft'] = { 'mft' } +input.suffixes['mppool'] = { 'pool' } +input.suffixes['graphic/figure'] = { 'eps', 'epsi' } +input.suffixes['texpool'] = { 'pool' } +input.suffixes['PostScript header'] = { 'pro' } +input.suffixes['ist'] = { 'ist' } +input.suffixes['web'] = { 'web', 'ch' } +input.suffixes['cweb'] = { 'w', 'web', 'ch' } +input.suffixes['cmap files'] = { 'cmap' } +input.suffixes['lig files'] = { 'lig' } +input.suffixes['bitmap font'] = { } +input.suffixes['MetaPost support'] = { } +input.suffixes['TeX system documentation'] = { } +input.suffixes['TeX system sources'] = { } +input.suffixes['dvips config'] = { } +input.suffixes['type42 fonts'] = { } +input.suffixes['web2c files'] = { } +input.suffixes['other text files'] = { } +input.suffixes['other binary files'] = { } +input.suffixes['opentype fonts'] = { 'otf' } + +input.suffixes['fmt'] = { 'fmt' } +input.suffixes['texmfscripts'] = { 'rb','lua','py','pl' } + +input.suffixes['pdftex config'] = { } +input.suffixes['Troff fonts'] = { } + +input.suffixes['ls-R'] = { } + +--[[ldx-- +<p>If you wondered abou tsome of the previous mappings, how about +the next bunch:</p> +--ldx]]-- + +input.formats['bib'] = '' +input.formats['bst'] = '' +input.formats['mft'] = '' +input.formats['ist'] = '' +input.formats['web'] = '' +input.formats['cweb'] = '' +input.formats['MetaPost support'] = '' +input.formats['TeX system documentation'] = '' +input.formats['TeX system sources'] = '' +input.formats['Troff fonts'] = '' +input.formats['dvips config'] = '' +input.formats['graphic/figure'] = '' +input.formats['ls-R'] = '' +input.formats['other text files'] = '' +input.formats['other binary files'] = '' + +input.formats['gf'] = '' +input.formats['pk'] = '' +input.formats['base'] = 'MFBASES' +input.formats['cnf'] = '' +input.formats['mem'] = 'MPMEMS' +input.formats['mf'] = 'MFINPUTS' +input.formats['mfpool'] = 'MFPOOL' +input.formats['mppool'] = 'MPPOOL' +input.formats['texpool'] = 'TEXPOOL' +input.formats['PostScript header'] = 'TEXPSHEADERS' +input.formats['cmap files'] = 'CMAPFONTS' +input.formats['type42 fonts'] = 'T42FONTS' +input.formats['web2c files'] = 'WEB2C' +input.formats['pdftex config'] = 'PDFTEXCONFIG' +input.formats['texmfscripts'] = 'TEXMFSCRIPTS' +input.formats['bitmap font'] = '' +input.formats['lig files'] = 'LIGFONTS' + +-- end library merge + +-- We initialize some characteristics of this program. We need to +-- do this before we load the libraries, else own.name will not be +-- properly set (handy for selfcleaning the file). It's an ugly +-- looking piece of code. + +own = { } + +own.libs = { -- todo: check which ones are really needed + 'l-string.lua', + 'l-lpeg.lua', + 'l-table.lua', + 'l-io.lua', + 'l-number.lua', + 'l-set.lua', + 'l-os.lua', + 'l-md5.lua', + 'l-file.lua', + 'l-url.lua', + 'l-dir.lua', + 'l-boolean.lua', + 'l-unicode.lua', + 'l-utils.lua', + 'luat-lib.lua', + 'luat-inp.lua', + 'luat-tmp.lua', + 'luat-zip.lua', + 'luat-tex.lua', + 'luat-kps.lua', +} + +-- We need this hack till luatex is fixed. + +if arg and arg[0] == 'luatex' and arg[1] == "--luaonly" then + arg[-1]=arg[0] arg[0]=arg[2] for k=3,#arg do arg[k-2]=arg[k] end arg[#arg]=nil arg[#arg]=nil +end + +-- End of hack. + +own.name = (environment and environment.ownname) or arg[0] or 'luatools.lua' +own.path = string.match(own.name,"^(.+)[\\/].-$") or "." +own.list = { '.' } + +if own.path ~= '.' then + table.insert(own.list,own.path) +end + +table.insert(own.list,own.path.."/../../../tex/context/base") +table.insert(own.list,own.path.."/mtx") +table.insert(own.list,own.path.."/../sources") + +function locate_libs() + for _, lib in pairs(own.libs) do + for _, pth in pairs(own.list) do + local filename = string.gsub(pth .. "/" .. lib,"\\","/") + local codeblob = loadfile(filename) + if codeblob then + codeblob() + own.list = { pth } -- speed up te search + break + end + end + end +end + +if not input then + locate_libs() +end + +if not input then + print("") + print("Luatools is unable to start up due to lack of libraries. You may") + print("try to run 'lua luatools.lua --selfmerge' in the path where this") + print("script is located (normally under ..../scripts/context/lua) which") + print("will make luatools library independent.") + os.exit() +end + +instance = input.reset() +input.verbose = environment.arguments["verbose"] or false +input.banner = 'LuaTools | ' +utils.report = input.report + +input.defaultlibs = { -- not all are needed + 'l-string.lua', 'l-lpeg.lua', 'l-table.lua', 'l-boolean.lua', 'l-number.lua', 'l-set.lua', 'l-unicode.lua', + 'l-md5.lua', 'l-os.lua', 'l-io.lua', 'l-file.lua', 'l-url.lua', 'l-dir.lua', 'l-utils.lua', 'l-tex.lua', + 'luat-env.lua', 'luat-lib.lua', 'luat-inp.lua', 'luat-tmp.lua', 'luat-zip.lua', 'luat-tex.lua' +} + +-- todo: use environment.argument() instead of environment.arguments[] + +instance.engine = environment.arguments["engine"] or 'luatex' +instance.progname = environment.arguments["progname"] or 'context' +instance.luaname = environment.arguments["luafile"] or "" -- environment.ownname or "" +instance.lualibs = environment.arguments["lualibs"] or table.concat(input.defaultlibs,",") +instance.allresults = environment.arguments["all"] or false +instance.pattern = environment.arguments["pattern"] or nil +instance.sortdata = environment.arguments["sort"] or false +instance.kpseonly = not environment.arguments["all"] or false +instance.my_format = environment.arguments["format"] or instance.format +instance.lsrmode = environment.arguments["lsr"] or false + +if type(instance.pattern) == 'boolean' then + input.report("invalid pattern specification") -- toto, force verbose for one message + instance.pattern = nil +end + +if environment.arguments["trace"] then input.settrace(environment.arguments["trace"]) end + +if environment.arguments["minimize"] then + if input.validators.visibility[instance.progname] then + instance.validfile = input.validators.visibility[instance.progname] + end +end + +function input.my_prepare_a(instance) + input.resetconfig(instance) + input.identify_cnf(instance) + input.load_lua(instance) + input.expand_variables(instance) + input.load_cnf(instance) + input.expand_variables(instance) +end + +function input.my_prepare_b(instance) + input.my_prepare_a(instance) + input.load_hash(instance) + input.automount(instance) +end + +-- barename + +if not messages then messages = { } end + +messages.no_ini_file = [[ +There is no lua initialization file found. This file can be forced by the +"--progname" directive, or specified with "--luaname", or it is derived +automatically from the formatname (aka jobname). It may be that you have +to regenerate the file database using "luatools --generate". +]] + +messages.help = [[ +--generate generate file database +--variables show configuration variables +--expansions show expanded variables +--configurations show configuration order +--expand-braces expand complex variable +--expand-path expand variable (resolve paths) +--expand-var expand variable (resolve references) +--show-path show path expansion of ... +--var-value report value of variable +--find-file report file location +--find-path report path of file +--make or --ini make luatex format +--run or --fmt= run luatex format +--luafile=str lua inifile (default is <progname>.lua) +--lualibs=list libraries to assemble (optional when --compile) +--compile assemble and compile lua inifile +--mkii force context mkii mode (only for testing, not usable!) +--verbose give a bit more info +--minimize optimize lists for format +--all show all found files +--sort sort cached data +--engine=str target engine +--progname=str format or backend +--pattern=str filter variables +--lsr use lsr and cnf directly +]] + +function input.my_make_format(instance,texname) + if texname and texname ~= "" then + if input.usecache then + local path = file.join(caches.setpath(instance,"formats")) -- maybe platform + if path and lfs then + lfs.chdir(path) + end + end + local barename = texname:gsub("%.%a+$","") + if barename == texname then + texname = texname .. ".tex" + end + local fullname = input.find_files(instance,texname)[1] or "" + if fullname == "" then + input.report("no tex file with name",texname) + else + local luaname, lucname, luapath, lualibs = "", "", "", { } + -- the following is optional, since context.lua can also + -- handle this collect and compile business + if environment.arguments["compile"] then + if luaname == "" then luaname = barename end + input.report("creating initialization file " .. luaname) + luapath = file.dirname(luaname) + if luapath == "" then + luapath = file.dirname(texname) + end + if luapath == "" then + luapath = file.dirname(input.find_files(instance,texname)[1] or "") + end + lualibs = string.split(instance.lualibs,",") + luaname = file.basename(barename .. ".lua") + lucname = file.basename(barename .. ".luc") + -- todo: when this fails, we can just copy the merged libraries from + -- luatools since they are normally the same, at least for context + if lualibs[1] then + local firstlib = file.join(luapath,lualibs[1]) + if not lfs.isfile(firstlib) then + local foundname = input.find_files(instance,lualibs[1])[1] + if foundname then + input.report("located library path : " .. luapath) + luapath = file.dirname(foundname) + end + end + end + input.report("using library path : " .. luapath) + input.report("using lua libraries: " .. table.join(lualibs," ")) + utils.merger.selfcreate(lualibs,luapath,luaname) + if utils.lua.compile(luaname, lucname) and io.exists(lucname) then + luaname = lucname + input.report("using compiled initialization file " .. lucname) + else + input.report("using uncompiled initialization file " .. luaname) + end + else + for _, v in pairs({instance.luaname, instance.progname, barename}) do + v = string.gsub(v..".lua","%.lua%.lua$",".lua") + if v and (v ~= "") then + luaname = input.find_files(instance,v)[1] or "" + if luaname ~= "" then + break + end + end + end + end + if luaname == "" then + input.reportlines(messages.no_ini_file) + input.report("texname : " .. texname) + input.report("luaname : " .. instance.luaname) + input.report("progname : " .. instance.progname) + input.report("barename : " .. barename) + else + input.report("using lua initialization file " .. luaname) + local flags = { "--ini" } + if environment.arguments["mkii"] then + flags[#flags+1] = "--progname=" .. instance.progname + else + flags[#flags+1] = "--lua=" .. string.quote(luaname) + end + local bs = (environment.platform == "unix" and "\\\\") or "\\" -- todo: make a function + local command = "luatex ".. table.concat(flags," ") .. " " .. string.quote(fullname) .. " " .. bs .. "dump" + input.report("running command: " .. command .. "\n") + os.spawn(command) + end + end + else + input.report("no tex file given") + end +end + +function input.my_run_format(instance,name,data,more) + -- hm, rather old code here; we can now use the file.whatever functions + if name and (name ~= "") then + local barename = name:gsub("%.%a+$","") + local fmtname = "" + if input.usecache then + local path = file.join(caches.setpath(instance,"formats")) -- maybe platform + fmtname = file.join(path,barename..".fmt") or "" + end + if fmtname == "" then + fmtname = input.find_files(instance,barename..".fmt")[1] or "" + end + fmtname = input.clean_path(fmtname) + barename = fmtname:gsub("%.%a+$","") + if fmtname == "" then + input.report("no format with name",name) + else + local luaname = barename .. ".luc" + local f = io.open(luaname) + if not f then + luaname = barename .. ".lua" + f = io.open(luaname) + end + if f then + f:close() + local command = "luatex --fmt=" .. string.quote(barename) .. " --lua=" .. string.quote(luaname) .. " " .. string.quote(data) .. " " .. string.quote(more) + input.report("running command: " .. command) + os.spawn(command) + else + input.report("using format name",fmtname) + input.report("no luc/lua with name",barename) + end + end + end +end + +-- helpers for verbose lists + +input.listers = input.listers or { } + +local function tabstr(str) + if type(str) == 'table' then + return table.concat(str," | ") + else + return str + end +end + +local function list(instance,list) + local pat = string.upper(instance.pattern or "","") + for _,key in pairs(table.sortedkeys(list)) do + if instance.pattern == "" or string.find(key:upper(),pat) then + if instance.kpseonly then + if instance.kpsevars[key] then + print(format("%s=%s",key,tabstr(list[key]))) + end + else + print(format('%s %s=%s',(instance.kpsevars[key] and 'K') or 'E',key,tabstr(list[key]))) + end + end + end +end + +function input.listers.variables (instance) list(instance,instance.variables ) end +function input.listers.expansions(instance) list(instance,instance.expansions) end + +function input.listers.configurations(instance) + for _,key in pairs(table.sortedkeys(instance.kpsevars)) do + if not instance.pattern or (instance.pattern=="") or key:find(instance.pattern) then + print(key.."\n") + for i,c in ipairs(instance.order) do + local str = c[key] + if str then + print(format("\t%s\t\t%s",i,input.aux.tabstr(str))) + end + end + print() + end + end +end + +input.report(banner,"\n") + +local ok = true + +if environment.arguments["find-file"] then + input.my_prepare_b(instance) + instance.format = environment.arguments["format"] or instance.format + if instance.pattern then + instance.allresults = true + input.for_files(instance, input.find_files, { instance.pattern }, instance.my_format) + else + input.for_files(instance, input.find_files, environment.files, instance.my_format) + end +elseif environment.arguments["find-path"] then + input.my_prepare_b(instance) + local path = input.find_file(instance, environment.files[1], instance.my_format) + if input.verbose then + input.report(file.dirname(path)) + else + print(file.dirname(path)) + end +--~ elseif environment.arguments["first-writable-path"] then +--~ input.my_prepare_b(instance) +--~ input.report(input.first_writable_path(instance,environment.files[1] or ".")) +elseif environment.arguments["run"] then + input.my_prepare_a(instance) -- ! no need for loading databases + input.verbose = true + input.my_run_format(instance,environment.files[1] or "",environment.files[2] or "",environment.files[3] or "") +elseif environment.arguments["fmt"] then + input.my_prepare_a(instance) -- ! no need for loading databases + input.verbose = true + input.my_run_format(instance,environment.arguments["fmt"], environment.files[1] or "",environment.files[2] or "") +elseif environment.arguments["expand-braces"] then + input.my_prepare_a(instance) + input.for_files(instance, input.expand_braces, environment.files) +elseif environment.arguments["expand-path"] then + input.my_prepare_a(instance) + input.for_files(instance, input.expand_path, environment.files) +elseif environment.arguments["expand-var"] or environment.arguments["expand-variable"] then + input.my_prepare_a(instance) + input.for_files(instance, input.expand_var, environment.files) +elseif environment.arguments["show-path"] or environment.arguments["path-value"] then + input.my_prepare_a(instance) + input.for_files(instance, input.show_path, environment.files) +elseif environment.arguments["var-value"] or environment.arguments["show-value"] then + input.my_prepare_a(instance) + input.for_files(instance, input.var_value, environment.files) +elseif environment.arguments["format-path"] then + input.my_prepare_b(instance) + input.report(caches.setpath(instance,"format")) +elseif instance.pattern then -- brrr + input.my_prepare_b(instance) + instance.format = environment.arguments["format"] or instance.format + instance.allresults = true + input.for_files(instance, input.find_files, { instance.pattern }, instance.my_format) +elseif environment.arguments["generate"] then + instance.renewcache = true + input.verbose = true + input.my_prepare_b(instance) +elseif environment.arguments["make"] or environment.arguments["ini"] or environment.arguments["compile"] then + input.my_prepare_b(instance) + input.verbose = true + input.my_make_format(instance,environment.files[1] or "") +elseif environment.arguments["selfmerge"] then + utils.merger.selfmerge(own.name,own.libs,own.list) +elseif environment.arguments["selfclean"] then + utils.merger.selfclean(own.name) +elseif environment.arguments["selfupdate"] then + input.my_prepare_b(instance) + input.verbose = true + input.update_script(instance,own.name,"luatools") +elseif environment.arguments["variables"] or environment.arguments["show-variables"] then + input.my_prepare_a(instance) + input.listers.variables(instance) +elseif environment.arguments["expansions"] or environment.arguments["show-expansions"] then + input.my_prepare_a(instance) + input.listers.expansions(instance) +elseif environment.arguments["configurations"] or environment.arguments["show-configurations"] then + input.my_prepare_a(instance) + input.listers.configurations(instance) +elseif environment.arguments["help"] or (environment.files[1]=='help') or (#environment.files==0) then + if not input.verbose then + input.verbose = true + input.report(banner,"\n") + end + input.reportlines(messages.help) +else + input.my_prepare_b(instance) + input.for_files(instance, input.find_files, environment.files, instance.my_format) +end + +if input.verbose then + input.report("") + input.report(string.format("runtime: %0.3f seconds",os.runtime())) +end + +--~ if ok then +--~ input.report("exit code: 0") os.exit(0) +--~ else +--~ input.report("exit code: 1") os.exit(1) +--~ end + +if environment.platform == "unix" then + io.write("\n") +end diff --git a/Master/texmf-dist/scripts/context/stubs/unix/mtxrun b/Master/texmf-dist/scripts/context/stubs/unix/mtxrun new file mode 100755 index 00000000000..040214178b8 --- /dev/null +++ b/Master/texmf-dist/scripts/context/stubs/unix/mtxrun @@ -0,0 +1,8549 @@ +#!/usr/bin/env texlua + +if not modules then modules = { } end modules ['mtxrun'] = { + version = 1.001, + comment = "runner, lua replacement for texmfstart.rb", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- one can make a stub: +-- +-- #!/bin/sh +-- env LUATEXDIR=/....../texmf/scripts/context/lua luatex --luaonly mtxrun.lua "$@" + +-- filename : mtxrun.lua +-- comment : companion to context.tex +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +-- This script is based on texmfstart.rb but does not use kpsewhich to +-- locate files. Although kpse is a library it never came to opening up +-- its interface to other programs (esp scripting languages) and so we +-- do it ourselves. The lua variant evolved out of an experimental ruby +-- one. Interesting is that using a scripting language instead of c does +-- not have a speed penalty. Actually the lua variant is more efficient, +-- especially when multiple calls to kpsewhich are involved. The lua +-- library also gives way more ocntrol. + +-- to be done / considered +-- +-- support for --exec or make it default +-- support for jar files (or maybe not, never used, too messy) +-- support for $RUBYINPUTS cum suis (if still needed) +-- remember for subruns: _CTX_K_V_#{original}_ +-- remember for subruns: _CTX_K_S_#{original}_ +-- remember for subruns: TEXMFSTART.#{original} [tex.rb texmfstart.rb] + +banner = "version 1.0.2 - 2007+ - PRAGMA ADE / CONTEXT" +texlua = true + +-- begin library merge +-- filename : l-string.lua +-- comment : split off from luat-lib +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-string'] = 1.001 + +--~ function string.split(str, pat) -- taken from the lua wiki +--~ local t = {n = 0} -- so this table has a length field, traverse with ipairs then! +--~ local fpat = "(.-)"..pat +--~ local last_end = 1 +--~ local s, e, cap = string.find(str, fpat, 1) +--~ while s ~= nil do +--~ if s~=1 or cap~="" then +--~ table.insert(t,cap) +--~ end +--~ last_end = e+1 +--~ s, e, cap = string.find(str, fpat, last_end) +--~ end +--~ if last_end<=string.len(str) then +--~ table.insert(t,(string.sub(str,last_end))) +--~ end +--~ return t +--~ end + +--~ function string:split(pat) -- taken from the lua wiki but adapted +--~ local t = { } -- self and colon usage (faster) +--~ local fpat = "(.-)"..pat +--~ local last_end = 1 +--~ local s, e, cap = self:find(fpat, 1) +--~ while s ~= nil do +--~ if s~=1 or cap~="" then +--~ t[#t+1] = cap +--~ end +--~ last_end = e+1 +--~ s, e, cap = self:find(fpat, last_end) +--~ end +--~ if last_end <= #self then +--~ t[#t+1] = self:sub(last_end) +--~ end +--~ return t +--~ end + +--~ a piece of brilliant code by Rici Lake (posted on lua list) -- only names changed +--~ +--~ function string:splitter(pat) +--~ local st, g = 1, self:gmatch("()"..pat.."()") +--~ local function splitter(self) +--~ if st then +--~ local s, f = g() +--~ local rv = self:sub(st, (s or 0)-1) +--~ st = f +--~ return rv +--~ end +--~ end +--~ return splitter, self +--~ end + +function string:splitter(pat) + -- by Rici Lake (posted on lua list) -- only names changed + -- p 79 ref man: () returns position of match + local st, g = 1, self:gmatch("()("..pat..")") + local function strgetter(self, segs, seps, sep, cap1, ...) + st = sep and seps + #sep + return self:sub(segs, (seps or 0) - 1), cap1 or sep, ... + end + local function strsplitter(self) + if st then return strgetter(self, st, g()) end + end + return strsplitter, self +end + +function string:split(separator) + local t = {} + for k in self:splitter(separator) do t[#t+1] = k end + return t +end + +-- faster than a string:split: + +function string:splitchr(chr) + if #self > 0 then + local t = { } + for s in string.gmatch(self..chr,"(.-)"..chr) do + t[#t+1] = s + end + return t + else + return { } + end +end + +--~ function string.piecewise(str, pat, fnc) -- variant of split +--~ local fpat = "(.-)"..pat +--~ local last_end = 1 +--~ local s, e, cap = string.find(str, fpat, 1) +--~ while s ~= nil do +--~ if s~=1 or cap~="" then +--~ fnc(cap) +--~ end +--~ last_end = e+1 +--~ s, e, cap = string.find(str, fpat, last_end) +--~ end +--~ if last_end <= #str then +--~ fnc((string.sub(str,last_end))) +--~ end +--~ end + +function string.piecewise(str, pat, fnc) -- variant of split + for k in string.splitter(str,pat) do fnc(k) end +end + +--~ function string.piecewise(str, pat, fnc) -- variant of split +--~ for k in str:splitter(pat) do fnc(k) end +--~ end + +--~ do if lpeg then + +--~ -- this alternative is 30% faster esp when we cache them +--~ -- problem: no expressions + +--~ splitters = { } + +--~ function string:split(separator) +--~ if #self > 0 then +--~ local split = splitters[separator] +--~ if not split then +--~ -- based on code by Roberto +--~ local p = lpeg.P(separator) +--~ local c = lpeg.C((1-p)^0) +--~ split = lpeg.Ct(c*(p*c)^0) +--~ splitters[separator] = split +--~ end +--~ return split:match(self) +--~ else +--~ return { } +--~ end +--~ end + +--~ string.splitchr = string.split + +--~ function string:piecewise(separator,fnc) +--~ for _,v in pairs(self:split(separator)) do +--~ fnc(v) +--~ end +--~ end + +--~ end end + +string.chr_to_esc = { + ["%"] = "%%", + ["."] = "%.", + ["+"] = "%+", ["-"] = "%-", ["*"] = "%*", + ["^"] = "%^", ["$"] = "%$", + ["["] = "%[", ["]"] = "%]", + ["("] = "%(", [")"] = "%)", + ["{"] = "%{", ["}"] = "%}" +} + +function string:esc() -- variant 2 + return (self:gsub("(.)",string.chr_to_esc)) +end + +function string.unquote(str) + return (str:gsub("^([\"\'])(.*)%1$","%2")) +end + +function string.quote(str) + return '"' .. str:unquote() .. '"' +end + +function string:count(pattern) -- variant 3 + local n = 0 + for _ in self:gmatch(pattern) do + n = n + 1 + end + return n +end + +function string:limit(n,sentinel) + if #self > n then + sentinel = sentinel or " ..." + return self:sub(1,(n-#sentinel)) .. sentinel + else + return self + end +end + +function string:strip() + return (self:gsub("^%s*(.-)%s*$", "%1")) +end + +--~ function string.strip(str) -- slightly different +--~ return (string.gsub(string.gsub(str,"^%s*(.-)%s*$","%1"),"%s+"," ")) +--~ end + +function string:is_empty() + return not self:find("%S") +end + +function string:enhance(pattern,action) + local ok, n = true, 0 + while ok do + ok = false + self = self:gsub(pattern, function(...) + ok, n = true, n + 1 + return action(...) + end) + end + return self, n +end + +--~ function string:enhance(pattern,action) +--~ local ok, n = 0, 0 +--~ repeat +--~ self, ok = self:gsub(pattern, function(...) +--~ n = n + 1 +--~ return action(...) +--~ end) +--~ until ok == 0 +--~ return self, n +--~ end + +--~ function string:to_hex() +--~ if self then +--~ return (self:gsub("(.)",function(c) +--~ return string.format("%02X",c:byte()) +--~ end)) +--~ else +--~ return "" +--~ end +--~ end + +--~ function string:from_hex() +--~ if self then +--~ return (self:gsub("(..)",function(c) +--~ return string.char(tonumber(c,16)) +--~ end)) +--~ else +--~ return "" +--~ end +--~ end + +string.chr_to_hex = { } +string.hex_to_chr = { } + +for i=0,255 do + local c, h = string.char(i), string.format("%02X",i) + string.chr_to_hex[c], string.hex_to_chr[h] = h, c +end + +--~ function string:to_hex() +--~ if self then return (self:gsub("(.)",string.chr_to_hex)) else return "" end +--~ end + +--~ function string:from_hex() +--~ if self then return (self:gsub("(..)",string.hex_to_chr)) else return "" end +--~ end + +function string:to_hex() + return ((self or ""):gsub("(.)",string.chr_to_hex)) +end + +function string:from_hex() + return ((self or ""):gsub("(..)",string.hex_to_chr)) +end + +if not string.characters then + + local function nextchar(str, index) + index = index + 1 + return (index <= #str) and index or nil, str:sub(index,index) + end + function string:characters() + return nextchar, self, 0 + end + local function nextbyte(str, index) + index = index + 1 + return (index <= #str) and index or nil, string.byte(str:sub(index,index)) + end + function string:bytes() + return nextbyte, self, 0 + end + +end + +--~ function string:padd(n,chr) +--~ return self .. self.rep(chr or " ",n-#self) +--~ end + +function string:rpadd(n,chr) + local m = n-#self + if m > 0 then + return self .. self.rep(chr or " ",m) + else + return self + end +end + +function string:lpadd(n,chr) + local m = n-#self + if m > 0 then + return self.rep(chr or " ",m) .. self + else + return self + end +end + +string.padd = string.rpadd + +function is_number(str) + return str:find("^[%-%+]?[%d]-%.?[%d+]$") == 1 +end + +--~ print(is_number("1")) +--~ print(is_number("1.1")) +--~ print(is_number(".1")) +--~ print(is_number("-0.1")) +--~ print(is_number("+0.1")) +--~ print(is_number("-.1")) +--~ print(is_number("+.1")) + +function string:split_settings() -- no {} handling, see l-aux for lpeg variant + if self:find("=") then + local t = { } + for k,v in self:gmatch("(%a+)=([^%,]*)") do + t[k] = v + end + return t + else + return nil + end +end + +local patterns_escapes = { + ["-"] = "%-", + ["."] = "%.", + ["+"] = "%+", + ["*"] = "%*", + ["%"] = "%%", + ["("] = "%)", + [")"] = "%)", + ["["] = "%[", + ["]"] = "%]", +} + +function string:pattesc() + return (self:gsub(".",patterns_escapes)) +end + +function string:tohash() + local t = { } + for s in self:gmatch("([^, ]+)") do -- lpeg + t[s] = true + end + return t +end + + +-- filename : l-lpeg.lua +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-lpeg'] = 1.001 + +--~ l-lpeg.lua : + +--~ lpeg.digit = lpeg.R('09')^1 +--~ lpeg.sign = lpeg.S('+-')^1 +--~ lpeg.cardinal = lpeg.P(lpeg.sign^0 * lpeg.digit^1) +--~ lpeg.integer = lpeg.P(lpeg.sign^0 * lpeg.digit^1) +--~ lpeg.float = lpeg.P(lpeg.sign^0 * lpeg.digit^0 * lpeg.P('.') * lpeg.digit^1) +--~ lpeg.number = lpeg.float + lpeg.integer +--~ lpeg.oct = lpeg.P("0") * lpeg.R('07')^1 +--~ lpeg.hex = lpeg.P("0x") * (lpeg.R('09') + lpeg.R('AF'))^1 +--~ lpeg.uppercase = lpeg.P("AZ") +--~ lpeg.lowercase = lpeg.P("az") + +--~ lpeg.eol = lpeg.S('\r\n\f')^1 -- includes formfeed +--~ lpeg.space = lpeg.S(' ')^1 +--~ lpeg.nonspace = lpeg.P(1-lpeg.space)^1 +--~ lpeg.whitespace = lpeg.S(' \r\n\f\t')^1 +--~ lpeg.nonwhitespace = lpeg.P(1-lpeg.whitespace)^1 + +local hash = { } + +function lpeg.anywhere(pattern) --slightly adapted from website + return lpeg.P { lpeg.P(pattern) + 1 * lpeg.V(1) } +end + +function lpeg.startswith(pattern) --slightly adapted + return lpeg.P(pattern) +end + +--~ g = lpeg.splitter(" ",function(s) ... end) -- gmatch:lpeg = 3:2 + +function lpeg.splitter(pattern, action) + return (((1-lpeg.P(pattern))^1)/action+1)^0 +end + +local crlf = lpeg.P("\r\n") +local cr = lpeg.P("\r") +local lf = lpeg.P("\n") +local space = lpeg.S(" \t\f\v") +local newline = crlf + cr + lf +local spacing = space^0 * newline + +local empty = spacing * lpeg.Cc("") +local nonempty = lpeg.Cs((1-spacing)^1) * spacing^-1 +local content = (empty + nonempty)^1 + +local capture = lpeg.Ct(content^0) + +function string:splitlines() + return capture:match(self) +end + + +-- filename : l-table.lua +-- comment : split off from luat-lib +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-table'] = 1.001 + +table.join = table.concat + +function table.strip(tab) + local lst = { } + for k, v in ipairs(tab) do + -- s = string.gsub(v, "^%s*(.-)%s*$", "%1") + s = v:gsub("^%s*(.-)%s*$", "%1") + if s == "" then + -- skip this one + else + lst[#lst+1] = s + end + end + return lst +end + +--~ function table.sortedkeys(tab) +--~ local srt = { } +--~ for key,_ in pairs(tab) do +--~ srt[#srt+1] = key +--~ end +--~ table.sort(srt) +--~ return srt +--~ end + +function table.sortedkeys(tab) + local srt, kind = { }, 0 -- 0=unknown 1=string, 2=number 3=mixed + for key,_ in pairs(tab) do + srt[#srt+1] = key + if kind == 3 then + -- no further check + else + local tkey = type(key) + if tkey == "string" then + -- if kind == 2 then kind = 3 else kind = 1 end + kind = (kind == 2 and 3) or 1 + elseif tkey == "number" then + -- if kind == 1 then kind = 3 else kind = 2 end + kind = (kind == 1 and 3) or 2 + else + kind = 3 + end + end + end + if kind == 0 or kind == 3 then + table.sort(srt,function(a,b) return (tostring(a) < tostring(b)) end) + else + table.sort(srt) + end + return srt +end + +function table.append(t, list) + for _,v in pairs(list) do + table.insert(t,v) + end +end + +function table.prepend(t, list) + for k,v in pairs(list) do + table.insert(t,k,v) + end +end + +function table.merge(t, ...) -- first one is target + t = t or {} + local lst = {...} + for i=1,#lst do + for k, v in pairs(lst[i]) do + t[k] = v + end + end + return t +end + +function table.merged(...) + local tmp, lst = { }, {...} + for i=1,#lst do + for k, v in pairs(lst[i]) do + tmp[k] = v + end + end + return tmp +end + +function table.imerge(t, ...) + local lst = {...} + for i=1,#lst do + local nst = lst[i] + for j=1,#nst do + t[#t+1] = nst[j] + end + end + return t +end + +function table.imerged(...) + local tmp, lst = { }, {...} + for i=1,#lst do + local nst = lst[i] + for j=1,#nst do + tmp[#tmp+1] = nst[j] + end + end + return tmp +end + +if not table.fastcopy then do + + local type, pairs, getmetatable, setmetatable = type, pairs, getmetatable, setmetatable + + local function fastcopy(old) -- fast one + if old then + local new = { } + for k,v in pairs(old) do + if type(v) == "table" then + new[k] = fastcopy(v) -- was just table.copy + else + new[k] = v + end + end + local mt = getmetatable(old) + if mt then + setmetatable(new,mt) + end + return new + else + return { } + end + end + + table.fastcopy = fastcopy + +end end + +if not table.copy then do + + local type, pairs, getmetatable, setmetatable = type, pairs, getmetatable, setmetatable + + local function copy(t, tables) -- taken from lua wiki, slightly adapted + tables = tables or { } + local tcopy = {} + if not tables[t] then + tables[t] = tcopy + end + for i,v in pairs(t) do -- brrr, what happens with sparse indexed + if type(i) == "table" then + if tables[i] then + i = tables[i] + else + i = copy(i, tables) + end + end + if type(v) ~= "table" then + tcopy[i] = v + elseif tables[v] then + tcopy[i] = tables[v] + else + tcopy[i] = copy(v, tables) + end + end + local mt = getmetatable(t) + if mt then + setmetatable(tcopy,mt) + end + return tcopy + end + + table.copy = copy + +end end + +-- rougly: copy-loop : unpack : sub == 0.9 : 0.4 : 0.45 (so in critical apps, use unpack) + +function table.sub(t,i,j) + return { unpack(t,i,j) } +end + +function table.replace(a,b) + for k,v in pairs(b) do + a[k] = v + end +end + +-- slower than #t on indexed tables (#t only returns the size of the numerically indexed slice) + +function table.is_empty(t) + return not t or not next(t) +end + +function table.one_entry(t) + local n = next(t) + return n and not next(t,n) +end + +function table.starts_at(t) + return ipairs(t,1)(t,0) +end + +do + + -- one of my first exercises in lua ... + + -- 34.055.092 32.403.326 arabtype.tma + -- 1.620.614 1.513.863 lmroman10-italic.tma + -- 1.325.585 1.233.044 lmroman10-regular.tma + -- 1.248.157 1.158.903 lmsans10-regular.tma + -- 194.646 153.120 lmtypewriter10-regular.tma + -- 1.771.678 1.658.461 palatinosanscom-bold.tma + -- 1.695.251 1.584.491 palatinosanscom-regular.tma + -- 13.736.534 13.409.446 zapfinoextraltpro.tma + + -- 13.679.038 11.774.106 arabtype.tmc + -- 886.248 754.944 lmroman10-italic.tmc + -- 729.828 466.864 lmroman10-regular.tmc + -- 688.482 441.962 lmsans10-regular.tmc + -- 128.685 95.853 lmtypewriter10-regular.tmc + -- 715.929 582.985 palatinosanscom-bold.tmc + -- 669.942 540.126 palatinosanscom-regular.tmc + -- 1.560.588 1.317.000 zapfinoextraltpro.tmc + + table.serialize_functions = true + table.serialize_compact = true + table.serialize_inline = true + + local function key(k) + if type(k) == "number" then -- or k:find("^%d+$") then + return "["..k.."]" + elseif noquotes and k:find("^%a[%a%d%_]*$") then + return k + else + return '["'..k..'"]' + end + end + + local function simple_table(t) + if #t > 0 then + local n = 0 + for _,v in pairs(t) do + n = n + 1 + end + if n == #t then + local tt = { } + for i=1,#t do + local v = t[i] + local tv = type(v) + if tv == "number" or tv == "boolean" then + tt[#tt+1] = tostring(v) + elseif tv == "string" then + tt[#tt+1] = ("%q"):format(v) + else + tt = nil + break + end + end + return tt + end + end + return nil + end + + local function serialize(root,name,handle,depth,level,reduce,noquotes,indexed) + handle = handle or print + reduce = reduce or false + if depth then + depth = depth .. " " + if indexed then + handle(("%s{"):format(depth)) + else + handle(("%s%s={"):format(depth,key(name))) + end + else + depth = "" + local tname = type(name) + if tname == "string" then + if name == "return" then + handle("return {") + else + handle(name .. "={") + end + elseif tname == "number" then + handle("[" .. name .. "]={") + elseif tname == "boolean" then + if name then + handle("return {") + else + handle("{") + end + else + handle("t={") + end + end + if root and next(root) then + local compact = table.serialize_compact + local inline = compact and table.serialize_inline + local first, last = nil, 0 -- #root cannot be trusted here + if compact then + for k,v in ipairs(root) do -- NOT: for k=1,#root do (why) + if not first then first = k end + last = last + 1 + end + end + for _,k in pairs(table.sortedkeys(root)) do + local v = root[k] + local t = type(v) + if compact and first and type(k) == "number" and k >= first and k <= last then + if t == "number" then + handle(("%s %s,"):format(depth,v)) + elseif t == "string" then + if reduce and (v:find("^[%-%+]?[%d]-%.?[%d+]$") == 1) then + handle(("%s %s,"):format(depth,v)) + else + handle(("%s %q,"):format(depth,v)) + end + elseif t == "table" then + if not next(v) then + handle(("%s {},"):format(depth)) + elseif inline then + local st = simple_table(v) + if st then + handle(("%s { %s },"):format(depth,table.concat(st,", "))) + else + serialize(v,k,handle,depth,level+1,reduce,noquotes,true) + end + else + serialize(v,k,handle,depth,level+1,reduce,noquotes,true) + end + elseif t == "boolean" then + handle(("%s %s,"):format(depth,tostring(v))) + elseif t == "function" then + if table.serialize_functions then + handle(('%s loadstring(%q),'):format(depth,string.dump(v))) + else + handle(('%s "function",'):format(depth)) + end + else + handle(("%s %q,"):format(depth,tostring(v))) + end + elseif k == "__p__" then -- parent + if false then + handle(("%s __p__=nil,"):format(depth)) + end + elseif t == "number" then + handle(("%s %s=%s,"):format(depth,key(k),v)) + elseif t == "string" then + if reduce and (v:find("^[%-%+]?[%d]-%.?[%d+]$") == 1) then + handle(("%s %s=%s,"):format(depth,key(k),v)) + else + handle(("%s %s=%q,"):format(depth,key(k),v)) + end + elseif t == "table" then + if not next(v) then + handle(("%s %s={},"):format(depth,key(k))) + elseif inline then + local st = simple_table(v) + if st then + handle(("%s %s={ %s },"):format(depth,key(k),table.concat(st,", "))) + else + serialize(v,k,handle,depth,level+1,reduce,noquotes) + end + else + serialize(v,k,handle,depth,level+1,reduce,noquotes) + end + elseif t == "boolean" then + handle(("%s %s=%s,"):format(depth,key(k),tostring(v))) + elseif t == "function" then + if table.serialize_functions then + handle(('%s %s=loadstring(%q),'):format(depth,key(k),string.dump(v))) + else + handle(('%s %s="function",'):format(depth,key(k))) + end + else + handle(("%s %s=%q,"):format(depth,key(k),tostring(v))) + -- handle(('%s %s=loadstring(%q),'):format(depth,key(k),string.dump(function() return v end))) + end + end + if level > 0 then + handle(("%s},"):format(depth)) + else + handle(("%s}"):format(depth)) + end + else + handle(("%s}"):format(depth)) + end + end + + --~ name: + --~ + --~ true : return { } + --~ false : { } + --~ nil : t = { } + --~ string : string = { } + --~ 'return' : return { } + --~ number : [number] = { } + + function table.serialize(root,name,reduce,noquotes) + local t = { } + local function flush(s) + t[#t+1] = s + end + serialize(root, name, flush, nil, 0, reduce, noquotes) + return table.concat(t,"\n") + end + + function table.tohandle(handle,root,name,reduce,noquotes) + serialize(root, name, handle, nil, 0, reduce, noquotes) + end + + -- sometimes tables are real use (zapfino extra pro is some 85M) in which + -- case a stepwise serialization is nice; actually, we could consider: + -- + -- for line in table.serializer(root,name,reduce,noquotes) do + -- ...(line) + -- end + -- + -- so this is on the todo list + + table.tofile_maxtab = 2*1024 + + function table.tofile(filename,root,name,reduce,noquotes) + local f = io.open(filename,'w') + if f then + local concat = table.concat + local maxtab = table.tofile_maxtab + if maxtab > 1 then + local t = { } + local function flush(s) + t[#t+1] = s + if #t > maxtab then + f:write(concat(t,"\n"),"\n") -- hm, write(sometable) should be nice + t = { } + end + end + serialize(root, name, flush, nil, 0, reduce, noquotes) + f:write(concat(t,"\n"),"\n") + else + local function flush(s) + f:write(s,"\n") + end + serialize(root, name, flush, nil, 0, reduce, noquotes) + end + f:close() + end + end + +end + +--~ t = { +--~ b = "123", +--~ a = "x", +--~ c = 1.23, +--~ d = "1.23", +--~ e = true, +--~ f = { +--~ d = "1.23", +--~ a = "x", +--~ b = "123", +--~ c = 1.23, +--~ e = true, +--~ f = { +--~ e = true, +--~ f = { +--~ e = true +--~ }, +--~ }, +--~ }, +--~ g = function() end +--~ } + +--~ print(table.serialize(t), "\n") +--~ print(table.serialize(t,"name"), "\n") +--~ print(table.serialize(t,false), "\n") +--~ print(table.serialize(t,true), "\n") +--~ print(table.serialize(t,"name",true), "\n") +--~ print(table.serialize(t,"name",true,true), "\n") + +do + + local function flatten(t,f,complete) + for i=1,#t do + local v = t[i] + if type(v) == "table" then + if complete or type(v[1]) == "table" then + flatten(v,f,complete) + else + f[#f+1] = v + end + else + f[#f+1] = v + end + end + end + + function table.flatten(t) + local f = { } + flatten(t,f,true) + return f + end + + function table.unnest(t) -- bad name + local f = { } + flatten(t,f,false) + return f + end + + table.flatten_one_level = table.unnest + +end + +function table.insert_before_value(t,value,str) + for i=1,#t do + if t[i] == value then + table.insert(t,i,str) + return + end + end + table.insert(t,1,str) +end + +function table.insert_after_value(t,value,str) + for i=1,#t do + if t[i] == value then + table.insert(t,i+1,str) + return + end + end + t[#t+1] = str +end + +function table.are_equal(a,b,n,m) + if #a == #b then + n = n or 1 + m = m or #a + for i=n,m do + local ai, bi = a[i], b[i] + if (ai==bi) or (type(ai)=="table" and type(bi)=="table" and table.are_equal(ai,bi)) then + -- continue + else + return false + end + end + return true + else + return false + end +end + +function table.compact(t) + if t then + for k,v in pairs(t) do + if not next(v) then + t[k] = nil + end + end + end +end + +function table.tohash(t) + local h = { } + for _, v in pairs(t) do -- no ipairs here + h[v] = true + end + return h +end + +function table.fromhash(t) + local h = { } + for k, v in pairs(t) do -- no ipairs here + if v then h[#h+1] = k end + end + return h +end + +function table.contains(t, v) + if t then + for i=1, #t do + if t[i] == v then + return true + end + end + end + return false +end + +function table.count(t) + local n, e = 0, next(t) + while e do + n, e = n + 1, next(t,e) + end + return n +end + +function table.swapped(t) + local s = { } + for k, v in pairs(t) do + s[v] = k + end + return s +end + +--~ function table.are_equal(a,b) +--~ return table.serialize(a) == table.serialize(b) +--~ end + +function table.clone(t,p) -- t is optional or nil or table + if not p then + t, p = { }, t or { } + elseif not t then + t = { } + end + setmetatable(t, { __index = function(_,key) return p[key] end }) + return t +end + + +function table.hexed(t,seperator) + local tt = { } + for i=1,#t do tt[i] = string.format("0x%04X",t[i]) end + return table.concat(tt,seperator or " ") +end + +function table.reverse_hash(h) + local r = { } + for k,v in pairs(h) do + r[v] = (k:gsub(" ","")):lower() + end + return r +end + +function table.reverse(t) + local tt = { } + if #t > 0 then + for i=#t,1,-1 do + tt[#tt+1] = t[i] + end + end + return tt +end + + +-- filename : l-io.lua +-- comment : split off from luat-lib +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-io'] = 1.001 + +if string.find(os.getenv("PATH"),";") then + io.fileseparator, io.pathseparator = "\\", ";" +else + io.fileseparator, io.pathseparator = "/" , ":" +end + +function io.loaddata(filename) + local f = io.open(filename,'rb') + if f then + local data = f:read('*all') + f:close() + return data + else + return nil + end +end + +function io.savedata(filename,data,joiner) + local f = io.open(filename, "wb") + if f then + if type(data) == "table" then + f:write(table.join(data,joiner or "")) + elseif type(data) == "function" then + data(f) + else + f:write(data) + end + f:close() + end +end + +function io.exists(filename) + local f = io.open(filename) + if f == nil then + return false + else + assert(f:close()) + return true + end +end + +function io.size(filename) + local f = io.open(filename) + if f == nil then + return 0 + else + local s = f:seek("end") + assert(f:close()) + return s + end +end + +function io.noflines(f) + local n = 0 + for _ in f:lines() do + n = n + 1 + end + f:seek('set',0) + return n +end + +do + + local sb = string.byte + + local nextchar = { + [ 4] = function(f) + return f:read(1,1,1,1) + end, + [ 2] = function(f) + return f:read(1,1) + end, + [ 1] = function(f) + return f:read(1) + end, + [-2] = function(f) + local a, b = f:read(1,1) + return b, a + end, + [-4] = function(f) + local a, b, c, d = f:read(1,1,1,1) + return d, c, b, a + end + } + + function io.characters(f,n) + if f then + return nextchar[n or 1], f + else + return nil, nil + end + end + +end + +do + + local sb = string.byte + +--~ local nextbyte = { +--~ [4] = function(f) +--~ local a = f:read(1) +--~ local b = f:read(1) +--~ local c = f:read(1) +--~ local d = f:read(1) +--~ if d then +--~ return sb(a), sb(b), sb(c), sb(d) +--~ else +--~ return nil, nil, nil, nil +--~ end +--~ end, +--~ [2] = function(f) +--~ local a = f:read(1) +--~ local b = f:read(1) +--~ if b then +--~ return sb(a), sb(b) +--~ else +--~ return nil, nil +--~ end +--~ end, +--~ [1] = function (f) +--~ local a = f:read(1) +--~ if a then +--~ return sb(a) +--~ else +--~ return nil +--~ end +--~ end, +--~ [-2] = function (f) +--~ local a = f:read(1) +--~ local b = f:read(1) +--~ if b then +--~ return sb(b), sb(a) +--~ else +--~ return nil, nil +--~ end +--~ end, +--~ [-4] = function(f) +--~ local a = f:read(1) +--~ local b = f:read(1) +--~ local c = f:read(1) +--~ local d = f:read(1) +--~ if d then +--~ return sb(d), sb(c), sb(b), sb(a) +--~ else +--~ return nil, nil, nil, nil +--~ end +--~ end +--~ } + + local nextbyte = { + [4] = function(f) + local a, b, c, d = f:read(1,1,1,1) + if d then + return sb(a), sb(b), sb(c), sb(d) + else + return nil, nil, nil, nil + end + end, + [2] = function(f) + local a, b = f:read(1,1) + if b then + return sb(a), sb(b) + else + return nil, nil + end + end, + [1] = function (f) + local a = f:read(1) + if a then + return sb(a) + else + return nil + end + end, + [-2] = function (f) + local a, b = f:read(1,1) + if b then + return sb(b), sb(a) + else + return nil, nil + end + end, + [-4] = function(f) + local a, b, c, d = f:read(1,1,1,1) + if d then + return sb(d), sb(c), sb(b), sb(a) + else + return nil, nil, nil, nil + end + end + } + + function io.bytes(f,n) + if f then + return nextbyte[n or 1], f + else + return nil, nil + end + end + +end + +function io.ask(question,default,options) + while true do + io.write(question) + if options then + io.write(string.format(" [%s]",table.concat(options,"|"))) + end + if default then + io.write(string.format(" [%s]",default)) + end + io.write(string.format(" ")) + local answer = io.read() + answer = answer:gsub("^%s*(.*)%s*$","%1") + if answer == "" and default then + return default + elseif not options then + return answer + else + for _,v in pairs(options) do + if v == answer then + return answer + end + end + local pattern = "^" .. answer + for _,v in pairs(options) do + if v:find(pattern) then + return v + end + end + end + end +end + + +-- filename : l-md5.lua +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-md5'] = 1.001 + +if md5 then do + + local function convert(str,fmt) + return (string.gsub(md5.sum(str),".",function(chr) return string.format(fmt,string.byte(chr)) end)) + end + + if not md5.HEX then function md5.HEX(str) return convert(str,"%02X") end end + if not md5.hex then function md5.hex(str) return convert(str,"%02x") end end + if not md5.dec then function md5.dec(str) return convert(str,"%03i") end end + +end end + + +-- filename : l-number.lua +-- comment : split off from luat-lib +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-number'] = 1.001 + +if not number then number = { } end + +-- a,b,c,d,e,f = number.toset(100101) + +function number.toset(n) + return (tostring(n)):match("(.?)(.?)(.?)(.?)(.?)(.?)(.?)(.?)") +end + +local format = string.format + +function number.toevenhex(n) + local s = format("%X",n) + if #s % 2 == 0 then + return s + else + return "0" .. s + end +end + +-- the lpeg way is slower on 8 digits, but faster on 4 digits, some 7.5% +-- on +-- +-- for i=1,1000000 do +-- local a,b,c,d,e,f,g,h = number.toset(12345678) +-- local a,b,c,d = number.toset(1234) +-- local a,b,c = number.toset(123) +-- end +-- +-- of course dedicated "(.)(.)(.)(.)" matches are even faster + +do + local one = lpeg.C(1-lpeg.S(''))^1 + + function number.toset(n) + return one:match(tostring(n)) + end +end + + + +-- filename : l-set.lua +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-set'] = 1.001 + +if not set then set = { } end + +do + + local nums = { } + local tabs = { } + local concat = table.concat + + set.create = table.tohash + + function set.tonumber(t) + if next(t) then + local s = "" + -- we could save mem by sorting, but it slows down + for k, v in pairs(t) do + if v then + -- why bother about the leading space + s = s .. " " .. k + end + end + if not nums[s] then + tabs[#tabs+1] = t + nums[s] = #tabs + end + return nums[s] + else + return 0 + end + end + + function set.totable(n) + if n == 0 then + return { } + else + return tabs[n] or { } + end + end + + function set.contains(n,s) + if type(n) == "table" then + return n[s] + elseif n == 0 then + return false + else + local t = tabs[n] + return t and t[s] + end + end + +end + +--~ local c = set.create{'aap','noot','mies'} +--~ local s = set.tonumber(c) +--~ local t = set.totable(s) +--~ print(t['aap']) +--~ local c = set.create{'zus','wim','jet'} +--~ local s = set.tonumber(c) +--~ local t = set.totable(s) +--~ print(t['aap']) +--~ print(t['jet']) +--~ print(set.contains(t,'jet')) +--~ print(set.contains(t,'aap')) + + + +-- filename : l-os.lua +-- comment : split off from luat-lib +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-os'] = 1.001 + +function os.resultof(command) + return io.popen(command,"r"):read("*all") +end + +if not os.exec then os.exec = os.execute end +if not os.spawn then os.spawn = os.execute end + +--~ os.type : windows | unix (new, we already guessed os.platform) +--~ os.name : windows | msdos | linux | macosx | solaris | .. | generic (new) + +if not io.fileseparator then + if string.find(os.getenv("PATH"),";") then + io.fileseparator, io.pathseparator, os.platform = "\\", ";", os.type or "windows" + else + io.fileseparator, io.pathseparator, os.platform = "/" , ":", os.type or "unix" + end +end + +os.platform = os.platform or os.type or (io.pathseparator == ";" and "windows") or "unix" + +function os.launch(str) + if os.platform == "windows" then + os.execute("start " .. str) -- os.spawn ? + else + os.execute(str .. " &") -- os.spawn ? + end +end + +if not os.setenv then + function os.setenv() return false end +end + +if not os.times then + -- utime = user time + -- stime = system time + -- cutime = children user time + -- cstime = children system time + function os.times() + return { + utime = os.gettimeofday(), -- user + stime = 0, -- system + cutime = 0, -- children user + cstime = 0, -- children system + } + end +end + +os.gettimeofday = os.gettimeofday or os.clock + +do + local startuptime = os.gettimeofday() + function os.runtime() + return os.gettimeofday() - startuptime + end +end + +--~ print(os.gettimeofday()-os.time()) +--~ os.sleep(1.234) +--~ print (">>",os.runtime()) +--~ print(os.date("%H:%M:%S",os.gettimeofday())) +--~ print(os.date("%H:%M:%S",os.time())) + + +-- filename : l-file.lua +-- comment : split off from luat-lib +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-file'] = 1.001 + +if not file then file = { } end + +function file.removesuffix(filename) + return filename:gsub("%.[%a%d]+$", "") +end + +function file.addsuffix(filename, suffix) + if not filename:find("%.[%a%d]+$") then + return filename .. "." .. suffix + else + return filename + end +end + +function file.replacesuffix(filename, suffix) + if not filename:find("%.[%a%d]+$") then + return filename .. "." .. suffix + else + return (filename:gsub("%.[%a%d]+$","."..suffix)) + end +end + +function file.dirname(name) + return name:match("^(.+)[/\\].-$") or "" +end + +function file.basename(name) + return name:match("^.+[/\\](.-)$") or name +end + +function file.nameonly(name) + return ((name:match("^.+[/\\](.-)$") or name):gsub("%..*$","")) +end + +function file.extname(name) + return name:match("^.+%.([^/\\]-)$") or "" +end + +file.suffix = file.extname + +function file.stripsuffix(name) + return (name:gsub("%.[%a%d]+$","")) +end + +--~ function file.join(...) +--~ local t = { ... } +--~ for i=1,#t do +--~ t[i] = (t[i]:gsub("\\","/")):gsub("/+$","") +--~ end +--~ return table.concat(t,"/") +--~ end + +--~ print(file.join("x/","/y")) +--~ print(file.join("http://","/y")) +--~ print(file.join("http://a","/y")) +--~ print(file.join("http:///a","/y")) +--~ print(file.join("//nas-1","/y")) + +function file.join(...) + local pth = table.concat({...},"/") + pth = pth:gsub("\\","/") + local a, b = pth:match("^(.*://)(.*)$") + if a and b then + return a .. b:gsub("//+","/") + end + a, b = pth:match("^(//)(.*)$") + if a and b then + return a .. b:gsub("//+","/") + end + return (pth:gsub("//+","/")) +end + +function file.is_writable(name) + local f = io.open(name, 'w') + if f then + f:close() + return true + else + return false + end +end + +function file.is_readable(name) + local f = io.open(name,'r') + if f then + f:close() + return true + else + return false + end +end + +--~ function file.split_path(str) +--~ if str:find(';') then +--~ return str:splitchr(";") +--~ else +--~ return str:splitchr(io.pathseparator) +--~ end +--~ end + +-- todo: lpeg + +function file.split_path(str) + local t = { } + str = str:gsub("\\", "/") + str = str:gsub("(%a):([;/])", "%1\001%2") + for name in str:gmatch("([^;:]+)") do + if name ~= "" then + name = name:gsub("\001",":") + t[#t+1] = name + end + end + return t +end + +function file.join_path(tab) + return table.concat(tab,io.pathseparator) -- can have trailing // +end + +--~ print('test' .. " == " .. file.collapse_path("test")) +--~ print("test/test" .. " == " .. file.collapse_path("test/test")) +--~ print("test/test/test" .. " == " .. file.collapse_path("test/test/test")) +--~ print("test/test" .. " == " .. file.collapse_path("test/../test/test")) +--~ print("test" .. " == " .. file.collapse_path("test/../test")) +--~ print("../test" .. " == " .. file.collapse_path("../test")) +--~ print("../test/" .. " == " .. file.collapse_path("../test/")) +--~ print("a/a" .. " == " .. file.collapse_path("a/b/c/../../a")) + +--~ function file.collapse_path(str) +--~ local ok, n = false, 0 +--~ while not ok do +--~ ok = true +--~ str, n = str:gsub("[^%./]+/%.%./", function(s) +--~ ok = false +--~ return "" +--~ end) +--~ end +--~ return (str:gsub("/%./","/")) +--~ end + +function file.collapse_path(str) + local n = 1 + while n > 0 do + str, n = str:gsub("([^/%.]+/%.%./)","") + end + return (str:gsub("/%./","/")) +end + +function file.robustname(str) + return (str:gsub("[^%a%d%/%-%.\\]+","-")) +end + +file.readdata = io.loaddata +file.savedata = io.savedata + +function file.copy(oldname,newname) + file.savedata(newname,io.loaddata(oldname)) +end + + +-- filename : l-dir.lua +-- comment : split off from luat-lib +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-dir'] = 1.001 + +dir = { } + +-- optimizing for no string.find (*) does not save time + +if lfs then do + +--~ local attributes = lfs.attributes +--~ local walkdir = lfs.dir +--~ +--~ local function glob_pattern(path,patt,recurse,action) +--~ local ok, scanner = xpcall(function() return walkdir(path) end, function() end) -- kepler safe +--~ if ok and type(scanner) == "function" then +--~ if not path:find("/$") then path = path .. '/' end +--~ for name in scanner do +--~ local full = path .. name +--~ local mode = attributes(full,'mode') +--~ if mode == 'file' then +--~ if name:find(patt) then +--~ action(full) +--~ end +--~ elseif recurse and (mode == "directory") and (name ~= '.') and (name ~= "..") then +--~ glob_pattern(full,patt,recurse,action) +--~ end +--~ end +--~ end +--~ end +--~ +--~ dir.glob_pattern = glob_pattern +--~ +--~ local function glob(pattern, action) +--~ local t = { } +--~ local action = action or function(name) t[#t+1] = name end +--~ local path, patt = pattern:match("^(.*)/*%*%*/*(.-)$") +--~ local recurse = path and patt +--~ if not recurse then +--~ path, patt = pattern:match("^(.*)/(.-)$") +--~ if not (path and patt) then +--~ path, patt = '.', pattern +--~ end +--~ end +--~ patt = patt:gsub("([%.%-%+])", "%%%1") +--~ patt = patt:gsub("%*", ".*") +--~ patt = patt:gsub("%?", ".") +--~ patt = "^" .. patt .. "$" +--~ -- print('path: ' .. path .. ' | pattern: ' .. patt .. ' | recurse: ' .. tostring(recurse)) +--~ glob_pattern(path,patt,recurse,action) +--~ return t +--~ end +--~ +--~ dir.glob = glob + + local attributes = lfs.attributes + local walkdir = lfs.dir + + local function glob_pattern(path,patt,recurse,action) + local ok, scanner + if path == "/" then + ok, scanner = xpcall(function() return walkdir(path..".") end, function() end) -- kepler safe + else + ok, scanner = xpcall(function() return walkdir(path) end, function() end) -- kepler safe + end + if ok and type(scanner) == "function" then + if not path:find("/$") then path = path .. '/' end + for name in scanner do + local full = path .. name + local mode = attributes(full,'mode') + if mode == 'file' then + if full:find(patt) then + action(full) + end + elseif recurse and (mode == "directory") and (name ~= '.') and (name ~= "..") then + glob_pattern(full,patt,recurse,action) + end + end + end + end + + dir.glob_pattern = glob_pattern + + --~ local function glob(pattern, action) + --~ local t = { } + --~ local path, rest, patt, recurse + --~ local action = action or function(name) t[#t+1] = name end + --~ local pattern = pattern:gsub("^%*%*","./**") + --~ local pattern = pattern:gsub("/%*/","/**/") + --~ path, rest = pattern:match("^(/)(.-)$") + --~ if path then + --~ path = path + --~ else + --~ path, rest = pattern:match("^([^/]*)/(.-)$") + --~ end + --~ if rest then + --~ patt = rest:gsub("([%.%-%+])", "%%%1") + --~ end + --~ patt = patt:gsub("%*", "[^/]*") + --~ patt = patt:gsub("%?", "[^/]") + --~ patt = patt:gsub("%[%^/%]%*%[%^/%]%*", ".*") + --~ if path == "" then path = "." end + --~ recurse = patt:find("%.%*/") ~= nil + --~ glob_pattern(path,patt,recurse,action) + --~ return t + --~ end + + local P, S, R, C, Cc, Cs, Ct, Cv, V = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.Cc, lpeg.Cs, lpeg.Ct, lpeg.Cv, lpeg.V + + local pattern = Ct { + [1] = (C(P(".") + P("/")^1) + C(R("az","AZ") * P(":") * P("/")^0) + Cc("./")) * V(2) * V(3), + [2] = C(((1-S("*?/"))^0 * P("/"))^0), + [3] = C(P(1)^0) + } + + local filter = Cs ( ( + P("**") / ".*" + + P("*") / "[^/]*" + + P("?") / "[^/]" + + P(".") / "%%." + + P("+") / "%%+" + + P("-") / "%%-" + + P(1) + )^0 ) + + local function glob(str,t) + if type(str) == "table" then + local t = t or { } + for _, s in ipairs(str) do + glob(s,t) + end + return t + else + local split = pattern:match(str) + if split then + local t = t or { } + local action = action or function(name) t[#t+1] = name end + local root, path, base = split[1], split[2], split[3] + local recurse = base:find("**") + local start = root .. path + local result = filter:match(start .. base) + glob_pattern(start,result,recurse,action) + return t + else + return { } + end + end + end + + dir.glob = glob + + --~ list = dir.glob("**/*.tif") + --~ list = dir.glob("/**/*.tif") + --~ list = dir.glob("./**/*.tif") + --~ list = dir.glob("oeps/**/*.tif") + --~ list = dir.glob("/oeps/**/*.tif") + + local function globfiles(path,recurse,func,files) -- func == pattern or function + if type(func) == "string" then + local s = func -- alas, we need this indirect way + func = function(name) return name:find(s) end + end + files = files or { } + for name in walkdir(path) do + if name:find("^%.") then + --- skip + elseif attributes(name,'mode') == "directory" then + if recurse then + globfiles(path .. "/" .. name,recurse,func,files) + end + elseif func then + if func(name) then + files[#files+1] = path .. "/" .. name + end + else + files[#files+1] = path .. "/" .. name + end + end + return files + end + + dir.globfiles = globfiles + + -- t = dir.glob("c:/data/develop/context/sources/**/????-*.tex") + -- t = dir.glob("c:/data/develop/tex/texmf/**/*.tex") + -- t = dir.glob("c:/data/develop/context/texmf/**/*.tex") + -- t = dir.glob("f:/minimal/tex/**/*") + -- print(dir.ls("f:/minimal/tex/**/*")) + -- print(dir.ls("*.tex")) + + function dir.ls(pattern) + return table.concat(glob(pattern),"\n") + end + + --~ mkdirs("temp") + --~ mkdirs("a/b/c") + --~ mkdirs(".","/a/b/c") + --~ mkdirs("a","b","c") + + local make_indeed = true -- false + + if string.find(os.getenv("PATH"),";") then + + function dir.mkdirs(...) + local str, pth = "", "" + for _, s in ipairs({...}) do + if s ~= "" then + if str ~= "" then + str = str .. "/" .. s + else + str = s + end + end + end + local first, middle, last + local drive = false + first, middle, last = str:match("^(//)(//*)(.*)$") + if first then + -- empty network path == local path + else + first, last = str:match("^(//)/*(.-)$") + if first then + middle, last = str:match("([^/]+)/+(.-)$") + if middle then + pth = "//" .. middle + else + pth = "//" .. last + last = "" + end + else + first, middle, last = str:match("^([a-zA-Z]:)(/*)(.-)$") + if first then + pth, drive = first .. middle, true + else + middle, last = str:match("^(/*)(.-)$") + if not middle then + last = str + end + end + end + end + for s in last:gmatch("[^/]+") do + if pth == "" then + pth = s + elseif drive then + pth, drive = pth .. s, false + else + pth = pth .. "/" .. s + end + if make_indeed and not lfs.isdir(pth) then + lfs.mkdir(pth) + end + end + return pth, (lfs.isdir(pth) == true) + end + +--~ print(dir.mkdirs("","","a","c")) +--~ print(dir.mkdirs("a")) +--~ print(dir.mkdirs("a:")) +--~ print(dir.mkdirs("a:/b/c")) +--~ print(dir.mkdirs("a:b/c")) +--~ print(dir.mkdirs("a:/bbb/c")) +--~ print(dir.mkdirs("/a/b/c")) +--~ print(dir.mkdirs("/aaa/b/c")) +--~ print(dir.mkdirs("//a/b/c")) +--~ print(dir.mkdirs("///a/b/c")) +--~ print(dir.mkdirs("a/bbb//ccc/")) + + function dir.expand_name(str) + local first, nothing, last = str:match("^(//)(//*)(.*)$") + if first then + first = lfs.currentdir() .. "/" + first = first:gsub("\\","/") + end + if not first then + first, last = str:match("^(//)/*(.*)$") + end + if not first then + first, last = str:match("^([a-zA-Z]:)(.*)$") + if first and not last:find("^/") then + local d = lfs.currentdir() + if lfs.chdir(first) then + first = lfs.currentdir() + first = first:gsub("\\","/") + end + lfs.chdir(d) + end + end + if not first then + first, last = lfs.currentdir(), str + first = first:gsub("\\","/") + end + last = last:gsub("//","/") + last = last:gsub("/%./","/") + last = last:gsub("^/*","") + first = first:gsub("/*$","") + if last == "" then + return first + else + return first .. "/" .. last + end + end + + else + + function dir.mkdirs(...) + local str, pth = "", "" + for _, s in ipairs({...}) do + if s ~= "" then + if str ~= "" then + str = str .. "/" .. s + else + str = s + end + end + end + str = str:gsub("/+","/") + if str:find("^/") then + pth = "/" + for s in str:gmatch("[^/]+") do + local first = (pth == "/") + if first then + pth = pth .. s + else + pth = pth .. "/" .. s + end + if make_indeed and not first and not lfs.isdir(pth) then + lfs.mkdir(pth) + end + end + else + pth = "." + for s in str:gmatch("[^/]+") do + pth = pth .. "/" .. s + if make_indeed and not lfs.isdir(pth) then + lfs.mkdir(pth) + end + end + end + return pth, (lfs.isdir(pth) == true) + end + +--~ print(dir.mkdirs("","","a","c")) +--~ print(dir.mkdirs("a")) +--~ print(dir.mkdirs("/a/b/c")) +--~ print(dir.mkdirs("/aaa/b/c")) +--~ print(dir.mkdirs("//a/b/c")) +--~ print(dir.mkdirs("///a/b/c")) +--~ print(dir.mkdirs("a/bbb//ccc/")) + + function dir.expand_name(str) + if not str:find("^/") then + str = lfs.currentdir() .. "/" .. str + end + str = str:gsub("//","/") + str = str:gsub("/%./","/") + return str + end + + end + + dir.makedirs = dir.mkdirs + +end end + + +-- filename : l-boolean.lua +-- comment : split off from luat-lib +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-boolean'] = 1.001 +if not boolean then boolean = { } end + +function boolean.tonumber(b) + if b then return 1 else return 0 end +end + +function toboolean(str,tolerant) + if tolerant then + local tstr = type(str) + if tstr == "string" then + return str == "true" or str == "yes" or str == "on" or str == "1" + elseif tstr == "number" then + return tonumber(str) ~= 0 + elseif tstr == "nil" then + return false + else + return str + end + elseif str == "true" then + return true + elseif str == "false" then + return false + else + return str + end +end + +function string.is_boolean(str) + if type(str) == "string" then + if str == "true" or str == "yes" or str == "on" then + return true + elseif str == "false" or str == "no" or str == "off" then + return false + end + end + return nil +end + +function boolean.alwaystrue() + return true +end + +function boolean.falsetrue() + return false +end + + +if not modules then modules = { } end modules ['l-xml'] = { + version = 1.001, + comment = "this module is the basis for the lxml-* ones", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- RJ: key=value ... lpeg.Ca(lpeg.Cc({}) * (pattern-producing-key-and-value / rawset)^0) + +-- some code may move to l-xmlext + +--[[ldx-- +<p>The parser used here is inspired by the variant discussed in the lua book, but +handles comment and processing instructions, has a different structure, provides +parent access; a first version used different tricky but was less optimized to we +went this route. First we had a find based parser, now we have an <l n='lpeg'/> based one. +The find based parser can be found in l-xml-edu.lua along with other older code.</p> + +<p>Expecially the lpath code is experimental, we will support some of xpath, but +only things that make sense for us; as compensation it is possible to hook in your +own functions. Apart from preprocessing content for <l n='context'/> we also need +this module for process management, like handling <l n='ctx'/> and <l n='rlx'/> +files.</p> + +<typing> +a/b/c /*/c +a/b/c/first() a/b/c/last() a/b/c/index(n) a/b/c/index(-n) +a/b/c/text() a/b/c/text(1) a/b/c/text(-1) a/b/c/text(n) +</typing> + +<p>Beware, the interface may change. For instance at, ns, tg, dt may get more +verbose names. Once the code is stable we will also remove some tracing and +optimize the code.</p> +--ldx]]-- + +xml = xml or { } +tex = tex or { } + +xml.trace_lpath = false +xml.trace_print = false +xml.trace_remap = false + +local format, concat = string.format, table.concat + +--~ local pairs, next, type = pairs, next, type + +-- todo: some things per xml file, liek namespace remapping + +--[[ldx-- +<p>First a hack to enable namespace resolving. A namespace is characterized by +a <l n='url'/>. The following function associates a namespace prefix with a +pattern. We use <l n='lpeg'/>, which in this case is more than twice as fast as a +find based solution where we loop over an array of patterns. Less code and +much cleaner.</p> +--ldx]]-- + +xml.xmlns = xml.xmlns or { } + +do + + local check = lpeg.P(false) + local parse = check + + --[[ldx-- + <p>The next function associates a namespace prefix with an <l n='url'/>. This + normally happens independent of parsing.</p> + + <typing> + xml.registerns("mml","mathml") + </typing> + --ldx]]-- + + function xml.registerns(namespace, pattern) -- pattern can be an lpeg + check = check + lpeg.C(lpeg.P(pattern:lower())) / namespace + parse = lpeg.P { lpeg.P(check) + 1 * lpeg.V(1) } + end + + --[[ldx-- + <p>The next function also registers a namespace, but this time we map a + given namespace prefix onto a registered one, using the given + <l n='url'/>. This used for attributes like <t>xmlns:m</t>.</p> + + <typing> + xml.checkns("m","http://www.w3.org/mathml") + </typing> + --ldx]]-- + + function xml.checkns(namespace,url) + local ns = parse:match(url:lower()) + if ns and namespace ~= ns then + xml.xmlns[namespace] = ns + end + end + + --[[ldx-- + <p>Next we provide a way to turn an <l n='url'/> into a registered + namespace. This used for the <t>xmlns</t> attribute.</p> + + <typing> + resolvedns = xml.resolvens("http://www.w3.org/mathml") + </typing> + + This returns <t>mml</t>. + --ldx]]-- + + function xml.resolvens(url) + return parse:match(url:lower()) or "" + end + + --[[ldx-- + <p>A namespace in an element can be remapped onto the registered + one efficiently by using the <t>xml.xmlns</t> table.</p> + --ldx]]-- + +end + +--[[ldx-- +<p>This version uses <l n='lpeg'/>. We follow the same approach as before, stack and top and +such. This version is about twice as fast which is mostly due to the fact that +we don't have to prepare the stream for cdata, doctype etc etc. This variant is +is dedicated to Luigi Scarso, who challenged me with 40 megabyte <l n='xml'/> files that +took 12.5 seconds to load (1.5 for file io and the rest for tree building). With +the <l n='lpeg'/> implementation we got that down to less 7.3 seconds. Loading the 14 +<l n='context'/> interface definition files (2.6 meg) went down from 1.05 seconds to 0.55.</p> + +<p>Next comes the parser. The rather messy doctype definition comes in many +disguises so it is no surprice that later on have to dedicate quite some +<l n='lpeg'/> code to it.</p> + +<typing> +<!DOCTYPE Something PUBLIC "... ..." "..." [ ... ] > +<!DOCTYPE Something PUBLIC "... ..." "..." > +<!DOCTYPE Something SYSTEM "... ..." [ ... ] > +<!DOCTYPE Something SYSTEM "... ..." > +<!DOCTYPE Something [ ... ] > +<!DOCTYPE Something > +</typing> + +<p>The code may look a bit complex but this is mostly due to the fact that we +resolve namespaces and attach metatables. There is only one public function:</p> + +<typing> +local x = xml.convert(somestring) +</typing> + +<p>An optional second boolean argument tells this function not to create a root +element.</p> +--ldx]]-- + +xml.strip_cm_and_dt = false -- an extra global flag, in case we have many includes + +do + + -- not just one big nested table capture (lpeg overflow) + + local remove, nsremap, resolvens = table.remove, xml.xmlns, xml.resolvens + + local stack, top, dt, at, xmlns, errorstr, entities = {}, {}, {}, {}, {}, nil, {} + + local mt = { __tostring = xml.text } + + function xml.check_error(top,toclose) + return "" + end + + local strip = false + local cleanup = false + + function xml.set_text_cleanup(fnc) + cleanup = fnc + end + + local function add_attribute(namespace,tag,value) + if tag == "xmlns" then + xmlns[#xmlns+1] = resolvens(value) + at[tag] = value + elseif namespace == "xmlns" then + xml.checkns(tag,value) + at["xmlns:" .. tag] = value + else + at[tag] = value + end + end + local function add_begin(spacing, namespace, tag) + if #spacing > 0 then + dt[#dt+1] = spacing + end + local resolved = (namespace == "" and xmlns[#xmlns]) or nsremap[namespace] or namespace + top = { ns=namespace or "", rn=resolved, tg=tag, at=at, dt={}, __p__ = stack[#stack] } + setmetatable(top, mt) + dt = top.dt + stack[#stack+1] = top + at = { } + end + local function add_end(spacing, namespace, tag) + if #spacing > 0 then + dt[#dt+1] = spacing + end + local toclose = remove(stack) + top = stack[#stack] + if #stack < 1 then + errorstr = format("nothing to close with %s %s", tag, xml.check_error(top,toclose) or "") + elseif toclose.tg ~= tag then -- no namespace check + errorstr = format("unable to close %s with %s %s", toclose.tg, tag, xml.check_error(top,toclose) or "") + end + dt = top.dt + dt[#dt+1] = toclose + if at.xmlns then + remove(xmlns) + end + end + local function add_empty(spacing, namespace, tag) + if #spacing > 0 then + dt[#dt+1] = spacing + end + local resolved = (namespace == "" and xmlns[#xmlns]) or nsremap[namespace] or namespace + top = stack[#stack] + dt = top.dt + local t = { ns=namespace or "", rn=resolved, tg=tag, at=at, dt={}, __p__ = top } + dt[#dt+1] = t + setmetatable(t, mt) + at = { } + if at.xmlns then + remove(xmlns) + end + end + local function add_text(text) + if cleanup and #text > 0 then + dt[#dt+1] = cleanup(text) + else + dt[#dt+1] = text + end + end + local function add_special(what, spacing, text) + if #spacing > 0 then + dt[#dt+1] = spacing + end + if strip and (what == "@cm@" or what == "@dt@") then + -- forget it + else + dt[#dt+1] = { special=true, ns="", tg=what, dt={text} } + end + end + local function set_message(txt) + errorstr = "garbage at the end of the file: " .. txt:gsub("([ \n\r\t]*)","") + end + + local P, S, R, C, V = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V + + local space = S(' \r\n\t') + local open = P('<') + local close = P('>') + local squote = S("'") + local dquote = S('"') + local equal = P('=') + local slash = P('/') + local colon = P(':') + local valid = R('az', 'AZ', '09') + S('_-.') + local name_yes = C(valid^1) * colon * C(valid^1) + local name_nop = C(P(true)) * C(valid^1) + local name = name_yes + name_nop + + local utfbom = P('\000\000\254\255') + P('\255\254\000\000') + + P('\255\254') + P('\254\255') + P('\239\187\191') -- no capture + + local spacing = C(space^0) + local justtext = C((1-open)^1) + local somespace = space^1 + local optionalspace = space^0 + + local value = (squote * C((1 - squote)^0) * squote) + (dquote * C((1 - dquote)^0) * dquote) + local attribute = (somespace * name * optionalspace * equal * optionalspace * value) / add_attribute + local attributes = attribute^0 + + local text = justtext / add_text + local balanced = P { "[" * ((1 - S"[]") + V(1))^0 * "]" } -- taken from lpeg manual, () example + + local emptyelement = (spacing * open * name * attributes * optionalspace * slash * close) / add_empty + local beginelement = (spacing * open * name * attributes * optionalspace * close) / add_begin + local endelement = (spacing * open * slash * name * optionalspace * close) / add_end + + local begincomment = open * P("!--") + local endcomment = P("--") * close + local begininstruction = open * P("?") + local endinstruction = P("?") * close + local begincdata = open * P("![CDATA[") + local endcdata = P("]]") * close + + local someinstruction = C((1 - endinstruction)^0) + local somecomment = C((1 - endcomment )^0) + local somecdata = C((1 - endcdata )^0) + + function entity(k,v) entities[k] = v end + + local begindoctype = open * P("!DOCTYPE") + local enddoctype = close + local beginset = P("[") + local endset = P("]") + local doctypename = C((1-somespace)^0) + local elementdoctype = optionalspace * P("<!ELEMENT") * (1-close)^0 * close + local entitydoctype = optionalspace * P("<!ENTITY") * somespace * (doctypename * somespace * value)/entity * optionalspace * close + local publicdoctype = doctypename * somespace * P("PUBLIC") * somespace * value * somespace * value * somespace + local systemdoctype = doctypename * somespace * P("SYSTEM") * somespace * value * somespace + local definitiondoctype= doctypename * somespace * beginset * P(elementdoctype + entitydoctype)^0 * optionalspace * endset + local simpledoctype = (1-close)^1 -- * balanced^0 + local somedoctype = C((somespace * (publicdoctype + systemdoctype + definitiondoctype + simpledoctype) * optionalspace)^0) + + local instruction = (spacing * begininstruction * someinstruction * endinstruction) / function(...) add_special("@pi@",...) end + local comment = (spacing * begincomment * somecomment * endcomment ) / function(...) add_special("@cm@",...) end + local cdata = (spacing * begincdata * somecdata * endcdata ) / function(...) add_special("@cd@",...) end + local doctype = (spacing * begindoctype * somedoctype * enddoctype ) / function(...) add_special("@dt@",...) end + + -- nicer but slower: + -- + -- local instruction = (lpeg.Cc("@pi@") * spacing * begininstruction * someinstruction * endinstruction) / add_special + -- local comment = (lpeg.Cc("@cm@") * spacing * begincomment * somecomment * endcomment ) / add_special + -- local cdata = (lpeg.Cc("@cd@") * spacing * begincdata * somecdata * endcdata ) / add_special + -- local doctype = (lpeg.Cc("@dt@") * spacing * begindoctype * somedoctype * enddoctype ) / add_special + + local trailer = space^0 * (justtext/set_message)^0 + + -- comment + emptyelement + text + cdata + instruction + V("parent"), -- 6.5 seconds on 40 MB database file + -- text + comment + emptyelement + cdata + instruction + V("parent"), -- 5.8 + -- text + V("parent") + emptyelement + comment + cdata + instruction, -- 5.5 + + local grammar = P { "preamble", + preamble = utfbom^0 * instruction^0 * (doctype + comment + instruction)^0 * V("parent") * trailer, + parent = beginelement * V("children")^0 * endelement, + children = text + V("parent") + emptyelement + comment + cdata + instruction, + } + + -- todo: xml.new + properties like entities and strip and such (store in root) + + function xml.convert(data, no_root, strip_cm_and_dt, given_entities) -- maybe use table met k/v (given_entities may disapear) + strip = strip_cm_and_dt or xml.strip_cm_and_dt + stack, top, at, xmlns, errorstr, result, entities = {}, {}, {}, {}, nil, nil, given_entities or {} + stack[#stack+1] = top + top.dt = { } + dt = top.dt + if not data or data == "" then + errorstr = "empty xml file" + elseif not grammar:match(data) then + errorstr = "invalid xml file" + else + errorstr = "" + end + if errorstr and errorstr ~= "" then + result = { dt = { { ns = "", tg = "error", dt = { errorstr }, at={}, er = true } }, error = true } + setmetatable(stack, mt) + if xml.error_handler then xml.error_handler("load",errorstr) end + else + result = stack[1] + end + if not no_root then + result = { special = true, ns = "", tg = '@rt@', dt = result.dt, at={}, entities = entities } + setmetatable(result, mt) + local rdt = result.dt + for k=1,#rdt do + local v = rdt[k] + if type(v) == "table" and not v.special then -- always table -) + result.ri = k -- rootindex + break + end + end + end + return result + end + + --[[ldx-- + <p>Packaging data in an xml like table is done with the following + function. Maybe it will go away (when not used).</p> + --ldx]]-- + + function xml.is_valid(root) + return root and root.dt and root.dt[1] and type(root.dt[1]) == "table" and not root.dt[1].er + end + + function xml.package(tag,attributes,data) + local ns, tg = tag:match("^(.-):?([^:]+)$") + local t = { ns = ns, tg = tg, dt = data or "", at = attributes or {} } + setmetatable(t, mt) + return t + end + + function xml.is_valid(root) + return root and not root.error + end + + xml.error_handler = (logs and logs.report) or (input and input.report) or print + +end + +--[[ldx-- +<p>We cannot load an <l n='lpeg'/> from a filehandle so we need to load +the whole file first. The function accepts a string representing +a filename or a file handle.</p> +--ldx]]-- + +function xml.load(filename) + if type(filename) == "string" then + local f = io.open(filename,'r') + if f then + local root = xml.convert(f:read("*all")) + f:close() + return root + else + return xml.convert("") + end + elseif filename then -- filehandle + return xml.convert(filename:read("*all")) + else + return xml.convert("") + end +end + +--[[ldx-- +<p>When we inject new elements, we need to convert strings to +valid trees, which is what the next function does.</p> +--ldx]]-- + +function xml.toxml(data) + if type(data) == "string" then + local root = { xml.convert(data,true) } + return (#root > 1 and root) or root[1] + else + return data + end +end + +--[[ldx-- +<p>For copying a tree we use a dedicated function instead of the +generic table copier. Since we know what we're dealing with we +can speed up things a bit. The second argument is not to be used!</p> +--ldx]]-- + +do + + function copy(old,tables) + if old then + tables = tables or { } + local new = { } + if not tables[old] then + tables[old] = new + end + for k,v in pairs(old) do + new[k] = (type(v) == "table" and (tables[v] or copy(v, tables))) or v + end + local mt = getmetatable(old) + if mt then + setmetatable(new,mt) + end + return new + else + return { } + end + end + + xml.copy = copy + +end + +--[[ldx-- +<p>In <l n='context'/> serializing the tree or parts of the tree is a major +actitivity which is why the following function is pretty optimized resulting +in a few more lines of code than needed. The variant that uses the formatting +function for all components is about 15% slower than the concatinating +alternative.</p> +--ldx]]-- + +do + + -- todo: add <?xml version='1.0' standalone='yes'?> when not present + + local fallbackhandle = (tex and tex.sprint) or io.write + + local function serialize(e, handle, textconverter, attributeconverter, specialconverter, nocommands) + if not e then + return + elseif not nocommands then + local ec = e.command + if ec ~= nil then -- we can have all kind of types + +if e.special then -- todo test for true/false + local etg, edt = e.tg, e.dt + local spc = specialconverter and specialconverter[etg] + if spc then +--~ print("SPECIAL",etg,table.serialize(specialconverter), spc) + local result = spc(edt[1]) + if result then + handle(result) + return + else + -- no need to handle any further + end + end +end + + local xc = xml.command + if xc then + xc(e,ec) + return + end + end + end + handle = handle or fallbackhandle + local etg = e.tg + if etg then + if e.special then + local edt = e.dt + local spc = specialconverter and specialconverter[etg] + if spc then + local result = spc(edt[1]) + if result then + handle(result) + else + -- no need to handle any further + end + elseif etg == "@pi@" then + -- handle(format("<?%s?>",edt[1])) + handle("<?" .. edt[1] .. "?>") + elseif etg == "@cm@" then + -- handle(format("<!--%s-->",edt[1])) + handle("<!--" .. edt[1] .. "-->") + elseif etg == "@cd@" then + -- handle(format("<![CDATA[%s]]>",edt[1])) + handle("<![CDATA[" .. edt[1] .. "]]>") + elseif etg == "@dt@" then + -- handle(format("<!DOCTYPE %s>",edt[1])) + handle("<!DOCTYPE " .. edt[1] .. ">") + elseif etg == "@rt@" then + serialize(edt,handle,textconverter,attributeconverter,specialconverter,nocommands) + end + else + local ens, eat, edt, ern = e.ns, e.at, e.dt, e.rn + local ats = eat and next(eat) and { } -- type test maybe faster + if ats then + if attributeconverter then + for k,v in pairs(eat) do + ats[#ats+1] = format('%s=%q',k,attributeconverter(v)) + end + else + for k,v in pairs(eat) do + ats[#ats+1] = format('%s=%q',k,v) + end + end + end + if ern and xml.trace_remap and ern ~= ens then +--~ if ats then +--~ ats[#ats+1] = format("xmlns:remapped='%s'",ern) +--~ else +--~ ats = { format("xmlns:remapped='%s'",ern) } +--~ end +--~ if ats then +--~ ats[#ats+1] = format("remappedns='%s'",ens or '-') +--~ else +--~ ats = { format("remappedns='%s'",ens or '-') } +--~ end +ens = ern + end + if ens ~= "" then + if edt and #edt > 0 then + if ats then + -- handle(format("<%s:%s %s>",ens,etg,concat(ats," "))) + handle("<" .. ens .. ":" .. etg .. " " .. concat(ats," ") .. ">") + else + -- handle(format("<%s:%s>",ens,etg)) + handle("<" .. ens .. ":" .. etg .. ">") + end + for i=1,#edt do + local e = edt[i] + if type(e) == "string" then + if textconverter then + handle(textconverter(e)) + else + handle(e) + end + else + serialize(e,handle,textconverter,attributeconverter,specialconverter,nocommands) + end + end + -- handle(format("</%s:%s>",ens,etg)) + handle("</" .. ens .. ":" .. etg .. ">") + else + if ats then + -- handle(format("<%s:%s %s/>",ens,etg,concat(ats," "))) + handle("<" .. ens .. ":" .. etg .. " " .. concat(ats," ") .. "/>") + else + -- handle(format("<%s:%s/>",ens,etg)) + handle("<" .. ens .. ":" .. etg .. "/>") + end + end + else + if edt and #edt > 0 then + if ats then + -- handle(format("<%s %s>",etg,concat(ats," "))) + handle("<" .. etg .. " " .. concat(ats," ") .. ">") + else + -- handle(format("<%s>",etg)) + handle("<" .. etg .. ">") + end + for i=1,#edt do + serialize(edt[i],handle,textconverter,attributeconverter,specialconverter,nocommands) + end + -- handle(format("</%s>",etg)) + handle("</" .. etg .. ">") + else + if ats then + -- handle(format("<%s %s/>",etg,concat(ats," "))) + handle("<" .. etg .. " " .. concat(ats," ") .. "/>") + else + -- handle(format("<%s/>",etg)) + handle("<" .. etg .. "/>") + end + end + end + end + elseif type(e) == "string" then + if textconverter then + handle(textconverter(e)) + else + handle(e) + end + else + for i=1,#e do + serialize(e[i],handle,textconverter,attributeconverter,specialconverter,nocommands) + end + end + end + + xml.serialize = serialize + + function xml.checkbom(root) -- can be made faster + if root.ri then + local dt, found = root.dt, false + for k,v in ipairs(dt) do + if type(v) == "table" and v.special and v.tg == "@pi" and v.dt:find("xml.*version=") then + found = true + break + end + end + if not found then + table.insert(dt, 1, { special=true, ns="", tg="@pi@", dt = { "xml version='1.0' standalone='yes'"} } ) + table.insert(dt, 2, "\n" ) + end + end + end + + --[[ldx-- + <p>At the cost of some 25% runtime overhead you can first convert the tree to a string + and then handle the lot.</p> + --ldx]]-- + + function xml.tostring(root) -- 25% overhead due to collecting + if root then + if type(root) == 'string' then + return root + elseif next(root) then -- next is faster than type (and >0 test) + local result = { } + serialize(root,function(s) result[#result+1] = s end) + return concat(result,"") + end + end + return "" + end + +end + +--[[ldx-- +<p>The next function operated on the content only and needs a handle function +that accepts a string.</p> +--ldx]]-- + +function xml.string(e,handle) + if not handle or (e.special and e.tg ~= "@rt@") then + -- nothing + elseif e.tg then + local edt = e.dt + if edt then + for i=1,#edt do + xml.string(edt[i],handle) + end + end + else + handle(e) + end +end + +--[[ldx-- +<p>How you deal with saving data depends on your preferences. For a 40 MB database +file the timing on a 2.3 Core Duo are as follows (time in seconds):</p> + +<lines> +1.3 : load data from file to string +6.1 : convert string into tree +5.3 : saving in file using xmlsave +6.8 : converting to string using xml.tostring +3.6 : saving converted string in file +</lines> + +<p>The save function is given below.</p> +--ldx]]-- + +function xml.save(root,name) + local f = io.open(name,"w") + if f then + xml.serialize(root,function(s) f:write(s) end) + f:close() + end +end + +--[[ldx-- +<p>A few helpers:</p> +--ldx]]-- + +function xml.body(root) + return (root.ri and root.dt[root.ri]) or root +end + +function xml.text(root) + return (root and xml.tostring(root)) or "" +end + +function xml.content(root) -- bugged + return (root and root.dt and xml.tostring(root.dt)) or "" +end + +--[[ldx-- +<p>The next helper erases an element but keeps the table as it is, +and since empty strings are not serialized (effectively) it does +not harm. Copying the table would take more time. Usage:</p> + +<typing> +dt[k] = xml.empty() or xml.empty(dt,k) +</typing> +--ldx]]-- + +function xml.empty(dt,k) + if dt and k then + dt[k] = "" + return dt[k] + else + return "" + end +end + +--[[ldx-- +<p>The next helper assigns a tree (or string). Usage:</p> + +<typing> +dt[k] = xml.assign(root) or xml.assign(dt,k,root) +</typing> +--ldx]]-- + +function xml.assign(dt,k,root) + if dt and k then + dt[k] = (type(root) == "table" and xml.body(root)) or root + return dt[k] + else + return xml.body(root) + end +end + +--[[ldx-- +<p>We've now arrived at an intersting part: accessing the tree using a subset +of <l n='xpath'/> and since we're not compatible we call it <l n='lpath'/>. We +will explain more about its usage in other documents.</p> +--ldx]]-- + +do + + xml.functions = xml.functions or { } + + local functions = xml.functions + + local actions = { + [10] = "stay", + [11] = "parent", + [12] = "subtree root", + [13] = "document root", + [14] = "any", + [15] = "many", + [16] = "initial", + [20] = "match", + [21] = "match one of", + [22] = "match and attribute eq", + [23] = "match and attribute ne", + [24] = "match one of and attribute eq", + [25] = "match one of and attribute ne", + [27] = "has attribute", + [28] = "has value", + [29] = "fast match", + [30] = "select", + [31] = "expression", + [40] = "processing instruction", + } + + --~ local function make_expression(str) --could also be an lpeg + --~ str = str:gsub("@([a-zA-Z%-_]+)", "(a['%1'] or '')") + --~ str = str:gsub("position%(%)", "i") + --~ str = str:gsub("text%(%)", "t") + --~ str = str:gsub("!=", "~=") + --~ str = str:gsub("([^=!~<>])=([^=!~<>])", "%1==%2") + --~ str = str:gsub("([a-zA-Z%-_]+)%(", "functions.%1(") + --~ return str, loadstring(format("return function(functions,i,a,t) return %s end", str))() + --~ end + + -- a rather dumb lpeg + + local P, S, R, C, V, Cc = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc + + local lp_position = P("position()") / "id" + local lp_text = P("text()") / "tx" + local lp_name = P("name()") / "((rt.ns~='' and rt.ns..':'..rt.tg) or '')" + local lp_tag = P("tag()") / "(rt.tg or '')" + local lp_ns = P("ns()") / "(rt.ns or '')" + local lp_noequal = P("!=") / "~=" + P("<=") + P(">=") + P("==") + local lp_doequal = P("=") / "==" + local lp_attribute = P("@") / "" * Cc("(at['") * R("az","AZ","--","__")^1 * Cc("'] or '')") + + local lp_function = C(R("az","AZ","--","__")^1) * P("(") / function(t) + if functions[t] then + return "functions." .. t .. "(" + else + return "functions.error(" + end + end + + local lparent = lpeg.P("(") + local rparent = lpeg.P(")") + local noparent = 1 - (lparent+rparent) + local nested = lpeg.P{lparent * (noparent + lpeg.V(1))^0 * rparent} + local value = lpeg.P(lparent * lpeg.C((noparent + nested)^0) * rparent) + +--~ local value = P { "(" * C(((1 - S("()")) + V(1))^0) * ")" } + + local lp_special = (C(P("name")+P("text")+P("tag"))) * value / function(t,s) + if functions[t] then + if s then + return "functions." .. t .. "(rt,k," .. s ..")" + else + return "functions." .. t .. "(rt,k)" + end + else + return "functions.error(" .. t .. ")" + end + end + + local converter = lpeg.Cs ( ( + lp_position + + lp_text + lp_name + -- fast one + lp_special + + lp_noequal + lp_doequal + + lp_attribute + + lp_function + + 1 )^1 ) + + local function make_expression(str) + str = converter:match(str) + return str, loadstring(format("return function(functions,id,at,tx,rt,k) return %s end", str))() + end + + local map = { } + + local space = S(' \r\n\t') + local squote = S("'") + local dquote = S('"') + local lparent = P('(') + local rparent = P(')') + local atsign = P('@') + local lbracket = P('[') + local rbracket = P(']') + local exclam = P('!') + local period = P('.') + local eq = P('==') + P('=') + local ne = P('<>') + P('!=') + local star = P('*') + local slash = P('/') + local colon = P(':') + local bar = P('|') + local hat = P('^') + local valid = R('az', 'AZ', '09') + S('_-') +--~ local name_yes = C(valid^1 + star) * colon * C(valid^1 + star) -- permits ns:* *:tg *:* +--~ local name_nop = C(P(true)) * C(valid^1) + local name_yes = C(valid^1 + star) * colon * C(valid^1 + star) -- permits ns:* *:tg *:* + local name_nop = Cc("*") * C(valid^1) + local name = name_yes + name_nop + local number = C((S('+-')^0 * R('09')^1)) / tonumber + local names = (bar^0 * name)^1 + local morenames = name * (bar^0 * name)^1 + local instructiontag = P('pi::') + local spacing = C(space^0) + local somespace = space^1 + local optionalspace = space^0 + local text = C(valid^0) + local value = (squote * C((1 - squote)^0) * squote) + (dquote * C((1 - dquote)^0) * dquote) + local empty = 1-slash + + local is_eq = lbracket * atsign * name * eq * value * rbracket + local is_ne = lbracket * atsign * name * ne * value * rbracket + local is_attribute = lbracket * atsign * name * rbracket + local is_value = lbracket * value * rbracket + local is_number = lbracket * number * rbracket + + local nobracket = 1-(lbracket+rbracket) -- must be improved + local is_expression = lbracket * C(((C(nobracket^1))/make_expression)) * rbracket + + local is_expression = lbracket * (C(nobracket^1))/make_expression * rbracket + + local is_one = name + local is_none = exclam * name + local is_one_of = ((lparent * names * rparent) + morenames) + local is_none_of = exclam * ((lparent * names * rparent) + morenames) + + local stay = (period ) + local parent = (period * period ) / function( ) map[#map+1] = { 11 } end + local subtreeroot = (slash + hat ) / function( ) map[#map+1] = { 12 } end + local documentroot = (hat * hat ) / function( ) map[#map+1] = { 13 } end + local any = (star ) / function( ) map[#map+1] = { 14 } end + local many = (star * star ) / function( ) map[#map+1] = { 15 } end + local initial = (hat * hat * hat ) / function( ) map[#map+1] = { 16 } end + + local match = (is_one ) / function(...) map[#map+1] = { 20, true , ... } end + local match_one_of = (is_one_of ) / function(...) map[#map+1] = { 21, true , ... } end + local dont_match = (is_none ) / function(...) map[#map+1] = { 20, false, ... } end + local dont_match_one_of = (is_none_of ) / function(...) map[#map+1] = { 21, false, ... } end + + local match_and_eq = (is_one * is_eq ) / function(...) map[#map+1] = { 22, true , ... } end + local match_and_ne = (is_one * is_ne ) / function(...) map[#map+1] = { 23, true , ... } end + local dont_match_and_eq = (is_none * is_eq ) / function(...) map[#map+1] = { 22, false, ... } end + local dont_match_and_ne = (is_none * is_ne ) / function(...) map[#map+1] = { 23, false, ... } end + + local match_one_of_and_eq = (is_one_of * is_eq ) / function(...) map[#map+1] = { 24, true , ... } end + local match_one_of_and_ne = (is_one_of * is_ne ) / function(...) map[#map+1] = { 25, true , ... } end + local dont_match_one_of_and_eq = (is_none_of * is_eq ) / function(...) map[#map+1] = { 24, false, ... } end + local dont_match_one_of_and_ne = (is_none_of * is_ne ) / function(...) map[#map+1] = { 25, false, ... } end + + local has_attribute = (is_one * is_attribute) / function(...) map[#map+1] = { 27, true , ... } end + local has_value = (is_one * is_value ) / function(...) map[#map+1] = { 28, true , ... } end + local dont_has_attribute = (is_none * is_attribute) / function(...) map[#map+1] = { 27, false, ... } end + local dont_has_value = (is_none * is_value ) / function(...) map[#map+1] = { 28, false, ... } end + local position = (is_one * is_number ) / function(...) map[#map+1] = { 30, true, ... } end + local dont_position = (is_none * is_number ) / function(...) map[#map+1] = { 30, false, ... } end + + local expression = (is_one * is_expression)/ function(...) map[#map+1] = { 31, true, ... } end + local dont_expression = (is_none * is_expression)/ function(...) map[#map+1] = { 31, false, ... } end + + local self_expression = ( is_expression) / function(...) if #map == 0 then map[#map+1] = { 11 } end + map[#map+1] = { 31, true, "*", "*", ... } end + local dont_self_expression = (exclam * is_expression) / function(...) if #map == 0 then map[#map+1] = { 11 } end + map[#map+1] = { 31, false, "*", "*", ... } end + + local instruction = (instructiontag * text ) / function(...) map[#map+1] = { 40, ... } end + local nothing = (empty ) / function( ) map[#map+1] = { 15 } end -- 15 ? + local crap = (1-slash)^1 + + -- a few ugly goodies: + + local docroottag = P('^^') / function( ) map[#map+1] = { 12 } end + local subroottag = P('^') / function( ) map[#map+1] = { 13 } end + local roottag = P('root::') / function( ) map[#map+1] = { 12 } end + local parenttag = P('parent::') / function( ) map[#map+1] = { 11 } end + local childtag = P('child::') + local selftag = P('self::') + + -- there will be more and order will be optimized + + local selector = ( + instruction + + many + any + + parent + stay + + dont_position + position + + dont_match_one_of_and_eq + dont_match_one_of_and_ne + + match_one_of_and_eq + match_one_of_and_ne + + dont_match_and_eq + dont_match_and_ne + + match_and_eq + match_and_ne + + dont_expression + expression + + dont_self_expression + self_expression + + has_attribute + has_value + + dont_match_one_of + match_one_of + + dont_match + match + + crap + empty + ) + + local grammar = P { "startup", + startup = (initial + documentroot + subtreeroot + roottag + docroottag + subroottag)^0 * V("followup"), + followup = ((slash + parenttag + childtag + selftag)^0 * selector)^1, + } + + local function compose(str) + if not str or str == "" then + -- wildcard + return true + elseif str == '/' then + -- root + return false + else + map = { } + grammar:match(str) + if #map == 0 then + return true + else + local m = map[1][1] + if #map == 1 then + if m == 14 or m == 15 then + -- wildcard + return true + elseif m == 12 then + -- root + return false + end + elseif #map == 2 and m == 12 and map[2][1] == 20 then + -- return { { 29, map[2][2], map[2][3], map[2][4], map[2][5] } } + map[2][1] = 29 + return { map[2] } + end + if m ~= 11 and m ~= 12 and m ~= 13 and m ~= 14 and m ~= 15 and m ~= 16 then + table.insert(map, 1, { 16 }) + end + -- print((table.serialize(map)):gsub("[ \n]+"," ")) + return map + end + end + end + + local cache = { } + + function xml.lpath(pattern,trace) + if type(pattern) == "string" then + local result = cache[pattern] + if not result then + result = compose(pattern) + cache[pattern] = result + end + if trace or xml.trace_lpath then + xml.lshow(result) + end + return result + else + return pattern + end + end + + local fallbackreport = (texio and texio.write) or io.write + + function xml.lshow(pattern,report) + report = report or fallbackreport + local lp = xml.lpath(pattern) + if lp == false then + report(" -: root\n") + elseif lp == true then + report(" -: wildcard\n") + else + if type(pattern) == "string" then + report(format("pattern: %s\n",pattern)) + end + for k,v in ipairs(lp) do + if #v > 1 then + local t = { } + for i=2,#v do + local vv = v[i] + if type(vv) == "string" then + t[#t+1] = (vv ~= "" and vv) or "#" + elseif type(vv) == "boolean" then + t[#t+1] = (vv and "==") or "<>" + end + end + report(format("%2i: %s %s -> %s\n", k,v[1],actions[v[1]],concat(t," "))) + else + report(format("%2i: %s %s\n", k,v[1],actions[v[1]])) + end + end + end + end + + function xml.xshow(e,...) -- also handy when report is given, use () to isolate first e + local t = { ... } + local report = (type(t[#t]) == "function" and t[#t]) or fallbackreport + if e == nil then + report("<!-- no element -->\n") + elseif type(e) ~= "table" then + report(tostring(e)) + elseif e.tg then + report(tostring(e) .. "\n") + else + for i=1,#e do + report(tostring(e[i]) .. "\n") + end + end + end + +end + +--[[ldx-- +<p>An <l n='lpath'/> is converted to a table with instructions for traversing the +tree. Hoever, simple cases are signaled by booleans. Because we don't know in +advance what we want to do with the found element the handle gets three arguments:</p> + +<lines> +<t>r</t> : the root element of the data table +<t>d</t> : the data table of the result +<t>t</t> : the index in the data table of the result +</lines> + +<p> Access to the root and data table makes it possible to construct insert and delete +functions.</p> +--ldx]]-- + +do + + local functions = xml.functions + + functions.contains = string.find + functions.find = string.find + functions.upper = string.upper + functions.lower = string.lower + functions.number = tonumber + functions.boolean = toboolean + + functions.oneof = function(s,...) -- slow + local t = {...} for i=1,#t do if s == t[i] then return true end end return false + end + functions.error = function(str) + xml.error_handler("unknown function in lpath expression",str) + return false + end + functions.text = function(root,k,n) -- unchecked, maybe one deeper + local t = type(t) + if t == "string" then + return t + else -- todo n + local rdt = root.dt + return (rdt and rdt[k]) or root[k] or "" + end + end + functions.name = function(root,k,n) + -- way too fuzzy + local found + if not k or not n then + local ns, tg = root.rn or root.ns or "", root.tg + if not tg then + for i=1,#root do + local e = root[i] + if type(e) == "table" then + found = e + break + end + end + elseif ns ~= "" then + return ns .. ":" .. tg + else + return tg + end + elseif n == 0 then + local e = root[k] + if type(e) ~= "table" then + found = e + end + elseif n < 0 then + for i=k-1,1,-1 do + local e = root[i] + if type(e) == "table" then + if n == -1 then + found = e + break + else + n = n + 1 + end + end + end + else +--~ print(k,n,#root) + for i=k+1,#root,1 do + local e = root[i] + if type(e) == "table" then + if n == 1 then + found = e + break + else + n = n - 1 + end + end + end + end + if found then + local ns, tg = found.rn or found.ns or "", found.tg + if ns ~= "" then + return ns .. ":" .. tg + else + return tg + end + else + return "" + end + end + + local function traverse(root,pattern,handle,reverse,index,parent,wildcard) -- multiple only for tags, not for namespaces + if not root then -- error + return false + elseif pattern == false then -- root + handle(root,root.dt,root.ri) + return false + elseif pattern == true then -- wildcard + local rootdt = root.dt + if rootdt then + local start, stop, step = 1, #rootdt, 1 + if reverse then + start, stop, step = stop, start, -1 + end + for k=start,stop,step do + if handle(root,rootdt,root.ri or k) then return false end + if not traverse(rootdt[k],true,handle,reverse) then return false end + end + end + return false + elseif root.dt then + index = index or 1 + local action = pattern[index] + local command = action[1] + if command == 29 then -- fast case /oeps + local rootdt = root.dt + for k=1,#rootdt do + local e = rootdt[k] + local tg = e.tg + if e.tg then + local ns = e.rn or e.ns + local ns_a, tg_a = action[3], action[4] + local matched = (ns_a == "*" or ns == ns_a) and (tg_a == "*" or tg == tg_a) + if not action[2] then matched = not matched end + if matched then + if handle(root,rootdt,k) then return false end + end + end + end + elseif command == 11 then -- parent + local ep = root.__p__ or parent + if index < #pattern then + if not traverse(ep,pattern,handle,reverse,index+1,root) then return false end + elseif handle(root,rootdt,k) then + return false + end + else + if (command == 16 or command == 12) and index == 1 then -- initial + -- wildcard = true + wildcard = command == 16 -- ok? + index = index + 1 + action = pattern[index] + command = action and action[1] or 0 -- something is wrong + end + if command == 11 then -- parent + local ep = root.__p__ or parent + if index < #pattern then + if not traverse(ep,pattern,handle,reverse,index+1,root) then return false end + elseif handle(root,rootdt,k) then + return false + end + else + local rootdt = root.dt + local start, stop, step, n, dn = 1, #rootdt, 1, 0, 1 + if command == 30 then + if action[5] < 0 then + start, stop, step = stop, start, -1 + dn = -1 + end + elseif reverse and index == #pattern then + start, stop, step = stop, start, -1 + end + local idx = 0 + for k=start,stop,step do -- we used to have functions for all but a case is faster + local e = rootdt[k] + local ns, tg = e.rn or e.ns, e.tg + if tg then + idx = idx + 1 + if command == 30 then + local ns_a, tg_a = action[3], action[4] + if tg == tg_a then + matched = ns_a == "*" or ns == ns_a + elseif tg_a == '*' then + matched, multiple = ns_a == "*" or ns == ns_a, true + else + matched = false + end + if not action[2] then matched = not matched end + if matched then + n = n + dn + if n == action[5] then + if index == #pattern then + if handle(root,rootdt,root.ri or k) then return false end + else + if not traverse(e,pattern,handle,reverse,index+1,root) then return false end + end + break + end + elseif wildcard then + if not traverse(e,pattern,handle,reverse,index,root,true) then return false end + end + else + local matched, multiple = false, false + if command == 20 then -- match + local ns_a, tg_a = action[3], action[4] + if tg == tg_a then + matched = ns_a == "*" or ns == ns_a + elseif tg_a == '*' then + matched, multiple = ns_a == "*" or ns == ns_a, true + else + matched = false + end + if not action[2] then matched = not matched end + elseif command == 21 then -- match one of + multiple = true + for i=3,#action,2 do + local ns_a, tg_a = action[i], action[i+1] + if (ns_a == "*" or ns == ns_a) and (tg == "*" or tg == tg_a) then + matched = true + break + end + end + if not action[2] then matched = not matched end + elseif command == 22 then -- eq + local ns_a, tg_a = action[3], action[4] + if tg == tg_a then + matched = ns_a == "*" or ns == ns_a + elseif tg_a == '*' then + matched, multiple = ns_a == "*" or ns == ns_a, true + else + matched = false + end + matched = matched and e.at[action[6]] == action[7] + elseif command == 23 then -- ne + local ns_a, tg_a = action[3], action[4] + if tg == tg_a then + matched = ns_a == "*" or ns == ns_a + elseif tg_a == '*' then + matched, multiple = ns_a == "*" or ns == ns_a, true + else + matched = false + end + if not action[2] then matched = not matched end + matched = mached and e.at[action[6]] ~= action[7] + elseif command == 24 then -- one of eq + multiple = true + for i=3,#action-2,2 do + local ns_a, tg_a = action[i], action[i+1] + if (ns_a == "*" or ns == ns_a) and (tg == "*" or tg == tg_a) then + matched = true + break + end + end + if not action[2] then matched = not matched end + matched = matched and e.at[action[#action-1]] == action[#action] + elseif command == 25 then -- one of ne + multiple = true + for i=3,#action-2,2 do + local ns_a, tg_a = action[i], action[i+1] + if (ns_a == "*" or ns == ns_a) and (tg == "*" or tg == tg_a) then + matched = true + break + end + end + if not action[2] then matched = not matched end + matched = matched and e.at[action[#action-1]] ~= action[#action] + elseif command == 27 then -- has attribute + local ns_a, tg_a = action[3], action[4] + if tg == tg_a then + matched = ns_a == "*" or ns == ns_a + elseif tg_a == '*' then + matched, multiple = ns_a == "*" or ns == ns_a, true + else + matched = false + end + if not action[2] then matched = not matched end + matched = matched and e.at[action[5]] + elseif command == 28 then -- has value + local edt, ns_a, tg_a = e.dt, action[3], action[4] + if tg == tg_a then + matched = ns_a == "*" or ns == ns_a + elseif tg_a == '*' then + matched, multiple = ns_a == "*" or ns == ns_a, true + else + matched = false + end + if not action[2] then matched = not matched end + matched = matched and edt and edt[1] == action[5] + elseif command == 31 then + local edt, ns_a, tg_a = e.dt, action[3], action[4] + if tg == tg_a then + matched = ns_a == "*" or ns == ns_a + elseif tg_a == '*' then + matched, multiple = ns_a == "*" or ns == ns_a, true + else + matched = false + end + if not action[2] then matched = not matched end + if matched then + matched = action[6](functions,idx,e.at or { },edt[1],rootdt,k) + end + end + if matched then -- combine tg test and at test + if index == #pattern then + if handle(root,rootdt,root.ri or k) then return false end + if wildcard then + if multiple then + if not traverse(e,pattern,handle,reverse,index,root,true) then return false end + else + -- maybe or multiple; anyhow, check on (section|title) vs just section and title in example in lxml + if not traverse(e,pattern,handle,reverse,index,root) then return false end + end + end + else + if not traverse(e,pattern,handle,reverse,index+1,root) then return false end + end + elseif command == 14 then -- any + if index == #pattern then + if handle(root,rootdt,root.ri or k) then return false end + else + if not traverse(e,pattern,handle,reverse,index+1,root) then return false end + end + elseif command == 15 then -- many + if index == #pattern then + if handle(root,rootdt,root.ri or k) then return false end + else + if not traverse(e,pattern,handle,reverse,index+1,root,true) then return false end + end + -- not here : 11 + elseif command == 11 then -- parent + local ep = e.__p__ or parent + if index < #pattern then + if not traverse(ep,pattern,handle,reverse,root,index+1) then return false end + elseif handle(root,rootdt,k) then + return false + end + elseif command == 40 and e.special and tg == "@pi@" then -- pi + local pi = action[2] + if pi ~= "" then + local pt = e.dt[1] + if pt and pt:find(pi) then + if handle(root,rootdt,k) then + return false + end + end + elseif handle(root,rootdt,k) then + return false + end + elseif wildcard then + if not traverse(e,pattern,handle,reverse,index,root,true) then return false end + end + end + else + -- not here : 11 + if command == 11 then -- parent + local ep = e.__p__ or parent + if index < #pattern then + if not traverse(ep,pattern,handle,reverse,index+1,root) then return false end + elseif handle(root,rootdt,k) then + return false + end + break -- else loop + end + end + end + end + end + end + return true + end + + xml.traverse = traverse + +end + +--[[ldx-- +<p>Next come all kind of locators and manipulators. The most generic function here +is <t>xml.filter(root,pattern)</t>. All registers functions in the filters namespace +can be path of a search path, as in:</p> + +<typing> +local r, d, k = xml.filter(root,"/a/b/c/position(4)" +</typing> +--ldx]]-- + +do + + local traverse, lpath, convert = xml.traverse, xml.lpath, xml.convert + + xml.filters = { } + + function xml.filters.default(root,pattern) + local rt, dt, dk + traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end) + return dt and dt[dk], rt, dt, dk + end + function xml.filters.attributes(root,pattern,arguments) + local rt, dt, dk + traverse(root, lpath(pattern), function(r,d,k) rt, dt, dk = r, d, k return true end) + local ekat = (dt and dt[dk] and dt[dk].at) or (rt and rt.at) + if ekat then + if arguments then + return ekat[arguments] or "", rt, dt, dk + else + return ekat, rt, dt, dk + end + else + return { }, rt, dt, dk + end + end + function xml.filters.reverse(root,pattern) + local rt, dt, dk + traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end, 'reverse') + return dt and dt[dk], rt, dt, dk + end + function xml.filters.count(root,pattern,everything) + local n = 0 + traverse(root, lpath(pattern), function(r,d,t) + if everything or type(d[t]) == "table" then + n = n + 1 + end + end) + return n + end + function xml.filters.elements(root, pattern) -- == all + local t = { } + traverse(root, lpath(pattern), function(r,d,k) + local e = d[k] + if e then + t[#t+1] = e + end + end) + return t + end + function xml.filters.texts(root, pattern) + local t = { } + traverse(root, lpath(pattern), function(r,d,k) + local e = d[k] + if e and e.dt then + t[#t+1] = e.dt + end + end) + return t + end + function xml.filters.first(root,pattern) + local rt, dt, dk + traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end) + return dt and dt[dk], rt, dt, dk + end + function xml.filters.last(root,pattern) + local rt, dt, dk + traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end, 'reverse') + return dt and dt[dk], rt, dt, dk + end + function xml.filters.index(root,pattern,arguments) + local rt, dt, dk, reverse, i = nil, nil, nil, false, tonumber(arguments or '1') or 1 + if i and i ~= 0 then + if i < 0 then + reverse, i = true, -i + end + traverse(root, lpath(pattern), function(r,d,k) rt, dt, dk, i = r, d, k, i-1 return i == 0 end, reverse) + if i == 0 then + return dt and dt[dk], rt, dt, dk + end + end + return nil, nil, nil, nil + end + function xml.filters.attribute(root,pattern,arguments) + local rt, dt, dk + traverse(root, lpath(pattern), function(r,d,k) rt, dt, dk = r, d, k return true end) + local ekat = (dt and dt[dk] and dt[dk].at) or (rt and rt.at) + return (ekat and (ekat[arguments] or ekat[arguments:gsub("^([\"\'])(.*)%1$","%2")])) or "" + end + function xml.filters.text(root,pattern,arguments) -- ?? why index, tostring slow + local dtk, rt, dt, dk = xml.filters.index(root,pattern,arguments) + if dtk then -- n + local dtkdt = dtk.dt + if not dtkdt then + return "", rt, dt, dk + elseif #dtkdt == 1 and type(dtkdt[1]) == "string" then + return dtkdt[1], rt, dt, dk + else + return xml.tostring(dtkdt), rt, dt, dk + end + else + return "", rt, dt, dk + end + end + function xml.filters.tag(root,pattern,n) + local tag = "" + traverse(root, lpath(pattern), function(r,d,k) + tag = xml.functions.tag(d,k,n and tonumber(n)) + return true + end) + return tag + end + function xml.filters.name(root,pattern,n) + local tag = "" + traverse(root, lpath(pattern), function(r,d,k) + tag = xml.functions.name(d,k,n and tonumber(n)) + return true + end) + return tag + end + + --[[ldx-- + <p>For splitting the filter function from the path specification, we can + use string matching or lpeg matching. Here the difference in speed is + neglectable but the lpeg variant is more robust.</p> + --ldx]]-- + + -- not faster but hipper ... although ... i can't get rid of the trailing / in the path + + local P, S, R, C, V, Cc = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc + + local slash = P('/') + local name = (R("az","AZ","--","__"))^1 + local path = C(((1-slash)^0 * slash)^1) + local argument = P { "(" * C(((1 - S("()")) + V(1))^0) * ")" } + local action = Cc(1) * path * C(name) * argument + local attribute = Cc(2) * path * P('@') * C(name) + local direct = Cc(3) * Cc("../*") * slash^0 * C(name) * argument + + local parser = direct + action + attribute + + local filters = xml.filters + local attribute_filter = xml.filters.attributes + local default_filter = xml.filters.default + + -- todo: also hash, could be gc'd + + function xml.filter(root,pattern) + local kind, a, b, c = parser:match(pattern) +--~ if xml.trace_lpath then +--~ print(pattern,kind,a,b,c) +--~ end + if kind == 1 or kind == 3 then + return (filters[b] or default_filter)(root,a,c) + elseif kind == 2 then + return attribute_filter(root,a,b) + else + return default_filter(root,pattern) + end + end + + --~ slightly faster, but first we need a proper test file + --~ + --~ local hash = { } + --~ + --~ function xml.filter(root,pattern) + --~ local h = hash[pattern] + --~ if not h then + --~ local kind, a, b, c = parser:match(pattern) + --~ if kind == 1 then + --~ h = { kind, filters[b] or default_filter, a, b, c } + --~ elseif kind == 2 then + --~ h = { kind, attribute_filter, a, b, c } + --~ else + --~ h = { kind, default_filter, a, b, c } + --~ end + --~ hash[pattern] = h + --~ end + --~ local kind = h[1] + --~ if kind == 1 then + --~ return h[2](root,h[2],h[4]) + --~ elseif kind == 2 then + --~ return h[2](root,h[2],h[3]) + --~ else + --~ return h[2](root,pattern) + --~ end + --~ end + + --[[ldx-- + <p>The following functions collect elements and texts.</p> + --ldx]]-- + + -- still somewhat bugged + + function xml.collect_elements(root, pattern, ignorespaces) + local rr, dd = { }, { } + traverse(root, lpath(pattern), function(r,d,k) + local dk = d and d[k] + if dk then + if ignorespaces and type(dk) == "string" and dk:find("[^%S]") then + -- ignore + else + local n = #rr+1 + rr[n], dd[n] = r, dk + end + end + end) + return dd, rr + end + + function xml.collect_texts(root, pattern, flatten) + local t = { } -- no r collector + traverse(root, lpath(pattern), function(r,d,k) + if d then + local ek = d[k] + local tx = ek and ek.dt + if flatten then + if tx then + t[#t+1] = xml.tostring(tx) or "" + else + t[#t+1] = "" + end + else + t[#t+1] = tx or "" + end + else + t[#t+1] = "" + end + end) + return t + end + + function xml.collect_tags(root, pattern, nonamespace) + local t = { } + xml.traverse(root, xml.lpath(pattern), function(r,d,k) + local dk = d and d[k] + if dk and type(dk) == "table" then + local ns, tg = e.ns, e.tg + if nonamespace then + t[#t+1] = tg -- if needed we can return an extra table + elseif ns == "" then + t[#t+1] = tg + else + t[#t+1] = ns .. ":" .. tg + end + end + end) + return #t > 0 and {} + end + + --[[ldx-- + <p>Often using an iterators looks nicer in the code than passing handler + functions. The <l n='lua'/> book describes how to use coroutines for that + purpose (<url href='http://www.lua.org/pil/9.3.html'/>). This permits + code like:</p> + + <typing> + for r, d, k in xml.elements(xml.load('text.xml'),"title") do + print(d[k]) + end + </typing> + + <p>Which will print all the titles in the document. The iterator variant takes + 1.5 times the runtime of the function variant which is due to the overhead in + creating the wrapper. So, instead of:</p> + + <typing> + function xml.filters.first(root,pattern) + for rt,dt,dk in xml.elements(root,pattern) + return dt and dt[dk], rt, dt, dk + end + return nil, nil, nil, nil + end + </typing> + + <p>We use the function variants in the filters.</p> + --ldx]]-- + + local wrap, yield = coroutine.wrap, coroutine.yield + + function xml.elements(root,pattern,reverse) + return wrap(function() traverse(root, lpath(pattern), yield, reverse) end) + end + + function xml.elements_only(root,pattern,reverse) + return wrap(function() traverse(root, lpath(pattern), function(r,d,k) yield(d[k]) end, reverse) end) + end + + function xml.each_element(root, pattern, handle, reverse) + local ok + traverse(root, lpath(pattern), function(r,d,k) ok = true handle(r,d,k) end, reverse) + return ok + end + + function xml.process_elements(root, pattern, handle) + traverse(root, lpath(pattern), function(r,d,k) + local dkdt = d[k].dt + if dkdt then + for i=1,#dkdt do + local v = dkdt[i] + if v.tg then handle(v) end + end + end + end) + end + + function xml.process_attributes(root, pattern, handle) + traverse(root, lpath(pattern), function(r,d,k) + local ek = d[k] + local a = ek.at or { } + handle(a) + if next(a) then -- next is faster than type (and >0 test) + ek.at = a + else + ek.at = nil + end + end) + end + + --[[ldx-- + <p>We've now arrives at the functions that manipulate the tree.</p> + --ldx]]-- + + function xml.inject_element(root, pattern, element, prepend) + if root and element then + local matches, collect = { }, nil + if type(element) == "string" then + element = convert(element,true) + end + if element then + collect = function(r,d,k) matches[#matches+1] = { r, d, k, element } end + traverse(root, lpath(pattern), collect) + for i=1,#matches do + local m = matches[i] + local r, d, k, element, edt = m[1], m[2], m[3], m[4], nil + if element.ri then + element = element.dt[element.ri].dt + else + element = element.dt + end + if r.ri then + edt = r.dt[r.ri].dt + else + edt = d and d[k] and d[k].dt + end + if edt then + local be, af + if prepend then + be, af = xml.copy(element), edt + else + be, af = edt, xml.copy(element) + end + for i=1,#af do + be[#be+1] = af[i] + end + if r.ri then + r.dt[r.ri].dt = be + else + d[k].dt = be + end + else + -- r.dt = element.dt -- todo + end + end + end + end + end + + -- todo: copy ! + + function xml.insert_element(root, pattern, element, before) -- todo: element als functie + if root and element then + if pattern == "/" then + xml.inject_element(root, pattern, element, before) + else + local matches, collect = { }, nil + if type(element) == "string" then + element = convert(element,true) + end + if element and element.ri then + element = element.dt[element.ri] + end + if element then + collect = function(r,d,k) matches[#matches+1] = { r, d, k, element } end + traverse(root, lpath(pattern), collect) + for i=#matches,1,-1 do + local m = matches[i] + local r, d, k, element = m[1], m[2], m[3], m[4] + if not before then k = k + 1 end + if element.tg then + table.insert(d,k,element) -- untested + elseif element.dt then + for _,v in ipairs(element.dt) do -- i added + table.insert(d,k,v) + k = k + 1 + end + end + end + end + end + end + end + + xml.insert_element_after = xml.insert_element + xml.insert_element_before = function(r,p,e) xml.insert_element(r,p,e,true) end + xml.inject_element_after = xml.inject_element + xml.inject_element_before = function(r,p,e) xml.inject_element(r,p,e,true) end + + function xml.delete_element(root, pattern) + local matches, deleted = { }, { } + local collect = function(r,d,k) matches[#matches+1] = { r, d, k } end + traverse(root, lpath(pattern), collect) + for i=#matches,1,-1 do + local m = matches[i] + deleted[#deleted+1] = table.remove(m[2],m[3]) + end + return deleted + end + + function xml.replace_element(root, pattern, element) + if type(element) == "string" then + element = convert(element,true) + end + if element and element.ri then + element = element.dt[element.ri] + end + if element then + traverse(root, lpath(pattern), function(rm, d, k) + d[k] = element.dt -- maybe not clever enough + end) + end + end + + local function load_data(name) -- == io.loaddata + local f, data = io.open(name), "" + if f then + data = f:read("*all",'b') -- 'b' ? + f:close() + end + return data + end + + function xml.include(xmldata,pattern,attribute,recursive,loaddata) + -- parse="text" (default: xml), encoding="" (todo) + -- attribute = attribute or 'href' + pattern = pattern or 'include' + loaddata = loaddata or load_data + local function include(r,d,k) + local ek, name = d[k], nil + if not attribute or attribute == "" then + local ekdt = ek.dt + name = (type(ekdt) == "table" and ekdt[1]) or ekdt + end + if not name then + if ek.at then + for a in (attribute or "href"):gmatch("([^|]+)") do + name = ek.at[a] + if name then break end + end + end + end + local data = (name and name ~= "" and loaddata(name)) or "" + if data == "" then + xml.empty(d,k) + elseif ek.at["parse"] == "text" then -- for the moment hard coded + d[k] = xml.escaped(data) + else + local xi = xml.convert(data) + if not xi then + xml.empty(d,k) + else + if recursive then + xml.include(xi,pattern,attribute,recursive,loaddata) + end + xml.assign(d,k,xi) + end + end + end + xml.each_element(xmldata, pattern, include) + end + + function xml.strip_whitespace(root, pattern) + traverse(root, lpath(pattern), function(r,d,k) + local dkdt = d[k].dt + if dkdt then -- can be optimized + local t = { } + for i=1,#dkdt do + local str = dkdt[i] + if type(str) == "string" and str:find("^[ \n\r\t]*$") then + -- stripped + else + t[#t+1] = str + end + end + d[k].dt = t + end + end) + end + + function xml.rename_space(root, oldspace, newspace) -- fast variant + local ndt = #root.dt + local rename = xml.rename_space + for i=1,ndt or 0 do + local e = root[i] + if type(e) == "table" then + if e.ns == oldspace then + e.ns = newspace + if e.rn then + e.rn = newspace + end + end + local edt = e.dt + if edt then + rename(edt, oldspace, newspace) + end + end + end + end + + function xml.remap_tag(root, pattern, newtg) + traverse(root, lpath(pattern), function(r,d,k) + d[k].tg = newtg + end) + end + function xml.remap_namespace(root, pattern, newns) + traverse(root, lpath(pattern), function(r,d,k) + d[k].ns = newns + end) + end + function xml.check_namespace(root, pattern, newns) + traverse(root, lpath(pattern), function(r,d,k) + local dk = d[k] + if (not dk.rn or dk.rn == "") and dk.ns == "" then + dk.rn = newns + end + end) + end + function xml.remap_name(root, pattern, newtg, newns, newrn) + traverse(root, lpath(pattern), function(r,d,k) + local dk = d[k] + dk.tg = newtg + dk.ns = newns + dk.rn = newrn + end) + end + + function xml.filters.found(root,pattern,check_content) + local found = false + traverse(root, lpath(pattern), function(r,d,k) + if check_content then + local dk = d and d[k] + found = dk and dk.dt and next(dk.dt) and true + else + found = true + end + return true + end) + return found + end + +end + +--[[ldx-- +<p>Here are a few synonyms.</p> +--ldx]]-- + +xml.filters.position = xml.filters.index + +xml.count = xml.filters.count +xml.index = xml.filters.index +xml.position = xml.filters.index +xml.first = xml.filters.first +xml.last = xml.filters.last +xml.found = xml.filters.found + +xml.each = xml.each_element +xml.process = xml.process_element +xml.strip = xml.strip_whitespace +xml.collect = xml.collect_elements +xml.all = xml.collect_elements + +xml.insert = xml.insert_element_after +xml.inject = xml.inject_element_after +xml.after = xml.insert_element_after +xml.before = xml.insert_element_before +xml.delete = xml.delete_element +xml.replace = xml.replace_element + +--[[ldx-- +<p>The following helper functions best belong to the <t>lmxl-ini</t> +module. Some are here because we need then in the <t>mk</t> +document and other manuals, others came up when playing with +this module. Since this module is also used in <l n='mtxrun'/> we've +put them here instead of loading mode modules there then needed.</p> +--ldx]]-- + +function xml.gsub(t,old,new) + if t.dt then + for k,v in ipairs(t.dt) do + if type(v) == "string" then + t.dt[k] = v:gsub(old,new) + else + xml.gsub(v,old,new) + end + end + end +end + +function xml.strip_leading_spaces(dk,d,k) -- cosmetic, for manual + if d and k and d[k-1] and type(d[k-1]) == "string" then + local s = d[k-1]:match("\n(%s+)") + xml.gsub(dk,"\n"..string.rep(" ",#s),"\n") + end +end + +function xml.serialize_path(root,lpath,handle) + local dk, r, d, k = xml.first(root,lpath) + dk = xml.copy(dk) + xml.strip_leading_spaces(dk,d,k) + xml.serialize(dk,handle) +end + +--~ xml.escapes = { ['&'] = '&', ['<'] = '<', ['>'] = '>', ['"'] = '"' } +--~ xml.unescapes = { } for k,v in pairs(xml.escapes) do xml.unescapes[v] = k end + +--~ function xml.escaped (str) return str:gsub("(.)" , xml.escapes ) end +--~ function xml.unescaped(str) return str:gsub("(&.-;)", xml.unescapes) end +--~ function xml.cleansed (str) return str:gsub("<.->" , '' ) end -- "%b<>" + +do + + local P, S, R, C, V, Cc, Cs = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc, lpeg.Cs + + -- 100 * 2500 * "oeps< oeps> oeps&" : gsub:lpeg|lpeg|lpeg + -- + -- 1021:0335:0287:0247 + + -- 10 * 1000 * "oeps< oeps> oeps& asfjhalskfjh alskfjh alskfjh alskfjh ;al J;LSFDJ" + -- + -- 1559:0257:0288:0190 (last one suggested by roberto) + + -- escaped = Cs((S("<&>") / xml.escapes + 1)^0) + -- escaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0) + local normal = (1 - S("<&>"))^0 + local special = P("<")/"<" + P(">")/">" + P("&")/"&" + local escaped = Cs(normal * (special * normal)^0) + + -- 100 * 1000 * "oeps< oeps> oeps&" : gsub:lpeg == 0153:0280:0151:0080 (last one by roberto) + + -- unescaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0) + -- unescaped = Cs((((P("&")/"") * (P("lt")/"<" + P("gt")/">" + P("amp")/"&") * (P(";")/"")) + 1)^0) + local normal = (1 - S"&")^0 + local special = P("<")/"<" + P(">")/">" + P("&")/"&" + local unescaped = Cs(normal * (special * normal)^0) + + -- 100 * 5000 * "oeps <oeps bla='oeps' foo='bar'> oeps </oeps> oeps " : gsub:lpeg == 623:501 msec (short tags, less difference) + + local cleansed = Cs(((P("<") * (1-P(">"))^0 * P(">"))/"" + 1)^0) + + function xml.escaped (str) return escaped :match(str) end + function xml.unescaped(str) return unescaped:match(str) end + function xml.cleansed (str) return cleansed :match(str) end + +end + +function xml.join(t,separator,lastseparator) + if #t > 0 then + local result = { } + for k,v in pairs(t) do + result[k] = xml.tostring(v) + end + if lastseparator then + return concat(result,separator or "",1,#result-1) .. (lastseparator or "") .. result[#result] + else + return concat(result,separator) + end + else + return "" + end +end + + +--[[ldx-- +<p>We provide (at least here) two entity handlers. The more extensive +resolver consults a hash first, tries to convert to <l n='utf'/> next, +and finaly calls a handler when defines. When this all fails, the +original entity is returned.</p> +--ldx]]-- + +do if unicode and unicode.utf8 then + + xml.entities = xml.entities or { } -- xml.entities.handler == function + + function xml.entities.handler(e) + return format("[%s]",e) + end + + local char = unicode.utf8.char + + local function toutf(s) + return char(tonumber(s,16)) + end + + local entities = xml.entities -- global entities + + function utfize(root) + local d = root.dt + for k=1,#d do + local dk = d[k] + if type(dk) == "string" then + -- test prevents copying if no match + if dk:find("&#x.-;") then + d[k] = dk:gsub("&#x(.-);",toutf) + end + else + utfize(dk) + end + end + end + + xml.utfize = utfize + + local function resolve(e) -- hex encoded always first, just to avoid mkii fallbacks + if e:find("#x") then + return char(tonumber(e:sub(3),16)) + else + local ee = entities[e] + if ee then + return ee + else + local h = xml.entities.handler + return (h and h(e)) or "&" .. e .. ";" + end + end + end + + local function resolve_entities(root) + if not root.special or root.tg == "@rt@" then + local d = root.dt + for k=1,#d do + local dk = d[k] + if type(dk) == "string" then + if dk:find("&.-;") then + d[k] = dk:gsub("&(.-);",resolve) + end + else + resolve_entities(dk) + end + end + end + end + + xml.resolve_entities = resolve_entities + + function xml.utfize_text(str) + if str:find("&#") then + return (str:gsub("&#x(.-);",toutf)) + else + return str + end + end + + function xml.resolve_text_entities(str) -- maybe an lpeg. maybe resolve inline + if str:find("&") then + return (str:gsub("&(.-);",resolve)) + else + return str + end + end + + function xml.show_text_entities(str) + if str:find("&") then + return (str:gsub("&(.-);","[%1]")) + else + return str + end + end + + -- experimental, this will be done differently + + function xml.merge_entities(root) + local documententities = root.entities + local allentities = xml.entities + if documententities then + for k, v in pairs(documententities) do + allentities[k] = v + end + end + end + +end end + +-- xml.set_text_cleanup(xml.show_text_entities) +-- xml.set_text_cleanup(xml.resolve_text_entities) + +--~ xml.lshow("/../../../a/(b|c)[@d='e']/f") +--~ xml.lshow("/../../../a/!(b|c)[@d='e']/f") +--~ xml.lshow("/../../../a/!b[@d!='e']/f") + +--~ x = xml.convert([[ +--~ <a> +--~ <b n='01'>01</b> +--~ <b n='02'>02</b> +--~ <b n='03'>03</b> +--~ <b n='04'>OK</b> +--~ <b n='05'>05</b> +--~ <b n='06'>06</b> +--~ <b n='07'>ALSO OK</b> +--~ </a> +--~ ]]) + +--~ xml.trace_lpath = true + +--~ xml.xshow(xml.first(x,"b[position() > 2 and position() < 5 and text() == 'ok']")) +--~ xml.xshow(xml.first(x,"b[position() > 2 and position() < 5 and text() == upper('ok')]")) +--~ xml.xshow(xml.first(x,"b[@n=='03' or @n=='08']")) +--~ xml.xshow(xml.all (x,"b[number(@n)>2 and number(@n)<6]")) +--~ xml.xshow(xml.first(x,"b[find(text(),'ALSO')]")) + +--~ str = [[ +--~ <?xml version="1.0" encoding="utf-8"?> +--~ <story line='mojca'> +--~ <windows>my secret</mouse> +--~ </story> +--~ ]] + +--~ x = xml.convert([[ +--~ <a><b n='01'>01</b><b n='02'>02</b><x>xx</x><b n='03'>03</b><b n='04'>OK</b></a> +--~ ]]) +--~ xml.xshow(xml.first(x,"b[tag(2) == 'x']")) +--~ xml.xshow(xml.first(x,"b[tag(1) == 'x']")) +--~ xml.xshow(xml.first(x,"b[tag(-1) == 'x']")) +--~ xml.xshow(xml.first(x,"b[tag(-2) == 'x']")) + +--~ print(xml.filter(x,"b/tag(2)")) +--~ print(xml.filter(x,"b/tag(1)")) + + +-- filename : l-utils.lua +-- comment : split off from luat-lib +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-utils'] = 1.001 + +if not utils then utils = { } end +if not utils.merger then utils.merger = { } end +if not utils.lua then utils.lua = { } end + +utils.merger.m_begin = "begin library merge" +utils.merger.m_end = "end library merge" +utils.merger.pattern = + "%c+" .. + "%-%-%s+" .. utils.merger.m_begin .. + "%c+(.-)%c+" .. + "%-%-%s+" .. utils.merger.m_end .. + "%c+" + +function utils.merger._self_fake_() + return + "-- " .. "created merged file" .. "\n\n" .. + "-- " .. utils.merger.m_begin .. "\n\n" .. + "-- " .. utils.merger.m_end .. "\n\n" +end + +function utils.report(...) + print(...) +end + +function utils.merger._self_load_(name) + local f, data = io.open(name), "" + if f then + data = f:read("*all") + f:close() + end + return data or "" +end + +function utils.merger._self_save_(name, data) + if data ~= "" then + local f = io.open(name,'w') + if f then + f:write(data) + f:close() + end + end +end + +function utils.merger._self_swap_(data,code) + if data ~= "" then + return (data:gsub(utils.merger.pattern, function(s) + return "\n\n" .. "-- "..utils.merger.m_begin .. "\n" .. code .. "\n" .. "-- "..utils.merger.m_end .. "\n\n" + end, 1)) + else + return "" + end +end + +function utils.merger._self_libs_(libs,list) + local result, f = { }, nil + if type(libs) == 'string' then libs = { libs } end + if type(list) == 'string' then list = { list } end + for _, lib in ipairs(libs) do + for _, pth in ipairs(list) do + local name = string.gsub(pth .. "/" .. lib,"\\","/") + f = io.open(name) + if f then + -- utils.report("merging library",name) + result[#result+1] = f:read("*all") + f:close() + list = { pth } -- speed up the search + break + else + -- utils.report("no library",name) + end + end + end + return table.concat(result, "\n\n") +end + +function utils.merger.selfcreate(libs,list,target) + if target then + utils.merger._self_save_( + target, + utils.merger._self_swap_( + utils.merger._self_fake_(), + utils.merger._self_libs_(libs,list) + ) + ) + end +end + +function utils.merger.selfmerge(name,libs,list,target) + utils.merger._self_save_( + target or name, + utils.merger._self_swap_( + utils.merger._self_load_(name), + utils.merger._self_libs_(libs,list) + ) + ) +end + +function utils.merger.selfclean(name) + utils.merger._self_save_( + name, + utils.merger._self_swap_( + utils.merger._self_load_(name), + "" + ) + ) +end + +utils.lua.compile_strip = true + +function utils.lua.compile(luafile, lucfile) + -- utils.report("compiling",luafile,"into",lucfile) + os.remove(lucfile) + local command = "-o " .. string.quote(lucfile) .. " " .. string.quote(luafile) + if utils.lua.compile_strip then + command = "-s " .. command + end + if os.spawn("texluac " .. command) == 0 then + return true + elseif os.spawn("luac " .. command) == 0 then + return true + else + return false + end +end + + + +-- filename : luat-lib.lua +-- comment : companion to luat-lib.tex +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['luat-lib'] = 1.001 + +-- mostcode moved to the l-*.lua and other luat-*.lua files + +-- os / io + +os.setlocale(nil,nil) -- useless feature and even dangerous in luatex + +-- os.platform + +-- mswin|bccwin|mingw|cygwin windows +-- darwin|rhapsody|nextstep macosx +-- netbsd|unix unix +-- linux linux + +if not io.fileseparator then + if string.find(os.getenv("PATH"),";") then + io.fileseparator, io.pathseparator, os.platform = "\\", ";", os.type or "windows" + else + io.fileseparator, io.pathseparator, os.platform = "/" , ":", os.type or "unix" + end +end + +os.platform = os.platform or os.type or (io.pathseparator == ";" and "windows") or "unix" + +-- arg normalization +-- +-- for k,v in pairs(arg) do print(k,v) end + +-- environment + +if not environment then environment = { } end + +environment.ownbin = environment.ownbin or arg[-2] or arg[-1] or arg[0] or "luatex" + +local ownpath = nil -- we could use a metatable here + +function environment.ownpath() + if not ownpath then + for p in string.gmatch(os.getenv("PATH"),"[^"..io.pathseparator.."]+") do + local b = file.join(p,environment.ownbin) + if lfs.isfile(b..".exe") or lfs.isfile(b) then + ownpath = p + break + end + end + if not ownpath then ownpath = '.' end + end + return ownpath +end + +if arg and (arg[0] == 'luatex' or arg[0] == 'luatex.exe') and arg[1] == "--luaonly" then + arg[-1]=arg[0] arg[0]=arg[2] for k=3,#arg do arg[k-2]=arg[k] end arg[#arg]=nil arg[#arg]=nil +end + +environment.arguments = { } +environment.files = { } +environment.sorted_argument_keys = nil + +environment.platform = os.platform + +function environment.initialize_arguments(arg) + environment.arguments = { } + environment.files = { } + environment.sorted_argument_keys = nil + for index, argument in pairs(arg) do + if index > 0 then + local flag, value = argument:match("^%-+(.+)=(.-)$") + if flag then + environment.arguments[flag] = string.unquote(value or "") + else + flag = argument:match("^%-+(.+)") + if flag then + environment.arguments[flag] = true + else + environment.files[#environment.files+1] = argument + end + end + end + end + environment.ownname = environment.ownname or arg[0] or 'unknown.lua' +end + +function environment.showarguments() + for k,v in pairs(environment.arguments) do + print(k .. " : " .. tostring(v)) + end + if #environment.files > 0 then + print("files : " .. table.concat(environment.files, " ")) + end +end + +function environment.setargument(name,value) + environment.arguments[name] = value +end + +function environment.argument(name) + if environment.arguments[name] then + return environment.arguments[name] + else + if not environment.sorted_argument_keys then + environment.sorted_argument_keys = { } + for _,v in pairs(table.sortedkeys(environment.arguments)) do + table.insert(environment.sorted_argument_keys, "^" .. v) + end + end + for _,v in pairs(environment.sorted_argument_keys) do + if name:find(v) then + return environment.arguments[v:sub(2,#v)] + end + end + end + return nil +end + +function environment.split_arguments(separator) -- rather special, cut-off before separator + local done, before, after = false, { }, { } + for _,v in ipairs(environment.original_arguments) do + if not done and v == separator then + done = true + elseif done then + after[#after+1] = v + else + before[#before+1] = v + end + end + return before, after +end + +function environment.reconstruct_commandline(arg) + if not arg then arg = environment.original_arguments end + local result = { } + for _,a in ipairs(arg) do -- ipairs 1 .. #n + local kk, vv = a:match("^(%-+.-)=(.+)$") + if kk and vv then + if vv:find(" ") then + result[#result+1] = kk .. "=" .. string.quote(vv) + else + result[#result+1] = a + end + elseif a:find(" ") then + result[#result+1] = string.quote(a) + else + result[#result+1] = a + end + end + return table.join(result," ") +end + +if arg then + environment.initialize_arguments(arg) + environment.original_arguments = arg + arg = { } -- prevent duplicate handling +end + + +-- filename : luat-inp.lua +-- comment : companion to luat-lib.tex +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +-- This lib is multi-purpose and can be loaded again later on so that +-- additional functionality becomes available. We will split this +-- module in components when we're done with prototyping. + +-- TODO: os.getenv -> os.env[] +-- TODO: instances.[hashes,cnffiles,configurations,522] -> ipairs (alles check, sneller) +-- TODO: check escaping in find etc, too much, too slow + +-- This is the first code I wrote for LuaTeX, so it needs some cleanup. + +-- To be considered: hash key lowercase, first entry in table filename +-- (any case), rest paths (so no need for optimization). Or maybe a +-- separate table that matches lowercase names to mixed case when +-- present. In that case the lower() cases can go away. I will do that +-- only when we run into problems with names ... well ... Iwona-Regular. + +-- Beware, loading and saving is overloaded in luat-tmp! + +if not versions then versions = { } end versions['luat-inp'] = 1.001 +if not environment then environment = { } end +if not file then file = { } end + +if environment.aleph_mode == nil then environment.aleph_mode = true end -- temp hack + +if not input then input = { } end +if not input.suffixes then input.suffixes = { } end +if not input.formats then input.formats = { } end +if not input.aux then input.aux = { } end + +if not input.suffixmap then input.suffixmap = { } end + +if not input.locators then input.locators = { } end -- locate databases +if not input.hashers then input.hashers = { } end -- load databases +if not input.generators then input.generators = { } end -- generate databases +if not input.filters then input.filters = { } end -- conversion filters + +local format = string.format + +input.locators.notfound = { nil } +input.hashers.notfound = { nil } +input.generators.notfound = { nil } + +input.cacheversion = '1.0.1' +input.banner = nil +input.verbose = false +input.debug = false +input.cnfname = 'texmf.cnf' +input.luaname = 'texmfcnf.lua' +input.lsrname = 'ls-R' +input.luasuffix = '.tma' +input.lucsuffix = '.tmc' + +-- we use a cleaned up list / format=any is a wildcard, as is *name + +input.formats['afm'] = 'AFMFONTS' input.suffixes['afm'] = { 'afm' } +input.formats['enc'] = 'ENCFONTS' input.suffixes['enc'] = { 'enc' } +input.formats['fmt'] = 'TEXFORMATS' input.suffixes['fmt'] = { 'fmt' } +input.formats['map'] = 'TEXFONTMAPS' input.suffixes['map'] = { 'map' } +input.formats['mp'] = 'MPINPUTS' input.suffixes['mp'] = { 'mp' } +input.formats['ocp'] = 'OCPINPUTS' input.suffixes['ocp'] = { 'ocp' } +input.formats['ofm'] = 'OFMFONTS' input.suffixes['ofm'] = { 'ofm', 'tfm' } +input.formats['otf'] = 'OPENTYPEFONTS' input.suffixes['otf'] = { 'otf' } -- 'ttf' +input.formats['opl'] = 'OPLFONTS' input.suffixes['opl'] = { 'opl' } +input.formats['otp'] = 'OTPINPUTS' input.suffixes['otp'] = { 'otp' } +input.formats['ovf'] = 'OVFFONTS' input.suffixes['ovf'] = { 'ovf', 'vf' } +input.formats['ovp'] = 'OVPFONTS' input.suffixes['ovp'] = { 'ovp' } +input.formats['tex'] = 'TEXINPUTS' input.suffixes['tex'] = { 'tex' } +input.formats['tfm'] = 'TFMFONTS' input.suffixes['tfm'] = { 'tfm' } +input.formats['ttf'] = 'TTFONTS' input.suffixes['ttf'] = { 'ttf', 'ttc' } +input.formats['pfb'] = 'T1FONTS' input.suffixes['pfb'] = { 'pfb', 'pfa' } +input.formats['vf'] = 'VFFONTS' input.suffixes['vf'] = { 'vf' } + +input.formats['fea'] = 'FONTFEATURES' input.suffixes['fea'] = { 'fea' } +input.formats['cid'] = 'FONTCIDMAPS' input.suffixes['cid'] = { 'cid', 'cidmap' } + +input.formats ['texmfscripts'] = 'TEXMFSCRIPTS' -- new +input.suffixes['texmfscripts'] = { 'rb', 'pl', 'py' } -- 'lua' + +input.formats ['lua'] = 'LUAINPUTS' -- new +input.suffixes['lua'] = { 'lua', 'luc', 'tma', 'tmc' } + +-- here we catch a few new thingies (todo: add these paths to context.tmf) +-- +-- FONTFEATURES = .;$TEXMF/fonts/fea// +-- FONTCIDMAPS = .;$TEXMF/fonts/cid// + +function input.checkconfigdata(instance) -- not yet ok, no time for debugging now + local function fix(varname,default) + local proname = varname .. "." .. instance.progname or "crap" + local p = instance.environment[proname] + local v = instance.environment[varname] + if not ((p and p ~= "") or (v and v ~= "")) then + instance.variables[varname] = default -- or environment? + end + end + fix("LUAINPUTS" , ".;$TEXINPUTS;$TEXMFSCRIPTS") + fix("FONTFEATURES", ".;$TEXMF/fonts/fea//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") + fix("FONTCIDMAPS" , ".;$TEXMF/fonts/cid//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") +end + +-- backward compatible ones + +input.alternatives = { } + +input.alternatives['map files'] = 'map' +input.alternatives['enc files'] = 'enc' +input.alternatives['cid files'] = 'cid' +input.alternatives['fea files'] = 'fea' +input.alternatives['opentype fonts'] = 'otf' +input.alternatives['truetype fonts'] = 'ttf' +input.alternatives['truetype collections'] = 'ttc' +input.alternatives['type1 fonts'] = 'pfb' + +-- obscure ones + +input.formats ['misc fonts'] = '' +input.suffixes['misc fonts'] = { } + +input.formats ['sfd'] = 'SFDFONTS' +input.suffixes ['sfd'] = { 'sfd' } +input.alternatives['subfont definition files'] = 'sfd' + +function input.reset() + + local instance = { } + + instance.rootpath = '' + instance.treepath = '' + instance.progname = environment.progname or 'context' + instance.engine = environment.engine or 'luatex' + instance.format = '' + instance.environment = { } + instance.variables = { } + instance.expansions = { } + instance.files = { } + instance.remap = { } + instance.configuration = { } + instance.setup = { } + instance.order = { } + instance.found = { } + instance.foundintrees = { } + instance.kpsevars = { } + instance.hashes = { } + instance.cnffiles = { } + instance.luafiles = { } + instance.lists = { } + instance.remember = true + instance.diskcache = true + instance.renewcache = false + instance.scandisk = true + instance.cachepath = nil + instance.loaderror = false + instance.smallcache = false + instance.savelists = true + instance.cleanuppaths = true + instance.allresults = false + instance.pattern = nil -- lists + instance.kpseonly = false -- lists + instance.cachefile = 'tmftools' + instance.loadtime = 0 + instance.starttime = 0 + instance.stoptime = 0 + instance.validfile = function(path,name) return true end + instance.data = { } -- only for loading + instance.force_suffixes = true + instance.dummy_path_expr = "^!*unset/*$" + instance.fakepaths = { } + instance.lsrmode = false + + if os.env then + -- store once, freeze and faster + for k,v in pairs(os.env) do + instance.environment[k] = input.bare_variable(v) + end + else + -- we will access os.env frequently + for k,v in pairs({'HOME','TEXMF','TEXMFCNF'}) do + local e = os.getenv(v) + if e then + -- input.report("setting",v,"to",input.bare_variable(e)) + instance.environment[v] = input.bare_variable(e) + end + end + end + + -- cross referencing + + for k, v in pairs(input.suffixes) do + for _, vv in pairs(v) do + if vv then + input.suffixmap[vv] = k + end + end + end + + return instance + +end + +function input.reset_hashes(instance) + instance.lists = { } + instance.found = { } +end + +function input.bare_variable(str) -- assumes str is a string + -- return string.gsub(string.gsub(string.gsub(str,"%s+$",""),'^"(.+)"$',"%1"),"^'(.+)'$","%1") + return (str:gsub("\s*([\"\']?)(.+)%1\s*", "%2")) +end + +if texio then + input.log = texio.write_nl +else + input.log = print +end + +function input.simple_logger(kind, name) + if name and name ~= "" then + if input.banner then + input.log(input.banner..kind..": "..name) + else + input.log("<<"..kind..": "..name..">>") + end + else + if input.banner then + input.log(input.banner..kind..": no name") + else + input.log("<<"..kind..": no name>>") + end + end +end + +function input.dummy_logger() +end + +function input.settrace(n) + input.trace = tonumber(n or 0) + if input.trace > 0 then + input.logger = input.simple_logger + input.verbose = true + else + input.logger = function() end + end +end + +function input.report(...) -- inefficient + if input.verbose then + if input.banner then + input.log(input.banner .. table.concat({...},' ')) + elseif input.logmode() == 'xml' then + input.log("<t>"..table.concat({...},' ').."</t>") + else + input.log("<<"..table.concat({...},' ')..">>") + end + end +end + +function input.reportlines(str) + if type(str) == "string" then + str = str:split("\n") + end + for _,v in pairs(str) do input.report(v) end +end + +input.settrace(tonumber(os.getenv("MTX.INPUT.TRACE") or os.getenv("MTX_INPUT_TRACE") or input.trace or 0)) + +-- These functions can be used to test the performance, especially +-- loading the database files. + +do + local clock = os.gettimeofday or os.clock + + function input.starttiming(instance) + if instance then + instance.starttime = clock() + if not instance.loadtime then + instance.loadtime = 0 + end + end + end + + function input.stoptiming(instance, report) + if instance then + local starttime = instance.starttime + if starttime then + local stoptime = clock() + local loadtime = stoptime - starttime + instance.stoptime = stoptime + instance.loadtime = instance.loadtime + loadtime + if report then + input.report('load time', format("%0.3f",loadtime)) + end + return loadtime + end + end + return 0 + end + +end + +function input.elapsedtime(instance) + return format("%0.3f",(instance and instance.loadtime) or 0) +end + +function input.report_loadtime(instance) + if instance then + input.report('total load time', input.elapsedtime(instance)) + end +end + +input.loadtime = input.elapsedtime + +function input.env(instance,key) + return instance.environment[key] or input.osenv(instance,key) +end + +function input.osenv(instance,key) + local ie = instance.environment + local value = ie[key] + if value == nil then + -- local e = os.getenv(key) + local e = os.env[key] + if e == nil then + -- value = "" -- false + else + value = input.bare_variable(e) + end + ie[key] = value + end + return value or "" +end + +-- we follow a rather traditional approach: +-- +-- (1) texmf.cnf given in TEXMFCNF +-- (2) texmf.cnf searched in TEXMF/web2c +-- +-- for the moment we don't expect a configuration file in a zip + +function input.identify_cnf(instance) + -- we no longer support treepath and rootpath (was handy for testing); + -- also we now follow the stupid route: if not set then just assume *one* + -- cnf file under texmf (i.e. distribution) + if #instance.cnffiles == 0 then + if input.env(instance,'TEXMFCNF') == "" then + local ownpath = environment.ownpath() or "." + if ownpath then + -- beware, this is tricky on my own system because at that location I do have + -- the raw tree that ends up in the zip; i.e. I cannot test this kind of mess + local function locate(filename,list) + local ownroot = input.normalize_name(file.join(ownpath,"../..")) + if not lfs.isdir(file.join(ownroot,"texmf")) then + ownroot = input.normalize_name(file.join(ownpath,"..")) + if not lfs.isdir(file.join(ownroot,"texmf")) then + input.verbose = true + input.report("error", "unable to identify cnf file") + return + end + end + local texmfcnf = file.join(ownroot,"texmf-local/web2c",filename) -- for minimals and myself + if not lfs.isfile(texmfcnf) then + texmfcnf = file.join(ownroot,"texmf/web2c",filename) + if not lfs.isfile(texmfcnf) then + input.verbose = true + input.report("error", "unable to locate",filename) + return + end + end + table.insert(list,texmfcnf) + local ie = instance.environment + if not ie['SELFAUTOPARENT'] then ie['SELFAUTOPARENT'] = ownroot end + if not ie['TEXMFCNF'] then ie['TEXMFCNF'] = file.dirname(texmfcnf) end + end + locate(input.luaname,instance.luafiles) + locate(input.cnfname,instance.cnffiles) + if #instance.luafiles == 0 and instance.cnffiles == 0 then + input.verbose = true + input.report("error", "unable to locate",filename) + os.exit() + end + -- here we also assume then TEXMF is set in the distribution, if this trickery is + -- used in the minimals, then users who don't use setuptex are on their own with + -- regards to extra trees + else + input.verbose = true + input.report("error", "unable to identify own path") + os.exit() + end + else + local t = input.split_path(input.env(instance,'TEXMFCNF')) + t = input.aux.expanded_path(instance,t) + input.aux.expand_vars(instance,t) + local function locate(filename,list) + for _,v in ipairs(t) do + local texmfcnf = input.normalize_name(file.join(v,filename)) + if lfs.isfile(texmfcnf) then + table.insert(list,texmfcnf) + end + end + end + locate(input.luaname,instance.luafiles) + locate(input.cnfname,instance.cnffiles) + end + end +end + +function input.load_cnf(instance) + local function loadoldconfigdata() + for _, fname in ipairs(instance.cnffiles) do + input.aux.load_cnf(instance,fname) + end + end + -- instance.cnffiles contain complete names now ! + if #instance.cnffiles == 0 then + input.report("no cnf files found (TEXMFCNF may not be set/known)") + else + instance.rootpath = instance.cnffiles[1] + for k,fname in ipairs(instance.cnffiles) do + instance.cnffiles[k] = input.normalize_name(fname:gsub("\\",'/')) + end + for i=1,3 do + instance.rootpath = file.dirname(instance.rootpath) + end + instance.rootpath = input.normalize_name(instance.rootpath) + instance.environment['SELFAUTOPARENT'] = instance.rootpath -- just to be sure + if instance.lsrmode then + loadoldconfigdata() + elseif instance.diskcache and not instance.renewcache then + input.loadoldconfig(instance,instance.cnffiles) + if instance.loaderror then + loadoldconfigdata() + input.saveoldconfig(instance) + end + else + loadoldconfigdata() + if instance.renewcache then + input.saveoldconfig(instance) + end + end + input.aux.collapse_cnf_data(instance) + end + input.checkconfigdata(instance) +end + +function input.load_lua(instance) + if #instance.luafiles == 0 then + -- yet harmless + else + instance.rootpath = instance.luafiles[1] + for k,fname in ipairs(instance.luafiles) do + instance.luafiles[k] = input.normalize_name(fname:gsub("\\",'/')) + end + for i=1,3 do + instance.rootpath = file.dirname(instance.rootpath) + end + instance.rootpath = input.normalize_name(instance.rootpath) + instance.environment['SELFAUTOPARENT'] = instance.rootpath -- just to be sure + input.loadnewconfig(instance) + input.aux.collapse_cnf_data(instance) + end + input.checkconfigdata(instance) +end + +function input.aux.collapse_cnf_data(instance) -- potential optmization: pass start index (setup and configuration are shared) + for _,c in ipairs(instance.order) do + for k,v in pairs(c) do + if not instance.variables[k] then + if instance.environment[k] then + instance.variables[k] = instance.environment[k] + else + instance.kpsevars[k] = true + instance.variables[k] = input.bare_variable(v) + end + end + end + end +end + +function input.aux.load_cnf(instance,fname) + fname = input.clean_path(fname) + local lname = fname:gsub("%.%a+$",input.luasuffix) + local f = io.open(lname) + if f then -- this will go + f:close() + local dname = file.dirname(fname) + if not instance.configuration[dname] then + input.aux.load_configuration(instance,dname,lname) + instance.order[#instance.order+1] = instance.configuration[dname] + end + else + f = io.open(fname) + if f then + input.report("loading", fname) + local line, data, n, k, v + local dname = file.dirname(fname) + if not instance.configuration[dname] then + instance.configuration[dname] = { } + instance.order[#instance.order+1] = instance.configuration[dname] + end + local data = instance.configuration[dname] + while true do + local line, n = f:read(), 0 + if line then + while true do -- join lines + line, n = line:gsub("\\%s*$", "") + if n > 0 then + line = line .. f:read() + else + break + end + end + if not line:find("^[%%#]") then + local k, v = (line:gsub("%s*%%.*$","")):match("%s*(.-)%s*=%s*(.-)%s*$") + if k and v and not data[k] then + data[k] = (v:gsub("[%%#].*",'')):gsub("~", "$HOME") + instance.kpsevars[k] = true + end + end + else + break + end + end + f:close() + else + input.report("skipping", fname) + end + end +end + +-- database loading + +function input.load_hash(instance) + input.locatelists(instance) + if instance.lsrmode then + input.loadlists(instance) + elseif instance.diskcache and not instance.renewcache then + input.loadfiles(instance) + if instance.loaderror then + input.loadlists(instance) + input.savefiles(instance) + end + else + input.loadlists(instance) + if instance.renewcache then + input.savefiles(instance) + end + end +end + +function input.aux.append_hash(instance,type,tag,name) + input.logger("= hash append",tag) + table.insert(instance.hashes, { ['type']=type, ['tag']=tag, ['name']=name } ) +end + +function input.aux.prepend_hash(instance,type,tag,name) + input.logger("= hash prepend",tag) + table.insert(instance.hashes, 1, { ['type']=type, ['tag']=tag, ['name']=name } ) +end + +function input.aux.extend_texmf_var(instance,specification) -- crap + if instance.environment['TEXMF'] then + input.report("extending environment variable TEXMF with", specification) + instance.environment['TEXMF'] = instance.environment['TEXMF']:gsub("^%{", function() + return "{" .. specification .. "," + end) + elseif instance.variables['TEXMF'] then + input.report("extending configuration variable TEXMF with", specification) + instance.variables['TEXMF'] = instance.variables['TEXMF']:gsub("^%{", function() + return "{" .. specification .. "," + end) + else + input.report("setting configuration variable TEXMF to", specification) + instance.variables['TEXMF'] = "{" .. specification .. "}" + end + if instance.variables['TEXMF']:find("%,") and not instance.variables['TEXMF']:find("^%{") then + input.report("adding {} to complex TEXMF variable, best do that yourself") + instance.variables['TEXMF'] = "{" .. instance.variables['TEXMF'] .. "}" + end + input.expand_variables(instance) + input.reset_hashes(instance) +end + +-- locators + +function input.locatelists(instance) + for _, path in pairs(input.simplified_list(input.expansion(instance,'TEXMF'))) do + path = file.collapse_path(path) + input.report("locating list of",path) + input.locatedatabase(instance,input.normalize_name(path)) + end +end + +function input.locatedatabase(instance,specification) + return input.methodhandler('locators', instance, specification) +end + +function input.locators.tex(instance,specification) + if specification and specification ~= '' and lfs.isdir(specification) then + input.logger('! tex locator', specification..' found') + input.aux.append_hash(instance,'file',specification,filename) + else + input.logger('? tex locator', specification..' not found') + end +end + +-- hashers + +function input.hashdatabase(instance,tag,name) + return input.methodhandler('hashers',instance,tag,name) +end + +function input.loadfiles(instance) + instance.loaderror = false + instance.files = { } + if not instance.renewcache then + for _, hash in ipairs(instance.hashes) do + input.hashdatabase(instance,hash.tag,hash.name) + if instance.loaderror then break end + end + end +end + +function input.hashers.tex(instance,tag,name) + input.aux.load_files(instance,tag) +end + +-- generators: + +function input.loadlists(instance) + for _, hash in ipairs(instance.hashes) do + input.generatedatabase(instance,hash.tag) + end +end + +function input.generatedatabase(instance,specification) + return input.methodhandler('generators', instance, specification) +end + +do + + local weird = lpeg.anywhere(lpeg.S("~`!#$%^&*()={}[]:;\"\'||<>,?\n\r\t")) + + function input.generators.tex(instance,specification) + local tag = specification + if not instance.lsrmode and lfs and lfs.dir then + input.report("scanning path",specification) + instance.files[tag] = { } + local files = instance.files[tag] + local n, m, r = 0, 0, 0 + local spec = specification .. '/' + local attributes = lfs.attributes + local directory = lfs.dir + local small = instance.smallcache + local function action(path) + local mode, full + if path then + full = spec .. path .. '/' + else + full = spec + end + for name in directory(full) do + if name:find("^%.") then + -- skip + -- elseif name:find("[%~%`%!%#%$%%%^%&%*%(%)%=%{%}%[%]%:%;\"\'%|%<%>%,%?\n\r\t]") then -- too much escaped + elseif weird:match(name) then + -- texio.write_nl("skipping " .. name) + -- skip + else + mode = attributes(full..name,'mode') + if mode == "directory" then + m = m + 1 + if path then + action(path..'/'..name) + else + action(name) + end + elseif path and mode == 'file' then + n = n + 1 + local f = files[name] + if f then + if not small then + if type(f) == 'string' then + files[name] = { f, path } + else + f[#f+1] = path + end + end + else + files[name] = path + local lower = name:lower() + if name ~= lower then + files["remap:"..lower] = name + r = r + 1 + end + end + end + end + end + end + action() + input.report(format("%s files found on %s directories with %s uppercase remappings",n,m,r)) + else + local fullname = file.join(specification,input.lsrname) + local path = '.' + local f = io.open(fullname) + if f then + instance.files[tag] = { } + local files = instance.files[tag] + local small = instance.smallcache + input.report("loading lsr file",fullname) + -- for line in f:lines() do -- much slower then the next one + for line in (f:read("*a")):gmatch("(.-)\n") do + if line:find("^[%a%d]") then + local fl = files[line] + if fl then + if not small then + if type(fl) == 'string' then + files[line] = { fl, path } -- table + else + fl[#fl+1] = path + end + end + else + files[line] = path -- string + local lower = line:lower() + if line ~= lower then + files["remap:"..lower] = line + end + end + else + path = line:match("%.%/(.-)%:$") or path -- match could be nil due to empty line + end + end + f:close() + end + end + end + +end + +-- savers, todo + +function input.savefiles(instance) + input.aux.save_data(instance, 'files', function(k,v) + return instance.validfile(k,v) -- path, name + end) +end + +-- A config (optionally) has the paths split in tables. Internally +-- we join them and split them after the expansion has taken place. This +-- is more convenient. + +function input.splitconfig(instance) + for i,c in ipairs(instance) do + for k,v in pairs(c) do + if type(v) == 'string' then + local t = file.split_path(v) + if #t > 1 then + c[k] = t + end + end + end + end +end +function input.joinconfig(instance) + for i,c in ipairs(instance.order) do + for k,v in pairs(c) do + if type(v) == 'table' then + c[k] = file.join_path(v) + end + end + end +end +function input.split_path(str) + if type(str) == 'table' then + return str + else + return file.split_path(str) + end +end +function input.join_path(str) + if type(str) == 'table' then + return file.join_path(str) + else + return str + end +end + +function input.splitexpansions(instance) + for k,v in pairs(instance.expansions) do + local t, h = { }, { } + for _,vv in pairs(file.split_path(v)) do + if vv ~= "" and not h[vv] then + t[#t+1] = vv + h[vv] = true + end + end + if #t > 1 then + instance.expansions[k] = t + else + instance.expansions[k] = t[1] + end + end +end + +-- end of split/join code + +function input.saveoldconfig(instance) + input.splitconfig(instance) + input.aux.save_data(instance, 'configuration', nil) + input.joinconfig(instance) +end + +input.configbanner = [[ +-- This is a Luatex configuration file created by 'luatools.lua' or +-- 'luatex.exe' directly. For comment, suggestions and questions you can +-- contact the ConTeXt Development Team. This configuration file is +-- not copyrighted. [HH & TH] +]] + +function input.serialize(files) + -- This version is somewhat optimized for the kind of + -- tables that we deal with, so it's much faster than + -- the generic serializer. This makes sense because + -- luatools and mtxtools are called frequently. Okay, + -- we pay a small price for properly tabbed tables. + local t = { } + local concat = table.concat + local sorted = table.sortedkeys + local function dump(k,v,m) + if type(v) == 'string' then + return m .. "['" .. k .. "']='" .. v .. "'," + elseif #v == 1 then + return m .. "['" .. k .. "']='" .. v[1] .. "'," + else + return m .. "['" .. k .. "']={'" .. concat(v,"','").. "'}," + end + end + t[#t+1] = "return {" + if instance.sortdata then + for _, k in pairs(sorted(files)) do + local fk = files[k] + if type(fk) == 'table' then + t[#t+1] = "\t['" .. k .. "']={" + for _, kk in pairs(sorted(fk)) do + t[#t+1] = dump(kk,fk[kk],"\t\t") + end + t[#t+1] = "\t}," + else + t[#t+1] = dump(k,fk,"\t") + end + end + else + for k, v in pairs(files) do + if type(v) == 'table' then + t[#t+1] = "\t['" .. k .. "']={" + for kk,vv in pairs(v) do + t[#t+1] = dump(kk,vv,"\t\t") + end + t[#t+1] = "\t}," + else + t[#t+1] = dump(k,v,"\t") + end + end + end + t[#t+1] = "}" + return concat(t,"\n") +end + +if not texmf then texmf = {} end -- no longer needed, at least not here + +function input.aux.save_data(instance, dataname, check, makename) -- untested without cache overload + for cachename, files in pairs(instance[dataname]) do + local name = (makename or file.join)(cachename,dataname) + local luaname, lucname = name .. input.luasuffix, name .. input.lucsuffix + input.report("preparing " .. dataname .. " for", luaname) + for k, v in pairs(files) do + if not check or check(v,k) then -- path, name + if type(v) == "table" and #v == 1 then + files[k] = v[1] + end + else + files[k] = nil -- false + end + end + local data = { + type = dataname, + root = cachename, + version = input.cacheversion, + date = os.date("%Y-%m-%d"), + time = os.date("%H:%M:%S"), + content = files, + } + local f = io.open(luaname,'w') + if f then + input.report("saving " .. dataname .. " in", luaname) + f:write(input.serialize(data)) + f:close() + input.report("compiling " .. dataname .. " to", lucname) + if not utils.lua.compile(luaname,lucname) then + input.report("compiling failed for " .. dataname .. ", deleting file " .. lucname) + os.remove(lucname) + end + else + input.report("unable to save " .. dataname .. " in " .. name..input.luasuffix) + end + end +end + +function input.aux.load_data(instance,pathname,dataname,filename,makename) -- untested without cache overload + filename = ((not filename or (filename == "")) and dataname) or filename + filename = (makename and makename(dataname,filename)) or file.join(pathname,filename) + local blob = loadfile(filename .. input.lucsuffix) or loadfile(filename .. input.luasuffix) + if blob then + local data = blob() + if data and data.content and data.type == dataname and data.version == input.cacheversion then + input.report("loading",dataname,"for",pathname,"from",filename) + instance[dataname][pathname] = data.content + else + input.report("skipping",dataname,"for",pathname,"from",filename) + instance[dataname][pathname] = { } + instance.loaderror = true + end + else + input.report("skipping",dataname,"for",pathname,"from",filename) + end +end + +-- some day i'll use the nested approach, but not yet (actually we even drop +-- engine/progname support since we have only luatex now) +-- +-- first texmfcnf.lua files are located, next the cached texmf.cnf files +-- +-- return { +-- TEXMFBOGUS = 'effe checken of dit werkt', +-- } + +function input.aux.load_texmfcnf(instance,dataname,pathname) + local filename = file.join(pathname,input.luaname) + local blob = loadfile(filename) + if blob then + local data = blob() + if data then + input.report("loading","configuration file",filename) + if true then + -- flatten to variable.progname + local t = { } + for k, v in pairs(data) do -- v = progname + if type(v) == "string" then + t[k] = v + else + for kk, vv in pairs(v) do -- vv = variable + if type(vv) == "string" then + t[vv.."."..v] = kk + end + end + end + end + instance[dataname][pathname] = t + else + instance[dataname][pathname] = data + end + else + input.report("skipping","configuration file",filename) + instance[dataname][pathname] = { } + instance.loaderror = true + end + else + input.report("skipping","configuration file",filename) + end +end + +function input.aux.load_configuration(instance,dname,lname) + input.aux.load_data(instance,dname,'configuration',lname and file.basename(lname)) +end +function input.aux.load_files(instance,tag) + input.aux.load_data(instance,tag,'files') +end + +function input.resetconfig(instance) + instance.configuration, instance.setup, instance.order, instance.loaderror = { }, { }, { }, false +end + +function input.loadnewconfig(instance) + for _, cnf in ipairs(instance.luafiles) do + local dname = file.dirname(cnf) + input.aux.load_texmfcnf(instance,'setup',dname) + instance.order[#instance.order+1] = instance.setup[dname] + if instance.loaderror then break end + end +end + +function input.loadoldconfig(instance) + if not instance.renewcache then + for _, cnf in ipairs(instance.cnffiles) do + local dname = file.dirname(cnf) + input.aux.load_configuration(instance,dname) + instance.order[#instance.order+1] = instance.configuration[dname] + if instance.loaderror then break end + end + end + input.joinconfig(instance) +end + +function input.expand_variables(instance) + instance.expansions = { } +--~ instance.environment['SELFAUTOPARENT'] = instance.environment['SELFAUTOPARENT'] or instance.rootpath + if instance.engine ~= "" then instance.environment['engine'] = instance.engine end + if instance.progname ~= "" then instance.environment['progname'] = instance.progname end + for k,v in pairs(instance.environment) do + local a, b = k:match("^(%a+)%_(.*)%s*$") + if a and b then + instance.expansions[a..'.'..b] = v + else + instance.expansions[k] = v + end + end + for k,v in pairs(instance.environment) do -- move environment to expansions + if not instance.expansions[k] then instance.expansions[k] = v end + end + for k,v in pairs(instance.variables) do -- move variables to expansions + if not instance.expansions[k] then instance.expansions[k] = v end + end + while true do + local busy = false + for k,v in pairs(instance.expansions) do + local s, n = v:gsub("%$([%a%d%_%-]+)", function(a) + busy = true + return instance.expansions[a] or input.env(instance,a) + end) + local s, m = s:gsub("%$%{([%a%d%_%-]+)%}", function(a) + busy = true + return instance.expansions[a] or input.env(instance,a) + end) + if n > 0 or m > 0 then + instance.expansions[k]= s + end + end + if not busy then break end + end + for k,v in pairs(instance.expansions) do + instance.expansions[k] = v:gsub("\\", '/') + end +end + +function input.aux.expand_vars(instance,lst) -- simple vars + for k,v in pairs(lst) do + lst[k] = v:gsub("%$([%a%d%_%-]+)", function(a) + return instance.variables[a] or input.env(instance,a) + end) + end +end + +function input.aux.expanded_var(instance,var) -- simple vars + return var:gsub("%$([%a%d%_%-]+)", function(a) + return instance.variables[a] or input.env(instance,a) + end) +end + +function input.aux.entry(instance,entries,name) + if name and (name ~= "") then + name = name:gsub('%$','') + local result = entries[name..'.'..instance.progname] or entries[name] + if result then + return result + else + result = input.env(instance,name) + if result then + instance.variables[name] = result + input.expand_variables(instance) + return instance.expansions[name] or "" + end + end + end + return "" +end +function input.variable(instance,name) + return input.aux.entry(instance,instance.variables,name) +end +function input.expansion(instance,name) + return input.aux.entry(instance,instance.expansions,name) +end + +function input.aux.is_entry(instance,entries,name) + if name and name ~= "" then + name = name:gsub('%$','') + return (entries[name..'.'..instance.progname] or entries[name]) ~= nil + else + return false + end +end + +function input.is_variable(instance,name) + return input.aux.is_entry(instance,instance.variables,name) +end +function input.is_expansion(instance,name) + return input.aux.is_entry(instance,instance.expansions,name) +end + +function input.simplified_list(str) + if type(str) == 'table' then + return str -- troubles ; ipv , in texmf + elseif str == '' then + return { } + else + local t = { } + for _,v in ipairs(string.splitchr(str:gsub("^\{(.+)\}$","%1"),",")) do + t[#t+1] = (v:gsub("^[%!]*(.+)[%/\\]*$","%1")) + end + return t + end +end + +function input.unexpanded_path_list(instance,str) + local pth = input.variable(instance,str) + local lst = input.split_path(pth) + return input.aux.expanded_path(instance,lst) +end +function input.unexpanded_path(instance,str) + return file.join_path(input.unexpanded_path_list(instance,str)) +end + +do + local done = { } + + function input.reset_extra_path(instance) + local ep = instance.extra_paths + if not ep then + ep, done = { }, { } + instance.extra_paths = ep + elseif #ep > 0 then + instance.lists, done = { }, { } + end + end + + function input.register_extra_path(instance,paths,subpaths) + local ep = instance.extra_paths or { } + local n = #ep + if paths and paths ~= "" then + if subpaths and subpaths ~= "" then + for p in paths:gmatch("[^,]+") do + -- we gmatch each step again, not that fast, but used seldom + for s in subpaths:gmatch("[^,]+") do + local ps = p .. "/" .. s + if not done[ps] then + ep[#ep+1] = input.clean_path(ps) + done[ps] = true + end + end + end + else + for p in paths:gmatch("[^,]+") do + if not done[p] then + ep[#ep+1] = input.clean_path(p) + done[p] = true + end + end + end + elseif subpaths and subpaths ~= "" then + for i=1,n do + -- we gmatch each step again, not that fast, but used seldom + for s in subpaths:gmatch("[^,]+") do + local ps = ep[i] .. "/" .. s + if not done[ps] then + ep[#ep+1] = input.clean_path(ps) + done[ps] = true + end + end + end + end + if #ep > 0 then + instance.extra_paths = ep -- register paths + end + if #ep > n then + instance.lists = { } -- erase the cache + end + end + +end + +function input.expanded_path_list(instance,str) + local function made_list(list) + local ep = instance.extra_paths + if not ep or #ep == 0 then + return list + else + local done, new = { }, { } + -- honour . .. ../.. but only when at the start + for k, v in ipairs(list) do + if not done[v] then + if v:find("^[%.%/]$") then + done[v] = true + new[#new+1] = v + else + break + end + end + end + -- first the extra paths + for k, v in ipairs(ep) do + if not done[v] then + done[v] = true + new[#new+1] = v + end + end + -- next the formal paths + for k, v in ipairs(list) do + if not done[v] then + done[v] = true + new[#new+1] = v + end + end + return new + end + end + if not str then + return ep or { } + elseif instance.savelists then + -- engine+progname hash + str = str:gsub("%$","") + if not instance.lists[str] then -- cached + local lst = made_list(input.split_path(input.expansion(instance,str))) + instance.lists[str] = input.aux.expanded_path(instance,lst) + end + return instance.lists[str] + else + local lst = input.split_path(input.expansion(instance,str)) + return made_list(input.aux.expanded_path(instance,lst)) + end +end + +function input.expand_path(instance,str) + return file.join_path(input.expanded_path_list(instance,str)) +end + +--~ function input.first_writable_path(instance,name) +--~ for _,v in pairs(input.expanded_path_list(instance,name)) do +--~ if file.is_writable(file.join(v,'luatex-cache.tmp')) then +--~ return v +--~ end +--~ end +--~ return "." +--~ end + +function input.expanded_path_list_from_var(instance,str) -- brrr + local tmp = input.var_of_format_or_suffix(str:gsub("%$","")) + if tmp ~= "" then + return input.expanded_path_list(instance,str) + else + return input.expanded_path_list(instance,tmp) + end +end +function input.expand_path_from_var(instance,str) + return file.join_path(input.expanded_path_list_from_var(instance,str)) +end + +function input.format_of_var(str) + return input.formats[str] or input.formats[input.alternatives[str]] or '' +end +function input.format_of_suffix(str) + return input.suffixmap[file.extname(str)] or 'tex' +end + +function input.variable_of_format(str) + return input.formats[str] or input.formats[input.alternatives[str]] or '' +end + +function input.var_of_format_or_suffix(str) + local v = input.formats[str] + if v then + return v + end + v = input.formats[input.alternatives[str]] + if v then + return v + end + v = input.suffixmap[file.extname(str)] + if v then + return input.formats[isf] + end + return '' +end + +function input.expand_braces(instance,str) -- output variable and brace expansion of STRING + local ori = input.variable(instance,str) + local pth = input.aux.expanded_path(instance,input.split_path(ori)) + return file.join_path(pth) +end + +-- {a,b,c,d} +-- a,b,c/{p,q,r},d +-- a,b,c/{p,q,r}/d/{x,y,z}// +-- a,b,c/{p,q/{x,y,z},r},d/{p,q,r} +-- a,b,c/{p,q/{x,y,z},r},d/{p,q,r} +-- a{b,c}{d,e}f +-- {a,b,c,d} +-- {a,b,c/{p,q,r},d} +-- {a,b,c/{p,q,r}/d/{x,y,z}//} +-- {a,b,c/{p,q/{x,y,z}},d/{p,q,r}} +-- {a,b,c/{p,q/{x,y,z},w}v,d/{p,q,r}} + +-- this one is better and faster, but it took me a while to realize +-- that this kind of replacement is cleaner than messy parsing and +-- fuzzy concatenating we can probably gain a bit with selectively +-- applying lpeg, but experiments with lpeg parsing this proved not to +-- work that well; the parsing is ok, but dealing with the resulting +-- table is a pain because we need to work inside-out recursively + +-- get rid of piecewise here, just a gmatch is ok + +function input.aux.splitpathexpr(str, t, validate) + -- no need for optimization, only called a few times, we can use lpeg for the sub + t = t or { } + local concat = table.concat + while true do + local done = false + while true do + local ok = false + str = str:gsub("([^{},]+){([^{}]-)}", function(a,b) + local t = { } + b:piecewise(",", function(s) t[#t+1] = a .. s end) + ok, done = true, true + return "{" .. concat(t,",") .. "}" + end) + if not ok then break end + end + while true do + local ok = false + str = str:gsub("{([^{}]-)}([^{},]+)", function(a,b) + local t = { } + a:piecewise(",", function(s) t[#t+1] = s .. b end) + ok, done = true, true + return "{" .. concat(t,",") .. "}" + end) + if not ok then break end + end + while true do + local ok = false + str = str:gsub("([,{]){([^{}]+)}([,}])", function(a,b,c) + ok, done = true, true + return a .. b .. c + end) + if not ok then break end + end + if not done then break end + end + while true do + local ok = false + str = str:gsub("{([^{}]-)}{([^{}]-)}", function(a,b) + local t = { } + a:piecewise(",", function(sa) + b:piecewise(",", function(sb) + t[#t+1] = sa .. sb + end) + end) + ok = true + return "{" .. concat(t,",") .. "}" + end) + if not ok then break end + end + while true do + local ok = false + str = str:gsub("{([^{}]-)}", function(a) + ok = true + return a + end) + if not ok then break end + end + if validate then + str:piecewise(",", function(s) + s = validate(s) + if s then t[#t+1] = s end + end) + else + str:piecewise(",", function(s) + t[#t+1] = s + end) + end + return t +end + +function input.aux.expanded_path(instance,pathlist) -- maybe not a list, just a path + -- a previous version fed back into pathlist + local newlist, ok = { }, false + for _,v in ipairs(pathlist) do + if v:find("[{}]") then + ok = true + break + end + end + if ok then + for _, v in ipairs(pathlist) do + input.aux.splitpathexpr(v, newlist, function(s) + s = file.collapse_path(s) + return s ~= "" and not s:find(instance.dummy_path_expr) and s + end) + end + else + for _,v in ipairs(pathlist) do + for vv in string.gmatch(v..',',"(.-),") do + vv = file.collapse_path(v) + if vv ~= "" then newlist[#newlist+1] = vv end + end + end + end + return newlist +end + +input.is_readable = { } + +function input.aux.is_readable(readable, name) + if input.trace > 2 then + if readable then + input.logger("+ readable", name) + else + input.logger("- readable", name) + end + end + return readable +end + +function input.is_readable.file(name) + -- return input.aux.is_readable(file.is_readable(name), name) + return input.aux.is_readable(input.aux.is_file(name), name) +end + +input.is_readable.tex = input.is_readable.file + +-- name +-- name/name + +function input.aux.collect_files(instance,names) + local filelist = { } + for _, fname in pairs(names) do + if fname then + if input.trace > 2 then + input.logger("? blobpath asked",fname) + end + local bname = file.basename(fname) + local dname = file.dirname(fname) + if dname == "" or dname:find("^%.") then + dname = false + else + dname = "/" .. dname .. "$" + end + for _, hash in ipairs(instance.hashes) do + local blobpath = hash.tag + local files = blobpath and instance.files[blobpath] + if files then + if input.trace > 2 then + input.logger('? blobpath do',blobpath .. " (" .. bname ..")") + end + local blobfile = files[bname] + if not blobfile then + local rname = "remap:"..bname + blobfile = files[rname] + if blobfile then + bname = files[rname] + blobfile = files[bname] + end + end + if blobfile then + if type(blobfile) == 'string' then + if not dname or blobfile:find(dname) then + filelist[#filelist+1] = { + hash.type, + file.join(blobpath,blobfile,bname), -- search + input.concatinators[hash.type](blobpath,blobfile,bname) -- result + } + end + else + for _, vv in pairs(blobfile) do + if not dname or vv:find(dname) then + filelist[#filelist+1] = { + hash.type, + file.join(blobpath,vv,bname), -- search + input.concatinators[hash.type](blobpath,vv,bname) -- result + } + end + end + end + end + elseif input.trace > 1 then + input.logger('! blobpath no',blobpath .. " (" .. bname ..")" ) + end + end + end + end + if #filelist > 0 then + return filelist + else + return nil + end +end + +function input.suffix_of_format(str) + if input.suffixes[str] then + return input.suffixes[str][1] + else + return "" + end +end + +function input.suffixes_of_format(str) + if input.suffixes[str] then + return input.suffixes[str] + else + return {} + end +end + +do + + -- called about 700 times for an empty doc (font initializations etc) + -- i need to weed the font files for redundant calls + + local letter = lpeg.R("az","AZ") + local separator = lpeg.P("://") + + local qualified = lpeg.P(".")^0 * lpeg.P("/") + letter*lpeg.P(":") + letter^1*separator + local rootbased = lpeg.P("/") + letter*lpeg.P(":") + + -- ./name ../name /name c: :// + function input.aux.qualified_path(filename) + return qualified:match(filename) + end + function input.aux.rootbased_path(filename) + return rootbased:match(filename) + end + + function input.normalize_name(original) + return original + end + + input.normalize_name = file.collapse_path + +end + +function input.aux.register_in_trees(instance,name) + if not name:find("^%.") then + instance.foundintrees[name] = (instance.foundintrees[name] or 0) + 1 -- maybe only one + end +end + +-- split the next one up, better for jit + +function input.aux.find_file(instance,filename) -- todo : plugin (scanners, checkers etc) + local result = { } + local stamp = nil + filename = input.normalize_name(filename) -- elsewhere + filename = file.collapse_path(filename:gsub("\\","/")) -- elsewhere + -- speed up / beware: format problem + if instance.remember then + stamp = filename .. "--" .. instance.engine .. "--" .. instance.progname .. "--" .. instance.format + if instance.found[stamp] then + input.logger('! remembered', filename) + return instance.found[stamp] + end + end + if filename:find('%*') then + input.logger('! wildcard', filename) + result = input.find_wildcard_files(instance,filename) + elseif input.aux.qualified_path(filename) then + if input.is_readable.file(filename) then + input.logger('! qualified', filename) + result = { filename } + else + local forcedname, ok = "", false + if file.extname(filename) == "" then + if instance.format == "" then + forcedname = filename .. ".tex" + if input.is_readable.file(forcedname) then + input.logger('! no suffix, forcing standard filetype tex') + result, ok = { forcedname }, true + end + else + for _, s in pairs(input.suffixes_of_format(instance.format)) do + forcedname = filename .. "." .. s + if input.is_readable.file(forcedname) then + input.logger('! no suffix, forcing format filetype', s) + result, ok = { forcedname }, true + break + end + end + end + end + if not ok then + input.logger('? qualified', filename) + end + end + else + -- search spec + local filetype, extra, done, wantedfiles, ext = '', nil, false, { }, file.extname(filename) + if ext == "" then + if not instance.force_suffixes then + wantedfiles[#wantedfiles+1] = filename + end + else + wantedfiles[#wantedfiles+1] = filename + end + if instance.format == "" then + if ext == "" then + local forcedname = filename .. '.tex' + wantedfiles[#wantedfiles+1] = forcedname + filetype = input.format_of_suffix(forcedname) + input.logger('! forcing filetype',filetype) + else + filetype = input.format_of_suffix(filename) + input.logger('! using suffix based filetype',filetype) + end + else + if ext == "" then + for _, s in pairs(input.suffixes_of_format(instance.format)) do + wantedfiles[#wantedfiles+1] = filename .. "." .. s + end + end + filetype = instance.format + input.logger('! using given filetype',filetype) + end + local typespec = input.variable_of_format(filetype) + local pathlist = input.expanded_path_list(instance,typespec) + if not pathlist or #pathlist == 0 then + -- no pathlist, access check only / todo == wildcard + if input.trace > 2 then + input.logger('? filename',filename) + input.logger('? filetype',filetype or '?') + input.logger('? wanted files',table.concat(wantedfiles," | ")) + end + for _, fname in pairs(wantedfiles) do + if fname and input.is_readable.file(fname) then + filename, done = fname, true + result[#result+1] = file.join('.',fname) + break + end + end + -- this is actually 'other text files' or 'any' or 'whatever' + local filelist = input.aux.collect_files(instance,wantedfiles) + local fl = filelist and filelist[1] + if fl then + filename = fl[3] + result[#result+1] = filename + done = true + end + else + -- list search + local filelist = input.aux.collect_files(instance,wantedfiles) + local doscan, recurse + if input.trace > 2 then + input.logger('? filename',filename) + -- if pathlist then input.logger('? path list',table.concat(pathlist," | ")) end + -- if filelist then input.logger('? file list',table.concat(filelist," | ")) end + end + -- a bit messy ... esp the doscan setting here + for _, path in pairs(pathlist) do + if path:find("^!!") then doscan = false else doscan = true end + if path:find("//$") then recurse = true else recurse = false end + local pathname = path:gsub("^!+", '') + done = false + -- using file list + if filelist and not (done and not instance.allresults) and recurse then + -- compare list entries with permitted pattern + pathname = pathname:gsub("([%-%.])","%%%1") -- this also influences + pathname = pathname:gsub("/+$", '/.*') -- later usage of pathname + pathname = pathname:gsub("//", '/.-/') -- not ok for /// but harmless + local expr = "^" .. pathname + -- input.debug('?',expr) + for _, fl in ipairs(filelist) do + local f = fl[2] + if f:find(expr) then + -- input.debug('T',' '..f) + if input.trace > 2 then + input.logger('= found in hash',f) + end + --- todo, test for readable + result[#result+1] = fl[3] + input.aux.register_in_trees(instance,f) -- for tracing used files + done = true + if not instance.allresults then break end + else + -- input.debug('F',' '..f) + end + end + end + if not done and doscan then + -- check if on disk / unchecked / does not work at all / also zips + if input.method_is_file(pathname) then -- ? + local pname = pathname:gsub("%.%*$",'') + if not pname:find("%*") then + local ppname = pname:gsub("/+$","") + if input.aux.can_be_dir(instance,ppname) then + for _, w in pairs(wantedfiles) do + local fname = file.join(ppname,w) + if input.is_readable.file(fname) then + if input.trace > 2 then + input.logger('= found by scanning',fname) + end + result[#result+1] = fname + done = true + if not instance.allresults then break end + end + end + else + -- no access needed for non existing path, speedup (esp in large tree with lots of fake) + end + end + end + end + if not done and doscan then + -- todo: slow path scanning + end + if done and not instance.allresults then break end + end + end + end + for k,v in pairs(result) do + result[k] = file.collapse_path(v) + end + if instance.remember then + instance.found[stamp] = result + end + return result +end + +input.aux._find_file_ = input.aux.find_file + +function input.aux.find_file(instance,filename) -- maybe make a lowres cache too + local result = input.aux._find_file_(instance,filename) + if #result == 0 then + local lowered = filename:lower() + if filename ~= lowered then + return input.aux._find_file_(instance,lowered) + end + end + return result +end + +if lfs and lfs.isfile then + input.aux.is_file = lfs.isfile -- to be done: use this +else + input.aux.is_file = file.is_readable +end + +if lfs and lfs.isdir then + function input.aux.can_be_dir(instance,name) + if not instance.fakepaths[name] then + if lfs.isdir(name) then + instance.fakepaths[name] = 1 -- directory + else + instance.fakepaths[name] = 2 -- no directory + end + end + return (instance.fakepaths[name] == 1) + end +else + function input.aux.can_be_dir() + return true + end +end + +if not input.concatinators then input.concatinators = { } end + +input.concatinators.tex = file.join +input.concatinators.file = input.concatinators.tex + +function input.find_files(instance,filename,filetype,mustexist) + if type(mustexist) == boolean then + -- all set + elseif type(filetype) == 'boolean' then + filetype, mustexist = nil, false + elseif type(filetype) ~= 'string' then + filetype, mustexist = nil, false + end + instance.format = filetype or '' + local t = input.aux.find_file(instance,filename,true) + instance.format = '' + return t +end + +function input.find_file(instance,filename,filetype,mustexist) + return (input.find_files(instance,filename,filetype,mustexist)[1] or "") +end + +function input.find_given_files(instance,filename) + local bname, result = file.basename(filename), { } + for k, hash in ipairs(instance.hashes) do + local files = instance.files[hash.tag] + local blist = files[bname] + if not blist then + local rname = "remap:"..bname + blist = files[rname] + if blist then + bname = files[rname] + blist = files[bname] + end + end + if blist then + if type(blist) == 'string' then + result[#result+1] = input.concatinators[hash.type](hash.tag,blist,bname) or "" + if not instance.allresults then break end + else + for kk,vv in pairs(blist) do + result[#result+1] = input.concatinators[hash.type](hash.tag,vv,bname) or "" + if not instance.allresults then break end + end + end + end + end + return result +end + +function input.find_given_file(instance,filename) + return (input.find_given_files(instance,filename)[1] or "") +end + +function input.find_wildcard_files(instance,filename) -- todo: remap: + local result = { } + local bname, dname = file.basename(filename), file.dirname(filename) + local path = dname:gsub("^*/","") + path = path:gsub("*",".*") + path = path:gsub("-","%%-") + if dname == "" then + path = ".*" + end + local name = bname + name = name:gsub("*",".*") + name = name:gsub("-","%%-") + path = path:lower() + name = name:lower() + local function doit(blist,bname,hash,allresults) + local done = false + if blist then + if type(blist) == 'string' then + -- make function and share code + if (blist:lower()):find(path) then + result[#result+1] = input.concatinators[hash.type](hash.tag,blist,bname) or "" + done = true + end + else + for kk,vv in pairs(blist) do + if (vv:lower()):find(path) then + result[#result+1] = input.concatinators[hash.type](hash.tag,vv,bname) or "" + done = true + if not allresults then break end + end + end + end + end + return done + end + local files, allresults, done = instance.files, instance.allresults, false + if name:find("%*") then + for k, hash in ipairs(instance.hashes) do + for kk, hh in pairs(files[hash.tag]) do + if not kk:find("^remap:") then + if (kk:lower()):find(name) then + if doit(hh,kk,hash,allresults) then done = true end + if done and not allresults then break end + end + end + end + end + else + for k, hash in ipairs(instance.hashes) do + if doit(files[hash.tag][bname],bname,hash,allresults) then done = true end + if done and not allresults then break end + end + end + return result +end + +function input.find_wildcard_file(instance,filename) + return (input.find_wildcard_files(instance,filename)[1] or "") +end + +-- main user functions + +function input.save_used_files_in_trees(instance, filename,jobname) + if not filename then filename = 'luatex.jlg' end + local f = io.open(filename,'w') + if f then + f:write("<?xml version='1.0' standalone='yes'?>\n") + f:write("<rl:job>\n") + if jobname then + f:write("\t<rl:name>" .. jobname .. "</rl:name>\n") + end + f:write("\t<rl:files>\n") + for _,v in pairs(table.sortedkeys(instance.foundintrees)) do + f:write("\t\t<rl:file n='" .. instance.foundintrees[v] .. "'>" .. v .. "</rl:file>\n") + end + f:write("\t</rl:files>\n") + f:write("</rl:usedfiles>\n") + f:close() + end +end + +function input.automount(instance) + -- implemented later +end + +function input.load(instance) + input.starttiming(instance) + input.resetconfig(instance) + input.identify_cnf(instance) + input.load_lua(instance) + input.expand_variables(instance) + input.load_cnf(instance) + input.expand_variables(instance) + input.load_hash(instance) + input.automount(instance) + input.stoptiming(instance) +end + +function input.for_files(instance, command, files, filetype, mustexist) + if files and #files > 0 then + local function report(str) + if input.verbose then + input.report(str) -- has already verbose + else + print(str) + end + end + if input.verbose then + report('') + end + for _, file in pairs(files) do + local result = command(instance,file,filetype,mustexist) + if type(result) == 'string' then + report(result) + else + for _,v in pairs(result) do + report(v) + end + end + end + end +end + +-- strtab + +function input.var_value(instance,str) -- output the value of variable $STRING. + return input.variable(instance,str) +end +function input.expand_var(instance,str) -- output variable expansion of STRING. + return input.expansion(instance,str) +end +function input.show_path(instance,str) -- output search path for file type NAME + return file.join_path(input.expanded_path_list(instance,input.format_of_var(str))) +end + +-- input.find_file(filename) +-- input.find_file(filename, filetype, mustexist) +-- input.find_file(filename, mustexist) +-- input.find_file(filename, filetype) + +function input.aux.register_file(files, name, path) + if files[name] then + if type(files[name]) == 'string' then + files[name] = { files[name], path } + else + files[name] = path + end + else + files[name] = path + end +end + +if not input.finders then input.finders = { } end +if not input.openers then input.openers = { } end +if not input.loaders then input.loaders = { } end + +input.finders.notfound = { nil } +input.openers.notfound = { nil } +input.loaders.notfound = { false, nil, 0 } + +function input.splitmethod(filename) + if not filename then + return { } -- safeguard + elseif type(filename) == "table" then + return filename -- already split + elseif not filename:find("://") then + return { scheme="file", path = filename, original=filename } -- quick hack + else + return url.hashed(filename) + end +end + +function input.method_is_file(filename) + return input.splitmethod(filename).scheme == 'file' +end + +function table.sequenced(t,sep) -- temp here + local s = { } + for k, v in pairs(t) do + s[#s+1] = k .. "=" .. v + end + return table.concat(s, sep or " | ") +end + +function input.methodhandler(what, instance, filename, filetype) -- ... + local specification = (type(filename) == "string" and input.splitmethod(filename)) or filename -- no or { }, let it bomb + local scheme = specification.scheme + if input[what][scheme] then + input.logger('= handler',specification.original .." -> " .. what .. " -> " .. table.sequenced(specification)) + return input[what][scheme](instance,filename,filetype) -- todo: specification + else + return input[what].tex(instance,filename,filetype) -- todo: specification + end +end + +-- also inside next test? + +function input.findtexfile(instance, filename, filetype) + return input.methodhandler('finders',instance, input.normalize_name(filename), filetype) +end +function input.opentexfile(instance,filename) + return input.methodhandler('openers',instance, input.normalize_name(filename)) +end + +function input.findbinfile(instance, filename, filetype) + return input.methodhandler('finders',instance, input.normalize_name(filename), filetype) +end +function input.openbinfile(instance,filename) + return input.methodhandler('loaders',instance, input.normalize_name(filename)) +end + +function input.loadbinfile(instance, filename, filetype) + local fname = input.findbinfile(instance, input.normalize_name(filename), filetype) + if fname and fname ~= "" then + return input.openbinfile(instance,fname) + else + return unpack(input.loaders.notfound) + end +end + +function input.texdatablob(instance, filename, filetype) + local ok, data, size = input.loadbinfile(instance, filename, filetype) + return data or "" +end + +input.loadtexfile = input.texdatablob + +function input.openfile(filename) -- brrr texmf.instance here / todo ! ! ! ! ! + local fullname = input.findtexfile(texmf.instance, filename) + if fullname and (fullname ~= "") then + return input.opentexfile(texmf.instance, fullname) + else + return nil + end +end + +function input.logmode() + return (os.getenv("MTX.LOG.MODE") or os.getenv("MTX_LOG_MODE") or "tex"):lower() +end + +-- this is a prelude to engine/progname specific configuration files +-- in which case we can omit files meant for other programs and +-- packages + +--- ctx + +-- maybe texinputs + font paths +-- maybe positive selection tex/context fonts/tfm|afm|vf|opentype|type1|map|enc + +input.validators = { } +input.validators.visibility = { } + +function input.validators.visibility.default(path, name) + return true +end + +function input.validators.visibility.context(path, name) + path = path[1] or path -- some day a loop + return not ( + path:find("latex") or +-- path:find("doc") or + path:find("tex4ht") or + path:find("source") or +-- path:find("config") or +-- path:find("metafont") or + path:find("lists$") or + name:find("%.tpm$") or + name:find("%.bak$") + ) +end + +-- todo: describe which functions are public (maybe input.private. ... ) + +-- beware: i need to check where we still need a / on windows: + +function input.clean_path(str) +--~ return (((str:gsub("\\","/")):gsub("^!+","")):gsub("//+","//")) + if str then + return ((str:gsub("\\","/")):gsub("^!+","")) + else + return nil + end +end + +function input.do_with_path(name,func) + for _, v in pairs(input.expanded_path_list(instance,name)) do + func("^"..input.clean_path(v)) + end +end + +function input.do_with_var(name,func) + func(input.aux.expanded_var(name)) +end + +function input.with_files(instance,pattern,handle) + for _, hash in ipairs(instance.hashes) do + local blobpath = hash.tag + local blobtype = hash.type + if blobpath then + local files = instance.files[blobpath] + if files then + for k,v in pairs(files) do + if k:find("^remap:") then + k = files[k] + v = files[k] -- chained + end + if k:find(pattern) then + if type(v) == "string" then + handle(blobtype,blobpath,v,k) + else + for _,vv in pairs(v) do + handle(blobtype,blobpath,vv,k) + end + end + end + end + end + end + end +end + +--~ function input.update_script(oldname,newname) -- oldname -> own.name, not per se a suffix +--~ newname = file.addsuffix(newname,"lua") +--~ local newscript = input.clean_path(input.find_file(instance, newname)) +--~ local oldscript = input.clean_path(oldname) +--~ input.report("old script", oldscript) +--~ input.report("new script", newscript) +--~ if oldscript ~= newscript and (oldscript:find(file.removesuffix(newname).."$") or oldscript:find(newname.."$")) then +--~ local newdata = io.loaddata(newscript) +--~ if newdata then +--~ input.report("old script content replaced by new content") +--~ io.savedata(oldscript,newdata) +--~ end +--~ end +--~ end + +function input.update_script(instance,oldname,newname) -- oldname -> own.name, not per se a suffix + local scriptpath = "scripts/context/lua" + newname = file.addsuffix(newname,"lua") + local oldscript = input.clean_path(oldname) + input.report("to be replaced old script", oldscript) + local newscripts = input.find_files(instance, newname) or { } + if #newscripts == 0 then + input.report("unable to locate new script") + else + for _, newscript in ipairs(newscripts) do + newscript = input.clean_path(newscript) + input.report("checking new script", newscript) + if oldscript == newscript then + input.report("old and new script are the same") + elseif not newscript:find(scriptpath) then + input.report("new script should come from",scriptpath) + elseif not (oldscript:find(file.removesuffix(newname).."$") or oldscript:find(newname.."$")) then + input.report("invalid new script name") + else + local newdata = io.loaddata(newscript) + if newdata then + input.report("old script content replaced by new content") + io.savedata(oldscript,newdata) + break + else + input.report("unable to load new script") + end + end + end + end +end + + +--~ print(table.serialize(input.aux.splitpathexpr("/usr/share/texmf-{texlive,tetex}", {}))) + +-- command line resolver: + +--~ print(input.resolve("abc env:tmp file:cont-en.tex path:cont-en.tex full:cont-en.tex rel:zapf/one/p-chars.tex")) + +do + + local resolvers = { } + + resolvers.environment = function(instance,str) + return input.clean_path(os.getenv(str) or os.getenv(str:upper()) or os.getenv(str:lower()) or "") + end + resolvers.relative = function(instance,str,n) + if io.exists(str) then + -- nothing + elseif io.exists("./" .. str) then + str = "./" .. str + else + local p = "../" + for i=1,n or 2 do + if io.exists(p .. str) then + str = p .. str + break + else + p = p .. "../" + end + end + end + return input.clean_path(str) + end + resolvers.locate = function(instance,str) + local fullname = input.find_given_file(instance,str) or "" + return input.clean_path((fullname ~= "" and fullname) or str) + end + resolvers.filename = function(instance,str) + local fullname = input.find_given_file(instance,str) or "" + return input.clean_path(file.basename((fullname ~= "" and fullname) or str)) + end + resolvers.pathname = function(instance,str) + local fullname = input.find_given_file(instance,str) or "" + return input.clean_path(file.dirname((fullname ~= "" and fullname) or str)) + end + + resolvers.env = resolvers.environment + resolvers.rel = resolvers.relative + resolvers.loc = resolvers.locate + resolvers.kpse = resolvers.locate + resolvers.full = resolvers.locate + resolvers.file = resolvers.filename + resolvers.path = resolvers.pathname + + local function resolve(instance,str) + if type(str) == "table" then + for k, v in pairs(str) do + str[k] = resolve(instance,v) or v + end + elseif str and str ~= "" then + str = str:gsub("([a-z]+):([^ ]+)", function(method,target) + if resolvers[method] then + return resolvers[method](instance,target) + else + return method .. ":" .. target + end + end) + end + return str + end + + input.resolve = resolve + +end + + +if not modules then modules = { } end modules ['luat-tmp'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +--[[ldx-- +<p>This module deals with caching data. It sets up the paths and +implements loaders and savers for tables. Best is to set the +following variable. When not set, the usual paths will be +checked. Personally I prefer the (users) temporary path.</p> + +</code> +TEXMFCACHE=$TMP;$TEMP;$TMPDIR;$TEMPDIR;$HOME;$TEXMFVAR;$VARTEXMF;. +</code> + +<p>Currently we do no locking when we write files. This is no real +problem because most caching involves fonts and the chance of them +being written at the same time is small. We also need to extend +luatools with a recache feature.</p> +--ldx]]-- + +caches = caches or { } +dir = dir or { } +texmf = texmf or { } + +caches.path = caches.path or nil +caches.base = caches.base or "luatex-cache" +caches.more = caches.more or "context" +caches.direct = false -- true is faster but may need huge amounts of memory +caches.trace = false +caches.tree = false +caches.paths = caches.paths or nil +caches.force = false + +input.usecache = not toboolean(os.getenv("TEXMFSHARECACHE") or "false",true) -- true + +function caches.temp(instance) + local function checkpath(cachepath) + if not cachepath or cachepath == "" then + return nil + elseif lfs.attributes(cachepath,"mode") == "directory" then -- lfs.isdir(cachepath) then + return cachepath + elseif caches.force or io.ask(string.format("Should I create the cache path %s?",cachepath), "no", { "yes", "no" }) == "yes" then + dir.mkdirs(cachepath) + return (lfs.attributes(cachepath,"mode") == "directory") and cachepath + else + return nil + end + end + local cachepath = input.expanded_path_list(instance,"TEXMFCACHE") + cachepath = cachepath and #cachepath > 0 and checkpath(cachepath[1]) + if not cachepath then + cachepath = os.getenv("TEXMFCACHE") or os.getenv("HOME") or os.getenv("HOMEPATH") or os.getenv("TMP") or os.getenv("TEMP") or os.getenv("TMPDIR") or nil + cachepath = checkpath(cachepath) + end + if not cachepath then + print("\nfatal error: there is no valid cache path defined\n") + os.exit() + elseif lfs.attributes(cachepath,"mode") ~= "directory" then + print(string.format("\nfatal error: cache path %s is not a directory\n",cachepath)) + os.exit() + end + function caches.temp(instance) + return cachepath + end + return cachepath +end + +function caches.configpath(instance) + return table.concat(instance.cnffiles,";") +end + +function caches.hashed(tree) + return md5.hex((tree:lower()):gsub("[\\\/]+","/")) +end + +function caches.treehash(instance) + local tree = caches.configpath(instance) + if not tree or tree == "" then + return false + else + return caches.hashed(tree) + end +end + +function caches.setpath(instance,...) + if not caches.path then + if not caches.path then + caches.path = caches.temp(instance) + end + caches.path = input.clean_path(caches.path) -- to be sure + if lfs then + caches.tree = caches.tree or caches.treehash(instance) + if caches.tree then + caches.path = dir.mkdirs(caches.path,caches.base,caches.more,caches.tree) + else + caches.path = dir.mkdirs(caches.path,caches.base,caches.more) + end + end + end + if not caches.path then + caches.path = '.' + end + caches.path = input.clean_path(caches.path) + if lfs and not table.is_empty({...}) then + local pth = dir.mkdirs(caches.path,...) + return pth + end + caches.path = dir.expand_name(caches.path) + return caches.path +end + +function caches.definepath(instance,category,subcategory) + return function() + return caches.setpath(instance,category,subcategory) + end +end + +function caches.setluanames(path,name) + return path .. "/" .. name .. ".tma", path .. "/" .. name .. ".tmc" +end + +function caches.loaddata(path,name) + local tmaname, tmcname = caches.setluanames(path,name) + local loader = loadfile(tmcname) or loadfile(tmaname) + if loader then + return loader() + else + return false + end +end + +function caches.is_writable(filepath,filename) + local tmaname, tmcname = caches.setluanames(filepath,filename) + return file.is_writable(tmaname) +end + +function caches.savedata(filepath,filename,data,raw) -- raw needed for file cache + local tmaname, tmcname = caches.setluanames(filepath,filename) + local reduce, simplify = true, true + if raw then + reduce, simplify = false, false + end + if caches.direct then + file.savedata(tmaname, table.serialize(data,'return',true,true)) + else + table.tofile (tmaname, data,'return',true,true) -- maybe not the last true + end + utils.lua.compile(tmaname, tmcname) +end + +-- here we use the cache for format loading (texconfig.[formatname|jobname]) + +--~ if tex and texconfig and texconfig.formatname and texconfig.formatname == "" then +if tex and texconfig and (not texconfig.formatname or texconfig.formatname == "") and texmf.instance then + if not texconfig.luaname then texconfig.luaname = "cont-en.lua" end -- or luc + texconfig.formatname = caches.setpath(texmf.instance,"formats") .. "/" .. texconfig.luaname:gsub("%.lu.$",".fmt") +end + +--[[ldx-- +<p>Once we found ourselves defining similar cache constructs +several times, containers were introduced. Containers are used +to collect tables in memory and reuse them when possible based +on (unique) hashes (to be provided by the calling function).</p> + +<p>Caching to disk is disabled by default. Version numbers are +stored in the saved table which makes it possible to change the +table structures without bothering about the disk cache.</p> + +<p>Examples of usage can be found in the font related code.</p> +--ldx]]-- + +containers = { } +containers.trace = false + +do -- local report + + local function report(container,tag,name) + if caches.trace or containers.trace or container.trace then + logs.report(string.format("%s cache",container.subcategory),string.format("%s: %s",tag,name or 'invalid')) + end + end + + local allocated = { } + + -- tracing + + function containers.define(category, subcategory, version, enabled) + return function() + if category and subcategory then + local c = allocated[category] + if not c then + c = { } + allocated[category] = c + end + local s = c[subcategory] + if not s then + s = { + category = category, + subcategory = subcategory, + storage = { }, + enabled = enabled, + version = version or 1.000, + trace = false, + path = caches.setpath(texmf.instance,category,subcategory), + } + c[subcategory] = s + end + return s + else + return nil + end + end + end + + function containers.is_usable(container, name) + return container.enabled and caches.is_writable(container.path, name) + end + + function containers.is_valid(container, name) + if name and name ~= "" then + local storage = container.storage[name] + return storage and not table.is_empty(storage) and storage.cache_version == container.version + else + return false + end + end + + function containers.read(container,name) + if container.enabled and not container.storage[name] then + container.storage[name] = caches.loaddata(container.path,name) + if containers.is_valid(container,name) then + report(container,"loaded",name) + else + container.storage[name] = nil + end + end + if container.storage[name] then + report(container,"reusing",name) + end + return container.storage[name] + end + + function containers.write(container, name, data) + if data then + data.cache_version = container.version + if container.enabled then + local unique, shared = data.unique, data.shared + data.unique, data.shared = nil, nil + caches.savedata(container.path, name, data) + report(container,"saved",name) + data.unique, data.shared = unique, shared + end + report(container,"stored",name) + container.storage[name] = data + end + return data + end + + function containers.content(container,name) + return container.storage[name] + end + +end + +-- since we want to use the cache instead of the tree, we will now +-- reimplement the saver. + +local save_data = input.aux.save_data + +input.cachepath = nil + +function input.aux.save_data(instance, dataname, check) + input.cachepath = input.cachepath or caches.definepath(instance,"trees") + save_data(instance, dataname, check, function(cachename,dataname) + if input.usecache then + return file.join(input.cachepath(),caches.hashed(cachename)) + else + return file.join(cachename,dataname) + end + end) +end + +local load_data = input.aux.load_data + +function input.aux.load_data(instance,pathname,dataname,filename) + input.cachepath = input.cachepath or caches.definepath(instance,"trees") + load_data(instance,pathname,dataname,filename,function(dataname,filename) + if input.usecache then + return file.join(input.cachepath(),caches.hashed(pathname)) + else + if not filename or (filename == "") then + filename = dataname + end + return file.join(pathname,filename) + end + end) +end + +-- we will make a better format, maybe something xml or just text or lua + +input.automounted = input.automounted or { } + +function input.automount(instance,usecache) + local mountpaths = input.simplified_list(input.expansion(instance,'TEXMFMOUNT')) + if table.is_empty(mountpaths) and usecache then + mountpaths = { caches.setpath(instance,"mount") } + end + if not table.is_empty(mountpaths) then + input.starttiming(instance) + for k, root in pairs(mountpaths) do + local f = io.open(root.."/url.tmi") + if f then + for line in f:lines() do + if line then + if line:find("^[%%#%-]") then -- or %W + -- skip + elseif line:find("^zip://") then + input.report("mounting",line) + table.insert(input.automounted,line) + input.usezipfile(instance,line) + end + end + end + f:close() + end + end + input.stoptiming(instance) + end +end + +-- store info in format + +input.storage = { } +input.storage.data = { } +input.storage.min = 0 -- 500 +input.storage.max = input.storage.min - 1 +input.storage.trace = false -- true +input.storage.done = 0 +input.storage.evaluators = { } +-- (evaluate,message,names) + +function input.storage.register(...) + input.storage.data[#input.storage.data+1] = { ... } +end + +function input.storage.evaluate(name) + input.storage.evaluators[#input.storage.evaluators+1] = name +end + +function input.storage.finalize() -- we can prepend the string with "evaluate:" + for _, t in ipairs(input.storage.evaluators) do + for i, v in pairs(t) do + if type(v) == "string" then + t[i] = loadstring(v)() + elseif type(v) == "table" then + for _, vv in pairs(v) do + if type(vv) == "string" then + t[i] = loadstring(vv)() + end + end + end + end + end +end + +function input.storage.dump() + for name, data in ipairs(input.storage.data) do + local evaluate, message, original, target = data[1], data[2], data[3] ,data[4] + local name, initialize, finalize, code = nil, "", "", "" + for str in target:gmatch("([^%.]+)") do + if name then + name = name .. "." .. str + else + name = str + end + initialize = string.format("%s %s = %s or {} ", initialize, name, name) + end + if evaluate then + finalize = "input.storage.evaluate(" .. name .. ")" + end + input.storage.max = input.storage.max + 1 + if input.storage.trace then + logs.report('storage',string.format('saving %s in slot %s',message,input.storage.max)) + code = + initialize .. + string.format("logs.report('storage','restoring %s from slot %s') ",message,input.storage.max) .. + table.serialize(original,name) .. + finalize + else + code = initialize .. table.serialize(original,name) .. finalize + end + lua.bytecode[input.storage.max] = loadstring(code) + end +end + +if lua.bytecode then -- from 0 upwards + local i = input.storage.min + while lua.bytecode[i] do + lua.bytecode[i]() + lua.bytecode[i] = nil + i = i + 1 + end + input.storage.done = i +end + + +if not modules then modules = { } end modules ['luat-log'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +--[[ldx-- +<p>This is a prelude to a more extensive logging module. For the sake +of parsing log files, in addition to the standard logging we will +provide an <l n='xml'/> structured file. Actually, any logging that +is hooked into callbacks will be \XML\ by default.</p> +--ldx]]-- + +input = input or { } +logs = logs or { } + +--[[ldx-- +<p>This looks pretty ugly but we need to speed things up a bit.</p> +--ldx]]-- + +logs.levels = { + ['error'] = 1, + ['warning'] = 2, + ['info'] = 3, + ['debug'] = 4 +} + +logs.functions = { + 'error', 'warning', 'info', 'debug', 'report', + 'start', 'stop', 'push', 'pop' +} + +logs.callbacks = { + 'start_page_number', + 'stop_page_number', + 'report_output_pages', + 'report_output_log' +} + +logs.xml = logs.xml or { } +logs.tex = logs.tex or { } + +logs.level = 0 + +do + local write_nl, write, format = texio.write_nl or print, texio.write or io.write, string.format + + if texlua then + write_nl = print + write = io.write + end + + function logs.xml.debug(category,str) + if logs.level > 3 then write_nl(format("<d category='%s'>%s</d>",category,str)) end + end + function logs.xml.info(category,str) + if logs.level > 2 then write_nl(format("<i category='%s'>%s</i>",category,str)) end + end + function logs.xml.warning(category,str) + if logs.level > 1 then write_nl(format("<w category='%s'>%s</w>",category,str)) end + end + function logs.xml.error(category,str) + if logs.level > 0 then write_nl(format("<e category='%s'>%s</e>",category,str)) end + end + function logs.xml.report(category,str) + write_nl(format("<r category='%s'>%s</r>",category,str)) + end + + function logs.xml.start() if logs.level > 0 then tw("<%s>" ) end end + function logs.xml.stop () if logs.level > 0 then tw("</%s>") end end + function logs.xml.push () if logs.level > 0 then tw("<!-- ") end end + function logs.xml.pop () if logs.level > 0 then tw(" -->" ) end end + + function logs.tex.debug(category,str) + if logs.level > 3 then write_nl(format("debug >> %s: %s" ,category,str)) end + end + function logs.tex.info(category,str) + if logs.level > 2 then write_nl(format("info >> %s: %s" ,category,str)) end + end + function logs.tex.warning(category,str) + if logs.level > 1 then write_nl(format("warning >> %s: %s",category,str)) end + end + function logs.tex.error(category,str) + if logs.level > 0 then write_nl(format("error >> %s: %s" ,category,str)) end + end + function logs.tex.report(category,str) + write_nl(format("report >> %s: %s" ,category,str)) + end + + function logs.set_level(level) + logs.level = logs.levels[level] or level + end + + function logs.set_method(method) + for _, v in pairs(logs.functions) do + logs[v] = logs[method][v] or function() end + end + if callback and input[method] then + for _, cb in pairs(logs.callbacks) do + callback.register(cb, input[method][cb]) + end + end + end + + function logs.xml.start_page_number() + write_nl(format("<p real='%s' page='%s' sub='%s'", tex.count[0], tex.count[1], tex.count[2])) + end + + function logs.xml.stop_page_number() + write("/>") + write_nl("") + end + + function logs.xml.report_output_pages(p,b) + write_nl(format("<v k='pages' v='%s'/>", p)) + write_nl(format("<v k='bytes' v='%s'/>", b)) + write_nl("") + end + + function logs.xml.report_output_log() + end + +end + +logs.set_level('error') +logs.set_method('tex') + + +if not modules then modules = { } end modules ['luat-sta'] = { + version = 1.001, + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +states = states or { } +states.data = states.data or { } +states.hash = states.hash or { } +states.tag = states.tag or "" +states.filename = states.filename or "" + +function states.save(filename,tag) + tag = tag or states.tag + filename = file.addsuffix(filename or states.filename,'lus') + io.savedata(filename, + "-- generator : luat-sta.lua\n" .. + "-- state tag : " .. tag .. "\n\n" .. + table.serialize(states.data[tag or states.tag] or {},true) + ) +end + +function states.load(filename,tag) + states.filename = filename + states.tag = tag or "whatever" + states.filename = file.addsuffix(states.filename,'lus') + states.data[states.tag], states.hash[states.tag] = (io.exists(filename) and dofile(filename)) or { }, { } +end + +function states.set_by_tag(tag,key,value,default,persistent) + local d, h = states.data[tag], states.hash[tag] + if d then + local dkey, hkey = key, key + local pre, post = key:match("(.+)%.([^%.]+)$") + if pre and post then + for k in pre:gmatch("[^%.]+") do + local dk = d[k] + if not dk then + dk = { } + d[k] = dk + end + d = dk + end + dkey, hkey = post, key + end + if type(value) == nil then + value = value or default + elseif persistent then + value = value or d[dkey] or default + else + value = value or default + end + d[dkey], h[hkey] = value, value + end +end + +function states.get_by_tag(tag,key,default) + local h = states.hash[tag] + if h and h[key] then + return h[key] + else + local d = states.data[tag] + if d then + for k in key:gmatch("[^%.]+") do + local dk = d[k] + if dk then + d = dk + else + return default + end + end + return d or default + end + end +end + +function states.set(key,value,default,persistent) + states.set_by_tag(states.tag,key,value,default,persistent) +end + +function states.get(key,default) + return states.get_by_tag(states.tag,key,default) +end + +--~ states.data.update = { +--~ ["version"] = { +--~ ["major"] = 0, +--~ ["minor"] = 1, +--~ }, +--~ ["rsync"] = { +--~ ["server"] = "contextgarden.net", +--~ ["module"] = "minimals", +--~ ["repository"] = "current", +--~ ["flags"] = "-rpztlv --stats", +--~ }, +--~ ["tasks"] = { +--~ ["update"] = true, +--~ ["make"] = true, +--~ ["delete"] = false, +--~ }, +--~ ["platform"] = { +--~ ["host"] = true, +--~ ["other"] = { +--~ ["mswin"] = false, +--~ ["linux"] = false, +--~ ["linux-64"] = false, +--~ ["osx-intel"] = false, +--~ ["osx-ppc"] = false, +--~ ["sun"] = false, +--~ }, +--~ }, +--~ ["context"] = { +--~ ["available"] = {"current", "beta", "alpha", "experimental"}, +--~ ["selected"] = "current", +--~ }, +--~ ["formats"] = { +--~ ["cont-en"] = true, +--~ ["cont-nl"] = true, +--~ ["cont-de"] = false, +--~ ["cont-cz"] = false, +--~ ["cont-fr"] = false, +--~ ["cont-ro"] = false, +--~ }, +--~ ["engine"] = { +--~ ["pdftex"] = { +--~ ["install"] = true, +--~ ["formats"] = { +--~ ["pdftex"] = true, +--~ }, +--~ }, +--~ ["luatex"] = { +--~ ["install"] = true, +--~ ["formats"] = { +--~ }, +--~ }, +--~ ["xetex"] = { +--~ ["install"] = true, +--~ ["formats"] = { +--~ ["xetex"] = false, +--~ }, +--~ }, +--~ ["metapost"] = { +--~ ["install"] = true, +--~ ["formats"] = { +--~ ["mpost"] = true, +--~ ["metafun"] = true, +--~ }, +--~ }, +--~ }, +--~ ["fonts"] = { +--~ }, +--~ ["doc"] = { +--~ }, +--~ ["modules"] = { +--~ ["f-urwgaramond"] = false, +--~ ["f-urwgothic"] = false, +--~ ["t-bnf"] = false, +--~ ["t-chromato"] = false, +--~ ["t-cmscbf"] = false, +--~ ["t-cmttbf"] = false, +--~ ["t-construction-plan"] = false, +--~ ["t-degrade"] = false, +--~ ["t-french"] = false, +--~ ["t-lettrine"] = false, +--~ ["t-lilypond"] = false, +--~ ["t-mathsets"] = false, +--~ ["t-tikz"] = false, +--~ ["t-typearea"] = false, +--~ ["t-vim"] = false, +--~ }, +--~ } + + +--~ states.save("teststate", "update") +--~ states.load("teststate", "update") + +--~ print(states.get_by_tag("update","rsync.server","unknown")) +--~ states.set_by_tag("update","rsync.server","oeps") +--~ print(states.get_by_tag("update","rsync.server","unknown")) +--~ states.save("teststate", "update") +--~ states.load("teststate", "update") +--~ print(states.get_by_tag("update","rsync.server","unknown")) + +-- end library merge + +own = { } + +own.libs = { -- todo: check which ones are really needed + 'l-string.lua', + 'l-lpeg.lua', + 'l-table.lua', + 'l-io.lua', + 'l-md5.lua', + 'l-number.lua', + 'l-set.lua', + 'l-os.lua', + 'l-file.lua', + 'l-dir.lua', + 'l-boolean.lua', + 'l-xml.lua', +-- 'l-unicode.lua', + 'l-utils.lua', +-- 'l-tex.lua', + 'luat-lib.lua', + 'luat-inp.lua', +-- 'luat-zip.lua', +-- 'luat-tex.lua', +-- 'luat-kps.lua', + 'luat-tmp.lua', + 'luat-log.lua', + 'luat-sta.lua', +} + +-- We need this hack till luatex is fixed. +-- +-- for k,v in pairs(arg) do print(k,v) end + +if arg and (arg[0] == 'luatex' or arg[0] == 'luatex.exe') and arg[1] == "--luaonly" then + arg[-1]=arg[0] arg[0]=arg[2] for k=3,#arg do arg[k-2]=arg[k] end arg[#arg]=nil arg[#arg]=nil +end + +-- End of hack. + +own.name = (environment and environment.ownname) or arg[0] or 'luatools.lua' + +own.path = string.match(own.name,"^(.+)[\\/].-$") or "." +own.list = { '.' } +if own.path ~= '.' then + table.insert(own.list,own.path) +end +table.insert(own.list,own.path.."/../../../tex/context/base") +table.insert(own.list,own.path.."/mtx") +table.insert(own.list,own.path.."/../sources") + +function locate_libs() + for _, lib in pairs(own.libs) do + for _, pth in pairs(own.list) do + local filename = string.gsub(pth .. "/" .. lib,"\\","/") + local codeblob = loadfile(filename) + if codeblob then + codeblob() + own.list = { pth } -- speed up te search + break + end + end + end +end + +if not input then + locate_libs() +end + +if not input then + print("") + print("Mtxrun is unable to start up due to lack of libraries. You may") + print("try to run 'lua mtxrun.lua --selfmerge' in the path where this") + print("script is located (normally under ..../scripts/context/lua) which") + print("will make this script library independent.") + os.exit() +end + +instance = input.reset() +input.verbose = environment.argument("verbose") or false +input.banner = 'MtxRun | ' +utils.report = input.report + +instance.engine = environment.argument("engine") or 'luatex' +instance.progname = environment.argument("progname") or 'context' +instance.lsrmode = environment.argument("lsr") or false + +-- use os.env or environment when available + +--~ function input.check_environment(tree) +--~ input.report('') +--~ os.setenv('TMP', os.getenv('TMP') or os.getenv('TEMP') or os.getenv('TMPDIR') or os.getenv('HOME')) +--~ if os.platform == 'linux' then +--~ os.setenv('TEXOS', os.getenv('TEXOS') or 'texmf-linux') +--~ elseif os.platform == 'windows' then +--~ os.setenv('TEXOS', os.getenv('TEXOS') or 'texmf-windows') +--~ elseif os.platform == 'macosx' then +--~ os.setenv('TEXOS', os.getenv('TEXOS') or 'texmf-macosx') +--~ end +--~ os.setenv('TEXOS', string.gsub(string.gsub(os.getenv('TEXOS'),"^[\\\/]*", ''),"[\\\/]*$", '')) +--~ os.setenv('TEXPATH', string.gsub(tree,"\/+$",'')) +--~ os.setenv('TEXMFOS', os.getenv('TEXPATH') .. "/" .. os.getenv('TEXOS')) +--~ input.report('') +--~ input.report("preset : TEXPATH => " .. os.getenv('TEXPATH')) +--~ input.report("preset : TEXOS => " .. os.getenv('TEXOS')) +--~ input.report("preset : TEXMFOS => " .. os.getenv('TEXMFOS')) +--~ input.report("preset : TMP => " .. os.getenv('TMP')) +--~ input.report('') +--~ end + +function input.check_environment(tree) + input.report('') + os.setenv('TMP', os.getenv('TMP') or os.getenv('TEMP') or os.getenv('TMPDIR') or os.getenv('HOME')) + os.setenv('TEXOS', os.getenv('TEXOS') or ("texmf-" .. os.currentplatform())) + os.setenv('TEXPATH', (tree or "tex"):gsub("\/+$",'')) + os.setenv('TEXMFOS', os.getenv('TEXPATH') .. "/" .. os.getenv('TEXOS')) + input.report('') + input.report("preset : TEXPATH => " .. os.getenv('TEXPATH')) + input.report("preset : TEXOS => " .. os.getenv('TEXOS')) + input.report("preset : TEXMFOS => " .. os.getenv('TEXMFOS')) + input.report("preset : TMP => " .. os.getenv('TMP')) + input.report('') +end + +function input.load_environment(name) -- todo: key=value as well as lua + local f = io.open(name) + if f then + for line in f:lines() do + if line:find("^[%%%#]") then + -- skip comment + else + local key, how, value = line:match("^(.-)%s*([<=>%?]+)%s*(.*)%s*$") + if how then + value = value:gsub("%%(.-)%%", function(v) return os.getenv(v) or "" end) + if how == "=" or how == "<<" then + os.setenv(key,value) + elseif how == "?" or how == "??" then + os.setenv(key,os.getenv(key) or value) + elseif how == "<" or how == "+=" then + if os.getenv(key) then + os.setenv(key,os.getenv(key) .. io.fileseparator .. value) + else + os.setenv(key,value) + end + elseif how == ">" or how == "=+" then + if os.getenv(key) then + os.setenv(key,value .. io.pathseparator .. os.getenv(key)) + else + os.setenv(key,value) + end + end + end + end + end + f:close() + end +end + +function input.load_tree(tree) + if tree and tree ~= "" then + local setuptex = 'setuptex.tmf' + if lfs.attributes(tree, "mode") == "directory" then -- check if not nil + setuptex = tree .. "/" .. setuptex + else + setuptex = tree + end + if io.exists(setuptex) then + input.check_environment(tree) + input.load_environment(setuptex) + end + end +end + +-- md5 extensions + +-- maybe md.md5 md.md5hex md.md5HEX + +if not md5 then md5 = { } end + +if not md5.sum then + function md5.sum(k) + return string.rep("x",16) + end +end + +function md5.hexsum(k) + return (string.gsub(md5.sum(k), ".", function(c) return string.format("%02x", string.byte(c)) end)) +end + +function md5.HEXsum(k) + return (string.gsub(md5.sum(k), ".", function(c) return string.format("%02X", string.byte(c)) end)) +end + +-- file extensions + +file.needs_updating_threshold = 1 + +function file.needs_updating(oldname,newname) -- size modification access change + local oldtime = lfs.attributes(oldname, modification) + local newtime = lfs.attributes(newname, modification) + if newtime >= oldtime then + return false + elseif oldtime - newtime < file.needs_updating_threshold then + return false + else + return true + end +end + +function file.mdchecksum(name) + if md5 then + local data = io.loadall(name) + if data then + return md5.HEXsum(data) + end + end + return nil +end + +function file.loadchecksum(name) + if md then + local data = io.loadall(name .. ".md5") + if data then + return string.gsub(md5.HEXsum(data),"%s$","") + end + end + return nil +end + +function file.savechecksum(name, checksum) + if not checksum then checksum = file.mdchecksum(name) end + if checksum then + local f = io.open(name .. ".md5","w") + if f then + f:write(checksum) + f:close() + return checksum + end + end + return nil +end + +os.arch = os.arch or function() + return os.resultof("uname -m") or "linux" +end + +function os.currentplatform(name, default) + local name = os.name or os.platform or name -- os.name is built in, os.platform is mine + if name then + if name == "windows" or name == "mswin" or name == "win32" or name == "msdos" then + return "mswin" + elseif name == "linux" then + local architecture = os.arch() + if architecture:find("x86_64") then + return "linux-64" + else + return "linux" + end + elseif name == "macosx" then + local architecture = os.arch() + if architecture:find("i386") then + return "osx-intel" + else + return "osx-ppc" + end + elseif name == "freebsd" then + return "freebsd" + end + end + return default or name +end + +-- it starts here + +input.runners = { } +input.runners.applications = { } + +input.runners.applications.lua = "luatex --luaonly" +input.runners.applications.pl = "perl" +input.runners.applications.py = "python" +input.runners.applications.rb = "ruby" + +input.runners.suffixes = { + 'rb', 'lua', 'py', 'pl' +} + +input.runners.registered = { + texexec = { 'texexec.rb', true }, + texutil = { 'texutil.rb', true }, + texfont = { 'texfont.pl', true }, + texshow = { 'texshow.pl', false }, + + makempy = { 'makempy.pl', true }, + mptopdf = { 'mptopdf.pl', true }, + pstopdf = { 'pstopdf.rb', true }, + + examplex = { 'examplex.rb', false }, + concheck = { 'concheck.rb', false }, + + runtools = { 'runtools.rb', true }, + textools = { 'textools.rb', true }, + tmftools = { 'tmftools.rb', true }, + ctxtools = { 'ctxtools.rb', true }, + rlxtools = { 'rlxtools.rb', true }, + pdftools = { 'pdftools.rb', true }, + mpstools = { 'mpstools.rb', true }, + exatools = { 'exatools.rb', true }, + xmltools = { 'xmltools.rb', true }, + luatools = { 'luatools.lua', true }, + mtxtools = { 'mtxtools.rb', true }, + + pdftrimwhite = { 'pdftrimwhite.pl', false } +} + +if not messages then messages = { } end + +messages.help = [[ +--script run an mtx script +--execute run a script or program +--resolve resolve prefixed arguments +--ctxlua run internally (using preloaded libs) +--locate locate given filename + +--autotree use texmf tree cf. env 'texmfstart_tree' or 'texmfstarttree' +--tree=pathtotree use given texmf tree (default file: 'setuptex.tmf') +--environment=name use given (tmf) environment file +--path=runpath go to given path before execution +--ifchanged=filename only execute when given file has changed (md checksum) +--iftouched=old,new only execute when given file has changed (time stamp) + +--make create stubs for (context related) scripts +--remove remove stubs (context related) scripts +--stubpath=binpath paths where stubs wil be written +--windows create windows (mswin) stubs +--unix create unix (linux) stubs + +--verbose give a bit more info +--engine=str target engine +--progname=str format or backend + +--edit launch editor with found file +--launch (--all) launch files (assume os support) + +--intern run script using built in libraries +]] + +function input.runners.my_prepare_a(instance) + input.resetconfig(instance) + input.identify_cnf(instance) + input.load_lua(instance) + input.expand_variables(instance) + input.load_cnf(instance) + input.expand_variables(instance) +end + +function input.runners.my_prepare_b(instance) + input.runners.my_prepare_a(instance) + input.load_hash(instance) + input.automount(instance) +end + +function input.runners.prepare(instance) + local checkname = environment.argument("ifchanged") + if checkname and checkname ~= "" then + local oldchecksum = file.loadchecksum(checkname) + local newchecksum = file.checksum(checkname) + if oldchecksum == newchecksum then + report("file '" .. checkname .. "' is unchanged") + return "skip" + else + report("file '" .. checkname .. "' is changed, processing started") + end + file.savechecksum(checkname) + end + local oldname, newname = string.split(environment.argument("iftouched") or "", ",") + if oldname and newname and oldname ~= "" and newname ~= "" then + if not file.needs_updating(oldname,newname) then + report("file '" .. oldname .. "' and '" .. newname .. "'have same age") + return "skip" + else + report("file '" .. newname .. "' is older than '" .. oldname .. "'") + end + end + local tree = environment.argument('tree') or "" + if environment.argument('autotree') then + tree = os.getenv('TEXMFSTART_TREE') or os.getenv('TEXMFSTARTTREE') or tree + end + if tree and tree ~= "" then + input.load_tree(tree) + end + local env = environment.argument('environment') or "" + if env and env ~= "" then + for _,e in pairs(string.split(env)) do + -- maybe force suffix when not given + input.load_tree(e) + end + end + local runpath = environment.argument("path") + if runpath and not dir.chdir(runpath) then + input.report("unable to change to path '" .. runpath .. "'") + return "error" + end + return "run" +end + +function input.runners.execute_script(instance,fullname,internal) + if fullname and fullname ~= "" then + local state = input.runners.prepare(instance) + if state == 'error' then + return false + elseif state == 'skip' then + return true + elseif state == "run" then + instance.progname = environment.argument("progname") or instance.progname + instance.format = environment.argument("format") or instance.format + local path, name, suffix, result = file.dirname(fullname), file.basename(fullname), file.extname(fullname), "" + if path ~= "" then + result = fullname + elseif name then + name = name:gsub("^int[%a]*:",function() + internal = true + return "" + end ) + name = name:gsub("^script:","") + if suffix == "" and input.runners.registered[name] and input.runners.registered[name][1] then + name = input.runners.registered[name][1] + suffix = file.extname(name) + end + if suffix == "" then + -- loop over known suffixes + for _,s in pairs(input.runners.suffixes) do + result = input.find_file(instance, name .. "." .. s, 'texmfscripts') + if result ~= "" then + break + end + end + elseif input.runners.applications[suffix] then + result = input.find_file(instance, name, 'texmfscripts') + else + -- maybe look on path + result = input.find_file(instance, name, 'other text files') + end + end + if result and result ~= "" then + if internal then + local before, after = environment.split_arguments(fullname) + arg = { } for _,v in pairs(after) do arg[#arg+1] = v end + dofile(result) + else + local binary = input.runners.applications[file.extname(result)] + if binary and binary ~= "" then + result = binary .. " " .. result + end + local before, after = environment.split_arguments(fullname) + local command = result .. " " .. environment.reconstruct_commandline(after) + input.report("") + input.report("executing: " .. command) + input.report("\n \n") + io.flush() + local code = os.exec(command) -- maybe spawn + return code == 0 + end + end + end + end + return false +end + +function input.runners.execute_program(instance,fullname) + if fullname and fullname ~= "" then + local state = input.runners.prepare(instance) + if state == 'error' then + return false + elseif state == 'skip' then + return true + elseif state == "run" then + local before, after = environment.split_arguments(fullname) + environment.initialize_arguments(after) + fullname = fullname:gsub("^bin:","") + local command = fullname .. " " .. environment.reconstruct_commandline(after) + input.report("") + input.report("executing: " .. command) + input.report("\n \n") + io.flush() + local code = os.exec(command) -- (fullname,unpack(after)) does not work / maybe spawn + return code == 0 + end + end + return false +end + +function input.runners.handle_stubs(instance,create) + local stubpath = environment.argument('stubpath') or '.' -- 'auto' no longer supported + local windows = environment.argument('windows') or environment.argument('mswin') or false + local unix = environment.argument('unix') or environment.argument('linux') or false + if not windows and not unix then + if environment.platform == "unix" then + unix = true + else + windows = true + end + end + for _,v in pairs(input.runners.registered) do + local name, doit = v[1], v[2] + if doit then + local base = string.gsub(file.basename(name), "%.(.-)$", "") + if create then + -- direct local command = input.runners.applications[file.extname(name)] .. " " .. name + local command = "luatex --luaonly mtxrun.lua " .. name + if windows then + io.savedata(base..".bat", {"@echo off", command.." %*"}, "\013\010") + input.report("windows stub for '" .. base .. "' created") + end + if unix then + io.savedata(base, {"#!/bin/sh", command..' "$@"'}, "\010") + input.report("unix stub for '" .. base .. "' created") + end + else + if windows and (os.remove(base..'.bat') or os.remove(base..'.cmd')) then + input.report("windows stub for '" .. base .. "' removed") + end + if unix and (os.remove(base) or os.remove(base..'.sh')) then + input.report("unix stub for '" .. base .. "' removed") + end + end + end + end +end + +function input.runners.resolve_string(instance,filename) + if filename and filename ~= "" then + input.runners.report_location(instance,input.resolve(instance,filename)) + end +end + +function input.runners.locate_file(instance,filename) + if filename and filename ~= "" then + input.runners.report_location(instance,input.find_given_file(instance,filename)) + end +end + +function input.runners.locate_platform(instance) + input.runners.report_location(instance,os.currentplatform()) +end + +function input.runners.report_location(instance,result) + if input.verbose then + input.report("") + if result and result ~= "" then + input.report(result) + else + input.report("not found") + end + else + io.write(result) + end +end + +function input.runners.edit_script(instance,filename) + local editor = os.getenv("MTXRUN_EDITOR") or os.getenv("TEXMFSTART_EDITOR") or os.getenv("EDITOR") or 'scite' + local rest = input.resolve(instance,filename) + if rest ~= "" then + os.launch(editor .. " " .. rest) + end +end + +function input.runners.save_script_session(filename, list) + local t = { } + for _, key in ipairs(list) do + t[key] = environment.arguments[key] + end + io.savedata(filename,table.serialize(t,true)) +end + +function input.runners.load_script_session(filename) + if lfs.isfile(filename) then + local t = io.loaddata(filename) + if t then + t = loadstring(t) + if t then t = t() end + for key, value in pairs(t) do + environment.arguments[key] = value + end + end + end +end + +input.runners.launchers = { + windows = { }, + unix = { } +} + +function input.launch(str) + -- maybe we also need to test on mtxrun.launcher.suffix environment + -- variable or on windows consult the assoc and ftype vars and such + local launchers = input.runners.launchers[os.platform] if launchers then + local suffix = file.extname(str) if suffix then + local runner = launchers[suffix] if runner then + str = runner .. " " .. str + end + end + end + os.launch(str) +end + +function input.runners.launch_file(instance,filename) + instance.allresults = true + input.verbose = true + local pattern = environment.arguments["pattern"] + if not pattern or pattern == "" then + pattern = filename + end + if not pattern or pattern == "" then + input.report("provide name or --pattern=") + else + local t = input.find_files(instance,pattern) + -- local t = input.aux.find_file(instance,"*/" .. pattern,true) + if t and #t > 0 then + if environment.arguments["all"] then + for _, v in pairs(t) do + input.report("launching", v) + input.launch(v) + end + else + input.report("launching", t[1]) + input.launch(t[1]) + end + else + input.report("no match for", pattern) + end + end +end + +function input.runners.execute_ctx_script(instance,filename,arguments) + local function found(name) + local path = file.dirname(name) + if path and path ~= "" then + return false + else + local fullname = own and own.path and file.join(own.path,name) + return io.exists(fullname) and fullname + end + end + local suffix = "" + if not filename:find("%.lua$") then suffix = ".lua" end + local fullname = filename + -- just <filename> + fullname = filename .. suffix + fullname = input.find_file(instance,fullname) + -- mtx-<filename> + if not fullname or fullname == "" then + fullname = "mtx-" .. filename .. suffix + fullname = found(fullname) or input.find_file(instance,fullname) + end + -- mtx-<filename>s + if not fullname or fullname == "" then + fullname = "mtx-" .. filename .. "s" .. suffix + fullname = found(fullname) or input.find_file(instance,fullname) + end + -- mtx-<filename minus trailing s> + if not fullname or fullname == "" then + fullname = "mtx-" .. filename:gsub("s$","") .. suffix + fullname = found(fullname) or input.find_file(instance,fullname) + end + -- that should do it + if fullname and fullname ~= "" then + local state = input.runners.prepare(instance) + if state == 'error' then + return false + elseif state == 'skip' then + return true + elseif state == "run" then + -- load and save ... kind of undocumented + arg = { } for _,v in pairs(arguments) do arg[#arg+1] = v end + environment.initialize_arguments(arg) + local loadname = environment.arguments['load'] + if loadname then + if type(loadname) ~= "string" then loadname = file.basename(fullname) end + loadname = file.replacesuffix(loadname,"cfg") + input.runners.load_script_session(loadname) + end + filename = environment.files[1] + if input.verbose then + input.report("using script: " .. fullname) + end + dofile(fullname) + local savename = environment.arguments['save'] + if savename and input.runners.save_list and not table.is_empty(input.runners.save_list or { }) then + if type(savename) ~= "string" then savename = file.basename(fullname) end + savename = file.replacesuffix(savename,"cfg") + input.runners.save_script_session(savename, input.runners.save_list) + end + return true + end + else + input.verbose = true + input.report("unknown script: " .. filename) + return false + end +end + +input.report(banner,"\n") + +function input.help(banner,message) + if not input.verbose then + input.verbose = true + input.report(banner,"\n") + end + input.reportlines(message) +end + +-- this is a bit dirty ... first we store the first filename and next we +-- split the arguments so that we only see the ones meant for this script +-- ... later we will use the second half + +local filename = environment.files[1] or "" +local ok = true + +local before, after = environment.split_arguments(filename) + +input.runners.my_prepare_b(instance) +before = input.resolve(instance,before) -- experimental here +after = input.resolve(instance,after) -- experimental here + +environment.initialize_arguments(before) + +if environment.argument("selfmerge") then + -- embed used libraries + utils.merger.selfmerge(own.name,own.libs,own.list) +elseif environment.argument("selfclean") then + -- remove embedded libraries + utils.merger.selfclean(own.name) +elseif environment.argument("selfupdate") then + input.verbose = true + input.update_script(instance,own.name,"mtxrun") +elseif environment.argument("ctxlua") or environment.argument("internal") then + -- run a script by loading it (using libs) + ok = input.runners.execute_script(instance,filename,true) +elseif environment.argument("script") then + -- run a script by loading it (using libs), pass args + ok = input.runners.execute_ctx_script(instance,filename,after) +elseif environment.argument("execute") then + -- execute script + ok = input.runners.execute_script(instance,filename) +elseif environment.argument("direct") then + -- equals bin: + ok = input.runners.execute_program(instance,filename) +elseif environment.argument("edit") then + -- edit file + input.runners.edit_script(instance,filename) +elseif environment.argument("launch") then + input.runners.launch_file(instance,filename) +elseif environment.argument("make") then + -- make stubs + input.runners.handle_stubs(instance,true) +elseif environment.argument("remove") then + -- remove stub + input.runners.handle_stubs(instance,false) +elseif environment.argument("resolve") then + -- resolve string + input.runners.resolve_string(instance,filename) +elseif environment.argument("locate") then + -- locate file + input.runners.locate_file(instance,filename) +elseif environment.argument("platform")then + -- locate platform + input.runners.locate_platform(instance) +elseif environment.argument("help") or filename=='help' or filename == "" then + input.help(banner,messages.help) + -- execute script + if filename:find("^bin:") then + ok = input.runners.execute_program(instance,filename) + else + ok = input.runners.execute_script(instance,filename) + end +end + +--~ if input.verbose then +--~ input.report("") +--~ input.report(string.format("runtime: %0.3f seconds",os.runtime())) +--~ end + +--~ if ok then +--~ input.report("exit code: 0") os.exit(0) +--~ else +--~ input.report("exit code: 1") os.exit(1) +--~ end + +if environment.platform == "unix" then + io.write("\n") +end diff --git a/Master/texmf-dist/scripts/context/stubs/unix/pdftrimwhite b/Master/texmf-dist/scripts/context/stubs/unix/pdftrimwhite new file mode 100755 index 00000000000..00b5f525aec --- /dev/null +++ b/Master/texmf-dist/scripts/context/stubs/unix/pdftrimwhite @@ -0,0 +1,2 @@ +#!/bin/sh +texmfstart pdftrimwhite.pl "$@" diff --git a/Master/texmf-dist/scripts/context/stubs/unix/texfind b/Master/texmf-dist/scripts/context/stubs/unix/texfind new file mode 100755 index 00000000000..c054bdf5218 --- /dev/null +++ b/Master/texmf-dist/scripts/context/stubs/unix/texfind @@ -0,0 +1,2 @@ +#!/bin/sh +texmfstart texfind "$@" diff --git a/Master/texmf-dist/scripts/context/stubs/unix/texshow b/Master/texmf-dist/scripts/context/stubs/unix/texshow new file mode 100755 index 00000000000..afd62c339b3 --- /dev/null +++ b/Master/texmf-dist/scripts/context/stubs/unix/texshow @@ -0,0 +1,2 @@ +#!/bin/sh +texmfstart texshow.pl "$@" |