From d7ccb42582f85acf30568913610ccf4d602023fb Mon Sep 17 00:00:00 2001
From: Taco Hoekwater This is a prelude to a more extensive logging module. For the sake
-of parsing log files, in addition to the standard logging we will
-provide an This looks pretty ugly but we need to speed things up a bit. This module deals with caching data. It sets up the paths and
-implements loaders and savers for tables. Best is to set the
-following variable. When not set, the usual paths will be
-checked. Personally I prefer the (users) temporary path. Currently we do no locking when we write files. This is no real
-problem because most caching involves fonts and the chance of them
-being written at the same time is small. We also need to extend
-luatools with a recache feature. Once we found ourselves defining similar cache constructs
-several times, containers were introduced. Containers are used
-to collect tables in memory and reuse them when possible based
-on (unique) hashes (to be provided by the calling function). Caching to disk is disabled by default. Version numbers are
-stored in the saved table which makes it possible to change the
-table structures without bothering about the disk cache. Examples of usage can be found in the font related code. This file is used when we want the input handlers to behave like
- How about just forgetting about them? If you wondered abou tsome of the previous mappings, how about
-the next bunch: This is a prelude to a more extensive logging module. We no longer
+provide The parser used here is inspired by the variant discussed in the lua book, but
handles comment and processing instructions, has a different structure, provides
@@ -4001,11 +6275,15 @@ The find based parser can be found in l-xml-edu.lua along with other older code.
Beware, the interface may change. For instance at, ns, tg, dt may get more
verbose names. Once the code is stable we will also remove some tracing and
optimize the code. I might even decide to reimplement the parser using the latest We cannot load an A few helpers: We've now arrived at an interesting part: accessing the tree using a subset
of This is the main filter function. It returns whatever is asked for. This is the main filter function. It returns whatever is asked for. Often using an iterators looks nicer in the code than passing handler
functions. The The following helper functions best belong to the Here are a few synonyms. This is a prelude to a more extensive logging module. For the sake
-of parsing log files, in addition to the standard logging we will
-provide an This module deals with caching data. It sets up the paths and
+implements loaders and savers for tables. Best is to set the
+following variable. When not set, the usual paths will be
+checked. Personally I prefer the (users) temporary path. This looks pretty ugly but we need to speed things up a bit. Currently we do no locking when we write files. This is no real
+problem because most caching involves fonts and the chance of them
+being written at the same time is small. We also need to extend
+luatools with a recache feature. 0 then
+ done = mkdirs(writable,...)
+ else
+ done = writable
+ end
+ w_cache[hash] = done
+ end
+ return done
end
-function logs.xml.stop_page_number()
- write("/>")
- write_nl("")
-end
+caches.getreadablepaths = getreadablepaths
+caches.getwritablepath = getwritablepath
-function logs.xml.report_output_pages(p,b)
- write_nl(format(" This module deals with caching data. It sets up the paths and
-implements loaders and savers for tables. Best is to set the
-following variable. When not set, the usual paths will be
-checked. Personally I prefer the (users) temporary path. Currently we do no locking when we write files. This is no real
-problem because most caching involves fonts and the chance of them
-being written at the same time is small. We also need to extend
-luatools with a recache feature.
Examples of usage can be found in the font related code.
--ldx]]-- -containers = containers or { } - +containers = containers or { } +local containers = containers containers.usecache = true +local report_containers = logs.reporter("resolvers","containers") + local function report(container,tag,name) if trace_cache or trace_containers then - logs.report(format("%s cache",container.subcategory),"%s: %s",tag,name or 'invalid') + report_containers("container: %s, tag: %s, name: %s",container.subcategory,tag,name or 'invalid') end end local allocated = { } --- tracing +local mt = { + __index = function(t,k) + if k == "writable" then + local writable = caches.getwritablepath(t.category,t.subcategory) or { "." } + t.writable = writable + return writable + elseif k == "readables" then + local readables = caches.getreadablepaths(t.category,t.subcategory) or { "." } + t.readables = readables + return readables + end + end, + __storage__ = true +} function containers.define(category, subcategory, version, enabled) - return function() - if category and subcategory then - local c = allocated[category] - if not c then - c = { } - allocated[category] = c - end - local s = c[subcategory] - if not s then - s = { - category = category, - subcategory = subcategory, - storage = { }, - enabled = enabled, - version = version or 1.000, - trace = false, - path = caches and caches.setpath and caches.setpath(category,subcategory), - } - c[subcategory] = s - end - return s - else - return nil + if category and subcategory then + local c = allocated[category] + if not c then + c = { } + allocated[category] = c + end + local s = c[subcategory] + if not s then + s = { + category = category, + subcategory = subcategory, + storage = { }, + enabled = enabled, + version = version or math.pi, -- after all, this is TeX + trace = false, + -- writable = caches.getwritablepath and caches.getwritablepath (category,subcategory) or { "." }, + -- readables = caches.getreadablepaths and caches.getreadablepaths(category,subcategory) or { "." }, + } + setmetatable(s,mt) + c[subcategory] = s end + return s end end function containers.is_usable(container, name) - return container.enabled and caches and caches.iswritable(container.path, name) + return container.enabled and caches and caches.is_writable(container.writable, name) end function containers.is_valid(container, name) @@ -10782,18 +12936,20 @@ function containers.is_valid(container, name) end function containers.read(container,name) - if container.enabled and caches and not container.storage[name] and containers.usecache then - container.storage[name] = caches.loaddata(container.path,name) - if containers.is_valid(container,name) then + local storage = container.storage + local stored = storage[name] + if not stored and container.enabled and caches and containers.usecache then + stored = caches.loaddata(container.readables,name) + if stored and stored.cache_version == container.version then report(container,"loaded",name) else - container.storage[name] = nil + stored = nil end - end - if container.storage[name] then + storage[name] = stored + elseif stored then report(container,"reusing",name) end - return container.storage[name] + return stored end function containers.write(container, name, data) @@ -10802,7 +12958,7 @@ function containers.write(container, name, data) if container.enabled and caches then local unique, shared = data.unique, data.shared data.unique, data.shared = nil, nil - caches.savedata(container.path, name, data) + caches.savedata(container.writable, name, data) report(container,"saved",name) data.unique, data.shared = unique, shared end @@ -10837,50 +12993,18 @@ local format, lower, gsub, find = string.format, string.lower, string.gsub, stri local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) --- since we want to use the cache instead of the tree, we will now --- reimplement the saver. - -local save_data = resolvers.save_data -local load_data = resolvers.load_data - -resolvers.cachepath = nil -- public, for tracing -resolvers.usecache = true -- public, for tracing +local report_mounts = logs.reporter("resolvers","mounts") -function resolvers.save_data(dataname) - save_data(dataname, function(cachename,dataname) - resolvers.usecache = not toboolean(resolvers.expansion("CACHEINTDS") or "false",true) - if resolvers.usecache then - resolvers.cachepath = resolvers.cachepath or caches.definepath("trees") - return file.join(resolvers.cachepath(),caches.hashed(cachename)) - else - return file.join(cachename,dataname) - end - end) -end - -function resolvers.load_data(pathname,dataname,filename) - load_data(pathname,dataname,filename,function(dataname,filename) - resolvers.usecache = not toboolean(resolvers.expansion("CACHEINTDS") or "false",true) - if resolvers.usecache then - resolvers.cachepath = resolvers.cachepath or caches.definepath("trees") - return file.join(resolvers.cachepath(),caches.hashed(pathname)) - else - if not filename or (filename == "") then - filename = dataname - end - return file.join(pathname,filename) - end - end) -end +local resolvers = resolvers -- we will make a better format, maybe something xml or just text or lua resolvers.automounted = resolvers.automounted or { } function resolvers.automount(usecache) - local mountpaths = resolvers.clean_path_list(resolvers.expansion('TEXMFMOUNT')) + local mountpaths = resolvers.cleanpathlist(resolvers.expansion('TEXMFMOUNT')) if (not mountpaths or #mountpaths == 0) and usecache then - mountpaths = { caches.setpath("mount") } + mountpaths = caches.getreadablepaths("mount") end if mountpaths and #mountpaths > 0 then statistics.starttiming(resolvers.instance) @@ -10894,7 +13018,7 @@ function resolvers.automount(usecache) -- skip elseif find(line,"^zip://") then if trace_locating then - logs.report("fileio","mounting %s",line) + report_mounts("mounting %s",line) end table.insert(resolvers.automounted,line) resolvers.usezipfile(line) @@ -10910,40 +13034,40 @@ end -- status info -statistics.register("used config path", function() return caches.configpath() end) -statistics.register("used cache path", function() return caches.temp() or "?" end) +statistics.register("used config file", function() return caches.configfiles() end) +statistics.register("used cache path", function() return caches.usedpaths() end) -- experiment (code will move) -function statistics.save_fmt_status(texname,formatbanner,sourcefile) -- texname == formatname +function statistics.savefmtstatus(texname,formatbanner,sourcefile) -- texname == formatname local enginebanner = status.list().banner if formatbanner and enginebanner and sourcefile then local luvname = file.replacesuffix(texname,"luv") local luvdata = { enginebanner = enginebanner, formatbanner = formatbanner, - sourcehash = md5.hex(io.loaddata(resolvers.find_file(sourcefile)) or "unknown"), + sourcehash = md5.hex(io.loaddata(resolvers.findfile(sourcefile)) or "unknown"), sourcefile = sourcefile, } io.savedata(luvname,table.serialize(luvdata,true)) end end -function statistics.check_fmt_status(texname) +function statistics.checkfmtstatus(texname) local enginebanner = status.list().banner if enginebanner and texname then local luvname = file.replacesuffix(texname,"luv") if lfs.isfile(luvname) then local luv = dofile(luvname) if luv and luv.sourcefile then - local sourcehash = md5.hex(io.loaddata(resolvers.find_file(luv.sourcefile)) or "unknown") + local sourcehash = md5.hex(io.loaddata(resolvers.findfile(luv.sourcefile)) or "unknown") local luvbanner = luv.enginebanner or "?" if luvbanner ~= enginebanner then - return string.format("engine mismatch (luv:%s <> bin:%s)",luvbanner,enginebanner) + return format("engine mismatch (luv: %s <> bin: %s)",luvbanner,enginebanner) end local luvhash = luv.sourcehash or "?" if luvhash ~= sourcehash then - return string.format("source mismatch (luv:%s <> bin:%s)",luvhash,sourcehash) + return format("source mismatch (luv: %s <> bin: %s)",luvhash,sourcehash) end else return "invalid status file" @@ -10968,25 +13092,30 @@ if not modules then modules = { } end modules ['data-zip'] = { license = "see context related readme files" } +-- partly redone .. needs testing + local format, find, match = string.format, string.find, string.match -local unpack = unpack or table.unpack local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) +local report_zip = logs.reporter("resolvers","zip") + -- zip:///oeps.zip?name=bla/bla.tex -- zip:///oeps.zip?tree=tex/texmf-local -- zip:///texmf.zip?tree=/tex/texmf -- zip:///texmf.zip?tree=/tex/texmf-local -- zip:///texmf-mine.zip?tree=/tex/texmf-projects -zip = zip or { } -zip.archives = zip.archives or { } -zip.registeredfiles = zip.registeredfiles or { } +local resolvers = resolvers -local finders, openers, loaders = resolvers.finders, resolvers.openers, resolvers.loaders -local locators, hashers, concatinators = resolvers.locators, resolvers.hashers, resolvers.concatinators +zip = zip or { } +local zip = zip -local archives = zip.archives +zip.archives = zip.archives or { } +local archives = zip.archives + +zip.registeredfiles = zip.registeredfiles or { } +local registeredfiles = zip.registeredfiles local function validzip(str) -- todo: use url splitter if not find(str,"^zip://") then @@ -11002,7 +13131,7 @@ function zip.openarchive(name) else local arch = archives[name] if not arch then - local full = resolvers.find_file(name) or "" + local full = resolvers.findfile(name) or "" arch = (full ~= "" and zip.open(full)) or false archives[name] = arch end @@ -11017,163 +13146,162 @@ function zip.closearchive(name) end end -function locators.zip(specification) -- where is this used? startup zips (untested) - specification = resolvers.splitmethod(specification) - local zipfile = specification.path - local zfile = zip.openarchive(name) -- tricky, could be in to be initialized tree +function resolvers.locators.zip(specification) + local archive = specification.filename + local zipfile = archive and archive ~= "" and zip.openarchive(archive) -- tricky, could be in to be initialized tree if trace_locating then - if zfile then - logs.report("fileio","zip locator, archive '%s' found",specification.original) + if zipfile then + report_zip("locator, archive '%s' found",archive) else - logs.report("fileio","zip locator, archive '%s' not found",specification.original) + report_zip("locator, archive '%s' not found",archive) end end end -function hashers.zip(tag,name) +function resolvers.hashers.zip(specification) + local archive = specification.filename if trace_locating then - logs.report("fileio","loading zip file '%s' as '%s'",name,tag) + report_zip("loading file '%s'",archive) end - resolvers.usezipfile(format("%s?tree=%s",tag,name)) + resolvers.usezipfile(specification.original) end -function concatinators.zip(tag,path,name) +function resolvers.concatinators.zip(zipfile,path,name) -- ok ? if not path or path == "" then - return format('%s?name=%s',tag,name) + return format('%s?name=%s',zipfile,name) else - return format('%s?name=%s/%s',tag,path,name) + return format('%s?name=%s/%s',zipfile,path,name) end end -function resolvers.isreadable.zip(name) - return true -end - -function finders.zip(specification,filetype) - specification = resolvers.splitmethod(specification) - if specification.path then - local q = url.query(specification.query) - if q.name then - local zfile = zip.openarchive(specification.path) +function resolvers.finders.zip(specification) + local original = specification.original + local archive = specification.filename + if archive then + local query = url.query(specification.query) + local queryname = query.name + if queryname then + local zfile = zip.openarchive(archive) if zfile then if trace_locating then - logs.report("fileio","zip finder, archive '%s' found",specification.path) + report_zip("finder, archive '%s' found",archive) end - local dfile = zfile:open(q.name) + local dfile = zfile:open(queryname) if dfile then dfile = zfile:close() if trace_locating then - logs.report("fileio","zip finder, file '%s' found",q.name) + report_zip("finder, file '%s' found",queryname) end return specification.original elseif trace_locating then - logs.report("fileio","zip finder, file '%s' not found",q.name) + report_zip("finder, file '%s' not found",queryname) end elseif trace_locating then - logs.report("fileio","zip finder, unknown archive '%s'",specification.path) + report_zip("finder, unknown archive '%s'",archive) end end end if trace_locating then - logs.report("fileio","zip finder, '%s' not found",filename) + report_zip("finder, '%s' not found",original) end - return unpack(finders.notfound) + return resolvers.finders.notfound() end -function openers.zip(specification) - local zipspecification = resolvers.splitmethod(specification) - if zipspecification.path then - local q = url.query(zipspecification.query) - if q.name then - local zfile = zip.openarchive(zipspecification.path) +function resolvers.openers.zip(specification) + local original = specification.original + local archive = specification.filename + if archive then + local query = url.query(specification.query) + local queryname = query.name + if queryname then + local zfile = zip.openarchive(archive) if zfile then if trace_locating then - logs.report("fileio","zip opener, archive '%s' opened",zipspecification.path) + report_zip("opener, archive '%s' opened",archive) end - local dfile = zfile:open(q.name) + local dfile = zfile:open(queryname) if dfile then - logs.show_open(specification) if trace_locating then - logs.report("fileio","zip opener, file '%s' found",q.name) + report_zip("opener, file '%s' found",queryname) end - return openers.text_opener(specification,dfile,'zip') + return resolvers.openers.helpers.textopener('zip',original,dfile) elseif trace_locating then - logs.report("fileio","zip opener, file '%s' not found",q.name) + report_zip("opener, file '%s' not found",queryname) end elseif trace_locating then - logs.report("fileio","zip opener, unknown archive '%s'",zipspecification.path) + report_zip("opener, unknown archive '%s'",archive) end end end if trace_locating then - logs.report("fileio","zip opener, '%s' not found",filename) + report_zip("opener, '%s' not found",original) end - return unpack(openers.notfound) + return resolvers.openers.notfound() end -function loaders.zip(specification) - specification = resolvers.splitmethod(specification) - if specification.path then - local q = url.query(specification.query) - if q.name then - local zfile = zip.openarchive(specification.path) +function resolvers.loaders.zip(specification) + local original = specification.original + local archive = specification.filename + if archive then + local query = url.query(specification.query) + local queryname = query.name + if queryname then + local zfile = zip.openarchive(archive) if zfile then if trace_locating then - logs.report("fileio","zip loader, archive '%s' opened",specification.path) + report_zip("loader, archive '%s' opened",archive) end - local dfile = zfile:open(q.name) + local dfile = zfile:open(queryname) if dfile then - logs.show_load(filename) + logs.show_load(original) if trace_locating then - logs.report("fileio","zip loader, file '%s' loaded",filename) + report_zip("loader, file '%s' loaded",original) end local s = dfile:read("*all") dfile:close() return true, s, #s elseif trace_locating then - logs.report("fileio","zip loader, file '%s' not found",q.name) + report_zip("loader, file '%s' not found",queryname) end elseif trace_locating then - logs.report("fileio","zip loader, unknown archive '%s'",specification.path) + report_zip("loader, unknown archive '%s'",archive) end end end if trace_locating then - logs.report("fileio","zip loader, '%s' not found",filename) + report_zip("loader, '%s' not found",original) end - return unpack(openers.notfound) + return resolvers.openers.notfound() end -- zip:///somefile.zip -- zip:///somefile.zip?tree=texmf-local -> mount -function resolvers.usezipfile(zipname) - zipname = validzip(zipname) - local specification = resolvers.splitmethod(zipname) - local zipfile = specification.path - if zipfile and not zip.registeredfiles[zipname] then - local tree = url.query(specification.query).tree or "" - local z = zip.openarchive(zipfile) +function resolvers.usezipfile(archive) + local specification = resolvers.splitmethod(archive) -- to be sure + local archive = specification.filename + if archive and not registeredfiles[archive] then + local z = zip.openarchive(archive) if z then - local instance = resolvers.instance + local tree = url.query(specification.query).tree or "" if trace_locating then - logs.report("fileio","zip registering, registering archive '%s'",zipname) - end - statistics.starttiming(instance) - resolvers.prepend_hash('zip',zipname,zipfile) - resolvers.extend_texmf_var(zipname) -- resets hashes too - zip.registeredfiles[zipname] = z - instance.files[zipname] = resolvers.register_zip_file(z,tree or "") - statistics.stoptiming(instance) + report_zip("registering, registering archive '%s'",archive) + end + statistics.starttiming(resolvers.instance) + resolvers.prependhash('zip',archive) + resolvers.extendtexmfvariable(archive) -- resets hashes too + registeredfiles[archive] = z + instance.files[archive] = resolvers.registerzipfile(z,tree) + statistics.stoptiming(resolvers.instance) elseif trace_locating then - logs.report("fileio","zip registering, unknown archive '%s'",zipname) + report_zip("registering, unknown archive '%s'",archive) end elseif trace_locating then - logs.report("fileio","zip registering, '%s' not found",zipname) + report_zip("registering, '%s' not found",archive) end end -function resolvers.register_zip_file(z,tree) +function resolvers.registerzipfile(z,tree) local files, filter = { }, "" if tree == "" then filter = "^(.+)/(.-)$" @@ -11181,9 +13309,9 @@ function resolvers.register_zip_file(z,tree) filter = format("^%s/(.+)/(.-)$",tree) end if trace_locating then - logs.report("fileio","zip registering, using filter '%s'",filter) + report_zip("registering, using filter '%s'",filter) end - local register, n = resolvers.register_file, 0 + local register, n = resolvers.registerfile, 0 for i in z:files() do local path, name = match(i.filename,filter) if path then @@ -11198,11 +13326,89 @@ function resolvers.register_zip_file(z,tree) n = n + 1 end end - logs.report("fileio","zip registering, %s files registered",n) + report_zip("registering, %s files registered",n) return files end +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['data-tre'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- \input tree://oeps1/**/oeps.tex + +local find, gsub, format = string.find, string.gsub, string.format + +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) + +local report_trees = logs.reporter("resolvers","trees") + +local resolvers = resolvers + +local done, found, notfound = { }, { }, resolvers.finders.notfound + +function resolvers.finders.tree(specification) + local spec = specification.filename + local fnd = found[spec] + if fnd == nil then + if spec ~= "" then + local path, name = file.dirname(spec), file.basename(spec) + if path == "" then path = "." end + local hash = done[path] + if not hash then + local pattern = path .. "/*" -- we will use the proper splitter + hash = dir.glob(pattern) + done[path] = hash + end + local pattern = "/" .. gsub(name,"([%.%-%+])", "%%%1") .. "$" + for k=1,#hash do + local v = hash[k] + if find(v,pattern) then + found[spec] = v + return v + end + end + end + fnd = notfound() -- false + found[spec] = fnd + end + return fnd +end + +function resolvers.locators.tree(specification) + local name = specification.filename + if name ~= '' and lfs.isdir(name) then + if trace_locating then + report_trees("locator '%s' found",name) + end + resolvers.appendhash('tree',name,false) -- don't cache + elseif trace_locating then + report_trees("locator '%s' not found",name) + end +end + +function resolvers.hashers.tree(specification) + local name = specification.filename + if trace_locating then + report_trees("analysing '%s'",name) + end + resolvers.methodhandler("hashers",name) +end + +resolvers.concatinators.tree = resolvers.concatinators.file +resolvers.generators.tree = resolvers.generators.file +resolvers.openers.tree = resolvers.openers.file +resolvers.loaders.tree = resolvers.loaders.file + + end -- of closure do -- create closure to overcome 200 locals limit @@ -11215,58 +13421,59 @@ if not modules then modules = { } end modules ['data-crl'] = { license = "see context related readme files" } -local gsub = string.gsub +-- this one is replaced by data-sch.lua -- -curl = curl or { } +local gsub = string.gsub -curl.cached = { } -curl.cachepath = caches.definepath("curl") +local resolvers = resolvers local finders, openers, loaders = resolvers.finders, resolvers.openers, resolvers.loaders -function curl.fetch(protocol, name) - local cachename = curl.cachepath() .. "/" .. gsub(name,"[^%a%d%.]+","-") --- cachename = gsub(cachename,"[\\/]", io.fileseparator) - cachename = gsub(cachename,"[\\]", "/") -- cleanup - if not curl.cached[name] then +resolvers.curl = resolvers.curl or { } +local curl = resolvers.curl + +local cached = { } + +local function runcurl(specification) + local original = specification.original + -- local scheme = specification.scheme + local cleanname = gsub(original,"[^%a%d%.]+","-") + local cachename = caches.setfirstwritablefile(cleanname,"curl") + if not cached[original] then if not io.exists(cachename) then - curl.cached[name] = cachename - local command = "curl --silent --create-dirs --output " .. cachename .. " " .. name -- no protocol .. "://" + cached[original] = cachename + local command = "curl --silent --create-dirs --output " .. cachename .. " " .. original os.spawn(command) end if io.exists(cachename) then - curl.cached[name] = cachename + cached[original] = cachename else - curl.cached[name] = "" + cached[original] = "" end end - return curl.cached[name] -end - -function finders.curl(protocol,filename) - local foundname = curl.fetch(protocol, filename) - return finders.generic(protocol,foundname,filetype) + return cached[original] end -function openers.curl(protocol,filename) - return openers.generic(protocol,filename) -end +-- old code: we could be cleaner using specification (see schemes) -function loaders.curl(protocol,filename) - return loaders.generic(protocol,filename) +local function finder(specification,filetype) + return resolvers.methodhandler("finders",runcurl(specification),filetype) end --- todo: metamethod +local opener = openers.file +local loader = loaders.file -function curl.install(protocol) - finders[protocol] = function (filename,filetype) return finders.curl(protocol,filename) end - openers[protocol] = function (filename) return openers.curl(protocol,filename) end - loaders[protocol] = function (filename) return loaders.curl(protocol,filename) end +local function install(scheme) + finders[scheme] = finder + openers[scheme] = opener + loaders[scheme] = loader end -curl.install('http') -curl.install('https') -curl.install('ftp') +resolvers.curl.install = install + +install('http') +install('https') +install('ftp') end -- of closure @@ -11287,9 +13494,13 @@ if not modules then modules = { } end modules ['data-lua'] = { local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) +local report_libraries = logs.reporter("resolvers","libraries") + local gsub, insert = string.gsub, table.insert local unpack = unpack or table.unpack +local resolvers, package = resolvers, package + local libformats = { 'luatexlibs', 'tex', 'texmfscripts', 'othertextfiles' } -- 'luainputs' local clibformats = { 'lib' } @@ -11298,7 +13509,7 @@ local _path_, libpaths, _cpath_, clibpaths function package.libpaths() if not _path_ or package.path ~= _path_ then _path_ = package.path - libpaths = file.split_path(_path_,";") + libpaths = file.splitpath(_path_,";") end return libpaths end @@ -11306,7 +13517,7 @@ end function package.clibpaths() if not _cpath_ or package.cpath ~= _cpath_ then _cpath_ = package.cpath - clibpaths = file.split_path(_cpath_,";") + clibpaths = file.splitpath(_cpath_,";") end return clibpaths end @@ -11315,18 +13526,18 @@ local function thepath(...) local t = { ... } t[#t+1] = "?.lua" local path = file.join(unpack(t)) if trace_locating then - logs.report("fileio","! appending '%s' to 'package.path'",path) + report_libraries("! appending '%s' to 'package.path'",path) end return path end local p_libpaths, a_libpaths = { }, { } -function package.append_libpath(...) +function package.appendtolibpath(...) insert(a_libpath,thepath(...)) end -function package.prepend_libpath(...) +function package.prependtolibpath(...) insert(p_libpaths,1,thepath(...)) end @@ -11337,31 +13548,30 @@ local function loaded(libpaths,name,simple) local libpath = libpaths[i] local resolved = gsub(libpath,"%?",simple) if trace_locating then -- more detail - logs.report("fileio","! checking for '%s' on 'package.path': '%s' => '%s'",simple,libpath,resolved) + report_libraries("! checking for '%s' on 'package.path': '%s' => '%s'",simple,libpath,resolved) end - if resolvers.isreadable.file(resolved) then + if file.is_readable(resolved) then if trace_locating then - logs.report("fileio","! lib '%s' located via 'package.path': '%s'",name,resolved) + report_libraries("! lib '%s' located via 'package.path': '%s'",name,resolved) end return loadfile(resolved) end end end - package.loaders[2] = function(name) -- was [#package.loaders+1] if trace_locating then -- mode detail - logs.report("fileio","! locating '%s'",name) + report_libraries("! locating '%s'",name) end for i=1,#libformats do local format = libformats[i] - local resolved = resolvers.find_file(name,format) or "" + local resolved = resolvers.findfile(name,format) or "" if trace_locating then -- mode detail - logs.report("fileio","! checking for '%s' using 'libformat path': '%s'",name,format) + report_libraries("! checking for '%s' using 'libformat path': '%s'",name,format) end if resolved ~= "" then if trace_locating then - logs.report("fileio","! lib '%s' located via environment: '%s'",name,resolved) + report_libraries("! lib '%s' located via environment: '%s'",name,resolved) end return loadfile(resolved) end @@ -11379,16 +13589,16 @@ package.loaders[2] = function(name) -- was [#package.loaders+1] for i=1,#clibformats do -- better have a dedicated loop local format = clibformats[i] - local paths = resolvers.expanded_path_list_from_var(format) + local paths = resolvers.expandedpathlistfromvariable(format) for p=1,#paths do local path = paths[p] local resolved = file.join(path,libname) if trace_locating then -- mode detail - logs.report("fileio","! checking for '%s' using 'clibformat path': '%s'",libname,path) + report_libraries("! checking for '%s' using 'clibformat path': '%s'",libname,path) end - if resolvers.isreadable.file(resolved) then + if file.is_readable(resolved) then if trace_locating then - logs.report("fileio","! lib '%s' located via 'clibformat': '%s'",libname,resolved) + report_libraries("! lib '%s' located via 'clibformat': '%s'",libname,resolved) end return package.loadlib(resolved,name) end @@ -11398,140 +13608,43 @@ package.loaders[2] = function(name) -- was [#package.loaders+1] local libpath = clibpaths[i] local resolved = gsub(libpath,"?",simple) if trace_locating then -- more detail - logs.report("fileio","! checking for '%s' on 'package.cpath': '%s'",simple,libpath) + report_libraries("! checking for '%s' on 'package.cpath': '%s'",simple,libpath) end - if resolvers.isreadable.file(resolved) then + if file.is_readable(resolved) then if trace_locating then - logs.report("fileio","! lib '%s' located via 'package.cpath': '%s'",name,resolved) + report_libraries("! lib '%s' located via 'package.cpath': '%s'",name,resolved) end return package.loadlib(resolved,name) end end -- just in case the distribution is messed up if trace_loading then -- more detail - logs.report("fileio","! checking for '%s' using 'luatexlibs': '%s'",name) + report_libraries("! checking for '%s' using 'luatexlibs': '%s'",name) end - local resolved = resolvers.find_file(file.basename(name),'luatexlibs') or "" + local resolved = resolvers.findfile(file.basename(name),'luatexlibs') or "" if resolved ~= "" then if trace_locating then - logs.report("fileio","! lib '%s' located by basename via environment: '%s'",name,resolved) + report_libraries("! lib '%s' located by basename via environment: '%s'",name,resolved) end return loadfile(resolved) end if trace_locating then - logs.report("fileio",'? unable to locate lib: %s',name) + report_libraries('? unable to locate lib: %s',name) end -- return "unable to locate " .. name end resolvers.loadlualib = require +-- -- -- -- -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['luat-kps'] = { - version = 1.001, - comment = "companion to luatools.lua", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - ---[[ldx-- -This file is used when we want the input handlers to behave like
-
How about just forgetting about them?
---ldx]]-- - -local suffixes = resolvers.suffixes -local formats = resolvers.formats - -suffixes['gf'] = { 'If you wondered abou tsome of the previous mappings, how about -the next bunch:
---ldx]]-- +package.append_libpath = appendtolibpath -- will become obsolete +package.prepend_libpath = prependtolibpath -- will become obsolete -formats['bib'] = '' -formats['bst'] = '' -formats['mft'] = '' -formats['ist'] = '' -formats['web'] = '' -formats['cweb'] = '' -formats['MetaPost support'] = '' -formats['TeX system documentation'] = '' -formats['TeX system sources'] = '' -formats['Troff fonts'] = '' -formats['dvips config'] = '' -formats['graphic/figure'] = '' -formats['ls-R'] = '' -formats['other text files'] = '' -formats['other binary files'] = '' - -formats['gf'] = '' -formats['pk'] = '' -formats['base'] = 'MFBASES' -formats['cnf'] = '' -formats['mem'] = 'MPMEMS' -formats['mf'] = 'MFINPUTS' -formats['mfpool'] = 'MFPOOL' -formats['mppool'] = 'MPPOOL' -formats['texpool'] = 'TEXPOOL' -formats['PostScript header'] = 'TEXPSHEADERS' -formats['cmap files'] = 'CMAPFONTS' -formats['type42 fonts'] = 'T42FONTS' -formats['web2c files'] = 'WEB2C' -formats['pdftex config'] = 'PDFTEXCONFIG' -formats['texmfscripts'] = 'TEXMFSCRIPTS' -formats['bitmap font'] = '' -formats['lig files'] = 'LIGFONTS' +package.obsolete.append_libpath = appendtolibpath -- will become obsolete +package.obsolete.prepend_libpath = prependtolibpath -- will become obsolete end -- of closure @@ -11547,49 +13660,54 @@ if not modules then modules = { } end modules ['data-aux'] = { } local find = string.find +local type, next = type, next local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) -function resolvers.update_script(oldname,newname) -- oldname -> own.name, not per se a suffix +local resolvers = resolvers + +local report_scripts = logs.reporter("resolvers","scripts") + +function resolvers.updatescript(oldname,newname) -- oldname -> own.name, not per se a suffix local scriptpath = "scripts/context/lua" newname = file.addsuffix(newname,"lua") - local oldscript = resolvers.clean_path(oldname) + local oldscript = resolvers.cleanpath(oldname) if trace_locating then - logs.report("fileio","to be replaced old script %s", oldscript) + report_scripts("to be replaced old script %s", oldscript) end - local newscripts = resolvers.find_files(newname) or { } + local newscripts = resolvers.findfiles(newname) or { } if #newscripts == 0 then if trace_locating then - logs.report("fileio","unable to locate new script") + report_scripts("unable to locate new script") end else for i=1,#newscripts do - local newscript = resolvers.clean_path(newscripts[i]) + local newscript = resolvers.cleanpath(newscripts[i]) if trace_locating then - logs.report("fileio","checking new script %s", newscript) + report_scripts("checking new script %s", newscript) end if oldscript == newscript then if trace_locating then - logs.report("fileio","old and new script are the same") + report_scripts("old and new script are the same") end elseif not find(newscript,scriptpath) then if trace_locating then - logs.report("fileio","new script should come from %s",scriptpath) + report_scripts("new script should come from %s",scriptpath) end elseif not (find(oldscript,file.removesuffix(newname).."$") or find(oldscript,newname.."$")) then if trace_locating then - logs.report("fileio","invalid new script name") + report_scripts("invalid new script name") end else local newdata = io.loaddata(newscript) if newdata then if trace_locating then - logs.report("fileio","old script content replaced by new content") + report_scripts("old script content replaced by new content") end io.savedata(oldscript,newdata) break elseif trace_locating then - logs.report("fileio","unable to load new script") + report_scripts("unable to load new script") end end end @@ -11609,72 +13727,133 @@ if not modules then modules = { } end modules ['data-tmf'] = { license = "see context related readme files" } -local find, gsub, match = string.find, string.gsub, string.match -local getenv, setenv = os.getenv, os.setenv +local resolvers = resolvers --- loads *.tmf files in minimal tree roots (to be optimized and documented) +local report_tds = logs.reporter("resolvers","tds") -function resolvers.check_environment(tree) - logs.simpleline() - setenv('TMP', getenv('TMP') or getenv('TEMP') or getenv('TMPDIR') or getenv('HOME')) - setenv('TEXOS', getenv('TEXOS') or ("texmf-" .. os.platform)) - setenv('TEXPATH', gsub(tree or "tex","\/+$",'')) - setenv('TEXMFOS', getenv('TEXPATH') .. "/" .. getenv('TEXOS')) - logs.simpleline() - logs.simple("preset : TEXPATH => %s", getenv('TEXPATH')) - logs.simple("preset : TEXOS => %s", getenv('TEXOS')) - logs.simple("preset : TEXMFOS => %s", getenv('TEXMFOS')) - logs.simple("preset : TMP => %s", getenv('TMP')) - logs.simple('') -end +-- = << +-- ? ?? +-- < += +-- > =+ -function resolvers.load_environment(name) -- todo: key=value as well as lua - local f = io.open(name) - if f then - for line in f:lines() do - if find(line,"^[%%%#]") then - -- skip comment - else - local key, how, value = match(line,"^(.-)%s*([<=>%?]+)%s*(.*)%s*$") - if how then - value = gsub(value,"%%(.-)%%", function(v) return getenv(v) or "" end) - if how == "=" or how == "<<" then - setenv(key,value) - elseif how == "?" or how == "??" then - setenv(key,getenv(key) or value) - elseif how == "<" or how == "+=" then - if getenv(key) then - setenv(key,getenv(key) .. io.fileseparator .. value) - else - setenv(key,value) - end - elseif how == ">" or how == "=+" then - if getenv(key) then - setenv(key,value .. io.pathseparator .. getenv(key)) - else - setenv(key,value) - end - end - end - end +function resolvers.load_tree(tree,resolve) + if type(tree) == "string" and tree ~= "" then + + local getenv, setenv = resolvers.getenv, resolvers.setenv + + -- later might listen to the raw osenv var as well + local texos = "texmf-" .. os.platform + + local oldroot = environment.texroot + local newroot = file.collapsepath(tree) + + local newtree = file.join(newroot,texos) + local newpath = file.join(newtree,"bin") + + if not lfs.isdir(newtree) then + report_tds("no '%s' under tree %s",texos,tree) + os.exit() end - f:close() + if not lfs.isdir(newpath) then + report_tds("no '%s/bin' under tree %s",texos,tree) + os.exit() + end + + local texmfos = newtree + + environment.texroot = newroot + environment.texos = texos + environment.texmfos = texmfos + + -- Beware, we need to obey the relocatable autoparent so we + -- set TEXMFCNF to its raw value. This is somewhat tricky when + -- we run a mkii job from within. Therefore, in mtxrun, there + -- is a resolve applied when we're in mkii/kpse mode. + + setenv('SELFAUTOPARENT', newroot) + setenv('SELFAUTODIR', newtree) + setenv('SELFAUTOLOC', newpath) + setenv('TEXROOT', newroot) + setenv('TEXOS', texos) + setenv('TEXMFOS', texmfos) + setenv('TEXMFCNF', resolvers.luacnfspec, not resolve) + setenv("PATH", newpath .. io.pathseparator .. getenv("PATH")) + + report_tds("changing from root '%s' to '%s'",oldroot,newroot) + report_tds("prepending '%s' to binary path",newpath) + report_tds() end end -function resolvers.load_tree(tree) - if tree and tree ~= "" then - local setuptex = 'setuptex.tmf' - if lfs.attributes(tree, "mode") == "directory" then -- check if not nil - setuptex = tree .. "/" .. setuptex - else - setuptex = tree + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['data-lst'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- used in mtxrun, can be loaded later .. todo + +local find, concat, upper, format = string.find, table.concat, string.upper, string.format + +resolvers.listers = resolvers.listers or { } + +local resolvers = resolvers + +local report_lists = logs.reporter("resolvers","lists") + +local function tabstr(str) + if type(str) == 'table' then + return concat(str," | ") + else + return str + end +end + +function resolvers.listers.variables(pattern) + local instance = resolvers.instance + local environment = instance.environment + local variables = instance.variables + local expansions = instance.expansions + local pattern = upper(pattern or "") + local configured = { } + local order = instance.order + for i=1,#order do + for k, v in next, order[i] do + if v ~= nil and configured[k] == nil then + configured[k] = v + end end - if io.exists(setuptex) then - resolvers.check_environment(tree) - resolvers.load_environment(setuptex) + end + local env = table.fastcopy(environment) + local var = table.fastcopy(variables) + local exp = table.fastcopy(expansions) + for key, value in table.sortedpairs(configured) do + if key ~= "" and (pattern == "" or find(upper(key),pattern)) then + report_lists(key) + report_lists(" env: %s",tabstr(rawget(environment,key)) or "unset") + report_lists(" var: %s",tabstr(configured[key]) or "unset") + report_lists(" exp: %s",tabstr(expansions[key]) or "unset") + report_lists(" res: %s",resolvers.resolve(expansions[key]) or "unset") end end + instance.environment = table.fastcopy(env) + instance.variables = table.fastcopy(var) + instance.expansions = table.fastcopy(exp) +end + +function resolvers.listers.configurations(report) + local configurations = resolvers.instance.specification + local report = report or texio.write_nl + for i=1,#configurations do + report(resolvers.resolve(configurations[i])) + end end @@ -11691,12 +13870,18 @@ if not modules then modules = { } end modules ['luat-sta'] = { -- this code is used in the updater -local gmatch, match = string.gmatch, string.match -local type = type +local gmatch, match = string.gmatch, string.match +local type = type + +states = states or { } +local states = states + +states.data = states.data or { } +local data = states.data + +states.hash = states.hash or { } +local hash = states.hash -states = states or { } -states.data = states.data or { } -states.hash = states.hash or { } states.tag = states.tag or "" states.filename = states.filename or "" @@ -11706,7 +13891,7 @@ function states.save(filename,tag) io.savedata(filename, "-- generator : luat-sta.lua\n" .. "-- state tag : " .. tag .. "\n\n" .. - table.serialize(states.data[tag or states.tag] or {},true) + table.serialize(data[tag or states.tag] or {},true) ) end @@ -11714,11 +13899,11 @@ function states.load(filename,tag) states.filename = filename states.tag = tag or "whatever" states.filename = file.addsuffix(states.filename,'lus') - states.data[states.tag], states.hash[states.tag] = (io.exists(filename) and dofile(filename)) or { }, { } + data[states.tag], hash[states.tag] = (io.exists(filename) and dofile(filename)) or { }, { } end -function states.set_by_tag(tag,key,value,default,persistent) - local d, h = states.data[tag], states.hash[tag] +local function set_by_tag(tag,key,value,default,persistent) + local d, h = data[tag], hash[tag] if d then if type(d) == "table" then local dkey, hkey = key, key @@ -11738,8 +13923,10 @@ function states.set_by_tag(tag,key,value,default,persistent) end dkey, hkey = post, key end - if type(value) == nil then - value = value or default + if value == nil then + value = default + elseif value == false then + -- special case elseif persistent then value = value or d[dkey] or default else @@ -11748,144 +13935,202 @@ function states.set_by_tag(tag,key,value,default,persistent) d[dkey], h[hkey] = value, value elseif type(d) == "string" then -- weird - states.data[tag], states.hash[tag] = value, value + data[tag], hash[tag] = value, value end end end -function states.get_by_tag(tag,key,default) - local h = states.hash[tag] +local function get_by_tag(tag,key,default) + local h = hash[tag] if h and h[key] then return h[key] else - local d = states.data[tag] + local d = data[tag] if d then for k in gmatch(key,"[^%.]+") do local dk = d[k] - if dk then + if dk ~= nil then d = dk else return default end end - return d or default + if d == false then + return false + else + return d or default + end end end end +states.set_by_tag = set_by_tag +states.get_by_tag = get_by_tag + function states.set(key,value,default,persistent) - states.set_by_tag(states.tag,key,value,default,persistent) + set_by_tag(states.tag,key,value,default,persistent) end function states.get(key,default) - return states.get_by_tag(states.tag,key,default) -end - ---~ states.data.update = { ---~ ["version"] = { ---~ ["major"] = 0, ---~ ["minor"] = 1, ---~ }, ---~ ["rsync"] = { ---~ ["server"] = "contextgarden.net", ---~ ["module"] = "minimals", ---~ ["repository"] = "current", ---~ ["flags"] = "-rpztlv --stats", ---~ }, ---~ ["tasks"] = { ---~ ["update"] = true, ---~ ["make"] = true, ---~ ["delete"] = false, ---~ }, ---~ ["platform"] = { ---~ ["host"] = true, ---~ ["other"] = { ---~ ["mswin"] = false, ---~ ["linux"] = false, ---~ ["linux-64"] = false, ---~ ["osx-intel"] = false, ---~ ["osx-ppc"] = false, ---~ ["sun"] = false, ---~ }, ---~ }, ---~ ["context"] = { ---~ ["available"] = {"current", "beta", "alpha", "experimental"}, ---~ ["selected"] = "current", ---~ }, ---~ ["formats"] = { ---~ ["cont-en"] = true, ---~ ["cont-nl"] = true, ---~ ["cont-de"] = false, ---~ ["cont-cz"] = false, ---~ ["cont-fr"] = false, ---~ ["cont-ro"] = false, ---~ }, ---~ ["engine"] = { ---~ ["pdftex"] = { ---~ ["install"] = true, ---~ ["formats"] = { ---~ ["pdftex"] = true, ---~ }, ---~ }, ---~ ["luatex"] = { ---~ ["install"] = true, ---~ ["formats"] = { ---~ }, ---~ }, ---~ ["xetex"] = { ---~ ["install"] = true, ---~ ["formats"] = { ---~ ["xetex"] = false, ---~ }, ---~ }, ---~ ["metapost"] = { ---~ ["install"] = true, ---~ ["formats"] = { ---~ ["mpost"] = true, ---~ ["metafun"] = true, ---~ }, ---~ }, ---~ }, ---~ ["fonts"] = { ---~ }, ---~ ["doc"] = { ---~ }, ---~ ["modules"] = { ---~ ["f-urwgaramond"] = false, ---~ ["f-urwgothic"] = false, ---~ ["t-bnf"] = false, ---~ ["t-chromato"] = false, ---~ ["t-cmscbf"] = false, ---~ ["t-cmttbf"] = false, ---~ ["t-construction-plan"] = false, ---~ ["t-degrade"] = false, ---~ ["t-french"] = false, ---~ ["t-lettrine"] = false, ---~ ["t-lilypond"] = false, ---~ ["t-mathsets"] = false, ---~ ["t-tikz"] = false, ---~ ["t-typearea"] = false, ---~ ["t-vim"] = false, ---~ }, ---~ } - ---~ states.save("teststate", "update") ---~ states.load("teststate", "update") - ---~ print(states.get_by_tag("update","rsync.server","unknown")) ---~ states.set_by_tag("update","rsync.server","oeps") ---~ print(states.get_by_tag("update","rsync.server","unknown")) ---~ states.save("teststate", "update") ---~ states.load("teststate", "update") ---~ print(states.get_by_tag("update","rsync.server","unknown")) + return get_by_tag(states.tag,key,default) +end + + + + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['luat-fmt'] = { + version = 1.001, + comment = "companion to mtxrun", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + + +local format = string.format + +local report_format = logs.reporter("resolvers","formats") + +-- helper for mtxrun + +local quoted = string.quoted + +local function primaryflags() -- not yet ok + local trackers = environment.argument("trackers") + local directives = environment.argument("directives") + local flags = "" + if trackers and trackers ~= "" then + flags = flags .. "--trackers=" .. quoted(trackers) + end + if directives and directives ~= "" then + flags = flags .. "--directives=" .. quoted(directives) + end + return flags +end + +function environment.make_format(name) + -- change to format path (early as we need expanded paths) + local olddir = lfs.currentdir() + local path = caches.getwritablepath("formats") or "" -- maybe platform + if path ~= "" then + lfs.chdir(path) + end + report_format("format path: %s",lfs.currentdir()) + -- check source file + local texsourcename = file.addsuffix(name,"mkiv") + local fulltexsourcename = resolvers.findfile(texsourcename,"tex") or "" + if fulltexsourcename == "" then + texsourcename = file.addsuffix(name,"tex") + fulltexsourcename = resolvers.findfile(texsourcename,"tex") or "" + end + if fulltexsourcename == "" then + report_format("no tex source file with name: %s (mkiv or tex)",name) + lfs.chdir(olddir) + return + else + report_format("using tex source file: %s",fulltexsourcename) + end + local texsourcepath = dir.expandname(file.dirname(fulltexsourcename)) -- really needed + -- check specification + local specificationname = file.replacesuffix(fulltexsourcename,"lus") + local fullspecificationname = resolvers.findfile(specificationname,"tex") or "" + if fullspecificationname == "" then + specificationname = file.join(texsourcepath,"context.lus") + fullspecificationname = resolvers.findfile(specificationname,"tex") or "" + end + if fullspecificationname == "" then + report_format("unknown stub specification: %s",specificationname) + lfs.chdir(olddir) + return + end + local specificationpath = file.dirname(fullspecificationname) + -- load specification + local usedluastub = nil + local usedlualibs = dofile(fullspecificationname) + if type(usedlualibs) == "string" then + usedluastub = file.join(file.dirname(fullspecificationname),usedlualibs) + elseif type(usedlualibs) == "table" then + report_format("using stub specification: %s",fullspecificationname) + local texbasename = file.basename(name) + local luastubname = file.addsuffix(texbasename,"lua") + local lucstubname = file.addsuffix(texbasename,"luc") + -- pack libraries in stub + report_format("creating initialization file: %s",luastubname) + utilities.merger.selfcreate(usedlualibs,specificationpath,luastubname) + -- compile stub file (does not save that much as we don't use this stub at startup any more) + local strip = resolvers.booleanvariable("LUACSTRIP", true) + if utilities.lua.compile(luastubname,lucstubname) and lfs.isfile(lucstubname) then + report_format("using compiled initialization file: %s",lucstubname) + usedluastub = lucstubname + else + report_format("using uncompiled initialization file: %s",luastubname) + usedluastub = luastubname + end + else + report_format("invalid stub specification: %s",fullspecificationname) + lfs.chdir(olddir) + return + end + -- generate format + local command = format("luatex --ini %s --lua=%s %s %sdump",primaryflags(),quoted(usedluastub),quoted(fulltexsourcename),os.platform == "unix" and "\\\\" or "\\") + report_format("running command: %s\n",command) + os.spawn(command) + -- remove related mem files + local pattern = file.removesuffix(file.basename(usedluastub)).."-*.mem" + -- report_format("removing related mplib format with pattern '%s'", pattern) + local mp = dir.glob(pattern) + if mp then + for i=1,#mp do + local name = mp[i] + report_format("removing related mplib format %s", file.basename(name)) + os.remove(name) + end + end + lfs.chdir(olddir) +end + +function environment.run_format(name,data,more) + -- hm, rather old code here; we can now use the file.whatever functions + if name and name ~= "" then + local barename = file.removesuffix(name) + local fmtname = caches.getfirstreadablefile(file.addsuffix(barename,"fmt"),"formats") + if fmtname == "" then + fmtname = resolvers.findfile(file.addsuffix(barename,"fmt")) or "" + end + fmtname = resolvers.cleanpath(fmtname) + if fmtname == "" then + report_format("no format with name: %s",name) + else + local barename = file.removesuffix(name) -- expanded name + local luaname = file.addsuffix(barename,"luc") + if not lfs.isfile(luaname) then + luaname = file.addsuffix(barename,"lua") + end + if not lfs.isfile(luaname) then + report_format("using format name: %s",fmtname) + report_format("no luc/lua with name: %s",barename) + else + local command = format("luatex %s --fmt=%s --lua=%s %s %s",primaryflags(),quoted(barename),quoted(luaname),quoted(data),more ~= "" and quoted(more) or "") + report_format("running command: %s",command) + os.spawn(command) + end + end + end +end end -- of closure -- end library merge -own = { } -- not local +own = { } -- not local, might change + +own.libs = { -- order can be made better -own.libs = { -- todo: check which ones are really needed 'l-string.lua', 'l-lpeg.lua', 'l-table.lua', @@ -11898,39 +14143,55 @@ own.libs = { -- todo: check which ones are really needed 'l-url.lua', 'l-dir.lua', 'l-boolean.lua', + 'l-unicode.lua', 'l-math.lua', --- 'l-unicode.lua', --- 'l-tex.lua', - 'l-utils.lua', - 'l-aux.lua', --- 'l-xml.lua', - 'trac-tra.lua', + + 'util-tab.lua', + 'util-sto.lua', + 'util-mrg.lua', + 'util-lua.lua', + 'util-prs.lua', + 'util-fmt.lua', + 'util-deb.lua', + + 'trac-inf.lua', + 'trac-set.lua', + 'trac-log.lua', + 'trac-pro.lua', + + 'luat-env.lua', -- can come before inf (as in mkiv) + 'lxml-tab.lua', 'lxml-lpt.lua', --- 'lxml-ent.lua', + -- 'lxml-ent.lua', 'lxml-mis.lua', 'lxml-aux.lua', 'lxml-xml.lua', - 'luat-env.lua', - 'trac-inf.lua', - 'trac-log.lua', - 'data-res.lua', + + 'data-ini.lua', + 'data-exp.lua', + 'data-env.lua', 'data-tmp.lua', + 'data-met.lua', + 'data-res.lua', 'data-pre.lua', 'data-inp.lua', 'data-out.lua', + 'data-fil.lua', 'data-con.lua', 'data-use.lua', -- 'data-tex.lua', -- 'data-bin.lua', 'data-zip.lua', + 'data-tre.lua', 'data-crl.lua', 'data-lua.lua', - 'data-kps.lua', -- so that we can replace kpsewhich 'data-aux.lua', -- updater - 'data-tmf.lua', -- tree files - -- needed ? - 'luat-sta.lua', -- states + 'data-tmf.lua', + 'data-lst.lua', + + 'luat-sta.lua', + 'luat-fmt.lua', } -- We need this hack till luatex is fixed. @@ -11943,35 +14204,62 @@ end -- End of hack. -own.name = (environment and environment.ownname) or arg[0] or 'luatools.lua' +local format, gsub, gmatch, match = string.format, string.gsub, string.gmatch, string.match +local concat = table.concat + +own.name = (environment and environment.ownname) or arg[0] or 'mtxrun.lua' +own.path = gsub(match(own.name,"^(.+)[\\/].-$") or ".","\\","/") + +local ownpath, owntree = own.path, environment and environment.ownpath or own.path + +own.list = { + '.', + ownpath , + ownpath .. "/../sources", -- HH's development path + owntree .. "/../../texmf-local/tex/context/base", + owntree .. "/../../texmf-context/tex/context/base", + owntree .. "/../../texmf-dist/tex/context/base", + owntree .. "/../../texmf/tex/context/base", + owntree .. "/../../../texmf-local/tex/context/base", + owntree .. "/../../../texmf-context/tex/context/base", + owntree .. "/../../../texmf-dist/tex/context/base", + owntree .. "/../../../texmf/tex/context/base", +} +if own.path == "." then table.remove(own.list,1) end -own.path = string.match(own.name,"^(.+)[\\/].-$") or "." -own.list = { '.' } -if own.path ~= '.' then - table.insert(own.list,own.path) +local function locate_libs() + for l=1,#own.libs do + local lib = own.libs[l] + for p =1,#own.list do + local pth = own.list[p] + local filename = pth .. "/" .. lib + local found = lfs.isfile(filename) + if found then + package.path = package.path .. ";" .. pth .. "/?.lua" -- in case l-* does a require + return pth + end + end + end end -table.insert(own.list,own.path.."/../../texmf-dist/tex/context/base") -table.insert(own.list,own.path.."/../../../tex/context/base") -table.insert(own.list,own.path.."/mtx") -table.insert(own.list,own.path.."/../sources") -local function locate_libs() - for _, lib in pairs(own.libs) do - for _, pth in pairs(own.list) do - local filename = string.gsub(pth .. "/" .. lib,"\\","/") +local function load_libs() + local found = locate_libs() + if found then + for l=1,#own.libs do + local filename = found .. "/" .. own.libs[l] local codeblob = loadfile(filename) if codeblob then codeblob() - own.list = { pth } -- speed up te search - break end end + else + resolvers = nil end end if not resolvers then - locate_libs() + load_libs() end if not resolvers then @@ -11983,20 +14271,56 @@ if not resolvers then os.exit() end -logs.setprogram('MTXrun',"TDS Runner Tool 1.24",environment.arguments["verbose"] or false) +-- verbosity -local instance = resolvers.reset() +local e_verbose = environment.arguments["verbose"] + +if e_verbose then + trackers.enable("resolvers.locating") +end + +-- some common flags (also passed through environment) + +local e_silent = environment.argument("silent") +local e_noconsole = environment.argument("noconsole") + +local e_trackers = environment.argument("trackers") +local e_directives = environment.argument("directives") +local e_experiments = environment.argument("experiments") + +if e_silent == true then + e_silent = "*" +end -local trackspec = environment.argument("trackers") or environment.argument("track") +if type(e_silent) == "string" then + if type(e_directives) == "string" then + e_directives = format("%s,logs.blocked={%s}",e_directives,e_silent) + else + e_directives = format("logs.blocked={%s}",e_silent) + end +end -if trackspec then - trackers.enable(trackspec) +if e_noconsole then + if type(e_directives) == "string" then + e_directives = format("%s,logs.target=file",e_directives) + else + e_directives = format("logs.target=file") + end end -runners = runners or { } -- global -messages = messages or { } +if e_trackers then trackers .enable(e_trackers) end +if e_directives then directives .enable(e_directives) end +if e_experiments then experiments.enable(e_experiments) end + +if not environment.trackers then environment.trackers = e_trackers end +if not environment.directives then environment.directives = e_directives end +if not environment.experiments then environment.experiments = e_experiments end + +-- + +local instance = resolvers.reset() -messages.help = [[ +local helpinfo = [[ --script run an mtx script (lua prefered method) (--noquotes), no script gives list --execute run a script or program (texmfstart method) (--noquotes) --resolve resolve prefixed arguments @@ -12011,15 +14335,14 @@ messages.help = [[ --ifchanged=filename only execute when given file has changed (md checksum) --iftouched=old,new only execute when given file has changed (time stamp) ---make create stubs for (context related) scripts ---remove remove stubs (context related) scripts +--makestubs create stubs for (context related) scripts +--removestubs remove stubs (context related) scripts --stubpath=binpath paths where stubs wil be written --windows create windows (mswin) stubs --unix create unix (linux) stubs --verbose give a bit more info --trackers=list enable given trackers ---engine=str target engine --progname=str format or backend --edit launch editor with found file @@ -12032,8 +14355,35 @@ messages.help = [[ --forcekpse force using kpse (handy when no mkiv and cache installed but less functionality) --prefixes show supported prefixes + +--generate generate file database + +--variables show configuration variables +--configurations show configuration order + +--expand-braces expand complex variable +--expand-path expand variable (resolve paths) +--expand-var expand variable (resolve references) +--show-path show path expansion of ... +--var-value report value of variable +--find-file report file location +--find-path report path of file + +--pattern=str filter variables ]] +local application = logs.application { + name = "mtxrun", + banner = "ConTeXt TDS Runner Tool 1.30", + helpinfo = helpinfo, +} + +local report = application.report + +messages = messages or { } -- for the moment + +runners = runners or { } -- global (might become local) + runners.applications = { ["lua"] = "luatex --luaonly", ["luc"] = "luatex --luaonly", @@ -12056,7 +14406,7 @@ runners.registered = { makempy = { 'makempy.pl', true }, mptopdf = { 'mptopdf.pl', true }, pstopdf = { 'pstopdf.rb', true }, -- converts ps (and some more) images, does some cleaning (replaced) --- examplex = { 'examplex.rb', false }, + -- examplex = { 'examplex.rb', false }, concheck = { 'concheck.rb', false }, runtools = { 'runtools.rb', true }, textools = { 'textools.rb', true }, @@ -12065,9 +14415,9 @@ runners.registered = { rlxtools = { 'rlxtools.rb', true }, pdftools = { 'pdftools.rb', true }, mpstools = { 'mpstools.rb', true }, --- exatools = { 'exatools.rb', true }, + -- exatools = { 'exatools.rb', true }, xmltools = { 'xmltools.rb', true }, --- luatools = { 'luatools.lua', true }, + -- luatools = { 'luatools.lua', true }, mtxtools = { 'mtxtools.rb', true }, pdftrimwhite = { 'pdftrimwhite.pl', false } } @@ -12086,45 +14436,39 @@ end function runners.prepare() local checkname = environment.argument("ifchanged") - if checkname and checkname ~= "" then + if type(checkname) == "string" and checkname ~= "" then local oldchecksum = file.loadchecksum(checkname) local newchecksum = file.checksum(checkname) if oldchecksum == newchecksum then - logs.simple("file '%s' is unchanged",checkname) + if e_verbose then + report("file '%s' is unchanged",checkname) + end return "skip" - else - logs.simple("file '%s' is changed, processing started",checkname) + elseif e_verbose then + report("file '%s' is changed, processing started",checkname) end file.savechecksum(checkname) end - local oldname, newname = string.split(environment.argument("iftouched") or "", ",") - if oldname and newname and oldname ~= "" and newname ~= "" then - if not file.needs_updating(oldname,newname) then - logs.simple("file '%s' and '%s' have same age",oldname,newname) - return "skip" - else - logs.simple("file '%s' is older than '%s'",oldname,newname) - end - end - local tree = environment.argument('tree') or "" - if environment.argument('autotree') then - tree = os.getenv('TEXMFSTART_TREE') or os.getenv('TEXMFSTARTTREE') or tree - end - if tree and tree ~= "" then - resolvers.load_tree(tree) - end - local env = environment.argument('environment') or "" - if env and env ~= "" then - for _,e in pairs(string.split(env)) do - -- maybe force suffix when not given - resolvers.load_tree(e) + local touchname = environment.argument("iftouched") + if type(touchname) == "string" and touchname ~= "" then + local oldname, newname = string.split(touchname, ",") + if oldname and newname and oldname ~= "" and newname ~= "" then + if not file.needs_updating(oldname,newname) then + if e_verbose then + report("file '%s' and '%s' have same age",oldname,newname) + end + return "skip" + elseif e_verbose then + report("file '%s' is older than '%s'",oldname,newname) + end end end local runpath = environment.argument("path") - if runpath and not lfs.chdir(runpath) then - logs.simple("unable to change to path '%s'",runpath) + if type(runpath) == "string" and not lfs.chdir(runpath) then + report("unable to change to path '%s'",runpath) return "error" end + runners.prepare = function() end return "run" end @@ -12137,8 +14481,6 @@ function runners.execute_script(fullname,internal,nosplit) elseif state == 'skip' then return true elseif state == "run" then - instance.progname = environment.argument("progname") or instance.progname - instance.format = environment.argument("format") or instance.format local path, name, suffix, result = file.dirname(fullname), file.basename(fullname), file.extname(fullname), "" if path ~= "" then result = fullname @@ -12155,21 +14497,21 @@ function runners.execute_script(fullname,internal,nosplit) if suffix == "" then -- loop over known suffixes for _,s in pairs(runners.suffixes) do - result = resolvers.find_file(name .. "." .. s, 'texmfscripts') + result = resolvers.findfile(name .. "." .. s, 'texmfscripts') if result ~= "" then break end end elseif runners.applications[suffix] then - result = resolvers.find_file(name, 'texmfscripts') + result = resolvers.findfile(name, 'texmfscripts') else -- maybe look on path - result = resolvers.find_file(name, 'other text files') + result = resolvers.findfile(name, 'other text files') end end if result and result ~= "" then if not no_split then - local before, after = environment.split_arguments(fullname) -- already done + local before, after = environment.splitarguments(fullname) -- already done environment.arguments_before, environment.arguments_after = before, after end if internal then @@ -12181,12 +14523,12 @@ function runners.execute_script(fullname,internal,nosplit) if binary and binary ~= "" then result = binary .. " " .. result end - local command = result .. " " .. environment.reconstruct_commandline(environment.arguments_after,noquote) - if logs.verbose then - logs.simpleline() - logs.simple("executing: %s",command) - logs.simpleline() - logs.simpleline() + local command = result .. " " .. environment.reconstructcommandline(environment.arguments_after,noquote) + if e_verbose then + report() + report("executing: %s",command) + report() + report() io.flush() end -- no os.exec because otherwise we get the wrong return value @@ -12196,14 +14538,14 @@ function runners.execute_script(fullname,internal,nosplit) else if binary then binary = file.addsuffix(binary,os.binsuffix) - for p in string.gmatch(os.getenv("PATH"),"[^"..io.pathseparator.."]+") do + for p in gmatch(os.getenv("PATH"),"[^"..io.pathseparator.."]+") do if lfs.isfile(file.join(p,binary)) then return false end end - logs.simpleline() - logs.simple("This script needs '%s' which seems not to be installed.",binary) - logs.simpleline() + report() + report("This script needs '%s' which seems not to be installed.",binary) + report() end return false end @@ -12223,14 +14565,15 @@ function runners.execute_program(fullname) elseif state == 'skip' then return true elseif state == "run" then - local before, after = environment.split_arguments(fullname) - environment.initialize_arguments(after) + local before, after = environment.splitarguments(fullname) + for k=1,#after do after[k] = resolvers.resolve(after[k]) end + environment.initializearguments(after) fullname = fullname:gsub("^bin:","") - local command = fullname .. " " .. (environment.reconstruct_commandline(after or "",noquote) or "") - logs.simpleline() - logs.simple("executing: %s",command) - logs.simpleline() - logs.simpleline() + local command = fullname .. " " .. (environment.reconstructcommandline(after or "",noquote) or "") + report() + report("executing: %s",command) + report() + report() io.flush() local code = os.exec(command) -- (fullname,unpack(after)) does not work / maybe spawn return code == 0 @@ -12239,7 +14582,7 @@ function runners.execute_program(fullname) return false end --- the --usekpse flag will fallback on kpse (hm, we can better update mtx-stubs) +-- the --usekpse flag will fallback (not default) on kpse (hm, we can better update mtx-stubs) local windows_stub = '@echo off\013\010setlocal\013\010set ownpath=%%~dp0%%\013\010texlua "%%ownpath%%mtxrun.lua" --usekpse --execute %s %%*\013\010endlocal\013\010' local unix_stub = '#!/bin/sh\010mtxrun --usekpse --execute %s \"$@\"\010' @@ -12258,22 +14601,22 @@ function runners.handle_stubs(create) for _,v in pairs(runners.registered) do local name, doit = v[1], v[2] if doit then - local base = string.gsub(file.basename(name), "%.(.-)$", "") + local base = gsub(file.basename(name), "%.(.-)$", "") if create then if windows then - io.savedata(file.join(stubpath,base..".bat"),string.format(windows_stub,name)) - logs.simple("windows stub for '%s' created",base) + io.savedata(file.join(stubpath,base..".bat"),format(windows_stub,name)) + report("windows stub for '%s' created",base) end if unix then - io.savedata(file.join(stubpath,base),string.format(unix_stub,name)) - logs.simple("unix stub for '%s' created",base) + io.savedata(file.join(stubpath,base),format(unix_stub,name)) + report("unix stub for '%s' created",base) end else if windows and (os.remove(file.join(stubpath,base..'.bat')) or os.remove(file.join(stubpath,base..'.cmd'))) then - logs.simple("windows stub for '%s' removed", base) + report("windows stub for '%s' removed", base) end if unix and (os.remove(file.join(stubpath,base)) or os.remove(file.join(stubpath,base..'.sh'))) then - logs.simple("unix stub for '%s' removed",base) + report("unix stub for '%s' removed",base) end end end @@ -12289,7 +14632,7 @@ end function runners.locate_file(filename) -- differs from texmfstart where locate appends .com .exe .bat ... todo if filename and filename ~= "" then - runners.report_location(resolvers.find_given_file(filename)) + runners.report_location(resolvers.findgivenfile(filename)) end end @@ -12298,28 +14641,28 @@ function runners.locate_platform() end function runners.report_location(result) - if logs.verbose then - logs.simpleline() + if e_verbose then + reportline() if result and result ~= "" then - logs.simple(result) + report(result) else - logs.simple("not found") + report("not found") end else io.write(result) end end -function runners.edit_script(filename) -- we assume that vim is present on most systems - local editor = os.getenv("MTXRUN_EDITOR") or os.getenv("TEXMFSTART_EDITOR") or os.getenv("EDITOR") or 'vim' +function runners.edit_script(filename) -- we assume that gvim is present on most systems (todo: also in cnf file) + local editor = os.getenv("MTXRUN_EDITOR") or os.getenv("TEXMFSTART_EDITOR") or os.getenv("EDITOR") or 'gvim' local rest = resolvers.resolve(filename) if rest ~= "" then local command = editor .. " " .. rest - if logs.verbose then - logs.simpleline() - logs.simple("starting editor: %s",command) - logs.simple_line() - logs.simple_line() + if e_verbose then + report() + report("starting editor: %s",command) + report() + report() end os.launch(command) end @@ -12362,33 +14705,33 @@ end function runners.launch_file(filename) instance.allresults = true - logs.setverbose(true) + trackers.enable("resolvers.locating") local pattern = environment.arguments["pattern"] if not pattern or pattern == "" then pattern = filename end if not pattern or pattern == "" then - logs.simple("provide name or --pattern=") + report("provide name or --pattern=") else - local t = resolvers.find_files(pattern) + local t = resolvers.findfiles(pattern) if not t or #t == 0 then - t = resolvers.find_files("*/" .. pattern) + t = resolvers.findfiles("*/" .. pattern) end if not t or #t == 0 then - t = resolvers.find_files("*/" .. pattern .. "*") + t = resolvers.findfiles("*/" .. pattern .. "*") end if t and #t > 0 then if environment.arguments["all"] then for _, v in pairs(t) do - logs.simple("launching %s", v) + report("launching %s", v) resolvers.launch(v) end else - logs.simple("launching %s", t[1]) + report("launching %s", t[1]) resolvers.launch(t[1]) end else - logs.simple("no match for %s", pattern) + report("no match for %s", pattern) end end end @@ -12421,33 +14764,45 @@ function runners.find_mtx_script(filename) local mtxprefix = (filename:find("^mtx%-") and "") or "mtx-" -- context namespace, mtx-This is a prelude to a more extensive logging module. We no longer
+provide
The parser used here is inspired by the variant discussed in the lua book, but handles comment and processing instructions, has a different structure, provides @@ -4001,11 +6275,15 @@ The find based parser can be found in l-xml-edu.lua along with other older code.
Beware, the interface may change. For instance at, ns, tg, dt may get more verbose names. Once the code is stable we will also remove some tracing and optimize the code.
+ +I might even decide to reimplement the parser using the latest
We cannot load an
A few helpers:
--ldx]]-- ---~ xmlsetproperty(root,"settings",settings) function xml.settings(e) while e do @@ -5150,6 +7435,7 @@ function xml.makestandalone(root) end end end + return root end @@ -5166,11 +7452,14 @@ if not modules then modules = { } end modules ['lxml-pth'] = { } -- e.ni is only valid after a filter run +-- todo: B/C/[get first match] local concat, remove, insert = table.concat, table.remove, table.insert local type, next, tonumber, tostring, setmetatable, loadstring = type, next, tonumber, tostring, setmetatable, loadstring local format, upper, lower, gmatch, gsub, find, rep = string.format, string.upper, string.lower, string.gmatch, string.gsub, string.find, string.rep -local lpegmatch = lpeg.match +local lpegmatch, lpegpatterns = lpeg.match, lpeg.patterns + +local setmetatableindex = table.setmetatableindex -- beware, this is not xpath ... e.g. position is different (currently) and -- we have reverse-sibling as reversed preceding sibling @@ -5202,23 +7491,33 @@ local trace_lpath = false if trackers then trackers.register("xml.path", local trace_lparse = false if trackers then trackers.register("xml.parse", function(v) trace_lparse = v end) end local trace_lprofile = false if trackers then trackers.register("xml.profile", function(v) trace_lpath = v trace_lparse = v trace_lprofile = v end) end +local report_lpath = logs.reporter("xml","lpath") + --[[ldx--We've now arrived at an interesting part: accessing the tree using a subset
of
This is the main filter function. It returns whatever is asked for.
+--ldx]]-- + +function xml.filter(root,pattern) -- no longer funny attribute handling here + return applylpath(root,pattern) +end + -- internal (parsed) expressions.child = function(e,pattern) - return parse_apply({ e },pattern) -- todo: cache + return applylpath(e,pattern) -- todo: cache end expressions.count = function(e,pattern) - local collected = parse_apply({ e },pattern) -- todo: cache + local collected = applylpath(e,pattern) -- todo: cache return (collected and #collected) or 0 end @@ -6188,7 +8492,7 @@ expressions.oneof = function(s,...) -- slow local t = {...} for i=1,#t do if s == t[i] then return true end end return false end expressions.error = function(str) - xml.error_handler("unknown function in lpath expression",tostring(str or "?")) + xml.errorhandler("unknown function in lpath expression",tostring(str or "?")) return false end expressions.undefined = function(s) @@ -6217,8 +8521,8 @@ expressions.boolean = toboolean -- user interface local function traverse(root,pattern,handle) - logs.report("xml","use 'xml.selection' instead for '%s'",pattern) - local collected = parse_apply({ root },pattern) + report_lpath("use 'xml.selection' instead for '%s'",pattern) + local collected = applylpath(root,pattern) if collected then for c=1,#collected do local e = collected[c] @@ -6229,7 +8533,7 @@ local function traverse(root,pattern,handle) end local function selection(root,pattern,handle) - local collected = parse_apply({ root },pattern) + local collected = applylpath(root,pattern) if collected then if handle then for c=1,#collected do @@ -6241,19 +8545,9 @@ local function selection(root,pattern,handle) end end -xml.parse_parser = parser -xml.parse_pattern = parse_pattern -xml.parse_apply = parse_apply xml.traverse = traverse -- old method, r, d, k xml.selection = selection -- new method, simple handle -local lpath = parse_pattern - -xml.lpath = lpath - -function xml.cached_patterns() - return cache -end -- generic function finalizer (independant namespace) @@ -6265,13 +8559,13 @@ local function dofunction(collected,fnc) f(collected[c]) end else - logs.report("xml","unknown function '%s'",fnc) + report_lpath("unknown function '%s'",fnc) end end end -xml.finalizers.xml["function"] = dofunction -xml.finalizers.tex["function"] = dofunction +finalizers.xml["function"] = dofunction +finalizers.tex["function"] = dofunction -- functions @@ -6363,14 +8657,6 @@ expressions.tag = function(e,n) -- only tg end end ---[[ldx-- -This is the main filter function. It returns whatever is asked for.
---ldx]]-- - -function xml.filter(root,pattern) -- no longer funny attribute handling here - return parse_apply({ root },pattern) -end - --[[ldx--Often using an iterators looks nicer in the code than passing handler
functions. The
The following helper functions best belong to the
Here are a few synonyms.
--ldx]]-- -xml.each = xml.each_element -xml.process = xml.process_element -xml.strip = xml.strip_whitespace -xml.collect = xml.collect_elements -xml.all = xml.collect_elements - -xml.insert = xml.insert_element_after -xml.inject = xml.inject_element_after -xml.after = xml.insert_element_after -xml.before = xml.insert_element_before -xml.delete = xml.delete_element -xml.replace = xml.replace_element +xml.all = xml.each +xml.insert = xml.insertafter +xml.inject = xml.injectafter +xml.after = xml.insertafter +xml.before = xml.insertbefore +xml.process = xml.each + +-- obsolete + +xml.obsolete = xml.obsolete or { } +local obsolete = xml.obsolete + +xml.strip_whitespace = xml.strip obsolete.strip_whitespace = xml.strip +xml.collect_elements = xml.collect obsolete.collect_elements = xml.collect +xml.delete_element = xml.delete obsolete.delete_element = xml.delete +xml.replace_element = xml.replace obsolete.replace_element = xml.replacet +xml.each_element = xml.each obsolete.each_element = xml.each +xml.process_elements = xml.process obsolete.process_elements = xml.process +xml.insert_element_after = xml.insertafter obsolete.insert_element_after = xml.insertafter +xml.insert_element_before = xml.insertbefore obsolete.insert_element_before = xml.insertbefore +xml.inject_element_after = xml.injectafter obsolete.inject_element_after = xml.injectafter +xml.inject_element_before = xml.injectbefore obsolete.inject_element_before = xml.injectbefore +xml.process_attributes = xml.processattributes obsolete.process_attributes = xml.processattributes +xml.collect_texts = xml.collecttexts obsolete.collect_texts = xml.collecttexts +xml.inject_element = xml.inject obsolete.inject_element = xml.inject +xml.remap_tag = xml.remaptag obsolete.remap_tag = xml.remaptag +xml.remap_name = xml.remapname obsolete.remap_name = xml.remapname +xml.remap_namespace = xml.remapnamespace obsolete.remap_namespace = xml.remapnamespace end -- of closure @@ -7084,10 +9375,15 @@ if not modules then modules = { } end modules ['lxml-xml'] = { license = "see context related readme files" } +local concat = string.concat + +local xml = xml + local finalizers = xml.finalizers.xml local xmlfilter = xml.filter -- we could inline this one for speed local xmltostring = xml.tostring local xmlserialize = xml.serialize +local xmlcollected = xml.collected local function first(collected) -- wrong ? return collected and collected[1] @@ -7101,15 +9397,8 @@ local function all(collected) return collected end -local function reverse(collected) - if collected then - local reversed = { } - for c=#collected,1,-1 do - reversed[#reversed+1] = collected[c] - end - return reversed - end -end + +local reverse = table.reversed local function attribute(collected,name) if collected and #collected > 0 then @@ -7200,11 +9489,12 @@ end local function texts(collected) if collected then - local t = { } + local t, n = { }, 0 for c=1,#collected do - local e = collection[c] + local e = collected[c] if e and e.dt then - t[#t+1] = e.dt + n = n + 1 + t[n] = e.dt end end return t @@ -7247,14 +9537,15 @@ end local function tags(collected,nonamespace) if collected then - local t = { } + local t, n = { }, 0 for c=1,#collected do local e = collected[c] local ns, tg = e.ns, e.tg + n = n + 1 if nonamespace or ns == "" then - t[#t+1] = tg + t[n] = tg else - t[#t+1] = ns .. ":" .. tg + t[n] = ns .. ":" .. tg end end return t @@ -7365,1812 +9656,1937 @@ xml.all = xml.filter xml.index = xml.position xml.found = xml.filter +-- a nice one: + +local function totable(x) + local t = { } + for e in xmlcollected(x[1] or x,"/*") do + t[e.tg] = xmltostring(e.dt) or "" + end + return next(t) and t or nil +end + +xml.table = totable +finalizers.table = totable + +local function textonly(e,t) + if e then + local edt = e.dt + if edt then + for i=1,#edt do + local e = edt[i] + if type(e) == "table" then + textonly(e,t) + else + t[#t+1] = e + end + end + end + end + return t +end + +function xml.textonly(e) -- no pattern + return concat(textonly(e,{})) +end + end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['luat-env'] = { +if not modules then modules = { } end modules ['data-ini'] = { version = 1.001, comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" + license = "see context related readme files", } --- A former version provided functionality for non embeded core --- scripts i.e. runtime library loading. Given the amount of --- Lua code we use now, this no longer makes sense. Much of this --- evolved before bytecode arrays were available and so a lot of --- code has disappeared already. +local gsub, find, gmatch = string.gsub, string.find, string.gmatch +local concat = table.concat +local next, type = next, type -local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) +local filedirname, filebasename, fileextname, filejoin = file.dirname, file.basename, file.extname, file.join -local format, sub, match, gsub, find = string.format, string.sub, string.match, string.gsub, string.find -local unquote, quote = string.unquote, string.quote +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) +local trace_detail = false trackers.register("resolvers.details", function(v) trace_detail = v end) +local trace_expansions = false trackers.register("resolvers.expansions", function(v) trace_expansions = v end) --- precautions +local report_initialization = logs.reporter("resolvers","initialization") -os.setlocale(nil,nil) -- useless feature and even dangerous in luatex +local ostype, osname, ossetenv, osgetenv = os.type, os.name, os.setenv, os.getenv -function os.setlocale() - -- no way you can mess with it -end +-- The code here used to be part of a data-res but for convenience +-- we now split it over multiple files. As this file is now the +-- starting point we introduce resolvers here. --- dirty tricks +resolvers = resolvers or { } +local resolvers = resolvers + +-- We don't want the kpse library to kick in. Also, we want to be able to +-- execute programs. Control over execution is implemented later. + +texconfig.kpse_init = false +texconfig.shell_escape = 't' + +kpse = { original = kpse } + +setmetatable(kpse, { + __index = function(kp,name) + report_initialization("fatal error: kpse library is accessed (key: %s)",name) + os.exit() + end +} ) + +-- First we check a couple of environment variables. Some might be +-- set already but we need then later on. We start with the system +-- font path. + +do + + local osfontdir = osgetenv("OSFONTDIR") + + if osfontdir and osfontdir ~= "" then + -- ok + elseif osname == "windows" then + ossetenv("OSFONTDIR","c:/windows/fonts//") + elseif osname == "macosx" then + ossetenv("OSFONTDIR","$HOME/Library/Fonts//;/Library/Fonts//;/System/Library/Fonts//") + end -if arg and (arg[0] == 'luatex' or arg[0] == 'luatex.exe') and arg[1] == "--luaonly" then - arg[-1]=arg[0] arg[0]=arg[2] for k=3,#arg do arg[k-2]=arg[k] end arg[#arg]=nil arg[#arg]=nil end -if profiler and os.env["MTX_PROFILE_RUN"] == "YES" then - profiler.start("luatex-profile.log") +-- Next comes the user's home path. We need this as later on we have +-- to replace ~ with its value. + +do + + local homedir = osgetenv(ostype == "windows" and 'USERPROFILE' or 'HOME') or '' + + if not homedir or homedir == "" then + homedir = string.char(127) -- we need a value, later we wil trigger on it + end + + homedir = file.collapsepath(homedir) + + ossetenv("HOME", homedir) -- can be used in unix cnf files + ossetenv("USERPROFILE",homedir) -- can be used in windows cnf files + + environment.homedir = homedir + end --- environment +-- The following code sets the name of the own binary and its +-- path. This is fallback code as we have os.selfdir now. -environment = environment or { } -environment.arguments = { } -environment.files = { } -environment.sortedflags = nil +do -if not environment.jobname or environment.jobname == "" then if tex then environment.jobname = tex.jobname end end -if not environment.version or environment.version == "" then environment.version = "unknown" end -if not environment.jobname then environment.jobname = "unknown" end + local args = environment.originalarguments or arg -- this needs a cleanup -function environment.initialize_arguments(arg) - local arguments, files = { }, { } - environment.arguments, environment.files, environment.sortedflags = arguments, files, nil - for index=1,#arg do - local argument = arg[index] - if index > 0 then - local flag, value = match(argument,"^%-+(.-)=(.-)$") - if flag then - arguments[flag] = unquote(value or "") - else - flag = match(argument,"^%-+(.+)") - if flag then - arguments[flag] = true - else - files[#files+1] = argument + local ownbin = environment.ownbin or args[-2] or arg[-2] or args[-1] or arg[-1] or arg[0] or "luatex" + local ownpath = environment.ownpath or os.selfdir + + ownbin = file.collapsepath(ownbin) + ownpath = file.collapsepath(ownpath) + + if not ownpath or ownpath == "" or ownpath == "unset" then + ownpath = args[-1] or arg[-1] + ownpath = ownpath and filedirname(gsub(ownpath,"\\","/")) + if not ownpath or ownpath == "" then + ownpath = args[-0] or arg[-0] + ownpath = ownpath and filedirname(gsub(ownpath,"\\","/")) + end + local binary = ownbin + if not ownpath or ownpath == "" then + ownpath = ownpath and filedirname(binary) + end + if not ownpath or ownpath == "" then + if os.binsuffix ~= "" then + binary = file.replacesuffix(binary,os.binsuffix) + end + local path = osgetenv("PATH") + if path then + for p in gmatch(path,"[^"..io.pathseparator.."]+") do + local b = filejoin(p,binary) + if lfs.isfile(b) then + -- we assume that after changing to the path the currentdir function + -- resolves to the real location and use this side effect here; this + -- trick is needed because on the mac installations use symlinks in the + -- path instead of real locations + local olddir = lfs.currentdir() + if lfs.chdir(p) then + local pp = lfs.currentdir() + if trace_locating and p ~= pp then + report_initialization("following symlink '%s' to '%s'",p,pp) + end + ownpath = pp + lfs.chdir(olddir) + else + if trace_locating then + report_initialization("unable to check path '%s'",p) + end + ownpath = p + end + break + end end end end + if not ownpath or ownpath == "" then + ownpath = "." + report_initialization("forcing fallback ownpath .") + elseif trace_locating then + report_initialization("using ownpath '%s'",ownpath) + end end - environment.ownname = environment.ownname or arg[0] or 'unknown.lua' + + environment.ownbin = ownbin + environment.ownpath = ownpath + end -function environment.setargument(name,value) - environment.arguments[name] = value +resolvers.ownpath = environment.ownpath + +function resolvers.getownpath() + return environment.ownpath end --- todo: defaults, better checks e.g on type (boolean versus string) --- --- tricky: too many hits when we support partials unless we add --- a registration of arguments so from now on we have 'partial' +-- The self variables permit us to use only a few (or even no) +-- environment variables. -function environment.argument(name,partial) - local arguments, sortedflags = environment.arguments, environment.sortedflags - if arguments[name] then - return arguments[name] - elseif partial then - if not sortedflags then - sortedflags = table.sortedkeys(arguments) - for k=1,#sortedflags do - sortedflags[k] = "^" .. sortedflags[k] - end - environment.sortedflags = sortedflags - end - -- example of potential clash: ^mode ^modefile - for k=1,#sortedflags do - local v = sortedflags[k] - if find(name,v) then - return arguments[sub(v,2,#v)] - end - end +do + + local ownpath = environment.ownpath or dir.current() + + if ownpath then + ossetenv('SELFAUTOLOC', file.collapsepath(ownpath)) + ossetenv('SELFAUTODIR', file.collapsepath(ownpath .. "/..")) + ossetenv('SELFAUTOPARENT', file.collapsepath(ownpath .. "/../..")) + else + report_initialization("error: unable to locate ownpath") + os.exit() end - return nil + end -environment.argument("x",true) +-- The running os: -function environment.split_arguments(separator) -- rather special, cut-off before separator - local done, before, after = false, { }, { } - local original_arguments = environment.original_arguments - for k=1,#original_arguments do - local v = original_arguments[k] - if not done and v == separator then - done = true - elseif done then - after[#after+1] = v - else - before[#before+1] = v - end - end - return before, after +-- todo: check is context sits here os.platform is more trustworthy +-- that the bin check as mtx-update runs from another path + +local texos = environment.texos or osgetenv("TEXOS") +local texmfos = environment.texmfos or osgetenv('SELFAUTODIR') + +if not texos or texos == "" then + texos = file.basename(texmfos) end -function environment.reconstruct_commandline(arg,noquote) - arg = arg or environment.original_arguments - if noquote and #arg == 1 then - local a = arg[1] - a = resolvers.resolve(a) - a = unquote(a) - return a - elseif #arg > 0 then - local result = { } - for i=1,#arg do - local a = arg[i] - a = resolvers.resolve(a) - a = unquote(a) - a = gsub(a,'"','\\"') -- tricky - if find(a," ") then - result[#result+1] = quote(a) - else - result[#result+1] = a - end - end - return table.join(result," ") - else - return "" +ossetenv('TEXMFOS', texmfos) -- full bin path +ossetenv('TEXOS', texos) -- partial bin parent +ossetenv('SELFAUTOSYSTEM',os.platform) -- bonus + +environment.texos = texos +environment.texmfos = texmfos + +-- The current root: + +local texroot = environment.texroot or osgetenv("TEXROOT") + +if not texroot or texroot == "" then + texroot = osgetenv('SELFAUTOPARENT') + ossetenv('TEXROOT',texroot) +end + +environment.texroot = file.collapsepath(texroot) + +-- Tracing. Todo ... + +function resolvers.settrace(n) -- no longer number but: 'locating' or 'detail' + if n then + trackers.disable("resolvers.*") + trackers.enable("resolvers."..n) end end -if arg then +resolvers.settrace(osgetenv("MTX_INPUT_TRACE")) + +-- todo: + +-- if profiler and osgetenv("MTX_PROFILE_RUN") == "YES" then +-- profiler.start("luatex-profile.log") +-- end + +-- a forward definition + +if not resolvers.resolve then + function resolvers.resolve (s) return s end + function resolvers.unresolve(s) return s end + function resolvers.repath (s) return s end +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['data-exp'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files", +} + +local format, find, gmatch, lower = string.format, string.find, string.gmatch, string.lower +local concat, sort = table.concat, table.sort +local lpegmatch, lpegpatterns = lpeg.match, lpeg.patterns +local Ct, Cs, Cc, P, C, S = lpeg.Ct, lpeg.Cs, lpeg.Cc, lpeg.P, lpeg.C, lpeg.S +local type, next = type, next - -- new, reconstruct quoted snippets (maybe better just remove the " then and add them later) - local newarg, instring = { }, false +local ostype = os.type +local collapsepath = file.collapsepath - for index=1,#arg do - local argument = arg[index] - if find(argument,"^\"") then - newarg[#newarg+1] = gsub(argument,"^\"","") - if not find(argument,"\"$") then - instring = true - end - elseif find(argument,"\"$") then - newarg[#newarg] = newarg[#newarg] .. " " .. gsub(argument,"\"$","") - instring = false - elseif instring then - newarg[#newarg] = newarg[#newarg] .. " " .. argument - else - newarg[#newarg+1] = argument - end - end - for i=1,-5,-1 do - newarg[i] = arg[i] - end +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) +local trace_expansions = false trackers.register("resolvers.expansions", function(v) trace_expansions = v end) - environment.initialize_arguments(newarg) - environment.original_arguments = newarg - environment.raw_arguments = arg +local report_expansions = logs.reporter("resolvers","expansions") - arg = { } -- prevent duplicate handling +local resolvers = resolvers -end +-- As this bit of code is somewhat special it gets its own module. After +-- all, when working on the main resolver code, I don't want to scroll +-- past this every time. See data-obs.lua for the gsub variant. --- weird place ... depends on a not yet loaded module +-- {a,b,c,d} +-- a,b,c/{p,q,r},d +-- a,b,c/{p,q,r}/d/{x,y,z}// +-- a,b,c/{p,q/{x,y,z},r},d/{p,q,r} +-- a,b,c/{p,q/{x,y,z},r},d/{p,q,r} +-- a{b,c}{d,e}f +-- {a,b,c,d} +-- {a,b,c/{p,q,r},d} +-- {a,b,c/{p,q,r}/d/{x,y,z}//} +-- {a,b,c/{p,q/{x,y,z}},d/{p,q,r}} +-- {a,b,c/{p,q/{x,y,z},w}v,d/{p,q,r}} +-- {$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c} -function environment.texfile(filename) - return resolvers.find_file(filename,'tex') +local function f_first(a,b) + local t, n = { }, 0 + for s in gmatch(b,"[^,]+") do + n = n + 1 ; t[n] = a .. s + end + return concat(t,",") end -function environment.luafile(filename) - local resolved = resolvers.find_file(filename,'tex') or "" - if resolved ~= "" then - return resolved +local function f_second(a,b) + local t, n = { }, 0 + for s in gmatch(a,"[^,]+") do + n = n + 1 ; t[n] = s .. b end - resolved = resolvers.find_file(filename,'texmfscripts') or "" - if resolved ~= "" then - return resolved + return concat(t,",") +end + +local function f_both(a,b) + local t, n = { }, 0 + for sa in gmatch(a,"[^,]+") do + for sb in gmatch(b,"[^,]+") do + n = n + 1 ; t[n] = sa .. sb + end end - return resolvers.find_file(filename,'luatexlibs') or "" + return concat(t,",") end -environment.loadedluacode = loadfile -- can be overloaded +local left = P("{") +local right = P("}") +local var = P((1 - S("{}" ))^0) +local set = P((1 - S("{},"))^0) +local other = P(1) ---~ function environment.loadedluacode(name) ---~ if os.spawn("texluac -s -o texluac.luc " .. name) == 0 then ---~ local chunk = loadstring(io.loaddata("texluac.luc")) ---~ os.remove("texluac.luc") ---~ return chunk ---~ else ---~ environment.loadedluacode = loadfile -- can be overloaded ---~ return loadfile(name) ---~ end ---~ end - -function environment.luafilechunk(filename) -- used for loading lua bytecode in the format - filename = file.replacesuffix(filename, "lua") - local fullname = environment.luafile(filename) - if fullname and fullname ~= "" then - if trace_locating then - logs.report("fileio","loading file %s", fullname) +local l_first = Cs( ( Cc("{") * (C(set) * left * C(var) * right / f_first) * Cc("}") + other )^0 ) +local l_second = Cs( ( Cc("{") * (left * C(var) * right * C(set) / f_second) * Cc("}") + other )^0 ) +local l_both = Cs( ( Cc("{") * (left * C(var) * right * left * C(var) * right / f_both) * Cc("}") + other )^0 ) +local l_rest = Cs( ( left * var * (left/"") * var * (right/"") * var * right + other )^0 ) + +local stripper_1 = lpeg.stripper ("{}@") +local replacer_1 = lpeg.replacer { { ",}", ",@}" }, { "{,", "{@," }, } + +local function splitpathexpr(str, newlist, validate) -- I couldn't resist lpegging it (nice exercise). + if trace_expansions then + report_expansions("expanding variable '%s'",str) + end + local t, ok, done = newlist or { }, false, false + local n = #t + str = lpegmatch(replacer_1,str) + repeat local old = str + repeat local old = str ; str = lpegmatch(l_first, str) until old == str + repeat local old = str ; str = lpegmatch(l_second,str) until old == str + repeat local old = str ; str = lpegmatch(l_both, str) until old == str + repeat local old = str ; str = lpegmatch(l_rest, str) until old == str + until old == str -- or not find(str,"{") + str = lpegmatch(stripper_1,str) + if validate then + for s in gmatch(str,"[^,]+") do + s = validate(s) + if s then + n = n + 1 ; t[n] = s + end end - return environment.loadedluacode(fullname) else - if trace_locating then - logs.report("fileio","unknown file %s", filename) + for s in gmatch(str,"[^,]+") do + n = n + 1 ; t[n] = s + end + end + if trace_expansions then + for k=1,#t do + report_expansions("% 4i: %s",k,t[k]) end - return nil end + return t end --- the next ones can use the previous ones / combine +-- We could make the previous one public. -function environment.loadluafile(filename, version) - local lucname, luaname, chunk - local basename = file.removesuffix(filename) - if basename == filename then - lucname, luaname = basename .. ".luc", basename .. ".lua" - else - lucname, luaname = nil, basename -- forced suffix - end - -- when not overloaded by explicit suffix we look for a luc file first - local fullname = (lucname and environment.luafile(lucname)) or "" - if fullname ~= "" then - if trace_locating then - logs.report("fileio","loading %s", fullname) - end - chunk = loadfile(fullname) -- this way we don't need a file exists check +local function validate(s) + s = collapsepath(s) -- already keeps the // + return s ~= "" and not find(s,"^!*unset/*$") and s +end + +resolvers.validatedpath = validate -- keeps the trailing // + +function resolvers.expandedpathfromlist(pathlist) + local newlist = { } + for k=1,#pathlist do + splitpathexpr(pathlist[k],newlist,validate) end - if chunk then - assert(chunk)() - if version then - -- we check of the version number of this chunk matches - local v = version -- can be nil - if modules and modules[filename] then - v = modules[filename].version -- new method - elseif versions and versions[filename] then - v = versions[filename] -- old method + return newlist +end + +local cleanup = lpeg.replacer { + { "!" , "" }, + { "\\" , "/" }, +} + +local homedir + +function resolvers.cleanpath(str) + if not homedir then + homedir = lpegmatch(cleanup,environment.homedir or "") + if homedir == string.char(127) or homedir == "" or not lfs.isdir(homedir) then + if trace_expansions then + report_expansions("no home dir set, ignoring dependent paths") end - if v == version then - return true - else - if trace_locating then - logs.report("fileio","version mismatch for %s: lua=%s, luc=%s", filename, v, version) + function resolvers.cleanpath(str) + if find(str,"~") then + return "" -- special case + else + return str and lpegmatch(cleanup,str) end - environment.loadluafile(filename) end else - return true - end - end - fullname = (luaname and environment.luafile(luaname)) or "" - if fullname ~= "" then - if trace_locating then - logs.report("fileio","loading %s", fullname) - end - chunk = loadfile(fullname) -- this way we don't need a file exists check - if not chunk then - if trace_locating then - logs.report("fileio","unknown file %s", filename) + cleanup = lpeg.replacer { + { "!" , "" }, + { "\\" , "/" }, + { "~" , homedir }, + } + function resolvers.cleanpath(str) + return str and lpegmatch(cleanup,str) end - else - assert(chunk)() - return true end end - return false + return resolvers.cleanpath(str) end +-- This one strips quotes and funny tokens. -end -- of closure - -do -- create closure to overcome 200 locals limit +local expandhome = P("~") / "$HOME" -- environment.homedir -if not modules then modules = { } end modules ['trac-inf'] = { - version = 1.001, - comment = "companion to trac-inf.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} +local dodouble = P('"')/"" * (expandhome + (1 - P('"')))^0 * P('"')/"" +local dosingle = P("'")/"" * (expandhome + (1 - P("'")))^0 * P("'")/"" +local dostring = (expandhome + 1 )^0 -local format = string.format +local stripper = Cs( + lpegpatterns.unspacer * (dosingle + dodouble + dostring) * lpegpatterns.unspacer +) -local statusinfo, n, registered = { }, 0, { } +function resolvers.checkedvariable(str) -- assumes str is a string + return lpegmatch(stripper,str) or str +end -statistics = statistics or { } +-- The path splitter: -statistics.enable = true -statistics.threshold = 0.05 +-- A config (optionally) has the paths split in tables. Internally +-- we join them and split them after the expansion has taken place. This +-- is more convenient. --- timing functions +local cache = { } -local clock = os.gettimeofday or os.clock +---- splitter = Ct(lpeg.splitat(S(ostype == "windows" and ";" or ":;"))) -- maybe add , +local splitter = Ct(lpeg.splitat(";")) -- as we move towards urls, prefixes and use tables we no longer do : -local notimer +local backslashswapper = lpeg.replacer("\\","/") -function statistics.hastimer(instance) - return instance and instance.starttime +local function splitconfigurationpath(str) -- beware, this can be either a path or a { specification } + if str then + local found = cache[str] + if not found then + if str == "" then + found = { } + else + local split = lpegmatch(splitter,lpegmatch(backslashswapper,str)) -- can be combined + found = { } + local noffound = 0 + for i=1,#split do + local s = split[i] + if not find(s,"^{*unset}*") then + noffound = noffound + 1 + found[noffound] = s + end + end + if trace_expansions then + report_expansions("splitting path specification '%s'",str) + for k=1,noffound do + report_expansions("% 4i: %s",k,found[k]) + end + end + cache[str] = found + end + end + return found + end end -function statistics.resettiming(instance) - if not instance then - notimer = { timing = 0, loadtime = 0 } +resolvers.splitconfigurationpath = splitconfigurationpath + +function resolvers.splitpath(str) + if type(str) == 'table' then + return str else - instance.timing, instance.loadtime = 0, 0 + return splitconfigurationpath(str) end end -function statistics.starttiming(instance) - if not instance then - notimer = { } - instance = notimer - end - local it = instance.timing - if not it then - it = 0 - end - if it == 0 then - instance.starttime = clock() - if not instance.loadtime then - instance.loadtime = 0 - end +function resolvers.joinpath(str) + if type(str) == 'table' then + return file.joinpath(str) else ---~ logs.report("system","nested timing (%s)",tostring(instance)) + return str end - instance.timing = it + 1 end -function statistics.stoptiming(instance, report) - if not instance then - instance = notimer - end - if instance then - local it = instance.timing - if it > 1 then - instance.timing = it - 1 - else - local starttime = instance.starttime - if starttime then - local stoptime = clock() - local loadtime = stoptime - starttime - instance.stoptime = stoptime - instance.loadtime = instance.loadtime + loadtime - if report then - statistics.report("load time %0.3f",loadtime) +-- The next function scans directories and returns a hash where the +-- entries are either strings or tables. + +-- starting with . or .. etc or funny char + + + + +local weird = P(".")^1 + lpeg.anywhere(S("~`!#$%^&*()={}[]:;\"\'||<>,?\n\r\t")) + +local attributes, directory = lfs.attributes, lfs.dir + +local function scan(files,spec,path,n,m,r) + local full = (path == "" and spec) or (spec .. path .. '/') + local dirs, nofdirs = { }, 0 + for name in directory(full) do + if not lpegmatch(weird,name) then + local mode = attributes(full..name,'mode') + if mode == 'file' then + n = n + 1 + local f = files[name] + if f then + if type(f) == 'string' then + files[name] = { f, path } + else + f[#f+1] = path + end + else -- probably unique anyway + files[name] = path + local lower = lower(name) + if name ~= lower then + files["remap:"..lower] = name + r = r + 1 + end + end + elseif mode == 'directory' then + m = m + 1 + nofdirs = nofdirs + 1 + if path ~= "" then + dirs[nofdirs] = path..'/'..name + else + dirs[nofdirs] = name end - instance.timing = 0 - return loadtime end end end - return 0 -end - -function statistics.elapsedtime(instance) - if not instance then - instance = notimer + if nofdirs > 0 then + sort(dirs) + for i=1,nofdirs do + files, n, m, r = scan(files,spec,dirs[i],n,m,r) + end end - return format("%0.3f",(instance and instance.loadtime) or 0) + return files, n, m, r end -function statistics.elapsedindeed(instance) - if not instance then - instance = notimer +function resolvers.scanfiles(path,branch) + if trace_locating then + report_expansions("scanning path '%s', branch '%s'",path, branch or path) + end + local realpath = resolvers.resolve(path) -- no shortcut + local files, n, m, r = scan({ },realpath .. '/',"",0,0,0) + files.__path__ = path -- can be selfautoparent:texmf-whatever + files.__files__ = n + files.__directories__ = m + files.__remappings__ = r + if trace_locating then + report_expansions("%s files found on %s directories with %s uppercase remappings",n,m,r) end - local t = (instance and instance.loadtime) or 0 - return t > statistics.threshold + return files end -function statistics.elapsedseconds(instance,rest) -- returns nil if 0 seconds - if statistics.elapsedindeed(instance) then - return format("%s seconds %s", statistics.elapsedtime(instance),rest or "") - end -end --- general function -function statistics.register(tag,fnc) - if statistics.enable and type(fnc) == "function" then - local rt = registered[tag] or (#statusinfo + 1) - statusinfo[rt] = { tag, fnc } - registered[tag] = rt - if #tag > n then n = #tag end - end -end +end -- of closure -function statistics.show(reporter) - if statistics.enable then - if not reporter then reporter = function(tag,data,n) texio.write_nl(tag .. " " .. data) end end - -- this code will move - local register = statistics.register - register("luatex banner", function() - return string.lower(status.banner) - end) - register("control sequences", function() - return format("%s of %s", status.cs_count, status.hash_size+status.hash_extra) - end) - register("callbacks", function() - local total, indirect = status.callbacks or 0, status.indirect_callbacks or 0 - return format("direct: %s, indirect: %s, total: %s", total-indirect, indirect, total) - end) - register("current memory usage", statistics.memused) - register("runtime",statistics.runtime) --- -- - for i=1,#statusinfo do - local s = statusinfo[i] - local r = s[2]() - if r then - reporter(s[1],r,n) +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['data-env'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files", +} + +local lower, gsub = string.lower, string.gsub + +local resolvers = resolvers + +local allocate = utilities.storage.allocate +local setmetatableindex = table.setmetatableindex +local fileextname = file.extname + +local formats = allocate() +local suffixes = allocate() +local dangerous = allocate() +local suffixmap = allocate() + +resolvers.formats = formats +resolvers.suffixes = suffixes +resolvers.dangerous = dangerous +resolvers.suffixmap = suffixmap + +local relations = allocate { -- todo: handlers also here + core = { + ofm = { -- will become obsolete + names = { "ofm", "omega font metric", "omega font metrics" }, + variable = 'OFMFONTS', + suffixes = { 'ofm', 'tfm' }, + }, + ovf = { -- will become obsolete + names = { "ovf", "omega virtual font", "omega virtual fonts" }, + variable = 'OVFFONTS', + suffixes = { 'ovf', 'vf' }, + }, + tfm = { + names = { "tfm", "tex font metric", "tex font metrics" }, + variable = 'TFMFONTS', + suffixes = { 'tfm' }, + }, + vf = { + names = { "vf", "virtual font", "virtual fonts" }, + variable = 'VFFONTS', + suffixes = { 'vf' }, + }, + otf = { + names = { "otf", "opentype", "opentype font", "opentype fonts"}, + variable = 'OPENTYPEFONTS', + suffixes = { 'otf' }, + }, + ttf = { + names = { "ttf", "truetype", "truetype font", "truetype fonts", "truetype collection", "truetype collections", "truetype dictionary", "truetype dictionaries" }, + variable = 'TTFONTS', + suffixes = { 'ttf', 'ttc', 'dfont' }, + }, + afm = { + names = { "afm", "adobe font metric", "adobe font metrics" }, + variable = "AFMFONTS", + suffixes = { "afm" }, + }, + pfb = { + names = { "pfb", "type1", "type 1", "type1 font", "type 1 font", "type1 fonts", "type 1 fonts" }, + variable = 'T1FONTS', + suffixes = { 'pfb', 'pfa' }, + }, + fea = { + names = { "fea", "font feature", "font features", "font feature file", "font feature files" }, + variable = 'FONTFEATURES', + suffixes = { 'fea' }, + }, + cid = { + names = { "cid", "cid map", "cid maps", "cid file", "cid files" }, + variable = 'FONTCIDMAPS', + suffixes = { 'cid', 'cidmap' }, + }, + fmt = { + names = { "fmt", "format", "tex format" }, + variable = 'TEXFORMATS', + suffixes = { 'fmt' }, + }, + mem = { -- will become obsolete + names = { 'mem', "metapost format" }, + variable = 'MPMEMS', + suffixes = { 'mem' }, + }, + mp = { + names = { "mp" }, + variable = 'MPINPUTS', + suffixes = { 'mp' }, + }, + tex = { + names = { "tex" }, + variable = 'TEXINPUTS', + suffixes = { 'tex', "mkiv", "mkiv", "mkii" }, + }, + icc = { + names = { "icc", "icc profile", "icc profiles" }, + variable = 'ICCPROFILES', + suffixes = { 'icc' }, + }, + texmfscripts = { + names = { "texmfscript", "texmfscripts", "script", "scripts" }, + variable = 'TEXMFSCRIPTS', + suffixes = { 'rb', 'pl', 'py' }, + }, + lua = { + names = { "lua" }, + variable = 'LUAINPUTS', + suffixes = { 'lua', 'luc', 'tma', 'tmc' }, + }, + lib = { + names = { "lib" }, + variable = 'CLUAINPUTS', + suffixes = os.libsuffix and { os.libsuffix } or { 'dll', 'so' }, + }, + bib = { + names = { 'bib' }, + suffixes = { 'bib' }, + }, + bst = { + names = { 'bst' }, + suffixes = { 'bst' }, + }, + fontconfig = { + names = { 'fontconfig', 'fontconfig file', 'fontconfig files' }, + variable = 'FONTCONFIG_PATH', + }, + }, + obsolete = { + enc = { + names = { "enc", "enc files", "enc file", "encoding files", "encoding file" }, + variable = 'ENCFONTS', + suffixes = { 'enc' }, + }, + map = { + names = { "map", "map files", "map file" }, + variable = 'TEXFONTMAPS', + suffixes = { 'map' }, + }, + lig = { + names = { "lig files", "lig file", "ligature file", "ligature files" }, + variable = 'LIGFONTS', + suffixes = { 'lig' }, + }, + opl = { + names = { "opl" }, + variable = 'OPLFONTS', + suffixes = { 'opl' }, + }, + ovp = { + names = { "ovp" }, + variable = 'OVPFONTS', + suffixes = { 'ovp' }, + }, + }, + kpse = { -- subset + base = { + names = { 'base', "metafont format" }, + variable = 'MFBASES', + suffixes = { 'base', 'bas' }, + }, + cmap = { + names = { 'cmap', 'cmap files', 'cmap file' }, + variable = 'CMAPFONTS', + suffixes = { 'cmap' }, + }, + cnf = { + names = { 'cnf' }, + suffixes = { 'cnf' }, + }, + web = { + names = { 'web' }, + suffixes = { 'web', 'ch' } + }, + cweb = { + names = { 'cweb' }, + suffixes = { 'w', 'web', 'ch' }, + }, + gf = { + names = { 'gf' }, + suffixes = { 'This is a prelude to a more extensive logging module. For the sake
-of parsing log files, in addition to the standard logging we will
-provide an
This module deals with caching data. It sets up the paths and +implements loaders and savers for tables. Best is to set the +following variable. When not set, the usual paths will be +checked. Personally I prefer the (users) temporary path.
-logs = logs or { } -logs.xml = logs.xml or { } -logs.tex = logs.tex or { } + +TEXMFCACHE=$TMP;$TEMP;$TMPDIR;$TEMPDIR;$HOME;$TEXMFVAR;$VARTEXMF;. + ---[[ldx-- -This looks pretty ugly but we need to speed things up a bit.
+Currently we do no locking when we write files. This is no real +problem because most caching involves fonts and the chance of them +being written at the same time is small. We also need to extend +luatools with a recache feature.
--ldx]]-- -logs.moreinfo = [[ -more information about ConTeXt and the tools that come with it can be found at: +local format, lower, gsub, concat = string.format, string.lower, string.gsub, table.concat +local mkdirs, isdir = dir.mkdirs, lfs.isdir -maillist : ntg-context@ntg.nl / http://www.ntg.nl/mailman/listinfo/ntg-context -webpage : http://www.pragma-ade.nl / http://tex.aanhet.net -wiki : http://contextgarden.net -]] +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) +local trace_cache = false trackers.register("resolvers.cache", function(v) trace_cache = v end) -logs.levels = { - ['error'] = 1, - ['warning'] = 2, - ['info'] = 3, - ['debug'] = 4, -} +local report_caches = logs.reporter("resolvers","caches") +local report_resolvers = logs.reporter("resolvers","caching") -logs.functions = { - 'report', 'start', 'stop', 'push', 'pop', 'line', 'direct', - 'start_run', 'stop_run', - 'start_page_number', 'stop_page_number', - 'report_output_pages', 'report_output_log', - 'report_tex_stat', 'report_job_stat', - 'show_open', 'show_close', 'show_load', -} +local resolvers = resolvers -logs.tracers = { -} +-- intermezzo -logs.level = 0 -logs.mode = string.lower((os.getenv("MTX.LOG.MODE") or os.getenv("MTX_LOG_MODE") or "tex")) +local directive_cleanup = false directives.register("system.compile.cleanup", function(v) directive_cleanup = v end) +local directive_strip = true directives.register("system.compile.strip", function(v) directive_strip = v end) -function logs.set_level(level) - logs.level = logs.levels[level] or level -end +local compile = utilities.lua.compile -function logs.set_method(method) - for _, v in next, logs.functions do - logs[v] = logs[method][v] or function() end - end +function utilities.lua.compile(luafile,lucfile,cleanup,strip) + if cleanup == nil then cleanup = directive_cleanup end + if strip == nil then strip = directive_strip end + return compile(luafile,lucfile,cleanup,strip) end --- tex logging +-- end of intermezzo -function logs.tex.report(category,fmt,...) -- new - if fmt then - write_nl(category .. " | " .. format(fmt,...)) - else - write_nl(category .. " |") - end -end +caches = caches or { } +local caches = caches -function logs.tex.line(fmt,...) -- new - if fmt then - write_nl(format(fmt,...)) +caches.base = caches.base or "luatex-cache" +caches.more = caches.more or "context" +caches.direct = false -- true is faster but may need huge amounts of memory +caches.tree = false +caches.force = true +caches.ask = false +caches.relocate = false +caches.defaults = { "TMPDIR", "TEMPDIR", "TMP", "TEMP", "HOME", "HOMEPATH" } + +local writable, readables, usedreadables = nil, { }, { } + +-- we could use a metatable for writable and readable but not yet + +local function identify() + -- Combining the loops makes it messy. First we check the format cache path + -- and when the last component is not present we try to create it. + local texmfcaches = resolvers.cleanpathlist("TEXMFCACHE") + if texmfcaches then + for k=1,#texmfcaches do + local cachepath = texmfcaches[k] + if cachepath ~= "" then + cachepath = resolvers.resolve(cachepath) + cachepath = resolvers.cleanpath(cachepath) + cachepath = file.collapsepath(cachepath) + local valid = isdir(cachepath) + if valid then + if file.is_readable(cachepath) then + readables[#readables+1] = cachepath + if not writable and file.is_writable(cachepath) then + writable = cachepath + end + end + elseif not writable and caches.force then + local cacheparent = file.dirname(cachepath) + if file.is_writable(cacheparent) then + if not caches.ask or io.ask(format("\nShould I create the cache path %s?",cachepath), "no", { "yes", "no" }) == "yes" then + mkdirs(cachepath) + if isdir(cachepath) and file.is_writable(cachepath) then + report_caches("created: %s",cachepath) + writable = cachepath + readables[#readables+1] = cachepath + end + end + end + end + end + end + end + -- As a last resort we check some temporary paths but this time we don't + -- create them. + local texmfcaches = caches.defaults + if texmfcaches then + for k=1,#texmfcaches do + local cachepath = texmfcaches[k] + cachepath = resolvers.expansion(cachepath) -- was getenv + if cachepath ~= "" then + cachepath = resolvers.resolve(cachepath) + cachepath = resolvers.cleanpath(cachepath) + local valid = isdir(cachepath) + if valid and file.is_readable(cachepath) then + if not writable and file.is_writable(cachepath) then + readables[#readables+1] = cachepath + writable = cachepath + break + end + end + end + end + end + -- Some extra checking. If we have no writable or readable path then we simply + -- quit. + if not writable then + report_caches("fatal error: there is no valid writable cache path defined") + os.exit() + elseif #readables == 0 then + report_caches("fatal error: there is no valid readable cache path defined") + os.exit() + end + -- why here + writable = dir.expandname(resolvers.cleanpath(writable)) -- just in case + -- moved here + local base, more, tree = caches.base, caches.more, caches.tree or caches.treehash() -- we have only one writable tree + if tree then + caches.tree = tree + writable = mkdirs(writable,base,more,tree) + for i=1,#readables do + readables[i] = file.join(readables[i],base,more,tree) + end else - write_nl("") + writable = mkdirs(writable,base,more) + for i=1,#readables do + readables[i] = file.join(readables[i],base,more) + end end + -- end + if trace_cache then + for i=1,#readables do + report_caches("using readable path '%s' (order %s)",readables[i],i) + end + report_caches("using writable path '%s'",writable) + end + identify = function() + return writable, readables + end + return writable, readables end ---~ function logs.tex.start_page_number() ---~ local real, user, sub = texcount.realpageno, texcount.userpageno, texcount.subpageno ---~ if real > 0 then ---~ if user > 0 then ---~ if sub > 0 then ---~ write(format("[%s.%s.%s",real,user,sub)) ---~ else ---~ write(format("[%s.%s",real,user)) ---~ end ---~ else ---~ write(format("[%s",real)) ---~ end ---~ else ---~ write("[-") ---~ end ---~ end - ---~ function logs.tex.stop_page_number() ---~ write("]") ---~ end - -local real, user, sub - -function logs.tex.start_page_number() - real, user, sub = texcount.realpageno, texcount.userpageno, texcount.subpageno -end - -function logs.tex.stop_page_number() - if real > 0 then - if user > 0 then - if sub > 0 then - logs.report("pages", "flushing realpage %s, userpage %s, subpage %s",real,user,sub) - else - logs.report("pages", "flushing realpage %s, userpage %s",real,user) +function caches.usedpaths() + local writable, readables = identify() + if #readables > 1 then + local result = { } + for i=1,#readables do + local readable = readables[i] + if usedreadables[i] or readable == writable then + result[#result+1] = format("readable: '%s' (order %s)",readable,i) end - else - logs.report("pages", "flushing realpage %s",real) end + result[#result+1] = format("writable: '%s'",writable) + return result else - logs.report("pages", "flushing page") + return writable end - io.flush() end -logs.tex.report_job_stat = statistics.show_job_stat - --- xml logging +function caches.configfiles() + return table.concat(resolvers.instance.specification,";") +end -function logs.xml.report(category,fmt,...) -- new - if fmt then - write_nl(format(" 0 then
+ done = mkdirs(writable,...)
+ else
+ done = writable
+ end
+ w_cache[hash] = done
+ end
+ return done
end
-function logs.xml.stop_page_number()
- write("/>")
- write_nl("")
-end
+caches.getreadablepaths = getreadablepaths
+caches.getwritablepath = getwritablepath
-function logs.xml.report_output_pages(p,b)
- write_nl(format("
This module deals with caching data. It sets up the paths and -implements loaders and savers for tables. Best is to set the -following variable. When not set, the usual paths will be -checked. Personally I prefer the (users) temporary path.
- - -TEXMFCACHE=$TMP;$TEMP;$TMPDIR;$TEMPDIR;$HOME;$TEXMFVAR;$VARTEXMF;. - - -Currently we do no locking when we write files. This is no real -problem because most caching involves fonts and the chance of them -being written at the same time is small. We also need to extend -luatools with a recache feature.
---ldx]]-- - -local format, lower, gsub = string.format, string.lower, string.gsub - -local trace_cache = false trackers.register("resolvers.cache", function(v) trace_cache = v end) -- not used yet - -caches = caches or { } - -caches.path = caches.path or nil -caches.base = caches.base or "luatex-cache" -caches.more = caches.more or "context" -caches.direct = false -- true is faster but may need huge amounts of memory -caches.tree = false -caches.paths = caches.paths or nil -caches.force = true -- changed for TEXLIVE by TH -caches.defaults = { "TEXMFCACHE", "TMPDIR", "TEMPDIR", "TMP", "TEMP", "HOME", "HOMEPATH" } - -function caches.temp() - local cachepath = nil - local function check(list,isenv) - if not cachepath then - for k=1,#list do - local v = list[k] - cachepath = (isenv and (os.env[v] or "")) or v or "" - if cachepath == "" then - -- next - else - cachepath = resolvers.clean_path(cachepath) - if lfs.isdir(cachepath) and file.iswritable(cachepath) then -- lfs.attributes(cachepath,"mode") == "directory" - break - elseif caches.force or io.ask(format("\nShould I create the cache path %s?",cachepath), "no", { "yes", "no" }) == "yes" then - dir.mkdirs(cachepath) - if lfs.isdir(cachepath) and file.iswritable(cachepath) then - break +function resolvers.dowithfilesintree(pattern,handle,before,after) -- will move, can be a nice iterator instead + local instance = resolvers.instance + local hashes = instance.hashes + for i=1,#hashes do + local hash = hashes[i] + local blobtype = hash.type + local blobpath = hash.name + if blobpath then + if before then + before(blobtype,blobpath,pattern) + end + local files = instance.files[blobpath] + local total, checked, done = 0, 0, 0 + if files then + for k,v in next, files do + total = total + 1 + if find(k,"^remap:") then + k = files[k] + v = k -- files[k] -- chained + end + if find(k,pattern) then + if type(v) == "string" then + checked = checked + 1 + if handle(blobtype,blobpath,v,k) then + done = done + 1 + end + else + checked = checked + #v + for i=1,#v do + if handle(blobtype,blobpath,v[i],k) then + done = done + 1 + end + end end end end - cachepath = nil + end + if after then + after(blobtype,blobpath,pattern,total,checked,done) end end end - check(resolvers.clean_path_list("TEXMFCACHE") or { }) - check(caches.defaults,true) - if not cachepath then - print("\nfatal error: there is no valid (writable) cache path defined\n") - os.exit() - elseif not lfs.isdir(cachepath) then -- lfs.attributes(cachepath,"mode") ~= "directory" - print(format("\nfatal error: cache path %s is not a directory\n",cachepath)) - os.exit() - end - cachepath = file.collapse_path(cachepath) - function caches.temp() - return cachepath - end - return cachepath -end - -function caches.configpath() - return table.concat(resolvers.instance.cnffiles,";") -end - -function caches.hashed(tree) - return md5.hex(gsub(lower(tree),"[\\\/]+","/")) -end - -function caches.treehash() - local tree = caches.configpath() - if not tree or tree == "" then - return false - else - return caches.hashed(tree) - end -end - -function caches.setpath(...) - if not caches.path then - if not caches.path then - caches.path = caches.temp() - end - caches.path = resolvers.clean_path(caches.path) -- to be sure - caches.tree = caches.tree or caches.treehash() - if caches.tree then - caches.path = dir.mkdirs(caches.path,caches.base,caches.more,caches.tree) - else - caches.path = dir.mkdirs(caches.path,caches.base,caches.more) - end - end - if not caches.path then - caches.path = '.' - end - caches.path = resolvers.clean_path(caches.path) - local dirs = { ... } - if #dirs > 0 then - local pth = dir.mkdirs(caches.path,...) - return pth - end - caches.path = dir.expand_name(caches.path) - return caches.path -end - -function caches.definepath(category,subcategory) - return function() - return caches.setpath(category,subcategory) - end -end - -function caches.setluanames(path,name) - return path .. "/" .. name .. ".tma", path .. "/" .. name .. ".tmc" -end - -function caches.loaddata(path,name) - local tmaname, tmcname = caches.setluanames(path,name) - local loader = loadfile(tmcname) or loadfile(tmaname) - if loader then - loader = loader() - collectgarbage("step") - return loader - else - return false - end -end - ---~ function caches.loaddata(path,name) ---~ local tmaname, tmcname = caches.setluanames(path,name) ---~ return dofile(tmcname) or dofile(tmaname) ---~ end - -function caches.iswritable(filepath,filename) - local tmaname, tmcname = caches.setluanames(filepath,filename) - return file.iswritable(tmaname) -end - -function caches.savedata(filepath,filename,data,raw) - local tmaname, tmcname = caches.setluanames(filepath,filename) - local reduce, simplify = true, true - if raw then - reduce, simplify = false, false - end - data.cache_uuid = os.uuid() - if caches.direct then - file.savedata(tmaname, table.serialize(data,'return',false,true,false)) -- no hex - else - table.tofile(tmaname, data,'return',false,true,false) -- maybe not the last true - end - local cleanup = resolvers.boolean_variable("PURGECACHE", false) - local strip = resolvers.boolean_variable("LUACSTRIP", true) - utils.lua.compile(tmaname, tmcname, cleanup, strip) end --- here we use the cache for format loading (texconfig.[formatname|jobname]) +resolvers.obsolete = resolvers.obsolete or { } +local obsolete = resolvers.obsolete ---~ if tex and texconfig and texconfig.formatname and texconfig.formatname == "" then -if tex and texconfig and (not texconfig.formatname or texconfig.formatname == "") and input and resolvers.instance then - if not texconfig.luaname then texconfig.luaname = "cont-en.lua" end -- or luc - texconfig.formatname = caches.setpath("formats") .. "/" .. gsub(texconfig.luaname,"%.lu.$",".fmt") -end +resolvers.find_file = resolvers.findfile obsolete.find_file = resolvers.findfile +resolvers.find_files = resolvers.findfiles obsolete.find_files = resolvers.findfiles end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-res'] = { +if not modules then modules = { } end modules ['data-pre'] = { version = 1.001, comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", @@ -10555,17 +12488,28 @@ if not modules then modules = { } end modules ['data-res'] = { license = "see context related readme files" } ---~ print(resolvers.resolve("abc env:tmp file:cont-en.tex path:cont-en.tex full:cont-en.tex rel:zapf/one/p-chars.tex")) +-- It could be interesting to hook the resolver in the file +-- opener so that unresolved prefixes travel around and we +-- get more abstraction. -local upper, lower, gsub = string.upper, string.lower, string.gsub +-- As we use this beforehand we will move this up in the chain +-- of loading. -local prefixes = { } + +local resolvers = resolvers +local prefixes = utilities.storage.allocate() +resolvers.prefixes = prefixes + +local gsub = string.gsub +local cleanpath, findgivenfile, expansion = resolvers.cleanpath, resolvers.findgivenfile, resolvers.expansion +local getenv = resolvers.getenv -- we can probably also use resolvers.expansion +local P, Cs, lpegmatch = lpeg.P, lpeg.Cs, lpeg.match prefixes.environment = function(str) - return resolvers.clean_path(os.getenv(str) or os.getenv(upper(str)) or os.getenv(lower(str)) or "") + return cleanpath(expansion(str)) end -prefixes.relative = function(str,n) +prefixes.relative = function(str,n) -- lfs.isfile if io.exists(str) then -- nothing elseif io.exists("./" .. str) then @@ -10581,7 +12525,7 @@ prefixes.relative = function(str,n) end end end - return resolvers.clean_path(str) + return cleanpath(str) end prefixes.auto = function(str) @@ -10593,18 +12537,34 @@ prefixes.auto = function(str) end prefixes.locate = function(str) - local fullname = resolvers.find_given_file(str) or "" - return resolvers.clean_path((fullname ~= "" and fullname) or str) + local fullname = findgivenfile(str) or "" + return cleanpath((fullname ~= "" and fullname) or str) end prefixes.filename = function(str) - local fullname = resolvers.find_given_file(str) or "" - return resolvers.clean_path(file.basename((fullname ~= "" and fullname) or str)) + local fullname = findgivenfile(str) or "" + return cleanpath(file.basename((fullname ~= "" and fullname) or str)) end prefixes.pathname = function(str) - local fullname = resolvers.find_given_file(str) or "" - return resolvers.clean_path(file.dirname((fullname ~= "" and fullname) or str)) + local fullname = findgivenfile(str) or "" + return cleanpath(file.dirname((fullname ~= "" and fullname) or str)) +end + +prefixes.selfautoloc = function(str) + return cleanpath(file.join(getenv('SELFAUTOLOC'),str)) +end + +prefixes.selfautoparent = function(str) + return cleanpath(file.join(getenv('SELFAUTOPARENT'),str)) +end + +prefixes.selfautodir = function(str) + return cleanpath(file.join(getenv('SELFAUTODIR'),str)) +end + +prefixes.home = function(str) + return cleanpath(file.join(getenv('HOME'),str)) end prefixes.env = prefixes.environment @@ -10633,26 +12593,71 @@ local function _resolve_(method,target) end end -local function resolve(str) - if type(str) == "table" then - for k=1,#str do - local v = str[k] - str[k] = resolve(v) or v +local resolved, abstract = { }, { } + +function resolvers.resetresolve(str) + resolved, abstract = { }, { } +end + +local function resolve(str) -- use schemes, this one is then for the commandline only + local res = resolved[str] + if not res then + res = gsub(str,"([a-z][a-z]+):([^ \"\']*)",_resolve_) + resolved[str] = res + abstract[res] = str + end + return res +end + +local function unresolve(str) + return abstract[str] or str +end + +resolvers.resolve = resolve +resolvers.unresolve = unresolve + +if os.uname then + + for k, v in next, os.uname() do + if not prefixes[k] then + prefixes[k] = function() return v end + end + end + +end + +if os.type == "unix" then + + local pattern + + local function makepattern(t,k,v) + local colon = P(":") + local p + for k, v in table.sortedpairs(prefixes) do + if p then + p = P(k) + p + else + p = P(k) + end + end + pattern = Cs((p * colon + colon/";" + P(1))^0) + if t then + t[k] = v end - elseif str and str ~= "" then - str = gsub(str,"([a-z]+):([^ \"\']*)",_resolve_) end - return str -end -resolvers.resolve = resolve + makepattern() -if os.uname then + getmetatable(prefixes).__newindex = makepattern - for k, v in next, os.uname() do - if not prefixes[k] then - prefixes[k] = function() return v end - end + function resolvers.repath(str) + return lpegmatch(pattern,str) + end + +else -- already the default: + + function resolvers.repath(str) + return str end end @@ -10670,13 +12675,23 @@ if not modules then modules = { } end modules ['data-inp'] = { license = "see context related readme files" } -resolvers.finders = resolvers.finders or { } -resolvers.openers = resolvers.openers or { } -resolvers.loaders = resolvers.loaders or { } +local allocate = utilities.storage.allocate +local resolvers = resolvers + +local methodhandler = resolvers.methodhandler +local registermethod = resolvers.registermethod -resolvers.finders.notfound = { nil } -resolvers.openers.notfound = { nil } -resolvers.loaders.notfound = { false, nil, 0 } +local finders = allocate { helpers = { }, notfound = function() end } +local openers = allocate { helpers = { }, notfound = function() end } +local loaders = allocate { helpers = { }, notfound = function() return false, nil, 0 end } + +registermethod("finders", finders, "uri") +registermethod("openers", openers, "uri") +registermethod("loaders", loaders, "uri") + +resolvers.finders = finders +resolvers.openers = openers +resolvers.loaders = loaders end -- of closure @@ -10691,15 +12706,23 @@ if not modules then modules = { } end modules ['data-out'] = { license = "see context related readme files" } -outputs = outputs or { } +local allocate = utilities.storage.allocate +local resolvers = resolvers + +local registermethod = resolvers.registermethod +local savers = allocate { helpers = { } } + +resolvers.savers = savers + +registermethod("savers", savers, "uri") end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['data-con'] = { +if not modules then modules = { } end modules ['data-fil'] = { version = 1.001, comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", @@ -10707,6 +12730,124 @@ if not modules then modules = { } end modules ['data-con'] = { license = "see context related readme files" } +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) + +local report_files = logs.reporter("resolvers","files") + +local resolvers = resolvers + +local finders, openers, loaders, savers = resolvers.finders, resolvers.openers, resolvers.loaders, resolvers.savers +local locators, hashers, generators, concatinators = resolvers.locators, resolvers.hashers, resolvers.generators, resolvers.concatinators + +local checkgarbage = utilities.garbagecollector and utilities.garbagecollector.check + +function locators.file(specification) + local name = specification.filename + local realname = resolvers.resolve(name) -- no shortcut + if realname and realname ~= '' and lfs.isdir(realname) then + if trace_locating then + report_files("file locator '%s' found as '%s'",name,realname) + end + resolvers.appendhash('file',name,true) -- cache + elseif trace_locating then + report_files("file locator '%s' not found",name) + end +end + +function hashers.file(specification) + local name = specification.filename + local content = caches.loadcontent(name,'files') + resolvers.registerfilehash(name,content,content==nil) +end + +function generators.file(specification) + local path = specification.filename + local content = resolvers.scanfiles(path) + resolvers.registerfilehash(path,content,true) +end + +concatinators.file = file.join + +function finders.file(specification,filetype) + local filename = specification.filename + local foundname = resolvers.findfile(filename,filetype) + if foundname and foundname ~= "" then + if trace_locating then + report_files("file finder: '%s' found",filename) + end + return foundname + else + if trace_locating then + report_files("file finder: %s' not found",filename) + end + return finders.notfound() + end +end + +-- The default textopener will be overloaded later on. + +function openers.helpers.textopener(tag,filename,f) + return { + reader = function() return f:read () end, + close = function() logs.show_close(filename) return f:close() end, + } +end + +function openers.file(specification,filetype) + local filename = specification.filename + if filename and filename ~= "" then + local f = io.open(filename,"r") + if f then + if trace_locating then + report_files("file opener, '%s' opened",filename) + end + return openers.helpers.textopener("file",filename,f) + end + end + if trace_locating then + report_files("file opener, '%s' not found",filename) + end + return openers.notfound() +end + +function loaders.file(specification,filetype) + local filename = specification.filename + if filename and filename ~= "" then + local f = io.open(filename,"rb") + if f then + logs.show_load(filename) + if trace_locating then + report_files("file loader, '%s' loaded",filename) + end + local s = f:read("*a") + if checkgarbage then + checkgarbage(#s) + end + f:close() + if s then + return true, s, #s + end + end + end + if trace_locating then + report_files("file loader, '%s' not found",filename) + end + return loaders.notfound() +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['data-con'] = { + version = 1.100, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + local format, lower, gsub = string.format, string.lower, string.gsub local trace_cache = false trackers.register("resolvers.cache", function(v) trace_cache = v end) @@ -10726,50 +12867,63 @@ table structures without bothering about the disk cache.Examples of usage can be found in the font related code.
--ldx]]-- -containers = containers or { } - +containers = containers or { } +local containers = containers containers.usecache = true +local report_containers = logs.reporter("resolvers","containers") + local function report(container,tag,name) if trace_cache or trace_containers then - logs.report(format("%s cache",container.subcategory),"%s: %s",tag,name or 'invalid') + report_containers("container: %s, tag: %s, name: %s",container.subcategory,tag,name or 'invalid') end end local allocated = { } --- tracing +local mt = { + __index = function(t,k) + if k == "writable" then + local writable = caches.getwritablepath(t.category,t.subcategory) or { "." } + t.writable = writable + return writable + elseif k == "readables" then + local readables = caches.getreadablepaths(t.category,t.subcategory) or { "." } + t.readables = readables + return readables + end + end, + __storage__ = true +} function containers.define(category, subcategory, version, enabled) - return function() - if category and subcategory then - local c = allocated[category] - if not c then - c = { } - allocated[category] = c - end - local s = c[subcategory] - if not s then - s = { - category = category, - subcategory = subcategory, - storage = { }, - enabled = enabled, - version = version or 1.000, - trace = false, - path = caches and caches.setpath and caches.setpath(category,subcategory), - } - c[subcategory] = s - end - return s - else - return nil + if category and subcategory then + local c = allocated[category] + if not c then + c = { } + allocated[category] = c + end + local s = c[subcategory] + if not s then + s = { + category = category, + subcategory = subcategory, + storage = { }, + enabled = enabled, + version = version or math.pi, -- after all, this is TeX + trace = false, + -- writable = caches.getwritablepath and caches.getwritablepath (category,subcategory) or { "." }, + -- readables = caches.getreadablepaths and caches.getreadablepaths(category,subcategory) or { "." }, + } + setmetatable(s,mt) + c[subcategory] = s end + return s end end function containers.is_usable(container, name) - return container.enabled and caches and caches.iswritable(container.path, name) + return container.enabled and caches and caches.is_writable(container.writable, name) end function containers.is_valid(container, name) @@ -10782,18 +12936,20 @@ function containers.is_valid(container, name) end function containers.read(container,name) - if container.enabled and caches and not container.storage[name] and containers.usecache then - container.storage[name] = caches.loaddata(container.path,name) - if containers.is_valid(container,name) then + local storage = container.storage + local stored = storage[name] + if not stored and container.enabled and caches and containers.usecache then + stored = caches.loaddata(container.readables,name) + if stored and stored.cache_version == container.version then report(container,"loaded",name) else - container.storage[name] = nil + stored = nil end - end - if container.storage[name] then + storage[name] = stored + elseif stored then report(container,"reusing",name) end - return container.storage[name] + return stored end function containers.write(container, name, data) @@ -10802,7 +12958,7 @@ function containers.write(container, name, data) if container.enabled and caches then local unique, shared = data.unique, data.shared data.unique, data.shared = nil, nil - caches.savedata(container.path, name, data) + caches.savedata(container.writable, name, data) report(container,"saved",name) data.unique, data.shared = unique, shared end @@ -10837,50 +12993,18 @@ local format, lower, gsub, find = string.format, string.lower, string.gsub, stri local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) --- since we want to use the cache instead of the tree, we will now --- reimplement the saver. - -local save_data = resolvers.save_data -local load_data = resolvers.load_data - -resolvers.cachepath = nil -- public, for tracing -resolvers.usecache = true -- public, for tracing +local report_mounts = logs.reporter("resolvers","mounts") -function resolvers.save_data(dataname) - save_data(dataname, function(cachename,dataname) - resolvers.usecache = not toboolean(resolvers.expansion("CACHEINTDS") or "false",true) - if resolvers.usecache then - resolvers.cachepath = resolvers.cachepath or caches.definepath("trees") - return file.join(resolvers.cachepath(),caches.hashed(cachename)) - else - return file.join(cachename,dataname) - end - end) -end - -function resolvers.load_data(pathname,dataname,filename) - load_data(pathname,dataname,filename,function(dataname,filename) - resolvers.usecache = not toboolean(resolvers.expansion("CACHEINTDS") or "false",true) - if resolvers.usecache then - resolvers.cachepath = resolvers.cachepath or caches.definepath("trees") - return file.join(resolvers.cachepath(),caches.hashed(pathname)) - else - if not filename or (filename == "") then - filename = dataname - end - return file.join(pathname,filename) - end - end) -end +local resolvers = resolvers -- we will make a better format, maybe something xml or just text or lua resolvers.automounted = resolvers.automounted or { } function resolvers.automount(usecache) - local mountpaths = resolvers.clean_path_list(resolvers.expansion('TEXMFMOUNT')) + local mountpaths = resolvers.cleanpathlist(resolvers.expansion('TEXMFMOUNT')) if (not mountpaths or #mountpaths == 0) and usecache then - mountpaths = { caches.setpath("mount") } + mountpaths = caches.getreadablepaths("mount") end if mountpaths and #mountpaths > 0 then statistics.starttiming(resolvers.instance) @@ -10894,7 +13018,7 @@ function resolvers.automount(usecache) -- skip elseif find(line,"^zip://") then if trace_locating then - logs.report("fileio","mounting %s",line) + report_mounts("mounting %s",line) end table.insert(resolvers.automounted,line) resolvers.usezipfile(line) @@ -10910,40 +13034,40 @@ end -- status info -statistics.register("used config path", function() return caches.configpath() end) -statistics.register("used cache path", function() return caches.temp() or "?" end) +statistics.register("used config file", function() return caches.configfiles() end) +statistics.register("used cache path", function() return caches.usedpaths() end) -- experiment (code will move) -function statistics.save_fmt_status(texname,formatbanner,sourcefile) -- texname == formatname +function statistics.savefmtstatus(texname,formatbanner,sourcefile) -- texname == formatname local enginebanner = status.list().banner if formatbanner and enginebanner and sourcefile then local luvname = file.replacesuffix(texname,"luv") local luvdata = { enginebanner = enginebanner, formatbanner = formatbanner, - sourcehash = md5.hex(io.loaddata(resolvers.find_file(sourcefile)) or "unknown"), + sourcehash = md5.hex(io.loaddata(resolvers.findfile(sourcefile)) or "unknown"), sourcefile = sourcefile, } io.savedata(luvname,table.serialize(luvdata,true)) end end -function statistics.check_fmt_status(texname) +function statistics.checkfmtstatus(texname) local enginebanner = status.list().banner if enginebanner and texname then local luvname = file.replacesuffix(texname,"luv") if lfs.isfile(luvname) then local luv = dofile(luvname) if luv and luv.sourcefile then - local sourcehash = md5.hex(io.loaddata(resolvers.find_file(luv.sourcefile)) or "unknown") + local sourcehash = md5.hex(io.loaddata(resolvers.findfile(luv.sourcefile)) or "unknown") local luvbanner = luv.enginebanner or "?" if luvbanner ~= enginebanner then - return string.format("engine mismatch (luv:%s <> bin:%s)",luvbanner,enginebanner) + return format("engine mismatch (luv: %s <> bin: %s)",luvbanner,enginebanner) end local luvhash = luv.sourcehash or "?" if luvhash ~= sourcehash then - return string.format("source mismatch (luv:%s <> bin:%s)",luvhash,sourcehash) + return format("source mismatch (luv: %s <> bin: %s)",luvhash,sourcehash) end else return "invalid status file" @@ -10968,25 +13092,30 @@ if not modules then modules = { } end modules ['data-zip'] = { license = "see context related readme files" } +-- partly redone .. needs testing + local format, find, match = string.format, string.find, string.match -local unpack = unpack or table.unpack local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) +local report_zip = logs.reporter("resolvers","zip") + -- zip:///oeps.zip?name=bla/bla.tex -- zip:///oeps.zip?tree=tex/texmf-local -- zip:///texmf.zip?tree=/tex/texmf -- zip:///texmf.zip?tree=/tex/texmf-local -- zip:///texmf-mine.zip?tree=/tex/texmf-projects -zip = zip or { } -zip.archives = zip.archives or { } -zip.registeredfiles = zip.registeredfiles or { } +local resolvers = resolvers -local finders, openers, loaders = resolvers.finders, resolvers.openers, resolvers.loaders -local locators, hashers, concatinators = resolvers.locators, resolvers.hashers, resolvers.concatinators +zip = zip or { } +local zip = zip -local archives = zip.archives +zip.archives = zip.archives or { } +local archives = zip.archives + +zip.registeredfiles = zip.registeredfiles or { } +local registeredfiles = zip.registeredfiles local function validzip(str) -- todo: use url splitter if not find(str,"^zip://") then @@ -11002,7 +13131,7 @@ function zip.openarchive(name) else local arch = archives[name] if not arch then - local full = resolvers.find_file(name) or "" + local full = resolvers.findfile(name) or "" arch = (full ~= "" and zip.open(full)) or false archives[name] = arch end @@ -11017,163 +13146,162 @@ function zip.closearchive(name) end end -function locators.zip(specification) -- where is this used? startup zips (untested) - specification = resolvers.splitmethod(specification) - local zipfile = specification.path - local zfile = zip.openarchive(name) -- tricky, could be in to be initialized tree +function resolvers.locators.zip(specification) + local archive = specification.filename + local zipfile = archive and archive ~= "" and zip.openarchive(archive) -- tricky, could be in to be initialized tree if trace_locating then - if zfile then - logs.report("fileio","zip locator, archive '%s' found",specification.original) + if zipfile then + report_zip("locator, archive '%s' found",archive) else - logs.report("fileio","zip locator, archive '%s' not found",specification.original) + report_zip("locator, archive '%s' not found",archive) end end end -function hashers.zip(tag,name) +function resolvers.hashers.zip(specification) + local archive = specification.filename if trace_locating then - logs.report("fileio","loading zip file '%s' as '%s'",name,tag) + report_zip("loading file '%s'",archive) end - resolvers.usezipfile(format("%s?tree=%s",tag,name)) + resolvers.usezipfile(specification.original) end -function concatinators.zip(tag,path,name) +function resolvers.concatinators.zip(zipfile,path,name) -- ok ? if not path or path == "" then - return format('%s?name=%s',tag,name) + return format('%s?name=%s',zipfile,name) else - return format('%s?name=%s/%s',tag,path,name) + return format('%s?name=%s/%s',zipfile,path,name) end end -function resolvers.isreadable.zip(name) - return true -end - -function finders.zip(specification,filetype) - specification = resolvers.splitmethod(specification) - if specification.path then - local q = url.query(specification.query) - if q.name then - local zfile = zip.openarchive(specification.path) +function resolvers.finders.zip(specification) + local original = specification.original + local archive = specification.filename + if archive then + local query = url.query(specification.query) + local queryname = query.name + if queryname then + local zfile = zip.openarchive(archive) if zfile then if trace_locating then - logs.report("fileio","zip finder, archive '%s' found",specification.path) + report_zip("finder, archive '%s' found",archive) end - local dfile = zfile:open(q.name) + local dfile = zfile:open(queryname) if dfile then dfile = zfile:close() if trace_locating then - logs.report("fileio","zip finder, file '%s' found",q.name) + report_zip("finder, file '%s' found",queryname) end return specification.original elseif trace_locating then - logs.report("fileio","zip finder, file '%s' not found",q.name) + report_zip("finder, file '%s' not found",queryname) end elseif trace_locating then - logs.report("fileio","zip finder, unknown archive '%s'",specification.path) + report_zip("finder, unknown archive '%s'",archive) end end end if trace_locating then - logs.report("fileio","zip finder, '%s' not found",filename) + report_zip("finder, '%s' not found",original) end - return unpack(finders.notfound) + return resolvers.finders.notfound() end -function openers.zip(specification) - local zipspecification = resolvers.splitmethod(specification) - if zipspecification.path then - local q = url.query(zipspecification.query) - if q.name then - local zfile = zip.openarchive(zipspecification.path) +function resolvers.openers.zip(specification) + local original = specification.original + local archive = specification.filename + if archive then + local query = url.query(specification.query) + local queryname = query.name + if queryname then + local zfile = zip.openarchive(archive) if zfile then if trace_locating then - logs.report("fileio","zip opener, archive '%s' opened",zipspecification.path) + report_zip("opener, archive '%s' opened",archive) end - local dfile = zfile:open(q.name) + local dfile = zfile:open(queryname) if dfile then - logs.show_open(specification) if trace_locating then - logs.report("fileio","zip opener, file '%s' found",q.name) + report_zip("opener, file '%s' found",queryname) end - return openers.text_opener(specification,dfile,'zip') + return resolvers.openers.helpers.textopener('zip',original,dfile) elseif trace_locating then - logs.report("fileio","zip opener, file '%s' not found",q.name) + report_zip("opener, file '%s' not found",queryname) end elseif trace_locating then - logs.report("fileio","zip opener, unknown archive '%s'",zipspecification.path) + report_zip("opener, unknown archive '%s'",archive) end end end if trace_locating then - logs.report("fileio","zip opener, '%s' not found",filename) + report_zip("opener, '%s' not found",original) end - return unpack(openers.notfound) + return resolvers.openers.notfound() end -function loaders.zip(specification) - specification = resolvers.splitmethod(specification) - if specification.path then - local q = url.query(specification.query) - if q.name then - local zfile = zip.openarchive(specification.path) +function resolvers.loaders.zip(specification) + local original = specification.original + local archive = specification.filename + if archive then + local query = url.query(specification.query) + local queryname = query.name + if queryname then + local zfile = zip.openarchive(archive) if zfile then if trace_locating then - logs.report("fileio","zip loader, archive '%s' opened",specification.path) + report_zip("loader, archive '%s' opened",archive) end - local dfile = zfile:open(q.name) + local dfile = zfile:open(queryname) if dfile then - logs.show_load(filename) + logs.show_load(original) if trace_locating then - logs.report("fileio","zip loader, file '%s' loaded",filename) + report_zip("loader, file '%s' loaded",original) end local s = dfile:read("*all") dfile:close() return true, s, #s elseif trace_locating then - logs.report("fileio","zip loader, file '%s' not found",q.name) + report_zip("loader, file '%s' not found",queryname) end elseif trace_locating then - logs.report("fileio","zip loader, unknown archive '%s'",specification.path) + report_zip("loader, unknown archive '%s'",archive) end end end if trace_locating then - logs.report("fileio","zip loader, '%s' not found",filename) + report_zip("loader, '%s' not found",original) end - return unpack(openers.notfound) + return resolvers.openers.notfound() end -- zip:///somefile.zip -- zip:///somefile.zip?tree=texmf-local -> mount -function resolvers.usezipfile(zipname) - zipname = validzip(zipname) - local specification = resolvers.splitmethod(zipname) - local zipfile = specification.path - if zipfile and not zip.registeredfiles[zipname] then - local tree = url.query(specification.query).tree or "" - local z = zip.openarchive(zipfile) +function resolvers.usezipfile(archive) + local specification = resolvers.splitmethod(archive) -- to be sure + local archive = specification.filename + if archive and not registeredfiles[archive] then + local z = zip.openarchive(archive) if z then - local instance = resolvers.instance + local tree = url.query(specification.query).tree or "" if trace_locating then - logs.report("fileio","zip registering, registering archive '%s'",zipname) - end - statistics.starttiming(instance) - resolvers.prepend_hash('zip',zipname,zipfile) - resolvers.extend_texmf_var(zipname) -- resets hashes too - zip.registeredfiles[zipname] = z - instance.files[zipname] = resolvers.register_zip_file(z,tree or "") - statistics.stoptiming(instance) + report_zip("registering, registering archive '%s'",archive) + end + statistics.starttiming(resolvers.instance) + resolvers.prependhash('zip',archive) + resolvers.extendtexmfvariable(archive) -- resets hashes too + registeredfiles[archive] = z + instance.files[archive] = resolvers.registerzipfile(z,tree) + statistics.stoptiming(resolvers.instance) elseif trace_locating then - logs.report("fileio","zip registering, unknown archive '%s'",zipname) + report_zip("registering, unknown archive '%s'",archive) end elseif trace_locating then - logs.report("fileio","zip registering, '%s' not found",zipname) + report_zip("registering, '%s' not found",archive) end end -function resolvers.register_zip_file(z,tree) +function resolvers.registerzipfile(z,tree) local files, filter = { }, "" if tree == "" then filter = "^(.+)/(.-)$" @@ -11181,9 +13309,9 @@ function resolvers.register_zip_file(z,tree) filter = format("^%s/(.+)/(.-)$",tree) end if trace_locating then - logs.report("fileio","zip registering, using filter '%s'",filter) + report_zip("registering, using filter '%s'",filter) end - local register, n = resolvers.register_file, 0 + local register, n = resolvers.registerfile, 0 for i in z:files() do local path, name = match(i.filename,filter) if path then @@ -11198,11 +13326,89 @@ function resolvers.register_zip_file(z,tree) n = n + 1 end end - logs.report("fileio","zip registering, %s files registered",n) + report_zip("registering, %s files registered",n) return files end +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['data-tre'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- \input tree://oeps1/**/oeps.tex + +local find, gsub, format = string.find, string.gsub, string.format + +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) + +local report_trees = logs.reporter("resolvers","trees") + +local resolvers = resolvers + +local done, found, notfound = { }, { }, resolvers.finders.notfound + +function resolvers.finders.tree(specification) + local spec = specification.filename + local fnd = found[spec] + if fnd == nil then + if spec ~= "" then + local path, name = file.dirname(spec), file.basename(spec) + if path == "" then path = "." end + local hash = done[path] + if not hash then + local pattern = path .. "/*" -- we will use the proper splitter + hash = dir.glob(pattern) + done[path] = hash + end + local pattern = "/" .. gsub(name,"([%.%-%+])", "%%%1") .. "$" + for k=1,#hash do + local v = hash[k] + if find(v,pattern) then + found[spec] = v + return v + end + end + end + fnd = notfound() -- false + found[spec] = fnd + end + return fnd +end + +function resolvers.locators.tree(specification) + local name = specification.filename + if name ~= '' and lfs.isdir(name) then + if trace_locating then + report_trees("locator '%s' found",name) + end + resolvers.appendhash('tree',name,false) -- don't cache + elseif trace_locating then + report_trees("locator '%s' not found",name) + end +end + +function resolvers.hashers.tree(specification) + local name = specification.filename + if trace_locating then + report_trees("analysing '%s'",name) + end + resolvers.methodhandler("hashers",name) +end + +resolvers.concatinators.tree = resolvers.concatinators.file +resolvers.generators.tree = resolvers.generators.file +resolvers.openers.tree = resolvers.openers.file +resolvers.loaders.tree = resolvers.loaders.file + + end -- of closure do -- create closure to overcome 200 locals limit @@ -11215,58 +13421,59 @@ if not modules then modules = { } end modules ['data-crl'] = { license = "see context related readme files" } -local gsub = string.gsub +-- this one is replaced by data-sch.lua -- -curl = curl or { } +local gsub = string.gsub -curl.cached = { } -curl.cachepath = caches.definepath("curl") +local resolvers = resolvers local finders, openers, loaders = resolvers.finders, resolvers.openers, resolvers.loaders -function curl.fetch(protocol, name) - local cachename = curl.cachepath() .. "/" .. gsub(name,"[^%a%d%.]+","-") --- cachename = gsub(cachename,"[\\/]", io.fileseparator) - cachename = gsub(cachename,"[\\]", "/") -- cleanup - if not curl.cached[name] then +resolvers.curl = resolvers.curl or { } +local curl = resolvers.curl + +local cached = { } + +local function runcurl(specification) + local original = specification.original + -- local scheme = specification.scheme + local cleanname = gsub(original,"[^%a%d%.]+","-") + local cachename = caches.setfirstwritablefile(cleanname,"curl") + if not cached[original] then if not io.exists(cachename) then - curl.cached[name] = cachename - local command = "curl --silent --create-dirs --output " .. cachename .. " " .. name -- no protocol .. "://" + cached[original] = cachename + local command = "curl --silent --create-dirs --output " .. cachename .. " " .. original os.spawn(command) end if io.exists(cachename) then - curl.cached[name] = cachename + cached[original] = cachename else - curl.cached[name] = "" + cached[original] = "" end end - return curl.cached[name] -end - -function finders.curl(protocol,filename) - local foundname = curl.fetch(protocol, filename) - return finders.generic(protocol,foundname,filetype) + return cached[original] end -function openers.curl(protocol,filename) - return openers.generic(protocol,filename) -end +-- old code: we could be cleaner using specification (see schemes) -function loaders.curl(protocol,filename) - return loaders.generic(protocol,filename) +local function finder(specification,filetype) + return resolvers.methodhandler("finders",runcurl(specification),filetype) end --- todo: metamethod +local opener = openers.file +local loader = loaders.file -function curl.install(protocol) - finders[protocol] = function (filename,filetype) return finders.curl(protocol,filename) end - openers[protocol] = function (filename) return openers.curl(protocol,filename) end - loaders[protocol] = function (filename) return loaders.curl(protocol,filename) end +local function install(scheme) + finders[scheme] = finder + openers[scheme] = opener + loaders[scheme] = loader end -curl.install('http') -curl.install('https') -curl.install('ftp') +resolvers.curl.install = install + +install('http') +install('https') +install('ftp') end -- of closure @@ -11287,9 +13494,13 @@ if not modules then modules = { } end modules ['data-lua'] = { local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) +local report_libraries = logs.reporter("resolvers","libraries") + local gsub, insert = string.gsub, table.insert local unpack = unpack or table.unpack +local resolvers, package = resolvers, package + local libformats = { 'luatexlibs', 'tex', 'texmfscripts', 'othertextfiles' } -- 'luainputs' local clibformats = { 'lib' } @@ -11298,7 +13509,7 @@ local _path_, libpaths, _cpath_, clibpaths function package.libpaths() if not _path_ or package.path ~= _path_ then _path_ = package.path - libpaths = file.split_path(_path_,";") + libpaths = file.splitpath(_path_,";") end return libpaths end @@ -11306,7 +13517,7 @@ end function package.clibpaths() if not _cpath_ or package.cpath ~= _cpath_ then _cpath_ = package.cpath - clibpaths = file.split_path(_cpath_,";") + clibpaths = file.splitpath(_cpath_,";") end return clibpaths end @@ -11315,18 +13526,18 @@ local function thepath(...) local t = { ... } t[#t+1] = "?.lua" local path = file.join(unpack(t)) if trace_locating then - logs.report("fileio","! appending '%s' to 'package.path'",path) + report_libraries("! appending '%s' to 'package.path'",path) end return path end local p_libpaths, a_libpaths = { }, { } -function package.append_libpath(...) +function package.appendtolibpath(...) insert(a_libpath,thepath(...)) end -function package.prepend_libpath(...) +function package.prependtolibpath(...) insert(p_libpaths,1,thepath(...)) end @@ -11337,31 +13548,30 @@ local function loaded(libpaths,name,simple) local libpath = libpaths[i] local resolved = gsub(libpath,"%?",simple) if trace_locating then -- more detail - logs.report("fileio","! checking for '%s' on 'package.path': '%s' => '%s'",simple,libpath,resolved) + report_libraries("! checking for '%s' on 'package.path': '%s' => '%s'",simple,libpath,resolved) end - if resolvers.isreadable.file(resolved) then + if file.is_readable(resolved) then if trace_locating then - logs.report("fileio","! lib '%s' located via 'package.path': '%s'",name,resolved) + report_libraries("! lib '%s' located via 'package.path': '%s'",name,resolved) end return loadfile(resolved) end end end - package.loaders[2] = function(name) -- was [#package.loaders+1] if trace_locating then -- mode detail - logs.report("fileio","! locating '%s'",name) + report_libraries("! locating '%s'",name) end for i=1,#libformats do local format = libformats[i] - local resolved = resolvers.find_file(name,format) or "" + local resolved = resolvers.findfile(name,format) or "" if trace_locating then -- mode detail - logs.report("fileio","! checking for '%s' using 'libformat path': '%s'",name,format) + report_libraries("! checking for '%s' using 'libformat path': '%s'",name,format) end if resolved ~= "" then if trace_locating then - logs.report("fileio","! lib '%s' located via environment: '%s'",name,resolved) + report_libraries("! lib '%s' located via environment: '%s'",name,resolved) end return loadfile(resolved) end @@ -11379,16 +13589,16 @@ package.loaders[2] = function(name) -- was [#package.loaders+1] for i=1,#clibformats do -- better have a dedicated loop local format = clibformats[i] - local paths = resolvers.expanded_path_list_from_var(format) + local paths = resolvers.expandedpathlistfromvariable(format) for p=1,#paths do local path = paths[p] local resolved = file.join(path,libname) if trace_locating then -- mode detail - logs.report("fileio","! checking for '%s' using 'clibformat path': '%s'",libname,path) + report_libraries("! checking for '%s' using 'clibformat path': '%s'",libname,path) end - if resolvers.isreadable.file(resolved) then + if file.is_readable(resolved) then if trace_locating then - logs.report("fileio","! lib '%s' located via 'clibformat': '%s'",libname,resolved) + report_libraries("! lib '%s' located via 'clibformat': '%s'",libname,resolved) end return package.loadlib(resolved,name) end @@ -11398,140 +13608,43 @@ package.loaders[2] = function(name) -- was [#package.loaders+1] local libpath = clibpaths[i] local resolved = gsub(libpath,"?",simple) if trace_locating then -- more detail - logs.report("fileio","! checking for '%s' on 'package.cpath': '%s'",simple,libpath) + report_libraries("! checking for '%s' on 'package.cpath': '%s'",simple,libpath) end - if resolvers.isreadable.file(resolved) then + if file.is_readable(resolved) then if trace_locating then - logs.report("fileio","! lib '%s' located via 'package.cpath': '%s'",name,resolved) + report_libraries("! lib '%s' located via 'package.cpath': '%s'",name,resolved) end return package.loadlib(resolved,name) end end -- just in case the distribution is messed up if trace_loading then -- more detail - logs.report("fileio","! checking for '%s' using 'luatexlibs': '%s'",name) + report_libraries("! checking for '%s' using 'luatexlibs': '%s'",name) end - local resolved = resolvers.find_file(file.basename(name),'luatexlibs') or "" + local resolved = resolvers.findfile(file.basename(name),'luatexlibs') or "" if resolved ~= "" then if trace_locating then - logs.report("fileio","! lib '%s' located by basename via environment: '%s'",name,resolved) + report_libraries("! lib '%s' located by basename via environment: '%s'",name,resolved) end return loadfile(resolved) end if trace_locating then - logs.report("fileio",'? unable to locate lib: %s',name) + report_libraries('? unable to locate lib: %s',name) end -- return "unable to locate " .. name end resolvers.loadlualib = require +-- -- -- -- -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['luat-kps'] = { - version = 1.001, - comment = "companion to luatools.lua", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - ---[[ldx-- -This file is used when we want the input handlers to behave like
-
How about just forgetting about them?
---ldx]]-- - -local suffixes = resolvers.suffixes -local formats = resolvers.formats - -suffixes['gf'] = { 'If you wondered abou tsome of the previous mappings, how about -the next bunch:
---ldx]]-- +package.append_libpath = appendtolibpath -- will become obsolete +package.prepend_libpath = prependtolibpath -- will become obsolete -formats['bib'] = '' -formats['bst'] = '' -formats['mft'] = '' -formats['ist'] = '' -formats['web'] = '' -formats['cweb'] = '' -formats['MetaPost support'] = '' -formats['TeX system documentation'] = '' -formats['TeX system sources'] = '' -formats['Troff fonts'] = '' -formats['dvips config'] = '' -formats['graphic/figure'] = '' -formats['ls-R'] = '' -formats['other text files'] = '' -formats['other binary files'] = '' - -formats['gf'] = '' -formats['pk'] = '' -formats['base'] = 'MFBASES' -formats['cnf'] = '' -formats['mem'] = 'MPMEMS' -formats['mf'] = 'MFINPUTS' -formats['mfpool'] = 'MFPOOL' -formats['mppool'] = 'MPPOOL' -formats['texpool'] = 'TEXPOOL' -formats['PostScript header'] = 'TEXPSHEADERS' -formats['cmap files'] = 'CMAPFONTS' -formats['type42 fonts'] = 'T42FONTS' -formats['web2c files'] = 'WEB2C' -formats['pdftex config'] = 'PDFTEXCONFIG' -formats['texmfscripts'] = 'TEXMFSCRIPTS' -formats['bitmap font'] = '' -formats['lig files'] = 'LIGFONTS' +package.obsolete.append_libpath = appendtolibpath -- will become obsolete +package.obsolete.prepend_libpath = prependtolibpath -- will become obsolete end -- of closure @@ -11547,49 +13660,54 @@ if not modules then modules = { } end modules ['data-aux'] = { } local find = string.find +local type, next = type, next local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) -function resolvers.update_script(oldname,newname) -- oldname -> own.name, not per se a suffix +local resolvers = resolvers + +local report_scripts = logs.reporter("resolvers","scripts") + +function resolvers.updatescript(oldname,newname) -- oldname -> own.name, not per se a suffix local scriptpath = "scripts/context/lua" newname = file.addsuffix(newname,"lua") - local oldscript = resolvers.clean_path(oldname) + local oldscript = resolvers.cleanpath(oldname) if trace_locating then - logs.report("fileio","to be replaced old script %s", oldscript) + report_scripts("to be replaced old script %s", oldscript) end - local newscripts = resolvers.find_files(newname) or { } + local newscripts = resolvers.findfiles(newname) or { } if #newscripts == 0 then if trace_locating then - logs.report("fileio","unable to locate new script") + report_scripts("unable to locate new script") end else for i=1,#newscripts do - local newscript = resolvers.clean_path(newscripts[i]) + local newscript = resolvers.cleanpath(newscripts[i]) if trace_locating then - logs.report("fileio","checking new script %s", newscript) + report_scripts("checking new script %s", newscript) end if oldscript == newscript then if trace_locating then - logs.report("fileio","old and new script are the same") + report_scripts("old and new script are the same") end elseif not find(newscript,scriptpath) then if trace_locating then - logs.report("fileio","new script should come from %s",scriptpath) + report_scripts("new script should come from %s",scriptpath) end elseif not (find(oldscript,file.removesuffix(newname).."$") or find(oldscript,newname.."$")) then if trace_locating then - logs.report("fileio","invalid new script name") + report_scripts("invalid new script name") end else local newdata = io.loaddata(newscript) if newdata then if trace_locating then - logs.report("fileio","old script content replaced by new content") + report_scripts("old script content replaced by new content") end io.savedata(oldscript,newdata) break elseif trace_locating then - logs.report("fileio","unable to load new script") + report_scripts("unable to load new script") end end end @@ -11609,72 +13727,133 @@ if not modules then modules = { } end modules ['data-tmf'] = { license = "see context related readme files" } -local find, gsub, match = string.find, string.gsub, string.match -local getenv, setenv = os.getenv, os.setenv +local resolvers = resolvers --- loads *.tmf files in minimal tree roots (to be optimized and documented) +local report_tds = logs.reporter("resolvers","tds") -function resolvers.check_environment(tree) - logs.simpleline() - setenv('TMP', getenv('TMP') or getenv('TEMP') or getenv('TMPDIR') or getenv('HOME')) - setenv('TEXOS', getenv('TEXOS') or ("texmf-" .. os.platform)) - setenv('TEXPATH', gsub(tree or "tex","\/+$",'')) - setenv('TEXMFOS', getenv('TEXPATH') .. "/" .. getenv('TEXOS')) - logs.simpleline() - logs.simple("preset : TEXPATH => %s", getenv('TEXPATH')) - logs.simple("preset : TEXOS => %s", getenv('TEXOS')) - logs.simple("preset : TEXMFOS => %s", getenv('TEXMFOS')) - logs.simple("preset : TMP => %s", getenv('TMP')) - logs.simple('') -end +-- = << +-- ? ?? +-- < += +-- > =+ -function resolvers.load_environment(name) -- todo: key=value as well as lua - local f = io.open(name) - if f then - for line in f:lines() do - if find(line,"^[%%%#]") then - -- skip comment - else - local key, how, value = match(line,"^(.-)%s*([<=>%?]+)%s*(.*)%s*$") - if how then - value = gsub(value,"%%(.-)%%", function(v) return getenv(v) or "" end) - if how == "=" or how == "<<" then - setenv(key,value) - elseif how == "?" or how == "??" then - setenv(key,getenv(key) or value) - elseif how == "<" or how == "+=" then - if getenv(key) then - setenv(key,getenv(key) .. io.fileseparator .. value) - else - setenv(key,value) - end - elseif how == ">" or how == "=+" then - if getenv(key) then - setenv(key,value .. io.pathseparator .. getenv(key)) - else - setenv(key,value) - end - end - end - end +function resolvers.load_tree(tree,resolve) + if type(tree) == "string" and tree ~= "" then + + local getenv, setenv = resolvers.getenv, resolvers.setenv + + -- later might listen to the raw osenv var as well + local texos = "texmf-" .. os.platform + + local oldroot = environment.texroot + local newroot = file.collapsepath(tree) + + local newtree = file.join(newroot,texos) + local newpath = file.join(newtree,"bin") + + if not lfs.isdir(newtree) then + report_tds("no '%s' under tree %s",texos,tree) + os.exit() end - f:close() + if not lfs.isdir(newpath) then + report_tds("no '%s/bin' under tree %s",texos,tree) + os.exit() + end + + local texmfos = newtree + + environment.texroot = newroot + environment.texos = texos + environment.texmfos = texmfos + + -- Beware, we need to obey the relocatable autoparent so we + -- set TEXMFCNF to its raw value. This is somewhat tricky when + -- we run a mkii job from within. Therefore, in mtxrun, there + -- is a resolve applied when we're in mkii/kpse mode. + + setenv('SELFAUTOPARENT', newroot) + setenv('SELFAUTODIR', newtree) + setenv('SELFAUTOLOC', newpath) + setenv('TEXROOT', newroot) + setenv('TEXOS', texos) + setenv('TEXMFOS', texmfos) + setenv('TEXMFCNF', resolvers.luacnfspec, not resolve) + setenv("PATH", newpath .. io.pathseparator .. getenv("PATH")) + + report_tds("changing from root '%s' to '%s'",oldroot,newroot) + report_tds("prepending '%s' to binary path",newpath) + report_tds() end end -function resolvers.load_tree(tree) - if tree and tree ~= "" then - local setuptex = 'setuptex.tmf' - if lfs.attributes(tree, "mode") == "directory" then -- check if not nil - setuptex = tree .. "/" .. setuptex - else - setuptex = tree + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['data-lst'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- used in mtxrun, can be loaded later .. todo + +local find, concat, upper, format = string.find, table.concat, string.upper, string.format + +resolvers.listers = resolvers.listers or { } + +local resolvers = resolvers + +local report_lists = logs.reporter("resolvers","lists") + +local function tabstr(str) + if type(str) == 'table' then + return concat(str," | ") + else + return str + end +end + +function resolvers.listers.variables(pattern) + local instance = resolvers.instance + local environment = instance.environment + local variables = instance.variables + local expansions = instance.expansions + local pattern = upper(pattern or "") + local configured = { } + local order = instance.order + for i=1,#order do + for k, v in next, order[i] do + if v ~= nil and configured[k] == nil then + configured[k] = v + end end - if io.exists(setuptex) then - resolvers.check_environment(tree) - resolvers.load_environment(setuptex) + end + local env = table.fastcopy(environment) + local var = table.fastcopy(variables) + local exp = table.fastcopy(expansions) + for key, value in table.sortedpairs(configured) do + if key ~= "" and (pattern == "" or find(upper(key),pattern)) then + report_lists(key) + report_lists(" env: %s",tabstr(rawget(environment,key)) or "unset") + report_lists(" var: %s",tabstr(configured[key]) or "unset") + report_lists(" exp: %s",tabstr(expansions[key]) or "unset") + report_lists(" res: %s",resolvers.resolve(expansions[key]) or "unset") end end + instance.environment = table.fastcopy(env) + instance.variables = table.fastcopy(var) + instance.expansions = table.fastcopy(exp) +end + +function resolvers.listers.configurations(report) + local configurations = resolvers.instance.specification + local report = report or texio.write_nl + for i=1,#configurations do + report(resolvers.resolve(configurations[i])) + end end @@ -11691,12 +13870,18 @@ if not modules then modules = { } end modules ['luat-sta'] = { -- this code is used in the updater -local gmatch, match = string.gmatch, string.match -local type = type +local gmatch, match = string.gmatch, string.match +local type = type + +states = states or { } +local states = states + +states.data = states.data or { } +local data = states.data + +states.hash = states.hash or { } +local hash = states.hash -states = states or { } -states.data = states.data or { } -states.hash = states.hash or { } states.tag = states.tag or "" states.filename = states.filename or "" @@ -11706,7 +13891,7 @@ function states.save(filename,tag) io.savedata(filename, "-- generator : luat-sta.lua\n" .. "-- state tag : " .. tag .. "\n\n" .. - table.serialize(states.data[tag or states.tag] or {},true) + table.serialize(data[tag or states.tag] or {},true) ) end @@ -11714,11 +13899,11 @@ function states.load(filename,tag) states.filename = filename states.tag = tag or "whatever" states.filename = file.addsuffix(states.filename,'lus') - states.data[states.tag], states.hash[states.tag] = (io.exists(filename) and dofile(filename)) or { }, { } + data[states.tag], hash[states.tag] = (io.exists(filename) and dofile(filename)) or { }, { } end -function states.set_by_tag(tag,key,value,default,persistent) - local d, h = states.data[tag], states.hash[tag] +local function set_by_tag(tag,key,value,default,persistent) + local d, h = data[tag], hash[tag] if d then if type(d) == "table" then local dkey, hkey = key, key @@ -11738,8 +13923,10 @@ function states.set_by_tag(tag,key,value,default,persistent) end dkey, hkey = post, key end - if type(value) == nil then - value = value or default + if value == nil then + value = default + elseif value == false then + -- special case elseif persistent then value = value or d[dkey] or default else @@ -11748,144 +13935,202 @@ function states.set_by_tag(tag,key,value,default,persistent) d[dkey], h[hkey] = value, value elseif type(d) == "string" then -- weird - states.data[tag], states.hash[tag] = value, value + data[tag], hash[tag] = value, value end end end -function states.get_by_tag(tag,key,default) - local h = states.hash[tag] +local function get_by_tag(tag,key,default) + local h = hash[tag] if h and h[key] then return h[key] else - local d = states.data[tag] + local d = data[tag] if d then for k in gmatch(key,"[^%.]+") do local dk = d[k] - if dk then + if dk ~= nil then d = dk else return default end end - return d or default + if d == false then + return false + else + return d or default + end end end end +states.set_by_tag = set_by_tag +states.get_by_tag = get_by_tag + function states.set(key,value,default,persistent) - states.set_by_tag(states.tag,key,value,default,persistent) + set_by_tag(states.tag,key,value,default,persistent) end function states.get(key,default) - return states.get_by_tag(states.tag,key,default) -end - ---~ states.data.update = { ---~ ["version"] = { ---~ ["major"] = 0, ---~ ["minor"] = 1, ---~ }, ---~ ["rsync"] = { ---~ ["server"] = "contextgarden.net", ---~ ["module"] = "minimals", ---~ ["repository"] = "current", ---~ ["flags"] = "-rpztlv --stats", ---~ }, ---~ ["tasks"] = { ---~ ["update"] = true, ---~ ["make"] = true, ---~ ["delete"] = false, ---~ }, ---~ ["platform"] = { ---~ ["host"] = true, ---~ ["other"] = { ---~ ["mswin"] = false, ---~ ["linux"] = false, ---~ ["linux-64"] = false, ---~ ["osx-intel"] = false, ---~ ["osx-ppc"] = false, ---~ ["sun"] = false, ---~ }, ---~ }, ---~ ["context"] = { ---~ ["available"] = {"current", "beta", "alpha", "experimental"}, ---~ ["selected"] = "current", ---~ }, ---~ ["formats"] = { ---~ ["cont-en"] = true, ---~ ["cont-nl"] = true, ---~ ["cont-de"] = false, ---~ ["cont-cz"] = false, ---~ ["cont-fr"] = false, ---~ ["cont-ro"] = false, ---~ }, ---~ ["engine"] = { ---~ ["pdftex"] = { ---~ ["install"] = true, ---~ ["formats"] = { ---~ ["pdftex"] = true, ---~ }, ---~ }, ---~ ["luatex"] = { ---~ ["install"] = true, ---~ ["formats"] = { ---~ }, ---~ }, ---~ ["xetex"] = { ---~ ["install"] = true, ---~ ["formats"] = { ---~ ["xetex"] = false, ---~ }, ---~ }, ---~ ["metapost"] = { ---~ ["install"] = true, ---~ ["formats"] = { ---~ ["mpost"] = true, ---~ ["metafun"] = true, ---~ }, ---~ }, ---~ }, ---~ ["fonts"] = { ---~ }, ---~ ["doc"] = { ---~ }, ---~ ["modules"] = { ---~ ["f-urwgaramond"] = false, ---~ ["f-urwgothic"] = false, ---~ ["t-bnf"] = false, ---~ ["t-chromato"] = false, ---~ ["t-cmscbf"] = false, ---~ ["t-cmttbf"] = false, ---~ ["t-construction-plan"] = false, ---~ ["t-degrade"] = false, ---~ ["t-french"] = false, ---~ ["t-lettrine"] = false, ---~ ["t-lilypond"] = false, ---~ ["t-mathsets"] = false, ---~ ["t-tikz"] = false, ---~ ["t-typearea"] = false, ---~ ["t-vim"] = false, ---~ }, ---~ } - ---~ states.save("teststate", "update") ---~ states.load("teststate", "update") - ---~ print(states.get_by_tag("update","rsync.server","unknown")) ---~ states.set_by_tag("update","rsync.server","oeps") ---~ print(states.get_by_tag("update","rsync.server","unknown")) ---~ states.save("teststate", "update") ---~ states.load("teststate", "update") ---~ print(states.get_by_tag("update","rsync.server","unknown")) + return get_by_tag(states.tag,key,default) +end + + + + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['luat-fmt'] = { + version = 1.001, + comment = "companion to mtxrun", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + + +local format = string.format + +local report_format = logs.reporter("resolvers","formats") + +-- helper for mtxrun + +local quoted = string.quoted + +local function primaryflags() -- not yet ok + local trackers = environment.argument("trackers") + local directives = environment.argument("directives") + local flags = "" + if trackers and trackers ~= "" then + flags = flags .. "--trackers=" .. quoted(trackers) + end + if directives and directives ~= "" then + flags = flags .. "--directives=" .. quoted(directives) + end + return flags +end + +function environment.make_format(name) + -- change to format path (early as we need expanded paths) + local olddir = lfs.currentdir() + local path = caches.getwritablepath("formats") or "" -- maybe platform + if path ~= "" then + lfs.chdir(path) + end + report_format("format path: %s",lfs.currentdir()) + -- check source file + local texsourcename = file.addsuffix(name,"mkiv") + local fulltexsourcename = resolvers.findfile(texsourcename,"tex") or "" + if fulltexsourcename == "" then + texsourcename = file.addsuffix(name,"tex") + fulltexsourcename = resolvers.findfile(texsourcename,"tex") or "" + end + if fulltexsourcename == "" then + report_format("no tex source file with name: %s (mkiv or tex)",name) + lfs.chdir(olddir) + return + else + report_format("using tex source file: %s",fulltexsourcename) + end + local texsourcepath = dir.expandname(file.dirname(fulltexsourcename)) -- really needed + -- check specification + local specificationname = file.replacesuffix(fulltexsourcename,"lus") + local fullspecificationname = resolvers.findfile(specificationname,"tex") or "" + if fullspecificationname == "" then + specificationname = file.join(texsourcepath,"context.lus") + fullspecificationname = resolvers.findfile(specificationname,"tex") or "" + end + if fullspecificationname == "" then + report_format("unknown stub specification: %s",specificationname) + lfs.chdir(olddir) + return + end + local specificationpath = file.dirname(fullspecificationname) + -- load specification + local usedluastub = nil + local usedlualibs = dofile(fullspecificationname) + if type(usedlualibs) == "string" then + usedluastub = file.join(file.dirname(fullspecificationname),usedlualibs) + elseif type(usedlualibs) == "table" then + report_format("using stub specification: %s",fullspecificationname) + local texbasename = file.basename(name) + local luastubname = file.addsuffix(texbasename,"lua") + local lucstubname = file.addsuffix(texbasename,"luc") + -- pack libraries in stub + report_format("creating initialization file: %s",luastubname) + utilities.merger.selfcreate(usedlualibs,specificationpath,luastubname) + -- compile stub file (does not save that much as we don't use this stub at startup any more) + local strip = resolvers.booleanvariable("LUACSTRIP", true) + if utilities.lua.compile(luastubname,lucstubname) and lfs.isfile(lucstubname) then + report_format("using compiled initialization file: %s",lucstubname) + usedluastub = lucstubname + else + report_format("using uncompiled initialization file: %s",luastubname) + usedluastub = luastubname + end + else + report_format("invalid stub specification: %s",fullspecificationname) + lfs.chdir(olddir) + return + end + -- generate format + local command = format("luatex --ini %s --lua=%s %s %sdump",primaryflags(),quoted(usedluastub),quoted(fulltexsourcename),os.platform == "unix" and "\\\\" or "\\") + report_format("running command: %s\n",command) + os.spawn(command) + -- remove related mem files + local pattern = file.removesuffix(file.basename(usedluastub)).."-*.mem" + -- report_format("removing related mplib format with pattern '%s'", pattern) + local mp = dir.glob(pattern) + if mp then + for i=1,#mp do + local name = mp[i] + report_format("removing related mplib format %s", file.basename(name)) + os.remove(name) + end + end + lfs.chdir(olddir) +end + +function environment.run_format(name,data,more) + -- hm, rather old code here; we can now use the file.whatever functions + if name and name ~= "" then + local barename = file.removesuffix(name) + local fmtname = caches.getfirstreadablefile(file.addsuffix(barename,"fmt"),"formats") + if fmtname == "" then + fmtname = resolvers.findfile(file.addsuffix(barename,"fmt")) or "" + end + fmtname = resolvers.cleanpath(fmtname) + if fmtname == "" then + report_format("no format with name: %s",name) + else + local barename = file.removesuffix(name) -- expanded name + local luaname = file.addsuffix(barename,"luc") + if not lfs.isfile(luaname) then + luaname = file.addsuffix(barename,"lua") + end + if not lfs.isfile(luaname) then + report_format("using format name: %s",fmtname) + report_format("no luc/lua with name: %s",barename) + else + local command = format("luatex %s --fmt=%s --lua=%s %s %s",primaryflags(),quoted(barename),quoted(luaname),quoted(data),more ~= "" and quoted(more) or "") + report_format("running command: %s",command) + os.spawn(command) + end + end + end +end end -- of closure -- end library merge -own = { } -- not local +own = { } -- not local, might change + +own.libs = { -- order can be made better -own.libs = { -- todo: check which ones are really needed 'l-string.lua', 'l-lpeg.lua', 'l-table.lua', @@ -11898,39 +14143,55 @@ own.libs = { -- todo: check which ones are really needed 'l-url.lua', 'l-dir.lua', 'l-boolean.lua', + 'l-unicode.lua', 'l-math.lua', --- 'l-unicode.lua', --- 'l-tex.lua', - 'l-utils.lua', - 'l-aux.lua', --- 'l-xml.lua', - 'trac-tra.lua', + + 'util-tab.lua', + 'util-sto.lua', + 'util-mrg.lua', + 'util-lua.lua', + 'util-prs.lua', + 'util-fmt.lua', + 'util-deb.lua', + + 'trac-inf.lua', + 'trac-set.lua', + 'trac-log.lua', + 'trac-pro.lua', + + 'luat-env.lua', -- can come before inf (as in mkiv) + 'lxml-tab.lua', 'lxml-lpt.lua', --- 'lxml-ent.lua', + -- 'lxml-ent.lua', 'lxml-mis.lua', 'lxml-aux.lua', 'lxml-xml.lua', - 'luat-env.lua', - 'trac-inf.lua', - 'trac-log.lua', - 'data-res.lua', + + 'data-ini.lua', + 'data-exp.lua', + 'data-env.lua', 'data-tmp.lua', + 'data-met.lua', + 'data-res.lua', 'data-pre.lua', 'data-inp.lua', 'data-out.lua', + 'data-fil.lua', 'data-con.lua', 'data-use.lua', -- 'data-tex.lua', -- 'data-bin.lua', 'data-zip.lua', + 'data-tre.lua', 'data-crl.lua', 'data-lua.lua', - 'data-kps.lua', -- so that we can replace kpsewhich 'data-aux.lua', -- updater - 'data-tmf.lua', -- tree files - -- needed ? - 'luat-sta.lua', -- states + 'data-tmf.lua', + 'data-lst.lua', + + 'luat-sta.lua', + 'luat-fmt.lua', } -- We need this hack till luatex is fixed. @@ -11943,35 +14204,62 @@ end -- End of hack. -own.name = (environment and environment.ownname) or arg[0] or 'luatools.lua' +local format, gsub, gmatch, match = string.format, string.gsub, string.gmatch, string.match +local concat = table.concat + +own.name = (environment and environment.ownname) or arg[0] or 'mtxrun.lua' +own.path = gsub(match(own.name,"^(.+)[\\/].-$") or ".","\\","/") + +local ownpath, owntree = own.path, environment and environment.ownpath or own.path + +own.list = { + '.', + ownpath , + ownpath .. "/../sources", -- HH's development path + owntree .. "/../../texmf-local/tex/context/base", + owntree .. "/../../texmf-context/tex/context/base", + owntree .. "/../../texmf-dist/tex/context/base", + owntree .. "/../../texmf/tex/context/base", + owntree .. "/../../../texmf-local/tex/context/base", + owntree .. "/../../../texmf-context/tex/context/base", + owntree .. "/../../../texmf-dist/tex/context/base", + owntree .. "/../../../texmf/tex/context/base", +} +if own.path == "." then table.remove(own.list,1) end -own.path = string.match(own.name,"^(.+)[\\/].-$") or "." -own.list = { '.' } -if own.path ~= '.' then - table.insert(own.list,own.path) +local function locate_libs() + for l=1,#own.libs do + local lib = own.libs[l] + for p =1,#own.list do + local pth = own.list[p] + local filename = pth .. "/" .. lib + local found = lfs.isfile(filename) + if found then + package.path = package.path .. ";" .. pth .. "/?.lua" -- in case l-* does a require + return pth + end + end + end end -table.insert(own.list,own.path.."/../../texmf-dist/tex/context/base") -table.insert(own.list,own.path.."/../../../tex/context/base") -table.insert(own.list,own.path.."/mtx") -table.insert(own.list,own.path.."/../sources") -local function locate_libs() - for _, lib in pairs(own.libs) do - for _, pth in pairs(own.list) do - local filename = string.gsub(pth .. "/" .. lib,"\\","/") +local function load_libs() + local found = locate_libs() + if found then + for l=1,#own.libs do + local filename = found .. "/" .. own.libs[l] local codeblob = loadfile(filename) if codeblob then codeblob() - own.list = { pth } -- speed up te search - break end end + else + resolvers = nil end end if not resolvers then - locate_libs() + load_libs() end if not resolvers then @@ -11983,20 +14271,56 @@ if not resolvers then os.exit() end -logs.setprogram('MTXrun',"TDS Runner Tool 1.24",environment.arguments["verbose"] or false) +-- verbosity -local instance = resolvers.reset() +local e_verbose = environment.arguments["verbose"] + +if e_verbose then + trackers.enable("resolvers.locating") +end + +-- some common flags (also passed through environment) + +local e_silent = environment.argument("silent") +local e_noconsole = environment.argument("noconsole") + +local e_trackers = environment.argument("trackers") +local e_directives = environment.argument("directives") +local e_experiments = environment.argument("experiments") + +if e_silent == true then + e_silent = "*" +end -local trackspec = environment.argument("trackers") or environment.argument("track") +if type(e_silent) == "string" then + if type(e_directives) == "string" then + e_directives = format("%s,logs.blocked={%s}",e_directives,e_silent) + else + e_directives = format("logs.blocked={%s}",e_silent) + end +end -if trackspec then - trackers.enable(trackspec) +if e_noconsole then + if type(e_directives) == "string" then + e_directives = format("%s,logs.target=file",e_directives) + else + e_directives = format("logs.target=file") + end end -runners = runners or { } -- global -messages = messages or { } +if e_trackers then trackers .enable(e_trackers) end +if e_directives then directives .enable(e_directives) end +if e_experiments then experiments.enable(e_experiments) end + +if not environment.trackers then environment.trackers = e_trackers end +if not environment.directives then environment.directives = e_directives end +if not environment.experiments then environment.experiments = e_experiments end + +-- + +local instance = resolvers.reset() -messages.help = [[ +local helpinfo = [[ --script run an mtx script (lua prefered method) (--noquotes), no script gives list --execute run a script or program (texmfstart method) (--noquotes) --resolve resolve prefixed arguments @@ -12011,15 +14335,14 @@ messages.help = [[ --ifchanged=filename only execute when given file has changed (md checksum) --iftouched=old,new only execute when given file has changed (time stamp) ---make create stubs for (context related) scripts ---remove remove stubs (context related) scripts +--makestubs create stubs for (context related) scripts +--removestubs remove stubs (context related) scripts --stubpath=binpath paths where stubs wil be written --windows create windows (mswin) stubs --unix create unix (linux) stubs --verbose give a bit more info --trackers=list enable given trackers ---engine=str target engine --progname=str format or backend --edit launch editor with found file @@ -12032,8 +14355,35 @@ messages.help = [[ --forcekpse force using kpse (handy when no mkiv and cache installed but less functionality) --prefixes show supported prefixes + +--generate generate file database + +--variables show configuration variables +--configurations show configuration order + +--expand-braces expand complex variable +--expand-path expand variable (resolve paths) +--expand-var expand variable (resolve references) +--show-path show path expansion of ... +--var-value report value of variable +--find-file report file location +--find-path report path of file + +--pattern=str filter variables ]] +local application = logs.application { + name = "mtxrun", + banner = "ConTeXt TDS Runner Tool 1.30", + helpinfo = helpinfo, +} + +local report = application.report + +messages = messages or { } -- for the moment + +runners = runners or { } -- global (might become local) + runners.applications = { ["lua"] = "luatex --luaonly", ["luc"] = "luatex --luaonly", @@ -12056,7 +14406,7 @@ runners.registered = { makempy = { 'makempy.pl', true }, mptopdf = { 'mptopdf.pl', true }, pstopdf = { 'pstopdf.rb', true }, -- converts ps (and some more) images, does some cleaning (replaced) --- examplex = { 'examplex.rb', false }, + -- examplex = { 'examplex.rb', false }, concheck = { 'concheck.rb', false }, runtools = { 'runtools.rb', true }, textools = { 'textools.rb', true }, @@ -12065,9 +14415,9 @@ runners.registered = { rlxtools = { 'rlxtools.rb', true }, pdftools = { 'pdftools.rb', true }, mpstools = { 'mpstools.rb', true }, --- exatools = { 'exatools.rb', true }, + -- exatools = { 'exatools.rb', true }, xmltools = { 'xmltools.rb', true }, --- luatools = { 'luatools.lua', true }, + -- luatools = { 'luatools.lua', true }, mtxtools = { 'mtxtools.rb', true }, pdftrimwhite = { 'pdftrimwhite.pl', false } } @@ -12086,45 +14436,39 @@ end function runners.prepare() local checkname = environment.argument("ifchanged") - if checkname and checkname ~= "" then + if type(checkname) == "string" and checkname ~= "" then local oldchecksum = file.loadchecksum(checkname) local newchecksum = file.checksum(checkname) if oldchecksum == newchecksum then - logs.simple("file '%s' is unchanged",checkname) + if e_verbose then + report("file '%s' is unchanged",checkname) + end return "skip" - else - logs.simple("file '%s' is changed, processing started",checkname) + elseif e_verbose then + report("file '%s' is changed, processing started",checkname) end file.savechecksum(checkname) end - local oldname, newname = string.split(environment.argument("iftouched") or "", ",") - if oldname and newname and oldname ~= "" and newname ~= "" then - if not file.needs_updating(oldname,newname) then - logs.simple("file '%s' and '%s' have same age",oldname,newname) - return "skip" - else - logs.simple("file '%s' is older than '%s'",oldname,newname) - end - end - local tree = environment.argument('tree') or "" - if environment.argument('autotree') then - tree = os.getenv('TEXMFSTART_TREE') or os.getenv('TEXMFSTARTTREE') or tree - end - if tree and tree ~= "" then - resolvers.load_tree(tree) - end - local env = environment.argument('environment') or "" - if env and env ~= "" then - for _,e in pairs(string.split(env)) do - -- maybe force suffix when not given - resolvers.load_tree(e) + local touchname = environment.argument("iftouched") + if type(touchname) == "string" and touchname ~= "" then + local oldname, newname = string.split(touchname, ",") + if oldname and newname and oldname ~= "" and newname ~= "" then + if not file.needs_updating(oldname,newname) then + if e_verbose then + report("file '%s' and '%s' have same age",oldname,newname) + end + return "skip" + elseif e_verbose then + report("file '%s' is older than '%s'",oldname,newname) + end end end local runpath = environment.argument("path") - if runpath and not lfs.chdir(runpath) then - logs.simple("unable to change to path '%s'",runpath) + if type(runpath) == "string" and not lfs.chdir(runpath) then + report("unable to change to path '%s'",runpath) return "error" end + runners.prepare = function() end return "run" end @@ -12137,8 +14481,6 @@ function runners.execute_script(fullname,internal,nosplit) elseif state == 'skip' then return true elseif state == "run" then - instance.progname = environment.argument("progname") or instance.progname - instance.format = environment.argument("format") or instance.format local path, name, suffix, result = file.dirname(fullname), file.basename(fullname), file.extname(fullname), "" if path ~= "" then result = fullname @@ -12155,21 +14497,21 @@ function runners.execute_script(fullname,internal,nosplit) if suffix == "" then -- loop over known suffixes for _,s in pairs(runners.suffixes) do - result = resolvers.find_file(name .. "." .. s, 'texmfscripts') + result = resolvers.findfile(name .. "." .. s, 'texmfscripts') if result ~= "" then break end end elseif runners.applications[suffix] then - result = resolvers.find_file(name, 'texmfscripts') + result = resolvers.findfile(name, 'texmfscripts') else -- maybe look on path - result = resolvers.find_file(name, 'other text files') + result = resolvers.findfile(name, 'other text files') end end if result and result ~= "" then if not no_split then - local before, after = environment.split_arguments(fullname) -- already done + local before, after = environment.splitarguments(fullname) -- already done environment.arguments_before, environment.arguments_after = before, after end if internal then @@ -12181,12 +14523,12 @@ function runners.execute_script(fullname,internal,nosplit) if binary and binary ~= "" then result = binary .. " " .. result end - local command = result .. " " .. environment.reconstruct_commandline(environment.arguments_after,noquote) - if logs.verbose then - logs.simpleline() - logs.simple("executing: %s",command) - logs.simpleline() - logs.simpleline() + local command = result .. " " .. environment.reconstructcommandline(environment.arguments_after,noquote) + if e_verbose then + report() + report("executing: %s",command) + report() + report() io.flush() end -- no os.exec because otherwise we get the wrong return value @@ -12196,14 +14538,14 @@ function runners.execute_script(fullname,internal,nosplit) else if binary then binary = file.addsuffix(binary,os.binsuffix) - for p in string.gmatch(os.getenv("PATH"),"[^"..io.pathseparator.."]+") do + for p in gmatch(os.getenv("PATH"),"[^"..io.pathseparator.."]+") do if lfs.isfile(file.join(p,binary)) then return false end end - logs.simpleline() - logs.simple("This script needs '%s' which seems not to be installed.",binary) - logs.simpleline() + report() + report("This script needs '%s' which seems not to be installed.",binary) + report() end return false end @@ -12223,14 +14565,15 @@ function runners.execute_program(fullname) elseif state == 'skip' then return true elseif state == "run" then - local before, after = environment.split_arguments(fullname) - environment.initialize_arguments(after) + local before, after = environment.splitarguments(fullname) + for k=1,#after do after[k] = resolvers.resolve(after[k]) end + environment.initializearguments(after) fullname = fullname:gsub("^bin:","") - local command = fullname .. " " .. (environment.reconstruct_commandline(after or "",noquote) or "") - logs.simpleline() - logs.simple("executing: %s",command) - logs.simpleline() - logs.simpleline() + local command = fullname .. " " .. (environment.reconstructcommandline(after or "",noquote) or "") + report() + report("executing: %s",command) + report() + report() io.flush() local code = os.exec(command) -- (fullname,unpack(after)) does not work / maybe spawn return code == 0 @@ -12239,7 +14582,7 @@ function runners.execute_program(fullname) return false end --- the --usekpse flag will fallback on kpse (hm, we can better update mtx-stubs) +-- the --usekpse flag will fallback (not default) on kpse (hm, we can better update mtx-stubs) local windows_stub = '@echo off\013\010setlocal\013\010set ownpath=%%~dp0%%\013\010texlua "%%ownpath%%mtxrun.lua" --usekpse --execute %s %%*\013\010endlocal\013\010' local unix_stub = '#!/bin/sh\010mtxrun --usekpse --execute %s \"$@\"\010' @@ -12258,22 +14601,22 @@ function runners.handle_stubs(create) for _,v in pairs(runners.registered) do local name, doit = v[1], v[2] if doit then - local base = string.gsub(file.basename(name), "%.(.-)$", "") + local base = gsub(file.basename(name), "%.(.-)$", "") if create then if windows then - io.savedata(file.join(stubpath,base..".bat"),string.format(windows_stub,name)) - logs.simple("windows stub for '%s' created",base) + io.savedata(file.join(stubpath,base..".bat"),format(windows_stub,name)) + report("windows stub for '%s' created",base) end if unix then - io.savedata(file.join(stubpath,base),string.format(unix_stub,name)) - logs.simple("unix stub for '%s' created",base) + io.savedata(file.join(stubpath,base),format(unix_stub,name)) + report("unix stub for '%s' created",base) end else if windows and (os.remove(file.join(stubpath,base..'.bat')) or os.remove(file.join(stubpath,base..'.cmd'))) then - logs.simple("windows stub for '%s' removed", base) + report("windows stub for '%s' removed", base) end if unix and (os.remove(file.join(stubpath,base)) or os.remove(file.join(stubpath,base..'.sh'))) then - logs.simple("unix stub for '%s' removed",base) + report("unix stub for '%s' removed",base) end end end @@ -12289,7 +14632,7 @@ end function runners.locate_file(filename) -- differs from texmfstart where locate appends .com .exe .bat ... todo if filename and filename ~= "" then - runners.report_location(resolvers.find_given_file(filename)) + runners.report_location(resolvers.findgivenfile(filename)) end end @@ -12298,28 +14641,28 @@ function runners.locate_platform() end function runners.report_location(result) - if logs.verbose then - logs.simpleline() + if e_verbose then + reportline() if result and result ~= "" then - logs.simple(result) + report(result) else - logs.simple("not found") + report("not found") end else io.write(result) end end -function runners.edit_script(filename) -- we assume that vim is present on most systems - local editor = os.getenv("MTXRUN_EDITOR") or os.getenv("TEXMFSTART_EDITOR") or os.getenv("EDITOR") or 'vim' +function runners.edit_script(filename) -- we assume that gvim is present on most systems (todo: also in cnf file) + local editor = os.getenv("MTXRUN_EDITOR") or os.getenv("TEXMFSTART_EDITOR") or os.getenv("EDITOR") or 'gvim' local rest = resolvers.resolve(filename) if rest ~= "" then local command = editor .. " " .. rest - if logs.verbose then - logs.simpleline() - logs.simple("starting editor: %s",command) - logs.simple_line() - logs.simple_line() + if e_verbose then + report() + report("starting editor: %s",command) + report() + report() end os.launch(command) end @@ -12362,33 +14705,33 @@ end function runners.launch_file(filename) instance.allresults = true - logs.setverbose(true) + trackers.enable("resolvers.locating") local pattern = environment.arguments["pattern"] if not pattern or pattern == "" then pattern = filename end if not pattern or pattern == "" then - logs.simple("provide name or --pattern=") + report("provide name or --pattern=") else - local t = resolvers.find_files(pattern) + local t = resolvers.findfiles(pattern) if not t or #t == 0 then - t = resolvers.find_files("*/" .. pattern) + t = resolvers.findfiles("*/" .. pattern) end if not t or #t == 0 then - t = resolvers.find_files("*/" .. pattern .. "*") + t = resolvers.findfiles("*/" .. pattern .. "*") end if t and #t > 0 then if environment.arguments["all"] then for _, v in pairs(t) do - logs.simple("launching %s", v) + report("launching %s", v) resolvers.launch(v) end else - logs.simple("launching %s", t[1]) + report("launching %s", t[1]) resolvers.launch(t[1]) end else - logs.simple("no match for %s", pattern) + report("no match for %s", pattern) end end end @@ -12421,33 +14764,45 @@ function runners.find_mtx_script(filename) local mtxprefix = (filename:find("^mtx%-") and "") or "mtx-" -- context namespace, mtx-%s | %s | %s | %s |