diff options
Diffstat (limited to 'Master/texmf-dist/tex/context/base/data-sch.lua')
| -rw-r--r-- | Master/texmf-dist/tex/context/base/data-sch.lua | 148 |
1 files changed, 88 insertions, 60 deletions
diff --git a/Master/texmf-dist/tex/context/base/data-sch.lua b/Master/texmf-dist/tex/context/base/data-sch.lua index e68b6cd01b9..6c905942a23 100644 --- a/Master/texmf-dist/tex/context/base/data-sch.lua +++ b/Master/texmf-dist/tex/context/base/data-sch.lua @@ -8,99 +8,127 @@ if not modules then modules = { } end modules ['data-sch'] = { local http = require("socket.http") local ltn12 = require("ltn12") - local gsub, concat, format = string.gsub, table.concat, string.format +local finders, openers, loaders = resolvers.finders, resolvers.openers, resolvers.loaders local trace_schemes = false trackers.register("resolvers.schemes",function(v) trace_schemes = v end) -schemes = schemes or { } +local report_schemes = logs.reporter("resolvers","schemes") + +local resolvers = resolvers -schemes.cached = { } -schemes.cachepath = caches.definepath("schemes") +resolvers.schemes = resolvers.schemes or { } +local schemes = resolvers.schemes schemes.threshold = 24 * 60 * 60 directives.register("schemes.threshold", function(v) schemes.threshold = tonumber(v) or schemes.threshold end) -local cached, loaded, reused = schemes.cached, { }, { } +local cleaners = { } -local finders, openers, loaders = resolvers.finders, resolvers.openers, resolvers.loaders +schemes.cleaners = cleaners + +function cleaners.none(specification) + return specification.original +end + +function cleaners.strip(specification) + return (gsub(specification.original,"[^%a%d%.]+","-")) +end + +function cleaners.md5(specification) + return file.addsuffix(md5.hex(specification.original),file.suffix(specification.path)) +end + +local cleaner = cleaners.strip -function schemes.curl(name,cachename) - local command = "curl --silent --create-dirs --output " .. cachename .. " " .. name -- no protocol .. "://" +directives.register("schemes.cleanmethod", function(v) cleaner = cleaners[v] or cleaners.strip end) + +function resolvers.schemes.cleanname(specification) + local hash = cleaner(specification) + if trace_schemes then + report_schemes("hashing %s to %s",specification.original,hash) + end + return hash +end + +local cached, loaded, reused, thresholds, handlers = { }, { }, { }, { }, { } + +local function runcurl(name,cachename) -- will use sockets instead or the curl library + local command = "curl --silent --create-dirs --output " .. cachename .. " " .. name os.spawn(command) end -function schemes.fetch(protocol,name,handler) - local cachename = schemes.cachepath() .. "/" .. gsub(name,"[^%a%d%.]+","-") - cachename = gsub(cachename,"[\\]", "/") -- cleanup - if not cached[name] then +local function fetch(specification) + local original = specification.original + local scheme = specification.scheme + local cleanname = schemes.cleanname(specification) + local cachename = caches.setfirstwritablefile(cleanname,"schemes") + if not cached[original] then statistics.starttiming(schemes) - if not io.exists(cachename) or (os.difftime(os.time(),lfs.attributes(cachename).modification) > schemes.threshold) then - cached[name] = cachename + if not io.exists(cachename) or (os.difftime(os.time(),lfs.attributes(cachename).modification) > + (thresholds[protocol] or schemes.threshold)) then + cached[original] = cachename + local handler = handlers[scheme] if handler then if trace_schemes then - logs.report("schemes","fetching '%s', protocol '%s', method 'built-in'",name,protocol) + report_schemes("fetching '%s', protocol '%s', method 'built-in'",original,scheme) end - io.flush() - handler(protocol,name,cachename) + logs.flush() + handler(specification,cachename) else if trace_schemes then - logs.report("schemes","fetching '%s', protocol '%s', method 'curl'",name,protocol) + report_schemes("fetching '%s', protocol '%s', method 'curl'",original,scheme) end - io.flush() - schemes.curl(name,cachename) + logs.flush() + runcurl(original,cachename) end end if io.exists(cachename) then - cached[name] = cachename + cached[original] = cachename if trace_schemes then - logs.report("schemes","using cached '%s', protocol '%s', cachename '%s'",name,protocol,cachename) + report_schemes("using cached '%s', protocol '%s', cachename '%s'",original,scheme,cachename) end else - cached[name] = "" + cached[original] = "" if trace_schemes then - logs.report("schemes","using missing '%s', protocol '%s'",name,protocol) + report_schemes("using missing '%s', protocol '%s'",original,scheme) end end - loaded[protocol] = loaded[protocol] + 1 + loaded[scheme] = loaded[scheme] + 1 statistics.stoptiming(schemes) else if trace_schemes then - logs.report("schemes","reusing '%s', protocol '%s'",name,protocol) + report_schemes("reusing '%s', protocol '%s'",original,scheme) end - reused[protocol] = reused[protocol] + 1 + reused[scheme] = reused[scheme] + 1 end - return cached[name] + return cached[original] end -function finders.schemes(protocol,filename,handler) - local foundname = schemes.fetch(protocol,filename,handler) - return finders.generic(protocol,foundname,filetype) +local function finder(specification,filetype) + return resolvers.methodhandler("finders",fetch(specification),filetype) end -function openers.schemes(protocol,filename) - return openers.generic(protocol,filename) +local opener = openers.file +local loader = loaders.file + +local function install(scheme,handler,threshold) + handlers [scheme] = handler + loaded [scheme] = 0 + reused [scheme] = 0 + finders [scheme] = finder + openers [scheme] = opener + loaders [scheme] = loader + thresholds[scheme] = threshold or schemes.threshold end -function loaders.schemes(protocol,filename) - return loaders.generic(protocol,filename) -end - --- could be metatable +schemes.install = install -function schemes.install(protocol,handler) - loaded [protocol] = 0 - reused [protocol] = 0 - finders[protocol] = function (filename,filetype) return finders.schemes(protocol,filename,handler) end - openers[protocol] = function (filename) return openers.schemes(protocol,filename) end - loaders[protocol] = function (filename) return loaders.schemes(protocol,filename) end -end - -local function http_handler(protocol,name,cachename) +local function http_handler(specification,cachename) local tempname = cachename .. ".tmp" local f = io.open(tempname,"wb") local status, message = http.request { - url = name, + url = specification.original, sink = ltn12.sink.file(f) } if not status then @@ -109,34 +137,34 @@ local function http_handler(protocol,name,cachename) os.remove(cachename) os.rename(tempname,cachename) end + return cachename end -schemes.install('http',http_handler) -schemes.install('https') -schemes.install('ftp') +install('http',http_handler) +install('https') -- see pod +install('ftp') statistics.register("scheme handling time", function() - local l, r = { }, { } + local l, r, nl, nr = { }, { }, 0, 0 for k, v in table.sortedhash(loaded) do if v > 0 then - l[#l+1] = k .. ":" .. v + nl = nl + 1 + l[nl] = k .. ":" .. v end end for k, v in table.sortedhash(reused) do if v > 0 then - r[#r+1] = k .. ":" .. v + nr = nr + 1 + r[nr] = k .. ":" .. v end end - local n = #l + #r + local n = nl + nr if n > 0 then - l = (#l > 0 and concat(l)) or "none" - r = (#r > 0 and concat(r)) or "none" + l = (nl > 0 and concat(l)) or "none" + r = (nr > 0 and concat(r)) or "none" return format("%s seconds, %s processed, threshold %s seconds, loaded: %s, reused: %s", statistics.elapsedtime(schemes), n, schemes.threshold, l, r) else return nil end end) - ---~ trace_schemes = true ---~ print(schemes.fetch("http","http://www.pragma-ade.com/show-man.pdf",http_handler)) |