1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
|
if not modules then modules = { } end modules ['data-sch'] = {
version = 1.001,
comment = "companion to luat-lib.mkiv",
author = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
copyright = "PRAGMA ADE / ConTeXt Development Team",
license = "see context related readme files"
}
local http = require("socket.http")
local ltn12 = require("ltn12")
local gsub, concat, format = string.gsub, table.concat, string.format
local finders, openers, loaders = resolvers.finders, resolvers.openers, resolvers.loaders
local trace_schemes = false trackers.register("resolvers.schemes",function(v) trace_schemes = v end)
local report_schemes = logs.reporter("resolvers","schemes")
local resolvers = resolvers
resolvers.schemes = resolvers.schemes or { }
local schemes = resolvers.schemes
schemes.threshold = 24 * 60 * 60
directives.register("schemes.threshold", function(v) schemes.threshold = tonumber(v) or schemes.threshold end)
local cleaners = { }
schemes.cleaners = cleaners
function cleaners.none(specification)
return specification.original
end
function cleaners.strip(specification)
return (gsub(specification.original,"[^%a%d%.]+","-")) -- so we keep periods
end
function cleaners.md5(specification)
return file.addsuffix(md5.hex(specification.original),file.suffix(specification.path))
end
local cleaner = cleaners.strip
directives.register("schemes.cleanmethod", function(v) cleaner = cleaners[v] or cleaners.strip end)
function resolvers.schemes.cleanname(specification)
local hash = cleaner(specification)
if trace_schemes then
report_schemes("hashing %s to %s",specification.original,hash)
end
return hash
end
local cached, loaded, reused, thresholds, handlers = { }, { }, { }, { }, { }
local function runcurl(name,cachename) -- will use sockets instead or the curl library
local command = "curl --silent --create-dirs --output " .. cachename .. " " .. name
os.spawn(command)
end
local function fetch(specification)
local original = specification.original
local scheme = specification.scheme
local cleanname = schemes.cleanname(specification)
local cachename = caches.setfirstwritablefile(cleanname,"schemes")
if not cached[original] then
statistics.starttiming(schemes)
if not io.exists(cachename) or (os.difftime(os.time(),lfs.attributes(cachename).modification) >
(thresholds[protocol] or schemes.threshold)) then
cached[original] = cachename
local handler = handlers[scheme]
if handler then
if trace_schemes then
report_schemes("fetching '%s', protocol '%s', method 'built-in'",original,scheme)
end
logs.flush()
handler(specification,cachename)
else
if trace_schemes then
report_schemes("fetching '%s', protocol '%s', method 'curl'",original,scheme)
end
logs.flush()
runcurl(original,cachename)
end
end
if io.exists(cachename) then
cached[original] = cachename
if trace_schemes then
report_schemes("using cached '%s', protocol '%s', cachename '%s'",original,scheme,cachename)
end
else
cached[original] = ""
if trace_schemes then
report_schemes("using missing '%s', protocol '%s'",original,scheme)
end
end
loaded[scheme] = loaded[scheme] + 1
statistics.stoptiming(schemes)
else
if trace_schemes then
report_schemes("reusing '%s', protocol '%s'",original,scheme)
end
reused[scheme] = reused[scheme] + 1
end
return cached[original]
end
local function finder(specification,filetype)
return resolvers.methodhandler("finders",fetch(specification),filetype)
end
local opener = openers.file
local loader = loaders.file
local function install(scheme,handler,threshold)
handlers [scheme] = handler
loaded [scheme] = 0
reused [scheme] = 0
finders [scheme] = finder
openers [scheme] = opener
loaders [scheme] = loader
thresholds[scheme] = threshold or schemes.threshold
end
schemes.install = install
local function http_handler(specification,cachename)
local tempname = cachename .. ".tmp"
local f = io.open(tempname,"wb")
local status, message = http.request {
url = specification.original,
sink = ltn12.sink.file(f)
}
if not status then
os.remove(tempname)
else
os.remove(cachename)
os.rename(tempname,cachename)
end
return cachename
end
install('http',http_handler)
install('https') -- see pod
install('ftp')
statistics.register("scheme handling time", function()
local l, r, nl, nr = { }, { }, 0, 0
for k, v in table.sortedhash(loaded) do
if v > 0 then
nl = nl + 1
l[nl] = k .. ":" .. v
end
end
for k, v in table.sortedhash(reused) do
if v > 0 then
nr = nr + 1
r[nr] = k .. ":" .. v
end
end
local n = nl + nr
if n > 0 then
l = (nl > 0 and concat(l)) or "none"
r = (nr > 0 and concat(r)) or "none"
return format("%s seconds, %s processed, threshold %s seconds, loaded: %s, reused: %s",
statistics.elapsedtime(schemes), n, schemes.threshold, l, r)
else
return nil
end
end)
|