1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
|
if not modules then modules = { } end modules ['data-sch'] = {
version = 1.001,
comment = "companion to luat-lib.mkiv",
author = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
copyright = "PRAGMA ADE / ConTeXt Development Team",
license = "see context related readme files"
}
local load, tonumber = load, tonumber
local gsub, concat, format = string.gsub, table.concat, string.format
local finders, openers, loaders = resolvers.finders, resolvers.openers, resolvers.loaders
local trace_schemes = false trackers.register("resolvers.schemes",function(v) trace_schemes = v end)
local report_schemes = logs.reporter("resolvers","schemes")
local http = require("socket.http")
local ltn12 = require("ltn12")
if mbox then mbox = nil end -- useless and even bugged (helper overwrites lib)
local resolvers = resolvers
local schemes = resolvers.schemes or { }
resolvers.schemes = schemes
local cleaners = { }
schemes.cleaners = cleaners
local threshold = 24 * 60 * 60
directives.register("schemes.threshold", function(v) threshold = tonumber(v) or threshold end)
function cleaners.none(specification)
return specification.original
end
-- function cleaners.strip(specification)
-- -- todo: only keep suffix periods, so after the last
-- return (gsub(specification.original,"[^%a%d%.]+","-")) -- so we keep periods
-- end
function cleaners.strip(specification) -- keep suffixes
local path, name = file.splitbase(specification.original)
if path == "" then
return (gsub(name,"[^%a%d%.]+","-"))
else
return (gsub((gsub(path,"%.","-") .. "-" .. name),"[^%a%d%.]+","-"))
end
end
function cleaners.md5(specification)
return file.addsuffix(md5.hex(specification.original),file.suffix(specification.path))
end
local cleaner = cleaners.strip
directives.register("schemes.cleanmethod", function(v) cleaner = cleaners[v] or cleaners.strip end)
function resolvers.schemes.cleanname(specification)
local hash = cleaner(specification)
if trace_schemes then
report_schemes("hashing %a to %a",specification.original,hash)
end
return hash
end
local cached = { }
local loaded = { }
local reused = { }
local thresholds = { }
local handlers = { }
local runner = sandbox.registerrunner {
name = "curl resolver",
method = "execute",
program = "curl",
template = "--silent --insecure --create-dirs --output %cachename% %original%",
checkers = {
cachename = "cache",
original = "url",
}
}
local function fetch(specification)
local original = specification.original
local scheme = specification.scheme
local cleanname = schemes.cleanname(specification)
local cachename = caches.setfirstwritablefile(cleanname,"schemes")
if not cached[original] then
statistics.starttiming(schemes)
if not io.exists(cachename) or (os.difftime(os.time(),lfs.attributes(cachename).modification) > (thresholds[protocol] or threshold)) then
cached[original] = cachename
local handler = handlers[scheme]
if handler then
if trace_schemes then
report_schemes("fetching %a, protocol %a, method %a",original,scheme,"built-in")
end
logs.flush()
handler(specification,cachename)
else
if trace_schemes then
report_schemes("fetching %a, protocol %a, method %a",original,scheme,"curl")
end
logs.flush()
runner {
original = original,
cachename = cachename,
}
end
end
if io.exists(cachename) then
cached[original] = cachename
if trace_schemes then
report_schemes("using cached %a, protocol %a, cachename %a",original,scheme,cachename)
end
else
cached[original] = ""
if trace_schemes then
report_schemes("using missing %a, protocol %a",original,scheme)
end
end
loaded[scheme] = loaded[scheme] + 1
statistics.stoptiming(schemes)
else
if trace_schemes then
report_schemes("reusing %a, protocol %a",original,scheme)
end
reused[scheme] = reused[scheme] + 1
end
return cached[original]
end
local function finder(specification,filetype)
return resolvers.methodhandler("finders",fetch(specification),filetype)
end
local opener = openers.file
local loader = loaders.file
local function install(scheme,handler,newthreshold)
handlers [scheme] = handler
loaded [scheme] = 0
reused [scheme] = 0
finders [scheme] = finder
openers [scheme] = opener
loaders [scheme] = loader
thresholds[scheme] = newthreshold or threshold
end
schemes.install = install
local function http_handler(specification,cachename)
local tempname = cachename .. ".tmp"
local f = io.open(tempname,"wb")
local status, message = http.request {
url = specification.original,
sink = ltn12.sink.file(f)
}
if not status then
os.remove(tempname)
else
os.remove(cachename)
os.rename(tempname,cachename)
end
return cachename
end
install('http',http_handler)
install('https') -- see pod
install('ftp')
statistics.register("scheme handling time", function()
local l, r, nl, nr = { }, { }, 0, 0
for k, v in table.sortedhash(loaded) do
if v > 0 then
nl = nl + 1
l[nl] = k .. ":" .. v
end
end
for k, v in table.sortedhash(reused) do
if v > 0 then
nr = nr + 1
r[nr] = k .. ":" .. v
end
end
local n = nl + nr
if n > 0 then
l = nl > 0 and concat(l) or "none"
r = nr > 0 and concat(r) or "none"
return format("%s seconds, %s processed, threshold %s seconds, loaded: %s, reused: %s",
statistics.elapsedtime(schemes), n, threshold, l, r)
else
return nil
end
end)
-- We provide a few more helpers:
----- http = require("socket.http")
local httprequest = http.request
local toquery = url.toquery
local function fetchstring(url,data)
local q = data and toquery(data)
if q then
url = url .. "?" .. q
end
local reply = httprequest(url)
return reply -- just one argument
end
schemes.fetchstring = fetchstring
function schemes.fetchtable(url,data)
local reply = fetchstring(url,data)
if reply then
local s = load("return " .. reply)
if s then
return s()
end
end
end
|
