summaryrefslogtreecommitdiff
path: root/Master/texmf-dist/tex/context/base/mkiv/data-sch.lua
blob: 2e880e99740a33ba2e9b1071cb422ae6ee667db5 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
if not modules then modules = { } end modules ['data-sch'] = {
    version   = 1.001,
    comment   = "companion to luat-lib.mkiv",
    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
    copyright = "PRAGMA ADE / ConTeXt Development Team",
    license   = "see context related readme files"
}

local load, tonumber = load, tonumber
local gsub, concat, format = string.gsub, table.concat, string.format
local finders, openers, loaders = resolvers.finders, resolvers.openers, resolvers.loaders

local trace_schemes  = false  trackers.register("resolvers.schemes",function(v) trace_schemes = v end)
local report_schemes = logs.reporter("resolvers","schemes")

local http           = require("socket.http")
local ltn12          = require("ltn12")

if mbox then mbox = nil end -- useless and even bugged (helper overwrites lib)

local resolvers      = resolvers
local schemes        = resolvers.schemes or { }
resolvers.schemes    = schemes

local cleaners       = { }
schemes.cleaners     = cleaners

local threshold      = 24 * 60 * 60

directives.register("schemes.threshold", function(v) threshold = tonumber(v) or threshold end)

function cleaners.none(specification)
    return specification.original
end

-- function cleaners.strip(specification)
--     -- todo: only keep suffix periods, so after the last
--     return (gsub(specification.original,"[^%a%d%.]+","-")) -- so we keep periods
-- end

function cleaners.strip(specification) -- keep suffixes
    local path, name = file.splitbase(specification.original)
    if path == "" then
        return (gsub(name,"[^%a%d%.]+","-"))
    else
        return (gsub((gsub(path,"%.","-") .. "-" .. name),"[^%a%d%.]+","-"))
    end
end

function cleaners.md5(specification)
    return file.addsuffix(md5.hex(specification.original),file.suffix(specification.path))
end

local cleaner = cleaners.strip

directives.register("schemes.cleanmethod", function(v) cleaner = cleaners[v] or cleaners.strip end)

function resolvers.schemes.cleanname(specification)
    local hash = cleaner(specification)
    if trace_schemes then
        report_schemes("hashing %a to %a",specification.original,hash)
    end
    return hash
end

local cached     = { }
local loaded     = { }
local reused     = { }
local thresholds = { }
local handlers   = { }
local runner     = sandbox.registerrunner {
    name     = "curl resolver",
    method   = "execute",
    program  = "curl",
    template = "--silent --insecure --create-dirs --output %cachename% %original%",
    checkers = {
        cachename = "cache",
        original  = "url",
    }
}

local function fetch(specification)
    local original  = specification.original
    local scheme    = specification.scheme
    local cleanname = schemes.cleanname(specification)
    local cachename = caches.setfirstwritablefile(cleanname,"schemes")
    if not cached[original] then
        statistics.starttiming(schemes)
        if not io.exists(cachename) or (os.difftime(os.time(),lfs.attributes(cachename).modification) > (thresholds[protocol] or threshold)) then
            cached[original] = cachename
            local handler = handlers[scheme]
            if handler then
                if trace_schemes then
                    report_schemes("fetching %a, protocol %a, method %a",original,scheme,"built-in")
                end
                logs.flush()
                handler(specification,cachename)
            else
                if trace_schemes then
                    report_schemes("fetching %a, protocol %a, method %a",original,scheme,"curl")
                end
                logs.flush()
                runner {
                    original  = original,
                    cachename = cachename,
                }
            end
        end
        if io.exists(cachename) then
            cached[original] = cachename
            if trace_schemes then
                report_schemes("using cached %a, protocol %a, cachename %a",original,scheme,cachename)
            end
        else
            cached[original] = ""
            if trace_schemes then
                report_schemes("using missing %a, protocol %a",original,scheme)
            end
        end
        loaded[scheme] = loaded[scheme] + 1
        statistics.stoptiming(schemes)
    else
        if trace_schemes then
            report_schemes("reusing %a, protocol %a",original,scheme)
        end
        reused[scheme] = reused[scheme] + 1
    end
    return cached[original]
end

local function finder(specification,filetype)
    return resolvers.methodhandler("finders",fetch(specification),filetype)
end

local opener = openers.file
local loader = loaders.file

local function install(scheme,handler,newthreshold)
    handlers  [scheme] = handler
    loaded    [scheme] = 0
    reused    [scheme] = 0
    finders   [scheme] = finder
    openers   [scheme] = opener
    loaders   [scheme] = loader
    thresholds[scheme] = newthreshold or threshold
end

schemes.install = install

local function http_handler(specification,cachename)
    local tempname = cachename .. ".tmp"
    local f = io.open(tempname,"wb")
    local status, message = http.request {
        url  = specification.original,
        sink = ltn12.sink.file(f)
    }
    if not status then
        os.remove(tempname)
    else
        os.remove(cachename)
        os.rename(tempname,cachename)
    end
    return cachename
end

install('http',http_handler)
install('https') -- see pod
install('ftp')

statistics.register("scheme handling time", function()
    local l, r, nl, nr = { }, { }, 0, 0
    for k, v in table.sortedhash(loaded) do
        if v > 0 then
            nl = nl + 1
            l[nl] = k .. ":" .. v
        end
    end
    for k, v in table.sortedhash(reused) do
        if v > 0 then
            nr = nr + 1
            r[nr] = k .. ":" .. v
        end
    end
    local n = nl + nr
    if n > 0 then
        l = nl > 0 and concat(l) or "none"
        r = nr > 0 and concat(r) or "none"
        return format("%s seconds, %s processed, threshold %s seconds, loaded: %s, reused: %s",
            statistics.elapsedtime(schemes), n, threshold, l, r)
    else
        return nil
    end
end)

-- We provide a few more helpers:

----- http        = require("socket.http")
local httprequest = http.request
local toquery     = url.toquery

local function fetchstring(url,data)
    local q = data and toquery(data)
    if q then
        url = url .. "?" .. q
    end
    local reply = httprequest(url)
    return reply -- just one argument
end

schemes.fetchstring = fetchstring

function schemes.fetchtable(url,data)
    local reply = fetchstring(url,data)
    if reply then
        local s = load("return " .. reply)
        if s then
            return s()
        end
    end
end