summaryrefslogtreecommitdiff
path: root/Master/texmf-dist/scripts/context/lua/mtx-watch.lua
blob: 95323f571cf3f6681d3de20fed31713ea93f8c32 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
if not modules then modules = { } end modules ['mtx-watch'] = {
    version   = 1.001,
    comment   = "companion to mtxrun.lua",
    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
    copyright = "PRAGMA ADE / ConTeXt Development Team",
    license   = "see context related readme files"
}

local helpinfo = [[
<?xml version="1.0"?>
<application>
 <metadata>
  <entry name="name">mtx-watch</entry>
  <entry name="detail">ConTeXt Request Watchdog</entry>
  <entry name="version">1.00</entry>
 </metadata>
 <flags>
  <category name="basic">
   <subcategory>
    <flag name="logpath"><short>optional path for log files</short></flag>
    <flag name="watch"><short>watch given path [<ref name="delay]"/></short></flag>
    <flag name="pipe"><short>use pipe instead of execute</short></flag>
    <flag name="delay"><short>delay between sweeps</short></flag>
    <flag name="automachine"><short>replace /machine/ in path /servername/</short></flag>
    <flag name="collect"><short>condense log files</short></flag>
    <flag name="cleanup" value="delay"><short>remove files in given path [<ref name="force]"/></short></flag>
    <flag name="showlog"><short>show log data</short></flag>
   </subcategory>
  </category>
 </flags>
</application>
]]

local application = logs.application {
    name     = "mtx-watch",
    banner   = "ConTeXt Request Watchdog 1.00",
    helpinfo = helpinfo,
}

local report = application.report

scripts       = scripts       or { }
scripts.watch = scripts.watch or { }

local format, concat, difftime, time = string.format, table.concat, os.difftime, os.time
local next, type = next, type
local basename, dirname, joinname = file.basename, file.dirname, file.join
local lfsdir, lfsattributes = lfs.dir, lfs.attributes

-- the machine/instance matches the server app we use

local machine  = socket.dns.gethostname() or "unknown-machine"
local instance = string.match(machine,"(%d+)$") or "0"

function scripts.watch.save_exa_modes(joblog,ctmname)
    local values = joblog and joblog.values
    if values then
        local t= { }
        t[#t+1] = "<?xml version='1.0' standalone='yes'?>\n"
        t[#t+1] = "<exa:variables xmlns:exa='htpp://www.pragma-ade.com/schemas/exa-variables.rng'>"
        for k, v in next, joblog.values do
            t[#t+1] = format("\t<exa:variable label='%s'>%s</exa:variable>", k, tostring(v))
        end
        t[#t+1] = "</exa:variables>"
        io.savedata(ctmname,concat(t,"\n"))
    else
        os.remove(ctmname)
    end
end

local function toset(t)
    if type(t) == "table" then
        return concat(t,",")
    else
        return t
    end
end

local function noset(t)
    if type(t) == "table" then
        return t[1]
    else
        return t
    end
end

-- todo: split order (o-name.luj) and combine with atime to determine sort order.

local function glob(files,path) -- some day: sort by name (order prefix) and atime
    for name in lfsdir(path) do
        if name:find("^%.") then
            -- skip . and ..
        else
            name = path .. "/" .. name
            local a = lfsattributes(name)
            if not a then
                -- weird
            elseif a.mode == "directory" then
                if name:find("graphics$") or name:find("figures$") or name:find("resources$") then
                    -- skip these too
                else
                    glob(files,name)
                end
            elseif name:find(".%luj$") then
                local bname = basename(name)
                local dname = dirname(name)
                local order = tonumber(bname:match("^(%d+)")) or 0
                files[#files+1] = { dname, bname, order }
            end
        end
    end
end

local clock = os.gettimeofday or (socket and socket.gettime) or os.time -- we cannot trust os.clock on linux

-- local function filenamesort(a,b)
--     local fa, da = a[1], a[2]
--     local fb, db = b[1], b[2]
--     if da == db then
--         return fa < fb
--     else
--         return da < db
--     end
-- end

local function filenamesort(a,b)
    local fa, oa = a[2], a[3]
    local fb, ob = b[2], b[3]
    if fa == fb then
        if oa == ob then
            return a[1] < b[1] -- order file  dir
        else
            return oa < ob     -- order file
        end
    else
        if oa == ob then
            return fa < fb     -- order file
        else
            return oa < ob     -- order file
        end
    end
end

function scripts.watch.watch()
    local delay   = tonumber(environment.argument("delay") or 5) or 5
    if delay == 0 then
        delay = .25
    end
    local logpath = environment.argument("logpath") or ""
    local pipe    = environment.argument("pipe")    or false
    local watcher = "mtxwatch.run"
    local paths   = environment.files
    if #paths > 0 then
        if environment.argument("automachine") then
            logpath = string.gsub(logpath,"/machine/","/"..machine.."/")
            for i=1,#paths do
                paths[i] = string.gsub(paths[i],"/machine/","/"..machine.."/")
            end
        end
        for i=1,#paths do
            report("watching path %s",paths[i])
        end
        local function process()
            local done = false
            for i=1,#paths do
                local path = paths[i]
                lfs.chdir(path)
                local files = { }
                glob(files,path)
                table.sort(files,filenamesort)
--                 for name, time in next, files do
                for i=1,#files do
                    local f = files[i]
                    local dirname = f[1]
                    local basename = f[2] -- we can use that later on
                    local name = joinname(dirname,basename)
                --~ local ok, joblog = xpcall(function() return dofile(name) end, function() end )
                    local ok, joblog = pcall(dofile,name)
                    if ok and joblog then
                        if joblog.status == "processing" then
                            report("aborted job, %s added to queue",name)
                            joblog.status = "queued"
                            io.savedata(name, table.serialize(joblog,true))
                        elseif joblog.status == "queued" then
                            local command = joblog.command
                            if command then
                                local replacements = {
                                    inputpath  = toset((joblog.paths and joblog.paths.input ) or "."),
                                    outputpath = noset((joblog.paths and joblog.paths.output) or "."),
                                    filename   = joblog.filename or "",
                                }
                                -- todo: revision path etc
                                command = command:gsub("%%(.-)%%", replacements)
                                if command ~= "" then
                                    joblog.status = "processing"
                                    joblog.runtime = clock()
                                    io.savedata(name, table.serialize(joblog,true))
                                    report("running: %s", command)
                                    local newpath = file.dirname(name)
                                    io.flush()
                                    local result = ""
                                    local ctmname = file.basename(replacements.filename)
                                    if ctmname == "" then ctmname = name end -- use self as fallback
                                    ctmname = file.replacesuffix(ctmname,"ctm")
                                    if newpath ~= "" and newpath ~= "." then
                                        local oldpath = lfs.currentdir()
                                        lfs.chdir(newpath)
                                        scripts.watch.save_exa_modes(joblog,ctmname)
                                        if pipe then result = os.resultof(command) else result = os.spawn(command) end
                                        lfs.chdir(oldpath)
                                    else
                                        scripts.watch.save_exa_modes(joblog,ctmname)
                                        if pipe then result = os.resultof(command) else result = os.spawn(command) end
                                    end
                                    report("return value: %s", result)
                                    done = true
                                    local path, base = replacements.outputpath, file.basename(replacements.filename)
                                    joblog.runtime = clock() - joblog.runtime
                                    if base ~= "" then
                                        joblog.result = file.replacesuffix(file.join(path,base),"pdf")
                                        joblog.size   = lfs.attributes(joblog.result,"size")
                                    end
                                    joblog.status = "finished"
                                else
                                    joblog.status = "invalid command"
                                end
                            else
                                joblog.status = "no command"
                            end
                            -- pcall, when error sleep + again
                            -- todo: just one log file and append
                            io.savedata(name, table.serialize(joblog,true))
                            if logpath and logpath ~= "" then
                                local name = file.join(logpath,os.uuid() .. ".lua")
                                io.savedata(name, table.serialize(joblog,true))
                                report("saving joblog in %s",name)
                            end
                        end
                    end
                end
            end
        end
        local n, start = 0, time()
        local wtime = 0
        local function wait()
            io.flush()
            if not done then
                n = n + 1
                if n >= 10 then
                    report("run time: %i seconds, memory usage: %0.3g MB", difftime(time(),start), (status.luastate_bytes/1024)/1000)
                    n = 0
                end
                local ttime = 0
                while ttime <= delay do
                    local wt = lfs.attributes(watcher,"mtime")
                    if wt and wt ~= wtime then
                        -- fast signal that there is a request
                        wtime = wt
                        break
                    end
                    ttime = ttime + 0.2
                    os.sleep(0.2)
                end
            end
        end
        local cleanupdelay, cleanup = environment.argument("cleanup"), false
        if cleanupdelay then
            local lasttime = time()
            cleanup = function()
                local currenttime = time()
                local delta = difftime(currenttime,lasttime)
                if delta > cleanupdelay then
                    lasttime = currenttime
                    for i=1,#paths do
                        local path = paths[i]
                        if string.find(path,"%.") then
                            -- safeguard, we want a fully qualified path
                        else
                            local files = dir.glob(file.join(path,"*"))
                            for i=1,#files do
                                local name = files[i]
                                local filetime = lfs.attributes(name,"modification")
                                local delta = difftime(currenttime,filetime)
                                if delta > cleanupdelay then
                                 -- report("cleaning up '%s'",name)
                                    os.remove(name)
                                end
                            end
                        end
                    end
                end
            end
        else
            cleanup = function()
                -- nothing
            end
        end
        while true do
            if false then
--~             if true then
                process()
                cleanup()
                wait()
            else
                pcall(process)
                pcall(cleanup)
                pcall(wait)
            end
        end
    else
        report("no paths to watch")
    end
end

function scripts.watch.collect_logs(path) -- clean 'm up too
    path = path or environment.argument("logpath") or ""
    path = (path == "" and ".") or path
    local files = dir.globfiles(path,false,"^%d+%.lua$")
    local collection = { }
    local valid = table.tohash({"filename","result","runtime","size","status"})
    for i=1,#files do
        local name = files[i]
        local t = dofile(name)
        if t and type(t) == "table" and t.status then
            for k, v in next, t do
                if not valid[k] then
                    t[k] = nil
                end
            end
            collection[name:gsub("[^%d]","")] = t
        end
    end
    return collection
end

function scripts.watch.save_logs(collection,path) -- play safe
    if collection and next(collection) then
        path = path or environment.argument("logpath") or ""
        path = (path == "" and ".") or path
        local filename = format("%s/collected-%s.lua",path,tostring(time()))
        io.savedata(filename,table.serialize(collection,true))
        local check = dofile(filename)
        for k,v in next, check do
            if not collection[k] then
                report("error in saving file")
                os.remove(filename)
                return false
            end
        end
        for k,v in next, check do
            os.remove(format("%s.lua",k))
        end
        return true
    else
        return false
    end
end

function scripts.watch.collect_collections(path) -- removes duplicates
    path = path or environment.argument("logpath") or ""
    path = (path == "" and ".") or path
    local files = dir.globfiles(path,false,"^collected%-%d+%.lua$")
    local collection = { }
    for i=1,#files do
        local name = files[i]
        local t = dofile(name)
        if t and type(t) == "table" then
            for k, v in next, t do
                collection[k] = v
            end
        end
    end
    return collection
end

function scripts.watch.show_logs(path) -- removes duplicates
    local collection = scripts.watch.collect_collections(path) or { }
    local max = 0
    for k,v in next, collection do
        v = v.filename or "?"
        if #v > max then max = #v end
    end
 -- print(max)
    local sorted = table.sortedkeys(collection)
    for k=1,#sorted do
        local v = sorted[k]
        local c = collection[v]
        local f, s, r, n = c.filename or "?", c.status or "?", c.runtime or 0, c.size or 0
        report("%s  %s  %3i  %8i  %s",string.padd(f,max," "),string.padd(s,10," "),r,n,v)
    end
end

function scripts.watch.cleanup_stale_files() -- removes duplicates
    local path  = environment.files[1]
    local delay = tonumber(environment.argument("cleanup"))
    local force = environment.argument("force")
    if not path or path == "." then
        report("provide qualified path")
    elseif not delay then
        report("missing --cleanup=delay")
    else
        report("dryrun, use --force for real cleanup")
        local files = dir.glob(file.join(path,"*"))
        local rtime = time()
        for i=1,#files do
            local name = files[i]
            local mtime = lfs.attributes(name,"modification")
            local delta = difftime(rtime,mtime)
            if delta > delay then
                report("cleaning up '%s'",name)
                if force then
                    os.remove(name)
                end
            end
        end
    end
end

if environment.argument("watch") then
    scripts.watch.watch()
elseif environment.argument("collect") then
    scripts.watch.save_logs(scripts.watch.collect_logs())
elseif environment.argument("cleanup") then
    scripts.watch.save_logs(scripts.watch.cleanup_stale_files())
elseif environment.argument("showlog") then
    scripts.watch.show_logs()
elseif environment.argument("exporthelp") then
    application.export(environment.argument("exporthelp"),environment.files[1])
else
    application.help()
end