diff options
Diffstat (limited to 'Master/texmf-dist/scripts/context/lua/mtx-watch.lua')
-rw-r--r-- | Master/texmf-dist/scripts/context/lua/mtx-watch.lua | 401 |
1 files changed, 261 insertions, 140 deletions
diff --git a/Master/texmf-dist/scripts/context/lua/mtx-watch.lua b/Master/texmf-dist/scripts/context/lua/mtx-watch.lua index 2e4dcf6efec..10f01cf8698 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-watch.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-watch.lua @@ -9,160 +9,247 @@ if not modules then modules = { } end modules ['mtx-watch'] = { scripts = scripts or { } scripts.watch = scripts.watch or { } -do +local format, concat, difftime, time = string.format, table.concat, os.difftime, os.time +local next, type = next, type - function scripts.watch.save_exa_modes(joblog,ctmname) +-- the machine/instance matches the server app we use + +local machine = socket.dns.gethostname() or "unknown-machine" +local instance = string.match(machine,"(%d+)$") or "0" + +function scripts.watch.save_exa_modes(joblog,ctmname) + local values = joblog and joblog.values + if values then local t= { } - if joblog then - t[#t+1] = "<?xml version='1.0' standalone='yes'?>\n" - t[#t+1] = "<exa:variables xmlns:exa='htpp://www.pragma-ade.com/schemas/exa-variables.rng'>" - if joblog.values then - for k, v in pairs(joblog.values) do - t[#t+1] = string.format("\t<exa:variable label='%s'>%s</exa:variable>", k, tostring(v)) + t[#t+1] = "<?xml version='1.0' standalone='yes'?>\n" + t[#t+1] = "<exa:variables xmlns:exa='htpp://www.pragma-ade.com/schemas/exa-variables.rng'>" + for k, v in next, joblog.values do + t[#t+1] = format("\t<exa:variable label='%s'>%s</exa:variable>", k, tostring(v)) + end + t[#t+1] = "</exa:variables>" + io.savedata(ctmname,concat(t,"\n")) + else + os.remove(ctmname) + end +end + +local function toset(t) + if type(t) == "table" then + return concat(t,",") + else + return t + end +end + +local function noset(t) + if type(t) == "table" then + return t[1] + else + return t + end +end + +local lfsdir, lfsattributes = lfs.dir, lfs.attributes + +local function glob(files,path) + for name in lfsdir(path) do + if name:find("^%.") then + -- skip . and .. + else + name = path .. "/" .. name + local a = lfsattributes(name) + if not a then + -- weird + elseif a.mode == "directory" then + if name:find("graphics$") or name:find("figures$") or name:find("resources$") then + -- skip these too + else + glob(files,name) end - else - t[#t+1] = "<!-- no modes -->" + elseif name:find(".%luj$") then + files[name] = a.change or a.ctime or a.modification or a.mtime end - t[#t+1] = "</exa:variables>" end - os.remove(ctmname) - io.savedata(ctmname,table.concat(t,"\n")) end +end - function scripts.watch.watch() - local delay = environment.argument("delay") or 5 - local logpath = environment.argument("logpath") or "" - local pipe = environment.argument("pipe") or false - if #environment.files > 0 then - for _, path in ipairs(environment.files) do - logs.report("watch", "watching path ".. path) +local clock = os.gettimeofday or os.time -- we cannot trust os.clock on linux + +function scripts.watch.watch() + local delay = tonumber(environment.argument("delay") or 5) or 5 + if delay == 0 then + delay = .25 + end + local logpath = environment.argument("logpath") or "" + local pipe = environment.argument("pipe") or false + local watcher = "mtxwatch.run" + local paths = environment.files + if #paths > 0 then + if environment.argument("automachine") then + logpath = string.gsub(logpath,"/machine/","/"..machine.."/") + for i=1,#paths do + paths[i] = string.gsub(paths[i],"/machine/","/"..machine.."/") end - local function glob(files,path) - for name in lfs.dir(path) do - if name:find("^%.") then - -- skip . and .. - else - name = path .. "/" .. name - local a = lfs.attributes(name) - if not a then - -- weird - elseif a.mode == "directory" then - if name:find("graphics$") or name:find("figures$") or name:find("resources$") then - -- skip these too + end + for i=1,#paths do + logs.report("watch", "watching path ".. paths[i]) + end + local function process() + local done = false + for i=1,#paths do + local path = paths[i] + lfs.chdir(path) + local files = { } + glob(files,path) + table.sort(files) -- what gets sorted here, todo: by time + for name, time in next, files do + --~ local ok, joblog = xpcall(function() return dofile(name) end, function() end ) + local ok, joblog = pcall(dofile,name) + if ok and joblog then + if joblog.status == "processing" then + logs.report("watch",format("aborted job, %s added to queue",name)) + joblog.status = "queued" + io.savedata(name, table.serialize(joblog,true)) + elseif joblog.status == "queued" then + local command = joblog.command + if command then + local replacements = { + inputpath = toset((joblog.paths and joblog.paths.input ) or "."), + outputpath = noset((joblog.paths and joblog.paths.output) or "."), + filename = joblog.filename or "", + } + -- todo: revision path etc + command = command:gsub("%%(.-)%%", replacements) + if command ~= "" then + joblog.status = "processing" + joblog.runtime = clock() + io.savedata(name, table.serialize(joblog,true)) + logs.report("watch",format("running: %s", command)) + local newpath = file.dirname(name) + io.flush() + local result = "" + local ctmname = file.basename(replacements.filename) + if ctmname == "" then ctmname = name end -- use self as fallback + ctmname = file.replacesuffix(ctmname,"ctm") + if newpath ~= "" and newpath ~= "." then + local oldpath = lfs.currentdir() + lfs.chdir(newpath) + scripts.watch.save_exa_modes(joblog,ctmname) + if pipe then result = os.resultof(command) else result = os.spawn(command) end + lfs.chdir(oldpath) + else + scripts.watch.save_exa_modes(joblog,ctmname) + if pipe then result = os.resultof(command) else result = os.spawn(command) end + end + logs.report("watch",format("return value: %s", result)) + done = true + local path, base = replacements.outputpath, file.basename(replacements.filename) + joblog.runtime = clock() - joblog.runtime + if base ~= "" then + joblog.result = file.replacesuffix(file.join(path,base),"pdf") + joblog.size = lfs.attributes(joblog.result,"size") + end + joblog.status = "finished" + else + joblog.status = "invalid command" + end else - glob(files,name) + joblog.status = "no command" + end + -- pcall, when error sleep + again + -- todo: just one log file and append + io.savedata(name, table.serialize(joblog,true)) + if logpath and logpath ~= "" then + local name = file.join(logpath,os.uuid() .. ".lua") + io.savedata(name, table.serialize(joblog,true)) + logs.report("watch", "saving joblog in " .. name) end - elseif name:find(".%luj$") then - files[name] = a.change or a.ctime or a.modification or a.mtime end end end end - local function toset(t) - if type(t) == "table" then - return table.concat(t,",") - else - return t + end + local n, start = 0, time() +--~ local function wait() +--~ io.flush() +--~ if not done then +--~ n = n + 1 +--~ if n >= 10 then +--~ logs.report("watch", format("run time: %i seconds, memory usage: %0.3g MB", difftime(time(),start), (status.luastate_bytes/1024)/1000)) +--~ n = 0 +--~ end +--~ os.sleep(delay) +--~ end +--~ end + local wtime = 0 + local function wait() + io.flush() + if not done then + n = n + 1 + if n >= 10 then + logs.report("watch", format("run time: %i seconds, memory usage: %0.3g MB", difftime(time(),start), (status.luastate_bytes/1024)/1000)) + n = 0 end - end - local function noset(t) - if type(t) == "table" then - return t[1] - else - return t + local ttime = 0 + while ttime <= delay do + local wt = lfs.attributes(watcher,"mtime") + if wt and wt ~= wtime then + -- fast signal that there is a request + wtime = wt + break + end + ttime = ttime + 0.2 + os.sleep(0.2) end end - local function process() - local done = false - for _, path in ipairs(environment.files) do - lfs.chdir(path) - local files = { } - glob(files,path) - table.sort(files) -- what gets sorted here - for name, time in pairs(files) do - --~ local ok, joblog = xpcall(function() return dofile(name) end, function() end ) - local ok, joblog = pcall(dofile,name) - if ok and joblog then - if joblog.status == "processing" then - logs.report("watch",string.format("aborted job, %s added to queue",name)) - joblog.status = "queued" - io.savedata(name, table.serialize(joblog,true)) - elseif joblog.status == "queued" then - local command = joblog.command - if command then - local replacements = { - inputpath = toset((joblog.paths and joblog.paths.input ) or "."), - outputpath = noset((joblog.paths and joblog.paths.output) or "."), - filename = joblog.filename or "", - } - command = command:gsub("%%(.-)%%", replacements) - if command ~= "" then - joblog.status = "processing" - joblog.runtime = os.time() -- os.clock() - io.savedata(name, table.serialize(joblog,true)) - logs.report("watch",string.format("running: %s", command)) - local newpath = file.dirname(name) - io.flush() - local result = "" - local ctmname = file.basename(replacements.filename) - if ctmname == "" then ctmname = name end -- use self as fallback - ctmname = file.replacesuffix(ctmname,"ctm") - if newpath ~= "" and newpath ~= "." then - local oldpath = lfs.currentdir() - lfs.chdir(newpath) - scripts.watch.save_exa_modes(joblog,ctmname) - if pipe then result = os.resultof(command) else result = os.spawn(command) end - lfs.chdir(oldpath) - else - scripts.watch.save_exa_modes(joblog,ctmname) - if pipe then result = os.resultof(command) else result = os.spawn(command) end - end - logs.report("watch",string.format("return value: %s", result)) - done = true - local path, base = replacements.outputpath, file.basename(replacements.filename) - joblog.runtime = os.time() - joblog.runtime -- os.clock() - joblog.runtime - joblog.result = file.replacesuffix(file.join(path,base),"pdf") - joblog.size = lfs.attributes(joblog.result,"size") - joblog.status = "finished" - else - joblog.status = "invalid command" - end - else - joblog.status = "no command" - end - -- pcall, when error sleep + again - io.savedata(name, table.serialize(joblog,true)) - if logpath ~= "" then - local name = string.format("%s/%s%04i%09i.lua", logpath, os.time(), math.floor((os.clock()*100)%1000), math.random(99999999)) - io.savedata(name, table.serialize(joblog,true)) - logs.report("watch", "saving joblog ".. name) + end + local cleanupdelay, cleanup = environment.argument("cleanup"), false + if cleanupdelay then + local lasttime = time() + cleanup = function() + local currenttime = time() + local delta = difftime(currenttime,lasttime) + if delta > cleanupdelay then + lasttime = currenttime + for i=1,#paths do + local path = paths[i] + if string.find(path,"%.") then + -- safeguard, we want a fully qualified path + else + local files = dir.glob(file.join(path,"*")) + for i=1,#files do + local name = files[i] + local filetime = lfs.attributes(name,"modification") + local delta = difftime(currenttime,filetime) + if delta > cleanupdelay then + -- logs.report("watch",format("cleaning up '%s'",name)) + os.remove(name) end end end end end end - local n, start = 0, os.clock() - local function wait() - io.flush() - if not done then - n = n + 1 - if n >= 10 then - logs.report("watch", string.format("run time: %i seconds, memory usage: %0.3g MB", os.clock() - start, (status.luastate_bytes/1024)/1000)) - n = 0 - end - os.sleep(delay) - end + else + cleanup = function() + -- nothing end - while true do + end + while true do + if false then +--~ if true then + process() + cleanup() + wait() + else pcall(process) + pcall(cleanup) pcall(wait) end - else - logs.report("watch", "no paths to watch") end + else + logs.report("watch", "no paths to watch") end - end function scripts.watch.collect_logs(path) -- clean 'm up too @@ -171,10 +258,11 @@ function scripts.watch.collect_logs(path) -- clean 'm up too local files = dir.globfiles(path,false,"^%d+%.lua$") local collection = { } local valid = table.tohash({"filename","result","runtime","size","status"}) - for _, name in ipairs(files) do + for i=1,#files do + local name = files[i] local t = dofile(name) if t and type(t) == "table" and t.status then - for k, v in pairs(t) do + for k, v in next, t do if not valid[k] then t[k] = nil end @@ -186,21 +274,21 @@ function scripts.watch.collect_logs(path) -- clean 'm up too end function scripts.watch.save_logs(collection,path) -- play safe - if collection and not table.is_empty(collection) then + if collection and next(collection) then path = path or environment.argument("logpath") or "" path = (path == "" and ".") or path - local filename = string.format("%s/collected-%s.lua",path,tostring(os.time())) + local filename = format("%s/collected-%s.lua",path,tostring(time())) io.savedata(filename,table.serialize(collection,true)) local check = dofile(filename) - for k,v in pairs(check) do + for k,v in next, check do if not collection[k] then logs.error("watch", "error in saving file") os.remove(filename) return false end end - for k,v in pairs(check) do - os.remove(string.format("%s.lua",k)) + for k,v in next, check do + os.remove(format("%s.lua",k)) end return true else @@ -213,10 +301,11 @@ function scripts.watch.collect_collections(path) -- removes duplicates path = (path == "" and ".") or path local files = dir.globfiles(path,false,"^collected%-%d+%.lua$") local collection = { } - for _, name in ipairs(files) do + for i=1,#files do + local name = files[i] local t = dofile(name) if t and type(t) == "table" then - for k, v in pairs(t) do + for k, v in next, t do collection[k] = v end end @@ -227,26 +316,56 @@ end function scripts.watch.show_logs(path) -- removes duplicates local collection = scripts.watch.collect_collections(path) or { } local max = 0 - for k,v in pairs(collection) do + for k,v in next, collection do v = v.filename or "?" if #v > max then max = #v end end - print(max) - for k,v in ipairs(table.sortedkeys(collection)) do + -- print(max) + local sorted = table.sortedkeys(collection) + for k=1,#sorted do + local v = sorted[k] local c = collection[v] local f, s, r, n = c.filename or "?", c.status or "?", c.runtime or 0, c.size or 0 - logs.report("watch", string.format("%s %s %3i %8i %s",string.padd(f,max," "),string.padd(s,10," "),r,n,v)) + logs.report("watch", format("%s %s %3i %8i %s",string.padd(f,max," "),string.padd(s,10," "),r,n,v)) + end +end + +function scripts.watch.cleanup_stale_files() -- removes duplicates + local path = environment.files[1] + local delay = tonumber(environment.argument("cleanup")) + local force = environment.argument("force") + if not path or path == "." then + logs.report("watch","provide qualified path") + elseif not delay then + logs.report("watch","missing --cleanup=delay") + else + logs.report("watch","dryrun, use --force for real cleanup") + local files = dir.glob(file.join(path,"*")) + local rtime = time() + for i=1,#files do + local name = files[i] + local mtime = lfs.attributes(name,"modification") + local delta = difftime(rtime,mtime) + if delta > delay then + logs.report("watch",format("cleaning up '%s'",name)) + if force then + os.remove(name) + end + end + end end end -logs.extendbanner("Watchdog 1.00",true) +logs.extendbanner("ConTeXt Request Watchdog 1.00",true) messages.help = [[ --logpath optional path for log files ---watch watch given path +--watch watch given path [--delay] --pipe use pipe instead of execute --delay delay between sweeps +--automachine replace /machine/ in path /<servername>/ --collect condense log files +--cleanup=delay remove files in given path [--force] --showlog show log data ]] @@ -254,6 +373,8 @@ if environment.argument("watch") then scripts.watch.watch() elseif environment.argument("collect") then scripts.watch.save_logs(scripts.watch.collect_logs()) +elseif environment.argument("cleanup") then + scripts.watch.save_logs(scripts.watch.cleanup_stale_files()) elseif environment.argument("showlog") then scripts.watch.show_logs() else |