1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
|
if not modules then modules = { } end modules ['core-uti'] = {
version = 1.001,
comment = "companion to core-uti.mkiv",
author = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
copyright = "PRAGMA ADE / ConTeXt Development Team",
license = "see context related readme files"
}
-- todo: keep track of changes here (hm, track access, and only true when
-- accessed and changed)
--[[ldx--
<p>A utility file has always been part of <l n='context'/> and with
the move to <l n='luatex'/> we also moved a lot of multi-pass info
to a <l n='lua'/> table. Instead of loading a <l n='tex'/> based
utility file under different setups, we now load a table once. This
saves much runtime but at the cost of more memory usage.</p>
--ldx]]--
local math = math
local format, match = string.format, string.match
local next, type, tostring, tonumber = next, type, tostring, tonumber
local concat = table.concat
local definetable = utilities.tables.definetable
local accesstable = utilities.tables.accesstable
local migratetable = utilities.tables.migratetable
local serialize = table.serialize
local packers = utilities.packers
local allocate = utilities.storage.allocate
local mark = utilities.storage.mark
local getrandom = utilities.randomizer.get
local setrandomseedi = utilities.randomizer.setseedi
local getrandomseed = utilities.randomizer.getseed
local implement = interfaces.implement
local texgetcount = tex.getcount
local report_passes = logs.reporter("job","passes")
job = job or { }
local job = job
job.version = 1.32
job.packversion = 1.02
-- some day we will implement loading of other jobs and then we need
-- job.jobs
--[[ldx--
<p>Variables are saved using in the previously defined table and passed
onto <l n='tex'/> using the following method. Of course one can also
directly access the variable using a <l n='lua'/> call.</p>
--ldx]]--
local savelist, comment = { }, { }
function job.comment(key,value)
if type(key) == "table" then
for k, v in next, key do
comment[k] = v
end
else
comment[key] = value
end
end
job.comment("version",job.version)
local enabled = true
local initialized = false
directives.register("job.save",function(v) enabled = v end)
function job.disablesave()
enabled = false -- for instance called when an error
end
function job.initialize(loadname,savename)
if not initialized then
if not loadname or loadname == "" then
loadname = tex.jobname .. ".tuc"
end
if not savename or savename == "" then
savename = tex.jobname .. ".tua"
end
job.load(loadname) -- has to come after structure is defined !
luatex.registerstopactions(function()
if enabled then
job.save(savename)
end
end)
initialized = true
end
end
function job.register(collected, tobesaved, initializer, finalizer, serializer)
savelist[#savelist+1] = { collected, tobesaved, initializer, finalizer, serializer }
end
-- as an example we implement variables
local tobesaved, collected, checksums = allocate(), allocate(), allocate()
local jobvariables = {
collected = collected,
tobesaved = tobesaved,
checksums = checksums,
}
-- if not checksums.old then checksums.old = md5.HEX("old") end -- used in experiment
-- if not checksums.new then checksums.new = md5.HEX("new") end -- used in experiment
job.variables = jobvariables
local function initializer()
checksums = jobvariables.checksums
end
job.register('job.variables.checksums', 'job.variables.checksums', initializer)
local rmethod, rvalue
local collectedmacros, tobesavedmacros
local ctx_setxvalue = context.setxvalue
local function initializer()
tobesaved = jobvariables.tobesaved
collected = jobvariables.collected
--
rvalue = collected.randomseed
if not rvalue then
rvalue = getrandom("initialize")
setrandomseedi(rvalue)
rmethod = "initialized"
else
setrandomseedi(rvalue)
rmethod = "resumed"
end
tobesaved.randomseed = rvalue
--
collectedmacros = collected.macros
tobesavedmacros = tobesaved.macros
if not collectedmacros then
collectedmacros = { }
collected.macros = collectedmacros
end
if not tobesavedmacros then
tobesavedmacros = { }
tobesaved.macros = tobesavedmacros
end
-- will become collected.macros
for cs, value in next, collectedmacros do
if type(value) == "string" then -- safeguard
ctx_setxvalue(cs,value)
end
end
end
job.register('job.variables.collected', tobesaved, initializer)
function jobvariables.save(cs,value)
tobesavedmacros[cs] = value
end
function jobvariables.restore(cs)
return collectedmacros[cs] or tobesavedmacros[cs]
end
function job.getrandomseed()
return tobesaved.randomseed or getrandomseed()
end
-- checksums
function jobvariables.getchecksum(tag)
return checksums[tag] -- no default
end
function jobvariables.makechecksum(data)
return data and md5.HEX(data) -- no default
end
function jobvariables.setchecksum(tag,checksum)
checksums[tag] = checksum
end
--
local packlist = {
"numbers",
"ownnumbers",
"metadata",
"sectiondata",
"prefixdata",
"numberdata",
"pagedata",
"directives",
"specification",
"processors", -- might become key under directives or metadata
-- "references", -- we need to rename of them as only one packs (not structures.lists.references)
}
local skiplist = {
"datasets",
"userdata",
"positions",
}
-- not ok as we can have arbitrary keys in userdata and dataset so some day we
-- might need a bit more granularity, like skippers
local jobpacker = packers.new(packlist,job.packversion,skiplist) -- jump number when changs in hash
job.pack = true
-- job.pack = false
directives.register("job.pack",function(v) job.pack = v end)
local _save_, _load_, _others_ = { }, { }, { } -- registers timing
function job.save(filename) -- we could return a table but it can get pretty large
statistics.starttiming(_save_)
local f = io.open(filename,'w')
if f then
f:write("local utilitydata = { }\n\n")
f:write(serialize(comment,"utilitydata.comment",true),"\n\n")
for l=1,#savelist do
-- f:write("do\n\n") -- no solution for the jit limitatione either
local list = savelist[l]
local target = format("utilitydata.%s",list[1])
local data = list[2]
local finalizer = list[4]
local serializer = list[5]
if type(data) == "string" then
data = utilities.tables.accesstable(data)
end
if type(finalizer) == "function" then
finalizer()
end
if job.pack then
packers.pack(data,jobpacker,true)
end
local definer, name = definetable(target,true,true) -- no first and no last
if serializer then
f:write(definer,"\n\n",serializer(data,name,true),"\n\n")
else
f:write(definer,"\n\n",serialize(data,name,true),"\n\n")
end
-- f:write("end\n\n")
end
if job.pack then
packers.strip(jobpacker)
-- f:write("do\n\n")
f:write(serialize(jobpacker,"utilitydata.job.packed",true),"\n\n")
-- f:write("end\n\n")
end
f:write("return utilitydata")
f:close()
end
statistics.stoptiming(_save_)
end
local function load(filename)
if lfs.isfile(filename) then
local function dofile(filename)
local result = loadstring(io.loaddata(filename))
if result then
return result()
else
return nil
end
end
local okay, data = pcall(dofile,filename)
if okay and type(data) == "table" then
local jobversion = job.version
local datacomment = data.comment
local dataversion = datacomment and datacomment.version or "?"
if dataversion ~= jobversion then
report_passes("version mismatch: %s <> %s",dataversion,jobversion)
else
return data
end
else
os.remove(filename) -- probably a bad file (or luajit overflow as it cannot handle large tables well)
report_passes("removing stale job data file %a, restart job, message: %s%s",filename,tostring(data),
jit and " (try luatex instead of luajittex)" or "")
os.exit(true) -- trigger second run
end
end
end
function job.load(filename)
statistics.starttiming(_load_)
local utilitydata = load(filename)
if utilitydata then
local jobpacker = utilitydata.job.packed
local handlers = { }
for i=1,#savelist do
local list = savelist[i]
local target = list[1]
local initializer = list[3]
local result = accesstable(target,utilitydata)
if result then
local done = packers.unpack(result,jobpacker,true)
if done then
migratetable(target,mark(result))
if type(initializer) == "function" then
handlers[#handlers+1] = { initializer, result }
end
else
report_passes("pack version mismatch")
end
end
end
-- so we have all tables available (unpacked)
for i=1,#handlers do
local handler = handlers[i]
handler[1](handler[2])
end
end
statistics.stoptiming(_load_)
end
function job.loadother(filename)
statistics.starttiming(_load_)
_others_[#_others_+1] = file.nameonly(filename)
local utilitydata = load(filename)
if utilitydata then
local jobpacker = utilitydata.job.packed
local unpacked = { }
for l=1,#savelist do
local list = savelist[l]
local target = list[1]
local result = accesstable(target,utilitydata)
local done = packers.unpack(result,jobpacker,true)
if done then
migratetable(target,result,unpacked)
end
end
unpacked.job.packed = nil -- nicer in inspecting
return unpacked
end
statistics.stoptiming(_load_)
end
-- function job.keep(filename)
-- local suffix = file.suffix(filename)
-- local base = file.removesuffix(filename)
-- if suffix == "" then
-- suffix = "tuc"
-- end
-- for i=1,10 do
-- local tmpname = format("%s-%s-%02d.tmp",base,suffix,i)
-- if lfs.isfile(tmpname) then
-- os.remove(tmpname)
-- report_passes("removing %a",tmpname)
-- end
-- end
-- if lfs.isfile(filename) then
-- local tmpname = format("%s-%s-%02d.tmp",base,suffix,environment.currentrun or 1)
-- report_passes("copying %a into %a",filename,tmpname)
-- file.copy(filename,tmpname)
-- else
-- report_passes("no file %a, nothing kept",filename)
-- end
-- end
-- eventually this will end up in strc-ini
statistics.register("startup time", function()
return statistics.elapsedseconds(statistics,"including runtime option file processing")
end)
statistics.register("jobdata time",function()
if enabled then
if #_others_ > 0 then
return format("%s seconds saving, %s seconds loading, other files: %s",statistics.elapsedtime(_save_),statistics.elapsedtime(_load_),concat(_others_," "))
else
return format("%s seconds saving, %s seconds loading",statistics.elapsedtime(_save_),statistics.elapsedtime(_load_))
end
else
if #_others_ > 0 then
return format("nothing saved, %s seconds loading, other files: %s",statistics.elapsedtime(_load_),concat(_others_," "))
else
return format("nothing saved, %s seconds loading",statistics.elapsedtime(_load_))
end
end
end)
statistics.register("callbacks", function()
local c_internal = status.callbacks or 0
local c_file = status.indirect_callbacks or 0
local c_direct = status.direct_callbacks or 0
local c_late = backends.getcallbackstate().count
local c_function = status.function_callbacks or 0
local c_total = c_internal + c_file + c_direct + c_late + c_function
local n_pages = structures.pages.nofpages or 0
local c_average = n_pages > 0 and math.round(c_total/n_pages) or 0
local result = format (
"internal: %s, file: %s, direct: %s, late: %s, function %s, total: %s (%s per page)",
c_internal, c_file, c_direct, c_late, c_function, c_total, c_average
)
statistics.callbacks = function()
return result
end
return result
end)
statistics.register("randomizer", function()
if rmethod and rvalue then
return format("%s with value %s",rmethod,rvalue)
end
end)
-- a sort of joke (for ctx meeting)
-- local kg_per_watt_per_second = 1 / 15000000
-- local watts_per_core = 50
-- local speedup_by_other_engine = 1.2
-- local used_wood_factor = watts_per_core * kg_per_watt_per_second / speedup_by_other_engine
-- local used_wood_factor = (50 / 15000000) / 1.2
function statistics.formatruntime(runtime)
if not environment.initex then -- else error when testing as not counters yet
-- stoptiming(statistics) -- to be sure
local shipped = texgetcount('nofshipouts')
local pages = texgetcount('realpageno')
if pages > shipped then
pages = shipped
end
runtime = tonumber(runtime)
if shipped > 0 or pages > 0 then
local persecond = (runtime > 0) and (shipped/runtime) or pages
if pages == 0 then
pages = shipped
end
return format("%0.3f seconds, %i processed pages, %i shipped pages, %.3f pages/second",runtime,pages,shipped,persecond)
else
return format("%0.3f seconds",runtime)
end
end
end
implement {
name = "savevariable",
actions = job.variables.save,
arguments = "2 strings",
}
implement {
name = "setjobcomment",
actions = job.comment,
arguments = { { "*" } }
}
implement {
name = "initializejob",
actions = job.initialize
}
implement {
name = "disablejobsave",
actions = job.disablesave
}
|