diff options
author | Karl Berry <karl@freefriends.org> | 2024-02-27 16:48:07 +0000 |
---|---|---|
committer | Karl Berry <karl@freefriends.org> | 2024-02-27 16:48:07 +0000 |
commit | ff3b6d10a05ce9bffa52a27ffa6e11e5ecf22c72 (patch) | |
tree | 35152df3770855d042b005decdfa9d63b686f446 /Master/texmf-dist/scripts | |
parent | 4624d267ff83f197bbce4cf296d291465069296c (diff) |
context 2024.02.27 09:18, and luametatex
git-svn-id: svn://tug.org/texlive/trunk@70189 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Master/texmf-dist/scripts')
26 files changed, 1708 insertions, 533 deletions
diff --git a/Master/texmf-dist/scripts/context/lua/mtx-context.lua b/Master/texmf-dist/scripts/context/lua/mtx-context.lua index 2991dbcc42c..1030e7a23b5 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-context.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-context.lua @@ -1,5 +1,3 @@ --- 8dec23 --socket added to restart per akira/hans (mail of 7dec23). - if not modules then modules = { } end modules['mtx-context'] = { version = 1.001, comment = "companion to mtxrun.lua", @@ -13,7 +11,7 @@ if not modules then modules = { } end modules['mtx-context'] = { local type, next, tostring, tonumber = type, next, tostring, tonumber local format, gmatch, match, gsub, find = string.format, string.gmatch, string.match, string.gsub, string.find -local quote, validstring, splitstring = string.quote, string.valid, string.split +local quote, validstring, splitstring, splitlines = string.quote, string.valid, string.split, string.splitlines local sort, concat, insert, sortedhash, tohash = table.sort, table.concat, table.insert, table.sortedhash, table.tohash local settings_to_array = utilities.parsers.settings_to_array local appendtable = table.append @@ -35,9 +33,13 @@ local removefile = os.remove local renamefile = os.rename local formatters = string.formatters +local starttiming = statistics.starttiming +local stoptiming = statistics.stoptiming +local elapsedtime = statistics.elapsedtime + local application = logs.application { name = "mtx-context", - banner = "ConTeXt Process Management 1.04", + banner = "ConTeXt Process Management 1.06", -- helpinfo = helpinfo, -- table with { category_a = text_1, category_b = text_2 } or helpstring or xml_blob helpinfo = "mtx-context.xml", } @@ -472,7 +474,7 @@ end local pdfview -- delayed local function pdf_open(name,method) - statistics.starttiming("pdfview") + starttiming("pdfview") pdfview = pdfview or dofile(resolvers.findfile("l-pdfview.lua","tex")) pdfview.setmethod(method) report(pdfview.status()) @@ -481,12 +483,12 @@ local function pdf_open(name,method) pdfname = name .. ".pdf" -- agressive end pdfview.open(pdfname) - statistics.stoptiming("pdfview") - report("pdfview overhead: %s seconds",statistics.elapsedtime("pdfview")) + stoptiming("pdfview") + report("pdfview overhead: %s seconds",elapsedtime("pdfview")) end local function pdf_close(name,method) - statistics.starttiming("pdfview") + starttiming("pdfview") pdfview = pdfview or dofile(resolvers.findfile("l-pdfview.lua","tex")) pdfview.setmethod(method) local pdfname = filenewsuffix(name,"pdf") @@ -495,7 +497,7 @@ local function pdf_close(name,method) end pdfname = name .. ".pdf" -- agressive pdfview.close(pdfname) - statistics.stoptiming("pdfview") + stoptiming("pdfview") end -- result file handling @@ -613,17 +615,20 @@ local function flags_to_string(flags,prefix) -- context flags get prepended by c: ... this will move to the sbx module local t = { } for k, v in table.sortedhash(flags) do + local p if prefix then - k = format("c:%s",k) + p = format("c:%s",k) + else + p = k end if not v or v == "" or v == '""' then -- no need to flag false elseif v == true then - t[#t+1] = format('--%s',k) + t[#t+1] = format('--%s',p) elseif type(v) == "string" then - t[#t+1] = format('--%s=%s',k,quote(v)) + t[#t+1] = format('--%s=%s',p,quote(v)) else - t[#t+1] = format('--%s=%s',k,tostring(v)) + t[#t+1] = format('--%s=%s',p,tostring(v)) end end return concat(t," ") @@ -696,6 +701,7 @@ function scripts.context.run(ctxdata,filename) local a_jithash = getargument("jithash") local a_permitloadlib = getargument("permitloadlib") local a_texformat = getargument("texformat") + local a_notuc = getargument("notuc") local a_keeptuc = getargument("keeptuc") local a_keeplog = getargument("keeplog") local a_keeppdf = getargument("keeppdf") @@ -910,6 +916,8 @@ function scripts.context.run(ctxdata,filename) -- local directives = { } -- + -- todo: handle these at the tex end + -- if a_nodates then directives[#directives+1] = format("backend.date=%s",type(a_nodates) == "string" and a_nodates or "no") end @@ -922,6 +930,11 @@ function scripts.context.run(ctxdata,filename) directives[#directives+1] = format("system.profile=%s",tonumber(a_profile) or 0) end -- + -- if a_notuc then + -- removefile(fileaddsuffix(jobname,"tuc")) + -- directives[#directives+1] = "job.save=no" -- handled at tex end + -- end + -- for i=1,#synctex_runfiles do removefile(fileaddsuffix(jobname,synctex_runfiles[i])) end @@ -1207,10 +1220,10 @@ do -- more or less copied from mtx-plain.lua: local function engine(texengine,texformat) local command = string.format('%s --ini --etex --8bit %s \\dump',texengine,fileaddsuffix(texformat,"mkii")) if environment.arguments.silent then - statistics.starttiming() + starttiming() local command = format("%s > temp.log",command) local result = os.execute(command) - local runtime = statistics.stoptiming() + local runtime = stoptiming() if result ~= 0 then print(format("%s silent make > fatal error when making format %q",texengine,texformat)) -- we use a basic print else @@ -1249,7 +1262,7 @@ do -- more or less copied from mtx-plain.lua: fmtpathspec = "." end end - fmtpathspec = string.splitlines(fmtpathspec)[1] or fmtpathspec + fmtpathspec = splitlines(fmtpathspec)[1] or fmtpathspec fmtpathspec = fmtpathspec and file.splitpath(fmtpathspec) local fmtpath = nil if fmtpathspec then @@ -1728,6 +1741,189 @@ function scripts.context.extra() end end +-- experiment + +do + + local popen = io.popen + local close = io.close + ----- read = io.read + local gobble = io.gobble or function(f) f:read("l") end + ----- clock = os.clock + local ticks = lua.getpreciseticks + local seconds = lua.getpreciseseconds + + local f_runner = formatters['context %s "%s"'] + local f_command = formatters['%s %s'] + + function scripts.context.parallel() + if getargument("pattern") then + environment.files = dir.glob(getargument("pattern")) + end + local files = environment.files + local total = files and #files or 0 + local list = nil + if getargument("parallellist") then + list = { } + for i=1,#files do + -- could be an lpeg + local name = files[i] + local data = splitlines(io.loaddata(name) or "") + if data then + for i=1,#data do + local line = data[i] + if string.find(line,"^context ") then + list[#list+1] = line + end + end + end + end + files = list + total = #list + end + if not list and total == 1 then + -- Beware: "--parallel" and "--terminal" are passed to the single run but this + -- is normally harmless. + scripts.context.autoctx() + elseif total > 0 then + local results = { } + local start = starttiming("parallel") + local terminal = environment.argument("terminal") + local runners = tonumber(environment.argument("parallel")) or 8 + local process = { } + local count = 0 + -- a hack + local passthese = environment.arguments_after + for i=1,#passthese do + local a = passthese[i] + if string.find(a,"^%-%-parallel") or string.find(a,"^%-%-terminal") or not find(a,"^%-%-") then + passthese[i] = "" + end + end + passthese = table.unique(passthese) + -- end of hack + local arguments = environment.reconstructcommandline(passthese) + local whattodo = list and "command" or "filename" + while true do + local done = false + for i=1,runners do + local pi = process[i] + if pi then + local s + if terminal then + s = pi.handle:read("l") + if s then + done = true + report("%02i : %s",i,s) + goto done + end + else + s = gobble(pi.handle) + if s then + done = true + goto done + end + end + if not s then + local r, detail, n = close(pi.handle) + stoptiming(pi.timer) + pi.result = (not r or n > 0) and "error" or "done" + pi.time = elapsedtime(pi.timer) + pi.handle = nil + pi.timer = nil + if terminal then + report() + end + report("process %02i, index %02i, %s %a, status %a, runtime %0.3f",i,pi.count,whattodo,pi.filename,pi.result,pi.time) + if terminal then + report() + end + process[i] = false + results[pi.count] = pi + end + end + count = count + 1 + if count > total then + -- we're done + else + local timer = "parallel:" .. i + local filename = files[count] + local dirname = file.dirname(filename) + local basename = file.basename(filename) + if dirname ~= "." and dirname ~= "./" then + dir.push(dirname) + end + local command = list and f_command(basename,arguments) or f_runner(arguments,basename) + starttiming(timer) + local result = popen(command) + if dirname ~= "." then + dir.pop() + end + local status = nil + if result then + process[i] = { + handle = result, + result = "start", + filename = filename, + count = count, + time = 0, + timer = timer, + } + status = process[i] + else + status = { + result = "error", + count = count, + filename = filename, + time = 0, + } + stoptiming(timer) + end + results[count] = status + if terminal then + report() + end + report("process %02i, index %02i, %s %a, status %a",i,status.count,whattodo,status.filename,status.result) + if terminal then + report() + end + done = true + end + ::done:: + end + if not done then + break + end + end + stoptiming("parallel") + results.runtime = elapsedtime("parallel") + report() + report("files: %i, runtime: %s",total,results.runtime) + report() + local errors = { } + for i=1,total do + local ri = results[i] + local result = ri.result + local filename = ri.filename + if result == "error" then + errors[#errors+1] = filename + end + report("index %02i, %s %a, status %a, runtime %0.3f ",ri.count,whattodo,filename,result,ri.time) + end + if #errors > 0 then + report() + report("errors in:") + report() + for i=1,#errors do + report(" %s",errors[i]) + end + end + report() + end + end + +end + -- todo: we need to do a dummy run local function showsetter() @@ -1748,6 +1944,35 @@ function scripts.context.logcategories() scripts.context.run() end +function scripts.context.find(pattern) + if pattern ~= "" then + local found = resolvers.findfile("luametatex.tex") + if found and found ~= "" then + for i=1,6 do + found = file.pathpart(found) + end + end + local files = dir.glob(found.."**.tex") + for i=1,#files do + local f = files[i] + local d = io.loaddata(f) + if find(d,pattern) then + local l = splitlines(d) + report(f) + report("") + for i=1,#l do + if find(l[i],pattern) then + report("%5i %s",i,l[i]) + end + end + report("") + end + end + else + report("there is nothing to search for") + end +end + function scripts.context.timed(action) statistics.timed(action,true) end @@ -1817,7 +2042,9 @@ else multipass_forcedruns = tonumber(getargument("forcedruns")) or nil end -if getargument("run") then +if getargument("parallel") or getargument("parallellist") then + scripts.context.timed(scripts.context.parallel) +elseif getargument("run") then scripts.context.timed(scripts.context.autoctx) elseif getargument("make") then scripts.context.timed(function() scripts.context.make() end) @@ -1832,7 +2059,7 @@ elseif getargument("touch") then scripts.context.touch() elseif getargument("pages") then scripts.context.pages() -elseif getargument("expert") then +elseif getargument("expert") then -- or getargument("special") then application.help("expert", "special") elseif getargument("showmodules") or getargument("modules") then scripts.context.modules() @@ -1856,6 +2083,15 @@ elseif getargument("showdirectives") or getargument("directives") == true then elseif getargument("showlogcategories") then scripts.context.logcategories() elseif environment.filenames[1] or getargument("nofile") then + -- -- + -- -- How compatible is this ... we might want to resolve the wildcard at the TeX, so + -- -- we just keep this as unsuported feature (but at least we know of this case): + -- + -- if not getargument("pattern") and find(environment.filenames[1],"%*") then + -- environment.filenames = dir.glob(environment.filenames[1]) + -- -- setargument("pattern",dir.glob(environment.filenames[1])) + -- end + -- scripts.context.timed(scripts.context.autoctx) elseif getargument("pipe") then scripts.context.timed(scripts.context.pipe) @@ -1865,6 +2101,9 @@ elseif getargument("purge") then elseif getargument("purgeall") then -- only when no filename given, supports --pattern scripts.context.purge(true,nil,true) +elseif getargument("find") then + -- context --find="%\framed" + scripts.context.find(getargument("find")) elseif getargument("pattern") then environment.filenames = dir.glob(getargument("pattern")) scripts.context.timed(scripts.context.autoctx) diff --git a/Master/texmf-dist/scripts/context/lua/mtx-context.xml b/Master/texmf-dist/scripts/context/lua/mtx-context.xml index bf341c21a9f..bd585a703de 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-context.xml +++ b/Master/texmf-dist/scripts/context/lua/mtx-context.xml @@ -4,7 +4,7 @@ <metadata> <entry name="name">mtx-context</entry> <entry name="detail">ConTeXt Process Management</entry> - <entry name="version">1.04</entry> + <entry name="version">1.05</entry> <entry name="comment">external helpinfo file</entry> </metadata> <flags> @@ -68,7 +68,7 @@ <short>set directive variables (show list with <ref name="showdirectives"/>)</short> </flag> <flag name="silent" value="list"> - <short>disable logcatgories (show list with <ref name="showlogcategories"/>)</short> + <short>disable logcategories (show list with <ref name="showlogcategories"/>)</short> </flag> <flag name="strip"> <short>strip Lua code (only meant for production where no errors are expected)</short> @@ -88,6 +88,9 @@ <flag name="purgeresult"> <short>purge result file before run</short> </flag> + <flag name="find" value="string"> + <short>search for the given pattern in the documentation sources</short> + </flag> </subcategory> <subcategory> <flag name="forcexml"> @@ -177,9 +180,17 @@ <flag name="global"> <short>assume given file present elsewhere</short> </flag> - <flag name="nofile"> + <flag name="nofile"> <short>use dummy file as jobname</short> </flag> + <flag name="forceinput"> + <short>use this file as input file instead of the given one</short> + </flag> + </subcategory> + <subcategory> + <flag name="expert"> + <short>show some more options</short> + </flag> </subcategory> </category> <category name="expert"> @@ -208,6 +219,9 @@ <flag name="keeplog"> <short>keep previous log files (jobname-log-[run].tmp)</short> </flag> + <flag name="notuc"> + <short>don't write a tuc file</short> + </flag> <flag name="lmtx"> <short>force lmtx mode (when available)</short> </flag> @@ -253,6 +267,14 @@ </flag> </subcategory> <subcategory> + <flag name="parallel"> + <short>process multiple files at the same time (you can specify a number of processes, default: 8)</short> + </flag> + <flag name="parallellist"> + <short>process commands in this file (lines that start with 'context')</short> + </flag> + </subcategory> + <subcategory> <flag name="sandbox"> <short>process file in a limited environment</short> </flag> diff --git a/Master/texmf-dist/scripts/context/lua/mtx-convert.lua b/Master/texmf-dist/scripts/context/lua/mtx-convert.lua index 61ce1504072..c24d77262e1 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-convert.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-convert.lua @@ -172,10 +172,10 @@ end if environment.arguments.convertall then convert.convertall() -elseif environment.files[1] then - convert.convertgiven() elseif environment.argument("exporthelp") then application.export(environment.argument("exporthelp"),environment.files[1]) +elseif environment.files[1] then + convert.convertgiven() else application.help() end diff --git a/Master/texmf-dist/scripts/context/lua/mtx-ctan.lua b/Master/texmf-dist/scripts/context/lua/mtx-ctan.lua index 6de51caa3ce..7603e363bd3 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-ctan.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-ctan.lua @@ -141,6 +141,7 @@ end local function checkedpattern(pattern) if pattern then + pattern = string.topattern(pattern,true) return lower(shaped(pattern)) end end @@ -291,7 +292,7 @@ scripts.ctan.topics = json and found[#found+1] = { key or name, details } end else - if strfound(pattern,name) or strfound(pattern,details) then + if strfound(pattern,key or name) or strfound(pattern,details) then found[#found+1] = { key or name, details } end end diff --git a/Master/texmf-dist/scripts/context/lua/mtx-fixpdf.lua b/Master/texmf-dist/scripts/context/lua/mtx-fixpdf.lua new file mode 100644 index 00000000000..78949a69d22 --- /dev/null +++ b/Master/texmf-dist/scripts/context/lua/mtx-fixpdf.lua @@ -0,0 +1,203 @@ +if not modules then modules = { } end modules ['mtx-fixpdf'] = { + version = 1.002, + comment = "companion to mtxrun.lua", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local find, lower = string.find, string.lower +local formatters = string.formatters +local nameonly, suffix = file.nameonly, file.suffix + +local helpinfo = [[ +<?xml version="1.0"?> +<application> + <metadata> + <entry name="name">mtx-tools</entry> + <entry name="detail">Some File Related Goodies</entry> + <entry name="version">1.01</entry> + </metadata> + <flags> + <category name="basic"> + <subcategory> + <flag name="uncompress"><short>uncompress using qpdf</short></flag> + <flag name="validate"><short>validate using verapdf</short></flag> + <flag name="check"><short>check verification result </short></flag> + <flag name="convert"><short>convert using context</short></flag> + <flag name="compare"><short>compare result with original</short></flag> + <flag name="compactor"><short>use given compactor</short></flag> + <flag name="standard"><short>use given standard</short></flag> + </subcategory> + <subcategory> + <flag name="pattern"><short>processs files according to pattern</short></flag> + <flag name="silent"><short>suppress messages</short></flag> + <flag name="once"><short>run only once</short></flag> + <flag name="resolution"><short>use this resolution when comparing</short></flag> + </subcategory> + </category> + </flags> +</application> +]] + +local application = logs.application { + name = "mtx-fixpdfs", + banner = "Making PDF files more compliant", + helpinfo = helpinfo, +} + +local report = application.report +local writeln = (logs and logs.writer) or (texio and texio.write_nl) or print + +scripts = scripts or { } +scripts.fixpdf = scripts.fixpdf or { } + +local function getfiles() + local pattern = environment.arguments.pattern + if pattern then + return dir.glob(pattern) or { } + else + return environment.files + end +end + +local function validfile(filename) + name = nameonly(filename) + return suffix(filename) == "pdf" and not find(name,"%-out$") and not find(name,"%-uncompressed$") and name or false +end + +function scripts.fixpdf.uncompress() + local files = environment.files + for i=1,#files do + local name = validfile(files[i]) + if name then + local command = formatters["qpdf --stream-data=uncompress --object-streams=disable %s.pdf %s-uncompressed.pdf"](name,name) + report("uncompressing %s.pdf -> %s-uncompressed.pdf",name,name) + os.execute(command) + end + end +end + +function scripts.fixpdf.validate() + local files = getfiles() + for i=1,#files do + local name = validfile(files[i]) + if name then + local command = formatters["verapdf %s-out.pdf > %s-out.txt"](name,name) + report("validating %s-out.pdf -> %s-out.txt",name,name) + os.execute(command) + end + end +end + +function scripts.fixpdf.check() + local files = getfiles() + for i=1,#files do + local name = validfile(files[i]) + if name then + local data = io.loaddata(formatters["%s-out.txt"](name)) + local atad = data and lower(data) + local t = { } + local compliant = atad and find(atad,"pdf file is compliant with validation profile requirements") + local hasmasks = atad and find(atad,"smask") + local fontissue = atad and find(atad,"errormessage.*font") + if compliant then + t = { "okay" } + else + if fontissue then + t[#t+1] = "font" + end + if hasmasks then + t[#t+1] = "mask" + end + end + if #t == 0 then + t = { "issue" } + end + report("checking %s-out.pdf : % t",name,t) + end + end +end + +function scripts.fixpdf.convert() + local files = getfiles() + local compactor = environment.arguments.compactor + local standard = environment.arguments.standard + local silent = environment.arguments.silent and "--batch --silent" or "" + local once = environment.arguments.silent and "--once " or "" + local nocompression = environment.arguments.nocompression and "--nocompression" or "" + local extrastyle = environment.arguments.extrastyle or "" + if type(compactor) ~= "string" then + compactor = "yes" + end + if type(standard) ~= "string" then + standard = "PDF/A-1a:2005" + end + for i=1,#files do + local name = validfile(files[i]) + if name then + -- "--batch --silent --once" can speed up if needed + local command = formatters["context --global s-pdf-fix.mkxl --compactor=%s --extrastyle=%s --standard=%s %s %s %s --pdffile=%s.pdf --result=%s-out"](compactor,extrastyle,standard,once,silent,nocompression,name,name) + report("converting %s.pdf -> %s-out.pdf",name,name) + os.execute(command) + end + end +end + +function scripts.fixpdf.compare() + local files = getfiles() + local resolution = tonumber(environment.arguments.resolution) or 72 + local tempdir = "temp-compare" + local workdir = lfs.currentdir() + dir.makedirs(tempdir) + if lfs.chdir(tempdir) then + for i=1,#files do + local name = validfile(files[i]) + if name then + local files = dir.glob("*.png") + for i=1,#files do + os.remove(files[i]) + end + print(oldname) + local command = string.format('mutool draw -c gray -o temp-old-%%d.png -r %i -A 0/0 -N ../%s.pdf',resolution,name) + print(command) + os.execute(command) + print(newname) + local command = string.format('mutool draw -c gray -o temp-new-%%d.png -r %i -A 0/0 -N ../%s-out.pdf',resolution,name) + print(command) + os.execute(command) + local files = dir.glob(string.format("temp-new-*.png",newpngname)) + for i=1,#files do + local command = string.format('gm compare -colorspace gray -highlight-color red -file temp-diff-%i.png temp-old-%i.png temp-new-%i.png',i,i,i) + print(command) + os.execute(command) + end + os.remove("context-extra.pdf") + command = string.format('context --extra=convert --pattern="temp-diff-*.png" --once --purgeall --result=%s-compare',name) + print(command) + os.execute(command) + local files = dir.glob("*.png") + for i=1,#files do + os.remove(files[i]) + end + -- we keep the file on temp-compare + end + end + end +end + +if environment.argument("uncompress") then + scripts.fixpdf.uncompress() +elseif environment.argument("validate") then + scripts.fixpdf.validate() +elseif environment.argument("check") then + scripts.fixpdf.check() +elseif environment.argument("convert") then + scripts.fixpdf.convert() +elseif environment.argument("compare") then + scripts.fixpdf.compare() +elseif environment.argument("exporthelp") then + application.export(environment.argument("exporthelp"),environment.files[1]) +else + application.help() +end diff --git a/Master/texmf-dist/scripts/context/lua/mtx-fonts.lua b/Master/texmf-dist/scripts/context/lua/mtx-fonts.lua index 746a010c8a2..3d43e9fc49b 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-fonts.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-fonts.lua @@ -16,7 +16,10 @@ local lower, gsub = string.lower, string.gsub local concat = table.concat local write_nl = (logs and logs.writer) or (texio and texio.write_nl) or print -local otlversion = 3.133 +local versions = { + otl = 3.140, + one = 1.520, +} local helpinfo = [[ <?xml version="1.0"?> @@ -288,14 +291,19 @@ local function showfeatures(tag,specification) for f,ff in table.sortedhash(data) do local done = false for s, ss in table.sortedhash(ff) do - if s == "*" then s = "all" end - if ss ["*"] then ss["*"] = nil ss.all = true end + local s = s == "*" and all or s + if ss["*"] then + ss["*"] = nil + ss.all = true + end + local name if done then - f = "" + name = "" else done = true + name = f end - report(" %-8s %-8s %-8s",f,s,concat(table.sortedkeys(ss), " ")) -- todo: padd 4 + report(" %-8s %-8s %-8s",name,s,concat(table.sortedkeys(ss), " ")) -- todo: padd 4 end end end @@ -471,10 +479,22 @@ end function scripts.fonts.unpack() local name = removesuffix(basename(givenfiles[1] or "")) if name and name ~= "" then - local cacheid = getargument("cache") or "otl" - local cache = containers.define("fonts", cacheid, otlversion, true) -- cache is temp - local cleanname = containers.cleanname(name) - local data = containers.read(cache,cleanname) + local cacheid = false + local cache = false + local cleanname = false + local data = false + local list = { getargument("cache") or false, "otl", "one" } + for i=1,#list do + cacheid = list[i] + if cacheid then + cache = containers.define("fonts", cacheid, versions[cacheid], true) -- cache is temp + cleanname = containers.cleanname(name) + data = containers.read(cache,cleanname) + if data then + break + end + end + end if data then local savename = addsuffix(cleanname .. "-unpacked","tma") report("fontsave, saving data in %s",savename) diff --git a/Master/texmf-dist/scripts/context/lua/mtx-grep.lua b/Master/texmf-dist/scripts/context/lua/mtx-grep.lua index e4a2a8d2f07..ae27e9d734e 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-grep.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-grep.lua @@ -23,6 +23,7 @@ local helpinfo = [[ <flag name="pattern"><short>search for pattern (optional)</short></flag> <flag name="count"><short>count matches only</short></flag> <flag name="nocomment"><short>skip lines that start with %% or #</short></flag> + <flag name="noattic"><short>skip files that hh considers irrelevant</short></flag> <flag name="n"><short>show at most n matches</short></flag> <flag name="first"><short>only show first match</short></flag> <flag name="match"><short>return the match (if it is one)</short></flag> @@ -81,33 +82,41 @@ function scripts.grep.find(pattern, files, offset) local nofmatches, noffiles, nofmatchedfiles = 0, 0, 0 local n, m, check = 0, 0, nil local name = "" + local noattic = environment.argument("noattic") local count = environment.argument("count") local nocomment = environment.argument("nocomment") local max = tonumber(environment.argument("n")) or (environment.argument("first") and 1) or false local domatch = environment.argument("match") + -- for me: + local function skip(name) + return noattic and (find(name,"attic") or find(name,"backup") or find(name,"old") or find(name,"keep") or find(name,"install") or find(name,"texmf")) + end + -- if environment.argument("xml") then for i=offset or 1, #files do local globbed = dir.glob(files[i]) for i=1,#globbed do name = globbed[i] - local data = xml.load(name) - if data and not data.error then - n, m, noffiles = 0, 0, noffiles + 1 - if count then - for c in xml.collected(data,pattern) do - m = m + 1 - end - if m > 0 then - nofmatches = nofmatches + m - nofmatchedfiles = nofmatchedfiles + 1 - write_nl(format("%5i %s",m,name)) - io.flush() - end - else - for c in xml.collected(data,pattern) do - m = m + 1 - if not max or m <= max then - write_nl(format("%s: %s",name,xml.tostring(c))) + if not skip(name) then + local data = xml.load(name) + if data and not data.error then + n, m, noffiles = 0, 0, noffiles + 1 + if count then + for c in xml.collected(data,pattern) do + m = m + 1 + end + if m > 0 then + nofmatches = nofmatches + m + nofmatchedfiles = nofmatchedfiles + 1 + write_nl(format("%5i %s",m,name)) + io.flush() + end + else + for c in xml.collected(data,pattern) do + m = m + 1 + if not max or m <= max then + write_nl(format("%s: %s",name,xml.tostring(c))) + end end end end @@ -173,7 +182,7 @@ function scripts.grep.find(pattern, files, offset) local globbed = dir.glob(files[i]) for i=1,#globbed do name = globbed[i] - if not find(name,"/%.") then + if not find(name,"/%.") and not skip(name) then local data = io.loaddata(name) if data then n, m, noffiles = 0, 0, noffiles + 1 diff --git a/Master/texmf-dist/scripts/context/lua/mtx-install-modules.lua b/Master/texmf-dist/scripts/context/lua/mtx-install-modules.lua new file mode 100644 index 00000000000..1fa9396d653 --- /dev/null +++ b/Master/texmf-dist/scripts/context/lua/mtx-install-modules.lua @@ -0,0 +1,373 @@ +if not modules then modules = { } end modules ['mtx-install-modules'] = { + version = 1.234, + comment = "companion to mtxrun.lua", + author = "Hans Hagen", + copyright = "ConTeXt Development Team", + license = "see context related readme files" +} + +-- Installing tikz is a bit tricky because there are many packages involved and it's +-- sort of impossible to derive from the names what to include in the installation. +-- I tried to use the ctan scrips we ship but there is no way to reliably derive a +-- set from the topics or packages using the web api (there are also some +-- inconsistencies between the json and xml interfaces that will not be fixed). A +-- wildcard pull of everything tikz/pgf is likely to fail or at least gives files we +-- don't want and/or need, the solution is to be specific. +-- +-- After that was implemented the script changed name and now also installs the +-- third party modules. +-- +-- We use curl and not the built in socket library because all kind of ssl and +-- redirection can kick in and who know how it evolves. +-- +-- We use the context unzipper because we cannot be sure if unzip is present on the +-- system. In many cases windows, linux and osx installations lack it by default. +-- +-- This script should be run in the tex root where there is also a texmf-context sub +-- directory; it will quit otherwise. The modules path will be created when absent. +-- +-- Maybe some day we can get the modules from ctan but then we need a consistent +-- names and such. + +local find = string.find + +local helpinfo = [[ +<?xml version="1.0"?> +<application> + <metadata> + <entry name="name">mtx-install</entry> + <entry name="detail">ConTeXt Installer</entry> + <entry name="version">2.01</entry> + </metadata> + <flags> + <category name="basic"> + <subcategory> + <flag name="list"><short>list modules</short></flag> + <flag name="installed"><short>list installed modules</short></flag> + <flag name="install"><short>install modules</short></flag> + <flag name="uninstall"><short>uninstall modules</short></flag> + <flag name="module"><short>install (zip) file(s)</short></flag> + </subcategory> + </category> + </flags> + <examples> + <category> + <title>Examples</title> + <subcategory> + <example><command>mtxrun --script install-modules --list</command></example> + </subcategory> + <subcategory> + <example><command>mtxrun --script install-modules --install filter letter</command></example> + <example><command>mtxrun --script install-modules --install tikz</command></example> + <example><command>mtxrun --script install-modules --install --all</command></example> + </subcategory> + <subcategory> + <example><command>mtxrun --script install-modules --install --module t-letter.zip</command></example> + <example><command>mtxrun --script install-modules --uninstall --module t-letter.zip</command></example> + </subcategory> + <subcategory> + <example><command>mtxrun --script install-modules --installed</command></example> + </subcategory> + </category> + </examples> +</application> +]] + +local application = logs.application { + name = "mtx-install-modules", + banner = "ConTeXt Module Installer 1.00", + helpinfo = helpinfo, +} + +local report = application.report + +scripts = scripts or { } +scripts.modules = scripts.modules or { } + +local okay, curl = pcall(require,"libs-imp-curl") + +local fetched = curl and curl.fetch and function(str) + local data, message = curl.fetch { + url = str, + followlocation = true, + sslverifyhost = false, + sslverifypeer = false, + } + if not data then + report("some error: %s",message) + end + return data +end or function(str) + -- So, no redirect to http, which means that we cannot use the built in socket + -- library. What if the client is happy with http? + local data = os.resultof("curl -sSL " .. str) + return data +end + +-- We use some abstraction: + +local urls = { + ctan = "https://mirrors.ctan.org/install", + modules = "https://modules.contextgarden.net/dl" +} + +-- Some package this in the root which is asking for conflicts. + +-- local badones = { +-- -- "LICENSE", +-- -- "README", +-- -- "README.md", +-- -- "VERSION", +-- } + +local tmpzipfile = "temp.zip" +local checkdir = "texmf-context" +local targetdir = "texmf-modules" +local basefile = "mtx-install-imp-modules.lua" +local lists = false + +-- local lists = { +-- ["tikz"] = { +-- url = "ctan", +-- zips = { }, -- table of zip files +-- wipes = { }, -- (nested) table of delete patterns +-- }, +-- ["filter"] = { +-- url = "modules", +-- zips = { "t-filter.zip" } +-- }, +-- } + +local function loadlists() + if not lists then + lists = { } + local mainfile = resolvers.findfile(basefile) + if mainfile and mainfile ~= "" then + local path = file.pathpart(mainfile) + local files = dir.glob((path == "" and "." or path) .. "/mtx-install-imp*.lua") + for i=1,#files do + local name = files[i] + local data = table.load(name) + if data then + local entries = data.lists + if entries then + report("loading entries from file %a",name) + for entry, data in table.sortedhash(entries) do + if lists[entry] then + report("entry %a already set from %a",entry,name) + else + lists[entry] = data + end + end + else + report("no entries in file %a",name) + end + end + end + else + report("base file %a is not found",basefile) + end + report() + end +end + +local function validate(n) + return not ( + find(n,"latex") + or find(n,"lualatex") + or find(n,"plain") + or find(n,"optex") + -- or find(n,"luatex") + -- or find(n,"pdftex") + ) +end + +local function install(list,wipe) + if type(list) ~= "table" then + report("unknown specification") + end + local zips = list.zips + local wipes = list.wipes + if type(zips) ~= "table" then + report("incomplete specification") + else + -- report("installing into %a",targetdir) + for i=1,#zips do + local remote = list.url + local where = zips[i] + local hash = file.addsuffix(sha2.HASH256(where),"tma") + local data = table.load(hash) + if data then + local name = data.name + local list = data.list + if name and list then + report() + report("removing %i old files for %a",#list,name) + report() + for i=1,#list do + os.remove(list[i]) + end + end + end + if not wipe then + if remote then + where = (urls[remote] or remote) .. "/" .. where + end + local data = fetched(where) + if string.find(data,"^PK") then + io.savedata(tmpzipfile,data) + report("from %a",where) + report("into %a",targetdir) + local done = utilities.zipfiles.unzipdir { + zipname = tmpzipfile, + path = ".", + verbose = "steps", + collect = true, + validate = validate, + } + table.save(hash,{ name = where, list = done }) + os.remove(tmpzipfile) + else + report("unknown %a",where) + end + end + end + + local function wiper(wipes) + for i=1,#wipes do + local s = wipes[i] + if type(s) == "table" then + wiper(s) + elseif type(s) == "string" then + local t = dir.glob(s) + report("wiping %i files in %a",#t,s) + for i=1,#t do +-- report("wiping %a",t[i]) + os.remove(t[i]) + end + end + end + end + + if type(wipes) == "table" then + wiper(wipes) + end + end +end + +function scripts.modules.list() + loadlists() + for k, v in table.sortedhash(lists) do + report("%-20s: %-36s : % t",k,urls[v.url],v.zips) + end +end + +function scripts.modules.installed() + local files = dir.glob(targetdir .. "/*.tma") + if files then + for i=1,#files do + local data = table.load(files[i]) + if data then + local name = data.name + local list = data.list + if name and list then + report("%4i : %s",#list,name) + end + end + end + end +end + +function scripts.modules.install(wipe) + local curdir = dir.current() + local done = false + if not lfs.isdir(checkdir) then + report("unknown subdirectory %a",checkdir) + elseif not dir.mkdirs(targetdir) then + report("unable to create %a",targetdir) + elseif not lfs.chdir(targetdir) then + report("unable to go into %a",targetdir) + elseif environment.argument("module") or environment.argument("modules") then + local files = environment.files + if #files == 0 then + report("no archive names provided") + else + for i=1,#files do + local name = files[i] + if url.hasscheme(name) then + install({ url = false, zips = { file.addsuffix(name,"zip") } }, wipe) + else + loadlists() + install({ url = "modules", zips = { file.addsuffix(name,"zip") } }, wipe) + end + end + done = files + end + else + loadlists() + local files = environment.argument("all") and table.sortedkeys(lists) or environment.files + if #files == 0 then + report("no module names provided") + else + for i=1,#files do + local name = files[i] + local list = lists[name] + if list then + install(list,wipe) + end + end + done = files + end + end + if done then + -- + -- for i=1,#badones do + -- os.remove(badones[i]) + -- end + local okay = false + local files = dir.glob("*") + for i=1,#files do + local name = files[i] + if file.suffix(name) == "tma" then + -- keep it + else + if not okay then + report() + okay = true + end + report("removed %a",name) + os.remove(name) + end + end + -- + report() + report("renewing file database") + report() + resolvers.renewcache() + resolvers.load() + report() + report("installed: % t",done) + report() + end + lfs.chdir(curdir) +end + +function scripts.modules.uninstall() + scripts.modules.install(true) +end + +if environment.argument("list") then + scripts.modules.list() +elseif environment.argument("installed") then + scripts.modules.installed() +elseif environment.argument("install") then + scripts.modules.install() +elseif environment.argument("uninstall") then + scripts.modules.uninstall() +elseif environment.argument("exporthelp") then + application.export(environment.argument("exporthelp"),environment.files[1]) +else + application.help() + report("") +end + diff --git a/Master/texmf-dist/scripts/context/lua/mtx-install-tikz.lua b/Master/texmf-dist/scripts/context/lua/mtx-install-tikz.lua deleted file mode 100644 index 74a2922418a..00000000000 --- a/Master/texmf-dist/scripts/context/lua/mtx-install-tikz.lua +++ /dev/null @@ -1,162 +0,0 @@ -if not modules then modules = { } end modules ['mtx-install-tikz'] = { - version = 1.234, - comment = "companion to mtxrun.lua", - author = "Hans Hagen", - copyright = "ConTeXt Development Team", - license = "see context related readme files" -} - --- Installing tikz is a bit tricky because there are many packages involved and it's --- sort of impossible to derive from the names what to include in the installation. --- I tried to use the ctan scrips we ship but there is no way to reliably derive a --- set from the topics or packages using the web api (there are also some --- inconsistencies between the jkson and xml interfaces that will not be fixed). A --- wildcard pull of everything tikz/pgf is likely to fail or at least gives files we --- don't want and/or need, the solution is to be specific. --- --- We use curl and not the built in socket library because all kind of ssl and --- redirection can kick in and who know how it evolves. --- --- We use the context unzipper because we cannot be sure if unzip is present on the --- system. In many cases windows, linux and osx installations lack it by default. --- --- This script has no help info etc as it's a rather dumb downloader. One can always --- do a better job than this suboptimal quick hack. Let me know if I forget to wipe --- something. --- --- mtxrun --script install-tikz --- --- It should be run in the tex root and will quit when there is no texmf-context --- path found. The modules path will be created when absent. - -local okay, curl = pcall(require,"libs-imp-curl") - -local fetched = curl and curl.fetch and function(str) - local data, message = curl.fetch { - url = str, - followlocation = true, - sslverifyhost = false, - sslverifypeer = false, - } - if not data then - report("some error: %s",message) - end - return data -end or function(str) - -- So, no redirect to http, which means that we cannot use the built in socket - -- library. What if the client is happy with http? - local data = os.resultof("curl -sSL " .. str) - return data -end - -local ctanurl = "https://mirrors.ctan.org/install" -local tmpzipfile = "temp.zip" -local checkdir = "texmf-context" -local targetdir = "texmf-modules" - -local report = logs.reporter("install") - -scripts = scripts or { } -scripts.ctan = scripts.ctan or { } - -function scripts.ctan.install(list) - if type(list) ~= "table"then - report("unknown specification") - end - local zips = list.zips - local wipes = list.wipes - if type(zips) ~= "table" or type(wipes) ~= "table" then - report("incomplete specification") - elseif not lfs.isdir(checkdir) then - report("unknown subdirectory %a",checkdir) - elseif not dir.mkdirs(targetdir) then - report("unable to create %a",targetdir) - elseif not lfs.chdir(targetdir) then - report("unable to go into %a",targetdir) - else - report("installing into %a",targetdir) - for i=1,#zips do - local where = ctanurl .. "/" .. zips[i] - local data = fetched(where) - if string.find(data,"^PK") then - io.savedata(tmpzipfile,data) - report("from %a",where) - report("into %a",targetdir) - utilities.zipfiles.unzipdir { - zipname = tmpzipfile, - path = ".", - verbose = "steps", - } - os.remove(tmpzipfile) - else - report("unknown %a",where) - end - end - - local function wiper(wipes) - for i=1,#wipes do - local s = wipes[i] - if type(s) == "table" then - wiper(s) - elseif type(s) == "string" then - local t = dir.glob(s) - report("wiping %i files in %a",#t,s) - for i=1,#t do - os.remove(t[i]) - end - end - end - end - - wiper(wipes) - - report("renewing file database") - resolvers.renewcache() - resolvers.load() - end -end - -local function wipers(s) - return { - "tex/context/third/" ..s.. "/**", - "doc/context/third/" ..s.. "/**", - "source/context/third/" ..s.. "/**", - - "tex/context/" ..s.. "/**", - "doc/context/" ..s.. "/**", - "source/context/" ..s.. "/**", - - "scripts/" ..s.. "/**", - } -end - -local defaults = { - "tex/latex/**", - "tex/plain/**", - - "doc/latex/**", - "doc/plain/**", - "doc/generic/**", - - "source/latex/**", - "source/plain/**", - "source/generic/**", -} - -local lists = { - tikz = { - zips = { - "graphics/pgf/base/pgf.tds.zip", - "graphics/pgf/contrib/pgfplots.tds.zip", - "graphics/pgf/contrib/circuitikz.tds.zip", - }, - wipes = { - wipers("pgf"), - wipers("pgfplots"), - wipers("circuitikz"), - defaults, - } - }, -} - -scripts.ctan.install(lists.tikz) diff --git a/Master/texmf-dist/scripts/context/lua/mtx-install.lua b/Master/texmf-dist/scripts/context/lua/mtx-install.lua index 4b590128245..d66278dc83f 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-install.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-install.lua @@ -40,14 +40,15 @@ local gsub, find, escapedpattern = string.gsub, string.find, string.escapedpatte local round = math.round local savetable, loadtable, sortedhash = table.save, table.load, table.sortedhash local copyfile, joinfile, filesize, dirname, addsuffix, basename = file.copy, file.join, file.size, file.dirname, file.addsuffix, file.basename -local isdir, isfile, walkdir, pushdir, popdir, currentdir = lfs.isdir, lfs.isfile, lfs.dir, lfs.chdir, dir.push, dir.pop, currentdir +local isdir, isfile, walkdir, pushdir, popdir, currentdir = lfs.isdir, lfs.isfile, lfs.dir, dir.push, dir.pop, dir.current local mkdirs, globdir = dir.mkdirs, dir.glob local osremove, osexecute, ostype, resultof = os.remove, os.execute, os.type, os.resultof -local savedata = io.savedata +local savedata, loaddata = io.savedata, io.loaddata local formatters = string.formatters local httprequest = socket.http.request -local usecurl = false +local usecurl = false +local protocol = "http" local function checkcurl() local s = resultof("curl --version") @@ -63,7 +64,11 @@ local function fetch(url) data, detail = httprequest(url) end if type(data) ~= "string" then - data = false + data = false + detail = "download failed" + elseif #data == 0 then + data = false + detail = "download failed, zero length" elseif #data < 2048 then local n, t = find(data,"<head>%s*<title>%s*(%d+)%s(.-)</title>") if tonumber(n) then @@ -100,36 +105,33 @@ local texformats = { local platforms = { ["mswin"] = "mswin", - ["windows"] = "mswin", + -- ["windows"] = "mswin", ["win32"] = "mswin", ["win"] = "mswin", - ["arm32"] = "windows-arm32", - ["windows-arm32"] = "windows-arm32", -- - ["mswin-64"] = "win64", + ["windows"] = "win64", ["windows-64"] = "win64", ["win64"] = "win64", - ["arm64"] = "windows-arm64", - ["windows-arm64"] = "windows-arm64", -- ["linux"] = "linux", ["linux-32"] = "linux", ["linux32"] = "linux", -- + -- ["linux"] = "linux-64", ["linux-64"] = "linux-64", ["linux64"] = "linux-64", -- - ["linuxmusl-64"] = "linuxmusl", ["linuxmusl"] = "linuxmusl", + ["linuxmusl-64"] = "linuxmusl-64", -- - ["linux-armhf"] = "linux-armhf", + -- ["linux-armhf"] = "linux-armhf", -- - ["openbsd"] = "openbsd7.3", - ["openbsd-i386"] = "openbsd7.3", - ["openbsd-amd64"] = "openbsd7.3-amd64", + ["openbsd"] = "openbsd-amd64", + -- ["openbsd-i386"] = "openbsd7.3", + ["openbsd-amd64"] = "openbsd-amd64", -- - ["freebsd"] = "freebsd", - ["freebsd-i386"] = "freebsd", + ["freebsd"] = "freebsd-amd64", + -- ["freebsd-i386"] = "freebsd", ["freebsd-amd64"] = "freebsd-amd64", -- -- ["kfreebsd"] = "kfreebsd-i386", @@ -151,6 +153,7 @@ local platforms = { -- ["macosx"] = "osx-64", ["osx"] = "osx-64", + ["osx-intel"] = "osx-64", ["osx-64"] = "osx-64", ["osx-arm"] = "osx-arm64", ["osx-arm64"] = "osx-arm64", @@ -187,7 +190,7 @@ function install.identify() local name = files[i] local size = filesize(name) local base = gsub(name,pattern,"") - local data = io.loaddata(name) + local data = loaddata(name) if data and #data > 0 then local stamp = hashdata(data) details[i] = { base, size, stamp } @@ -218,6 +221,8 @@ function install.identify() date = os.date("%Y-%m-%d"), }) + os.exit() + end local function disclaimer() @@ -245,15 +250,15 @@ function install.update() return ok end - local function download(what,url,target,total,done,oldhash) + local function download(what,url,target,total,done,hash) local data = fetch(url .. "/" .. target) - if data then + if type(data) == "string" and #data > 0 then if total and done then report("%-8s : %3i %% : %8i : %s",what,round(100*done/total),#data,target) else report("%-8s : %8i : %s",what,#data,target) end - if oldhash and oldhash ~= hashdata(data) then + if hash and hash ~= hashdata(data) then return "different hash value" elseif not validdir(dirname(target)) then return "wrong target directory" @@ -336,8 +341,9 @@ function install.update() report("fetching %a",zipurl) local zipdata = fetch(zipurl) if zipdata then - io.savedata(zipfile,zipdata) + savedata(zipfile,zipdata) else + osremove(zipfile) zipfile = false end end @@ -353,6 +359,7 @@ function install.update() path = ".", -- verbose = true, verbose = "steps", + collect = true, -- sort of a check } if utilities.zipfiles.unzipdir(specification) then @@ -361,7 +368,6 @@ function install.update() else osremove(zipfile) end - end count = #new @@ -379,7 +385,10 @@ function install.update() local target = joinfile(tree,name) done = done + size if not skiplist or not skiplist[basename(name)] then - download("new",url,target,total,done) + local message = download("new",url,target,total,done) + if message then + report("%s",message) + end else report("skipping %s",target) end @@ -490,7 +499,7 @@ function install.update() for i=1,#list do local host = list[i] - local data, status, detail = fetch("http://" .. host .. "/" .. instance .. "/tex/status.tma") + local data, status, detail = fetch(protocol .. "://" .. host .. "/" .. instance .. "/tex/status.tma") if status == 200 and type(data) == "string" then local t = loadstring(data) if type(t) == "function" then @@ -508,7 +517,7 @@ function install.update() return end - local url = "http://" .. server .. "/" .. instance .. "/" + local url = protocol .. "://" .. server .. "/" .. instance .. "/" local texmfplatform = "texmf-" .. platform @@ -557,8 +566,8 @@ function install.update() local cdir = currentdir() local pdir = pushdir(binpath) - report("current : %S",cdir) - report("target : %S",pdir) + report("current : %S",cdir or "<unable to check the curent path>") + report("target : %S",pdir or "<unable to change to the binary path>") if pdir ~= cdir then @@ -634,22 +643,45 @@ function install.update() report("") report("update, done") + +end + +function install.modules() + report("--modules in not yet implemented") +end + +function install.fonts() + report("--fonts in not yet implemented") +end + +function install.goodies() + report("--goodies in not yet implemented") end if environment.argument("secure") then usecurl = checkcurl() - if not usecurl then + if usecurl then + protocol = "https" + else report("no curl installed, quitting") os.exit() end end +io.output():setvbuf("no") + if environment.argument("identify") then install.identify() elseif environment.argument("install") then install.update() elseif environment.argument("update") then install.update() +elseif environment.argument("modules") then + install.modules() +elseif environment.argument("fonts") then + install.fonts() +elseif environment.argument("goodies") then + install.goodies() elseif environment.argument("exporthelp") then application.export(environment.argument("exporthelp"),environment.files[1]) else @@ -657,3 +689,4 @@ else report("") disclaimer() end + diff --git a/Master/texmf-dist/scripts/context/lua/mtx-kpse.lua b/Master/texmf-dist/scripts/context/lua/mtx-kpse.lua index 7542751ac5f..663d10383fe 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-kpse.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-kpse.lua @@ -77,13 +77,6 @@ local argument = environment.argument local files = environment.files local target = files[1] -if argument("progname") or argument("programname") then - kpse.set_program_name(argument("progname")) -else - application.help() - return -end - local function printtable(result) if type(result) == "table" then for i=1,#result do @@ -94,7 +87,15 @@ end if argument("exporthelp") then application.export(environment.argument("exporthelp"),target) -elseif argument("filetypes") or argument("formats") then + return +elseif argument("progname") or argument("programname") then + kpse.set_program_name(argument("progname")) +else + application.help() + return +end + +if argument("filetypes") or argument("formats") then print(table.concat(kpse.get_file_types()," ")) elseif type(target) == "string" and target ~= "" then if argument("findfiles") or argument("find-files") then diff --git a/Master/texmf-dist/scripts/context/lua/mtx-patterns.lua b/Master/texmf-dist/scripts/context/lua/mtx-patterns.lua index 7f5c1c55629..648ec8aa95e 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-patterns.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-patterns.lua @@ -121,9 +121,9 @@ scripts.patterns.list = { { "hr", "hyph-hr", "croatian" }, -- { "??", "hyph-hsb", "upper sorbian" }, { "hu", "hyph-hu", "hungarian" }, - -- { "hy", "hyph-hy", "armenian" }, + { "hy", "hyph-hy", "armenian" }, -- { "??", "hyph-ia", "interlingua" }, - -- { "??", "hyph-id", "indonesian" }, + { "id", "hyph-id", "indonesian" }, { "is", "hyph-is", "icelandic" }, { "it", "hyph-it", "italian" }, -- { "??", "hyph-kmr", "kurmanji" }, diff --git a/Master/texmf-dist/scripts/context/lua/mtx-pdf.lua b/Master/texmf-dist/scripts/context/lua/mtx-pdf.lua index 05973d2365d..53ba7cb93c7 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-pdf.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-pdf.lua @@ -25,16 +25,23 @@ local helpinfo = [[ <subcategory> <flag name="info"><short>show some info about the given file</short></flag> <flag name="metadata"><short>show metadata xml blob</short></flag> + <flag name="formdata"><short>show formdata</short></flag> <flag name="pretty"><short>replace newlines in metadata</short></flag> - <flag name="fonts"><short>show used fonts (<ref name="detail)"/></short></flag> - <flag name="object"><short>show object"/></short></flag> - <flag name="links"><short>show links"/></short></flag> + <flag name="fonts"><short>show used fonts (<ref name="detail"/>)</short></flag> + <flag name="object"><short>show object</short></flag> + <flag name="links"><short>show links</short></flag> + <flag name="sign"><short>sign document (assumes signature template)</short></flag> + <flag name="verify"><short>verify document</short></flag> + <flag name="detail"><short>print detail to the console</short></flag> </subcategory> <subcategory> <example><command>mtxrun --script pdf --info foo.pdf</command></example> <example><command>mtxrun --script pdf --metadata foo.pdf</command></example> <example><command>mtxrun --script pdf --metadata --pretty foo.pdf</command></example> <example><command>mtxrun --script pdf --stream=4 foo.pdf</command></example> + <example><command>mtxrun --script pdf --sign --certificate=somesign.pem --password=test --uselibrary somefile</command></example> + <example><command>mtxrun --script pdf --verify --certificate=somesign.pem --password=test --uselibrary somefile</command></example> + <example><command>mtxrun --script pdf --detail=nofpages somefile</command></example> </subcategory> </category> </flags> @@ -55,9 +62,11 @@ elseif CONTEXTLMTXMODE then dofile(resolvers.findfile("util-dim.lua","tex")) dofile(resolvers.findfile("lpdf-ini.lmt","tex")) dofile(resolvers.findfile("lpdf-pde.lmt","tex")) + dofile(resolvers.findfile("lpdf-sig.lmt","tex")) else dofile(resolvers.findfile("lpdf-pde.lua","tex")) end +dofile(resolvers.findfile("util-jsn.lua","tex")) scripts = scripts or { } scripts.pdf = scripts.pdf or { } @@ -68,13 +77,13 @@ local function loadpdffile(filename) if not filename or filename == "" then report("no filename given") elseif not lfs.isfile(filename) then - report("unknown file '%s'",filename) + report("unknown file %a",filename) else - local pdffile = lpdf.epdf.load(filename) + local pdffile = lpdf.epdf.load(filename) if pdffile then return pdffile else - report("no valid pdf file '%s'",filename) + report("no valid pdf file %a",filename) end end end @@ -82,10 +91,10 @@ end function scripts.pdf.info(filename) local pdffile = loadpdffile(filename) if pdffile then - local catalog = pdffile.Catalog - local info = pdffile.Info - local pages = pdffile.pages - local nofpages = pdffile.nofpages + local catalog = pdffile.Catalog + local info = pdffile.Info + local pages = pdffile.pages + local nofpages = pdffile.nofpages local unset = "<unset>" @@ -130,7 +139,7 @@ function scripts.pdf.info(filename) -- if details then local annotations = 0 - for i=1, nofpages do + for i=1,nofpages do local page = pages[i] local a = page.Annots if a then @@ -167,6 +176,189 @@ function scripts.pdf.info(filename) end end +function scripts.pdf.detail(filename,detail) + if detail == "pages" or detail == "nofpages" then + local pdffile = loadpdffile(filename) + print(pdffile and pdffile.nofpages or 0) + end +end + +local function flagstoset(flag,flags) + local t = { } + if flags then + for k, v in next, flags do + if (flag & v) ~= 0 then + t[k] = true + end + end + end + return t +end + +function scripts.pdf.formdata(filename,save) + local pdffile = loadpdffile(filename) + if pdffile then + local widgets = pdffile.widgets + if widgets then + local results = { { "type", "name", "value" } } + for i=1,#widgets do + local annotation = widgets[i] + local parent = annotation.Parent or { } + local name = annotation.T or parent.T + local what = annotation.FT or parent.FT + if name and what then + local value = annotation.V and tostring(annotation.V) or "" + if value and value ~= "" then + local wflags = flagstoset(annotation.Ff or parent.Ff or 0, widgetflags) + if what == "Tx" then + if wflags.MultiLine then + wflags.MultiLine = nil + what = "text" + else + what = "line" + end + local default = annotation.V or "" + elseif what == "Btn" then + if wflags.Radio or wflags.RadiosInUnison then + what = "radio" + elseif wflags.PushButton then + what = "push" + else + what = "check" + end + elseif what == "Ch" then + -- F Ff FT Opt T | AA OC (rest follows) + if wflags.PopUp then + wflags.PopUp = nil + if wflags.Edit then + what = "combo" + else + what = "popup" + end + else + what = "choice" + end + elseif what == "Sig" then + what = "signature" + else + what = nil + end + if what then + results[#results+1] = { what, name, value } + end + end + end + end + if save then + local values = { } + for i=2,#results do + local result= results[i] + values[#values+1] = { + type = result[1], + name = result[2], + value = result[3], + } + end + local data = { + filename = filename, + values = values, + } + local name = file.nameonly(filename) .. "-formdata" + if save == "json" then + name = file.addsuffix(name,"json") + io.savedata(name,utilities.json.tojson(data)) + elseif save then + name = file.addsuffix(name,"lua") + table.save(name,data) + end + report("") + report("%i widgets found, %i values saved in %a",#widgets,#results-1,name) + report("") + end + utilities.formatters.formatcolumns(results) + report(results[1]) + report("") + for i=2,#results do + report(results[i]) + end + report("") + end + end +end + +function scripts.pdf.signature(filename,save) + local pdffile = loadpdffile(filename) + if pdffile then + local widgets = pdffile.widgets + if widgets then + for i=1,#widgets do + local annotation = widgets[i] + local parent = annotation.Parent or { } + local name = annotation.T or parent.T + local what = annotation.FT or parent.FT + if what == "Sig" then + local value = annotation.V + if value then + local contents = tostring(value.Contents) or "" + report("") + if save then + local name = file.nameonly(filename) .. "-signature.bin" + report("signature saved in %a",name) + io.savedata(name,string.tobytes(contents)) + else + report("signature: %s",contents) + end + report("") + return + end + end + end + end + report("there is no signature") + end +end + +function scripts.pdf.sign(filename,save) + local pdffile = file.addsuffix(filename,"pdf") + if not lfs.isfile(pdffile) then + report("invalid pdf file %a",pdffile) + return + end + local certificate = environment.argument("certificate") + local password = environment.argument("password") + if type(certificate) ~= "string" or type(password) ~= "string" then + report("provide --certificate and --password") + return + end + lpdf.sign { + filename = pdffile, + certificate = certificate, + password = password, + purge = environment.argument("purge"), + uselibrary = environment.argument("uselibrary"), + } +end + +function scripts.pdf.verify(filename,save) + local pdffile = file.addsuffix(filename,"pdf") + if not lfs.isfile(pdffile) then + report("invalid pdf file %a",pdffile) + return + end + local certificate = environment.argument("certificate") + local password = environment.argument("password") + if type(certificate) ~= "string" or type(password) ~= "string" then + report("provide --certificate and --password") + return + end + lpdf.verify { + filename = pdffile, + certificate = certificate, + password = password, + uselibrary = environment.argument("uselibrary"), + } +end + function scripts.pdf.metadata(filename,pretty) local pdffile = loadpdffile(filename) if pdffile then @@ -187,7 +379,7 @@ end local expanded = lpdf.epdf.expanded local function getfonts(pdffile) - local usedfonts = { } + local usedfonts = { } local function collect(where,tag) local resources = where.Resources @@ -196,6 +388,9 @@ local function getfonts(pdffile) if fontlist then for k, v in expanded(fontlist) do usedfonts[tag and (tag .. "." .. k) or k] = v + if v.Subtype == "Type3" then + collect(v,tag and (tag .. "." .. k) or k) + end end end local objects = resources.XObject @@ -214,6 +409,8 @@ local function getfonts(pdffile) return usedfonts end +-- todo: fromunicode16 + local function getunicodes(font) local cid = font.ToUnicode if cid then @@ -228,9 +425,9 @@ local function getunicodes(font) setmetatableindex(counts, function(t,k) t[k] = 0 return 0 end) for s in gmatch(cid,"beginbfrange%s*(.-)%s*endbfrange") do for first, last, offset in gmatch(s,"<([^>]+)>%s+<([^>]+)>%s+<([^>]+)>") do - first = tonumber(first,16) - last = tonumber(last,16) - offset = tonumber(offset,16) + local first = tonumber(first,16) + local last = tonumber(last,16) + local offset = tonumber(offset,16) offset = offset - first for i=first,last do local c = i + offset @@ -267,7 +464,7 @@ function scripts.pdf.fonts(filename) indices = getunicodes(v) local codes = { } local chars = { } - local freqs = { } + -- local freqs = { } local names = { } if counts then codes = sortedkeys(counts) @@ -276,10 +473,10 @@ function scripts.pdf.fonts(filename) if k > 32 then local c = utfchar(k) chars[i] = c - freqs[i] = format("U+%05X %s %s",k,counts[k] > 1 and "+" or " ", c) + -- freqs[i] = format("U+%05X %s %s",k,counts[k] > 1 and "+" or " ", c) else chars[i] = k == 32 and "SPACE" or format("U+%03X",k) - freqs[i] = format("U+%05X %s --",k,counts[k] > 1 and "+" or " ") + -- freqs[i] = format("U+%05X %s --",k,counts[k] > 1 and "+" or " ") end end if basefont and unicode then @@ -302,20 +499,40 @@ function scripts.pdf.fonts(filename) end end end + if not basefont then + local fontdescriptor = v.FontDescriptor + if fontdescriptor then + basefont = fontdescriptor.FontName + end + end found[k] = { basefont = basefont or "no basefont", encoding = (d and "custom n=" .. #d) or "no encoding", subtype = subtype or "no subtype", - unicode = tounicode and "unicode" or "no vector", + unicode = unicode and "unicode" or "no vector", chars = chars, codes = codes, - freqs = freqs, + -- freqs = freqs, names = names, } end + local haschar = false + + local list = { } + for k, v in next, found do + local s = string.gsub(k,"(%d+)",function(s) return string.format("%05i",tonumber(s)) end) + list[s] = { k, v } + if #v.chars > 0 then + haschar = true + end + end + if details then for k, v in sortedhash(found) do +-- for s, f in sortedhash(list) do +-- local k = f[1] +-- local v = f[2] report("id : %s", k) report("basefont : %s", v.basefont) report("encoding : % t", v.names) @@ -335,16 +552,17 @@ function scripts.pdf.fonts(filename) report("") end else - local haschar = false - for k, v in sortedhash(found) do - if #v.chars > 0 then - haschar = true - break - end - end local results = { { "id", "basefont", "encoding", "subtype", "unicode", haschar and "characters" or nil } } - for k, v in sortedhash(found) do - results[#results+1] = { k, v.basefont, v.encoding, v.subtype, v.unicode, haschar and concat(v.chars," ") or nil } + local shared = { } + for s, f in sortedhash(list) do + local k = f[1] + local v = f[2] + local basefont = v.basefont + local characters = shared[basefont] or (haschar and concat(v.chars," ")) or nil + results[#results+1] = { k, v.basefont, v.encoding, v.subtype, v.unicode, characters } + if not shared[basefont] then + shared[basefont] = "shared with " .. k + end end utilities.formatters.formatcolumns(results) report(results[1]) @@ -382,43 +600,65 @@ function scripts.pdf.links(filename,asked) local reverse = swapped(pages) + local function banner(pagenumber) + report("") + report("annotations @ page %i",pagenumber) + report("") + end + local function show(pagenumber) local page = pages[pagenumber] local annots = page.Annots if annots then - report("") - report("annotations @ page %i",pagenumber) - report("") + local done = false for i=1,#annots do - local annot = annots[i] - if annot.Subtype == "Link" then - local A = annot.A - if A then - local S = A.S - local D = A.D - if S == "GoTo" then - if D then + local annotation = annots[i] + local a = annotation.A + if not a then + local d = annotation.Dest + if d then + a = { S = "GoTo", D = d } -- no need for a dict + end + end + if a then + local S = a.S + if S == "GoTo" then + local D = a.D + if D then + local D1 = D[1] + local R1 = reverse[D1] + if not done then + banner(pagenumber) + done = true + end + if tonumber(R1) then + report("intern, page % 4i",R1 or 0) + else + report("intern, name %s",tostring(D1)) + end + end + elseif S == "GoToR" then + local D = a.D + if D then + local F = A.F + if F then local D1 = D[1] - local R1 = reverse[D1] - if tonumber(R1) then - report("intern, page % 4i",R1 or 0) - else - report("intern, name %s",tostring(D1)) + if not done then + banner(pagenumber) + done = true end - end - elseif S == "GoToR" then - if D then - local F = A.F - if F then - local D1 = D[1] - if tonumber(D1) then - report("extern, page % 4i, file %s",D1 + 1,F) - else - report("extern, page % 4i, file %s, name %s",0,F,D[1]) - end + if tonumber(D1) then + report("extern, page % 4i, file %s",D1 + 1,F) + else + report("extern, page % 4i, file %s, name %s",0,F,D[1]) end end end + elseif S == "URI" then + local URI = a.URI + if URI then + report("extern, uri %a",URI) + end end end end @@ -483,12 +723,22 @@ elseif environment.argument("info") then scripts.pdf.info(filename) elseif environment.argument("metadata") then scripts.pdf.metadata(filename,environment.argument("pretty")) +elseif environment.argument("formdata") then + scripts.pdf.formdata(filename,environment.argument("save")) elseif environment.argument("fonts") then scripts.pdf.fonts(filename) elseif environment.argument("object") then scripts.pdf.object(filename,tonumber(environment.argument("object"))) elseif environment.argument("links") then scripts.pdf.links(filename,tonumber(environment.argument("page"))) +elseif environment.argument("signature") then + scripts.pdf.signature(filename,environment.argument("save")) +elseif environment.argument("sign") then + scripts.pdf.sign(filename) +elseif environment.argument("detail") then + scripts.pdf.detail(filename,environment.argument("detail")) +elseif environment.argument("verify") then + scripts.pdf.verify(filename) elseif environment.argument("exporthelp") then application.export(environment.argument("exporthelp"),filename) else diff --git a/Master/texmf-dist/scripts/context/lua/mtx-server-ctx-fonttest.lua b/Master/texmf-dist/scripts/context/lua/mtx-server-ctx-fonttest.lua index d5437587b2b..f1146b95f01 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-server-ctx-fonttest.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-server-ctx-fonttest.lua @@ -541,15 +541,20 @@ local function show_font(currentfont,detail) for f,ff in table.sortedhash(data) do local done = false for s, ss in table.sortedhash(ff) do - if s == "*" then s = "all" end - if ss ["*"] then ss["*"] = nil ss.all = true end + local s = s == "*" and "all" or s + if ss ["*"] then + ss["*"] = nil + ss.all = true + end + local name if done then - f = "" + name = "" else done = true + name = f end - local title = fonts.handlers.otf.tables.features[f] or "" - result[#result+1] = formatters["<tr><td width='50%%'>%s </td><td><tt>%s </tt></td><td><tt>%s </tt></td><td><tt>%s </tt></td></tr>"](title,f,s,concat(table.sortedkeys(ss)," ")) + local title = fonts.handlers.otf.tables.features[name] or "" + result[#result+1] = formatters["<tr><td width='50%%'>%s </td><td><tt>%s </tt></td><td><tt>%s </tt></td><td><tt>%s </tt></td></tr>"](title,name,s,concat(table.sortedkeys(ss)," ")) end end result[#result+1] = "</table>" diff --git a/Master/texmf-dist/scripts/context/lua/mtx-server.lua b/Master/texmf-dist/scripts/context/lua/mtx-server.lua index 4f7b9b5919a..85373a5a0b2 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-server.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-server.lua @@ -287,6 +287,7 @@ end handlers.luc = handlers.lua handlers.html = handlers.htm +handlers.pdf = handlers.generic local indices = { "index.htm", "index.html" } local portnumber = 8088 diff --git a/Master/texmf-dist/scripts/context/lua/mtx-synctex.lua b/Master/texmf-dist/scripts/context/lua/mtx-synctex.lua index f94a9605f3b..00e43f8703c 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-synctex.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-synctex.lua @@ -123,7 +123,7 @@ local function findlocation(filename,page,xpos,ypos) if find(line,"^}") then break else - -- we only look at positive cases + -- we only look at positive cases / could be sped up but it is not critical local f, l, x, y, w, h, d = match(line,"^h(.-),(.-):(.-),(.-):(.-),(.-),(.-)$") if f and f ~= 0 then x = tonumber(x) @@ -284,6 +284,8 @@ local function findlocation(filename,page,xpos,ypos,tolerance) locate( s,-s) if fi ~= 0 then tl = s ; goto done end locate(-s, s) if fi ~= 0 then tl = s ; goto done end locate(-s,-s) if fi ~= 0 then tl = s ; goto done end + locate( 0, s) if fi ~= 0 then tl = s ; goto done end + locate( 0,-s) if fi ~= 0 then tl = s ; goto done end end break else diff --git a/Master/texmf-dist/scripts/context/lua/mtx-unicode.lua b/Master/texmf-dist/scripts/context/lua/mtx-unicode.lua index ad42797b70d..cc58036703b 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-unicode.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-unicode.lua @@ -92,7 +92,7 @@ local application = logs.application { } local gmatch, match, gsub, find, lower, upper, format = string.gmatch, string.match, string.gsub, string.find, string.lower, string.upper, string.format -local concat, sort = table.concat, table.sort +local concat, sort, sortedhash = table.concat, table.sort, table.sortedhash local split, splitlines, strip = string.split, string.splitlines, string.strip local are_equal = table.are_equal local tonumber, tostring, rawget = tonumber, tostring, rawget @@ -424,13 +424,13 @@ function scripts.unicode.update() if l or r then mathextensible = 'm' -- mixed else - mathextensible = "u" -- up + mathextensible = "u" -- up end elseif d then if l or r then mathextensible = 'm' -- mixed else - mathextensible = "d" -- down + mathextensible = "d" -- down end elseif l and r then mathextensible = "h" -- horizontal @@ -449,7 +449,7 @@ function scripts.unicode.update() end end -- we need the hash .. add missing specials - for unicode, data in table.sortedhash(characterdata) do + for unicode, data in sortedhash(characterdata) do if not data.specials or data.comment and find(data.comment,"check special") then local description = data.description local b, m = match(description,"^(.+) WITH (.+)$") @@ -609,7 +609,7 @@ function scripts.unicode.load() index = splitindex(textdata.index), } -- - for k, v in table.sortedhash(textfiles) do + for k, v in sortedhash(textfiles) do report("using: %s",v) end return true @@ -673,7 +673,18 @@ end function scripts.unicode.save(filename) if preamble then - local data = table.serialize(characters.data,"characters.data", { hexify = true, noquotes = true }) + -- for k, v in next, characters.data do + -- v.adobename = nil + -- end + -- + characters.data[0x1FE3].uccode={ 0x3C5, 0x308, 0x301 } + characters.data[0x1FD3].uccode={ 0x3B9, 0x308, 0x301 } + characters.data[0x00DF].uccode={ 0x53, 0x53 } + -- + local data = table.serialize(characters.data,"characters.data", { + hexify = true, + noquotes = true, + }) data = gsub(data, "%{%s+%[0xFE0E%]=\"text style\",%s+%[0xFE0F%]=\"emoji style\",%s+%}", "variants_emoji" @@ -779,7 +790,7 @@ function scripts.unicode.extras() -- old code -- v.synonym = nil -- v.synonyms = nil -- end - for k, v in table.sortedhash(index) do + for k, v in sortedhash(index) do local d = data[v] if d and d.description ~= upper(k) then local synonyms = d.synonyms @@ -803,6 +814,53 @@ function scripts.unicode.extras() -- old code end end end +-- -- +-- for name, block in sortedhash(characters.blocks) do +-- if block.math then +-- local isalphabet = find(name,"lowercase") or find(name,"uppercase") or find(name,"letterlike") +-- for unicode=block.first,block.last do +-- local c = data[unicode] +-- if c.mathspec and #c.mathspec == 0 then +-- c.mathspec = { c.mathspec } +-- end +-- if c.mathclass then +-- if isalphabet and c.mathclass ~= "variable" then +-- report("CHECK %C : %s",unicode,c.description) +-- end +-- elseif c.mathspec then +-- -- skip +-- else +-- report("%s : %C : %s",name,unicode,c.description) +-- if isalphabet then +-- c.mathclass = "variable" +-- end +-- end +-- end +-- local gaps = block.gaps +-- if gaps then +-- for gap, unicode in sortedhash(gaps) do +-- local c = data[u] +-- if c.mathspec and #c.mathspec == 0 then +-- c.mathspec = { c.mathspec } +-- end +-- if c.mathclass then +-- if isalphabet and c.mathclass ~= "variable" then +-- report("CHECK %C : %s",gap,c.description) +-- end +-- -- skip +-- elseif c.mathspec then +-- -- skip +-- else +-- report("%s : %U -> %C : %s",name,gap,unicode,c.description) +-- if isalphabet then +-- c.mathclass = "variable" +-- end +-- end +-- end +-- end +-- end +-- end + end do @@ -907,7 +965,7 @@ else scripts.unicode.save("char-def-new.lua") scripts.unicode.emoji("char-emj-new.lua") report("saved file %a","char-def-new.lua") - report("saved file %a (current 14.0, check for updates, see above!)","char-emj-new.lua") + report("saved file %a (current 15.1, check for updates, see above!)","char-emj-new.lua") else report("nothing to do") end diff --git a/Master/texmf-dist/scripts/context/lua/mtx-update.lua b/Master/texmf-dist/scripts/context/lua/mtx-update.lua index 19e3e686581..7989bd7d46a 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-update.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-update.lua @@ -158,12 +158,12 @@ update.goodies = { } update.platforms = { - ["mswin"] = "mswin", - ["windows"] = "mswin", - ["win32"] = "mswin", - ["win"] = "mswin", + -- ["mswin"] = "mswin", + -- ["windows"] = "mswin", + -- ["win32"] = "mswin", + -- ["win"] = "mswin", -- - ["mswin-64"] = "win64", + ["windows"] = "win64", ["windows-64"] = "win64", ["win64"] = "win64", -- @@ -171,19 +171,21 @@ update.platforms = { ["linux-32"] = "linux", ["linux32"] = "linux", -- + -- ["linux"] = "linux-64", ["linux-64"] = "linux-64", ["linux64"] = "linux-64", -- + ["linuxmusl"] = "linuxmusl", ["linuxmusl-64"] = "linuxmusl-64", -- - ["linux-armhf"] = "linux-armhf", + -- ["linux-armhf"] = "linux-armhf", -- - ["openbsd"] = "openbsd7.3", - ["openbsd-i386"] = "openbsd7.3", - ["openbsd-amd64"] = "openbsd7.3-amd64", + ["openbsd"] = "openbsd-amd64", + -- ["openbsd-i386"] = "openbsd7.3", + ["openbsd-amd64"] = "openbsd-amd64", -- - ["freebsd"] = "freebsd", - ["freebsd-i386"] = "freebsd", + ["freebsd"] = "freebsd-amd64", + -- ["freebsd-i386"] = "freebsd", ["freebsd-amd64"] = "freebsd-amd64", -- -- ["kfreebsd"] = "kfreebsd-i386", @@ -205,6 +207,7 @@ update.platforms = { -- ["macosx"] = "osx-64", ["osx"] = "osx-64", + ["osx-intel"] = "osx-64", ["osx-64"] = "osx-64", ["osx-arm"] = "osx-arm64", ["osx-arm64"] = "osx-arm64", @@ -218,8 +221,8 @@ update.platforms = { } local windowsplatform = { - ["mswin"] = true, - ["win32"] = true, + ["mswin"] = true, -- hm + ["win32"] = true, -- hm ["win64"] = true, } @@ -682,15 +685,11 @@ if scripts.savestate then -- states.set("formats.metafun", true) for r in gmatch(environment.argument("extras") or "","([^, ]+)") do -- for old times sake - if r ~= "all" and not find(r,"^[a-z]%-") then - r = "t-" .. r - end + local r = (r ~= "all" and not find(r,"^[a-z]%-") and ("t-" .. r)) or r states.set("modules." .. r, true) end for r in gmatch(environment.argument("modules") or "","([^, ]+)") do - if r ~= "all" and not find(r,"^[a-z]%-") then - r = "t-" .. r - end + local r = (r ~= "all" and not find(r,"^[a-z]%-") and ("t-" .. r)) or r states.set("modules." .. r, true) end for r in gmatch(environment.argument("fonts") or "","([^, ]+)") do diff --git a/Master/texmf-dist/scripts/context/lua/mtx-watch.lua b/Master/texmf-dist/scripts/context/lua/mtx-watch.lua index 7815dafdd7a..e46bb39fdad 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-watch.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-watch.lua @@ -91,7 +91,7 @@ local function glob(files,path) -- some day: sort by name (order prefix) and ati if name:find("^%.") then -- skip . and .. else - name = path .. "/" .. name + local name = path .. "/" .. name local a = lfsattributes(name) if not a then -- weird diff --git a/Master/texmf-dist/scripts/context/lua/mtxrun.lua b/Master/texmf-dist/scripts/context/lua/mtxrun.lua index 8c28a90e38a..59bc6e65928 100644 --- a/Master/texmf-dist/scripts/context/lua/mtxrun.lua +++ b/Master/texmf-dist/scripts/context/lua/mtxrun.lua @@ -1,6 +1,4 @@ #!/usr/bin/env texlua --- 8dec23 --socket added to runners.applications lua and luc per hans --- (mail of 7dec23). if not modules then modules = { } end modules ['mtxrun'] = { version = 1.001, @@ -2143,7 +2141,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-table"] = package.loaded["l-table"] or true --- original size: 41758, stripped down to: 22643 +-- original size: 42643, stripped down to: 23053 if not modules then modules={} end modules ['l-table']={ version=1.001, @@ -2435,19 +2433,22 @@ local function copy(t,tables) tables[t]=tcopy end for i,v in next,t do + local k if type(i)=="table" then if tables[i] then - i=tables[i] + k=tables[i] else - i=copy(i,tables) + k=copy(i,tables) end + else + k=i end if type(v)~="table" then - tcopy[i]=v + tcopy[k]=v elseif tables[v] then - tcopy[i]=tables[v] + tcopy[k]=tables[v] else - tcopy[i]=copy(v,tables) + tcopy[k]=copy(v,tables) end end local mt=getmetatable(t) @@ -2644,7 +2645,11 @@ local function do_serialize(root,name,depth,level,indexed) elseif tv=="number" then if tk=="number" then if hexify then - handle(format("%s [0x%X]=0x%X,",depth,k,v)) + if accurate then + handle(format("%s [0x%X]=%q,",depth,k,v)) + else + handle(format("%s [0x%X]=%s,",depth,k,v)) + end elseif accurate then handle(format("%s [%s]=%q,",depth,k,v)) else @@ -2652,7 +2657,11 @@ local function do_serialize(root,name,depth,level,indexed) end elseif tk=="boolean" then if hexify then - handle(format("%s [%s]=0x%X,",depth,k and "true" or "false",v)) + if accurate then + handle(format("%s [%s]=%q,",depth,k and "true" or "false",v)) + else + handle(format("%s [%s]=%s,",depth,k and "true" or "false",v)) + end elseif accurate then handle(format("%s [%s]=%q,",depth,k and "true" or "false",v)) else @@ -2661,7 +2670,11 @@ local function do_serialize(root,name,depth,level,indexed) elseif tk~="string" then elseif noquotes and not reserved[k] and lpegmatch(propername,k) then if hexify then - handle(format("%s %s=0x%X,",depth,k,v)) + if accurate then + handle(format("%s %s=%q,",depth,k,v)) + else + handle(format("%s %s=0x%X,",depth,k,v)) + end elseif accurate then handle(format("%s %s=%q,",depth,k,v)) else @@ -2669,7 +2682,11 @@ local function do_serialize(root,name,depth,level,indexed) end else if hexify then - handle(format("%s [%q]=0x%X,",depth,k,v)) + if accurate then + handle(format("%s [%q]=%q,",depth,k,v)) + else + handle(format("%s [%q]=0x%X,",depth,k,v)) + end elseif accurate then handle(format("%s [%q]=%q,",depth,k,v)) else @@ -3850,7 +3867,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-os"] = package.loaded["l-os"] or true --- original size: 20585, stripped down to: 10701 +-- original size: 20686, stripped down to: 10791 if not modules then modules={} end modules ['l-os']={ version=1.001, @@ -3990,6 +4007,8 @@ os.type=os.type or (io.pathseparator==";" and "windows") or "unix" os.name=os.name or (os.type=="windows" and "mswin" ) or "linux" if os.type=="windows" then os.libsuffix,os.binsuffix,os.binsuffixes='dll','exe',{ 'exe','cmd','bat' } +elseif os.name=="macosx" then + os.libsuffix,os.binsuffix,os.binsuffixes='dylib','',{ '' } else os.libsuffix,os.binsuffix,os.binsuffixes='so','',{ '' } end @@ -6585,7 +6604,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-str"] = package.loaded["util-str"] or true --- original size: 46322, stripped down to: 24128 +-- original size: 46976, stripped down to: 24530 if not modules then modules={} end modules ['util-str']={ version=1.001, @@ -6779,6 +6798,7 @@ local p_prune_intospace=Cs (noleading*(notrailing+intospace+1 )^0 ) local p_retain_normal=Cs ((normalline+normalempty )^0 ) local p_retain_collapse=Cs ((normalline+doubleempty )^0 ) local p_retain_noempty=Cs ((normalline+singleempty )^0 ) +local p_collapse_all=Cs (stripstart*(stripend+((whitespace+newline)^1/" ")+1)^0 ) local striplinepatterns={ ["prune"]=p_prune_normal, ["prune and collapse"]=p_prune_collapse, @@ -6787,6 +6807,7 @@ local striplinepatterns={ ["retain"]=p_retain_normal, ["retain and collapse"]=p_retain_collapse, ["retain and no empty"]=p_retain_noempty, + ["collapse all"]=p_collapse_all, ["collapse"]=patterns.collapser, } setmetatable(striplinepatterns,{ __index=function(t,k) return p_prune_collapse end }) @@ -7463,6 +7484,7 @@ end strings.formatters.add=add patterns.xmlescape=Cs((P("<")/"<"+P(">")/">"+P("&")/"&"+P('"')/"""+anything)^0) patterns.texescape=Cs((C(S("#$%\\{}"))/"\\%1"+anything)^0) +patterns.ctxescape=Cs((C(S("#$%\\{}|"))/"\\%1"+anything)^0) patterns.luaescape=Cs(((1-S('"\n'))^1+P('"')/'\\"'+P('\n')/'\\n"')^0) patterns.luaquoted=Cs(Cc('"')*((1-S('"\n'))^1+P('"')/'\\"'+P('\n')/'\\n"')^0*Cc('"')) add(formatters,"xml",[[lpegmatch(xmlescape,%s)]],{ xmlescape=patterns.xmlescape }) @@ -7533,6 +7555,20 @@ do return cache[s] end end +if CONTEXTLMTXMODE and CONTEXTLMTXMODE>0 then + local t={ + ["#"]="#H", + ["\n"]="#L", + ['"']="#Q", + ["\r"]="#R", + [" "]="#S", + ["\t"]="#T", + ["\\"]="#X", + } + function string.texhashed(s) + return (gsub(s,".",t)) + end +end end -- of closure @@ -9349,7 +9385,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-prs"] = package.loaded["util-prs"] or true --- original size: 25716, stripped down to: 16805 +-- original size: 26298, stripped down to: 17137 if not modules then modules={} end modules ['util-prs']={ version=1.001, @@ -9645,7 +9681,17 @@ hashes.settings_to_set=table.setmetatableindex(function(t,k) t[k]=v return v end) -getmetatable(hashes.settings_to_set).__mode="kv" +function parsers.settings_to_set(str) + return str and lpegmatch(pattern,str) or {} +end +local pattern=Ct((C((1-S(", "))^1)*S(", ")^0)^1) +hashes.settings_to_list=table.setmetatableindex(function(t,k) + local v=k and lpegmatch(pattern,k) or {} + t[k]=v + return v +end) +getmetatable(hashes.settings_to_set ).__mode="kv" +getmetatable(hashes.settings_to_list).__mode="kv" function parsers.simple_hash_to_string(h,separator) local t={} local tn=0 @@ -9667,7 +9713,7 @@ local splitter=lpeg.tsplitat(" ") function parsers.options_to_array(str) return str and lpegmatch(splitter,str) or {} end -local value=P(lbrace*C((nobrace+nestedbraces)^0)*rbrace)+C(digit^1*lparent*(noparent+nestedparents)^1*rparent)+C((nestedbraces+(1-comma))^1) +local value=P(lbrace*C((nobrace+nestedbraces)^0)*rbrace)+C(digit^1*lparent*(noparent+nestedparents)^1*rparent)+C((nestedbraces+(1-comma))^1)+Cc("") local pattern_a=spaces*Ct(value*(separator*value)^0) local function repeater(n,str) if not n then @@ -9689,7 +9735,7 @@ local function repeater(n,str) end end end -local value=P(lbrace*C((nobrace+nestedbraces)^0)*rbrace)+(C(digit^1)/tonumber*lparent*Cs((noparent+nestedparents)^1)*rparent)/repeater+C((nestedbraces+(1-comma))^1) +local value=P(lbrace*C((nobrace+nestedbraces)^0)*rbrace)+(C(digit^1)/tonumber*lparent*Cs((noparent+nestedparents)^1)*rparent)/repeater+C((nestedbraces+(1-comma))^1)+Cc("") local pattern_b=spaces*Ct(value*(separator*value)^0) function parsers.settings_to_array_with_repeat(str,expand) if expand then @@ -11903,7 +11949,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-soc-imp-http"] = package.loaded["util-soc-imp-http"] or true --- original size: 12624, stripped down to: 9598 +-- original size: 13055, stripped down to: 9818 local tostring,tonumber,setmetatable,next,type=tostring,tonumber,setmetatable,next,type @@ -11955,7 +12001,7 @@ local function receiveheaders(sock,headers) while line~="" do local name,value=skipsocket(2,find(line,"^(.-):%s*(.*)")) if not (name and value) then - return nil,"malformed reponse headers" + return nil,"malformed response headers" end name=lower(name) line,err=sock:receive("*l") @@ -12058,9 +12104,13 @@ function methods.sendbody(self,headers,source,step) end function methods.receivestatusline(self) local try=self.try - local status=try(self.c:receive(5)) + local status,err=try(self.c:receive(5)) if status~="HTTP/" then - return nil,status + if err=="timeout" then + return 408 + else + return nil,status + end end status=try(self.c:receive("*l",status)) local code=skipsocket(2,find(status,"HTTP/%d*%.%d* (%d%d%d)")) @@ -12165,7 +12215,7 @@ local function adjustrequest(originalrequest) end return request end -local maxredericts=4 +local maxredericts=5 local validredirects={ [301]=true,[302]=true,[303]=true,[307]=true } local validmethods={ [false]=true,GET=true,HEAD=true } local function shouldredirect(request,code,headers) @@ -12184,7 +12234,8 @@ local function shouldredirect(request,code,headers) local method=request.method local redirect=request.redirect local redirects=request.nredirects or 0 - return redirect and validredirects[code] and validmethods[method] and redirects<=maxredericts + local maxredirects=request.maxredirects or maxredirects + return redirect and validredirects[code] and validmethods[method] and redirects<=maxredirects end local function shouldreceivebody(request,code) if request.method=="HEAD" then @@ -12207,6 +12258,7 @@ tredirect=function(request,location) headers=request.headers, proxy=request.proxy, nredirects=(request.nredirects or 0)+1, + maxredirects=request.maxredirects or maxredirects, create=request.create, } if not headers then @@ -12229,10 +12281,13 @@ trequest=function(originalrequest) local code,status=connection:receivestatusline() if not code then connection:receive09body(status,request.sink,request.step) + connection:close() return 1,200 + elseif code==408 then + return 1,code end while code==100 do - headers=connection:receiveheaders() + connection:receiveheaders() code,status=connection:receivestatusline() end headers=connection:receiveheaders() @@ -12285,7 +12340,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-soc-imp-ftp"] = package.loaded["util-soc-imp-ftp"] or true --- original size: 10357, stripped down to: 8548 +-- original size: 10345, stripped down to: 8538 local setmetatable,type,next=setmetatable,type,next @@ -12396,7 +12451,7 @@ function methods.port(self,address,port) local tp=self.tp self.pasvt=nil if not address then - address,port=try(tp:getsockname()) + address=try(tp:getsockname()) self.server=try(bindsocket(address,0)) address,port=try(self.server:getsockname()) try(self.server:settimeout(ftp.TIMEOUT)) @@ -12413,7 +12468,7 @@ function methods.eprt(self,family,address,port) local tp=self.tp self.pasvt=nil if not address then - address,port=try(tp:getsockname()) + address=try(tp:getsockname()) self.server=try(bindsocket(address,0)) address,port=try(self.server:getsockname()) try(self.server:settimeout(ftp.TIMEOUT)) @@ -12897,7 +12952,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-set"] = package.loaded["trac-set"] or true --- original size: 14568, stripped down to: 9644 +-- original size: 14574, stripped down to: 9650 if not modules then modules={} end modules ['trac-set']={ version=1.001, @@ -12982,10 +13037,10 @@ local function set(t,what,newvalue) else value=is_boolean(value,value,true) end - w=topattern(w,true,true) + local p=topattern(w,true,true) for name,functions in sortedhash(data) do if done[name] then - elseif find(name,w) then + elseif find(name,p) then done[name]=true for i=1,#functions do functions[i](value) @@ -13267,7 +13322,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-log"] = package.loaded["trac-log"] or true --- original size: 16017, stripped down to: 11045 +-- original size: 16046, stripped down to: 11072 if not modules then modules={} end modules ['trac-log']={ version=1.001, @@ -13282,6 +13337,7 @@ local concat,insert,remove=table.concat,table.insert,table.remove local topattern=string.topattern local utfchar=utf.char local datetime=os.date +local sleep=os.sleep local openfile=io.open local write_nl=print local write=io.write @@ -13554,9 +13610,9 @@ local function setblocked(category,value) if v then v.state=value else - c=topattern(c,true,true) + local p=topattern(c,true,true) for k,v in next,data do - if find(k,c) then + if find(k,p) then v.state=value end end @@ -14156,7 +14212,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-lua"] = package.loaded["util-lua"] or true --- original size: 7149, stripped down to: 4997 +-- original size: 7166, stripped down to: 5009 if not modules then modules={} end modules ['util-lua']={ version=1.001, @@ -14190,6 +14246,7 @@ luautilities.suffixes={ tma="tma", tmc=(CONTEXTLMTXMODE and CONTEXTLMTXMODE>0 and "tmd") or (jit and "tmb") or "tmc", lua="lua", + lmt="lmt", luc=(CONTEXTLMTXMODE and CONTEXTLMTXMODE>0 and "lud") or (jit and "lub") or "luc", lui="lui", luv="luv", @@ -14334,7 +14391,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-deb"] = package.loaded["util-deb"] or true --- original size: 10593, stripped down to: 7102 +-- original size: 10416, stripped down to: 7076 if not modules then modules={} end modules ['util-deb']={ version=1.001, @@ -14499,10 +14556,7 @@ function debugger.showstats(printer,threshold) realtime=realtime+real end totaltime=totaltime+total - if line<0 then - line=0 - end - dataset[#dataset+1]={ real,total,count,name,source,line } + dataset[#dataset+1]={ real,total,count,name,source,line<0 and 0 or line } end end end @@ -15498,7 +15552,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-env"] = package.loaded["util-env"] or true --- original size: 9738, stripped down to: 5531 +-- original size: 10014, stripped down to: 5102 if not modules then modules={} end modules ['util-env']={ version=1.001, @@ -15513,22 +15567,7 @@ local unquoted,quoted,optionalquoted=string.unquoted,string.quoted,string.option local concat,insert,remove=table.concat,table.insert,table.remove environment=environment or {} local environment=environment -local setlocale=os.setlocale -setlocale(nil,nil) -local report=logs.reporter("system") -function os.setlocale(a,b) - if a or b then - if report then - report() - report("You're messing with os.locale in a supposedly locale neutral enviroment. From") - report("now on are on your own and without support. Crashes or unexpected side effects") - report("can happen but don't bother the luatex and context developer team with it.") - report() - report=nil - end - setlocale(a,b) - end -end +os.setlocale(nil,nil) function os.setlocale() end local validengines=allocate { ["luatex"]=true, ["luajittex"]=true, @@ -15606,7 +15645,8 @@ function environment.setargument(name,value) environment.arguments[name]=value end function environment.getargument(name,partial) - local arguments,sortedflags=environment.arguments,environment.sortedflags + local arguments=environment.arguments + local sortedflags=environment.sortedflags if arguments[name] then return arguments[name] elseif partial then @@ -15713,7 +15753,7 @@ do -- create closure to overcome 200 locals limit package.loaded["luat-env"] = package.loaded["luat-env"] or true --- original size: 6267, stripped down to: 4115 +-- original size: 6293, stripped down to: 4141 if not modules then modules={} end modules ['luat-env']={ version=1.001, @@ -15786,7 +15826,7 @@ local function strippable(filename) return false end end -function environment.luafilechunk(filename,silent,macros) +function environment.luafilechunk(filename,silent,macros,optional) filename=file.replacesuffix(filename,"lua") local fullname=environment.luafile(filename) if fullname and fullname~="" then @@ -15796,7 +15836,7 @@ function environment.luafilechunk(filename,silent,macros) end return data else - if not silent then + if not optional and not silent then report_lua("unknown file %a",filename) end return nil @@ -15878,7 +15918,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-zip"] = package.loaded["util-zip"] or true --- original size: 23730, stripped down to: 14293 +-- original size: 33051, stripped down to: 16324 if not modules then modules={} end modules ['util-zip']={ version=1.001, @@ -15904,11 +15944,13 @@ end local files=utilities.files local openfile=files.open local closefile=files.close +local getsize=files.size local readstring=files.readstring local readcardinal2=files.readcardinal2le local readcardinal4=files.readcardinal4le local setposition=files.setposition local getposition=files.getposition +local skipbytes=files.skip local band=bit32.band local rshift=bit32.rshift local lshift=bit32.lshift @@ -15931,6 +15973,55 @@ local openzipfile,closezipfile,unzipfile,foundzipfile,getziphash,getziplist do handle=openfile(name,0), } end + local function update(handle,data) + position=data.offset + setposition(handle,position) + local signature=readstring(handle,4) + if signature=="PK\3\4" then + local version=readcardinal2(handle) + local flag=readcardinal2(handle) + local method=readcardinal2(handle) + skipbytes(handle,4) + local crc32=readcardinal4(handle) + local compressed=readcardinal4(handle) + local uncompressed=readcardinal4(handle) + local namelength=readcardinal2(handle) + local extralength=readcardinal2(handle) + local filename=readstring(handle,namelength) + local descriptor=band(flag,8)~=0 + local encrypted=band(flag,1)~=0 + local acceptable=method==0 or method==8 + local skipped=0 + local size=0 + if encrypted then + size=readcardinal2(handle) + skipbytes(handle,size) + skipped=skipped+size+2 + skipbytes(8) + skipped=skipped+8 + size=readcardinal2(handle) + skipbytes(handle,size) + skipped=skipped+size+2 + size=readcardinal4(handle) + skipbytes(handle,size) + skipped=skipped+size+4 + size=readcardinal2(handle) + skipbytes(handle,size) + skipped=skipped+size+2 + end + if acceptable then + if filename~=data.filename then + else + position=position+30+namelength+extralength+skipped + data.position=position + return position + end + else + end + end + data.position=false + return false + end local function collect(z) if not z.list then local list={} @@ -15938,72 +16029,68 @@ local openzipfile,closezipfile,unzipfile,foundzipfile,getziphash,getziplist do local position=0 local index=0 local handle=z.handle - while true do - setposition(handle,position) - local signature=readstring(handle,4) - if signature=="PK\3\4" then - local version=readcardinal2(handle) - local flag=readcardinal2(handle) - local method=readcardinal2(handle) - local filetime=readcardinal2(handle) - local filedate=readcardinal2(handle) - local crc32=readcardinal4(handle) - local compressed=readcardinal4(handle) - local uncompressed=readcardinal4(handle) - local namelength=readcardinal2(handle) - local extralength=readcardinal2(handle) - local filename=readstring(handle,namelength) - local descriptor=band(flag,8)~=0 - local encrypted=band(flag,1)~=0 - local acceptable=method==0 or method==8 - local skipped=0 - local size=0 - if encrypted then - size=readcardinal2(handle) - skipbytes(size) - skipped=skipped+size+2 - skipbytes(8) - skipped=skipped+8 - size=readcardinal2(handle) - skipbytes(size) - skipped=skipped+size+2 - size=readcardinal4(handle) - skipbytes(size) - skipped=skipped+size+4 - size=readcardinal2(handle) - skipbytes(size) - skipped=skipped+size+2 - end - position=position+30+namelength+extralength+skipped - if descriptor then - setposition(handle,position+compressed) - crc32=readcardinal4(handle) - compressed=readcardinal4(handle) - uncompressed=readcardinal4(handle) - end - if acceptable then - index=index+1 - local data={ - filename=filename, - index=index, - position=position, - method=method, - compressed=compressed, - uncompressed=uncompressed, - crc32=crc32, - encrypted=encrypted, - } - hash[filename]=data - list[index]=data - else + local size=getsize(handle) + for i=size-4,size-64*1024,-1 do + setposition(handle,i) + local enddirsignature=readcardinal4(handle) + if enddirsignature==0x06054B50 then + local thisdisknumber=readcardinal2(handle) + local centraldisknumber=readcardinal2(handle) + local thisnofentries=readcardinal2(handle) + local totalnofentries=readcardinal2(handle) + local centralsize=readcardinal4(handle) + local centraloffset=readcardinal4(handle) + local commentlength=readcardinal2(handle) + local comment=readstring(handle,length) + if size-i>=22 then + if thisdisknumber==centraldisknumber then + setposition(handle,centraloffset) + while true do + if readcardinal4(handle)==0x02014B50 then + skipbytes(handle,4) + local flag=readcardinal2(handle) + local method=readcardinal2(handle) + skipbytes(handle,4) + local crc32=readcardinal4(handle) + local compressed=readcardinal4(handle) + local uncompressed=readcardinal4(handle) + local namelength=readcardinal2(handle) + local extralength=readcardinal2(handle) + local commentlength=readcardinal2(handle) + skipbytes(handle,8) + local headeroffset=readcardinal4(handle) + local filename=readstring(handle,namelength) + skipbytes(handle,extralength+commentlength) + local descriptor=band(flag,8)~=0 + local encrypted=band(flag,1)~=0 + local acceptable=method==0 or method==8 + if acceptable then + index=index+1 + local data={ + filename=filename, + index=index, + position=nil, + method=method, + compressed=compressed, + uncompressed=uncompressed, + crc32=crc32, + encrypted=encrypted, + offset=headeroffset, + } + hash[filename]=data + list[index]=data + end + else + break + end + end + end + break end - position=position+compressed - else - break end - z.list=list - z.hash=hash end + z.list=list + z.hash=hash end end function getziplist(z) @@ -16042,7 +16129,10 @@ local openzipfile,closezipfile,unzipfile,foundzipfile,getziphash,getziplist do local handle=z.handle local position=data.position local compressed=data.compressed - if compressed>0 then + if position==nil then + position=update(handle,data) + end + if position and compressed>0 then setposition(handle,position) local result=readstring(handle,compressed) if data.method==8 then @@ -16253,8 +16343,10 @@ if xzip then closezip(zipf) end end - local function unzipdir(zipname,path,verbose) + local function unzipdir(zipname,path,verbose,collect,validate) if type(zipname)=="table" then + validate=zipname.validate + collect=zipname.collect verbose=zipname.verbose path=zipname.path zipname=zipname.zipname @@ -16275,34 +16367,47 @@ if xzip then local done=0 local steps=verbose=="steps" local time=steps and osclock() + if collect then + collect={} + else + collect=false + end for i=1,count do local l=list[i] local n=l.filename - local d=unzipfile(z,n) - if d then - local p=filejoin(path,n) - if mkdirs(dirname(p)) then - if steps then - total=total+#d - done=done+1 - if done>=step then - done=0 - logwriter(format("%4i files of %4i done, %10i bytes, %0.3f seconds",i,count,total,osclock()-time)) + if not validate or validate(n) then + local d=unzipfile(z,n) + if d then + local p=filejoin(path,n) + if mkdirs(dirname(p)) then + if steps then + total=total+#d + done=done+1 + if done>=step then + done=0 + logwriter(format("%4i files of %4i done, %10i bytes, %0.3f seconds",i,count,total,osclock()-time)) + end + elseif verbose then + logwriter(n) + end + savedata(p,d) + if collect then + collect[#collect+1]=p end - elseif verbose then - logwriter(n) end - savedata(p,d) + else + logwriter(format("problem with file %s",n)) end else - logwriter(format("problem with file %s",n)) end end if steps then logwriter(format("%4i files of %4i done, %10i bytes, %0.3f seconds",count,count,total,osclock()-time)) end closezipfile(z) - return true + if collect then + return collect + end else closezipfile(z) end @@ -16440,7 +16545,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-tab"] = package.loaded["lxml-tab"] or true --- original size: 62221, stripped down to: 36225 +-- original size: 62465, stripped down to: 36432 if not modules then modules={} end modules ['lxml-tab']={ version=1.001, @@ -16824,8 +16929,13 @@ do [ [[}]] ]="&U+7D;", [ [[~]] ]="&U+7E;", } - local privates_n={ + local privates_n={ } + utilities.storage.mark(privates_u) + utilities.storage.mark(privates_p) + utilities.storage.mark(privates_s) + utilities.storage.mark(privates_x) + utilities.storage.mark(privates_n) local escaped=utf.remapper(privates_u,"dynamic") local unprivatized=utf.remapper(privates_p,"dynamic") local unspecialized=utf.remapper(privates_s,"dynamic") @@ -17549,7 +17659,11 @@ local f_attribute=formatters['%s=%q'] local function verbose_element(e,handlers,escape) local handle=handlers.handle local serialize=handlers.serialize - local ens,etg,eat,edt,ern=e.ns,e.tg,e.at,e.dt,e.rn + local ens=e.ns + local etg=e.tg + local eat=e.at + local edt=e.dt + local ern=e.rn local ats=eat and next(eat) and {} if ats then local n=0 @@ -17921,7 +18035,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-lpt"] = package.loaded["lxml-lpt"] or true --- original size: 54589, stripped down to: 31258 +-- original size: 54738, stripped down to: 31395 if not modules then modules={} end modules ['lxml-lpt']={ version=1.001, @@ -18925,9 +19039,11 @@ expressions.print=function(...) print(...) return true end -expressions.find=find -expressions.upper=upper -expressions.lower=lower +expressions.find=function(str,...) + return str and find(str,...) +end +expressions.upper=function(str) return str and upper(str) or "" end +expressions.lower=function(str) return str and lower(str) or "" end expressions.number=tonumber expressions.boolean=toboolean function expressions.contains(str,pattern) @@ -21053,7 +21169,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-exp"] = package.loaded["data-exp"] or true --- original size: 18179, stripped down to: 10432 +-- original size: 18185, stripped down to: 10438 if not modules then modules={} end modules ['data-exp']={ version=1.001, @@ -21135,10 +21251,10 @@ local function splitpathexpr(str,newlist,validate) str=lpegmatch(stripper_1,str) if validate then for s in gmatch(str,"[^,]+") do - s=validate(s) - if s then + local v=validate(s) + if v then n=n+1 - t[n]=s + t[n]=v end end else @@ -21461,7 +21577,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-env"] = package.loaded["data-env"] or true --- original size: 9501, stripped down to: 6413 +-- original size: 9758, stripped down to: 6523 if not modules then modules={} end modules ['data-env']={ version=1.001, @@ -21475,6 +21591,7 @@ local next,rawget=next,rawget local resolvers=resolvers local allocate=utilities.storage.allocate local setmetatableindex=table.setmetatableindex +local sortedhash=table.sortedhash local suffixonly=file.suffixonly local formats=allocate() local suffixes=allocate() @@ -21576,7 +21693,7 @@ local relations=allocate { lua={ names={ "lua" }, variable='LUAINPUTS', - suffixes={ luasuffixes.lua,luasuffixes.luc,luasuffixes.tma,luasuffixes.tmc }, + suffixes={ luasuffixes.lmt,luasuffixes.lua,luasuffixes.luc,luasuffixes.tma,luasuffixes.tmc }, usertype=true, }, lib={ @@ -21677,8 +21794,8 @@ local relations=allocate { } resolvers.relations=relations function resolvers.updaterelations() - for category,categories in next,relations do - for name,relation in next,categories do + for category,categories in sortedhash(relations) do + for name,relation in sortedhash(categories) do local rn=relation.names local rv=relation.variable if rn and rv then @@ -21691,7 +21808,9 @@ function resolvers.updaterelations() suffixes[rni]=rs for i=1,#rs do local rsi=rs[i] - suffixmap[rsi]=rni + if not suffixmap[rsi] then + suffixmap[rsi]=rni + end end end end @@ -22283,7 +22402,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-res"] = package.loaded["data-res"] or true --- original size: 70711, stripped down to: 44839 +-- original size: 70699, stripped down to: 45019 if not modules then modules={} end modules ['data-res']={ version=1.001, @@ -22348,18 +22467,37 @@ local criticalvars={ "TEXMF", "TEXOS", } -if environment.default_texmfcnf then - resolvers.luacnfspec="home:texmf/web2c;"..environment.default_texmfcnf -else +do local texroot=environment.texroot - resolvers.luacnfspec="home:texmf/web2c;selfautoparent:/texmf-local/web2c;selfautoparent:/texmf-context/web2c;selfautoparent:/texmf/web2c" - if texroot and isdir(texroot.."/texmf-context") then + resolvers.luacnfspec={ + "home:texmf/web2c", + "selfautoparent:/texmf-local/web2c", + "selfautoparent:/texmf-context/web2c", + "selfautoparent:/texmf/web2c", + } + if environment.default_texmfcnf then + resolvers.luacnfspec={ + "home:texmf/web2c", + environment.default_texmfcnf, + } + elseif texroot and isdir(texroot.."/texmf-context") then elseif texroot and isdir(texroot.."/texmf-dist") then - resolvers.luacnfspec="home:texmf/web2c;selfautoparent:/texmf-local/web2c;selfautoparent:/texmf-dist/web2c;selfautoparent:/texmf/web2c" + resolvers.luacnfspec={ + "home:texmf/web2c", + "selfautoparent:/texmf-local/web2c", + "selfautoparent:", + "selfautoparent:/texmf-dist/web2c", + "selfautoparent:/texmf/web2c", + } elseif ostype~="windows" and isdir("/etc/texmf/web2c") then - resolvers.luacnfspec="home:texmf/web2c;/etc/texmf/web2c;selfautodir:/share/texmf/web2c" + resolvers.luacnfspec={ + "home:texmf/web2c", + "/etc/texmf/web2c", + "selfautodir:/share/texmf/web2c", + } else end + resolvers.luacnfspec=concat(resolvers.luacnfspec,";") end local unset_variable="unset" local formats=resolvers.formats @@ -23200,7 +23338,7 @@ end local preparetreepattern=Cs((P(".")/"%%."+P("-")/"%%-"+P(1))^0*Cc("$")) local collect_instance_files local function find_analyze(filename,askedformat,allresults) - local filetype='' + local filetype="" local filesuffix=suffixonly(filename) local wantedfiles={} wantedfiles[#wantedfiles+1]=filename @@ -23208,7 +23346,7 @@ local function find_analyze(filename,askedformat,allresults) if filesuffix=="" or not suffixmap[filesuffix] then local defaultsuffixes=resolvers.defaultsuffixes for i=1,#defaultsuffixes do - local forcedname=filename..'.'..defaultsuffixes[i] + local forcedname=filename.."."..defaultsuffixes[i] wantedfiles[#wantedfiles+1]=forcedname filetype=formatofsuffix(forcedname) if trace_locating then @@ -23552,7 +23690,8 @@ collect_instance_files=function(filename,askedformat,allresults) local result={} local status={} local done={} - for k,r in next,results do + for k=1,#results do + local r=results[k] local method,list=r[1],r[2] if method and list then for i=1,#list do @@ -25750,7 +25889,7 @@ do -- create closure to overcome 200 locals limit package.loaded["luat-sta"] = package.loaded["luat-sta"] or true --- original size: 5703, stripped down to: 2321 +-- original size: 5608, stripped down to: 2321 if not modules then modules={} end modules ['luat-sta']={ version=1.001, @@ -26140,8 +26279,8 @@ end -- of closure -- used libraries : l-bit32.lua l-lua.lua l-macro.lua l-sandbox.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-sha.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-fil.lua util-sac.lua util-sto.lua util-prs.lua util-fmt.lua util-soc-imp-reset.lua util-soc-imp-socket.lua util-soc-imp-copas.lua util-soc-imp-ltn12.lua util-soc-imp-mime.lua util-soc-imp-url.lua util-soc-imp-headers.lua util-soc-imp-tp.lua util-soc-imp-http.lua util-soc-imp-ftp.lua util-soc-imp-smtp.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-tpl.lua util-sbx.lua util-mrg.lua util-env.lua luat-env.lua util-zip.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua libs-ini.lua luat-sta.lua luat-fmt.lua -- skipped libraries : - --- original bytes : 1037585 --- stripped bytes : 408686 +-- original bytes : 1050273 +-- stripped bytes : 417643 -- end library merge diff --git a/Master/texmf-dist/scripts/context/stubs-mkiv/unix/contextjit b/Master/texmf-dist/scripts/context/stubs-mkiv/unix/contextjit deleted file mode 100755 index d1374801b2e..00000000000 --- a/Master/texmf-dist/scripts/context/stubs-mkiv/unix/contextjit +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -luajittex --luaonly `kpsewhich mtxrun.lua` --script context "$@" diff --git a/Master/texmf-dist/scripts/context/stubs-mkiv/unix/luatools b/Master/texmf-dist/scripts/context/stubs-mkiv/unix/luatools deleted file mode 100755 index c17b483be53..00000000000 --- a/Master/texmf-dist/scripts/context/stubs-mkiv/unix/luatools +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -mtxrun --script base "$@" diff --git a/Master/texmf-dist/scripts/context/stubs-mkiv/unix/mtxrunjit b/Master/texmf-dist/scripts/context/stubs-mkiv/unix/mtxrunjit deleted file mode 100755 index 06cd8a8acbb..00000000000 --- a/Master/texmf-dist/scripts/context/stubs-mkiv/unix/mtxrunjit +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -luajittex --luaonly `kpsewhich mtxrun.lua` "$@" diff --git a/Master/texmf-dist/scripts/context/stubs-mkiv/win64/contextjit.cmd b/Master/texmf-dist/scripts/context/stubs-mkiv/win64/contextjit.cmd deleted file mode 100755 index ec963b1a62a..00000000000 --- a/Master/texmf-dist/scripts/context/stubs-mkiv/win64/contextjit.cmd +++ /dev/null @@ -1,4 +0,0 @@ -@echo off - -for /f "usebackq tokens=*" %%M in (`kpsewhich mtxrun.lua`) do set MTX=%%M -luajittex --luaonly --socket %MTX% --script context %* diff --git a/Master/texmf-dist/scripts/context/stubs-mkiv/win64/luatools.cmd b/Master/texmf-dist/scripts/context/stubs-mkiv/win64/luatools.cmd deleted file mode 100755 index a20c7e0c901..00000000000 --- a/Master/texmf-dist/scripts/context/stubs-mkiv/win64/luatools.cmd +++ /dev/null @@ -1,2 +0,0 @@ -@echo off -mtxrun --script base %* diff --git a/Master/texmf-dist/scripts/context/stubs-mkiv/win64/mtxrunjit.cmd b/Master/texmf-dist/scripts/context/stubs-mkiv/win64/mtxrunjit.cmd deleted file mode 100755 index 39507f2214a..00000000000 --- a/Master/texmf-dist/scripts/context/stubs-mkiv/win64/mtxrunjit.cmd +++ /dev/null @@ -1,4 +0,0 @@ -@echo off - -for /f "usebackq tokens=*" %%M in (`kpsewhich mtxrun.lua`) do set MTX=%%M -luajittex --luaonly --socket %MTX% %* |