diff options
Diffstat (limited to 'Master/texmf-dist/scripts/context/lua')
14 files changed, 3317 insertions, 1543 deletions
diff --git a/Master/texmf-dist/scripts/context/lua/mtx-bibtex.lua b/Master/texmf-dist/scripts/context/lua/mtx-bibtex.lua index c81fd596f10..92036e3a5ed 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-bibtex.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-bibtex.lua @@ -19,6 +19,7 @@ local helpinfo = [[ <subcategory> <flag name="toxml"><short>convert bibtex database(s) to xml</short></flag> <flag name="tolua"><short>convert bibtex database(s) to lua</short></flag> + <flag name="search"><short>seatch bibtex database(s)</short></flag> </subcategory> </category> </flags> @@ -29,6 +30,7 @@ local helpinfo = [[ <example><command>mtxrun --script bibtex --tolua bibl-001.bib</command></example> <example><command>mtxrun --script bibtex --tolua --simple bibl-001.bib</command></example> <example><command>mtxrun --script bibtex --toxml bibl-001.bib bibl-002.bib bibl-003.bib biblio.xml</command></example> + <example><command>mtxrun --script bibtex --search --list --pattern=match(author:foo) bar.bib</command></example> </subcategory> </category> </examples> @@ -43,7 +45,9 @@ local application = logs.application { local report = application.report +require("util-seq") require("publ-dat") +require("publ-fnd") scripts = scripts or { } scripts.bibtex = scripts.bibtex or { } @@ -57,7 +61,7 @@ function scripts.bibtex.toxml(files) if filetype == "xml" then target = filename elseif filetype == "bib" then - bibtex.load(instance,filename) + bibtex.load { dataset = instance, filename = filename } else -- not supported end @@ -77,7 +81,8 @@ function scripts.bibtex.tolua(files) if filetype == "lua" then target = filename elseif filetype == "bib" then - bibtex.load(instance,filename) + bibtex.load { dataset = instance, filename = filename } + else -- not supported end @@ -92,7 +97,48 @@ function scripts.bibtex.tolua(files) end end -if environment.arguments.toxml then +function scripts.bibtex.search(files,pattern,list) + if pattern then + local dataset = publications.datasets["whatever"] + for i=1,#files do + local filename = resolvers.findfile(files[i]) + if filename and filename ~= "" then + publications.load { dataset = "whatever", filename = filename } + end + end + local found = publications.search(dataset,pattern) + local tags = table.sortedkeys(found) + if #tags == 0 then + report("no match") + elseif list then + report("%s matches:",#tags) + local result = { } + local luadata = dataset.luadata + for i=1,#tags do + local tag = tags[i] + local entry = luadata[tag] + result[i] = { + tag, + entry.year, + entry.author, + entry.title, + } + end + utilities.formatters.formatcolumns(result) + logs.newline() + for i=1,#result do + texio.write_nl(result[i]) + end + logs.newline() + else + report("%s matches: % t",#tags,tags) + end + end +end + +if environment.arguments.search then + scripts.bibtex.search(environment.files,environment.arguments.pattern,environment.arguments.list) +elseif environment.arguments.toxml then scripts.bibtex.toxml(environment.files) elseif environment.arguments.tolua then scripts.bibtex.tolua(environment.files) diff --git a/Master/texmf-dist/scripts/context/lua/mtx-check.lua b/Master/texmf-dist/scripts/context/lua/mtx-check.lua index 9f52509ec7e..c456b44143e 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-check.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-check.lua @@ -21,7 +21,7 @@ local helpinfo = [[ <flags> <category name="basic"> <subcategory> - <flag name="convert"><short>check tex file for errors</short></flag> + <flag name="check"><short>check tex file for errors</short></flag> </subcategory> </category> </flags> @@ -34,17 +34,17 @@ local application = logs.application { helpinfo = helpinfo, } -local report = application.report +local report = application.report -scripts = scripts or { } -scripts.checker = scripts.checker or { } +scripts = scripts or { } +scripts.checker = scripts.checker or { } -local validator = { } +local validator = { } -validator.n = 1 -validator.errors = { } -validator.trace = false -validator.direct = false +validator.n = 1 +validator.errors = { } +validator.trace = false +validator.direct = false validator.printer = print validator.tracer = print @@ -68,27 +68,24 @@ local progress = function(position, data, kind) end end -local i_m, d_m = P("$"), P("$$") -local l_s, r_s = P("["), P("]") -local l_g, r_g = P("{"), P("}") +local i_m, d_m = P("$"), P("$$") +local l_s, r_s = P("["), P("]") +local l_g, r_g = P("{"), P("}") -local okay = lpeg.P("{[}") + lpeg.P("{]}") +local okay = lpeg.P("{[}") + lpeg.P("{]}") -local esc = P("\\") -local cr = P("\r") -local lf = P("\n") -local crlf = P("\r\n") -local space = S(" \t\f\v") -local newline = crlf + cr + lf +local esc = P("\\") +local space = S(" \t\f\v") +local newline = lpeg.patterns.newline -local line = newline / function() validator.n = validator.n + 1 end +local line = newline / function() validator.n = validator.n + 1 end local startluacode = P("\\startluacode") local stopluacode = P("\\stopluacode") -local somecode = startluacode * (1-stopluacode)^1 * stopluacode +local somecode = startluacode * (1-stopluacode)^1 * stopluacode -local stack = { } +local stack = { } local function push(p,s) -- print("start",p,s) @@ -117,7 +114,7 @@ local contextgrammar = P { "tokens", ["start"] = start, ["stop"] = stop, ["whatever"] = line + esc * 1 + C(P("%") * (1-line)^0), - ["grouped"] = l_g * (V("whatever") + V("grouped") + V("setup") + V("display") + V("inline") + (1 - l_g - r_g))^0 * r_g, + ["grouped"] = l_g * (V("whatever") + V("grouped") + V("setup") + V("display") + V("inline") + line + (1 - l_g - r_g))^0 * r_g, ["setup"] = l_s * (okay + V("whatever") + V("grouped") + V("setup") + V("display") + V("inline") + (1 - l_s - r_s))^0 * r_s, ["display"] = d_m * (V("whatever") + V("grouped") + (1 - d_m))^0 * d_m, ["inline"] = i_m * (V("whatever") + V("grouped") + (1 - i_m))^0 * i_m, diff --git a/Master/texmf-dist/scripts/context/lua/mtx-context.lua b/Master/texmf-dist/scripts/context/lua/mtx-context.lua index 37172327140..919dbbb8cd4 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-context.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-context.lua @@ -28,10 +28,13 @@ local fileaddsuffix = file.addsuffix local filenewsuffix = file.replacesuffix local removesuffix = file.removesuffix local validfile = lfs.isfile +local removefile = os.remove +local renamefile = os.rename +local formatters = string.formatters local application = logs.application { name = "mtx-context", - banner = "ConTeXt Process Management 0.60", + banner = "ConTeXt Process Management 0.61", -- helpinfo = helpinfo, -- table with { category_a = text_1, category_b = text_2 } or helpstring or xml_blob helpinfo = "mtx-context.xml", } @@ -279,11 +282,47 @@ local function multipass_changed(oldhash, newhash) return false end -local function multipass_copyluafile(jobname) +local f_tempfile = formatters["%s-%s-%02d.tmp"] + +local function backup(run,kind,filename) + if run == 1 then + for i=1,10 do + local tmpname = f_tempfile(jobname,kind,i) + if validfile(tmpname) then + removefile(tmpname) + report("removing %a",tmpname) + end + end + end + if validfile(filename) then + local tmpname = f_tempfile(jobname,kind,run or 1) + report("copying %a into %a",filename,tmpname) + file.copy(filename,tmpname) + else + report("no file %a, nothing kept",filename) + end +end + +local function multipass_copyluafile(jobname,run) local tuaname, tucname = jobname..".tua", jobname..".tuc" if validfile(tuaname) then - os.remove(tucname) - os.rename(tuaname,tucname) + if run then + backup(run,"tuc",tucname) + report("copying %a into %a",tuaname,tucname) + report() + end + removefile(tucname) + renamefile(tuaname,tucname) + end +end + +local function multipass_copylogfile(jobname,run) + local logname = jobname..".log" + if validfile(logname) then + if run then + backup(run,"log",logname) + report() + end end end @@ -348,8 +387,8 @@ local function result_push_purge(oldbase,newbase) for _, suffix in next, usedsuffixes.after do local oldname = fileaddsuffix(oldbase,suffix) local newname = fileaddsuffix(newbase,suffix) - os.remove(newname) - os.remove(oldname) + removefile(newname) + removefile(oldname) end end @@ -358,10 +397,10 @@ local function result_push_keep(oldbase,newbase) local oldname = fileaddsuffix(oldbase,suffix) local newname = fileaddsuffix(newbase,suffix) local tmpname = "keep-"..oldname - os.remove(tmpname) - os.rename(oldname,tmpname) - os.remove(oldname) - os.rename(newname,oldname) + removefile(tmpname) + renamefile(oldname,tmpname) + removefile(oldname) + renamefile(newname,oldname) end end @@ -369,8 +408,8 @@ local function result_save_error(oldbase,newbase) for _, suffix in next, usedsuffixes.keep do local oldname = fileaddsuffix(oldbase,suffix) local newname = fileaddsuffix(newbase,suffix) - os.remove(newname) -- to be sure - os.rename(oldname,newname) + removefile(newname) -- to be sure + renamefile(oldname,newname) end end @@ -378,8 +417,8 @@ local function result_save_purge(oldbase,newbase) for _, suffix in next, usedsuffixes.after do local oldname = fileaddsuffix(oldbase,suffix) local newname = fileaddsuffix(newbase,suffix) - os.remove(newname) -- to be sure - os.rename(oldname,newname) + removefile(newname) -- to be sure + renamefile(oldname,newname) end end @@ -388,9 +427,9 @@ local function result_save_keep(oldbase,newbase) local oldname = fileaddsuffix(oldbase,suffix) local newname = fileaddsuffix(newbase,suffix) local tmpname = "keep-"..oldname - os.remove(newname) - os.rename(oldname,newname) - os.rename(tmpname,oldname) + removefile(newname) + renamefile(oldname,newname) + renamefile(tmpname,oldname) end end @@ -549,6 +588,19 @@ function scripts.context.run(ctxdata,filename) local a_jiton = getargument("jiton") local a_jithash = getargument("jithash") local a_texformat = getargument("texformat") + local a_keeptuc = getargument("keeptuc") + local a_keeplog = getargument("keeplog") + + -- the following flag is not officially supported because i cannot forsee + -- side effects (so no bug reports please) .. we provide --sandbox that + -- does similar things but tries to ensure that context works as expected + + local a_safer = getargument("safer") + + if a_safer then + report("warning: using the luatex safer options, processing is not guaranteed") + end + -- a_batchmode = (a_batchmode and "batchmode") or (a_nonstopmode and "nonstopmode") or (a_scrollmode and "scrollmode") or nil a_synctex = check_synctex(a_synctex) @@ -592,6 +644,22 @@ function scripts.context.run(ctxdata,filename) scripts.context.make(formatname,a_engine) formatfile, scriptfile = resolvers.locateformat(formatname) end + -- + local function combine(key) + local flag = validstring(environment[key]) + local plus = analysis[key] + if flag and plus then + return plus .. "," .. flag -- flag wins + else + return flag or plus -- flag wins + end + end + local a_trackers = analysis.trackers + local a_experiments = analysis.experiments + local directives = combine("directives") + local trackers = combine("trackers") + local experiments = combine("experiments") + -- if formatfile and scriptfile then local suffix = validstring(getargument("suffix")) local resultname = validstring(getargument("result")) @@ -639,9 +707,9 @@ function scripts.context.run(ctxdata,filename) local maxnofruns = once and 1 or multipass_nofruns -- local c_flags = { - directives = validstring(environment.directives), -- gets passed via mtxrun - trackers = validstring(environment.trackers), -- gets passed via mtxrun - experiments = validstring(environment.experiments), -- gets passed via mtxrun + directives = directives, -- gets passed via mtxrun + trackers = trackers, -- gets passed via mtxrun + experiments = experiments, -- gets passed via mtxrun -- result = validstring(resultname), input = validstring(getargument("input") or filename), -- alternative input @@ -662,6 +730,7 @@ function scripts.context.run(ctxdata,filename) ["interaction"] = a_batchmode, ["synctex"] = a_synctex, ["no-parse-first-line"] = true, + ["safer"] = a_safer, -- ["no-mktex"] = true, -- ["file-line-error-style"] = true, ["fmt"] = formatfile, @@ -671,10 +740,6 @@ function scripts.context.run(ctxdata,filename) ["jithash"] = a_jithash, } -- - if a_synctex then - report("warning: synctex is enabled") -- can add upto 5% runtime - end - -- if not a_timing then -- okay elseif c_flags.usemodule then @@ -691,6 +756,15 @@ function scripts.context.run(ctxdata,filename) c_flags.directives = "system.profile" end -- + if a_synctex then + report("warning: synctex is enabled") -- can add upto 5% runtime + if c_flags.directives then + c_flags.directives = format("system.synctex=%s,%s",a_synctex,c_flags.directives) + else + c_flags.directives = format("system.synctex=%s",a_synctex) + end + end + -- -- kindofrun: 1:first run, 2:successive run, 3:once, 4:last of maxruns -- for currentrun=1,maxnofruns do @@ -704,9 +778,10 @@ function scripts.context.run(ctxdata,filename) -- local command = luatex_command(l_flags,c_flags,mainfile,a_engine) -- - report("run %s: %s",i,command) + report("run %s: %s",currentrun,command) print("") -- cleaner, else continuation on same line local returncode, errorstring = os.spawn(command) + -- todo: remake format when no proper format is found if not returncode then report("fatal error: no return code, message: %s",errorstring or "?") if resultname then @@ -715,7 +790,8 @@ function scripts.context.run(ctxdata,filename) os.exit(1) break elseif returncode == 0 then - multipass_copyluafile(jobname) + multipass_copyluafile(jobname,a_keeptuc and currentrun) + multipass_copylogfile(jobname,a_keeplog and currentrun) if not multipass_forcedruns then newhash = multipass_hashfiles(jobname) if multipass_changed(oldhash,newhash) then @@ -852,7 +928,7 @@ function scripts.context.pipe() -- still used? scripts.context.purge_job(filename) elseif getargument("purgeall") then scripts.context.purge_job(filename,true) - os.remove(filename) + removefile(filename) end else if formatname then @@ -1046,11 +1122,11 @@ local special_runfiles = { local function purge_file(dfile,cfile) if cfile and validfile(cfile) then - if os.remove(dfile) then + if removefile(dfile) then return filebasename(dfile) end elseif dfile then - if os.remove(dfile) then + if removefile(dfile) then return filebasename(dfile) end end @@ -1144,8 +1220,8 @@ local function touch(path,name,versionpattern,kind,kindpattern) end if newdata ~= "" and (oldversion ~= newversion or oldkind ~= newkind or newdata ~= olddata) then local backup = filenewsuffix(name,"tmp") - os.remove(backup) - os.rename(name,backup) + removefile(backup) + renamefile(name,backup) io.savedata(name,newdata) return name, oldversion, newversion, oldkind, newkind end diff --git a/Master/texmf-dist/scripts/context/lua/mtx-context.xml b/Master/texmf-dist/scripts/context/lua/mtx-context.xml index af8d5cc20e5..c4109328900 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-context.xml +++ b/Master/texmf-dist/scripts/context/lua/mtx-context.xml @@ -142,7 +142,7 @@ <short>assume given file present elsewhere</short> </flag> <flag name="nofile"> - <short>use dummy file as jobname</short> + <short>use dummy file as jobname</short> </flag> </subcategory> </category> @@ -152,16 +152,22 @@ <short>update context version number (also provide <ref name="expert"/>, optionally provide <ref name="basepath"/>)</short> </flag> <flag name="nostatistics"> - <short>omit runtime statistics at the end of the run</short> + <short>omit runtime statistics at the end of the run</short> </flag> <flag name="update"> - <short>update context from website (not to be confused with contextgarden)</short> + <short>update context from website (not to be confused with contextgarden)</short> </flag> - <flag name="profile"> - <short>profile job (use: mtxrun <ref name="script"/> profile <ref name="analyze"/>)</short> + <flag name="profile"> + <short>profile job (use: mtxrun <ref name="script"/> profile <ref name="analyze"/>)</short> </flag> - <flag name="timing"> - <short>generate timing and statistics overview</short> + <flag name="timing"> + <short>generate timing and statistics overview</short> + </flag> + <flag name="keeptuc"> + <short>keep previous tuc files (jobname-tuc-[run].tmp)</short> + </flag> + <flag name="keeplog"> + <short>keep previous log files (jobname-log-[run].tmp)</short> </flag> </subcategory> <subcategory> @@ -190,6 +196,11 @@ <short>do not check for file and enter scroll mode (<ref name="dummyfile"/>=whatever.tmp)</short> </flag> </subcategory> + <subcategory> + <flag name="sandbox"> + <short>process file in a limited environment</short> + </flag> + </subcategory> </category> </flags> </application> diff --git a/Master/texmf-dist/scripts/context/lua/mtx-convert.lua b/Master/texmf-dist/scripts/context/lua/mtx-convert.lua index b76b3baaf62..d5dba075acf 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-convert.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-convert.lua @@ -48,7 +48,7 @@ local convert = scripts.convert convert.converters = convert.converters or { } local converters = convert.converters -local gsprogram = os.type == "windows" and "gswin32c" or "gs" +local gsprogram = (os.type == "windows" and (os.which("gswin64c.exe") or os.which("gswin32c.exe"))) or "gs" local gstemplate_eps = "%s -q -sDEVICE=pdfwrite -dPDFSETTINGS=/prepress -dEPSCrop -dNOPAUSE -dSAFER -dNOCACHE -dBATCH -dAutoRotatePages=/None -dProcessColorModel=/DeviceCMYK -sOutputFile=%s %s -c quit" local gstemplate_ps = "%s -q -sDEVICE=pdfwrite -dPDFSETTINGS=/prepress -dNOPAUSE -dSAFER -dNOCACHE -dBATCH -dAutoRotatePages=/None -dProcessColorModel=/DeviceCMYK -sOutputFile=%s %s -c quit" diff --git a/Master/texmf-dist/scripts/context/lua/mtx-epub.lua b/Master/texmf-dist/scripts/context/lua/mtx-epub.lua index 11f0a202453..956ce4931a9 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-epub.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-epub.lua @@ -18,8 +18,89 @@ if not modules then modules = { } end modules ['mtx-epub'] = { -- first we need a decent strategy to export them. More information will be -- available on the wiki. -local format, gsub = string.format, string.gsub -local concat = table.concat +-- META-INF +-- container.xml +-- OEBPS +-- content.opf +-- toc.ncx +-- images +-- styles +-- mimetype + +-- todo: +-- +-- remove m_k_v_i prefixes +-- remap fonts %mono% in css so that we can replace +-- coverpage tests +-- split up + +-- todo: automated cover page: +-- +-- \startMPpage +-- StartPage ; +-- fill Page withcolor .5red ; +-- numeric n ; +-- for i=10 downto 1 : +-- n := i * PaperWidth/40 ; +-- draw +-- lrcorner Page shifted (0,n) +-- % -- lrcorner Page +-- -- lrcorner Page shifted (-n,0) +-- % -- cycle +-- withpen pencircle scaled 1mm +-- withcolor white ; +-- endfor ; +-- picture p ; p := image ( +-- draw +-- anchored.top( +-- textext.bot("\tttf Some Title") +-- xsized .8PaperWidth +-- ,center topboundary Page +-- ) +-- withcolor white ; +-- ) ; +-- picture q ; q := image ( +-- draw +-- anchored.top( +-- textext.bot("\tttf An Author") +-- xsized .4PaperWidth +-- shifted (0,-PaperHeight/40) +-- ,center bottomboundary p +-- ) +-- withcolor white ; +-- ) ; +-- draw p ; +-- draw q ; +-- StopPage ; +-- \stopMPpage + +local format, gsub, find = string.format, string.gsub, string.find +local concat, sortedhash = table.concat, table.sortedhash + +local formatters = string.formatters +local replacetemplate = utilities.templates.replace + +local addsuffix = file.addsuffix +local nameonly = file.nameonly +local basename = file.basename +local pathpart = file.pathpart +local joinfile = file.join +local suffix = file.suffix +local addsuffix = file.addsuffix +local removesuffix = file.removesuffix +local replacesuffix = file.replacesuffix + +local copyfile = file.copy +local removefile = os.remove + +local needsupdating = file.needsupdating + +local isdir = lfs.isdir +local isfile = lfs.isfile +local mkdir = lfs.mkdir + +local pushdir = dir.push +local popdir = dir.pop local helpinfo = [[ <?xml version="1.0"?> @@ -27,12 +108,17 @@ local helpinfo = [[ <metadata> <entry name="name">mtx-epub</entry> <entry name="detail">ConTeXt EPUB Helpers</entry> - <entry name="version">0.12</entry> + <entry name="version">1.10</entry> </metadata> <flags> <category name="basic"> <subcategory> <flag name="make"><short>create epub zip file</short></flag> + <flag name="purge"><short>remove obsolete files</short></flag> + <flag name="rename"><short>rename images to sane names</short></flag> + <flag name="svgmath"><short>convert mathml to svg</short></flag> + <flag name="svgstyle"><short>use given tex style for svg generation (overloads style in specification)</short></flag> + <flag name="all"><short>assume: --purge --rename --svgmath (for fast testing)</short></flag> </subcategory> </category> </flags> @@ -49,10 +135,12 @@ local helpinfo = [[ local application = logs.application { name = "mtx-epub", - banner = "ConTeXt EPUB Helpers 0.12", + banner = "ConTeXt EPUB Helpers 1.10", helpinfo = helpinfo, } +local report = application.report + -- script code scripts = scripts or { } @@ -60,91 +148,142 @@ scripts.epub = scripts.epub or { } local mimetype = "application/epub+zip" -local container = [[ +local t_container = [[ <?xml version="1.0" encoding="UTF-8"?> <container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container"> <rootfiles> - <rootfile full-path="OEBPS/%s" media-type="application/oebps-package+xml"/> + <rootfile full-path="OEBPS/%rootfile%" media-type="application/oebps-package+xml"/> </rootfiles> </container> ]] -local package = [[ +-- urn:uuid: + +-- <dc:identifier id="%identifier%" opf:scheme="UUID">%uuid%</dc:identifier> + +local t_package = [[ <?xml version="1.0" encoding="UTF-8"?> -<package version="2.0" xmlns="http://www.idpf.org/2007/opf" unique-identifier="%s"> +<package xmlns="http://www.idpf.org/2007/opf" unique-identifier="%identifier%" version="3.0"> <metadata xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:opf="http://www.idpf.org/2007/opf"> - <dc:title>%s</dc:title> - <dc:language>%s</dc:language> - <dc:identifier id="%s" opf:scheme="UUID">urn:uuid:%s</dc:identifier> - <dc:creator>%s</dc:creator> - <dc:date>%s</dc:date> - <meta name="cover" content="%s" /> + <dc:title>%title%</dc:title> + <dc:language>%language%</dc:language> + <dc:identifier id="%identifier%">%uuid%</dc:identifier> + <dc:creator>%creator%</dc:creator> + <dc:date>%date%</dc:date> + <!-- + <dc:subject>%subject%</dc:subject> + <dc:description>%description%</dc:description> + <dc:publisher>%publisher%</dc:publisher> + <dc:source>%source%</dc:source> + <dc:relation>%relation%</dc:relation> + <dc:coverage>%coverage%</dc:coverage> + <dc:rights>%rights%</dc:rights> + --> + <meta name="cover" content="%coverpage%" /> + <meta name="generator" content="ConTeXt MkIV" /> + <meta property="dcterms:modified">%date%</meta> </metadata> <manifest> -%s +%manifest% </manifest> <spine toc="ncx"> <itemref idref="cover-xhtml" /> - <itemref idref="%s" /> + <itemref idref="%rootfile%" /> </spine> </package> ]] -local item = [[ <item id="%s" href="%s" media-type="%s"/>]] -local toc = [[ -<?xml version="1.0"?> +local t_item = [[ <item id="%id%" href="%filename%" media-type="%mime%" />]] +local t_prop = [[ <item id="%id%" href="%filename%" media-type="%mime%" properties="%properties%" />]] + +-- <!DOCTYPE ncx PUBLIC "-//NISO//DTD ncx 2005-1//EN" "http://www.daisy.org/z3986/2005/ncx-2005-1.dtd"> + +local t_toc = [[ +<?xml version="1.0" encoding="UTF-8"?> -<!DOCTYPE ncx PUBLIC "-//NISO//DTD ncx 2005-1//EN" "http://www.daisy.org/z3986/2005/ncx-2005-1.dtd"> +<!-- this is no longer needed in epub 3.0+ --> <ncx xmlns="http://www.daisy.org/z3986/2005/ncx/" version="2005-1"> <head> - <meta name="dtb:uid" content="%s" /> + <meta name="generator" content="ConTeXt MkIV" /> + <meta name="dtb:uid" content="%identifier%" /> <meta name="dtb:depth" content="2" /> <meta name="dtb:totalPgeCount" content="0" /> <meta name="dtb:maxPageNumber" content="0" /> </head> <docTitle> - <text>%s</text> + <text>%title%</text> </docTitle> + <docAuthor> + <text>%author%</text> + </docAuthor> + <navMap> <navPoint id="np-1" playOrder="1"> <navLabel> <text>start</text> </navLabel> - <content src="%s"/> + <content src="%root%"/> </navPoint> </navMap> </ncx> ]] -local coverxhtml = [[ +local t_navtoc = [[ <?xml version="1.0" encoding="UTF-8"?> -<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd"> +<html xmlns="http://www.w3.org/1999/xhtml" xmlns:epub="http://www.idpf.org/2007/ops"> + <head> + <meta charset="utf-8" /> + <title>navtoc</title> + </head> + <body> + <div class="navtoc"> + <!-- <nav epub:type="lot"> --> + <nav epub:type="toc" id="navtoc"> + <ol> + <li><a href="%root%">document</a></li> + </ol> + </nav> + </div> + </body> +</html> +]] + +-- <html xmlns="http://www.w3.org/1999/xhtml"> +-- <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd"> + +local t_coverxhtml = [[ +<?xml version="1.0" encoding="UTF-8"?> <html xmlns="http://www.w3.org/1999/xhtml"> <head> - <title>cover.xhtml</title> + <meta charset="utf-8" /> + <title>cover page</title> </head> <body> - <div> - <img src="%s" alt="The cover image" style="max-width: 100%%;" /> + <div class="coverpage"> + %content% </div> </body> </html> ]] +local t_coverimg = [[ + <img src="%image%" alt="The cover image" style="max-width: 100%%;" /> +]] + -- We need to figure out what is permitted. Numbers only seem to give -- problems is some applications as do names with dashes. Also the -- optional toc is supposed to be there and although id's are by @@ -154,12 +293,13 @@ local coverxhtml = [[ local function dumbid(filename) -- return (string.gsub(os.uuid(),"%-%","")) -- to be tested - return file.nameonly(filename) .. "-" .. file.suffix(filename) + return nameonly(filename) .. "-" .. suffix(filename) end local mimetypes = { xhtml = "application/xhtml+xml", xml = "application/xhtml+xml", + html = "application/html", css = "text/css", svg = "image/svg+xml", png = "image/png", @@ -175,204 +315,601 @@ local idmakers = { default = function(filename) return dumbid(filename) end, } --- specification = { --- name = "document", --- identifier = "123", --- root = "a.xhtml", --- files = { --- "a.xhtml", --- "b.css", --- "c.png", --- } --- } - -local function locateimages(oldname,newname,subpath) +local function relocateimages(imagedata,oldname,newname,subpath,rename) local data = io.loaddata(oldname) - local images = { } - local done = gsub(data,"(background%-image *: * url%()(.-)(%))", function(before,name,after) - if subpath then - name = file.join(subpath,name) + if data then + subpath = joinfile("..",subpath) + report("relocating images") + local n = 0 + local done = gsub(data,[[(id=")(.-)(".-background%-image *: *url%()(.-)(%))]], function(s1,id,s2,name,s3) + local data = imagedata[id] + if data then + local newname = data[id].newname + if newname then + if subpath then + name = joinfile(subpath,basename(newname)) + else + name = basename(newname) + end + -- name = url.addscheme(name) + end + if newname then + n = n + 1 + if rename then + name = joinfile(subpath,addsuffix(id,suffix(name))) + end + return s1 .. id .. s2 .. name .. s3 + end + end + end) + report("%s images relocated in %a",n,newname) + if newname then + io.savedata(newname,done) end - images[#images+1] = name - return before .. name .. after - end) - if newname then - io.savedata(done,newname) end return images end +function reportobsolete(oldfiles,newfiles,purge) + + for i=1,#oldfiles do oldfiles[i] = gsub(oldfiles[i],"^[%./]+","") end + for i=1,#newfiles do newfiles[i] = gsub(newfiles[i],"^[%./]+","") end + + local old = table.tohash(oldfiles) + local new = table.tohash(newfiles) + local done = false + + for name in sortedhash(old) do + if not new[name] then + if not done then + report() + if purge then + report("removing obsolete files:") + else + report("obsolete files:") + end + report() + done = true + end + report(" %s",name) + if purge then + removefile(name) + end + end + end + + if done then + report() + end + + return done + +end + + local zippers = { { name = "zip", + binary = "zip", uncompressed = "zip %s -X -0 %s", compressed = "zip %s -X -9 -r %s", }, { - name = "7zip (7z)", + name = "7z (7zip)", + binary = "7z", uncompressed = "7z a -tzip -mx0 %s %s", compressed = "7z a -tzip %s %s", }, } -function scripts.epub.make() +function scripts.epub.make(purge,rename,svgmath,svgstyle) + + -- one can enter a jobname or jobname-export but the simple jobname is + -- preferred local filename = environment.files[1] - if filename and filename ~= "" and type(filename) == "string" then + if not filename or filename == "" or type(filename) ~= "string" then + report("provide filename") + return + end + + local specpath, specname, specfull - filename = file.basename(filename) - local specfile = file.replacesuffix(filename,"specification") - local specification = lfs.isfile(specfile) and dofile(specfile) or { } + if isdir(filename) then + specpath = filename + specname = addsuffix(specpath,"lua") + specfull = joinfile(specpath,specname) + end + + if not specfull or not isfile(specfull) then + specpath = filename .. "-export" + specname = addsuffix(filename .. "-pub","lua") + specfull = joinfile(specpath,specname) + end + + if not specfull or not isfile(specfull) then + report("unknown specificaton file %a for %a",specfull or "?",filename) + return + end --- inspect(specification) + local specification = dofile(specfull) + + if not specification or not next(specification) then + report("invalid specificaton file %a",specfile) + return + end + + report("using specification file %a",specfull) + + -- images: { ... url = location ... } + + local defaultcoverpage = "cover.xhtml" + + local name = specification.name or nameonly(filename) + local identifier = specification.identifier or "" + local htmlfiles = specification.htmlfiles or { } + local styles = specification.styles or { } + local images = specification.images or { } + local htmlroot = specification.htmlroot or htmlfiles[1] or "" + local language = specification.language or "en" + local creator = specification.creator or "context mkiv" + local author = specification.author or "anonymous" + local title = specification.title or name + local subtitle = specification.subtitle or "" + local imagefile = specification.imagefile or "" + local imagepath = specification.imagepath or "images" + local stylepath = specification.stylepath or "styles" + local coverpage = specification.firstpage or defaultcoverpage + + if type(svgstyle) == "string" and not svgstyle then + svgstyle = specification.svgstyle or "" + end - local name = specification.name or file.removesuffix(filename) - local identifier = specification.identifier or os.uuid(true) - local files = specification.files or { file.addsuffix(filename,"xhtml") } - local images = specification.images or { } - local root = specification.root or files[1] - local language = specification.language or "en" - local creator = specification.author or "My Self" - local title = specification.title or "My Title" - local firstpage = specification.firstpage or "" - local lastpage = specification.lastpage or "" + local obsolete = false - -- identifier = gsub(identifier,"[^a-zA-z0-9]","") + if #htmlfiles == 0 then + report("no html files specified") + return + end + if htmlroot == "" then + report("no html root file specified") + return + end + + if subtitle ~= "" then + title = format("%s, %s",title,subtitle) + end + + local htmlsource = specpath + local imagesource = joinfile(specpath,imagepath) + local stylesource = joinfile(specpath,stylepath) + + -- once we're here we can start moving files to the right spot; first we deal + -- with images + + -- ["image-1"]={ + -- height = "7.056cm", + -- name = "file:///t:/sources/cow.svg", + -- page = "1", + -- width = "9.701cm", + -- } + + -- end of todo + + local pdftosvg = os.which("mudraw") and formatters[ [[mudraw -o "%s" "%s" %s]] ] + + local f_svgpage = formatters["%s-page-%s.svg"] + local f_svgname = formatters["%s.svg"] + + local notupdated = 0 + local updated = 0 + local skipped = 0 + local oldfiles = dir.glob(file.join(imagesource,"*")) + local newfiles = { } + + if not pdftosvg then + report("the %a binary is not present","mudraw") + end + + -- a coverpage file has to be in the root of the export tree + + if not coverpage then + report("no cover page (image) defined") + elseif suffix(coverpage) ~= "xhtml" then + report("using cover page %a",coverpage) + local source = coverpage + local target = joinfile(htmlsource,coverpage) + htmlfiles[#htmlfiles+1 ] = coverpage + report("copying coverpage %a to %a",source,target) + copyfile(source,target) + elseif isfile(coverpage) then + report("using cover page image %a",coverpage) + images.cover = { + height = "100%", + width = "100%", + page = "1", + name = url.filename(coverpage), + used = coverpage, + } + local data = replacetemplate(t_coverxhtml, { + content = replacetemplate(t_coverimg, { + image = coverpage, + }) + }) + coverpage = defaultcoverpage + local target = joinfile(htmlsource,coverpage) + report("saving coverpage to %a",target) + io.savedata(target,data) + htmlfiles[#htmlfiles+1 ] = coverpage + else + report("cover page image %a is not present",coverpage) + coverpage = false + end + + if not coverpage then + local data = replacetemplate(t_coverxhtml, { + content = "no cover page" + }) + coverpage = defaultcoverpage + local target = joinfile(htmlsource,coverpage) + report("saving dummy coverpage to %a",target) + io.savedata(target,data) + htmlfiles[#htmlfiles+1 ] = coverpage + end - if firstpage ~= "" then - images[firstpage] = firstpage + for id, data in sortedhash(images) do + local name = url.filename(data.name) + local used = url.filename(data.used) + local base = basename(used) + local page = tonumber(data.page) or 1 + -- todo : check timestamp and prefix, rename to image-* + if suffix(used) == "pdf" then + -- todo: pass svg name + if page > 1 then + name = f_svgpage(nameonly(name),page) + else + name = f_svgname(nameonly(name)) + end + local source = used + local target = joinfile(imagesource,name) + if needsupdating(source,target) then + if pdftosvg then + local command = pdftosvg(target,source,page) + report("running command %a",command) + os.execute(command) + updated = updated + 1 + else + skipped = skipped + 1 + end + else + notupdated = notupdated + 1 + end + newfiles[#newfiles+1] = target + else + name = basename(used) + local source = used + local target = joinfile(imagesource,name) + if needsupdating(source,target) then + report("copying %a to %a",source,target) + copyfile(source,target) + updated = updated + 1 + else + notupdated = notupdated + 1 + -- no message + end + newfiles[#newfiles+1] = target end - if lastpage ~= "" then - images[lastpage] = lastpage + local target = newfiles[#newfiles] + if suffix(target) == "svg" and isfile(target) then + local data = io.loaddata(target) + if data then + local done = gsub(data,"<!(DOCTYPE.-)>","<!-- %1 -->",1) + if data ~= done then + report("doctype fixed in %a",target) + io.savedata(target,data) + end + end end + data.newname = name -- without path + end + + report("%s images checked, %s updated, %s kept, %s skipped",updated + notupdated + skipped,updated,notupdated,skipped) - identifier = "BookId" -- weird requirement - - local epubname = name - local epubpath = file.replacesuffix(name,"tree") - local epubfile = file.replacesuffix(name,"epub") - local epubroot = file.replacesuffix(name,"opf") - local epubtoc = "toc.ncx" - local epubcover = "cover.xhtml" - - application.report("creating paths in tree %s",epubpath) - lfs.mkdir(epubpath) - lfs.mkdir(file.join(epubpath,"META-INF")) - lfs.mkdir(file.join(epubpath,"OEBPS")) - - local used = { } - - local function copyone(filename) - local suffix = file.suffix(filename) - local mime = mimetypes[suffix] - if mime then - local idmaker = idmakers[suffix] or idmakers.default - local target = file.join(epubpath,"OEBPS",filename) - file.copy(filename,target) - application.report("copying %s to %s",filename,target) - used[#used+1] = format(item,idmaker(filename),filename,mime) + if reportobsolete(oldfiles,newfiles,purge) then + obsolete = true + end + + -- here we can decide not to make an epub + + local uuid = format("urn:uuid:%s",os.uuid(true)) -- os.uuid() + local identifier = "bookid" -- for now + + local epubname = removesuffix(name) + local epubpath = name .. "-epub" + local epubfile = replacesuffix(name,"epub") + local epubroot = replacesuffix(name,"opf") + local epubtoc = "toc.ncx" + local epubmimetypes = "mimetype" + local epubcontainer = "container.xml" + local epubnavigator = "nav.xhtml" + + local metapath = "META-INF" + local datapath = "OEBPS" + + local oldfiles = dir.glob(file.join(epubpath,"**/*")) + local newfiles = { } + + report("creating paths in tree %a",epubpath) + + if not isdir(epubpath) then + mkdir(epubpath) + end + if not isdir(epubpath) then + report("unable to create path %a",epubpath) + return + end + + local metatarget = joinfile(epubpath,metapath) + local htmltarget = joinfile(epubpath,datapath) + local styletarget = joinfile(epubpath,datapath,stylepath) + local imagetarget = joinfile(epubpath,datapath,imagepath) + + mkdir(metatarget) + mkdir(htmltarget) + mkdir(styletarget) + mkdir(imagetarget) + + local used = { } + local notupdated = 0 + local updated = 0 + + local oldimagespecification = joinfile(htmlsource,imagefile) + local newimagespecification = joinfile(htmltarget,imagefile) + + report("removing %a",newimagespecification) + -- removefile(newimagespecification) -- because we update that one + + local function registerone(path,filename,mathml) + local suffix = suffix(filename) + local mime = mimetypes[suffix] + if mime then + local idmaker = idmakers[suffix] or idmakers.default + local fullname = path and joinfile(path,filename) or filename + if mathml then + used[#used+1] = replacetemplate(t_prop, { + id = idmaker(filename), + filename = fullname, + mime = mime, + properties = "mathml", + } ) + else + used[#used+1] = replacetemplate(t_item, { + id = idmaker(filename), + filename = fullname, + mime = mime, + } ) end + return true + end + end + + local function registerandcopyfile(check,path,name,sourcepath,targetpath,newname,image) + + if name == "" then + report("ignoring unknown image") + return + end + + if newname then + newname = replacesuffix(newname,suffix(name)) + else + newname = name end - copyone("cover.xhtml") - copyone("toc.ncx") + local source = joinfile(sourcepath,name) + local target = joinfile(targetpath,newname) + local mathml = false - local function copythem(files) - for i=1,#files do - local filename = files[i] - if type(filename) == "string" then - copyone(filename) + if suffix(source) == "xhtml" then + if find(io.loaddata(source),"MathML") then + mathml = true -- inbelievable: the property is only valid when there is mathml + end + else + report("checking image %a -> %a",source,target) + end + if registerone(path,newname,mathml) then + if not check or needsupdating(source,target) or mathml and svgmath then + report("copying %a to %a",source,target) + copyfile(source,target) + updated = updated + 1 + else + notupdated = notupdated + 1 + end + newfiles[#newfiles+1] = target + if mathml and svgmath then + report() + report("converting mathml into svg in %a",target) + report() + local status, total, unique = moduledata.svgmath.convert(target,svgstyle) + report() + if status then + report("%s formulas converted, %s are unique",total,unique) + else + report("warning: %a in %a",total,target) end + report() end end + end - copythem(files) + -- local nofdummies = 0 + -- local dummyname = formatters["dummy-figure-%03i"] + -- local makedummy = formatters["context --extra=dummies --noconsole --once --result=%s"] + -- + -- local function registerandcopydummy(targetpath,name) + -- nofdummies = nofdummies + 1 + -- local newname = dummyname(nofdummies) + -- local target = joinfile(targetpath,newname) + -- if not isfile(target) then + -- pushdir(targetpath) + -- report("generating dummy %a for %a",newname,name or "unknown") + -- os.execute(makedummy(newname)) + -- popdir() + -- end + -- return newname + -- end + + for image, data in sortedhash(images) do + -- if data.used == "" then + -- data.newname = registerandcopydummy(imagetarget,data.name) + -- end + registerandcopyfile(true,imagepath,data.newname,imagesource,imagetarget,rename and image,true) + end + for i=1,#styles do + registerandcopyfile(false,stylepath,styles[i],stylesource,styletarget) + end + for i=1,#htmlfiles do + registerandcopyfile(false,false,htmlfiles[i],htmlsource,htmltarget) + end - local theimages = { } + relocateimages(images,oldimagespecification,oldimagespecification,imagepath,rename) + relocateimages(images,oldimagespecification,newimagespecification,imagepath,rename) - for k, v in table.sortedpairs(images) do - theimages[#theimages+1] = k - if not lfs.isfile(k) and file.suffix(k) == "svg" and file.suffix(v) == "pdf" then - local command = format("inkscape --export-plain-svg=%s %s",k,v) - application.report("running command '%s'\n\n",command) - os.execute(command) - end + report("%s files registered, %s updated, %s kept",updated + notupdated,updated,notupdated) + + local function saveinfile(what,name,data) + report("saving %s in %a",what,name) + io.savedata(name,data) + newfiles[#newfiles+1] = name + end + + used[#used+1] = replacetemplate(t_prop, { + id = "nav", + filename = epubnavigator, + properties = "nav", + mime = "application/xhtml+xml", + }) + + registerone(false,epubtoc) + + saveinfile("navigation data",joinfile(htmltarget,epubnavigator),replacetemplate(t_navtoc, { -- version 3.0 + root = htmlroot, + } ) ) + + saveinfile("used mimetypes",joinfile(epubpath,epubmimetypes),mimetype) + + saveinfile("version 2.0 container",joinfile(metatarget,epubcontainer),replacetemplate(t_container, { + rootfile = epubroot + } ) ) + + local idmaker = idmakers[suffix(htmlroot)] or idmakers.default + + saveinfile("package specification",joinfile(htmltarget,epubroot),replacetemplate(t_package, { + identifier = identifier, + title = title, + language = language, + uuid = uuid, + creator = creator, + date = os.date("!%Y-%m-%dT%H:%M:%SZ"), + coverpage = idmaker(coverpage), + manifest = concat(used,"\n"), + rootfile = idmaker(htmlroot) + } ) ) + + -- t_toc is replaced by t_navtoc in >= 3 + + saveinfile("table of contents",joinfile(htmltarget,epubtoc), replacetemplate(t_toc, { + identifier = uuid, -- identifier, + title = title, + author = author, + root = htmlroot, + } ) ) + + report("creating archive\n\n") + + pushdir(epubpath) + + removefile(epubfile) + + local usedzipper = false + + local function zipped(zipper) + local ok = os.execute(format(zipper.uncompressed,epubfile,epubmimetypes)) + if ok == 0 then + os.execute(format(zipper.compressed,epubfile,metapath)) + os.execute(format(zipper.compressed,epubfile,datapath)) + usedzipper = zipper.name + return true end + end + + -- nice way + + for i=1,#zippers do + if os.which(zippers[i].binary) and zipped(zippers[i]) then + break + end + end - copythem(theimages) - - local idmaker = idmakers[file.suffix(root)] or idmakers.default - - container = format(container, - epubroot - ) - package = format(package, - identifier, - title, - language, - identifier, - os.uuid(), - creator, - os.date("!%Y-%m-%dT%H:%M:%SZ"), - idmaker(firstpage), - concat(used,"\n"), - idmaker(root) - ) - toc = format(toc, - identifier, - title, - root - ) - coverxhtml = format(coverxhtml, - firstpage - ) - - io.savedata(file.join(epubpath,"mimetype"),mimetype) - io.savedata(file.join(epubpath,"META-INF","container.xml"),container) - io.savedata(file.join(epubpath,"OEBPS",epubroot),package) - io.savedata(file.join(epubpath,"OEBPS",epubtoc),toc) - io.savedata(file.join(epubpath,"OEBPS",epubcover),coverxhtml) - - application.report("creating archive\n\n") - - lfs.chdir(epubpath) - os.remove(epubfile) - - local done = false + -- trial and error + if not usedzipper then for i=1,#zippers do - local zipper = zippers[i] - if os.execute(format(zipper.uncompressed,epubfile,"mimetype")) then - os.execute(format(zipper.compressed,epubfile,"META-INF")) - os.execute(format(zipper.compressed,epubfile,"OEBPS")) - done = zipper.name + if zipped(zippers[i]) then break end end + end - lfs.chdir("..") + popdir() - if done then - application.report("epub archive made using %s: %s",done,file.join(epubpath,epubfile)) - else - local list = { } - for i=1,#zippers do - list[#list+1] = zipper.name - end - application.report("no epub archive made, install one of: %s",concat(list," ")) + if usedzipper then + local treefile = joinfile(epubpath,epubfile) + removefile(epubfile) + copyfile(treefile,epubfile) + if isfile(epubfile) then + removefile(treefile) end + report("epub archive made using %s: %s",usedzipper,epubfile) + else + local list = { } + for i=1,#zippers do + list[#list+1] = zippers[i].name + end + report("no epub archive made, install one of: % | t",list) + end + + if reportobsolete(oldfiles,newfiles,purge) then + obsolete = true + end + if obsolete and not purge then + report("use --purge to remove obsolete files") end end -- -if environment.argument("make") then - scripts.epub.make() -elseif environment.argument("exporthelp") then - application.export(environment.argument("exporthelp"),environment.files[1]) +local a_exporthelp = environment.argument("exporthelp") +local a_make = environment.argument("make") +local a_all = environment.argument("all") +local a_purge = a_all or environment.argument("purge") +local a_rename = a_all or environment.argument("rename") +local a_svgmath = a_all or environment.argument("svgmath") +local a_svgstyle = environment.argument("svgstyle") + +if a_make and a_svgmath then + require("x-math-svg") +end + +if a_make then + scripts.epub.make(a_purge,a_rename,a_svgmath,a_svgstyle) +elseif a_exporthelp then + application.export(a_exporthelp,environment.files[1]) else application.help() end + +-- java -jar d:\epubcheck\epubcheck-3.0.1.jar -v 3.0 -mode xhtml mkiv-publications.tree\mkiv-publications.epub diff --git a/Master/texmf-dist/scripts/context/lua/mtx-fcd.lua b/Master/texmf-dist/scripts/context/lua/mtx-fcd.lua index 2fcb9a2c7e9..76087cc37ea 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-fcd.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-fcd.lua @@ -247,7 +247,7 @@ end local function fcd_find() found = { } - pattern = environment.files[1] or "" + pattern = lower(environment.files[1] or "") if pattern ~= "" then pattern = string.escapedpattern(pattern) local paths = hash.paths @@ -255,7 +255,7 @@ local function fcd_find() local paths = paths[i][2] for i=1,#paths do local path = paths[i] - if find(path,pattern) then + if find(lower(path),pattern) then found[#found+1] = path end end diff --git a/Master/texmf-dist/scripts/context/lua/mtx-fonts.lua b/Master/texmf-dist/scripts/context/lua/mtx-fonts.lua index 675d9fb129e..694e6a6496f 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-fonts.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-fonts.lua @@ -38,7 +38,7 @@ local helpinfo = [[ <flag name="filter" value="list"><short>key-value pairs</short></flag> <flag name="all"><short>show all found instances (combined with other flags)</short></flag> <flag name="info"><short>give more details</short></flag> - <flag name="track" value="list"><short>enable trackers</short></flag> + <flag name="trackers" value="list"><short>enable trackers</short></flag> <flag name="statistics"><short>some info about the database</short></flag> </subcategory> </category> diff --git a/Master/texmf-dist/scripts/context/lua/mtx-interface.lua b/Master/texmf-dist/scripts/context/lua/mtx-interface.lua index 82cefd63817..1640f0891fd 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-interface.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-interface.lua @@ -248,7 +248,7 @@ function scripts.interface.editor(editor,split,forcedinterfaces) local mappings = { } local environments = { } local x = xml.load(keyfile) - for e, d, k in xml.elements(x,"cd:command") do + for e, d, k in xml.elements(x,"/cd:interface/cd:commands/cd:command") do -- somehow this was variable local at = d[k].at local name, value = at.name, at.value if name and value then @@ -256,7 +256,7 @@ function scripts.interface.editor(editor,split,forcedinterfaces) end end local x = xml.load(xmlfile) - for e, d, k in xml.elements(x,"cd:command") do + for e, d, k in xml.elements(x,"/cd:interface/cd:command") do local at = d[k].at local name, type = at.name, at["type"] if name and name ~= "" then @@ -322,7 +322,7 @@ function scripts.interface.check() if f then f:write("\\starttext\n") local x = xml.load(xmlfile) - for e, d, k in xml.elements(x,"cd:command") do + for e, d, k in xml.elements(x,"/cd:interface/cd:command") do local dk = d[k] local at = dk.at if at then @@ -384,6 +384,7 @@ function scripts.interface.interfaces() return a .. b .. c .. b end) end + -- we could just replace attributes for language, _ in next, commands.setuplayout do local texresult, xmlresult = { }, { } texresult[#texresult+1] = format("%% this file is auto-generated, don't edit this file\n%%") @@ -403,6 +404,7 @@ function scripts.interface.interfaces() report("saving interface translations '%s'",xmlfilename) if language ~= "en" and xmldata ~= "" then local newdata = xmldata:gsub("(<cd:interface.*language=.)en(.)","%1"..language.."%2",1) +-- newdata = replace(newdata, 'cd:command', 'name', interface.commands, interface.elements, language) newdata = replace(newdata, 'cd:string', 'value', interface.commands, interface.elements, language) newdata = replace(newdata, 'cd:variable' , 'value', interface.variables, nil, language) newdata = replace(newdata, 'cd:parameter', 'name', interface.constants, nil, language) diff --git a/Master/texmf-dist/scripts/context/lua/mtx-metapost.lua b/Master/texmf-dist/scripts/context/lua/mtx-metapost.lua index 08daec978dc..6306125d405 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-metapost.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-metapost.lua @@ -6,6 +6,8 @@ if not modules then modules = { } end modules ['mtx-metapost'] = { -- this was m license = "see context related readme files" } +-- todo: load map files + local helpinfo = [[ <?xml version="1.0"?> <application> @@ -60,24 +62,42 @@ local function assumes_latex(filename) return find(d,"\\documentstyle") or find(d,"\\documentclass") or find(d,"\\begin{document}") end +local basemaps = "original-base.map,original-ams-base.map,original-ams-euler.map,original-public-lm.map" + +local wrapper = "\\starttext\n%s\n%s\\stoptext" +local loadmap = "\\loadmapfile[%s]\n" local template = "\\startTEXpage\n\\convertMPtoPDF{%s}{1}{1}\n\\stopTEXpage" local texified = "\\starttext\n%s\n\\stoptext" local splitter = "\\startTEXpage\\externalfigure[%s][page=%s]\\stopTEXpage" local tempname = "mptopdf-temp.tex" -local function do_convert(filename) +local function do_mapfiles(mapfiles) + local maps = { } + for i=1,#mapfiles do + local mapfile = mapfiles[i] + application.report("using map file %a",mapfile) + maps[i] = format(loadmap,mapfile) + end + return table.concat(maps) +end + +local function do_convert(filename,mapfiles) if find(filename,".%d+$") or find(filename,"%.mps$") then - io.savedata(tempname,format(template,filename)) + local body = format(template,filename) + local maps = do_mapfiles(mapfiles) + io.savedata(tempname,format(wrapper,maps,body)) local resultname = format("%s-%s.pdf",file.nameonly(filename),file.suffix(filename)) local result = os.execute(format([[context --once --batch --purge --result=%s "%s"]],resultname,tempname)) return lfs.isfile(resultname) and resultname end end -local function do_split(filename,numbers) +local function do_split(filename,numbers,mapfiles) local name = file.nameonly(filename) + local maps = do_mapfiles(mapfiles) for i=1,#numbers do - io.savedata(tempname,format(splitter,file.addsuffix(name,"pdf"),i)) + local body = format(splitter,file.addsuffix(name,"pdf"),i) + io.savedata(tempname,format(wrapper,maps,body)) local resultname = format("%s-%s.pdf",name,numbers[i]) local result = os.execute(format([[context --once --batch --purge --result=%s "%s"]],resultname,tempname)) end @@ -99,12 +119,12 @@ local function do_texify(str) return format(texified,str), numbers end -local function do_convert_all(filename) +local function do_convert_all(filename,mapfiles) local results = dir.glob(file.nameonly(filename) .. ".*") -- reset local report = { } for i=1,#results do local filename = results[i] - local resultname = do_convert(filename) + local resultname = do_convert(filename,mapfiles) if resultname then report[#report+1] = { filename, resultname } end @@ -121,8 +141,8 @@ local function do_convert_all(filename) end end -local function do_convert_one(filename) - local resultname = do_convert(filename) +local function do_convert_one(filename,mapfiles) + local resultname = do_convert(filename,mapfiles) if resultname then report("%s => %s", filename,resultname) else @@ -131,17 +151,13 @@ local function do_convert_one(filename) end function scripts.mptopdf.convertall() - local rawmp = environment.arguments.rawmp or false - local metafun = environment.arguments.metafun or false - local latex = environment.arguments.latex or false - local pattern = environment.arguments.pattern or false - local split = environment.arguments.split or false - local files - if pattern then - files = dir.glob(file.nameonly(filename)) - else - files = environment.files - end + local rawmp = environment.arguments.rawmp or false + local metafun = environment.arguments.metafun or false + local latex = environment.arguments.latex or false + local pattern = environment.arguments.pattern or false + local split = environment.arguments.split or false + local files = pattern and dir.glob(file.nameonly(filename)) or environment.files + local mapfiles = utilities.parsers.settings_to_array(environment.arguments.mapfiles or basemaps) if #files > 0 then for i=1,#files do local filename = files[i] @@ -168,16 +184,16 @@ function scripts.mptopdf.convertall() local done = os.execute(command) if done then if convert then - do_convert_all(filename) + do_convert_all(filename,mapfiles) elseif split then - do_split(filename,numbers) + do_split(filename,numbers,mapfiles) -- already pdf, maybe optionally split end else report("error while processing mp file '%s'", filename) end else - do_convert_one(filename) + do_convert_one(filename,mapfiles) end end else diff --git a/Master/texmf-dist/scripts/context/lua/mtx-patterns.lua b/Master/texmf-dist/scripts/context/lua/mtx-patterns.lua index 5e2b2d902ca..efd2e9faf47 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-patterns.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-patterns.lua @@ -29,6 +29,7 @@ local helpinfo = [[ <flag name="specification"><short>additional patterns: e.g.: =cy,hyph-cy,welsh</short></flag> <flag name="compress"><short>compress data</short></flag> <flag name="words"><short>update words in given file</short></flag> + <flag name="hyphenate"><short>show hypephenated words</short></flag> </subcategory> </category> </flags> @@ -40,6 +41,7 @@ local helpinfo = [[ <example><command>mtxrun --script pattern --check --path=c:/data/develop/svn-hyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns</command></example> <example><command>mtxrun --script pattern --convert --path=c:/data/develop/svn-hyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns/tex --destination=e:/tmp/patterns</command></example> <example><command>mtxrun --script pattern --convert --path=c:/data/develop/svn-hyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns/txt --destination=e:/tmp/patterns</command></example> + <example><command>mtxrun --script pattern --hyphenate --language=nl --left=3 nogalwiedes inderdaad</command></example> </subcategory> </category> </examples> @@ -625,6 +627,49 @@ function scripts.patterns.words() end end +-- mtxrun --script patterns --hyphenate --language=nl nogalwiedes --left=3 +-- +-- hyphenator | +-- hyphenator | . n o g a l w i e d e s . . n o g a l w i e d e s . +-- hyphenator | .0n4 0 4 0 0 0 0 0 0 0 0 0 0 +-- hyphenator | 0o0g0a4l0 0 4 0 0 4 0 0 0 0 0 0 0 +-- hyphenator | 1g0a0 0 4 1 0 4 0 0 0 0 0 0 0 +-- hyphenator | 0l1w0 0 4 1 0 4 1 0 0 0 0 0 0 +-- hyphenator | 4i0e0 0 4 1 0 4 1 4 0 0 0 0 0 +-- hyphenator | 0i0e3d0e0 0 4 1 0 4 1 4 0 3 0 0 0 +-- hyphenator | 0e1d0 0 4 1 0 4 1 4 0 3 0 0 0 +-- hyphenator | 1d0e0 0 4 1 0 4 1 4 0 3 0 0 0 +-- hyphenator | 0d0e2s0 0 4 1 0 4 1 4 0 3 0 2 0 +-- hyphenator | 4s0. 0 4 1 0 4 1 4 0 3 0 4 0 +-- hyphenator | .0n4o1g0a4l1w4i0e3d0e4s0. . n o-g a l-w i e-d e s . +-- hyphenator | +-- mtx-patterns | nl 3 3 : nogalwiedes : nogal-wie-des + +function scripts.patterns.hyphenate() + require("lang-hyp") + local traditional = languages.hyphenators.traditional + local left = tonumber(environment.arguments.left) or 3 + local right = tonumber(environment.arguments.right) or 3 + local language = environment.arguments.language or "us" + local dictionary = traditional.loadpatterns(language) + local words = environment.files + local specification = { + leftcharmin = left, + rightcharmin = right, + leftchar = false, + rightchar = false, + } + trackers.enable("hyphenator.steps") + for i=1,#words do + local word = words[i] + report("%s %s %s : %s : %s", + language, left, right, + word, + traditional.injecthyphens(dictionary,word,specification) + ) + end +end + if environment.argument("check") then scripts.patterns.prepare() scripts.patterns.check() @@ -633,6 +678,8 @@ elseif environment.argument("convert") then scripts.patterns.convert() elseif environment.argument("words") then scripts.patterns.words() -- for the moment here +elseif environment.argument("hyphenate") then + scripts.patterns.hyphenate() -- for the moment here elseif environment.argument("exporthelp") then application.export(environment.argument("exporthelp"),environment.files[1]) else diff --git a/Master/texmf-dist/scripts/context/lua/mtx-server.lua b/Master/texmf-dist/scripts/context/lua/mtx-server.lua index 5466bfe80a2..dba07f1d584 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-server.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-server.lua @@ -278,6 +278,20 @@ handlers.html = handlers.htm local indices = { "index.htm", "index.html" } local portnumber = 31415 -- pi suits tex +local newline = lpeg.patterns.newline +local spacer = lpeg.patterns.spacer +local whitespace = lpeg.patterns.whitespace +local method = lpeg.P("GET") + + lpeg.P("POST") +local identify = (1-method)^0 + * lpeg.C(method) + * spacer^1 + * lpeg.C((1-spacer)^1) + * spacer^1 + * lpeg.P("HTTP/") + * (1-whitespace)^0 + * lpeg.C(lpeg.P(1)^0) + function scripts.webserver.run(configuration) -- check configuration configuration.port = tonumber(configuration.port or os.getenv("MTX_SERVER_PORT") or portnumber) or portnumber @@ -329,17 +343,24 @@ function scripts.webserver.run(configuration) local from = client:getpeername() report("request from: %s",tostring(from)) report("request data: %s",tostring(request)) - local fullurl = string.match(request,"GET (.+) HTTP/.*$") or "" -- todo: more clever / post - if fullurl == "" then + -- local fullurl = string.match(request,"(GET) (.+) HTTP/.*$") or "" -- todo: more clever / post + -- if fullurl == "" then +-- print("!!!!",request) + local method, fullurl, body = lpeg.match(identify,request) + if method == "" or fullurl == "" then report("no url") errormessage(client,configuration,404) else + + -- todo: method: POST + fullurl = url.unescapeget(fullurl) report("requested url: %s",fullurl) -- fullurl = socket.url.unescape(fullurl) -- happens later local hashed = url.hashed(fullurl) local query = url.query(hashed.query) local filename = hashed.path -- hm, not query? + hashed.body = body if script then filename = script report("forced script: %s",filename) diff --git a/Master/texmf-dist/scripts/context/lua/mtx-update.lua b/Master/texmf-dist/scripts/context/lua/mtx-update.lua index aedc48041e3..daf4f5b16ef 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-update.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-update.lua @@ -183,6 +183,8 @@ scripts.update.platforms = { ["linux-64"] = "linux-64", ["linux64"] = "linux-64", -- + ["linux-armhf"] = "linux-armhf", + -- ["freebsd"] = "freebsd", -- ["freebsd-amd64"] = "freebsd-amd64", diff --git a/Master/texmf-dist/scripts/context/lua/mtxrun.lua b/Master/texmf-dist/scripts/context/lua/mtxrun.lua index 5cb5c8d2416..48a9a27a250 100755 --- a/Master/texmf-dist/scripts/context/lua/mtxrun.lua +++ b/Master/texmf-dist/scripts/context/lua/mtxrun.lua @@ -56,7 +56,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-lua"] = package.loaded["l-lua"] or true --- original size: 3409, stripped down to: 1763 +-- original size: 3888, stripped down to: 2197 if not modules then modules={} end modules ['l-lua']={ version=1.001, @@ -139,6 +139,13 @@ end if lua then lua.mask=load([[τεχ = 1]]) and "utf" or "ascii" end +local flush=io.flush +if flush then + local execute=os.execute if execute then function os.execute(...) flush() return execute(...) end end + local exec=os.exec if exec then function os.exec (...) flush() return exec (...) end end + local spawn=os.spawn if spawn then function os.spawn (...) flush() return spawn (...) end end + local popen=io.popen if popen then function io.popen (...) flush() return popen (...) end end +end end -- of closure @@ -437,7 +444,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-lpeg"] = package.loaded["l-lpeg"] or true --- original size: 29983, stripped down to: 16202 +-- original size: 36225, stripped down to: 19891 if not modules then modules={} end modules ['l-lpeg']={ version=1.001, @@ -498,6 +505,8 @@ patterns.utfbom_16_le=utfbom_16_le patterns.utfbom_8=utfbom_8 patterns.utf_16_be_nl=P("\000\r\000\n")+P("\000\r")+P("\000\n") patterns.utf_16_le_nl=P("\r\000\n\000")+P("\r\000")+P("\n\000") +patterns.utf_32_be_nl=P("\000\000\000\r\000\000\000\n")+P("\000\000\000\r")+P("\000\000\000\n") +patterns.utf_32_le_nl=P("\r\000\000\000\n\000\000\000")+P("\r\000\000\000")+P("\n\000\000\000") patterns.utf8one=R("\000\127") patterns.utf8two=R("\194\223")*utf8next patterns.utf8three=R("\224\239")*utf8next*utf8next @@ -527,9 +536,21 @@ patterns.nonwhitespace=nonwhitespace local stripper=spacer^0*C((spacer^0*nonspacer^1)^0) local fullstripper=whitespace^0*C((whitespace^0*nonwhitespace^1)^0) local collapser=Cs(spacer^0/""*nonspacer^0*((spacer^0/" "*nonspacer^1)^0)) +local b_collapser=Cs(whitespace^0/""*(nonwhitespace^1+whitespace^1/" ")^0) +local e_collapser=Cs((whitespace^1*P(-1)/""+nonwhitespace^1+whitespace^1/" ")^0) +local m_collapser=Cs((nonwhitespace^1+whitespace^1/" ")^0) +local b_stripper=Cs(spacer^0/""*(nonspacer^1+spacer^1/" ")^0) +local e_stripper=Cs((spacer^1*P(-1)/""+nonspacer^1+spacer^1/" ")^0) +local m_stripper=Cs((nonspacer^1+spacer^1/" ")^0) patterns.stripper=stripper patterns.fullstripper=fullstripper patterns.collapser=collapser +patterns.b_collapser=b_collapser +patterns.m_collapser=m_collapser +patterns.e_collapser=e_collapser +patterns.b_stripper=b_stripper +patterns.m_stripper=m_stripper +patterns.e_stripper=e_stripper patterns.lowercase=lowercase patterns.uppercase=uppercase patterns.letter=patterns.lowercase+patterns.uppercase @@ -566,9 +587,12 @@ patterns.integer=sign^-1*digit^1 patterns.unsigned=digit^0*period*digit^1 patterns.float=sign^-1*patterns.unsigned patterns.cunsigned=digit^0*comma*digit^1 +patterns.cpunsigned=digit^0*(period+comma)*digit^1 patterns.cfloat=sign^-1*patterns.cunsigned +patterns.cpfloat=sign^-1*patterns.cpunsigned patterns.number=patterns.float+patterns.integer patterns.cnumber=patterns.cfloat+patterns.integer +patterns.cpnumber=patterns.cpfloat+patterns.integer patterns.oct=zero*octdigit^1 patterns.octal=patterns.oct patterns.HEX=zero*P("X")*(digit+uppercase)^1 @@ -985,23 +1009,40 @@ function lpeg.append(list,pp,delayed,checked) end return p end +local p_false=P(false) +local p_true=P(true) local function make(t) - local p + local function making(t) + local p=p_false + local keys=sortedkeys(t) + for i=1,#keys do + local k=keys[i] + if k~="" then + local v=t[k] + if v==true then + p=p+P(k)*p_true + elseif v==false then + else + p=p+P(k)*making(v) + end + end + end + if t[""] then + p=p+p_true + end + return p + end + local p=p_false local keys=sortedkeys(t) for i=1,#keys do local k=keys[i] - local v=t[k] - if not p then - if next(v) then - p=P(k)*make(v) - else - p=P(k) - end - else - if next(v) then - p=p+P(k)*make(v) + if k~="" then + local v=t[k] + if v==true then + p=p+P(k)*p_true + elseif v==false then else - p=p+P(k) + p=p+P(k)*making(v) end end end @@ -1009,13 +1050,73 @@ local function make(t) end function lpeg.utfchartabletopattern(list) local tree={} - for i=1,#list do - local t=tree - for c in gmatch(list[i],".") do - if not t[c] then - t[c]={} + local n=#list + if n==0 then + for s in next,list do + local t=tree + local p,pk + for c in gmatch(s,".") do + if t==true then + t={ [c]=true,[""]=true } + p[pk]=t + p=t + t=false + elseif t==false then + t={ [c]=false } + p[pk]=t + p=t + t=false + else + local tc=t[c] + if not tc then + tc=false + t[c]=false + end + p=t + t=tc + end + pk=c + end + if t==false then + p[pk]=true + elseif t==true then + else + t[""]=true + end + end + else + for i=1,n do + local s=list[i] + local t=tree + local p,pk + for c in gmatch(s,".") do + if t==true then + t={ [c]=true,[""]=true } + p[pk]=t + p=t + t=false + elseif t==false then + t={ [c]=false } + p[pk]=t + p=t + t=false + else + local tc=t[c] + if not tc then + tc=false + t[c]=false + end + p=t + t=tc + end + pk=c + end + if t==false then + p[pk]=true + elseif t==true then + else + t[""]=true end - t=t[c] end end return make(tree) @@ -1055,6 +1156,65 @@ local case_2=period*(digit-trailingzeros)^1*(trailingzeros/"") local number=digit^1*(case_1+case_2) local stripper=Cs((number+1)^0) lpeg.patterns.stripzeros=stripper +local byte_to_HEX={} +local byte_to_hex={} +local byte_to_dec={} +local hex_to_byte={} +for i=0,255 do + local H=format("%02X",i) + local h=format("%02x",i) + local d=format("%03i",i) + local c=char(i) + byte_to_HEX[c]=H + byte_to_hex[c]=h + byte_to_dec[c]=d + hex_to_byte[h]=c + hex_to_byte[H]=c +end +local hextobyte=P(2)/hex_to_byte +local bytetoHEX=P(1)/byte_to_HEX +local bytetohex=P(1)/byte_to_hex +local bytetodec=P(1)/byte_to_dec +local hextobytes=Cs(hextobyte^0) +local bytestoHEX=Cs(bytetoHEX^0) +local bytestohex=Cs(bytetohex^0) +local bytestodec=Cs(bytetodec^0) +patterns.hextobyte=hextobyte +patterns.bytetoHEX=bytetoHEX +patterns.bytetohex=bytetohex +patterns.bytetodec=bytetodec +patterns.hextobytes=hextobytes +patterns.bytestoHEX=bytestoHEX +patterns.bytestohex=bytestohex +patterns.bytestodec=bytestodec +function string.toHEX(s) + if not s or s=="" then + return s + else + return lpegmatch(bytestoHEX,s) + end +end +function string.tohex(s) + if not s or s=="" then + return s + else + return lpegmatch(bytestohex,s) + end +end +function string.todec(s) + if not s or s=="" then + return s + else + return lpegmatch(bytestodec,s) + end +end +function string.tobytes(s) + if not s or s=="" then + return s + else + return lpegmatch(hextobytes,s) + end +end end -- of closure @@ -1082,7 +1242,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-string"] = package.loaded["l-string"] or true --- original size: 5671, stripped down to: 2827 +-- original size: 5694, stripped down to: 2827 if not modules then modules={} end modules ['l-string']={ version=1.001, @@ -1187,7 +1347,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-table"] = package.loaded["l-table"] or true --- original size: 31860, stripped down to: 20846 +-- original size: 33830, stripped down to: 21894 if not modules then modules={} end modules ['l-table']={ version=1.001, @@ -1230,11 +1390,12 @@ function table.keys(t) end end local function compare(a,b) - local ta,tb=type(a),type(b) - if ta==tb then + local ta=type(a) + local tb=type(b) + if ta==tb and ta=="number" then return a<b else - return tostring(a)<tostring(b) + return tostring(a)<tostring(b) end end local function sortedkeys(tab) @@ -1334,14 +1495,14 @@ local function sortedhash(t,cmp) end local n=0 local m=#s - local function kv(s) + local function kv() if n<m then n=n+1 local k=s[n] return k,t[k] end end - return kv,s + return kv else return nothing end @@ -1506,7 +1667,7 @@ local function simple_table(t) if tv=="number" then nt=nt+1 if hexify then - tt[nt]=format("0x%04X",v) + tt[nt]=format("0x%X",v) else tt[nt]=tostring(v) end @@ -1537,7 +1698,7 @@ local function do_serialize(root,name,depth,level,indexed) local tn=type(name) if tn=="number" then if hexify then - handle(format("%s[0x%04X]={",depth,name)) + handle(format("%s[0x%X]={",depth,name)) else handle(format("%s[%s]={",depth,name)) end @@ -1554,7 +1715,7 @@ local function do_serialize(root,name,depth,level,indexed) end end end - if root and next(root) then + if root and next(root)~=nil then local first,last=nil,0 if compact then last=#root @@ -1576,7 +1737,7 @@ local function do_serialize(root,name,depth,level,indexed) if compact and first and tk=="number" and k>=first and k<=last then if tv=="number" then if hexify then - handle(format("%s 0x%04X,",depth,v)) + handle(format("%s 0x%X,",depth,v)) else handle(format("%s %s,",depth,v)) end @@ -1587,7 +1748,7 @@ local function do_serialize(root,name,depth,level,indexed) handle(format("%s %q,",depth,v)) end elseif tv=="table" then - if not next(v) then + if next(v)==nil then handle(format("%s {},",depth)) elseif inline then local st=simple_table(v) @@ -1617,25 +1778,25 @@ local function do_serialize(root,name,depth,level,indexed) elseif tv=="number" then if tk=="number" then if hexify then - handle(format("%s [0x%04X]=0x%04X,",depth,k,v)) + handle(format("%s [0x%X]=0x%X,",depth,k,v)) else handle(format("%s [%s]=%s,",depth,k,v)) end elseif tk=="boolean" then if hexify then - handle(format("%s [%s]=0x%04X,",depth,k and "true" or "false",v)) + handle(format("%s [%s]=0x%X,",depth,k and "true" or "false",v)) else handle(format("%s [%s]=%s,",depth,k and "true" or "false",v)) end elseif noquotes and not reserved[k] and lpegmatch(propername,k) then if hexify then - handle(format("%s %s=0x%04X,",depth,k,v)) + handle(format("%s %s=0x%X,",depth,k,v)) else handle(format("%s %s=%s,",depth,k,v)) end else if hexify then - handle(format("%s [%q]=0x%04X,",depth,k,v)) + handle(format("%s [%q]=0x%X,",depth,k,v)) else handle(format("%s [%q]=%s,",depth,k,v)) end @@ -1644,7 +1805,7 @@ local function do_serialize(root,name,depth,level,indexed) if reduce and tonumber(v) then if tk=="number" then if hexify then - handle(format("%s [0x%04X]=%s,",depth,k,v)) + handle(format("%s [0x%X]=%s,",depth,k,v)) else handle(format("%s [%s]=%s,",depth,k,v)) end @@ -1658,7 +1819,7 @@ local function do_serialize(root,name,depth,level,indexed) else if tk=="number" then if hexify then - handle(format("%s [0x%04X]=%q,",depth,k,v)) + handle(format("%s [0x%X]=%q,",depth,k,v)) else handle(format("%s [%s]=%q,",depth,k,v)) end @@ -1671,10 +1832,10 @@ local function do_serialize(root,name,depth,level,indexed) end end elseif tv=="table" then - if not next(v) then + if next(v)==nil then if tk=="number" then if hexify then - handle(format("%s [0x%04X]={},",depth,k)) + handle(format("%s [0x%X]={},",depth,k)) else handle(format("%s [%s]={},",depth,k)) end @@ -1690,7 +1851,7 @@ local function do_serialize(root,name,depth,level,indexed) if st then if tk=="number" then if hexify then - handle(format("%s [0x%04X]={ %s },",depth,k,concat(st,", "))) + handle(format("%s [0x%X]={ %s },",depth,k,concat(st,", "))) else handle(format("%s [%s]={ %s },",depth,k,concat(st,", "))) end @@ -1710,7 +1871,7 @@ local function do_serialize(root,name,depth,level,indexed) elseif tv=="boolean" then if tk=="number" then if hexify then - handle(format("%s [0x%04X]=%s,",depth,k,v and "true" or "false")) + handle(format("%s [0x%X]=%s,",depth,k,v and "true" or "false")) else handle(format("%s [%s]=%s,",depth,k,v and "true" or "false")) end @@ -1726,7 +1887,7 @@ local function do_serialize(root,name,depth,level,indexed) local f=getinfo(v).what=="C" and dump(dummy) or dump(v) if tk=="number" then if hexify then - handle(format("%s [0x%04X]=load(%q),",depth,k,f)) + handle(format("%s [0x%X]=load(%q),",depth,k,f)) else handle(format("%s [%s]=load(%q),",depth,k,f)) end @@ -1741,7 +1902,7 @@ local function do_serialize(root,name,depth,level,indexed) else if tk=="number" then if hexify then - handle(format("%s [0x%04X]=%q,",depth,k,tostring(v))) + handle(format("%s [0x%X]=%q,",depth,k,tostring(v))) else handle(format("%s [%s]=%q,",depth,k,tostring(v))) end @@ -1795,7 +1956,7 @@ local function serialize(_handle,root,name,specification) end elseif tname=="number" then if hexify then - handle(format("[0x%04X]={",name)) + handle(format("[0x%X]={",name)) else handle("["..name.."]={") end @@ -1813,7 +1974,7 @@ local function serialize(_handle,root,name,specification) local dummy=root._w_h_a_t_e_v_e_r_ root._w_h_a_t_e_v_e_r_=nil end - if next(root) then + if next(root)~=nil then do_serialize(root,name,"",0) end end @@ -1942,14 +2103,25 @@ local function identical(a,b) end table.identical=identical table.are_equal=are_equal -function table.compact(t) - if t then - for k,v in next,t do - if not next(v) then - t[k]=nil +local function sparse(old,nest,keeptables) + local new={} + for k,v in next,old do + if not (v=="" or v==false) then + if nest and type(v)=="table" then + v=sparse(v,nest) + if keeptables or next(v)~=nil then + new[k]=v + end + else + new[k]=v end end end + return new +end +table.sparse=sparse +function table.compact(t) + return sparse(t,true,true) end function table.contains(t,v) if t then @@ -2054,10 +2226,10 @@ function table.sub(t,i,j) return { unpack(t,i,j) } end function table.is_empty(t) - return not t or not next(t) + return not t or next(t)==nil end function table.has_one_entry(t) - return t and not next(t,next(t)) + return t and next(t,next(t))==nil end function table.loweredkeys(t) local l={} @@ -2102,6 +2274,44 @@ function table.values(t,s) return {} end end +function table.filtered(t,pattern,sort,cmp) + if t and type(pattern)=="string" then + if sort then + local s + if cmp then + s=sortedhashkeys(t,function(a,b) return cmp(t,a,b) end) + else + s=sortedkeys(t) + end + local n=0 + local m=#s + local function kv(s) + while n<m do + n=n+1 + local k=s[n] + if find(k,pattern) then + return k,t[k] + end + end + end + return kv,s + else + local n=next(t) + local function iterator() + while n~=nil do + local k=n + n=next(t,k) + if find(k,pattern) then + return k,t[k] + end + end + end + return iterator,t + end + else + return nothing + end +end end -- of closure @@ -2110,7 +2320,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-io"] = package.loaded["l-io"] or true --- original size: 8824, stripped down to: 6347 +-- original size: 8643, stripped down to: 6232 if not modules then modules={} end modules ['l-io']={ version=1.001, @@ -2417,8 +2627,6 @@ function io.readstring(f,n,m) local str=gsub(f:read(n),"\000","") return str end -if not io.i_limiter then function io.i_limiter() end end -if not io.o_limiter then function io.o_limiter() end end end -- of closure @@ -2645,7 +2853,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-os"] = package.loaded["l-os"] or true --- original size: 16093, stripped down to: 9704 +-- original size: 15761, stripped down to: 9403 if not modules then modules={} end modules ['l-os']={ version=1.001, @@ -2719,13 +2927,10 @@ if not os.__getenv__ then setmetatable(os.env,{ __index=__index,__newindex=__newindex } ) end end -local execute,spawn,exec,iopopen,ioflush=os.execute,os.spawn or os.execute,os.exec or os.execute,io.popen,io.flush -function os.execute(...) ioflush() return execute(...) end -function os.spawn (...) ioflush() return spawn (...) end -function os.exec (...) ioflush() return exec (...) end -function io.popen (...) ioflush() return iopopen(...) end +local execute=os.execute +local iopopen=io.popen function os.resultof(command) - local handle=io.popen(command,"r") + local handle=iopopen(command,"r") if handle then local result=handle:read("*all") or "" handle:close() @@ -2754,7 +2959,7 @@ local launchers={ unix="$BROWSER %s &> /dev/null &", } function os.launch(str) - os.execute(format(launchers[os.name] or launchers.unix,str)) + execute(format(launchers[os.name] or launchers.unix,str)) end if not os.times then function os.times() @@ -3029,7 +3234,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-file"] = package.loaded["l-file"] or true --- original size: 18308, stripped down to: 9948 +-- original size: 20949, stripped down to: 9945 if not modules then modules={} end modules ['l-file']={ version=1.001, @@ -3043,41 +3248,28 @@ local file=file if not lfs then lfs=optionalrequire("lfs") end -if not lfs then - lfs={ - getcurrentdir=function() - return "." - end, - attributes=function() - return nil - end, - isfile=function(name) - local f=io.open(name,'rb') - if f then - f:close() - return true - end - end, - isdir=function(name) - print("you need to load lfs") - return false - end - } -elseif not lfs.isfile then - local attributes=lfs.attributes - function lfs.isdir(name) - return attributes(name,"mode")=="directory" - end - function lfs.isfile(name) - return attributes(name,"mode")=="file" - end -end local insert,concat=table.insert,table.concat local match,find,gmatch=string.match,string.find,string.gmatch local lpegmatch=lpeg.match local getcurrentdir,attributes=lfs.currentdir,lfs.attributes local checkedsplit=string.checkedsplit local P,R,S,C,Cs,Cp,Cc,Ct=lpeg.P,lpeg.R,lpeg.S,lpeg.C,lpeg.Cs,lpeg.Cp,lpeg.Cc,lpeg.Ct +local tricky=S("/\\")*P(-1) +local attributes=lfs.attributes +if sandbox then + sandbox.redefine(lfs.isfile,"lfs.isfile") + sandbox.redefine(lfs.isdir,"lfs.isdir") +end +function lfs.isdir(name) + if lpegmatch(tricky,name) then + return attributes(name,"mode")=="directory" + else + return attributes(name.."/.","mode")=="directory" + end +end +function lfs.isfile(name) + return attributes(name,"mode")=="file" +end local colon=P(":") local period=P(".") local periods=P("..") @@ -3279,28 +3471,30 @@ local isroot=fwslash^1*-1 local hasroot=fwslash^1 local reslasher=lpeg.replacer(S("\\/"),"/") local deslasher=lpeg.replacer(S("\\/")^1,"/") -function file.join(...) - local lst={... } - local one=lst[1] +function file.join(one,two,three,...) + if not two then + return one=="" and one or lpegmatch(stripper,one) + end + if one=="" then + return lpegmatch(stripper,three and concat({ two,three,... },"/") or two) + end if lpegmatch(isnetwork,one) then local one=lpegmatch(reslasher,one) - local two=lpegmatch(deslasher,concat(lst,"/",2)) + local two=lpegmatch(deslasher,three and concat({ two,three,... },"/") or two) if lpegmatch(hasroot,two) then return one..two else return one.."/"..two end elseif lpegmatch(isroot,one) then - local two=lpegmatch(deslasher,concat(lst,"/",2)) + local two=lpegmatch(deslasher,three and concat({ two,three,... },"/") or two) if lpegmatch(hasroot,two) then return two else return "/"..two end - elseif one=="" then - return lpegmatch(stripper,concat(lst,"/",2)) else - return lpegmatch(deslasher,concat(lst,"/")) + return lpegmatch(deslasher,concat({ one,two,three,... },"/")) end end local drivespec=R("az","AZ")^1*colon @@ -3474,7 +3668,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-md5"] = package.loaded["l-md5"] or true --- original size: 3760, stripped down to: 2088 +-- original size: 3248, stripped down to: 2266 if not modules then modules={} end modules ['l-md5']={ version=1.001, @@ -3492,14 +3686,20 @@ if not md5 then } end local md5,file=md5,file -local gsub,format,byte=string.gsub,string.format,string.byte -local md5sum=md5.sum -local function convert(str,fmt) - return (gsub(md5sum(str),".",function(chr) return format(fmt,byte(chr)) end)) -end -if not md5.HEX then function md5.HEX(str) return convert(str,"%02X") end end -if not md5.hex then function md5.hex(str) return convert(str,"%02x") end end -if not md5.dec then function md5.dec(str) return convert(str,"%03i") end end +local gsub=string.gsub +do + local patterns=lpeg and lpeg.patterns + if patterns then + local bytestoHEX=patterns.bytestoHEX + local bytestohex=patterns.bytestohex + local bytestodec=patterns.bytestodec + local lpegmatch=lpeg.match + local md5sum=md5.sum + if not md5.HEX then function md5.HEX(str) if str then return lpegmatch(bytestoHEX,md5sum(str)) end end end + if not md5.hex then function md5.hex(str) if str then return lpegmatch(bytestohex,md5sum(str)) end end end + if not md5.dec then function md5.dec(str) if str then return lpegmatch(bytestodec,md5sum(str)) end end end + end +end function file.needsupdating(oldname,newname,threshold) local oldtime=lfs.attributes(oldname,"modification") if oldtime then @@ -3556,7 +3756,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-url"] = package.loaded["l-url"] or true --- original size: 12292, stripped down to: 5585 +-- original size: 12531, stripped down to: 5721 if not modules then modules={} end modules ['l-url']={ version=1.001, @@ -3642,19 +3842,25 @@ local splitquery=Cf (Ct("")*P { "sequence", pair=Cg(key*equal*value), },rawset) local function hashed(str) - if str=="" then + if not str or str=="" then return { scheme="invalid", original=str, } end - local s=split(str) - local rawscheme=s[1] - local rawquery=s[4] - local somescheme=rawscheme~="" - local somequery=rawquery~="" + local detailed=split(str) + local rawscheme="" + local rawquery="" + local somescheme=false + local somequery=false + if detailed then + rawscheme=detailed[1] + rawquery=detailed[4] + somescheme=rawscheme~="" + somequery=rawquery~="" + end if not somescheme and not somequery then - s={ + return { scheme="file", authority="", path=str, @@ -3664,28 +3870,28 @@ local function hashed(str) noscheme=true, filename=str, } - else - local authority,path,filename=s[2],s[3] - if authority=="" then - filename=path - elseif path=="" then - filename="" - else - filename=authority.."/"..path - end - s={ - scheme=rawscheme, - authority=authority, - path=path, - query=lpegmatch(unescaper,rawquery), - queries=lpegmatch(splitquery,rawquery), - fragment=s[5], - original=str, - noscheme=false, - filename=filename, - } end - return s + local authority=detailed[2] + local path=detailed[3] + local filename=nil + if authority=="" then + filename=path + elseif path=="" then + filename="" + else + filename=authority.."/"..path + end + return { + scheme=rawscheme, + authority=authority, + path=path, + query=lpegmatch(unescaper,rawquery), + queries=lpegmatch(splitquery,rawquery), + fragment=detailed[5], + original=str, + noscheme=false, + filename=filename, + } end url.split=split url.hasscheme=hasscheme @@ -3719,7 +3925,7 @@ function url.construct(hash) end return lpegmatch(escaper,concat(fullurl)) end -local pattern=Cs(noslash*R("az","AZ")*(S(":|")/":")*noslash*P(1)^0) +local pattern=Cs(slash^-1/""*R("az","AZ")*((S(":|")/":")+P(":"))*slash*P(1)^0) function url.filename(filename) local spec=hashed(filename) local path=spec.path @@ -3767,7 +3973,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-dir"] = package.loaded["l-dir"] or true --- original size: 14788, stripped down to: 9096 +-- original size: 16765, stripped down to: 11003 if not modules then modules={} end modules ['l-dir']={ version=1.001, @@ -3777,7 +3983,7 @@ if not modules then modules={} end modules ['l-dir']={ license="see context related readme files" } local type,select=type,select -local find,gmatch,match,gsub=string.find,string.gmatch,string.match,string.gsub +local find,gmatch,match,gsub,sub=string.find,string.gmatch,string.match,string.gsub,string.sub local concat,insert,remove,unpack=table.concat,table.insert,table.remove,table.unpack local lpegmatch=lpeg.match local P,S,R,C,Cc,Cs,Ct,Cv,V=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.Cc,lpeg.Cs,lpeg.Ct,lpeg.Cv,lpeg.V @@ -3786,53 +3992,126 @@ local dir=dir local lfs=lfs local attributes=lfs.attributes local walkdir=lfs.dir -local isdir=lfs.isdir -local isfile=lfs.isfile +local isdir=lfs.isdir +local isfile=lfs.isfile local currentdir=lfs.currentdir local chdir=lfs.chdir local mkdir=lfs.mkdir local onwindows=os.type=="windows" or find(os.getenv("PATH"),";",1,true) -if not isdir then - function isdir(name) - local a=attributes(name) - return a and a.mode=="directory" +if onwindows then + local tricky=S("/\\")*P(-1) + isdir=function(name) + if lpegmatch(tricky,name) then + return attributes(name,"mode")=="directory" + else + return attributes(name.."/.","mode")=="directory" + end + end + isfile=function(name) + return attributes(name,"mode")=="file" end lfs.isdir=isdir -end -if not isfile then - function isfile(name) - local a=attributes(name) - return a and a.mode=="file" + lfs.isfile=isfile +else + isdir=function(name) + return attributes(name,"mode")=="directory" end + isfile=function(name) + return attributes(name,"mode")=="file" + end + lfs.isdir=isdir lfs.isfile=isfile end function dir.current() return (gsub(currentdir(),"\\","/")) end -local lfsisdir=isdir -local function isdir(path) - path=gsub(path,"[/\\]+$","") - return lfsisdir(path) +local function glob_pattern_function(path,patt,recurse,action) + if isdir(path) then + local usedpath + if path=="/" then + usedpath="/." + elseif not find(path,"/$") then + usedpath=path.."/." + path=path.."/" + else + usedpath=path + end + local dirs + for name in walkdir(usedpath) do + if name~="." and name~=".." then + local full=path..name + local mode=attributes(full,'mode') + if mode=='file' then + if not patt or find(full,patt) then + action(full) + end + elseif recurse and mode=="directory" then + if not dirs then + dirs={ full } + else + dirs[#dirs+1]=full + end + end + end + end + if dirs then + for i=1,#dirs do + glob_pattern_function(dirs[i],patt,recurse,action) + end + end + end end -lfs.isdir=isdir -local function globpattern(path,patt,recurse,action) - if path=="/" then - path=path.."." - elseif not find(path,"/$") then - path=path..'/' - end - if isdir(path) then - for name in walkdir(path) do - local full=path..name - local mode=attributes(full,'mode') - if mode=='file' then - if find(full,patt) then - action(full) +local function glob_pattern_table(path,patt,recurse,result) + if not result then + result={} + end + if isdir(path) then + local usedpath + if path=="/" then + usedpath="/." + elseif not find(path,"/$") then + usedpath=path.."/." + path=path.."/" + else + usedpath=path + end + local dirs + for name in walkdir(usedpath) do + if name~="." and name~=".." then + local full=path..name + local mode=attributes(full,'mode') + if mode=='file' then + if not patt or find(full,patt) then + result[#result+1]=full + end + elseif recurse and mode=="directory" then + if not dirs then + dirs={ full } + else + dirs[#dirs+1]=full + end end - elseif recurse and (mode=="directory") and (name~='.') and (name~="..") then - globpattern(full,patt,recurse,action) end end + if dirs then + for i=1,#dirs do + glob_pattern_table(dirs[i],patt,recurse,result) + end + end + end + return result +end +local function globpattern(path,patt,recurse,method) + local kind=type(method) + if patt and sub(patt,1,-3)==path then + patt=false + end + if kind=="function" then + return glob_pattern_function(path,patt,recurse,method) + elseif kind=="table" then + return glob_pattern_table(path,patt,recurse,method) + else + return glob_pattern_table(path,patt,recurse,{}) end end dir.globpattern=globpattern @@ -3845,34 +4124,40 @@ local function collectpattern(path,patt,recurse,result) ok,scanner,first=xpcall(function() return walkdir(path) end,function() end) end if ok and type(scanner)=="function" then - if not find(path,"/$") then path=path..'/' end + if not find(path,"/$") then + path=path..'/' + end for name in scanner,first do - local full=path..name - local attr=attributes(full) - local mode=attr.mode - if mode=='file' then - if find(full,patt) then + if name=="." then + elseif name==".." then + else + local full=path..name + local attr=attributes(full) + local mode=attr.mode + if mode=='file' then + if find(full,patt) then + result[name]=attr + end + elseif recurse and mode=="directory" then + attr.list=collectpattern(full,patt,recurse) result[name]=attr end - elseif recurse and (mode=="directory") and (name~='.') and (name~="..") then - attr.list=collectpattern(full,patt,recurse) - result[name]=attr end end end return result end dir.collectpattern=collectpattern -local separator -if onwindows then +local separator,pattern +if onwindows then local slash=S("/\\")/"/" - pattern=Ct { + pattern={ [1]=(Cs(P(".")+slash^1)+Cs(R("az","AZ")*P(":")*slash^0)+Cc("./"))*V(2)*V(3), [2]=Cs(((1-S("*?/\\"))^0*slash)^0), [3]=Cs(P(1)^0) } -else - pattern=Ct { +else + pattern={ [1]=(C(P(".")+P("/")^1)+Cc("./"))*V(2)*V(3), [2]=C(((1-S("*?/"))^0*P("/"))^0), [3]=C(P(1)^0) @@ -3890,9 +4175,8 @@ local function glob(str,t) elseif isfile(str) then t(str) else - local split=lpegmatch(pattern,str) - if split then - local root,path,base=split[1],split[2],split[3] + local root,path,base=lpegmatch(pattern,str) + if root and path and base then local recurse=find(base,"**",1,true) local start=root..path local result=lpegmatch(filter,start..base) @@ -3914,16 +4198,12 @@ local function glob(str,t) return { str } end else - local split=lpegmatch(pattern,str) - if split then - local t=t or {} - local action=action or function(name) t[#t+1]=name end - local root,path,base=split[1],split[2],split[3] + local root,path,base=lpegmatch(pattern,str) + if root and path and base then local recurse=find(base,"**",1,true) local start=root..path local result=lpegmatch(filter,start..base) - globpattern(start,result,recurse,action) - return t + return globpattern(start,result,recurse,t) else return {} end @@ -4071,47 +4351,51 @@ else end end dir.makedirs=dir.mkdirs -if onwindows then - function dir.expandname(str) - local first,nothing,last=match(str,"^(//)(//*)(.*)$") - if first then - first=dir.current().."/" - end - if not first then - first,last=match(str,"^(//)/*(.*)$") - end - if not first then - first,last=match(str,"^([a-zA-Z]:)(.*)$") - if first and not find(last,"^/") then - local d=currentdir() - if chdir(first) then - first=dir.current() +do + local chdir=sandbox and sandbox.original(chdir) or chdir + if onwindows then + local xcurrentdir=dir.current + function dir.expandname(str) + local first,nothing,last=match(str,"^(//)(//*)(.*)$") + if first then + first=xcurrentdir().."/" + end + if not first then + first,last=match(str,"^(//)/*(.*)$") + end + if not first then + first,last=match(str,"^([a-zA-Z]:)(.*)$") + if first and not find(last,"^/") then + local d=currentdir() + if chdir(first) then + first=xcurrentdir() + end + chdir(d) end - chdir(d) + end + if not first then + first,last=xcurrentdir(),str + end + last=gsub(last,"//","/") + last=gsub(last,"/%./","/") + last=gsub(last,"^/*","") + first=gsub(first,"/*$","") + if last=="" or last=="." then + return first + else + return first.."/"..last end end - if not first then - first,last=dir.current(),str - end - last=gsub(last,"//","/") - last=gsub(last,"/%./","/") - last=gsub(last,"^/*","") - first=gsub(first,"/*$","") - if last=="" or last=="." then - return first - else - return first.."/"..last - end - end -else - function dir.expandname(str) - if not find(str,"^/") then - str=currentdir().."/"..str + else + function dir.expandname(str) + if not find(str,"^/") then + str=currentdir().."/"..str + end + str=gsub(str,"//","/") + str=gsub(str,"/%./","/") + str=gsub(str,"(.)/%.$","%1") + return str end - str=gsub(str,"//","/") - str=gsub(str,"/%./","/") - str=gsub(str,"(.)/%.$","%1") - return str end end file.expandname=dir.expandname @@ -4154,7 +4438,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-boolean"] = package.loaded["l-boolean"] or true --- original size: 1809, stripped down to: 1527 +-- original size: 1850, stripped down to: 1568 if not modules then modules={} end modules ['l-boolean']={ version=1.001, @@ -4208,11 +4492,11 @@ function string.booleanstring(str) return str=="yes" or str=="on" or str=="t" end end -function string.is_boolean(str,default) +function string.is_boolean(str,default,strict) if type(str)=="string" then - if str=="true" or str=="yes" or str=="on" or str=="t" or str=="1" then + if str=="true" or str=="yes" or str=="on" or str=="t" or (not strict and str=="1") then return true - elseif str=="false" or str=="no" or str=="off" or str=="f" or str=="0" then + elseif str=="false" or str=="no" or str=="off" or str=="f" or (not strict and str=="0") then return false end end @@ -4226,7 +4510,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-unicode"] = package.loaded["l-unicode"] or true --- original size: 33706, stripped down to: 14938 +-- original size: 37233, stripped down to: 15684 if not modules then modules={} end modules ['l-unicode']={ version=1.001, @@ -4242,7 +4526,9 @@ local type=type local char,byte,format,sub,gmatch=string.char,string.byte,string.format,string.sub,string.gmatch local concat=table.concat local P,C,R,Cs,Ct,Cmt,Cc,Carg,Cp=lpeg.P,lpeg.C,lpeg.R,lpeg.Cs,lpeg.Ct,lpeg.Cmt,lpeg.Cc,lpeg.Carg,lpeg.Cp -local lpegmatch,patterns=lpeg.match,lpeg.patterns +local lpegmatch=lpeg.match +local patterns=lpeg.patterns +local tabletopattern=lpeg.utfchartabletopattern local bytepairs=string.bytepairs local finder=lpeg.finder local replacer=lpeg.replacer @@ -4251,7 +4537,7 @@ local utfgmatch=utf.gmatch local p_utftype=patterns.utftype local p_utfstricttype=patterns.utfstricttype local p_utfoffset=patterns.utfoffset -local p_utf8char=patterns.utf8char +local p_utf8char=patterns.utf8character local p_utf8byte=patterns.utf8byte local p_utfbom=patterns.utfbom local p_newline=patterns.newline @@ -4450,15 +4736,52 @@ if not utf.sub then end end end -function utf.remapper(mapping) - local pattern=Cs((p_utf8char/mapping)^0) - return function(str) - if not str or str=="" then - return "" +function utf.remapper(mapping,option) + local variant=type(mapping) + if variant=="table" then + if option=="dynamic" then + local pattern=false + table.setmetatablenewindex(mapping,function(t,k,v) rawset(t,k,v) pattern=false end) + return function(str) + if not str or str=="" then + return "" + else + if not pattern then + pattern=Cs((tabletopattern(mapping)/mapping+p_utf8char)^0) + end + return lpegmatch(pattern,str) + end + end + elseif option=="pattern" then + return Cs((tabletopattern(mapping)/mapping+p_utf8char)^0) else - return lpegmatch(pattern,str) + local pattern=Cs((tabletopattern(mapping)/mapping+p_utf8char)^0) + return function(str) + if not str or str=="" then + return "" + else + return lpegmatch(pattern,str) + end + end,pattern + end + elseif variant=="function" then + if option=="pattern" then + return Cs((p_utf8char/mapping+p_utf8char)^0) + else + local pattern=Cs((p_utf8char/mapping+p_utf8char)^0) + return function(str) + if not str or str=="" then + return "" + else + return lpegmatch(pattern,str) + end + end,pattern + end + else + return function(str) + return str or "" end - end,pattern + end end function utf.replacer(t) local r=replacer(t,false,false,true) @@ -4508,190 +4831,157 @@ function utf.magic(f) end local utf16_to_utf8_be,utf16_to_utf8_le local utf32_to_utf8_be,utf32_to_utf8_le -local utf_16_be_linesplitter=patterns.utfbom_16_be^-1*lpeg.tsplitat(patterns.utf_16_be_nl) -local utf_16_le_linesplitter=patterns.utfbom_16_le^-1*lpeg.tsplitat(patterns.utf_16_le_nl) -if bytepairs then - utf16_to_utf8_be=function(t) - if type(t)=="string" then - t=lpegmatch(utf_16_be_linesplitter,t) - end - local result={} - for i=1,#t do - local r,more=0,0 - for left,right in bytepairs(t[i]) do - if right then - local now=256*left+right - if more>0 then - now=(more-0xD800)*0x400+(now-0xDC00)+0x10000 - more=0 - r=r+1 - result[r]=utfchar(now) - elseif now>=0xD800 and now<=0xDBFF then - more=now - else - r=r+1 - result[r]=utfchar(now) - end - end - end - t[i]=concat(result,"",1,r) - end - return t +local utf_16_be_getbom=patterns.utfbom_16_be^-1 +local utf_16_le_getbom=patterns.utfbom_16_le^-1 +local utf_32_be_getbom=patterns.utfbom_32_be^-1 +local utf_32_le_getbom=patterns.utfbom_32_le^-1 +local utf_16_be_linesplitter=utf_16_be_getbom*lpeg.tsplitat(patterns.utf_16_be_nl) +local utf_16_le_linesplitter=utf_16_le_getbom*lpeg.tsplitat(patterns.utf_16_le_nl) +local utf_32_be_linesplitter=utf_32_be_getbom*lpeg.tsplitat(patterns.utf_32_be_nl) +local utf_32_le_linesplitter=utf_32_le_getbom*lpeg.tsplitat(patterns.utf_32_le_nl) +local more=0 +local p_utf16_to_utf8_be=C(1)*C(1)/function(left,right) + local now=256*byte(left)+byte(right) + if more>0 then + now=(more-0xD800)*0x400+(now-0xDC00)+0x10000 + more=0 + return utfchar(now) + elseif now>=0xD800 and now<=0xDBFF then + more=now + return "" + else + return utfchar(now) + end +end +local p_utf16_to_utf8_le=C(1)*C(1)/function(right,left) + local now=256*byte(left)+byte(right) + if more>0 then + now=(more-0xD800)*0x400+(now-0xDC00)+0x10000 + more=0 + return utfchar(now) + elseif now>=0xD800 and now<=0xDBFF then + more=now + return "" + else + return utfchar(now) + end +end +local p_utf32_to_utf8_be=C(1)*C(1)*C(1)*C(1)/function(a,b,c,d) + return utfchar(256*256*256*byte(a)+256*256*byte(b)+256*byte(c)+byte(d)) +end +local p_utf32_to_utf8_le=C(1)*C(1)*C(1)*C(1)/function(a,b,c,d) + return utfchar(256*256*256*byte(d)+256*256*byte(c)+256*byte(b)+byte(a)) +end +p_utf16_to_utf8_be=P(true)/function() more=0 end*utf_16_be_getbom*Cs(p_utf16_to_utf8_be^0) +p_utf16_to_utf8_le=P(true)/function() more=0 end*utf_16_le_getbom*Cs(p_utf16_to_utf8_le^0) +p_utf32_to_utf8_be=P(true)/function() more=0 end*utf_32_be_getbom*Cs(p_utf32_to_utf8_be^0) +p_utf32_to_utf8_le=P(true)/function() more=0 end*utf_32_le_getbom*Cs(p_utf32_to_utf8_le^0) +patterns.utf16_to_utf8_be=p_utf16_to_utf8_be +patterns.utf16_to_utf8_le=p_utf16_to_utf8_le +patterns.utf32_to_utf8_be=p_utf32_to_utf8_be +patterns.utf32_to_utf8_le=p_utf32_to_utf8_le +utf16_to_utf8_be=function(s) + if s and s~="" then + return lpegmatch(p_utf16_to_utf8_be,s) + else + return s end - utf16_to_utf8_le=function(t) - if type(t)=="string" then - t=lpegmatch(utf_16_le_linesplitter,t) - end - local result={} - for i=1,#t do - local r,more=0,0 - for left,right in bytepairs(t[i]) do - if right then - local now=256*right+left - if more>0 then - now=(more-0xD800)*0x400+(now-0xDC00)+0x10000 - more=0 - r=r+1 - result[r]=utfchar(now) - elseif now>=0xD800 and now<=0xDBFF then - more=now - else - r=r+1 - result[r]=utfchar(now) - end - end - end - t[i]=concat(result,"",1,r) - end - return t +end +local utf16_to_utf8_be_t=function(t) + if not t then + return nil + elseif type(t)=="string" then + t=lpegmatch(utf_16_be_linesplitter,t) end - utf32_to_utf8_be=function(t) - if type(t)=="string" then - t=lpegmatch(utflinesplitter,t) - end - local result={} - for i=1,#t do - local r,more=0,-1 - for a,b in bytepairs(t[i]) do - if a and b then - if more<0 then - more=256*256*256*a+256*256*b - else - r=r+1 - result[t]=utfchar(more+256*a+b) - more=-1 - end - else - break - end - end - t[i]=concat(result,"",1,r) + for i=1,#t do + local s=t[i] + if s~="" then + t[i]=lpegmatch(p_utf16_to_utf8_be,s) end - return t end - utf32_to_utf8_le=function(t) - if type(t)=="string" then - t=lpegmatch(utflinesplitter,t) - end - local result={} - for i=1,#t do - local r,more=0,-1 - for a,b in bytepairs(t[i]) do - if a and b then - if more<0 then - more=256*b+a - else - r=r+1 - result[t]=utfchar(more+256*256*256*b+256*256*a) - more=-1 - end - else - break - end - end - t[i]=concat(result,"",1,r) - end - return t + return t +end +utf16_to_utf8_le=function(s) + if s and s~="" then + return lpegmatch(p_utf16_to_utf8_le,s) + else + return s end -else - utf16_to_utf8_be=function(t) - if type(t)=="string" then - t=lpegmatch(utf_16_be_linesplitter,t) - end - local result={} - for i=1,#t do - local r,more=0,0 - for left,right in gmatch(t[i],"(.)(.)") do - if left=="\000" then - r=r+1 - result[r]=utfchar(byte(right)) - elseif right then - local now=256*byte(left)+byte(right) - if more>0 then - now=(more-0xD800)*0x400+(now-0xDC00)+0x10000 - more=0 - r=r+1 - result[r]=utfchar(now) - elseif now>=0xD800 and now<=0xDBFF then - more=now - else - r=r+1 - result[r]=utfchar(now) - end - end - end - t[i]=concat(result,"",1,r) +end +local utf16_to_utf8_le_t=function(t) + if not t then + return nil + elseif type(t)=="string" then + t=lpegmatch(utf_16_le_linesplitter,t) + end + for i=1,#t do + local s=t[i] + if s~="" then + t[i]=lpegmatch(p_utf16_to_utf8_le,s) end - return t end - utf16_to_utf8_le=function(t) - if type(t)=="string" then - t=lpegmatch(utf_16_le_linesplitter,t) + return t +end +utf32_to_utf8_be=function(s) + if s and s~="" then + return lpegmatch(p_utf32_to_utf8_be,s) + else + return s + end +end +local utf32_to_utf8_be_t=function(t) + if not t then + return nil + elseif type(t)=="string" then + t=lpegmatch(utf_32_be_linesplitter,t) + end + for i=1,#t do + local s=t[i] + if s~="" then + t[i]=lpegmatch(p_utf32_to_utf8_be,s) end - local result={} - for i=1,#t do - local r,more=0,0 - for left,right in gmatch(t[i],"(.)(.)") do - if right=="\000" then - r=r+1 - result[r]=utfchar(byte(left)) - elseif right then - local now=256*byte(right)+byte(left) - if more>0 then - now=(more-0xD800)*0x400+(now-0xDC00)+0x10000 - more=0 - r=r+1 - result[r]=utfchar(now) - elseif now>=0xD800 and now<=0xDBFF then - more=now - else - r=r+1 - result[r]=utfchar(now) - end - end - end - t[i]=concat(result,"",1,r) + end + return t +end +utf32_to_utf8_le=function(s) + if s and s~="" then + return lpegmatch(p_utf32_to_utf8_le,s) + else + return s + end +end +local utf32_to_utf8_le_t=function(t) + if not t then + return nil + elseif type(t)=="string" then + t=lpegmatch(utf_32_le_linesplitter,t) + end + for i=1,#t do + local s=t[i] + if s~="" then + t[i]=lpegmatch(p_utf32_to_utf8_le,s) end - return t end - utf32_to_utf8_le=function() return {} end - utf32_to_utf8_be=function() return {} end + return t end +utf.utf16_to_utf8_le_t=utf16_to_utf8_le_t +utf.utf16_to_utf8_be_t=utf16_to_utf8_be_t +utf.utf32_to_utf8_le_t=utf32_to_utf8_le_t +utf.utf32_to_utf8_be_t=utf32_to_utf8_be_t utf.utf16_to_utf8_le=utf16_to_utf8_le utf.utf16_to_utf8_be=utf16_to_utf8_be utf.utf32_to_utf8_le=utf32_to_utf8_le utf.utf32_to_utf8_be=utf32_to_utf8_be -function utf.utf8_to_utf8(t) +function utf.utf8_to_utf8_t(t) return type(t)=="string" and lpegmatch(utflinesplitter,t) or t end -function utf.utf16_to_utf8(t,endian) - return endian and utf16_to_utf8_be(t) or utf16_to_utf8_le(t) or t +function utf.utf16_to_utf8_t(t,endian) + return endian and utf16_to_utf8_be_t(t) or utf16_to_utf8_le_t(t) or t end -function utf.utf32_to_utf8(t,endian) - return endian and utf32_to_utf8_be(t) or utf32_to_utf8_le(t) or t +function utf.utf32_to_utf8_t(t,endian) + return endian and utf32_to_utf8_be_t(t) or utf32_to_utf8_le_t(t) or t end -local function little(c) - local b=byte(c) +local function little(b) if b<0x10000 then return char(b%256,b/256) else @@ -4700,8 +4990,7 @@ local function little(c) return char(b1%256,b1/256,b2%256,b2/256) end end -local function big(c) - local b=byte(c) +local function big(b) if b<0x10000 then return char(b/256,b%256) else @@ -4710,27 +4999,29 @@ local function big(c) return char(b1/256,b1%256,b2/256,b2%256) end end -local _,l_remap=utf.remapper(little) -local _,b_remap=utf.remapper(big) -function utf.utf8_to_utf16_be(str,nobom) +local l_remap=Cs((p_utf8byte/little+P(1)/"")^0) +local b_remap=Cs((p_utf8byte/big+P(1)/"")^0) +local function utf8_to_utf16_be(str,nobom) if nobom then return lpegmatch(b_remap,str) else return char(254,255)..lpegmatch(b_remap,str) end end -function utf.utf8_to_utf16_le(str,nobom) +local function utf8_to_utf16_le(str,nobom) if nobom then return lpegmatch(l_remap,str) else return char(255,254)..lpegmatch(l_remap,str) end end +utf.utf8_to_utf16_be=utf8_to_utf16_be +utf.utf8_to_utf16_le=utf8_to_utf16_le function utf.utf8_to_utf16(str,littleendian,nobom) if littleendian then - return utf.utf8_to_utf16_le(str,nobom) + return utf8_to_utf16_le(str,nobom) else - return utf.utf8_to_utf16_be(str,nobom) + return utf8_to_utf16_be(str,nobom) end end local pattern=Cs ( @@ -4746,16 +5037,16 @@ function utf.xstring(s) return format("0x%05X",type(s)=="number" and s or utfbyte(s)) end function utf.toeight(str) - if not str then + if not str or str=="" then return nil end local utftype=lpegmatch(p_utfstricttype,str) if utftype=="utf-8" then - return sub(str,4) - elseif utftype=="utf-16-le" then - return utf16_to_utf8_le(str) + return sub(str,4) elseif utftype=="utf-16-be" then - return utf16_to_utf8_ne(str) + return utf16_to_utf8_be(str) + elseif utftype=="utf-16-le" then + return utf16_to_utf8_le(str) else return str end @@ -4834,7 +5125,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-math"] = package.loaded["l-math"] or true --- original size: 915, stripped down to: 836 +-- original size: 974, stripped down to: 890 if not modules then modules={} end modules ['l-math']={ version=1.001, @@ -4844,6 +5135,9 @@ if not modules then modules={} end modules ['l-math']={ license="see context related readme files" } local floor,sin,cos,tan=math.floor,math.sin,math.cos,math.tan +if not math.ceiling then + math.ceiling=math.ceil +end if not math.round then function math.round(x) return floor(x+0.5) end end @@ -4871,7 +5165,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-str"] = package.loaded["util-str"] or true --- original size: 33063, stripped down to: 18397 +-- original size: 34407, stripped down to: 18852 if not modules then modules={} end modules ['util-str']={ version=1.001, @@ -5001,10 +5295,12 @@ local striplinepatterns={ ["retain"]=p_retain_normal, ["retain and collapse"]=p_retain_collapse, ["retain and no empty"]=p_retain_noempty, + ["collapse"]=patterns.collapser, } +setmetatable(striplinepatterns,{ __index=function(t,k) return p_prune_collapse end }) strings.striplinepatterns=striplinepatterns function strings.striplines(str,how) - return str and lpegmatch(how and striplinepatterns[how] or p_prune_collapse,str) or str + return str and lpegmatch(striplinepatterns[how],str) or str end strings.striplong=strings.striplines function strings.nice(str) @@ -5044,10 +5340,10 @@ string.tracedchars=tracedchars strings.tracers=tracedchars function string.tracedchar(b) if type(b)=="number" then - return tracedchars[b] or (utfchar(b).." (U+"..format('%05X',b)..")") + return tracedchars[b] or (utfchar(b).." (U+"..format("%05X",b)..")") else local c=utfbyte(b) - return tracedchars[c] or (b.." (U+"..format('%05X',c)..")") + return tracedchars[c] or (b.." (U+"..(c and format("%05X",c) or "?????")..")") end end function number.signed(i) @@ -5184,12 +5480,12 @@ local format_f=function(f) n=n+1 return format("format('%%%sf',a%s)",f,n) end -local format_F=function(f) +local format_F=function(f) n=n+1 if not f or f=="" then - return format("(((a%s > -0.0000000005 and a%s < 0.0000000005) and '0') or (a%s == 1 and '1') or format('%%.9f',a%s))",n,n,n,n) + return format("(((a%s > -0.0000000005 and a%s < 0.0000000005) and '0') or format((a%s %% 1 == 0) and '%%i' or '%%.9f',a%s))",n,n,n,n) else - return format("((a%s == 0 and '0') or (a%s == 1 and '1') or format('%%%sf',a%s))",n,n,f,n) + return format("format((a%s %% 1 == 0) and '%%i' or '%%%sf',a%s)",n,f,n) end end local format_g=function(f) @@ -5417,7 +5713,6 @@ local builder=Cs { "start", +V("j")+V("J") +V("m")+V("M") +V("z") -+V("*") )+V("*") )*(P(-1)+Carg(1)) )^0, @@ -5461,6 +5756,7 @@ local builder=Cs { "start", ["a"]=(prefix_any*P("a"))/format_a, ["A"]=(prefix_any*P("A"))/format_A, ["*"]=Cs(((1-P("%"))^1+P("%%")/"%%")^1)/format_rest, + ["?"]=Cs(((1-P("%"))^1 )^1)/format_rest, ["!"]=Carg(2)*prefix_any*P("!")*C((1-P("!"))^1)*P("!")/format_extension, } local direct=Cs ( @@ -5535,6 +5831,15 @@ else add(formatters,"tex",[[lpegmatch(texescape,%s)]],{ texescape=lpeg.patterns.texescape }) add(formatters,"lua",[[lpegmatch(luaescape,%s)]],{ luaescape=lpeg.patterns.luaescape }) end +local dquote=patterns.dquote +local equote=patterns.escaped+dquote/'\\"'+1 +local space=patterns.space +local cquote=Cc('"') +local pattern=Cs(dquote*(equote-P(-2))^0*dquote) ++Cs(cquote*(equote-space)^0*space*equote^0*cquote) +function string.optionalquoted(str) + return lpegmatch(pattern,str) or str +end end -- of closure @@ -5543,7 +5848,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-tab"] = package.loaded["util-tab"] or true --- original size: 23985, stripped down to: 16069 +-- original size: 24267, stripped down to: 16260 if not modules then modules={} end modules ['util-tab']={ version=1.001, @@ -5633,6 +5938,15 @@ function tables.removevalue(t,value) end end end +function tables.replacevalue(t,oldvalue,newvalue) + if oldvalue and newvalue then + for i=1,#t do + if t[i]==oldvalue then + t[i]=newvalue + end + end + end +end function tables.insertbeforevalue(t,value,extra) for i=1,#t do if t[i]==extra then @@ -6026,7 +6340,7 @@ function table.serialize(root,name,specification) end depth=depth+1 end - if root and next(root) then + if root and next(root)~=nil then local first=nil local last=0 last=#root @@ -6051,7 +6365,7 @@ function table.serialize(root,name,specification) elseif tv=="string" then n=n+1 t[n]=f_val_str(depth,v) elseif tv=="table" then - if not next(v) then + if next(v)==nil then n=n+1 t[n]=f_val_not(depth) else local st=simple_table(v) @@ -6081,7 +6395,7 @@ function table.serialize(root,name,specification) n=n+1 t[n]=f_key_boo_value_str(depth,k,v) end elseif tv=="table" then - if not next(v) then + if next(v)==nil then if tk=="number" then n=n+1 t[n]=f_key_num_value_not(depth,k,v) elseif tk=="string" then @@ -6139,7 +6453,7 @@ function table.serialize(root,name,specification) local dummy=root._w_h_a_t_e_v_e_r_ root._w_h_a_t_e_v_e_r_=nil end - if next(root) then + if next(root)~=nil then do_serialize(root,name,1,0) end end @@ -6302,7 +6616,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-prs"] = package.loaded["util-prs"] or true --- original size: 19618, stripped down to: 14012 +-- original size: 21751, stripped down to: 15108 if not modules then modules={} end modules ['util-prs']={ version=1.001, @@ -6458,11 +6772,30 @@ function parsers.settings_to_array(str,strict) return { str } end end -local separator=space^0*comma*space^0 -local value=P(lbrace*C((nobrace+nestedbraces)^0)*rbrace)+C((nestedbraces+(1-(space^0*(comma+P(-1)))))^0) -local withvalue=Carg(1)*value/function(f,s) return f(s) end -local pattern_a=spaces*Ct(value*(separator*value)^0) -local pattern_b=spaces*withvalue*(separator*withvalue)^0 +local cache_a={} +local cache_b={} +function parsers.groupedsplitat(symbol,withaction) + if not symbol then + symbol="," + end + local pattern=(withaction and cache_b or cache_a)[symbol] + if not pattern then + local symbols=S(symbol) + local separator=space^0*symbols*space^0 + local value=P(lbrace*C((nobrace+nestedbraces)^0)*rbrace)+C((nestedbraces+(1-(space^0*(symbols+P(-1)))))^0) + if withaction then + local withvalue=Carg(1)*value/function(f,s) return f(s) end + pattern=spaces*withvalue*(separator*withvalue)^0 + cache_b[symbol]=pattern + else + pattern=spaces*Ct(value*(separator*value)^0) + cache_a[symbol]=pattern + end + end + return pattern +end +local pattern_a=parsers.groupedsplitat(",",false) +local pattern_b=parsers.groupedsplitat(",",true) function parsers.stripped_settings_to_array(str) if not str or str=="" then return {} @@ -6524,12 +6857,9 @@ function parsers.array_to_string(a,separator) return "" end end -function parsers.settings_to_set(str,t) - t=t or {} - for s in gmatch(str,"[^, ]+") do - t[s]=true - end - return t +local pattern=Cf(Ct("")*Cg(C((1-S(", "))^1)*S(", ")^0*Cc(true))^1,rawset) +function utilities.parsers.settings_to_set(str,t) + return str and lpegmatch(pattern,str) or {} end function parsers.simple_hash_to_string(h,separator) local t,tn={},0 @@ -6541,12 +6871,16 @@ function parsers.simple_hash_to_string(h,separator) end return concat(t,separator or ",") end -local str=C((1-whitespace-equal)^1) +local str=Cs(lpegpatterns.unquoted)+C((1-whitespace-equal)^1) local setting=Cf(Carg(1)*(whitespace^0*Cg(str*whitespace^0*(equal*whitespace^0*str+Cc(""))))^1,rawset) local splitter=setting^1 function utilities.parsers.options_to_hash(str,target) return str and lpegmatch(splitter,str,1,target or {}) or {} end +local splitter=lpeg.tsplitat(" ") +function utilities.parsers.options_to_array(str) + return str and lpegmatch(splitter,str) or {} +end local value=P(lbrace*C((nobrace+nestedbraces)^0)*rbrace)+C(digit^1*lparent*(noparent+nestedparents)^1*rparent)+C((nestedbraces+(1-comma))^1) local pattern_a=spaces*Ct(value*(separator*value)^0) local function repeater(n,str) @@ -6668,7 +7002,7 @@ function parsers.rfc4180splitter(specification) local field=escaped+non_escaped+Cc("") local record=Ct(field*(separator*field)^1) local headerline=record*Cp() - local wholeblob=Ct((newline^-1*record)^0) + local wholeblob=Ct((newline^(specification.strict and -1 or 1)*record)^0) return function(data,getheader) if getheader then local header,position=lpegmatch(headerline,data) @@ -6705,13 +7039,13 @@ function parsers.stepper(str,n,action) lpegmatch(stepper,str,1,n,action or print) end end -local pattern_math=Cs((P("%")/"\\percent "+P("^")*Cc("{")*lpegpatterns.integer*Cc("}")+P(1))^0) -local pattern_text=Cs((P("%")/"\\percent "+(P("^")/"\\high")*Cc("{")*lpegpatterns.integer*Cc("}")+P(1))^0) +local pattern_math=Cs((P("%")/"\\percent "+P("^")*Cc("{")*lpegpatterns.integer*Cc("}")+anything)^0) +local pattern_text=Cs((P("%")/"\\percent "+(P("^")/"\\high")*Cc("{")*lpegpatterns.integer*Cc("}")+anything)^0) patterns.unittotex=pattern function parsers.unittotex(str,textmode) return lpegmatch(textmode and pattern_text or pattern_math,str) end -local pattern=Cs((P("^")/"<sup>"*lpegpatterns.integer*Cc("</sup>")+P(1))^0) +local pattern=Cs((P("^")/"<sup>"*lpegpatterns.integer*Cc("</sup>")+anything)^0) function parsers.unittoxml(str) return lpegmatch(pattern,str) end @@ -6783,6 +7117,18 @@ function utilities.parsers.runtime(time) local seconds=mod(time,60) return days,hours,minutes,seconds end +local spacing=whitespace^0 +local apply=P("->") +local method=C((1-apply)^1) +local token=lbrace*C((1-rbrace)^1)*rbrace+C(anything^1) +local pattern=spacing*(method*spacing*apply+Carg(1))*spacing*token +function utilities.parsers.splitmethod(str,default) + if str then + return lpegmatch(pattern,str,1,default or false) + else + return default or false,"" + end +end end -- of closure @@ -6872,7 +7218,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-set"] = package.loaded["trac-set"] or true --- original size: 12365, stripped down to: 8799 +-- original size: 12482, stripped down to: 8864 if not modules then modules={} end modules ['trac-set']={ version=1.001, @@ -6900,7 +7246,7 @@ function setters.initialize(filename,name,values) local data=setter.data if data then for key,newvalue in next,values do - local newvalue=is_boolean(newvalue,newvalue) + local newvalue=is_boolean(newvalue,newvalue,true) local functions=data[key] if functions then local oldvalue=functions.value @@ -6954,7 +7300,7 @@ local function set(t,what,newvalue) elseif not value then value=false else - value=is_boolean(value,value) + value=is_boolean(value,value,true) end w=topattern(w,true,true) for name,functions in next,data do @@ -7093,6 +7439,7 @@ function setters.new(name) report=function(...) setters.report (setter,...) end, enable=function(...) enable (setter,...) end, disable=function(...) disable (setter,...) end, + reset=function(...) reset (setter,...) end, register=function(...) register(setter,...) end, list=function(...) list (setter,...) end, show=function(...) show (setter,...) end, @@ -7184,7 +7531,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-log"] = package.loaded["trac-log"] or true --- original size: 25613, stripped down to: 16617 +-- original size: 29359, stripped down to: 20483 if not modules then modules={} end modules ['trac-log']={ version=1.001, @@ -7193,15 +7540,18 @@ if not modules then modules={} end modules ['trac-log']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } +local next,type,select,print=next,type,select,print local write_nl,write=texio and texio.write_nl or print,texio and texio.write or io.write local format,gmatch,find=string.format,string.gmatch,string.find local concat,insert,remove=table.concat,table.insert,table.remove local topattern=string.topattern -local next,type,select=next,type,select local utfchar=utf.char +local datetime=os.date +local openfile=io.open local setmetatableindex=table.setmetatableindex local formatters=string.formatters local texgetcount=tex and tex.getcount +local variant="default" logs=logs or {} local logs=logs local moreinfo=[[ @@ -7211,32 +7561,122 @@ maillist : ntg-context@ntg.nl / http://www.ntg.nl/mailman/listinfo/ntg-context webpage : http://www.pragma-ade.nl / http://tex.aanhet.net wiki : http://contextgarden.net ]] -utilities.strings.formatters.add ( +formatters.add ( formatters,"unichr", [["U+" .. format("%%05X",%s) .. " (" .. utfchar(%s) .. ")"]] ) -utilities.strings.formatters.add ( +formatters.add ( formatters,"chruni", [[utfchar(%s) .. " (U+" .. format("%%05X",%s) .. ")"]] ) local function ignore() end setmetatableindex(logs,function(t,k) t[k]=ignore;return ignore end) local report,subreport,status,settarget,setformats,settranslations -local direct,subdirect,writer,pushtarget,poptarget,setlogfile,settimedlog,setprocessor,setformatters +local direct,subdirect,writer,pushtarget,poptarget,setlogfile,settimedlog,setprocessor,setformatters,newline if tex and (tex.jobname or tex.formatname) then - local valueiskey={ __index=function(t,k) t[k]=k return k end } - local target="term and log" + local function useluawrites() + local texio_write_nl=texio.write_nl + local texio_write=texio.write + local io_write=io.write + write_nl=function(target,...) + if not io_write then + io_write=io.write + end + if target=="term and log" then + texio_write_nl("log",...) + texio_write_nl("term","") + io_write(...) + elseif target=="log" then + texio_write_nl("log",...) + elseif target=="term" then + texio_write_nl("term","") + io_write(...) + elseif target~="none" then + texio_write_nl("log",target,...) + texio_write_nl("term","") + io_write(target,...) + end + end + write=function(target,...) + if not io_write then + io_write=io.write + end + if target=="term and log" then + texio_write("log",...) + io_write(...) + elseif target=="log" then + texio_write("log",...) + elseif target=="term" then + io_write(...) + elseif target~="none" then + texio_write("log",target,...) + io_write(target,...) + end + end + texio.write=write + texio.write_nl=write_nl + useluawrites=ignore + end + local whereto="both" + local target=nil + local targets=nil + local formats=table.setmetatableindex("self") + local translations=table.setmetatableindex("self") + local report_yes,subreport_yes,direct_yes,subdirect_yes,status_yes + local report_nop,subreport_nop,direct_nop,subdirect_nop,status_nop + local variants={ + default={ + formats={ + report_yes=formatters["%-15s > %s\n"], + report_nop=formatters["%-15s >\n"], + direct_yes=formatters["%-15s > %s"], + direct_nop=formatters["%-15s >"], + subreport_yes=formatters["%-15s > %s > %s\n"], + subreport_nop=formatters["%-15s > %s >\n"], + subdirect_yes=formatters["%-15s > %s > %s"], + subdirect_nop=formatters["%-15s > %s >"], + status_yes=formatters["%-15s : %s\n"], + status_nop=formatters["%-15s :\n"], + }, + targets={ + logfile="log", + log="log", + file="log", + console="term", + terminal="term", + both="term and log", + }, + }, + ansi={ + formats={ + report_yes=formatters["[0;33m%-15s [0;1m>[0m %s\n"], + report_nop=formatters["[0;33m%-15s [0;1m>[0m\n"], + direct_yes=formatters["[0;33m%-15s [0;1m>[0m %s"], + direct_nop=formatters["[0;33m%-15s [0;1m>[0m"], + subreport_yes=formatters["[0;33m%-15s [0;1m>[0;35m %s [0;1m>[0m %s\n"], + subreport_nop=formatters["[0;33m%-15s [0;1m>[0;35m %s [0;1m>[0m\n"], + subdirect_yes=formatters["[0;33m%-15s [0;1m>[0;35m %s [0;1m>[0m %s"], + subdirect_nop=formatters["[0;33m%-15s [0;1m>[0;35m %s [0;1m>[0m"], + status_yes=formatters["[0;33m%-15s [0;1m:[0m %s\n"], + status_nop=formatters["[0;33m%-15s [0;1m:[0m\n"], + }, + targets={ + logfile="none", + log="none", + file="none", + console="term", + terminal="term", + both="term", + }, + } + } logs.flush=io.flush - local formats={} setmetatable(formats,valueiskey) - local translations={} setmetatable(translations,valueiskey) writer=function(...) write_nl(target,...) end newline=function() write_nl(target,"\n") end - local report_yes=formatters["%-15s > %s\n"] - local report_nop=formatters["%-15s >\n"] report=function(a,b,c,...) if c then write_nl(target,report_yes(translations[a],formatters[formats[b]](c,...))) @@ -7248,8 +7688,6 @@ if tex and (tex.jobname or tex.formatname) then write_nl(target,"\n") end end - local direct_yes=formatters["%-15s > %s"] - local direct_nop=formatters["%-15s >"] direct=function(a,b,c,...) if c then return direct_yes(translations[a],formatters[formats[b]](c,...)) @@ -7261,8 +7699,6 @@ if tex and (tex.jobname or tex.formatname) then return "" end end - local subreport_yes=formatters["%-15s > %s > %s\n"] - local subreport_nop=formatters["%-15s > %s >\n"] subreport=function(a,s,b,c,...) if c then write_nl(target,subreport_yes(translations[a],translations[s],formatters[formats[b]](c,...))) @@ -7274,8 +7710,6 @@ if tex and (tex.jobname or tex.formatname) then write_nl(target,"\n") end end - local subdirect_yes=formatters["%-15s > %s > %s"] - local subdirect_nop=formatters["%-15s > %s >"] subdirect=function(a,s,b,c,...) if c then return subdirect_yes(translations[a],translations[s],formatters[formats[b]](c,...)) @@ -7287,8 +7721,6 @@ if tex and (tex.jobname or tex.formatname) then return "" end end - local status_yes=formatters["%-15s : %s\n"] - local status_nop=formatters["%-15s :\n"] status=function(a,b,c,...) if c then write_nl(target,status_yes(translations[a],formatters[formats[b]](c,...))) @@ -7300,16 +7732,13 @@ if tex and (tex.jobname or tex.formatname) then write_nl(target,"\n") end end - local targets={ - logfile="log", - log="log", - file="log", - console="term", - terminal="term", - both="term and log", - } - settarget=function(whereto) - target=targets[whereto or "both"] or targets.both + settarget=function(askedwhereto) + whereto=askedwhereto or whereto or "both" + target=targets[whereto] + if not target then + whereto="both" + target=targets[whereto] + end if target=="term" or target=="term and log" then logs.flush=io.flush else @@ -7338,21 +7767,74 @@ if tex and (tex.jobname or tex.formatname) then writeline(target,f(...)) end end - setformatters=function(f) - report_yes=f.report_yes or report_yes - report_nop=f.report_nop or report_nop - subreport_yes=f.subreport_yes or subreport_yes - subreport_nop=f.subreport_nop or subreport_nop - direct_yes=f.direct_yes or direct_yes - direct_nop=f.direct_nop or direct_nop - subdirect_yes=f.subdirect_yes or subdirect_yes - subdirect_nop=f.subdirect_nop or subdirect_nop - status_yes=f.status_yes or status_yes - status_nop=f.status_nop or status_nop - end + setformatters=function(specification) + local t=nil + local f=nil + local d=variants.default + if not specification then + elseif type(specification)=="table" then + t=specification.targets + f=specification.formats or specification + else + local v=variants[specification] + if v then + t=v.targets + f=v.formats + variant=specification + end + end + targets=t or d.targets + target=targets[whereto] or target + if f then + d=d.formats + else + f=d.formats + d=f + end + setmetatableindex(f,d) + report_yes=f.report_yes + report_nop=f.report_nop + subreport_yes=f.subreport_yes + subreport_nop=f.subreport_nop + direct_yes=f.direct_yes + direct_nop=f.direct_nop + subdirect_yes=f.subdirect_yes + subdirect_nop=f.subdirect_nop + status_yes=f.status_yes + status_nop=f.status_nop + if variant=="ansi" then + useluawrites() + end + settarget(whereto) + end + setformatters(variant) setlogfile=ignore settimedlog=ignore else + local report_yes,subreport_yes,status_yes + local report_nop,subreport_nop,status_nop + local variants={ + default={ + formats={ + report_yes=formatters["%-15s | %s"], + report_nop=formatters["%-15s |"], + subreport_yes=formatters["%-15s | %s | %s"], + subreport_nop=formatters["%-15s | %s |"], + status_yes=formatters["%-15s : %s\n"], + status_nop=formatters["%-15s :\n"], + }, + }, + ansi={ + formats={ + report_yes=formatters["[0;32m%-15s [0;1m|[0m %s"], + report_nop=formatters["[0;32m%-15s [0;1m|[0m"], + subreport_yes=formatters["[0;32m%-15s [0;1m|[0;31m %s [0;1m|[0m %s"], + subreport_nop=formatters["[0;32m%-15s [0;1m|[0;31m %s [0;1m|[0m"], + status_yes=formatters["[0;32m%-15s [0;1m:[0m %s\n"], + status_nop=formatters["[0;32m%-15s [0;1m:[0m\n"], + }, + }, + } logs.flush=ignore writer=function(s) write_nl(s) @@ -7360,8 +7842,6 @@ else newline=function() write_nl("\n") end - local report_yes=formatters["%-15s | %s"] - local report_nop=formatters["%-15s |"] report=function(a,b,c,...) if c then write_nl(report_yes(a,formatters[b](c,...))) @@ -7373,8 +7853,6 @@ else write_nl("") end end - local subreport_yes=formatters["%-15s | %s | %s"] - local subreport_nop=formatters["%-15s | %s |"] subreport=function(a,sub,b,c,...) if c then write_nl(subreport_yes(a,sub,formatters[b](c,...))) @@ -7386,8 +7864,6 @@ else write_nl("") end end - local status_yes=formatters["%-15s : %s\n"] - local status_nop=formatters["%-15s :\n"] status=function(a,b,c,...) if c then write_nl(status_yes(a,formatters[b](c,...))) @@ -7412,14 +7888,34 @@ else writeline(f(s)) end end - setformatters=function(f) - report_yes=f.report_yes or report_yes - report_nop=f.report_nop or report_nop - subreport_yes=f.subreport_yes or subreport_yes - subreport_nop=f.subreport_nop or subreport_nop - status_yes=f.status_yes or status_yes - status_nop=f.status_nop or status_nop - end + setformatters=function(specification) + local f=nil + local d=variants.default + if specification then + if type(specification)=="table" then + f=specification.formats or specification + else + local v=variants[specification] + if v then + f=v.formats + end + end + end + if f then + d=d.formats + else + f=d.formats + d=f + end + setmetatableindex(f,d) + report_yes=f.report_yes + report_nop=f.report_nop + subreport_yes=f.subreport_yes + subreport_nop=f.subreport_nop + status_yes=f.status_yes + status_nop=f.status_nop + end + setformatters(variant) setlogfile=function(name,keepopen) if name and name~="" then local localtime=os.localtime @@ -7672,13 +8168,13 @@ end local simple=logs.reporter("comment") logs.simple=simple logs.simpleline=simple -function logs.setprogram () end -function logs.extendbanner() end -function logs.reportlines () end -function logs.reportbanner() end -function logs.reportline () end -function logs.simplelines () end -function logs.help () end +logs.setprogram=ignore +logs.extendbanner=ignore +logs.reportlines=ignore +logs.reportbanner=ignore +logs.reportline=ignore +logs.simplelines=ignore +logs.help=ignore local Carg,C,lpegmatch=lpeg.Carg,lpeg.C,lpeg.match local p_newline=lpeg.patterns.newline local linewise=( @@ -7755,10 +8251,11 @@ function logs.application(t) end return t end -function logs.system(whereto,process,jobname,category,...) - local message=formatters["%s %s => %s => %s => %s\r"](os.date("%d/%m/%y %H:%m:%S"),process,jobname,category,format(...)) +local f_syslog=formatters["%s %s => %s => %s => %s\r"] +function logs.system(whereto,process,jobname,category,fmt,arg,...) + local message=f_syslog(datetime("%d/%m/%y %H:%m:%S"),process,jobname,category,arg==nil and fmt or format(fmt,arg,...)) for i=1,10 do - local f=io.open(whereto,"a") + local f=openfile(whereto,"a") if f then f:write(message) f:close() @@ -7820,7 +8317,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-inf"] = package.loaded["trac-inf"] or true --- original size: 7011, stripped down to: 5590 +-- original size: 6704, stripped down to: 5343 if not modules then modules={} end modules ['trac-inf']={ version=1.001, @@ -7993,15 +8490,6 @@ function statistics.tracefunction(base,tag,...) statistics.register(formatters["%s.%s"](tag,name),function() return serialize(stat,"calls") end) end end -commands=commands or {} -function commands.resettimer(name) - resettiming(name or "whatever") - starttiming(name or "whatever") -end -function commands.elapsedtime(name) - stoptiming(name or "whatever") - context(elapsedtime(name or "whatever")) -end end -- of closure @@ -8568,7 +9056,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-tpl"] = package.loaded["util-tpl"] or true --- original size: 6251, stripped down to: 3488 +-- original size: 7100, stripped down to: 3978 if not modules then modules={} end modules ['util-tpl']={ version=1.001, @@ -8610,7 +9098,7 @@ local sqlescape=lpeg.replacer { { "\r\n","\\n" }, { "\r","\\n" }, } -local sqlquoted=lpeg.Cs(lpeg.Cc("'")*sqlescape*lpeg.Cc("'")) +local sqlquoted=Cs(Cc("'")*sqlescape*Cc("'")) lpegpatterns.sqlescape=sqlescape lpegpatterns.sqlquoted=sqlquoted local luaescape=lpegpatterns.luaescape @@ -8633,12 +9121,24 @@ local quotedescapers={ local luaescaper=escapers.lua local quotedluaescaper=quotedescapers.lua local function replacekeyunquoted(s,t,how,recurse) - local escaper=how and escapers[how] or luaescaper - return escaper(replacekey(s,t,how,recurse)) + if how==false then + return replacekey(s,t,how,recurse) + else + local escaper=how and escapers[how] or luaescaper + return escaper(replacekey(s,t,how,recurse)) + end end local function replacekeyquoted(s,t,how,recurse) - local escaper=how and quotedescapers[how] or quotedluaescaper - return escaper(replacekey(s,t,how,recurse)) + if how==false then + return replacekey(s,t,how,recurse) + else + local escaper=how and quotedescapers[how] or quotedluaescaper + return escaper(replacekey(s,t,how,recurse)) + end +end +local function replaceoptional(l,m,r,t,how,recurse) + local v=t[l] + return v and v~="" and lpegmatch(replacer,r,1,t,how or "lua",recurse or false) or "" end local single=P("%") local double=P("%%") @@ -8653,11 +9153,16 @@ local nolquoted=lquoted/'' local norquoted=rquoted/'' local nolquotedq=lquotedq/'' local norquotedq=rquotedq/'' -local key=nosingle*((C((1-nosingle )^1)*Carg(1)*Carg(2)*Carg(3))/replacekey )*nosingle -local quoted=nolquotedq*((C((1-norquotedq)^1)*Carg(1)*Carg(2)*Carg(3))/replacekeyquoted )*norquotedq -local unquoted=nolquoted*((C((1-norquoted )^1)*Carg(1)*Carg(2)*Carg(3))/replacekeyunquoted)*norquoted +local noloptional=P("%?")/'' +local noroptional=P("?%")/'' +local nomoptional=P(":")/'' +local args=Carg(1)*Carg(2)*Carg(3) +local key=nosingle*((C((1-nosingle )^1)*args)/replacekey )*nosingle +local quoted=nolquotedq*((C((1-norquotedq )^1)*args)/replacekeyquoted )*norquotedq +local unquoted=nolquoted*((C((1-norquoted )^1)*args)/replacekeyunquoted)*norquoted +local optional=noloptional*((C((1-nomoptional)^1)*nomoptional*C((1-noroptional)^1)*args)/replaceoptional)*noroptional local any=P(1) - replacer=Cs((unquoted+quoted+escape+key+any)^0) + replacer=Cs((unquoted+quoted+escape+optional+key+any)^0) local function replace(str,mapping,how,recurse) if mapping and str then return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str @@ -8696,7 +9201,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-env"] = package.loaded["util-env"] or true --- original size: 8814, stripped down to: 5092 +-- original size: 8022, stripped down to: 5038 if not modules then modules={} end modules ['util-env']={ version=1.001, @@ -8707,7 +9212,7 @@ if not modules then modules={} end modules ['util-env']={ } local allocate,mark=utilities.storage.allocate,utilities.storage.mark local format,sub,match,gsub,find=string.format,string.sub,string.match,string.gsub,string.find -local unquoted,quoted=string.unquoted,string.quoted +local unquoted,quoted,optionalquoted=string.unquoted,string.quoted,string.optionalquoted local concat,insert,remove=table.concat,table.insert,table.remove environment=environment or {} local environment=environment @@ -8820,24 +9325,14 @@ function environment.splitarguments(separator) return before,after end function environment.reconstructcommandline(arg,noquote) + local resolveprefix=resolvers.resolve arg=arg or environment.originalarguments if noquote and #arg==1 then - local a=arg[1] - a=resolvers.resolve(a) - a=unquoted(a) - return a + return unquoted(resolveprefix and resolveprefix(arg[1]) or arg[1]) elseif #arg>0 then local result={} for i=1,#arg do - local a=arg[i] - a=resolvers.resolve(a) - a=unquoted(a) - a=gsub(a,'"','\\"') - if find(a," ",1,true) then - result[#result+1]=quoted(a) - else - result[#result+1]=a - end + result[i]=optionalquoted(resolveprefix and resolveprefix(arg[i]) or resolveprefix) end return concat(result," ") else @@ -9046,7 +9541,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-tab"] = package.loaded["lxml-tab"] or true --- original size: 42614, stripped down to: 26694 +-- original size: 43882, stripped down to: 26870 if not modules then modules={} end modules ['lxml-tab']={ version=1.001, @@ -9061,10 +9556,10 @@ if lpeg.setmaxstack then lpeg.setmaxstack(1000) end xml=xml or {} local xml=xml local concat,remove,insert=table.concat,table.remove,table.insert -local type,next,setmetatable,getmetatable,tonumber=type,next,setmetatable,getmetatable,tonumber +local type,next,setmetatable,getmetatable,tonumber,rawset=type,next,setmetatable,getmetatable,tonumber,rawset local lower,find,match,gsub=string.lower,string.find,string.match,string.gsub local utfchar=utf.char -local lpegmatch=lpeg.match +local lpegmatch,lpegpatterns=lpeg.match,lpeg.patterns local P,S,R,C,V,C,Cs=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.V,lpeg.C,lpeg.Cs local formatters=string.formatters xml.xmlns=xml.xmlns or {} @@ -9226,9 +9721,10 @@ local function fromdec(s) return formatters["d:%s"](s),true end end -local rest=(1-P(";"))^0 -local many=P(1)^0 -local parsedentity=P("&")*(P("#x")*(rest/fromhex)+P("#")*(rest/fromdec))*P(";")*P(-1)+(P("#x")*(many/fromhex)+P("#")*(many/fromdec)) +local p_rest=(1-P(";"))^0 +local p_many=P(1)^0 +local p_char=lpegpatterns.utf8character +local parsedentity=P("&")*(P("#x")*(p_rest/fromhex)+P("#")*(p_rest/fromdec))*P(";")*P(-1)+(P("#x")*(p_many/fromhex)+P("#")*(p_many/fromdec)) local predefined_unified={ [38]="&", [42]=""", @@ -9254,7 +9750,9 @@ local privates_u={ local privates_p={} local privates_n={ } -local escaped=utf.remapper(privates_u) +local escaped=utf.remapper(privates_u,"dynamic") +local unprivatized=utf.remapper(privates_p,"dynamic") +xml.unprivatized=unprivatized local function unescaped(s) local p=privates_n[s] if not p then @@ -9267,9 +9765,7 @@ local function unescaped(s) end return p end -local unprivatized=utf.remapper(privates_p) xml.privatetoken=unescaped -xml.unprivatized=unprivatized xml.privatecodes=privates_n local function handle_hex_entity(str) local h=hcache[str] @@ -9422,7 +9918,7 @@ local valid=R('az','AZ','09')+S('_-.') local name_yes=C(valid^1)*colon*C(valid^1) local name_nop=C(P(true))*C(valid^1) local name=name_yes+name_nop -local utfbom=lpeg.patterns.utfbom +local utfbom=lpegpatterns.utfbom local spacing=C(space^0) local anyentitycontent=(1-open-semicolon-space-close)^0 local hexentitycontent=R("AF","af","09")^0 @@ -9590,7 +10086,7 @@ local function _xmlconvert_(data,settings) reported_attribute_errors,mt,errorhandler=nil,nil,nil return result end -function xmlconvert(data,settings) +local function xmlconvert(data,settings) local ok,result=pcall(function() return _xmlconvert_(data,settings) end) if ok then return result @@ -9682,14 +10178,17 @@ function xml.checkbom(root) insert(dt,2,"\n" ) end end -local function verbose_element(e,handlers) +local f_attribute=formatters['%s=%q'] +local function verbose_element(e,handlers,escape) local handle=handlers.handle local serialize=handlers.serialize local ens,etg,eat,edt,ern=e.ns,e.tg,e.at,e.dt,e.rn local ats=eat and next(eat) and {} if ats then + local n=0 for k,v in next,eat do - ats[#ats+1]=formatters['%s=%q'](k,escaped(v)) + n=n+1 + ats[n]=f_attribute(k,escaped(v)) end end if ern and trace_entities and ern~=ens then @@ -10033,7 +10532,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-lpt"] = package.loaded["lxml-lpt"] or true --- original size: 48030, stripped down to: 30595 +-- original size: 48229, stripped down to: 30684 if not modules then modules={} end modules ['lxml-lpt']={ version=1.001, @@ -10418,8 +10917,8 @@ local lp_builtin=P ( P("ns")/"ll.ns" )*((spaces*P("(")*spaces*P(")"))/"") local lp_attribute=(P("@")+P("attribute::"))/""*Cc("(ll.at and ll.at['")*((R("az","AZ")+S("-_:"))^1)*Cc("'])") -lp_fastpos_p=P("+")^0*R("09")^1*P(-1)/"l==%0" -lp_fastpos_n=P("-")*R("09")^1*P(-1)/"(%0<0 and (#list+%0==l))" +local lp_fastpos_p=P("+")^0*R("09")^1*P(-1)/"l==%0" +local lp_fastpos_n=P("-")*R("09")^1*P(-1)/"(%0<0 and (#list+%0==l))" local lp_fastpos=lp_fastpos_n+lp_fastpos_p local lp_reserved=C("and")+C("or")+C("not")+C("div")+C("mod")+C("true")+C("false") local lp_lua_function=Cs((R("az","AZ","__")^1*(P(".")*R("az","AZ","__")^1)^1)*("("))/"%0" @@ -10598,7 +11097,7 @@ local function nodesettostring(set,nodetest) if not ns or ns=="" then ns="*" end if not tg or tg=="" then tg="*" end tg=(tg=="@rt@" and "[root]") or format("%s:%s",ns,tg) - t[i]=(directive and tg) or format("not(%s)",tg) + t[#t+1]=(directive and tg) or format("not(%s)",tg) end if nodetest==false then return format("not(%s)",concat(t,"|")) @@ -10864,7 +11363,6 @@ expressions.print=function(...) print(...) return true end -expressions.contains=find expressions.find=find expressions.upper=upper expressions.lower=lower @@ -10886,6 +11384,9 @@ function expressions.contains(str,pattern) end return false end +function xml.expressions.idstring(str) + return type(str)=="string" and gsub(str,"^#","") or "" +end local function traverse(root,pattern,handle) local collected=applylpath(root,pattern) if collected then @@ -11173,7 +11674,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-aux"] = package.loaded["lxml-aux"] or true --- original size: 23804, stripped down to: 16817 +-- original size: 28225, stripped down to: 20125 if not modules then modules={} end modules ['lxml-aux']={ version=1.001, @@ -11183,16 +11684,19 @@ if not modules then modules={} end modules ['lxml-aux']={ license="see context related readme files" } local trace_manipulations=false trackers.register("lxml.manipulations",function(v) trace_manipulations=v end) +local trace_inclusions=false trackers.register("lxml.inclusions",function(v) trace_inclusions=v end) local report_xml=logs.reporter("xml") local xml=xml -local xmlconvert,xmlcopy,xmlname=xml.convert,xml.copy,xml.name +local xmlcopy,xmlname=xml.copy,xml.name local xmlinheritedconvert=xml.inheritedconvert local xmlapplylpath=xml.applylpath local xmlfilter=xml.filter -local type,setmetatable,getmetatable=type,setmetatable,getmetatable +local type,next,setmetatable,getmetatable=type,next,setmetatable,getmetatable local insert,remove,fastcopy,concat=table.insert,table.remove,table.fastcopy,table.concat local gmatch,gsub,format,find,strip=string.gmatch,string.gsub,string.format,string.find,string.strip local utfbyte=utf.byte +local lpegmatch=lpeg.match +local striplinepatterns=utilities.strings.striplinepatterns local function report(what,pattern,c,e) report_xml("%s element %a, root %a, position %a, index %a, pattern %a",what,xmlname(e),xmlname(e.__p__),c,e.ni,pattern) end @@ -11247,13 +11751,15 @@ end function xml.each(root,pattern,handle,reverse) local collected=xmlapplylpath(root,pattern) if collected then - if reverse then - for c=#collected,1,-1 do - handle(collected[c]) - end - else - for c=1,#collected do - handle(collected[c]) + if handle then + if reverse then + for c=#collected,1,-1 do + handle(collected[c]) + end + else + for c=1,#collected do + handle(collected[c]) + end end end return collected @@ -11309,6 +11815,7 @@ local function redo_ni(d) end end end +xml.reindex=redo_ni local function xmltoelement(whatever,root) if not whatever then return nil @@ -11360,8 +11867,16 @@ function xml.delete(root,pattern) report('deleting',pattern,c,e) end local d=p.dt - remove(d,e.ni) - redo_ni(d) + local ni=e.ni + if ni<=#d then + if false then + p.dt[ni]="" + else + remove(d,ni) + redo_ni(d) + end + else + end end end end @@ -11481,28 +11996,40 @@ xml.insertafter=insert_element xml.insertbefore=function(r,p,e) insert_element(r,p,e,true) end xml.injectafter=inject_element xml.injectbefore=function(r,p,e) inject_element(r,p,e,true) end -local function include(xmldata,pattern,attribute,recursive,loaddata) +local function include(xmldata,pattern,attribute,recursive,loaddata,level) pattern=pattern or 'include' loaddata=loaddata or io.loaddata local collected=xmlapplylpath(xmldata,pattern) if collected then + if not level then + level=1 + end for c=1,#collected do local ek=collected[c] local name=nil local ekdt=ek.dt local ekat=ek.at - local epdt=ek.__p__.dt + local ekrt=ek.__p__ + local epdt=ekrt.dt if not attribute or attribute=="" then name=(type(ekdt)=="table" and ekdt[1]) or ekdt end if not name then for a in gmatch(attribute or "href","([^|]+)") do name=ekat[a] - if name then break end + if name then + break + end + end + end + local data=nil + if name and name~="" then + data=loaddata(name) or "" + if trace_inclusions then + report_xml("including %s bytes from %a at level %s by pattern %a and attribute %a (%srecursing)",#data,name,level,pattern,attribute or "",recursive and "" or "not ") end end - local data=(name and name~="" and loaddata(name)) or "" - if data=="" then + if not data or data=="" then epdt[ek.ni]="" elseif ekat["parse"]=="text" then epdt[ek.ni]=xml.escaped(data) @@ -11512,70 +12039,113 @@ local function include(xmldata,pattern,attribute,recursive,loaddata) epdt[ek.ni]="" else if recursive then - include(xi,pattern,attribute,recursive,loaddata) + include(xi,pattern,attribute,recursive,loaddata,level+1) + end + local child=xml.body(xi) + child.__p__=ekrt + child.__f__=name + epdt[ek.ni]=child + local inclusions=xmldata.settings.inclusions + if inclusions then + inclusions[#inclusions+1]=name + else + xmldata.settings.inclusions={ name } end - epdt[ek.ni]=xml.body(xi) end end end end end xml.include=include +function xml.inclusion(e,default) + while e do + local f=e.__f__ + if f then + return f + else + e=e.__p__ + end + end + return default +end +function xml.inclusions(e,sorted) + while e do + local settings=e.settings + if settings then + local inclusions=settings.inclusions + if inclusions then + inclusions=table.unique(inclusions) + if sorted then + table.sort(inclusions) + end + return inclusions + else + e=e.__p__ + end + else + e=e.__p__ + end + end +end +local b_collapser=lpeg.patterns.b_collapser +local m_collapser=lpeg.patterns.m_collapser +local e_collapser=lpeg.patterns.e_collapser +local b_stripper=lpeg.patterns.b_stripper +local m_stripper=lpeg.patterns.m_stripper +local e_stripper=lpeg.patterns.e_stripper +local lpegmatch=lpeg.match local function stripelement(e,nolines,anywhere) local edt=e.dt if edt then - if anywhere then - local t,n={},0 - for e=1,#edt do + local n=#edt + if n==0 then + return e + elseif anywhere then + local t={} + local m=0 + for e=1,n do local str=edt[e] if type(str)~="string" then - n=n+1 - t[n]=str + m=m+1 + t[m]=str elseif str~="" then if nolines then - str=gsub(str,"%s+"," ") + str=lpegmatch((n==1 and b_collapser) or (n==m and e_collapser) or m_collapser,str) + else + str=lpegmatch((n==1 and b_stripper) or (n==m and e_stripper) or m_stripper,str) end - str=gsub(str,"^%s*(.-)%s*$","%1") if str~="" then - n=n+1 - t[n]=str + m=m+1 + t[m]=str end end end e.dt=t else - if #edt>0 then - local str=edt[1] - if type(str)~="string" then - elseif str=="" then + local str=edt[1] + if type(str)=="string" then + if str~="" then + str=lpegmatch(nolines and b_collapser or b_stripper,str) + end + if str=="" then remove(edt,1) + n=n-1 else - if nolines then - str=gsub(str,"%s+"," ") - end - str=gsub(str,"^%s+","") - if str=="" then - remove(edt,1) - else - edt[1]=str - end + edt[1]=str end end - local nedt=#edt - if nedt>0 then - local str=edt[nedt] - if type(str)~="string" then - elseif str=="" then - remove(edt) - else - if nolines then - str=gsub(str,"%s+"," ") - end - str=gsub(str,"%s+$","") + if n>0 then + str=edt[n] + if type(str)=="string" then if str=="" then remove(edt) else - edt[nedt]=str + str=lpegmatch(nolines and e_collapser or e_stripper,str) + if str=="" then + remove(edt) + else + edt[n]=str + end end end end @@ -11761,8 +12331,8 @@ function xml.finalizers.xml.cdata(collected) end return "" end -function xml.insertcomment(e,str,n) - table.insert(e.dt,n or 1,{ +function xml.insertcomment(e,str,n) + insert(e.dt,n or 1,{ tg="@cm@", ns="", special=true, @@ -11770,7 +12340,25 @@ function xml.insertcomment(e,str,n) dt={ str }, }) end -function xml.setcdata(e,str) +function xml.insertcdata(e,str,n) + insert(e.dt,n or 1,{ + tg="@cd@", + ns="", + special=true, + at={}, + dt={ str }, + }) +end +function xml.setcomment(e,str,n) + e.dt={ { + tg="@cm@", + ns="", + special=true, + at={}, + dt={ str }, + } } +end +function xml.setcdata(e,str) e.dt={ { tg="@cd@", ns="", @@ -11840,7 +12428,7 @@ local function recurse(e,action) for i=1,#edt do local str=edt[i] if type(str)~="string" then - recurse(str,action,recursive) + recurse(str,action) elseif str~="" then edt[i]=action(str) end @@ -11858,6 +12446,65 @@ function helpers.recursetext(collected,action,recursive) end end end +local specials={ + ["@rt@"]="root", + ["@pi@"]="instruction", + ["@cm@"]="comment", + ["@dt@"]="declaration", + ["@cd@"]="cdata", +} +local function convert(x,strip,flat) + local ns=x.ns + local tg=x.tg + local at=x.at + local dt=x.dt + local node=flat and { + [0]=(not x.special and (ns~="" and ns..":"..tg or tg)) or nil, + } or { + _namespace=ns~="" and ns or nil, + _tag=not x.special and tg or nil, + _type=specials[tg] or "_element", + } + if at then + for k,v in next,at do + node[k]=v + end + end + local n=0 + for i=1,#dt do + local di=dt[i] + if type(di)=="table" then + if flat and di.special then + else + di=convert(di,strip,flat) + if di then + n=n+1 + node[n]=di + end + end + elseif strip then + di=lpegmatch(strip,di) + if di~="" then + n=n+1 + node[n]=di + end + else + n=n+1 + node[n]=di + end + end + if next(node) then + return node + end +end +function xml.totable(x,strip,flat) + if type(x)=="table" then + if strip then + strip=striplinepatterns[strip] + end + return convert(x,strip,flat) + end +end end -- of closure @@ -12414,7 +13061,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-ini"] = package.loaded["data-ini"] or true --- original size: 7927, stripped down to: 5528 +-- original size: 11085, stripped down to: 7662 if not modules then modules={} end modules ['data-ini']={ version=1.001, @@ -12423,14 +13070,15 @@ if not modules then modules={} end modules ['data-ini']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files", } +local next,type,getmetatable,rawset=next,type,getmetatable,rawset local gsub,find,gmatch,char=string.gsub,string.find,string.gmatch,string.char -local next,type=next,type local filedirname,filebasename,filejoin=file.dirname,file.basename,file.join +local ostype,osname,osuname,ossetenv,osgetenv=os.type,os.name,os.uname,os.setenv,os.getenv +local P,S,R,C,Cs,Cc,lpegmatch=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.Cs,lpeg.Cc,lpeg.match local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local trace_detail=false trackers.register("resolvers.details",function(v) trace_detail=v end) local trace_expansions=false trackers.register("resolvers.expansions",function(v) trace_expansions=v end) local report_initialization=logs.reporter("resolvers","initialization") -local ostype,osname,ossetenv,osgetenv=os.type,os.name,os.setenv,os.getenv resolvers=resolvers or {} local resolvers=resolvers texconfig.kpse_init=false @@ -12563,10 +13211,103 @@ if type(profiler)=="table" and not jit then profiler.start("luatex-profile.log") end) end -if not resolvers.resolve then - function resolvers.resolve (s) return s end - function resolvers.unresolve(s) return s end - function resolvers.repath (s) return s end +local prefixes=utilities.storage.allocate() +resolvers.prefixes=prefixes +local resolved={} +local abstract={} +local dynamic={} +function resolvers.resetresolve(str) + resolved,abstract={},{} +end +function resolvers.allprefixes(separator) + local all=table.sortedkeys(prefixes) + if separator then + for i=1,#all do + all[i]=all[i]..":" + end + end + return all +end +local function _resolve_(method,target) + local action=prefixes[method] + if action then + return action(target) + else + return method..":"..target + end +end +function resolvers.unresolve(str) + return abstract[str] or str +end +function resolvers.setdynamic(str) + dynamic[str]=true +end +local pattern=Cs((C(R("az")^2)*P(":")*C((1-S(" \"\';,"))^1)/_resolve_+P(1))^0) +local prefix=C(R("az")^2)*P(":") +local target=C((1-S(" \"\';,"))^1) +local notarget=(#S(";,")+P(-1))*Cc("") +local p_resolve=Cs(((prefix*(target+notarget))/_resolve_+P(1))^0) +local p_simple=prefix*P(-1) +local function resolve(str) + if type(str)=="table" then + local res={} + for i=1,#str do + res[i]=resolve(str[i]) + end + return res + end + local res=resolved[str] + if res then + return res + end + local simple=lpegmatch(p_simple,str) + local action=prefixes[simple] + if action then + local res=action(res) + if not dynamic[simple] then + resolved[simple]=res + abstract[res]=simple + end + return res + end + res=lpegmatch(p_resolve,str) + resolved[str]=res + abstract[res]=str + return res +end +resolvers.resolve=resolve +if type(osuname)=="function" then + for k,v in next,osuname() do + if not prefixes[k] then + prefixes[k]=function() return v end + end + end +end +if ostype=="unix" then + local pattern + local function makepattern(t,k,v) + if t then + rawset(t,k,v) + end + local colon=P(":") + for k,v in table.sortedpairs(prefixes) do + if p then + p=P(k)+p + else + p=P(k) + end + end + pattern=Cs((p*colon+colon/";"+P(1))^0) + end + makepattern() + table.setmetatablenewindex(prefixes,makepattern) + function resolvers.repath(str) + return lpegmatch(pattern,str) + end +else + function resolvers.repath(str) + return str + end end @@ -12576,7 +13317,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-exp"] = package.loaded["data-exp"] or true --- original size: 15317, stripped down to: 9723 +-- original size: 17216, stripped down to: 10657 if not modules then modules={} end modules ['data-exp']={ version=1.001, @@ -12590,12 +13331,16 @@ local concat,sort=table.concat,table.sort local lpegmatch,lpegpatterns=lpeg.match,lpeg.patterns local Ct,Cs,Cc,Carg,P,C,S=lpeg.Ct,lpeg.Cs,lpeg.Cc,lpeg.Carg,lpeg.P,lpeg.C,lpeg.S local type,next=type,next +local isdir=lfs.isdir local ostype=os.type -local collapsepath=file.collapsepath +local collapsepath,joinpath,basename=file.collapsepath,file.join,file.basename local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local trace_expansions=false trackers.register("resolvers.expansions",function(v) trace_expansions=v end) +local trace_globbing=true trackers.register("resolvers.globbing",function(v) trace_globbing=v end) local report_expansions=logs.reporter("resolvers","expansions") +local report_globbing=logs.reporter("resolvers","globbing") local resolvers=resolvers +local resolveprefix=resolvers.resolve local function f_both(a,b) local t,n={},0 for sb in gmatch(b,"[^,]+") do @@ -12685,35 +13430,27 @@ function resolvers.expandedpathfromlist(pathlist) end return newlist end -local cleanup=lpeg.replacer { - { "!","" }, - { "\\","/" }, -} -function resolvers.cleanpath(str) - local doslashes=(P("\\")/"/"+1)^0 - local donegation=(P("!")/"" )^0 - local homedir=lpegmatch(Cs(donegation*doslashes),environment.homedir or "") - if homedir=="~" or homedir=="" or not lfs.isdir(homedir) then - if trace_expansions then - report_expansions("no home dir set, ignoring dependent paths") - end - function resolvers.cleanpath(str) - if not str or find(str,"~",1,true) then - return "" - else - return lpegmatch(cleanup,str) +local usedhomedir=nil +local donegation=(P("!")/"" )^0 +local doslashes=(P("\\")/"/"+1)^0 +local function expandedhome() + if not usedhomedir then + usedhomedir=lpegmatch(Cs(donegation*doslashes),environment.homedir or "") + if usedhomedir=="~" or usedhomedir=="" or not isdir(usedhomedir) then + if trace_expansions then + report_expansions("no home dir set, ignoring dependent path using current path") end - end - else - local dohome=((P("~")+P("$HOME"))/homedir)^0 - local cleanup=Cs(donegation*dohome*doslashes) - function resolvers.cleanpath(str) - return str and lpegmatch(cleanup,str) or "" + usedhomedir="." end end - return resolvers.cleanpath(str) + return usedhomedir +end +local dohome=((P("~")+P("$HOME")+P("%HOME%"))/expandedhome)^0 +local cleanup=Cs(donegation*dohome*doslashes) +resolvers.cleanpath=function(str) + return str and lpegmatch(cleanup,str) or "" end -local expandhome=P("~")/"$HOME" +local expandhome=P("~")/"$HOME" local dodouble=P('"')/""*(expandhome+(1-P('"')))^0*P('"')/"" local dosingle=P("'")/""*(expandhome+(1-P("'")))^0*P("'")/"" local dostring=(expandhome+1 )^0 @@ -12765,46 +13502,67 @@ function resolvers.splitpath(str) end function resolvers.joinpath(str) if type(str)=='table' then - return file.joinpath(str) + return joinpath(str) else return str end end local attributes,directory=lfs.attributes,lfs.dir local weird=P(".")^1+lpeg.anywhere(S("~`!#$%^&*()={}[]:;\"\'||<>,?\n\r\t")) +local lessweird=P(".")^1+lpeg.anywhere(S("~`#$%^&*:;\"\'||<>,?\n\r\t")) local timer={} local scanned={} local nofscans=0 local scancache={} -local function scan(files,spec,path,n,m,r) - local full=(path=="" and spec) or (spec..path..'/') +local fullcache={} +local nofsharedscans=0 +local function scan(files,remap,spec,path,n,m,r,onlyone,tolerant) + local full=path=="" and spec or (spec..path..'/') local dirs={} local nofdirs=0 + local pattern=tolerant and lessweird or weird for name in directory(full) do - if not lpegmatch(weird,name) then - local mode=attributes(full..name,'mode') - if mode=='file' then + if not lpegmatch(pattern,name) then + local mode=attributes(full..name,"mode") + if mode=="file" then n=n+1 - local f=files[name] - if f then - if type(f)=='string' then - files[name]={ f,path } + local lower=lower(name) + local paths=files[lower] + if paths then + if onlyone then else - f[#f+1]=path + if type(paths)=="string" then + files[lower]={ paths,path } + else + paths[#paths+1]=path + end + if name~=lower then + local rl=remap[lower] + if not rl then + remap[lower]=name + r=r+1 + elseif trace_globbing and rl~=name then + report_globbing("confusing filename, name: %a, lower: %a, already: %a",name,lower,rl) + end + end end else - files[name]=path - local lower=lower(name) + files[lower]=path if name~=lower then - files["remap:"..lower]=name - r=r+1 + local rl=remap[lower] + if not rl then + remap[lower]=name + r=r+1 + elseif trace_globbing and rl~=name then + report_globbing("confusing filename, name: %a, lower: %a, already: %a",name,lower,rl) + end end end - elseif mode=='directory' then + elseif mode=="directory" then m=m+1 nofdirs=nofdirs+1 if path~="" then - dirs[nofdirs]=path..'/'..name + dirs[nofdirs]=path.."/"..name else dirs[nofdirs]=name end @@ -12814,107 +13572,69 @@ local function scan(files,spec,path,n,m,r) if nofdirs>0 then sort(dirs) for i=1,nofdirs do - files,n,m,r=scan(files,spec,dirs[i],n,m,r) + files,remap,n,m,r=scan(files,remap,spec,dirs[i],n,m,r,onlyonce,tolerant) end end scancache[sub(full,1,-2)]=files - return files,n,m,r + return files,remap,n,m,r end -local fullcache={} -function resolvers.scanfiles(path,branch,usecache) - statistics.starttiming(timer) - local realpath=resolvers.resolve(path) +function resolvers.scanfiles(path,branch,usecache,onlyonce,tolerant) + local realpath=resolveprefix(path) if usecache then - local files=fullcache[realpath] - if files then + local content=fullcache[realpath] + if content then if trace_locating then - report_expansions("using caches scan of path %a, branch %a",path,branch or path) + report_expansions("using cached scan of path %a, branch %a",path,branch or path) end - return files + nofsharedscans=nofsharedscans+1 + return content end end + statistics.starttiming(timer) if trace_locating then report_expansions("scanning path %a, branch %a",path,branch or path) end - local files,n,m,r=scan({},realpath..'/',"",0,0,0) - files.__path__=path - files.__files__=n - files.__directories__=m - files.__remappings__=r - if trace_locating then - report_expansions("%s files found on %s directories with %s uppercase remappings",n,m,r) - end - if usecache then - scanned[#scanned+1]=realpath - fullcache[realpath]=files - end - nofscans=nofscans+1 - statistics.stoptiming(timer) - return files -end -local function simplescan(files,spec,path) - local full=(path=="" and spec) or (spec..path..'/') - local dirs={} - local nofdirs=0 - for name in directory(full) do - if not lpegmatch(weird,name) then - local mode=attributes(full..name,'mode') - if mode=='file' then - if not files[name] then - files[name]=path - end - elseif mode=='directory' then - nofdirs=nofdirs+1 - if path~="" then - dirs[nofdirs]=path..'/'..name - else - dirs[nofdirs]=name - end - end - end - end - if nofdirs>0 then - sort(dirs) - for i=1,nofdirs do - files=simplescan(files,spec,dirs[i]) - end - end - return files -end -local simplecache={} -local nofsharedscans=0 -function resolvers.simplescanfiles(path,branch,usecache) - statistics.starttiming(timer) - local realpath=resolvers.resolve(path) - if usecache then - local files=simplecache[realpath] - if not files then - files=scancache[realpath] - if files then - nofsharedscans=nofsharedscans+1 - end + local content + if isdir(realpath) then + local files,remap,n,m,r=scan({},{},realpath..'/',"",0,0,0,onlyonce,tolerant) + content={ + metadata={ + path=path, + files=n, + directories=m, + remappings=r, + }, + files=files, + remap=remap, + } + if trace_locating then + report_expansions("%s files found on %s directories with %s uppercase remappings",n,m,r) end - if files then - if trace_locating then - report_expansions("using caches scan of path %a, branch %a",path,branch or path) - end - return files + else + content={ + metadata={ + path=path, + files=0, + directories=0, + remappings=0, + }, + files={}, + remap={}, + } + if trace_locating then + report_expansions("invalid path %a",realpath) end end - if trace_locating then - report_expansions("scanning path %a, branch %a",path,branch or path) - end - local files=simplescan({},realpath..'/',"") - if trace_locating then - report_expansions("%s files found",table.count(files)) - end if usecache then scanned[#scanned+1]=realpath - simplecache[realpath]=files + fullcache[realpath]=content end nofscans=nofscans+1 statistics.stoptiming(timer) - return files + return content +end +function resolvers.simplescanfiles(path,branch,usecache) + return resolvers.scanfiles(path,branch,usecache,true,true) end function resolvers.scandata() table.sort(scanned) @@ -12925,6 +13645,52 @@ function resolvers.scandata() paths=scanned, } end +function resolvers.get_from_content(content,path,name) + if not content then + return + end + local files=content.files + if not files then + return + end + local remap=content.remap + if not remap then + return + end + if name then + local used=lower(name) + return path,remap[used] or used + else + local name=path + local used=lower(name) + local path=files[used] + if path then + return path,remap[used] or used + end + end +end +local nothing=function() end +function resolvers.filtered_from_content(content,pattern) + if content and type(pattern)=="string" then + local pattern=lower(pattern) + local files=content.files + local remap=content.remap + if files and remap then + local n=next(files) + local function iterator() + while n do + local k=n + n=next(files,k) + if find(k,pattern) then + return files[k],remap and remap[k] or k + end + end + end + return iterator + end + end + return nothing +end end -- of closure @@ -12933,7 +13699,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-env"] = package.loaded["data-env"] or true --- original size: 8769, stripped down to: 6490 +-- original size: 9216, stripped down to: 6798 if not modules then modules={} end modules ['data-env']={ version=1.001, @@ -12951,10 +13717,12 @@ local formats=allocate() local suffixes=allocate() local dangerous=allocate() local suffixmap=allocate() +local usertypes=allocate() resolvers.formats=formats resolvers.suffixes=suffixes resolvers.dangerous=dangerous resolvers.suffixmap=suffixmap +resolvers.usertypes=usertypes local luasuffixes=utilities.lua.suffixes local relations=allocate { core={ @@ -13022,11 +13790,13 @@ local relations=allocate { names={ "mp" }, variable='MPINPUTS', suffixes={ 'mp','mpvi','mpiv','mpii' }, + usertype=true, }, tex={ names={ "tex" }, variable='TEXINPUTS', - suffixes={ 'tex',"mkvi","mkiv","mkii" }, + suffixes={ "tex","mkvi","mkiv","mkii","cld","lfg","xml" }, + usertype=true, }, icc={ names={ "icc","icc profile","icc profiles" }, @@ -13042,6 +13812,7 @@ local relations=allocate { names={ "lua" }, variable='LUAINPUTS', suffixes={ luasuffixes.lua,luasuffixes.luc,luasuffixes.tma,luasuffixes.tmc }, + usertype=true, }, lib={ names={ "lib" }, @@ -13050,11 +13821,15 @@ local relations=allocate { }, bib={ names={ 'bib' }, + variable='BIBINPUTS', suffixes={ 'bib' }, + usertype=true, }, bst={ names={ 'bst' }, + variable='BSTINPUTS', suffixes={ 'bst' }, + usertype=true, }, fontconfig={ names={ 'fontconfig','fontconfig file','fontconfig files' }, @@ -13136,8 +13911,9 @@ function resolvers.updaterelations() for name,relation in next,categories do local rn=relation.names local rv=relation.variable - local rs=relation.suffixes if rn and rv then + local rs=relation.suffixes + local ru=relation.usertype for i=1,#rn do local rni=lower(gsub(rn[i]," ","")) formats[rni]=rv @@ -13149,8 +13925,9 @@ function resolvers.updaterelations() end end end - end - if rs then + if ru then + usertypes[name]=true + end end end end @@ -13201,7 +13978,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tmp"] = package.loaded["data-tmp"] or true --- original size: 15532, stripped down to: 11648 +-- original size: 15681, stripped down to: 11761 if not modules then modules={} end modules ['data-tmp']={ version=1.100, @@ -13220,6 +13997,7 @@ local trace_cache=false trackers.register("resolvers.cache",function(v) trace_ca local report_caches=logs.reporter("resolvers","caches") local report_resolvers=logs.reporter("resolvers","caching") local resolvers=resolvers +local cleanpath=resolvers.cleanpath local directive_cleanup=false directives.register("system.compile.cleanup",function(v) directive_cleanup=v end) local directive_strip=false directives.register("system.compile.strip",function(v) directive_strip=v end) local compile=utilities.lua.compile @@ -13241,7 +14019,7 @@ caches.relocate=false caches.defaults={ "TMPDIR","TEMPDIR","TMP","TEMP","HOME","HOMEPATH" } local writable,readables,usedreadables=nil,{},{} local function identify() - local texmfcaches=resolvers.cleanpathlist("TEXMFCACHE") + local texmfcaches=resolvers.cleanpathlist("TEXMFCACHE") if texmfcaches then for k=1,#texmfcaches do local cachepath=texmfcaches[k] @@ -13495,10 +14273,12 @@ local content_state={} function caches.contentstate() return content_state or {} end -function caches.loadcontent(cachename,dataname) - local name=caches.hashed(cachename) - local full,path=caches.getfirstreadablefile(addsuffix(name,luasuffixes.lua),"trees") - local filename=file.join(path,name) +function caches.loadcontent(cachename,dataname,filename) + if not filename then + local name=caches.hashed(cachename) + local full,path=caches.getfirstreadablefile(addsuffix(name,luasuffixes.lua),"trees") + filename=file.join(path,name) + end local blob=loadfile(addsuffix(filename,luasuffixes.luc)) or loadfile(addsuffix(filename,luasuffixes.lua)) if blob then local data=blob() @@ -13530,10 +14310,12 @@ function caches.collapsecontent(content) end end end -function caches.savecontent(cachename,dataname,content) - local name=caches.hashed(cachename) - local full,path=caches.setfirstwritablefile(addsuffix(name,luasuffixes.lua),"trees") - local filename=file.join(path,name) +function caches.savecontent(cachename,dataname,content,filename) + if not filename then + local name=caches.hashed(cachename) + local full,path=caches.setfirstwritablefile(addsuffix(name,luasuffixes.lua),"trees") + filename=file.join(path,name) + end local luaname=addsuffix(filename,luasuffixes.lua) local lucname=addsuffix(filename,luasuffixes.luc) if trace_locating then @@ -13576,7 +14358,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-met"] = package.loaded["data-met"] or true --- original size: 5460, stripped down to: 4014 +-- original size: 5347, stripped down to: 4015 if not modules then modules={} end modules ['data-met']={ version=1.100, @@ -13604,7 +14386,7 @@ local function splitmethod(filename) if type(filename)=="table" then return filename end - filename=file.collapsepath(filename,".") + filename=file.collapsepath(filename,".") if not find(filename,"://",1,true) then return { scheme="file",path=filename,original=filename,filename=filename } end @@ -13695,7 +14477,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-res"] = package.loaded["data-res"] or true --- original size: 62045, stripped down to: 43116 +-- original size: 67003, stripped down to: 46291 if not modules then modules={} end modules ['data-res']={ version=1.001, @@ -13705,7 +14487,7 @@ if not modules then modules={} end modules ['data-res']={ license="see context related readme files", } local gsub,find,lower,upper,match,gmatch=string.gsub,string.find,string.lower,string.upper,string.match,string.gmatch -local concat,insert,sortedkeys=table.concat,table.insert,table.sortedkeys +local concat,insert,remove,sortedkeys,sortedhash=table.concat,table.insert,table.remove,table.sortedkeys,table.sortedhash local next,type,rawget=next,type,rawget local os=os local P,S,R,C,Cc,Cs,Ct,Carg=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.Cc,lpeg.Cs,lpeg.Ct,lpeg.Carg @@ -13714,17 +14496,23 @@ local formatters=string.formatters local filedirname=file.dirname local filebasename=file.basename local suffixonly=file.suffixonly +local addsuffix=file.addsuffix +local removesuffix=file.removesuffix local filejoin=file.join local collapsepath=file.collapsepath local joinpath=file.joinpath +local is_qualified_path=file.is_qualified_path local allocate=utilities.storage.allocate local settings_to_array=utilities.parsers.settings_to_array +local getcurrentdir=lfs.currentdir +local isfile=lfs.isfile +local isdir=lfs.isdir local setmetatableindex=table.setmetatableindex local luasuffixes=utilities.lua.suffixes -local getcurrentdir=lfs.currentdir local trace_locating=false trackers .register("resolvers.locating",function(v) trace_locating=v end) local trace_detail=false trackers .register("resolvers.details",function(v) trace_detail=v end) local trace_expansions=false trackers .register("resolvers.expansions",function(v) trace_expansions=v end) +local trace_paths=false trackers .register("resolvers.paths",function(v) trace_paths=v end) local resolve_otherwise=true directives.register("resolvers.otherwise",function(v) resolve_otherwise=v end) local report_resolving=logs.reporter("resolvers","resolving") local resolvers=resolvers @@ -13732,10 +14520,14 @@ local expandedpathfromlist=resolvers.expandedpathfromlist local checkedvariable=resolvers.checkedvariable local splitconfigurationpath=resolvers.splitconfigurationpath local methodhandler=resolvers.methodhandler +local filtered=resolvers.filtered_from_content +local lookup=resolvers.get_from_content +local cleanpath=resolvers.cleanpath +local resolveprefix=resolvers.resolve local initializesetter=utilities.setters.initialize local ostype,osname,osenv,ossetenv,osgetenv=os.type,os.name,os.env,os.setenv,os.getenv -resolvers.cacheversion='1.0.1' -resolvers.configbanner='' +resolvers.cacheversion="1.100" +resolvers.configbanner="" resolvers.homedir=environment.homedir resolvers.criticalvars=allocate { "SELFAUTOLOC","SELFAUTODIR","SELFAUTOPARENT","TEXMFCNF","TEXMF","TEXOS" } resolvers.luacnfname="texmfcnf.lua" @@ -13754,6 +14546,7 @@ end local unset_variable="unset" local formats=resolvers.formats local suffixes=resolvers.suffixes +local usertypes=resolvers.usertypes local dangerous=resolvers.dangerous local suffixmap=resolvers.suffixmap resolvers.defaultsuffixes={ "tex" } @@ -13762,7 +14555,7 @@ local instance=resolvers.instance or nil function resolvers.setenv(key,value,raw) if instance then instance.environment[key]=value - ossetenv(key,raw and value or resolvers.resolve(value)) + ossetenv(key,raw and value or resolveprefix(value)) end end local function getenv(key) @@ -13776,7 +14569,7 @@ local function getenv(key) end resolvers.getenv=getenv resolvers.env=getenv -local function resolve(k) +local function resolvevariable(k) return instance.expansions[k] end local dollarstripper=lpeg.stripper("$") @@ -13785,19 +14578,19 @@ local backslashswapper=lpeg.replacer("\\","/") local somevariable=P("$")/"" local somekey=C(R("az","AZ","09","__","--")^1) local somethingelse=P(";")*((1-S("!{}/\\"))^1*P(";")/"")+P(";")*(P(";")/"")+P(1) -local variableexpander=Cs((somevariable*(somekey/resolve)+somethingelse)^1 ) +local variableexpander=Cs((somevariable*(somekey/resolvevariable)+somethingelse)^1 ) local cleaner=P("\\")/"/"+P(";")*S("!{}/\\")^0*P(";")^1/";" local variablecleaner=Cs((cleaner+P(1))^0) -local somevariable=R("az","AZ","09","__","--")^1/resolve +local somevariable=R("az","AZ","09","__","--")^1/resolvevariable local variable=(P("$")/"")*(somevariable+(P("{")/"")*somevariable*(P("}")/"")) local variableresolver=Cs((variable+P(1))^0) local function expandedvariable(var) return lpegmatch(variableexpander,var) or var end -function resolvers.newinstance() - if trace_locating then +function resolvers.newinstance() + if trace_locating then report_resolving("creating instance") - end + end local environment,variables,expansions,order=allocate(),allocate(),allocate(),allocate() local newinstance={ environment=environment, @@ -13810,6 +14603,7 @@ function resolvers.newinstance() foundintrees=allocate(), hashes=allocate(), hashed=allocate(), + pathlists=false, specification=allocate(), lists=allocate(), data=allocate(), @@ -13822,6 +14616,7 @@ function resolvers.newinstance() savelists=true, pattern=nil, force_suffixes=true, + pathstack={}, } setmetatableindex(variables,function(t,k) local v @@ -13871,8 +14666,13 @@ function resolvers.reset() end local function reset_hashes() instance.lists={} + instance.pathlists=false instance.found={} end +local function reset_caches() + instance.lists={} + instance.pathlists=false +end local slash=P("/") local pathexpressionpattern=Cs ( Cc("^")*( @@ -13924,13 +14724,13 @@ local function identify_configuration_files() for i=1,#cnfpaths do local filepath=cnfpaths[i] local filename=collapsepath(filejoin(filepath,luacnfname)) - local realname=resolvers.resolve(filename) + local realname=resolveprefix(filename) if trace_locating then - local fullpath=gsub(resolvers.resolve(collapsepath(filepath)),"//","/") + local fullpath=gsub(resolveprefix(collapsepath(filepath)),"//","/") local weirdpath=find(fullpath,"/texmf.+/texmf") or not find(fullpath,"/web2c",1,true) report_resolving("looking for %a on %s path %a from specification %a",luacnfname,weirdpath and "weird" or "given",fullpath,filepath) end - if lfs.isfile(realname) then + if isfile(realname) then specification[#specification+1]=filename if trace_locating then report_resolving("found configuration file %a",realname) @@ -13952,7 +14752,7 @@ local function load_configuration_files() local filename=specification[i] local pathname=filedirname(filename) local filename=filejoin(pathname,luacnfname) - local realname=resolvers.resolve(filename) + local realname=resolveprefix(filename) local blob=loadfile(realname) if blob then local setups=instance.setups @@ -13960,7 +14760,7 @@ local function load_configuration_files() local parent=data and data.parent if parent then local filename=filejoin(pathname,parent) - local realname=resolvers.resolve(filename) + local realname=resolveprefix(filename) local blob=loadfile(realname) if blob then local parentdata=blob() @@ -13985,7 +14785,7 @@ local function load_configuration_files() elseif variables[k]==nil then if trace_locating and not warning then report_resolving("variables like %a in configuration file %a should move to the 'variables' subtable", - k,resolvers.resolve(filename)) + k,resolveprefix(filename)) warning=true end variables[k]=v @@ -14045,7 +14845,7 @@ local function locate_file_databases() local stripped=lpegmatch(inhibitstripper,path) if stripped~="" then local runtime=stripped==path - path=resolvers.cleanpath(path) + path=cleanpath(path) local spec=resolvers.splitmethod(stripped) if runtime and (spec.noscheme or spec.scheme=="file") then stripped="tree:///"..stripped @@ -14108,8 +14908,8 @@ function resolvers.renew(hashname) report_resolving("identifying tree %a",hashname) end end - local realpath=resolvers.resolve(hashname) - if lfs.isdir(realpath) then + local realpath=resolveprefix(hashname) + if isdir(realpath) then if trace_locating then report_resolving("using path %a",realpath) end @@ -14210,19 +15010,53 @@ end function resolvers.unexpandedpath(str) return joinpath(resolvers.unexpandedpathlist(str)) end +function resolvers.pushpath(name) + local pathstack=instance.pathstack + local lastpath=pathstack[#pathstack] + local pluspath=filedirname(name) + if lastpath then + lastpath=collapsepath(filejoin(lastpath,pluspath)) + else + lastpath=collapsepath(pluspath) + end + insert(pathstack,lastpath) + if trace_paths then + report_resolving("pushing path %a",lastpath) + end +end +function resolvers.poppath() + local pathstack=instance.pathstack + if trace_paths and #pathstack>0 then + report_resolving("popping path %a",pathstack[#pathstack]) + end + remove(pathstack) +end +function resolvers.stackpath() + local pathstack=instance.pathstack + local currentpath=pathstack[#pathstack] + return currentpath~="" and currentpath or nil +end local done={} function resolvers.resetextrapath() local ep=instance.extra_paths if not ep then - ep,done={},{} - instance.extra_paths=ep + done={} + instance.extra_paths={} elseif #ep>0 then - instance.lists,done={},{} + done={} + reset_caches() end end function resolvers.registerextrapath(paths,subpaths) - paths=settings_to_array(paths) - subpaths=settings_to_array(subpaths) + if not subpaths or subpaths=="" then + if not paths or path=="" then + return + elseif done[paths] then + return + end + end + local paths=settings_to_array(paths) + local subpaths=settings_to_array(subpaths) local ep=instance.extra_paths or {} local oldn=#ep local newn=oldn @@ -14237,7 +15071,7 @@ function resolvers.registerextrapath(paths,subpaths) local ps=p.."/"..s if not done[ps] then newn=newn+1 - ep[newn]=resolvers.cleanpath(ps) + ep[newn]=cleanpath(ps) done[ps]=true end end @@ -14247,7 +15081,7 @@ function resolvers.registerextrapath(paths,subpaths) local p=paths[i] if not done[p] then newn=newn+1 - ep[newn]=resolvers.cleanpath(p) + ep[newn]=cleanpath(p) done[p]=true end end @@ -14259,7 +15093,7 @@ function resolvers.registerextrapath(paths,subpaths) local ps=ep[i].."/"..s if not done[ps] then newn=newn+1 - ep[newn]=resolvers.cleanpath(ps) + ep[newn]=cleanpath(ps) done[ps]=true end end @@ -14268,52 +15102,70 @@ function resolvers.registerextrapath(paths,subpaths) if newn>0 then instance.extra_paths=ep end - if newn>oldn then - instance.lists={} + if newn~=oldn then + reset_caches() end end -local function made_list(instance,list) - local ep=instance.extra_paths - if not ep or #ep==0 then - return list +function resolvers.pushextrapath(path) + local paths=settings_to_array(path) + if instance.extra_stack then + insert(instance.extra_stack,1,paths) else - local done,new,newn={},{},0 - for k=1,#list do - local v=list[k] - if not done[v] then - if find(v,"^[%.%/]$") then - done[v]=true - newn=newn+1 - new[newn]=v - else - break - end - end - end - for k=1,#ep do - local v=ep[k] + instance.extra_stack={ paths } + end + reset_caches() +end +function resolvers.popextrapath() + if instance.extra_stack then + reset_caches() + return remove(instance.extra_stack,1) + end +end +local function made_list(instance,list,extra_too) + local done={} + local new={} + local newn=0 + local function add(p) + for k=1,#p do + local v=p[k] if not done[v] then done[v]=true newn=newn+1 new[newn]=v end end - for k=1,#list do - local v=list[k] - if not done[v] then - done[v]=true - newn=newn+1 - new[newn]=v + end + for k=1,#list do + local v=list[k] + if done[v] then + elseif find(v,"^[%.%/]$") then + done[v]=true + newn=newn+1 + new[newn]=v + else + break + end + end + if extra_too then + local es=instance.extra_stack + if es and #es>0 then + for k=1,#es do + add(es[k]) end end - return new + local ep=instance.extra_paths + if ep and #ep>0 then + add(ep) + end end + add(list) + return new end function resolvers.cleanpathlist(str) local t=resolvers.expandedpathlist(str) if t then for i=1,#t do - t[i]=collapsepath(resolvers.cleanpath(t[i])) + t[i]=collapsepath(cleanpath(t[i])) end end return t @@ -14321,22 +15173,22 @@ end function resolvers.expandpath(str) return joinpath(resolvers.expandedpathlist(str)) end -function resolvers.expandedpathlist(str) +function resolvers.expandedpathlist(str,extra_too) if not str then return {} - elseif instance.savelists then + elseif instance.savelists then str=lpegmatch(dollarstripper,str) local lists=instance.lists local lst=lists[str] if not lst then - local l=made_list(instance,resolvers.splitpath(resolvers.expansion(str))) + local l=made_list(instance,resolvers.splitpath(resolvers.expansion(str)),extra_too) lst=expandedpathfromlist(l) lists[str]=lst end return lst else local lst=resolvers.splitpath(resolvers.expansion(str)) - return made_list(instance,expandedpathfromlist(lst)) + return made_list(instance,expandedpathfromlist(lst),extra_too) end end function resolvers.expandedpathlistfromvariable(str) @@ -14347,6 +15199,13 @@ end function resolvers.expandpathfromvariable(str) return joinpath(resolvers.expandedpathlistfromvariable(str)) end +function resolvers.cleanedpathlist(v) + local t=resolvers.expandedpathlist(v) + for i=1,#t do + t[i]=resolvers.resolve(resolvers.cleanpath(t[i])) + end + return t +end function resolvers.expandbraces(str) local ori=str local pth=expandedpathfromlist(resolvers.splitpath(ori)) @@ -14363,7 +15222,7 @@ function resolvers.registerfilehash(name,content,someerror) end end local function isreadable(name) - local readable=lfs.isfile(name) + local readable=isfile(name) if trace_detail then if readable then report_resolving("file %a is readable",name) @@ -14373,70 +15232,57 @@ local function isreadable(name) end return readable end -local function collect_files(names) - local filelist,noffiles={},0 +local function collect_files(names) + local filelist={} + local noffiles=0 + local function check(hash,root,pathname,path,name) + if not pathname or find(path,pathname) then + local variant=hash.type + local search=filejoin(root,path,name) + local result=methodhandler('concatinators',variant,root,path,name) + if trace_detail then + report_resolving("match: variant %a, search %a, result %a",variant,search,result) + end + noffiles=noffiles+1 + filelist[noffiles]={ variant,search,result } + end + end for k=1,#names do - local fname=names[k] + local filename=names[k] if trace_detail then - report_resolving("checking name %a",fname) + report_resolving("checking name %a",filename) end - local bname=filebasename(fname) - local dname=filedirname(fname) - if dname=="" or find(dname,"^%.") then - dname=false + local basename=filebasename(filename) + local pathname=filedirname(filename) + if pathname=="" or find(pathname,"^%.") then + pathname=false else - dname=gsub(dname,"%*",".*") - dname="/"..dname.."$" + pathname=gsub(pathname,"%*",".*") + pathname="/"..pathname.."$" end local hashes=instance.hashes for h=1,#hashes do local hash=hashes[h] - local blobpath=hash.name - local files=blobpath and instance.files[blobpath] - if files then + local hashname=hash.name + local content=hashname and instance.files[hashname] + if content then if trace_detail then - report_resolving("deep checking %a, base %a, pattern %a",blobpath,bname,dname) - end - local blobfile=files[bname] - if not blobfile then - local rname="remap:"..bname - blobfile=files[rname] - if blobfile then - bname=files[rname] - blobfile=files[bname] - end + report_resolving("deep checking %a, base %a, pattern %a",hashname,basename,pathname) end - if blobfile then - local blobroot=files.__path__ or blobpath - if type(blobfile)=='string' then - if not dname or find(blobfile,dname) then - local variant=hash.type - local search=filejoin(blobroot,blobfile,bname) - local result=methodhandler('concatinators',hash.type,blobroot,blobfile,bname) - if trace_detail then - report_resolving("match: variant %a, search %a, result %a",variant,search,result) - end - noffiles=noffiles+1 - filelist[noffiles]={ variant,search,result } - end + local path,name=lookup(content,basename) + if path then + local metadata=content.metadata + local realroot=metadata and metadata.path or hashname + if type(path)=="string" then + check(hash,realroot,pathname,path,name) else - for kk=1,#blobfile do - local vv=blobfile[kk] - if not dname or find(vv,dname) then - local variant=hash.type - local search=filejoin(blobroot,vv,bname) - local result=methodhandler('concatinators',hash.type,blobroot,vv,bname) - if trace_detail then - report_resolving("match: variant %a, search %a, result %a",variant,search,result) - end - noffiles=noffiles+1 - filelist[noffiles]={ variant,search,result } - end + for i=1,#path do + check(hash,realroot,pathname,path[i],name) end end end elseif trace_locating then - report_resolving("no match in %a (%s)",blobpath,bname) + report_resolving("no match in %a (%s)",hashname,basename) end end end @@ -14461,7 +15307,7 @@ end local function can_be_dir(name) local fakepaths=instance.fakepaths if not fakepaths[name] then - if lfs.isdir(name) then + if isdir(name) then fakepaths[name]=1 else fakepaths[name]=2 @@ -14477,10 +15323,11 @@ local function find_analyze(filename,askedformat,allresults) if askedformat=="" then if ext=="" or not suffixmap[ext] then local defaultsuffixes=resolvers.defaultsuffixes + local formatofsuffix=resolvers.formatofsuffix for i=1,#defaultsuffixes do local forcedname=filename..'.'..defaultsuffixes[i] wantedfiles[#wantedfiles+1]=forcedname - filetype=resolvers.formatofsuffix(forcedname) + filetype=formatofsuffix(forcedname) if trace_locating then report_resolving("forcing filetype %a",filetype) end @@ -14520,14 +15367,14 @@ local function find_wildcard(filename,allresults) if trace_locating then report_resolving("checking wildcard %a",filename) end - local method,result=resolvers.findwildcardfiles(filename) + local result=resolvers.findwildcardfiles(filename) if result then return "wildcard",result end end end local function find_qualified(filename,allresults,askedformat,alsostripped) - if not file.is_qualified_path(filename) then + if not is_qualified_path(filename) then return end if trace_locating then @@ -14601,33 +15448,66 @@ local function check_subpath(fname) return fname end end -local function find_intree(filename,filetype,wantedfiles,allresults) +local function makepathlist(list,filetype) local typespec=resolvers.variableofformat(filetype) - local pathlist=resolvers.expandedpathlist(typespec) - local method="intree" + local pathlist=resolvers.expandedpathlist(typespec,filetype and usertypes[filetype]) + local entry={} if pathlist and #pathlist>0 then - local filelist=collect_files(wantedfiles) + for k=1,#pathlist do + local path=pathlist[k] + local prescanned=find(path,'^!!') + local resursive=find(path,'//$') + local pathname=lpegmatch(inhibitstripper,path) + local expression=makepathexpression(pathname) + local barename=gsub(pathname,"/+$","") + barename=resolveprefix(barename) + local scheme=url.hasscheme(barename) + local schemename=gsub(barename,"%.%*$",'') + entry[k]={ + path=path, + pathname=pathname, + prescanned=prescanned, + recursive=recursive, + expression=expression, + barename=barename, + scheme=scheme, + schemename=schemename, + } + end + entry.typespec=typespec + list[filetype]=entry + else + list[filetype]=false + end + return entry +end +local function find_intree(filename,filetype,wantedfiles,allresults) + local pathlists=instance.pathlists + if not pathlists then + pathlists=setmetatableindex(allocate(),makepathlist) + instance.pathlists=pathlists + end + local pathlist=pathlists[filetype] + if pathlist then + local method="intree" + local filelist=collect_files(wantedfiles) local dirlist={} + local result={} if filelist then for i=1,#filelist do dirlist[i]=filedirname(filelist[i][3]).."/" end end if trace_detail then - report_resolving("checking filename %a",filename) + report_resolving("checking filename %a in tree",filename) end - local resolve=resolvers.resolve - local result={} for k=1,#pathlist do - local path=pathlist[k] - local pathname=lpegmatch(inhibitstripper,path) - local doscan=path==pathname - if not find (pathname,'//$') then - doscan=false - end + local entry=pathlist[k] + local path=entry.path + local pathname=entry.pathname local done=false if filelist then - local expression=makepathexpression(pathname) + local expression=entry.expression if trace_detail then report_resolving("using pattern %a for path %a",expression,pathname) end @@ -14635,8 +15515,8 @@ local function find_intree(filename,filetype,wantedfiles,allresults) local fl=filelist[k] local f=fl[2] local d=dirlist[k] - if find(d,expression) or find(resolve(d),expression) then - result[#result+1]=resolve(fl[3]) + if find(d,expression) or find(resolveprefix(d),expression) then + result[#result+1]=resolveprefix(fl[3]) done=true if allresults then if trace_detail then @@ -14657,56 +15537,62 @@ local function find_intree(filename,filetype,wantedfiles,allresults) method="database" else method="filesystem" - pathname=gsub(pathname,"/+$","") - pathname=resolve(pathname) - local scheme=url.hasscheme(pathname) + local scheme=entry.scheme if not scheme or scheme=="file" then - local pname=gsub(pathname,"%.%*$",'') + local pname=entry.schemename if not find(pname,"*",1,true) then if can_be_dir(pname) then - for k=1,#wantedfiles do - local w=wantedfiles[k] - local fname=check_subpath(filejoin(pname,w)) - if fname then - result[#result+1]=fname - done=true - if not allresults then - break - end + if not done and not entry.prescanned then + if trace_detail then + report_resolving("quick root scan for %a",pname) end - end - if not done and doscan then - local files=resolvers.simplescanfiles(pname,false,true) for k=1,#wantedfiles do local w=wantedfiles[k] - local subpath=files[w] - if not subpath or subpath=="" then - elseif type(subpath)=="string" then - local fname=check_subpath(filejoin(pname,subpath,w)) - if fname then - result[#result+1]=fname - done=true - if not allresults then - break - end + local fname=check_subpath(filejoin(pname,w)) + if fname then + result[#result+1]=fname + done=true + if not allresults then + break end - else - for i=1,#subpath do - local sp=subpath[i] - if sp=="" then - else - local fname=check_subpath(filejoin(pname,sp,w)) - if fname then - result[#result+1]=fname - done=true - if not allresults then - break + end + end + if not done and entry.recursive then + if trace_detail then + report_resolving("scanning filesystem for %a",pname) + end + local files=resolvers.simplescanfiles(pname,false,true) + for k=1,#wantedfiles do + local w=wantedfiles[k] + local subpath=files[w] + if not subpath or subpath=="" then + elseif type(subpath)=="string" then + local fname=check_subpath(filejoin(pname,subpath,w)) + if fname then + result[#result+1]=fname + done=true + if not allresults then + break + end + end + else + for i=1,#subpath do + local sp=subpath[i] + if sp=="" then + else + local fname=check_subpath(filejoin(pname,sp,w)) + if fname then + result[#result+1]=fname + done=true + if not allresults then + break + end end end end - end - if done and not allresults then - break + if done and not allresults then + break + end end end end @@ -14714,6 +15600,18 @@ local function find_intree(filename,filetype,wantedfiles,allresults) end else end + else + for k=1,#wantedfiles do + local pname=entry.barename + local fname=methodhandler('finders',pname.."/"..wantedfiles[k]) + if fname then + result[#result+1]=fname + done=true + if not allresults then + break + end + end + end end end if done and not allresults then @@ -14748,10 +15646,13 @@ local function find_otherwise(filename,filetype,wantedfiles,allresults) local filelist=collect_files(wantedfiles) local fl=filelist and filelist[1] if fl then - return "otherwise",{ resolvers.resolve(fl[3]) } + return "otherwise",{ resolveprefix(fl[3]) } end end collect_instance_files=function(filename,askedformat,allresults) + if not filename or filename=="" then + return {} + end askedformat=askedformat or "" filename=collapsepath(filename,".") filename=gsub(filename,"^%./",getcurrentdir().."/") @@ -14786,7 +15687,11 @@ collect_instance_files=function(filename,askedformat,allresults) else local method,result,stamp,filetype,wantedfiles if instance.remember then - stamp=formatters["%s--%s"](filename,askedformat) + if askedformat=="" then + stamp=formatters["%s::%s"](suffixonly(filename),filename) + else + stamp=formatters["%s::%s"](askedformat,filename) + end result=stamp and instance.found[stamp] if result then if trace_locating then @@ -14821,7 +15726,7 @@ collect_instance_files=function(filename,askedformat,allresults) end if stamp then if trace_locating then - report_resolving("remembering file %a",filename) + report_resolving("remembering file %a using hash %a",filename,stamp) end instance.found[stamp]=result end @@ -14829,6 +15734,9 @@ collect_instance_files=function(filename,askedformat,allresults) end end local function findfiles(filename,filetype,allresults) + if not filename or filename=="" then + return {} + end local result,status=collect_instance_files(filename,filetype or "",allresults) if not result or #result==0 then local lowered=lower(filename) @@ -14848,39 +15756,30 @@ function resolvers.findpath(filename,filetype) return filedirname(findfiles(filename,filetype,false)[1] or "") end local function findgivenfiles(filename,allresults) - local bname,result=filebasename(filename),{} + local base=filebasename(filename) + local result={} local hashes=instance.hashes - local noffound=0 + local function okay(hash,path,name) + local found=methodhandler('concatinators',hash.type,hash.name,path,name) + if found and found~="" then + result[#result+1]=resolveprefix(found) + return not allresults + end + end for k=1,#hashes do local hash=hashes[k] - local files=instance.files[hash.name] or {} - local blist=files[bname] - if not blist then - local rname="remap:"..bname - blist=files[rname] - if blist then - bname=files[rname] - blist=files[bname] - end - end - if blist then - if type(blist)=='string' then - local found=methodhandler('concatinators',hash.type,hash.name,blist,bname) or "" - if found~="" then - noffound=noffound+1 - result[noffound]=resolvers.resolve(found) - if not allresults then - break - end + local content=instance.files[hash.name] + if content then + local path,name=lookup(content,base) + if not path then + elseif type(path)=="string" then + if okay(hash,path,name) then + return result end else - for kk=1,#blist do - local vv=blist[kk] - local found=methodhandler('concatinators',hash.type,hash.name,vv,bname) or "" - if found~="" then - noffound=noffound+1 - result[noffound]=resolvers.resolve(found) - if not allresults then break end + for i=1,#path do + if okay(hash,path[i],name) then + return result end end end @@ -14894,64 +15793,80 @@ end function resolvers.findgivenfile(filename) return findgivenfiles(filename,false)[1] or "" end -local function doit(path,blist,bname,tag,variant,result,allresults) - local done=false - if blist and variant then - local resolve=resolvers.resolve - if type(blist)=='string' then - if find(lower(blist),path) then - local full=methodhandler('concatinators',variant,tag,blist,bname) or "" - result[#result+1]=resolve(full) - done=true - end - else - for kk=1,#blist do - local vv=blist[kk] - if find(lower(vv),path) then - local full=methodhandler('concatinators',variant,tag,vv,bname) or "" - result[#result+1]=resolve(full) - done=true - if not allresults then break end - end - end - end - end - return done -end local makewildcard=Cs( (P("^")^0*P("/")*P(-1)+P(-1))/".*"+(P("^")^0*P("/")/"")^0*(P("*")/".*"+P("-")/"%%-"+P(".")/"%%."+P("?")/"."+P("\\")/"/"+P(1))^0 ) function resolvers.wildcardpattern(pattern) return lpegmatch(makewildcard,pattern) or pattern end -local function findwildcardfiles(filename,allresults,result) - result=result or {} +local function findwildcardfiles(filename,allresults,result) + local result=result or {} local base=filebasename(filename) local dirn=filedirname(filename) local path=lower(lpegmatch(makewildcard,dirn) or dirn) local name=lower(lpegmatch(makewildcard,base) or base) - local files,done=instance.files,false + local files=instance.files if find(name,"*",1,true) then local hashes=instance.hashes + local function okay(found,path,base,hashname,hashtype) + if find(found,path) then + local full=methodhandler('concatinators',hashtype,hashname,found,base) + if full and full~="" then + result[#result+1]=resolveprefix(full) + return not allresults + end + end + end for k=1,#hashes do local hash=hashes[k] - local hashname,hashtype=hash.name,hash.type - for kk,hh in next,files[hashname] do - if not find(kk,"^remap:") then - if find(lower(kk),name) then - if doit(path,hh,kk,hashname,hashtype,result,allresults) then done=true end - if done and not allresults then break end + local hashname=hash.name + local hashtype=hash.type + if hashname and hashtype then + for found,base in filtered(files[hashname],name) do + if type(found)=='string' then + if okay(found,path,base,hashname,hashtype) then + break + end + else + for i=1,#found do + if okay(found[i],path,base,hashname,hashtype) then + break + end + end end end end end else + local function okayokay(found,path,base,hashname,hashtype) + if find(found,path) then + local full=methodhandler('concatinators',hashtype,hashname,found,base) + if full and full~="" then + result[#result+1]=resolveprefix(full) + return not allresults + end + end + end local hashes=instance.hashes for k=1,#hashes do local hash=hashes[k] - local hashname,hashtype=hash.name,hash.type - if doit(path,files[hashname][base],base,hashname,hashtype,result,allresults) then done=true end - if done and not allresults then break end + local hashname=hash.name + local hashtype=hash.type + if hashname and hashtype then + local found,base=lookup(content,base) + if not found then + elseif type(found)=='string' then + if okay(found,path,base,hashname,hashtype) then + break + end + else + for i=1,#found do + if okay(found[i],path,base,hashname,hashtype) then + break + end + end + end + end end end return result @@ -15024,7 +15939,7 @@ end function resolvers.dowithpath(name,func) local pathlist=resolvers.expandedpathlist(name) for i=1,#pathlist do - func("^"..resolvers.cleanpath(pathlist[i])) + func("^"..cleanpath(pathlist[i])) end end function resolvers.dowithvariable(name,func) @@ -15032,23 +15947,23 @@ function resolvers.dowithvariable(name,func) end function resolvers.locateformat(name) local engine=environment.ownmain or "luatex" - local barename=file.removesuffix(name) - local fullname=file.addsuffix(barename,"fmt") + local barename=removesuffix(name) + local fullname=addsuffix(barename,"fmt") local fmtname=caches.getfirstreadablefile(fullname,"formats",engine) or "" if fmtname=="" then fmtname=resolvers.findfile(fullname) - fmtname=resolvers.cleanpath(fmtname) + fmtname=cleanpath(fmtname) end if fmtname~="" then - local barename=file.removesuffix(fmtname) - local luaname=file.addsuffix(barename,luasuffixes.lua) - local lucname=file.addsuffix(barename,luasuffixes.luc) - local luiname=file.addsuffix(barename,luasuffixes.lui) - if lfs.isfile(luiname) then + local barename=removesuffix(fmtname) + local luaname=addsuffix(barename,luasuffixes.lua) + local lucname=addsuffix(barename,luasuffixes.luc) + local luiname=addsuffix(barename,luasuffixes.lui) + if isfile(luiname) then return barename,luiname - elseif lfs.isfile(lucname) then + elseif isfile(lucname) then return barename,lucname - elseif lfs.isfile(luaname) then + elseif isfile(luaname) then return barename,luaname end end @@ -15070,29 +15985,24 @@ function resolvers.dowithfilesintree(pattern,handle,before,after) local hash=hashes[i] local blobtype=hash.type local blobpath=hash.name - if blobpath then + if blobtype and blobpath then + local total=0 + local checked=0 + local done=0 if before then before(blobtype,blobpath,pattern) end - local files=instance.files[blobpath] - local total,checked,done=0,0,0 - if files then - for k,v in table.sortedhash(files) do - total=total+1 - if find(k,"^remap:") then - elseif find(k,pattern) then - if type(v)=="string" then - checked=checked+1 - if handle(blobtype,blobpath,v,k) then - done=done+1 - end - else - checked=checked+#v - for i=1,#v do - if handle(blobtype,blobpath,v[i],k) then - done=done+1 - end - end + for path,name in filtered(instance.files[blobpath],pattern) do + if type(path)=="string" then + checked=checked+1 + if handle(blobtype,blobpath,path,name) then + done=done+1 + end + else + checked=checked+#path + for i=1,#path do + if handle(blobtype,blobpath,path[i],name) then + done=done+1 end end end @@ -15103,8 +16013,8 @@ function resolvers.dowithfilesintree(pattern,handle,before,after) end end end -resolvers.obsolete=resolvers.obsolete or {} -local obsolete=resolvers.obsolete +local obsolete=resolvers.obsolete or {} +resolvers.obsolete=obsolete resolvers.find_file=resolvers.findfile obsolete.find_file=resolvers.findfile resolvers.find_files=resolvers.findfiles obsolete.find_files=resolvers.findfiles @@ -15115,7 +16025,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-pre"] = package.loaded["data-pre"] or true --- original size: 6643, stripped down to: 4401 +-- original size: 3950, stripped down to: 2935 if not modules then modules={} end modules ['data-pre']={ version=1.001, @@ -15125,44 +16035,51 @@ if not modules then modules={} end modules ['data-pre']={ license="see context related readme files" } local resolvers=resolvers -local prefixes=utilities.storage.allocate() -resolvers.prefixes=prefixes -local cleanpath,findgivenfile,expansion=resolvers.cleanpath,resolvers.findgivenfile,resolvers.expansion +local prefixes=resolvers.prefixes +local cleanpath=resolvers.cleanpath +local findgivenfile=resolvers.findgivenfile +local expansion=resolvers.expansion local getenv=resolvers.getenv -local P,S,R,C,Cs,Cc,lpegmatch=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.Cs,lpeg.Cc,lpeg.match -local joinpath,basename,dirname=file.join,file.basename,file.dirname -local getmetatable,rawset,type=getmetatable,rawset,type +local basename=file.basename +local dirname=file.dirname +local joinpath=file.join +local isfile=lfs.isfile prefixes.environment=function(str) return cleanpath(expansion(str)) end -prefixes.relative=function(str,n) - if io.exists(str) then - elseif io.exists("./"..str) then - str="./"..str - else - local p="../" - for i=1,n or 2 do - if io.exists(p..str) then - str=p..str - break - else - p=p.."../" +local function relative(str,n) + if not isfile(str) then + local pstr="./"..str + if isfile(pstr) then + str=pstr + else + local p="../" + for i=1,n or 2 do + local pstr=p..str + if isfile(pstr) then + str=pstr + break + else + p=p.."../" + end end end end return cleanpath(str) end +local function locate(str) + local fullname=findgivenfile(str) or "" + return cleanpath(fullname~="" and fullname or str) +end +prefixes.relative=relative +prefixes.locate=locate prefixes.auto=function(str) - local fullname=prefixes.relative(str) - if not lfs.isfile(fullname) then - fullname=prefixes.locate(str) + local fullname=relative(str) + if not isfile(fullname) then + fullname=locate(str) end return fullname end -prefixes.locate=function(str) - local fullname=findgivenfile(str) or "" - return cleanpath((fullname~="" and fullname) or str) -end prefixes.filename=function(str) local fullname=findgivenfile(str) or "" return cleanpath(basename((fullname~="" and fullname) or str)) @@ -15183,6 +16100,13 @@ end prefixes.home=function(str) return cleanpath(joinpath(getenv('HOME'),str)) end +prefixes.env=prefixes.environment +prefixes.rel=prefixes.relative +prefixes.loc=prefixes.locate +prefixes.kpse=prefixes.locate +prefixes.full=prefixes.locate +prefixes.file=prefixes.filename +prefixes.path=prefixes.pathname local function toppath() local inputstack=resolvers.inputstack if not inputstack then @@ -15195,98 +16119,22 @@ local function toppath() return pathname end end -resolvers.toppath=toppath -prefixes.toppath=function(str) - return cleanpath(joinpath(toppath(),str)) -end -prefixes.env=prefixes.environment -prefixes.rel=prefixes.relative -prefixes.loc=prefixes.locate -prefixes.kpse=prefixes.locate -prefixes.full=prefixes.locate -prefixes.file=prefixes.filename -prefixes.path=prefixes.pathname -function resolvers.allprefixes(separator) - local all=table.sortedkeys(prefixes) - if separator then - for i=1,#all do - all[i]=all[i]..":" - end - end - return all -end -local function _resolve_(method,target) - local action=prefixes[method] - if action then - return action(target) - else - return method..":"..target - end -end -local resolved,abstract={},{} -function resolvers.resetresolve(str) - resolved,abstract={},{} -end -local pattern=Cs((C(R("az")^2)*P(":")*C((1-S(" \"\';,"))^1)/_resolve_+P(1))^0) -local prefix=C(R("az")^2)*P(":") -local target=C((1-S(" \"\';,"))^1) -local notarget=(#S(";,")+P(-1))*Cc("") -local pattern=Cs(((prefix*(target+notarget))/_resolve_+P(1))^0) -local function resolve(str) - if type(str)=="table" then - local t={} - for i=1,#str do - t[i]=resolve(str[i]) - end - return t +local function jobpath() + local path=resolvers.stackpath() + if not path or path=="" then + return "." else - local res=resolved[str] - if not res then - res=lpegmatch(pattern,str) - resolved[str]=res - abstract[res]=str - end - return res - end -end -local function unresolve(str) - return abstract[str] or str -end -resolvers.resolve=resolve -resolvers.unresolve=unresolve -if type(os.uname)=="function" then - for k,v in next,os.uname() do - if not prefixes[k] then - prefixes[k]=function() return v end - end - end -end -if os.type=="unix" then - local pattern - local function makepattern(t,k,v) - if t then - rawset(t,k,v) - end - local colon=P(":") - for k,v in table.sortedpairs(prefixes) do - if p then - p=P(k)+p - else - p=P(k) - end - end - pattern=Cs((p*colon+colon/";"+P(1))^0) - end - makepattern() - getmetatable(prefixes).__newindex=makepattern - function resolvers.repath(str) - return lpegmatch(pattern,str) - end -else - function resolvers.repath(str) - return str + return path end end +resolvers.toppath=toppath +resolvers.jobpath=jobpath +prefixes.toppath=function(str) return cleanpath(joinpath(toppath(),str)) end +prefixes.jobpath=function(str) return cleanpath(joinpath(jobpath(),str)) end +resolvers.setdynamic("toppath") +resolvers.setdynamic("jobpath") +prefixes.jobfile=prefixes.jobpath +resolvers.setdynamic("jobfile") end -- of closure @@ -15348,7 +16196,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-fil"] = package.loaded["data-fil"] or true --- original size: 3801, stripped down to: 3231 +-- original size: 3863, stripped down to: 3310 if not modules then modules={} end modules ['data-fil']={ version=1.001, @@ -15360,30 +16208,31 @@ if not modules then modules={} end modules ['data-fil']={ local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local report_files=logs.reporter("resolvers","files") local resolvers=resolvers +local resolveprefix=resolvers.resolve local finders,openers,loaders,savers=resolvers.finders,resolvers.openers,resolvers.loaders,resolvers.savers local locators,hashers,generators,concatinators=resolvers.locators,resolvers.hashers,resolvers.generators,resolvers.concatinators local checkgarbage=utilities.garbagecollector and utilities.garbagecollector.check function locators.file(specification) - local name=specification.filename - local realname=resolvers.resolve(name) + local filename=specification.filename + local realname=resolveprefix(filename) if realname and realname~='' and lfs.isdir(realname) then if trace_locating then - report_files("file locator %a found as %a",name,realname) + report_files("file locator %a found as %a",filename,realname) end - resolvers.appendhash('file',name,true) + resolvers.appendhash('file',filename,true) elseif trace_locating then - report_files("file locator %a not found",name) + report_files("file locator %a not found",filename) end end function hashers.file(specification) - local name=specification.filename - local content=caches.loadcontent(name,'files') - resolvers.registerfilehash(name,content,content==nil) + local pathname=specification.filename + local content=caches.loadcontent(pathname,'files') + resolvers.registerfilehash(pathname,content,content==nil) end function generators.file(specification) - local path=specification.filename - local content=resolvers.scanfiles(path,false,true) - resolvers.registerfilehash(path,content,true) + local pathname=specification.filename + local content=resolvers.scanfiles(pathname,false,true) + resolvers.registerfilehash(pathname,content,true) end concatinators.file=file.join function finders.file(specification,filetype) @@ -15665,7 +16514,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-zip"] = package.loaded["data-zip"] or true --- original size: 8489, stripped down to: 6757 +-- original size: 8772, stripped down to: 6841 if not modules then modules={} end modules ['data-zip']={ version=1.001, @@ -15684,16 +16533,6 @@ zip.archives=zip.archives or {} local archives=zip.archives zip.registeredfiles=zip.registeredfiles or {} local registeredfiles=zip.registeredfiles -local limited=false -directives.register("system.inputmode",function(v) - if not limited then - local i_limiter=io.i_limiter(v) - if i_limiter then - zip.open=i_limiter.protect(zip.open) - limited=true - end - end -end) local function validzip(str) if not find(str,"^zip://") then return "zip:///"..str @@ -15708,7 +16547,7 @@ function zip.openarchive(name) local arch=archives[name] if not arch then local full=resolvers.findfile(name) or "" - arch=(full~="" and zip.open(full)) or false + arch=full~="" and zip.open(full) or false archives[name]=arch end return arch @@ -15867,31 +16706,42 @@ function resolvers.usezipfile(archive) end end function resolvers.registerzipfile(z,tree) - local files,filter={},"" - if tree=="" then - filter="^(.+)/(.-)$" - else - filter=format("^%s/(.+)/(.-)$",tree) - end + local names={} + local files={} + local remap={} + local n=0 + local filter=tree=="" and "^(.+)/(.-)$" or format("^%s/(.+)/(.-)$",tree) + local register=resolvers.registerfile if trace_locating then report_zip("registering: using filter %a",filter) end - local register,n=resolvers.registerfile,0 for i in z:files() do - local path,name=match(i.filename,filter) - if path then - if name and name~='' then - register(files,name,path) - n=n+1 - else + local filename=i.filename + local path,name=match(filename,filter) + if not path then + n=n+1 + register(names,filename,"") + local usedname=lower(filename) + files[usedname]="" + if usedname~=filename then + remap[usedname]=filename end - else - register(files,i.filename,'') + elseif name and name~="" then n=n+1 + register(names,name,path) + local usedname=lower(name) + files[usedname]=path + if usedname~=name then + remap[usedname]=name + end + else end end report_zip("registering: %s files registered",n) - return files + return { + files=files, + remap=remap, + } end @@ -15901,7 +16751,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tre"] = package.loaded["data-tre"] or true --- original size: 2508, stripped down to: 2074 +-- original size: 8479, stripped down to: 5580 if not modules then modules={} end modules ['data-tre']={ version=1.001, @@ -15910,42 +16760,64 @@ if not modules then modules={} end modules ['data-tre']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local find,gsub,format=string.find,string.gsub,string.format +local find,gsub,lower=string.find,string.gsub,string.lower +local basename,dirname,joinname=file.basename,file.dirname,file .join +local globdir,isdir,isfile=dir.glob,lfs.isdir,lfs.isfile +local P,lpegmatch=lpeg.P,lpeg.match local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local report_trees=logs.reporter("resolvers","trees") local resolvers=resolvers -local done,found,notfound={},{},resolvers.finders.notfound -function resolvers.finders.tree(specification) +local resolveprefix=resolvers.resolve +local notfound=resolvers.finders.notfound +local lookup=resolvers.get_from_content +local collectors={} +local found={} +function resolvers.finders.tree(specification) local spec=specification.filename - local fnd=found[spec] - if fnd==nil then + local okay=found[spec] + if okay==nil then if spec~="" then - local path,name=file.dirname(spec),file.basename(spec) - if path=="" then path="." end - local hash=done[path] - if not hash then - local pattern=path.."/*" - hash=dir.glob(pattern) - done[path]=hash + local path=dirname(spec) + local name=basename(spec) + if path=="" then + path="." + end + local names=collectors[path] + if not names then + local pattern=find(path,"/%*+$") and path or (path.."/*") + names=globdir(pattern) + collectors[path]=names end local pattern="/"..gsub(name,"([%.%-%+])","%%%1").."$" - for k=1,#hash do - local v=hash[k] - if find(v,pattern) then - found[spec]=v - return v + for i=1,#names do + local fullname=names[i] + if find(fullname,pattern) then + found[spec]=fullname + return fullname + end + end + local pattern=lower(pattern) + for i=1,#names do + local fullname=lower(names[i]) + if find(fullname,pattern) then + if isfile(fullname) then + found[spec]=fullname + return fullname + else + break + end end end end - fnd=notfound() - found[spec]=fnd + okay=notfound() + found[spec]=okay end - return fnd + return okay end function resolvers.locators.tree(specification) local name=specification.filename - local realname=resolvers.resolve(name) - if realname and realname~='' and lfs.isdir(realname) then + local realname=resolveprefix(name) + if realname and realname~='' and isdir(realname) then if trace_locating then report_trees("locator %a found",realname) end @@ -15956,16 +16828,110 @@ function resolvers.locators.tree(specification) end function resolvers.hashers.tree(specification) local name=specification.filename - if trace_locating then - report_trees("analysing %a",name) - end + report_trees("analyzing %a",name) resolvers.methodhandler("hashers",name) resolvers.generators.file(specification) end -resolvers.concatinators.tree=resolvers.concatinators.file -resolvers.generators.tree=resolvers.generators.file -resolvers.openers.tree=resolvers.openers.file -resolvers.loaders.tree=resolvers.loaders.file +local collectors={} +local splitter=lpeg.splitat("/**/") +local stripper=lpeg.replacer { [P("/")*P("*")^1*P(-1)]="" } +table.setmetatableindex(collectors,function(t,k) + local rootname=lpegmatch(stripper,k) + local dataname=joinname(rootname,"dirlist") + local content=caches.loadcontent(dataname,"files",dataname) + if not content then + content=resolvers.scanfiles(rootname,nil,nil,false,true) + caches.savecontent(dataname,"files",content,dataname) + end + t[k]=content + return content +end) +local function checked(root,p,n) + if p then + if type(p)=="table" then + for i=1,#p do + local fullname=joinname(root,p[i],n) + if isfile(fullname) then + return fullname + end + end + else + local fullname=joinname(root,p,n) + if isfile(fullname) then + return fullname + end + end + end + return notfound() +end +local function resolve(specification) + local filename=specification.filename + if filename~="" then + local root,rest=lpegmatch(splitter,filename) + if root and rest then + local path,name=dirname(rest),basename(rest) + if name~=rest then + local content=collectors[root] + local p,n=lookup(content,name) + if not p then + return notfound() + end + local pattern=".*/"..path.."$" + local istable=type(p)=="table" + if istable then + for i=1,#p do + local pi=p[i] + if pi==path or find(pi,pattern) then + local fullname=joinname(root,pi,n) + if isfile(fullname) then + return fullname + end + end + end + elseif p==path or find(p,pattern) then + local fullname=joinname(root,p,n) + if isfile(fullname) then + return fullname + end + end + local queries=specification.queries + if queries and queries.option=="fileonly" then + return checked(root,p,n) + else + return notfound() + end + end + end + local path,name=dirname(filename),basename(filename) + local root=lpegmatch(stripper,path) + local content=collectors[path] + local p,n=lookup(content,name) + if p then + return checked(root,p,n) + end + end + return notfound() +end +resolvers.finders .dirlist=resolve +resolvers.locators .dirlist=resolvers.locators .tree +resolvers.hashers .dirlist=resolvers.hashers .tree +resolvers.generators.dirlist=resolvers.generators.file +resolvers.openers .dirlist=resolvers.openers .file +resolvers.loaders .dirlist=resolvers.loaders .file +function resolvers.finders.dirfile(specification) + local queries=specification.queries + if queries then + queries.option="fileonly" + else + specification.queries={ option="fileonly" } + end + return resolve(specification) +end +resolvers.locators .dirfile=resolvers.locators .dirlist +resolvers.hashers .dirfile=resolvers.hashers .dirlist +resolvers.generators.dirfile=resolvers.generators.dirlist +resolvers.openers .dirfile=resolvers.openers .dirlist +resolvers.loaders .dirfile=resolvers.loaders .dirlist end -- of closure @@ -15974,7 +16940,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-sch"] = package.loaded["data-sch"] or true --- original size: 6213, stripped down to: 5160 +-- original size: 6569, stripped down to: 5304 if not modules then modules={} end modules ['data-sch']={ version=1.001, @@ -16000,8 +16966,13 @@ directives.register("schemes.threshold",function(v) threshold=tonumber(v) or thr function cleaners.none(specification) return specification.original end -function cleaners.strip(specification) - return (gsub(specification.original,"[^%a%d%.]+","-")) +function cleaners.strip(specification) + local path,name=file.splitbase(specification.original) + if path=="" then + return (gsub(name,"[^%a%d%.]+","-")) + else + return (gsub((gsub(path,"%.","-").."-"..name),"[^%a%d%.]+","-")) + end end function cleaners.md5(specification) return file.addsuffix(md5.hex(specification.original),file.suffix(specification.path)) @@ -16018,7 +16989,7 @@ end local cached,loaded,reused,thresholds,handlers={},{},{},{},{} local function runcurl(name,cachename) local command="curl --silent --insecure --create-dirs --output "..cachename.." "..name - os.spawn(command) + os.execute(command) end local function fetch(specification) local original=specification.original @@ -16150,7 +17121,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-lua"] = package.loaded["data-lua"] or true --- original size: 4237, stripped down to: 3177 +-- original size: 4313, stripped down to: 3227 if not modules then modules={} end modules ['data-lua']={ version=1.001, @@ -16159,7 +17130,7 @@ if not modules then modules={} end modules ['data-lua']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local resolvers,package=resolvers,package +local package,lpeg=package,lpeg local gsub=string.gsub local concat=table.concat local addsuffix=file.addsuffix @@ -16170,9 +17141,11 @@ local luaformats={ 'TEXINPUTS','LUAINPUTS' } local libformats={ 'CLUAINPUTS' } local helpers=package.helpers or {} local methods=helpers.methods or {} +local resolvers=resolvers +local resolveprefix=resolvers.resolve +helpers.report=logs.reporter("resolvers","libraries") trackers.register("resolvers.libraries",function(v) helpers.trace=v end) trackers.register("resolvers.locating",function(v) helpers.trace=v end) -helpers.report=logs.reporter("resolvers","libraries") helpers.sequence={ "already loaded", "preload table", @@ -16187,7 +17160,7 @@ helpers.sequence={ } local pattern=Cs(P("!")^0/""*(P("/")*P(-1)/"/"+P("/")^1/"/"+1)^0) function helpers.cleanpath(path) - return resolvers.resolve(lpegmatch(pattern,path)) + return resolveprefix(lpegmatch(pattern,path)) end local loadedaslib=helpers.loadedaslib local getextraluapaths=package.extraluapaths @@ -16324,7 +17297,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tmf"] = package.loaded["data-tmf"] or true --- original size: 2600, stripped down to: 1627 +-- original size: 2601, stripped down to: 1627 if not modules then modules={} end modules ['data-tmf']={ version=1.001, @@ -16380,7 +17353,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-lst"] = package.loaded["data-lst"] or true --- original size: 2654, stripped down to: 2301 +-- original size: 2734, stripped down to: 2354 if not modules then modules={} end modules ['data-lst']={ version=1.001, @@ -16389,10 +17362,13 @@ if not modules then modules={} end modules ['data-lst']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local find,concat,upper,format=string.find,table.concat,string.upper,string.format +local rawget,type,next=rawget,type,next +local find,concat,upper=string.find,table.concat,string.upper local fastcopy,sortedpairs=table.fastcopy,table.sortedpairs -resolvers.listers=resolvers.listers or {} local resolvers=resolvers +local listers=resolvers.listers or {} +resolvers.listers=listers +local resolveprefix=resolvers.resolve local report_lists=logs.reporter("resolvers","lists") local function tabstr(str) if type(str)=='table' then @@ -16401,7 +17377,7 @@ local function tabstr(str) return str end end -function resolvers.listers.variables(pattern) +function listers.variables(pattern) local instance=resolvers.instance local environment=instance.environment local variables=instance.variables @@ -16422,10 +17398,10 @@ function resolvers.listers.variables(pattern) for key,value in sortedpairs(configured) do if key~="" and (pattern=="" or find(upper(key),pattern)) then report_lists(key) - report_lists(" env: %s",tabstr(rawget(environment,key)) or "unset") - report_lists(" var: %s",tabstr(configured[key]) or "unset") - report_lists(" exp: %s",tabstr(expansions[key]) or "unset") - report_lists(" res: %s",tabstr(resolvers.resolve(expansions[key])) or "unset") + report_lists(" env: %s",tabstr(rawget(environment,key)) or "unset") + report_lists(" var: %s",tabstr(configured[key]) or "unset") + report_lists(" exp: %s",tabstr(expansions[key]) or "unset") + report_lists(" res: %s",tabstr(resolveprefix(expansions[key])) or "unset") end end instance.environment=fastcopy(env) @@ -16433,15 +17409,15 @@ function resolvers.listers.variables(pattern) instance.expansions=fastcopy(exp) end local report_resolved=logs.reporter("system","resolved") -function resolvers.listers.configurations() +function listers.configurations() local configurations=resolvers.instance.specification for i=1,#configurations do - report_resolved("file : %s",resolvers.resolve(configurations[i])) + report_resolved("file : %s",resolveprefix(configurations[i])) end report_resolved("") local list=resolvers.expandedpathfromlist(resolvers.splitpath(resolvers.luacnfspec)) for i=1,#list do - local li=resolvers.resolve(list[i]) + local li=resolveprefix(list[i]) if lfs.isdir(li) then report_resolved("path - %s",li) else @@ -16746,7 +17722,7 @@ do -- create closure to overcome 200 locals limit package.loaded["luat-fmt"] = package.loaded["luat-fmt"] or true --- original size: 5951, stripped down to: 4922 +-- original size: 5955, stripped down to: 4926 if not modules then modules={} end modules ['luat-fmt']={ version=1.001, @@ -16834,7 +17810,7 @@ function environment.make_format(name) end local command=format("%s --ini %s --lua=%s %s %sdump",engine,primaryflags(),quoted(usedluastub),quoted(fulltexsourcename),os.platform=="unix" and "\\\\" or "\\") report_format("running command: %s\n",command) - os.spawn(command) + os.execute(command) local pattern=file.removesuffix(file.basename(usedluastub)).."-*.mem" local mp=dir.glob(pattern) if mp then @@ -16869,7 +17845,7 @@ function environment.run_format(name,data,more) else local command=format("%s %s --fmt=%s --lua=%s %s %s",engine,primaryflags(),quoted(barename),quoted(luaname),quoted(data),more~="" and quoted(more) or "") report_format("running command: %s",command) - os.spawn(command) + os.execute(command) end end end @@ -16880,8 +17856,8 @@ end -- of closure -- used libraries : l-lua.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-sto.lua util-prs.lua util-fmt.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-mrg.lua util-tpl.lua util-env.lua luat-env.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua -- skipped libraries : - --- original bytes : 695180 --- stripped bytes : 246794 +-- original bytes : 739251 +-- stripped bytes : 264497 -- end library merge @@ -17107,6 +18083,7 @@ local helpinfo = [[ <category name="basic"> <subcategory> <flag name="script"><short>run an mtx script (lua prefered method) (<ref name="noquotes"/>), no script gives list</short></flag> + <flag name="evaluate"><short>run code passed on the commandline (between quotes)</short></flag> <flag name="execute"><short>run a script or program (texmfstart method) (<ref name="noquotes"/>)</short></flag> <flag name="resolve"><short>resolve prefixed arguments</short></flag> <flag name="ctxlua"><short>run internally (using preloaded libs)</short></flag> @@ -17132,6 +18109,7 @@ local helpinfo = [[ <flag name="verbose"><short>give a bit more info</short></flag> <flag name="trackers" value="list"><short>enable given trackers</short></flag> <flag name="progname" value="str"><short>format or backend</short></flag> + <flag name="systeminfo" value="str"><short>show current operating system, processor, etc</short></flag> </subcategory> <subcategory> <flag name="edit"><short>launch editor with found file</short></flag> @@ -17761,6 +18739,39 @@ function runners.associate(filename) os.launch(filename) end +function runners.evaluate(code,filename) -- for Luigi + if code == "loop" then + while true do + io.write("> ") + local code = io.read() + if code ~= "" then + local temp = string.match(code,"^= (.*)$") + if temp then + code = "print("..temp..")" + end + local compiled, message = loadstring(code) + if type(compiled) ~= "function" then + io.write("! " .. (message or code).."\n") + else + io.write(compiled()) + end + end + end + else + if type(code) ~= "string" or code == "" then + code = filename + end + if code ~= "" then + local compiled, message = loadstring(code) + if type(compiled) ~= "function" then + io.write("invalid lua code: " .. (message or code)) + return + end + io.write(compiled()) + end + end +end + function runners.gethelp(filename) local url = environment.argument("url") if url and url ~= "" then @@ -17772,6 +18783,15 @@ function runners.gethelp(filename) end end +function runners.systeminfo() + report("architecture : %s",os.platform or "<unset>") + report("operating system : %s",os.name or "<unset>") + report("file architecture : %s",os.type or "<unset>") + report("binary path : %s",os.selfdir or "<unset>") + report("binary suffix : %s",os.binsuffix or "<unset>") + report("library suffix : %s",os.libsuffix or "<unset>") +end + -- this is a bit dirty ... first we store the first filename and next we -- split the arguments so that we only see the ones meant for this script -- ... later we will use the second half @@ -17887,16 +18907,7 @@ end if e_argument("ansi") then - local formatters = string.formatters - - logs.setformatters { - report_yes = formatters["[1;32m%-15s [0;1m|[0m %s"], - report_nop = formatters["[1;32m%-15s [0;1m|[0m"], - subreport_yes = formatters["[1;32m%-15s [0;1m|[1;31m %s [0;1m|[0m %s"], - subreport_nop = formatters["[1;32m%-15s [0;1m|[1;31m %s [0;1m|[0m"], - status_yes = formatters["[1;32m%-15s [0;1m:[0m %s\n"], - status_nop = formatters["[1;32m%-15s [0;1m:[0m\n"], - } + logs.setformatters("ansi") local script = e_argument("script") or e_argument("scripts") @@ -17921,6 +18932,10 @@ if e_argument("script") or e_argument("scripts") then ok = runners.execute_ctx_script(filename) end +elseif e_argument("evaluate") then + + runners.evaluate(e_argument("evaluate"),filename) + elseif e_argument("selfmerge") then -- embed used libraries @@ -18213,6 +19228,10 @@ elseif e_argument("exporthelp") then runners.loadbase() application.export(e_argument("exporthelp"),filename) +elseif e_argument("systeminfo") then + + runners.systeminfo() + elseif e_argument("help") or filename=='help' or filename == "" then application.help() |