diff options
Diffstat (limited to 'Master/texmf-dist/tex/context/base/lxml-tab.lua')
-rw-r--r-- | Master/texmf-dist/tex/context/base/lxml-tab.lua | 308 |
1 files changed, 229 insertions, 79 deletions
diff --git a/Master/texmf-dist/tex/context/base/lxml-tab.lua b/Master/texmf-dist/tex/context/base/lxml-tab.lua index 7d38a8d5abc..0a43f435274 100644 --- a/Master/texmf-dist/tex/context/base/lxml-tab.lua +++ b/Master/texmf-dist/tex/context/base/lxml-tab.lua @@ -10,6 +10,10 @@ if not modules then modules = { } end modules ['lxml-tab'] = { -- stripping spaces from e.g. cont-en.xml saves .2 sec runtime so it's not worth the -- trouble +-- todo: when serializing optionally remap named entities to hex (if known in char-ent.lua) +-- maybe when letter -> utf, else name .. then we need an option to the serializer .. a bit +-- of work so we delay this till we cleanup + local trace_entities = false trackers.register("xml.entities", function(v) trace_entities = v end) local report_xml = logs and logs.reporter("xml","core") or function(...) print(format(...)) end @@ -35,10 +39,11 @@ local xml = xml --~ local xml = xml +local utf = unicode.utf8 local concat, remove, insert = table.concat, table.remove, table.insert local type, next, setmetatable, getmetatable, tonumber = type, next, setmetatable, getmetatable, tonumber local format, lower, find, match, gsub = string.format, string.lower, string.find, string.match, string.gsub -local utfchar = unicode.utf8.char +local utfchar, utffind, utfgsub = utf.char, utf.find, utf.gsub local lpegmatch = lpeg.match local P, S, R, C, V, C, Cs = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.C, lpeg.Cs @@ -151,9 +156,22 @@ element.</p> local nsremap, resolvens = xml.xmlns, xml.resolvens -local stack, top, dt, at, xmlns, errorstr, entities = { }, { }, { }, { }, { }, nil, { } -local strip, cleanup, utfize, resolve, resolve_predefined, unify_predefined = false, false, false, false, false, false -local dcache, hcache, acache = { }, { }, { } +local stack = { } +local top = { } +local dt = { } +local at = { } +local xmlns = { } +local errorstr = nil +local entities = { } +local strip = false +local cleanup = false +local utfize = false +local resolve_predefined = false +local unify_predefined = false + +local dcache = { } +local hcache = { } +local acache = { } local mt = { } @@ -319,7 +337,7 @@ local predefined_unified = { [42] = """, [47] = "'", [74] = "<", - [76] = "&gr;", + [76] = ">", } local predefined_simplified = { @@ -330,6 +348,58 @@ local predefined_simplified = { [76] = ">", gt = ">", } +local nofprivates = 0xF0000 -- shared but seldom used + +local privates_u = { -- unescaped + [ [[&]] ] = "&", + [ [["]] ] = """, + [ [[']] ] = "'", + [ [[<]] ] = "<", + [ [[>]] ] = ">", +} + +local privates_p = { +} + +local privates_n = { + -- keeps track of defined ones +} + +local function escaped(s) + if s == "" then + return "" + else -- if utffind(s,privates_u) then + return (utfgsub(s,".",privates_u)) + -- else + -- return s + end +end + +local function unescaped(s) + local p = privates_n[s] + if not p then + nofprivates = nofprivates + 1 + p = utfchar(nofprivates) + privates_n[s] = p + s = "&" .. s .. ";" -- todo: use char-ent to map to hex + privates_u[p] = s + privates_p[p] = s + end + return p +end + +local function unprivatized(s,resolve) + if s == "" then + return "" + else + return (utfgsub(s,".",privates_p)) + end +end + +xml.privatetoken = unescaped +xml.unprivatized = unprivatized +xml.privatecodes = privates_n + local function handle_hex_entity(str) local h = hcache[str] if not h then @@ -371,7 +441,7 @@ local function handle_dec_entity(str) if not n then report_xml("utfize, ignoring dec entity &#%s;",str) elseif trace_entities then - report_xml("utfize, converting dec entity &#%s; into %s",str,h) + report_xml("utfize, converting dec entity &#%s; into %s",str,d) end else if trace_entities then @@ -392,34 +462,44 @@ local function handle_any_entity(str) if not a then a = resolve_predefined and predefined_simplified[str] if a then - -- one of the predefined - elseif type(resolve) == "function" then - a = resolve(str) or entities[str] - else - a = entities[str] - end - if a then if trace_entities then - report_xml("resolved entity &%s; -> %s (internal)",str,a) + report_xml("resolved entity &%s; -> %s (predefined)",str,a) end - a = lpegmatch(parsedentity,a) or a else - local unknown_any_entity = placeholders.unknown_any_entity - if unknown_any_entity then - a = unknown_any_entity(str) or "" + if type(resolve) == "function" then + a = resolve(str) or entities[str] + else + a = entities[str] end if a then + if type(a) == "function" then + if trace_entities then + report_xml("expanding entity &%s; (function)",str) + end + a = a(str) or "" + end + a = lpegmatch(parsedentity,a) or a -- for nested if trace_entities then - report_xml("resolved entity &%s; -> %s (external)",str,a) + report_xml("resolved entity &%s; -> %s (internal)",str,a) end else - if trace_entities then - report_xml("keeping entity &%s;",str) + local unknown_any_entity = placeholders.unknown_any_entity + if unknown_any_entity then + a = unknown_any_entity(str) or "" end - if str == "" then - a = "&error;" + if a then + if trace_entities then + report_xml("resolved entity &%s; -> %s (external)",str,a) + end else - a = "&" .. str .. ";" + if trace_entities then + report_xml("keeping entity &%s;",str) + end + if str == "" then + a = "&error;" + else + a = "&" .. str .. ";" + end end end end @@ -434,18 +514,25 @@ local function handle_any_entity(str) else local a = acache[str] if not a then - if trace_entities then - report_xml("found entity &%s;",str) - end a = resolve_predefined and predefined_simplified[str] if a then -- one of the predefined acache[str] = a + if trace_entities then + report_xml("entity &%s; becomes %s",str,tostring(a)) + end elseif str == "" then + if trace_entities then + report_xml("invalid entity &%s;",str) + end a = "&error;" acache[str] = a else - a = "&" .. str .. ";" + if trace_entities then + report_xml("entity &%s; is made private",str) + end + -- a = "&" .. str .. ";" + a = unescaped(str) acache[str] = a end end @@ -495,7 +582,7 @@ local value = (squote * Cs((entity + (1 - squote))^0) * squote) + (dq local endofattributes = slash * close + close -- recovery of flacky html local whatever = space * name * optionalspace * equal -local wrongvalue = C(P(1-whatever-close)^1 + P(1-close)^1) / attribute_value_error +----- wrongvalue = C(P(1-whatever-close)^1 + P(1-close)^1) / attribute_value_error ----- wrongvalue = C(P(1-whatever-endofattributes)^1 + P(1-endofattributes)^1) / attribute_value_error ----- wrongvalue = C(P(1-space-endofattributes)^1) / attribute_value_error local wrongvalue = Cs(P(entity + (1-space-endofattributes))^1) / attribute_value_error @@ -530,6 +617,8 @@ local function normalentity(k,v ) entities[k] = v end local function systementity(k,v,n) entities[k] = v end local function publicentity(k,v,n) entities[k] = v end +-- todo: separate dtd parser + local begindoctype = open * P("!DOCTYPE") local enddoctype = close local beginset = P("[") @@ -537,17 +626,22 @@ local endset = P("]") local doctypename = C((1-somespace-close)^0) local elementdoctype = optionalspace * P("<!ELEMENT") * (1-close)^0 * close +local basiccomment = begincomment * ((1 - endcomment)^0) * endcomment + local normalentitytype = (doctypename * somespace * value)/normalentity local publicentitytype = (doctypename * somespace * P("PUBLIC") * somespace * value)/publicentity local systementitytype = (doctypename * somespace * P("SYSTEM") * somespace * value * somespace * P("NDATA") * somespace * doctypename)/systementity local entitydoctype = optionalspace * P("<!ENTITY") * somespace * (systementitytype + publicentitytype + normalentitytype) * optionalspace * close -local doctypeset = beginset * optionalspace * P(elementdoctype + entitydoctype + space)^0 * optionalspace * endset +-- we accept comments in doctypes + +local doctypeset = beginset * optionalspace * P(elementdoctype + entitydoctype + basiccomment + space)^0 * optionalspace * endset local definitiondoctype= doctypename * somespace * doctypeset local publicdoctype = doctypename * somespace * P("PUBLIC") * somespace * value * somespace * value * somespace * doctypeset local systemdoctype = doctypename * somespace * P("SYSTEM") * somespace * value * somespace * doctypeset local simpledoctype = (1-close)^1 -- * balanced^0 local somedoctype = C((somespace * (publicdoctype + systemdoctype + definitiondoctype + simpledoctype) * optionalspace)^0) +local somedoctype = C((somespace * (publicdoctype + systemdoctype + definitiondoctype + simpledoctype) * optionalspace)^0) local instruction = (spacing * begininstruction * someinstruction * endinstruction) / function(...) add_special("@pi@",...) end local comment = (spacing * begincomment * somecomment * endcomment ) / function(...) add_special("@cm@",...) end @@ -579,17 +673,31 @@ local grammar_unparsed_text = P { "preamble", children = unparsedtext + V("parent") + emptyelement + comment + cdata + instruction, } --- maybe we will add settinsg to result as well +-- maybe we will add settings to result as well -local function xmlconvert(data, settings) - settings = settings or { } -- no_root strip_cm_and_dt given_entities parent_root error_handler - strip = settings.strip_cm_and_dt - utfize = settings.utfize_entities - resolve = settings.resolve_entities +local function _xmlconvert_(data, settings) + settings = settings or { } -- no_root strip_cm_and_dt given_entities parent_root error_handler + -- + strip = settings.strip_cm_and_dt + utfize = settings.utfize_entities + resolve = settings.resolve_entities resolve_predefined = settings.resolve_predefined_entities -- in case we have escaped entities - unify_predefined = settings.unify_predefined_entities -- & -> & - cleanup = settings.text_cleanup - stack, top, at, xmlns, errorstr, entities = { }, { }, { }, { }, nil, settings.entities or { } + unify_predefined = settings.unify_predefined_entities -- & -> & + cleanup = settings.text_cleanup + entities = settings.entities or { } + -- + if utfize == nil then + settings.utfize_entities = true + utfize = true + end + if resolve_predefined == nil then + settings.resolve_predefined_entities = true + resolve_predefined = true + end + -- +--~ inspect(settings) + -- + stack, top, at, xmlns, errorstr = { }, { }, { }, { }, nil acache, hcache, dcache = { }, { }, { } -- not stored reported_attribute_errors = { } if settings.parent_root then @@ -627,7 +735,7 @@ local function xmlconvert(data, settings) else errorhandler = errorhandler or xml.errorhandler if errorhandler then - xml.errorhandler("load",errorstr) + xml.errorhandler(format("load error: %s",errorstr)) end end else @@ -641,7 +749,7 @@ local function xmlconvert(data, settings) local v = rdt[k] if type(v) == "table" and not v.special then -- always table -) result.ri = k -- rootindex -v.__p__ = result -- new, experiment, else we cannot go back to settings, we need to test this ! + v.__p__ = result -- new, experiment, else we cannot go back to settings, we need to test this ! break end end @@ -649,16 +757,42 @@ v.__p__ = result -- new, experiment, else we cannot go back to settings, we nee if errorstr and errorstr ~= "" then result.error = true end + result.statistics = { + entities = { + decimals = dcache, + hexadecimals = hcache, + names = acache, + } + } + strip, utfize, resolve, resolve_predefined = nil, nil, nil, nil + unify_predefined, cleanup, entities = nil, nil, nil + stack, top, at, xmlns, errorstr = nil, nil, nil, nil, nil + acache, hcache, dcache = nil, nil, nil + reported_attribute_errors, mt, errorhandler = nil, nil, nil return result end +-- Because we can have a crash (stack issues) with faulty xml, we wrap this one +-- in a protector: + +function xmlconvert(data,settings) + local ok, result = pcall(function() return _xmlconvert_(data,settings) end) + if ok then + return result + else + return _xmlconvert_("") + end +end + xml.convert = xmlconvert -function xml.inheritedconvert(data,xmldata) +function xml.inheritedconvert(data,xmldata) -- xmldata is parent local settings = xmldata.settings - settings.parent_root = xmldata -- to be tested + if settings then + settings.parent_root = xmldata -- to be tested + end -- settings.no_root = true - local xc = xmlconvert(data,settings) + local xc = xmlconvert(data,settings) -- hm, we might need to locate settings -- xc.settings = nil -- xc.entities = nil -- xc.special = nil @@ -687,7 +821,7 @@ function xml.is_valid(root) return root and not root.error end -xml.errorhandler = report +xml.errorhandler = report_xml --[[ldx-- <p>We cannot load an <l n='lpeg'/> from a filehandle so we need to load @@ -766,18 +900,15 @@ alternative.</p> function xml.checkbom(root) -- can be made faster if root.ri then - local dt, found = root.dt, false + local dt = root.dt for k=1,#dt do local v = dt[k] if type(v) == "table" and v.special and v.tg == "@pi@" and find(v.dt[1],"xml.*version=") then - found = true - break + return end end - if not found then - insert(dt, 1, { special=true, ns="", tg="@pi@", dt = { "xml version='1.0' standalone='yes'"} } ) - insert(dt, 2, "\n" ) - end + insert(dt, 1, { special=true, ns="", tg="@pi@", dt = { "xml version='1.0' standalone='yes'"} } ) + insert(dt, 2, "\n" ) end end @@ -788,14 +919,14 @@ and then handle the lot.</p> -- new experimental reorganized serialize -local function verbose_element(e,handlers) +local function verbose_element(e,handlers) -- options local handle = handlers.handle local serialize = handlers.serialize local ens, etg, eat, edt, ern = e.ns, e.tg, e.at, e.dt, e.rn local ats = eat and next(eat) and { } if ats then for k,v in next, eat do - ats[#ats+1] = format('%s=%q',k,v) + ats[#ats+1] = format('%s=%q',k,escaped(v)) end end if ern and trace_entities and ern ~= ens then @@ -811,7 +942,7 @@ local function verbose_element(e,handlers) for i=1,#edt do local e = edt[i] if type(e) == "string" then - handle(e) + handle(escaped(e)) else serialize(e,handlers) end @@ -832,11 +963,11 @@ local function verbose_element(e,handlers) handle("<",etg,">") end for i=1,#edt do - local ei = edt[i] - if type(ei) == "string" then - handle(ei) + local e = edt[i] + if type(e) == "string" then + handle(escaped(e)) -- option: hexify escaped entities else - serialize(ei,handlers) + serialize(e,handlers) end end handle("</",etg,">") @@ -871,7 +1002,7 @@ local function verbose_root(e,handlers) end local function verbose_text(e,handlers) - handlers.handle(e) + handlers.handle(escaped(e)) end local function verbose_document(e,handlers) @@ -999,20 +1130,33 @@ local result local xmlfilehandler = newhandlers { name = "file", - initialize = function(name) result = io.open(name,"wb") return result end, - finalize = function() result:close() return true end, - handle = function(...) result:write(...) end, + initialize = function(name) + result = io.open(name,"wb") + return result + end, + finalize = function() + result:close() + return true + end, + handle = function(...) + result:write(...) + end, } -- no checking on writeability here but not faster either -- -- local xmlfilehandler = newhandlers { --- initialize = function(name) io.output(name,"wb") return true end, --- finalize = function() io.close() return true end, +-- initialize = function(name) +-- io.output(name,"wb") +-- return true +-- end, +-- finalize = function() +-- io.close() +-- return true +-- end, -- handle = io.write, -- } - function xml.save(root,name) serialize(root,xmlfilehandler,name) end @@ -1021,28 +1165,34 @@ local result local xmlstringhandler = newhandlers { name = "string", - initialize = function() result = { } return result end, - finalize = function() return concat(result) end, - handle = function(...) result[#result+1] = concat { ... } end + initialize = function() + result = { } + return result + end, + finalize = function() + return concat(result) + end, + handle = function(...) + result[#result+1] = concat { ... } + end, } local function xmltostring(root) -- 25% overhead due to collecting - if root then - if type(root) == 'string' then - return root - else -- if next(root) then -- next is faster than type (and >0 test) - return serialize(root,xmlstringhandler) or "" - end + if not root then + return "" + elseif type(root) == 'string' then + return root + else -- if next(root) then -- next is faster than type (and >0 test) + return serialize(root,xmlstringhandler) or "" end - return "" end -local function xmltext(root) -- inline +local function __tostring(root) -- inline return (root and xmltostring(root)) or "" end initialize_mt = function(root) -- redefinition - mt = { __tostring = xmltext, __index = root } + mt = { __tostring = __tostring, __index = root } end xml.defaulthandlers = handlers @@ -1163,7 +1313,7 @@ xml.tocdata(e,"error") --ldx]]-- function xml.tocdata(e,wrapper) - local whatever = xmltostring(e.dt) + local whatever = type(e) == "table" and xmltostring(e.dt) or e or "" if wrapper then whatever = format("<%s>%s</%s>",wrapper,whatever,wrapper) end |