summaryrefslogtreecommitdiff
path: root/Master/texmf-dist/tex/context/base/lxml-tab.lua
diff options
context:
space:
mode:
Diffstat (limited to 'Master/texmf-dist/tex/context/base/lxml-tab.lua')
-rw-r--r--Master/texmf-dist/tex/context/base/lxml-tab.lua308
1 files changed, 229 insertions, 79 deletions
diff --git a/Master/texmf-dist/tex/context/base/lxml-tab.lua b/Master/texmf-dist/tex/context/base/lxml-tab.lua
index 7d38a8d5abc..0a43f435274 100644
--- a/Master/texmf-dist/tex/context/base/lxml-tab.lua
+++ b/Master/texmf-dist/tex/context/base/lxml-tab.lua
@@ -10,6 +10,10 @@ if not modules then modules = { } end modules ['lxml-tab'] = {
-- stripping spaces from e.g. cont-en.xml saves .2 sec runtime so it's not worth the
-- trouble
+-- todo: when serializing optionally remap named entities to hex (if known in char-ent.lua)
+-- maybe when letter -> utf, else name .. then we need an option to the serializer .. a bit
+-- of work so we delay this till we cleanup
+
local trace_entities = false trackers.register("xml.entities", function(v) trace_entities = v end)
local report_xml = logs and logs.reporter("xml","core") or function(...) print(format(...)) end
@@ -35,10 +39,11 @@ local xml = xml
--~ local xml = xml
+local utf = unicode.utf8
local concat, remove, insert = table.concat, table.remove, table.insert
local type, next, setmetatable, getmetatable, tonumber = type, next, setmetatable, getmetatable, tonumber
local format, lower, find, match, gsub = string.format, string.lower, string.find, string.match, string.gsub
-local utfchar = unicode.utf8.char
+local utfchar, utffind, utfgsub = utf.char, utf.find, utf.gsub
local lpegmatch = lpeg.match
local P, S, R, C, V, C, Cs = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.C, lpeg.Cs
@@ -151,9 +156,22 @@ element.</p>
local nsremap, resolvens = xml.xmlns, xml.resolvens
-local stack, top, dt, at, xmlns, errorstr, entities = { }, { }, { }, { }, { }, nil, { }
-local strip, cleanup, utfize, resolve, resolve_predefined, unify_predefined = false, false, false, false, false, false
-local dcache, hcache, acache = { }, { }, { }
+local stack = { }
+local top = { }
+local dt = { }
+local at = { }
+local xmlns = { }
+local errorstr = nil
+local entities = { }
+local strip = false
+local cleanup = false
+local utfize = false
+local resolve_predefined = false
+local unify_predefined = false
+
+local dcache = { }
+local hcache = { }
+local acache = { }
local mt = { }
@@ -319,7 +337,7 @@ local predefined_unified = {
[42] = "&quot;",
[47] = "&apos;",
[74] = "&lt;",
- [76] = "&gr;",
+ [76] = "&gt;",
}
local predefined_simplified = {
@@ -330,6 +348,58 @@ local predefined_simplified = {
[76] = ">", gt = ">",
}
+local nofprivates = 0xF0000 -- shared but seldom used
+
+local privates_u = { -- unescaped
+ [ [[&]] ] = "&amp;",
+ [ [["]] ] = "&quot;",
+ [ [[']] ] = "&apos;",
+ [ [[<]] ] = "&lt;",
+ [ [[>]] ] = "&gt;",
+}
+
+local privates_p = {
+}
+
+local privates_n = {
+ -- keeps track of defined ones
+}
+
+local function escaped(s)
+ if s == "" then
+ return ""
+ else -- if utffind(s,privates_u) then
+ return (utfgsub(s,".",privates_u))
+ -- else
+ -- return s
+ end
+end
+
+local function unescaped(s)
+ local p = privates_n[s]
+ if not p then
+ nofprivates = nofprivates + 1
+ p = utfchar(nofprivates)
+ privates_n[s] = p
+ s = "&" .. s .. ";" -- todo: use char-ent to map to hex
+ privates_u[p] = s
+ privates_p[p] = s
+ end
+ return p
+end
+
+local function unprivatized(s,resolve)
+ if s == "" then
+ return ""
+ else
+ return (utfgsub(s,".",privates_p))
+ end
+end
+
+xml.privatetoken = unescaped
+xml.unprivatized = unprivatized
+xml.privatecodes = privates_n
+
local function handle_hex_entity(str)
local h = hcache[str]
if not h then
@@ -371,7 +441,7 @@ local function handle_dec_entity(str)
if not n then
report_xml("utfize, ignoring dec entity &#%s;",str)
elseif trace_entities then
- report_xml("utfize, converting dec entity &#%s; into %s",str,h)
+ report_xml("utfize, converting dec entity &#%s; into %s",str,d)
end
else
if trace_entities then
@@ -392,34 +462,44 @@ local function handle_any_entity(str)
if not a then
a = resolve_predefined and predefined_simplified[str]
if a then
- -- one of the predefined
- elseif type(resolve) == "function" then
- a = resolve(str) or entities[str]
- else
- a = entities[str]
- end
- if a then
if trace_entities then
- report_xml("resolved entity &%s; -> %s (internal)",str,a)
+ report_xml("resolved entity &%s; -> %s (predefined)",str,a)
end
- a = lpegmatch(parsedentity,a) or a
else
- local unknown_any_entity = placeholders.unknown_any_entity
- if unknown_any_entity then
- a = unknown_any_entity(str) or ""
+ if type(resolve) == "function" then
+ a = resolve(str) or entities[str]
+ else
+ a = entities[str]
end
if a then
+ if type(a) == "function" then
+ if trace_entities then
+ report_xml("expanding entity &%s; (function)",str)
+ end
+ a = a(str) or ""
+ end
+ a = lpegmatch(parsedentity,a) or a -- for nested
if trace_entities then
- report_xml("resolved entity &%s; -> %s (external)",str,a)
+ report_xml("resolved entity &%s; -> %s (internal)",str,a)
end
else
- if trace_entities then
- report_xml("keeping entity &%s;",str)
+ local unknown_any_entity = placeholders.unknown_any_entity
+ if unknown_any_entity then
+ a = unknown_any_entity(str) or ""
end
- if str == "" then
- a = "&error;"
+ if a then
+ if trace_entities then
+ report_xml("resolved entity &%s; -> %s (external)",str,a)
+ end
else
- a = "&" .. str .. ";"
+ if trace_entities then
+ report_xml("keeping entity &%s;",str)
+ end
+ if str == "" then
+ a = "&error;"
+ else
+ a = "&" .. str .. ";"
+ end
end
end
end
@@ -434,18 +514,25 @@ local function handle_any_entity(str)
else
local a = acache[str]
if not a then
- if trace_entities then
- report_xml("found entity &%s;",str)
- end
a = resolve_predefined and predefined_simplified[str]
if a then
-- one of the predefined
acache[str] = a
+ if trace_entities then
+ report_xml("entity &%s; becomes %s",str,tostring(a))
+ end
elseif str == "" then
+ if trace_entities then
+ report_xml("invalid entity &%s;",str)
+ end
a = "&error;"
acache[str] = a
else
- a = "&" .. str .. ";"
+ if trace_entities then
+ report_xml("entity &%s; is made private",str)
+ end
+ -- a = "&" .. str .. ";"
+ a = unescaped(str)
acache[str] = a
end
end
@@ -495,7 +582,7 @@ local value = (squote * Cs((entity + (1 - squote))^0) * squote) + (dq
local endofattributes = slash * close + close -- recovery of flacky html
local whatever = space * name * optionalspace * equal
-local wrongvalue = C(P(1-whatever-close)^1 + P(1-close)^1) / attribute_value_error
+----- wrongvalue = C(P(1-whatever-close)^1 + P(1-close)^1) / attribute_value_error
----- wrongvalue = C(P(1-whatever-endofattributes)^1 + P(1-endofattributes)^1) / attribute_value_error
----- wrongvalue = C(P(1-space-endofattributes)^1) / attribute_value_error
local wrongvalue = Cs(P(entity + (1-space-endofattributes))^1) / attribute_value_error
@@ -530,6 +617,8 @@ local function normalentity(k,v ) entities[k] = v end
local function systementity(k,v,n) entities[k] = v end
local function publicentity(k,v,n) entities[k] = v end
+-- todo: separate dtd parser
+
local begindoctype = open * P("!DOCTYPE")
local enddoctype = close
local beginset = P("[")
@@ -537,17 +626,22 @@ local endset = P("]")
local doctypename = C((1-somespace-close)^0)
local elementdoctype = optionalspace * P("<!ELEMENT") * (1-close)^0 * close
+local basiccomment = begincomment * ((1 - endcomment)^0) * endcomment
+
local normalentitytype = (doctypename * somespace * value)/normalentity
local publicentitytype = (doctypename * somespace * P("PUBLIC") * somespace * value)/publicentity
local systementitytype = (doctypename * somespace * P("SYSTEM") * somespace * value * somespace * P("NDATA") * somespace * doctypename)/systementity
local entitydoctype = optionalspace * P("<!ENTITY") * somespace * (systementitytype + publicentitytype + normalentitytype) * optionalspace * close
-local doctypeset = beginset * optionalspace * P(elementdoctype + entitydoctype + space)^0 * optionalspace * endset
+-- we accept comments in doctypes
+
+local doctypeset = beginset * optionalspace * P(elementdoctype + entitydoctype + basiccomment + space)^0 * optionalspace * endset
local definitiondoctype= doctypename * somespace * doctypeset
local publicdoctype = doctypename * somespace * P("PUBLIC") * somespace * value * somespace * value * somespace * doctypeset
local systemdoctype = doctypename * somespace * P("SYSTEM") * somespace * value * somespace * doctypeset
local simpledoctype = (1-close)^1 -- * balanced^0
local somedoctype = C((somespace * (publicdoctype + systemdoctype + definitiondoctype + simpledoctype) * optionalspace)^0)
+local somedoctype = C((somespace * (publicdoctype + systemdoctype + definitiondoctype + simpledoctype) * optionalspace)^0)
local instruction = (spacing * begininstruction * someinstruction * endinstruction) / function(...) add_special("@pi@",...) end
local comment = (spacing * begincomment * somecomment * endcomment ) / function(...) add_special("@cm@",...) end
@@ -579,17 +673,31 @@ local grammar_unparsed_text = P { "preamble",
children = unparsedtext + V("parent") + emptyelement + comment + cdata + instruction,
}
--- maybe we will add settinsg to result as well
+-- maybe we will add settings to result as well
-local function xmlconvert(data, settings)
- settings = settings or { } -- no_root strip_cm_and_dt given_entities parent_root error_handler
- strip = settings.strip_cm_and_dt
- utfize = settings.utfize_entities
- resolve = settings.resolve_entities
+local function _xmlconvert_(data, settings)
+ settings = settings or { } -- no_root strip_cm_and_dt given_entities parent_root error_handler
+ --
+ strip = settings.strip_cm_and_dt
+ utfize = settings.utfize_entities
+ resolve = settings.resolve_entities
resolve_predefined = settings.resolve_predefined_entities -- in case we have escaped entities
- unify_predefined = settings.unify_predefined_entities -- &#038; -> &amp;
- cleanup = settings.text_cleanup
- stack, top, at, xmlns, errorstr, entities = { }, { }, { }, { }, nil, settings.entities or { }
+ unify_predefined = settings.unify_predefined_entities -- &#038; -> &amp;
+ cleanup = settings.text_cleanup
+ entities = settings.entities or { }
+ --
+ if utfize == nil then
+ settings.utfize_entities = true
+ utfize = true
+ end
+ if resolve_predefined == nil then
+ settings.resolve_predefined_entities = true
+ resolve_predefined = true
+ end
+ --
+--~ inspect(settings)
+ --
+ stack, top, at, xmlns, errorstr = { }, { }, { }, { }, nil
acache, hcache, dcache = { }, { }, { } -- not stored
reported_attribute_errors = { }
if settings.parent_root then
@@ -627,7 +735,7 @@ local function xmlconvert(data, settings)
else
errorhandler = errorhandler or xml.errorhandler
if errorhandler then
- xml.errorhandler("load",errorstr)
+ xml.errorhandler(format("load error: %s",errorstr))
end
end
else
@@ -641,7 +749,7 @@ local function xmlconvert(data, settings)
local v = rdt[k]
if type(v) == "table" and not v.special then -- always table -)
result.ri = k -- rootindex
-v.__p__ = result -- new, experiment, else we cannot go back to settings, we need to test this !
+ v.__p__ = result -- new, experiment, else we cannot go back to settings, we need to test this !
break
end
end
@@ -649,16 +757,42 @@ v.__p__ = result -- new, experiment, else we cannot go back to settings, we nee
if errorstr and errorstr ~= "" then
result.error = true
end
+ result.statistics = {
+ entities = {
+ decimals = dcache,
+ hexadecimals = hcache,
+ names = acache,
+ }
+ }
+ strip, utfize, resolve, resolve_predefined = nil, nil, nil, nil
+ unify_predefined, cleanup, entities = nil, nil, nil
+ stack, top, at, xmlns, errorstr = nil, nil, nil, nil, nil
+ acache, hcache, dcache = nil, nil, nil
+ reported_attribute_errors, mt, errorhandler = nil, nil, nil
return result
end
+-- Because we can have a crash (stack issues) with faulty xml, we wrap this one
+-- in a protector:
+
+function xmlconvert(data,settings)
+ local ok, result = pcall(function() return _xmlconvert_(data,settings) end)
+ if ok then
+ return result
+ else
+ return _xmlconvert_("")
+ end
+end
+
xml.convert = xmlconvert
-function xml.inheritedconvert(data,xmldata)
+function xml.inheritedconvert(data,xmldata) -- xmldata is parent
local settings = xmldata.settings
- settings.parent_root = xmldata -- to be tested
+ if settings then
+ settings.parent_root = xmldata -- to be tested
+ end
-- settings.no_root = true
- local xc = xmlconvert(data,settings)
+ local xc = xmlconvert(data,settings) -- hm, we might need to locate settings
-- xc.settings = nil
-- xc.entities = nil
-- xc.special = nil
@@ -687,7 +821,7 @@ function xml.is_valid(root)
return root and not root.error
end
-xml.errorhandler = report
+xml.errorhandler = report_xml
--[[ldx--
<p>We cannot load an <l n='lpeg'/> from a filehandle so we need to load
@@ -766,18 +900,15 @@ alternative.</p>
function xml.checkbom(root) -- can be made faster
if root.ri then
- local dt, found = root.dt, false
+ local dt = root.dt
for k=1,#dt do
local v = dt[k]
if type(v) == "table" and v.special and v.tg == "@pi@" and find(v.dt[1],"xml.*version=") then
- found = true
- break
+ return
end
end
- if not found then
- insert(dt, 1, { special=true, ns="", tg="@pi@", dt = { "xml version='1.0' standalone='yes'"} } )
- insert(dt, 2, "\n" )
- end
+ insert(dt, 1, { special=true, ns="", tg="@pi@", dt = { "xml version='1.0' standalone='yes'"} } )
+ insert(dt, 2, "\n" )
end
end
@@ -788,14 +919,14 @@ and then handle the lot.</p>
-- new experimental reorganized serialize
-local function verbose_element(e,handlers)
+local function verbose_element(e,handlers) -- options
local handle = handlers.handle
local serialize = handlers.serialize
local ens, etg, eat, edt, ern = e.ns, e.tg, e.at, e.dt, e.rn
local ats = eat and next(eat) and { }
if ats then
for k,v in next, eat do
- ats[#ats+1] = format('%s=%q',k,v)
+ ats[#ats+1] = format('%s=%q',k,escaped(v))
end
end
if ern and trace_entities and ern ~= ens then
@@ -811,7 +942,7 @@ local function verbose_element(e,handlers)
for i=1,#edt do
local e = edt[i]
if type(e) == "string" then
- handle(e)
+ handle(escaped(e))
else
serialize(e,handlers)
end
@@ -832,11 +963,11 @@ local function verbose_element(e,handlers)
handle("<",etg,">")
end
for i=1,#edt do
- local ei = edt[i]
- if type(ei) == "string" then
- handle(ei)
+ local e = edt[i]
+ if type(e) == "string" then
+ handle(escaped(e)) -- option: hexify escaped entities
else
- serialize(ei,handlers)
+ serialize(e,handlers)
end
end
handle("</",etg,">")
@@ -871,7 +1002,7 @@ local function verbose_root(e,handlers)
end
local function verbose_text(e,handlers)
- handlers.handle(e)
+ handlers.handle(escaped(e))
end
local function verbose_document(e,handlers)
@@ -999,20 +1130,33 @@ local result
local xmlfilehandler = newhandlers {
name = "file",
- initialize = function(name) result = io.open(name,"wb") return result end,
- finalize = function() result:close() return true end,
- handle = function(...) result:write(...) end,
+ initialize = function(name)
+ result = io.open(name,"wb")
+ return result
+ end,
+ finalize = function()
+ result:close()
+ return true
+ end,
+ handle = function(...)
+ result:write(...)
+ end,
}
-- no checking on writeability here but not faster either
--
-- local xmlfilehandler = newhandlers {
--- initialize = function(name) io.output(name,"wb") return true end,
--- finalize = function() io.close() return true end,
+-- initialize = function(name)
+-- io.output(name,"wb")
+-- return true
+-- end,
+-- finalize = function()
+-- io.close()
+-- return true
+-- end,
-- handle = io.write,
-- }
-
function xml.save(root,name)
serialize(root,xmlfilehandler,name)
end
@@ -1021,28 +1165,34 @@ local result
local xmlstringhandler = newhandlers {
name = "string",
- initialize = function() result = { } return result end,
- finalize = function() return concat(result) end,
- handle = function(...) result[#result+1] = concat { ... } end
+ initialize = function()
+ result = { }
+ return result
+ end,
+ finalize = function()
+ return concat(result)
+ end,
+ handle = function(...)
+ result[#result+1] = concat { ... }
+ end,
}
local function xmltostring(root) -- 25% overhead due to collecting
- if root then
- if type(root) == 'string' then
- return root
- else -- if next(root) then -- next is faster than type (and >0 test)
- return serialize(root,xmlstringhandler) or ""
- end
+ if not root then
+ return ""
+ elseif type(root) == 'string' then
+ return root
+ else -- if next(root) then -- next is faster than type (and >0 test)
+ return serialize(root,xmlstringhandler) or ""
end
- return ""
end
-local function xmltext(root) -- inline
+local function __tostring(root) -- inline
return (root and xmltostring(root)) or ""
end
initialize_mt = function(root) -- redefinition
- mt = { __tostring = xmltext, __index = root }
+ mt = { __tostring = __tostring, __index = root }
end
xml.defaulthandlers = handlers
@@ -1163,7 +1313,7 @@ xml.tocdata(e,"error")
--ldx]]--
function xml.tocdata(e,wrapper)
- local whatever = xmltostring(e.dt)
+ local whatever = type(e) == "table" and xmltostring(e.dt) or e or ""
if wrapper then
whatever = format("<%s>%s</%s>",wrapper,whatever,wrapper)
end