if not modules then modules = { } end modules ['lxml-tst'] = {
    version   = 1.001,
    comment   = "companion to lxml-ini.mkiv",
    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
    copyright = "PRAGMA ADE / ConTeXt Development Team",
    license   = "see context related readme files"
}

local utf = unicode.utf8

local utfchar = utf.char
local concat, insert, remove, gsub, find = table.concat, table.insert, table.remove
local format, sub, gsub, find, gmatch, match = string.format, string.sub, string.gsub, string.find, string.gmatch, string.match
local type, next, tonumber, tostring = type, next, tonumber, tostring
local lpegmatch = lpeg.match
local P, S, C, Cc = lpeg.P, lpeg.S, lpeg.C, lpeg.Cc

if not tex and not tex.sprint then
    tex = {
        sprint = function(catcodes,...) texio.write(table.concat{...}) end,
        print  = function(catcodes,...) texio.write(table.concat{...}) end,
        write  = function(         ...) texio.write(table.concat{...}) end,
    }
    commands = {
        writestatus = logs.report
    }
    resolvers.loadbinfile = function(filename) return true, io.loaddata(filename) end
end

local texsprint, texprint, texwrite = tex.sprint, tex.print, tex.write
local texcatcodes, ctxcatcodes, vrbcatcodes, notcatcodes = tex.texcatcodes, tex.ctxcatcodes, tex.vrbcatcodes, tex.notcatcodes

local xmlelements, xmlcollected, xmlsetproperty = xml.elements, xml.collected, xml.setproperty
local xmlparseapply, xmlwithelements = xml.parse_apply, xml.withelements
local xmlserialize, xmlcollect, xmltext, xmltostring = xml.serialize, xml.collect, xml.text, xml.tostring

local variables = (interfaces and interfaces.variables) or { }

local starttiming, stoptiming = statistics.starttiming, statistics.stoptiming

local trace_setups   = false  trackers.register("lxml.setups",   function(v) trace_setups   = v end)
local trace_loading  = false  trackers.register("lxml.loading",  function(v) trace_loading  = v end)
local trace_access   = false  trackers.register("lxml.access",   function(v) trace_access   = v end)
local trace_comments = false  trackers.register("lxml.comments", function(v) trace_comments = v end)

lxml              = lxml or { }
lxml.loaded       = lxml.loaded or { }

local loaded = lxml.loaded

-- print(contextdirective("context-mathml-directive function reduction yes "))
-- print(contextdirective("context-mathml-directive function "))

xml.defaultprotocol = "tex"

local finalizers  = xml.finalizers

finalizers.xml = finalizers.xml or { }
finalizers.tex = finalizers.tex or { }

-- this might look inefficient but it's actually rather efficient
-- because we avoid tokenization of leading spaces and xml can be
-- rather verbose (indented)

local newline   = lpeg.patterns.newline
local space     = lpeg.patterns.spacer
local ampersand = P("&")
local semicolon = P(";")
local spacing   = newline * space^0
local content   = C((1-spacing-ampersand)^1)
local verbose   = C((1-(space+newline))^1)
local entity    = ampersand * C((1-semicolon)^1) * semicolon

local xmltextcapture = (
    space^0 * newline^2  * Cc("")            / texprint  + -- better ^-2 ?
    space^0 * newline    * space^0 * Cc(" ") / texsprint +
    content                                  / function(str) return texsprint(notcatcodes,str) end + -- was just texsprint, current catcodes regime is notcatcodes
    entity                                  / xml.resolved_entity
)^0

local ctxtextcapture = (
    space^0 * newline^2  * Cc("")            / texprint  + -- better ^-2 ?
    space^0 * newline    * space^0 * Cc(" ") / texsprint +
    content                                  / function(str) return texsprint(ctxcatcodes,str) end + -- was just texsprint, current catcodes regime is notcatcodes
    entity                                   / xml.resolved_entity
)^0

local forceraw, rawroot = false, nil

function lxml.startraw()
    forceraw = true
end

function lxml.stopraw()
    forceraw = false
end

function lxml.rawroot()
    return rawroot
end

--~ function lxml.rawpath(rootid)
--~     if rawroot and type(rawroot) == "table" then
--~         local text, path, rp
--~         if not rawroot.dt then
--~             text, path, rp = "text", "", rawroot[0]
--~         else
--~             path, rp = "tree", "", rawroot.__p__
--~         end
--~         while rp do
--~             local rptg = rp.tg
--~             if rptg then
--~                 path = rptg .. "/" .. path
--~             end
--~             rp = rp.__p__
--~         end
--~         return { rootid, "/" .. path, text }
--~     end
--~ end

-- cdata

local linecommand   = "\\obeyedline"
local spacecommand  = "\\obeyedspace" -- "\\strut\\obeyedspace"
local beforecommand = ""
local aftercommand  = ""

local xmlverbosecapture = (
    newline / function( ) texsprint(texcatcodes,linecommand,"{}") end +
    verbose / function(s) texsprint(vrbcatcodes,s) end +
    space   / function( ) texsprint(texcatcodes,spacecommand,"{}") end
)^0

local function toverbatim(str)
    if beforecommand then texsprint(texcatcodes,beforecommand,"{}") end
    lpegmatch(xmlverbosecapture,str)
    if aftercommand  then texsprint(texcatcodes,aftercommand,"{}")  end
end

function lxml.set_verbatim(before,after,obeyedline,obeyedspace)
    beforecommand, aftercommand, linecommand, spacecommand = before, after, obeyedline, obeyedspace
end

local obeycdata = true

function lxml.set_cdata()
    obeycdata = true
end

function lxml.reset_cdata()
    obeycdata = false
end

-- cdata and verbatim

lxml.set_verbatim("\\xmlcdatabefore", "\\xmlcdataafter", "\\xmlcdataobeyedline", "\\xmlcdataobeyedspace")

-- local capture = (space^0*newline)^0 * capture * (space+newline)^0 * -1

function lxml.toverbatim(str)
    if beforecommand then texsprint(texcatcodes,beforecommand,"{}") end
    -- todo: add this to capture
    str = gsub(str,"^[ \t]+[\n\r]+","")
    str = gsub(str,"[ \t\n\r]+$","")
    lpegmatch(xmlverbosecapture,str)
    if aftercommand  then texsprint(texcatcodes,aftercommand,"{}")  end
end

-- storage

function lxml.store(id,root,filename)
    loaded[id] = root
    xmlsetproperty(root,"name",id)
    if filename then
        xmlsetproperty(root,"filename",filename)
    end
end

local splitter = lpeg.splitat("::")

lxml.idsplitter = splitter

function lxml.splitid(id)
    local d, i = lpegmatch(splitter,id)
    if d then
        return d, i
    else
        return "", id
    end
end

local function get_id(id, qualified)
    if id then
        local lid = loaded[id]
        if lid then
            return lid
        elseif type(id) == "table" then
            return id
        else
            local d, i = lpegmatch(splitter,id)
            if d then
                local ld = loaded[d]
                if ld then
                    local ldi = ld.index
                    if ldi then
                        local root = ldi[tonumber(i)]
                        if root then
                            if qualified then -- we need this else two args that confuse others
                                return root, d
                            else
                                return root
                            end
                        elseif trace_access then
                            logs.report("lxml","'%s' has no index entry '%s'",d,i)
                        end
                    elseif trace_access then
                        logs.report("lxml","'%s' has no index",d)
                    end
                elseif trace_access then
                    logs.report("lxml","'%s' is not loaded",d)
                end
            elseif trace_access then
                logs.report("lxml","'%s' is not loaded",i)
            end
        end
    elseif trace_access then
        logs.report("lxml","invalid id (nil)")
    end
end

lxml.id     = get_id
lxml.get_id = get_id

function lxml.root(id)
    return loaded[id]
end

-- index

local nofindices = 0

local function addindex(name,check_sum,force)
    local root = get_id(name)
    if root and (not root.index or force) then -- weird, only called once
        local n, index, maxindex, check = 0, root.index or { }, root.maxindex or 0, root.check or { }
        local function nest(root)
            local dt = root.dt
            if not root.ix then
                maxindex = maxindex + 1
                root.ix = maxindex
                check[maxindex] = root.tg -- still needed ?
                index[maxindex] = root
                n = n + 1
            end
            if dt then
                for k=1,#dt do
                    local dk = dt[k]
                    if type(dk) == "table" then
                        nest(dk)
                    end
                end
            end
        end
        nest(root)
        nofindices = nofindices + n
        --
        if type(name) ~= "string" then
            name = "unknown"
        end
        root.index = index
        root.maxindex = maxindex
        if trace_access then
            logs.report("lxml","%s indexed, %s nodes",tostring(name),maxindex)
        end
    end
end

lxml.addindex = addindex

-- another cache

local function lxmlparseapply(id,pattern) -- better inline, saves call
    return xmlparseapply({ get_id(id) }, pattern)
end

lxml.filter = lxmlparseapply

function lxml.filterlist(list,pattern)
    for s in gmatch(list,"[^, ]+") do -- we could cache a table
        lxmlparseapply(s,pattern)
    end
end

lxml["function"] = function(id,name)
    local f = xml.functions[name]
    return f and f(get_id(id))
end

-- rather new, indexed storage (backward refs), maybe i will merge this

function lxml.checkindex(name)
    local root = get_id(name)
    return (root and root.index) or 0
end

function lxml.withindex(name,n,command) -- will change as name is always there now
    local i, p = lpegmatch(splitter,n)
    if p then
        texsprint(ctxcatcodes,"\\xmlw{",command,"}{",n,"}")
    else
        texsprint(ctxcatcodes,"\\xmlw{",command,"}{",name,"::",n,"}")
    end
end

function lxml.getindex(name,n) -- will change as name is always there now
    local i, p = lpegmatch(splitter,n)
    if p then
        texsprint(ctxcatcodes,n)
    else
        texsprint(ctxcatcodes,name,"::",n)
    end
end

-- loading (to be redone, no overload)

xml.originalload = xml.originalload or xml.load

local noffiles, nofconverted = 0, 0

function xml.load(filename,settings)
    noffiles, nofconverted = noffiles + 1, nofconverted + 1
    starttiming(xml)
    local ok, data = resolvers.loadbinfile(filename)
    local xmltable = xml.convert((ok and data) or "",settings)
    stoptiming(xml)
    return xmltable
end

local entities = xml.entities

local function entityconverter(id,str)
    return entities[str] and "&"..str..";" -- feed back into tex end later
end

function lxml.convert(id,data,entities,compress)
    local settings = {
        unify_predefined_entities = true,
    }
    if compress and compress == variables.yes then
        settings.strip_cm_and_dt = true
    end
    if entities and entities == variables.yes then
        settings.utfize_entities = true
        settings.resolve_entities = function (str) return entityconverter(id,str) end
    end
    return xml.convert(data,settings)
end

function lxml.load(id,filename,compress,entities)
    filename = commands.preparedfile(filename)
    if trace_loading then
        commands.writestatus("lxml","loading file '%s' as '%s'",filename,id)
    end
    noffiles, nofconverted = noffiles + 1, nofconverted + 1
 -- local xmltable = xml.load(filename)
    local ok, data = resolvers.loadbinfile(filename)
    local xmltable = lxml.convert(id,(ok and data) or "",compress,entities)
    lxml.store(id,xmltable,filename)
    return xmltable, filename
end

function lxml.register(id,xmltable,filename)
    lxml.store(id,xmltable,filename)
    return xmltable
end

function lxml.include(id,pattern,attribute,recurse)
    starttiming(xml)
    local root = get_id(id)
    xml.include(root,pattern,attribute,recurse,function(filename)
        if filename then
            filename = commands.preparedfile(filename)
            if file.dirname(filename) == "" and root.filename then
                local dn = file.dirname(root.filename)
                if dn ~= "" then
                    filename = file.join(dn,filename)
                end
            end
            if trace_loading then
                commands.writestatus("lxml","including file: %s",filename)
            end
            noffiles, nofconverted = noffiles + 1, nofconverted + 1
            return resolvers.loadtexfile(filename) or ""
        else
            return ""
        end
    end)
    stoptiming(xml)
end

function xml.getbuffer(name,compress,entities) -- we need to make sure that commands are processed
    if not name or name == "" then
        name = tex.jobname
    end
    nofconverted = nofconverted + 1
    xmltostring(lxml.convert(name,concat(buffers.data[name] or {},""),compress,entities))
end

function lxml.loadbuffer(id,name,compress,entities)
    if not name or name == "" then
        name = tex.jobname
    end
    starttiming(xml)
    nofconverted = nofconverted + 1
    local xmltable = lxml.convert(id,buffers.collect(name or id,"\n"),compress,entities)
    lxml.store(id,xmltable)
    stoptiming(xml)
    return xmltable, name or id
end

function lxml.loaddata(id,str,compress,entities)
    starttiming(xml)
    nofconverted = nofconverted + 1
    local xmltable = lxml.convert(id,str or "",compress,entities)
    lxml.store(id,xmltable)
    stoptiming(xml)
    return xmltable, id
end

function lxml.loadregistered(id)
    return loaded[id], id
end

-- e.command:
--
-- string   : setup
-- true     : text (no <self></self>)
-- false    : ignore
-- function : call

local function tex_doctype(e,handlers)
    -- ignore
end

local function tex_comment(e,handlers)
    if trace_comments then
        logs.report("lxml","comment: %s",e.dt[1])
    end
end

local default_element_handler = xml.gethandlers("verbose").functions["@el@"]

local function tex_element(e,handlers)
    local command = e.command
    if command == nil then
        default_element_handler(e,handlers)
    elseif command == true then
        -- text (no <self></self>) / so, no mkii fallback then
        handlers.serialize(e.dt,handlers)
    elseif command == false then
        -- ignore
    else
        local tc = type(command)
        if tc == "string" then
            local rootname, ix = e.name, e.ix
            if rootname then
                if not ix then
                    addindex(rootname,false,true)
                    ix = e.ix
                end
                texsprint(ctxcatcodes,"\\xmlw{",command,"}{",rootname,"::",ix,"}")
            else
                logs.report("lxml", "fatal error: no index for '%s'",command)
                texsprint(ctxcatcodes,"\\xmlw{",command,"}{",ix or 0,"}")
            end
        elseif tc == "function" then
            command(e)
        end
    end
end

local pihandlers = { }  xml.pihandlers = pihandlers

local kind   = P("context-") * C((1-P("-"))^1) * P("-directive")
local space  = S(" \n\r")
local spaces = space^0
local class  = C((1-space)^0)
local key    = class
local value  = C(P(1-(space * -1))^0)

local parser = kind * spaces * class * spaces * key * spaces * value

pihandlers[#pihandlers+1] = function(str)
--  local kind, class, key, value = lpegmatch(parser,str)
    if str then
        local a, b, c, d = lpegmatch(parser,str)
        if d then
            texsprint(ctxcatcodes,"\\xmlcontextdirective{",a",}{",b,"}{",c,"}{",d,"}")
        end
    end
end

local function tex_pi(e,handlers)
    local str = e.dt[1]
    for i=1,#pihandlers do
        pihandlers[i](str)
    end
end

local function tex_cdata(e,handlers)
    if obeycdata then
        toverbatim(e.dt[1])
    end
end

local function tex_text(e)
    lpegmatch(xmltextcapture,e)
end

local function ctx_text(e)
    lpegmatch(ctxtextcapture,e)
end

local function tex_handle(...)
--  logs.report("lxml", "error while flushing: %s", concat { ... })
    texsprint(...) -- notcatcodes is active anyway
end

local xmltexhandler = xml.newhandlers {
    name       = "tex",
    handle     = tex_handle,
    functions  = {
     -- ["@dc@"]   = tex_document,
        ["@dt@"]   = tex_doctype,
     -- ["@rt@"]   = tex_root,
        ["@el@"]   = tex_element,
        ["@pi@"]   = tex_pi,
        ["@cm@"]   = tex_comment,
        ["@cd@"]   = tex_cdata,
        ["@tx@"]   = tex_text,
    }
}

lxml.xmltexhandler = xmltexhandler

function lxml.serialize(root)
    xmlserialize(root,xmltexhandler)
end

function lxml.setaction(id,pattern,action)
    local collected = lxmlparseapply(id,pattern)
    if collected then
        for c=1,#collected do
            collected[c].command = action
        end
    end
end

local function sprint(root)
    if root then
        local tr = type(root)
        if tr == "string" then -- can also be result of lpath
         -- rawroot = false
            lpegmatch(xmltextcapture,root)
        elseif tr == "table" then
            if forceraw then
                rawroot = root
                texwrite(xmltostring(root))
            else
                xmlserialize(root,xmltexhandler)
            end
        end
    end
end

local function tprint(root) -- we can move sprint inline
    local tr = type(root)
    if tr == "table" then
        local n = #root
        if n == 0 then
            -- skip
        else
            for i=1,n do
                sprint(root[i])
            end
        end
    elseif tr == "string" then
        lpegmatch(xmltextcapture,root)
    end
end

local function cprint(root) -- content
    if not root then
     -- rawroot = false
        -- quit
    elseif type(root) == 'string' then
     -- rawroot = false
        lpegmatch(xmltextcapture,root)
    else
        local rootdt = root.dt
        if forceraw then
            rawroot = root
            texwrite(xmltostring(rootdt or root))
        else
            xmlserialize(rootdt or root,xmltexhandler)
        end
    end
end

xml.sprint = sprint local xmlsprint = sprint  -- redo these names
xml.tprint = tprint local xmltprint = tprint
xml.cprint = cprint local xmlcprint = cprint

-- now we can flush

function lxml.main(id)
    xmlserialize(get_id(id),xmltexhandler) -- the real root (@rt@)
end

--~ -- lines (untested)
--~
--~ local buffer = { }
--~
--~ local xmllinescapture = (
--~     newline^2 / function()  buffer[#buffer+1] = "" end +
--~     newline   / function()  buffer[#buffer] = buffer[#buffer] .. " " end +
--~     content   / function(s) buffer[#buffer] = buffer[#buffer] ..  s  end
--~ )^0
--~
--~ local xmllineshandler = table.copy(xmltexhandler)
--~
--~ xmllineshandler.handle = function(...) lpegmatch(xmllinescapture,concat{ ... }) end
--~
--~ function lines(root)
--~     if not root then
--~      -- rawroot = false
--~      -- quit
--~     elseif type(root) == 'string' then
--~      -- rawroot = false
--~         lpegmatch(xmllinescapture,root)
--~     elseif next(root) then -- tr == 'table'
--~         xmlserialize(root,xmllineshandler)
--~     end
--~ end
--~
--~ function xml.lines(root) -- used at all?
--~     buffer = { "" }
--~     lines(root)
--~     return result
--~ end

local function to_text(e)
    if e.command == nil then
        local etg = e.tg
        if etg and e.special and etg ~= "@rt@" then
            e.command = false -- i.e. skip
        else
            e.command = true  -- i.e. no <self></self>
        end
    end
end

local function to_none(e)
    if e.command == nil then
        e.command = false -- i.e. skip
    end
end

-- setups

local setups = { }

function lxml.set_command_to_text(id)
    xmlwithelements(get_id(id),to_text)
end

function lxml.set_command_to_none(id)
    xmlwithelements(get_id(id),to_none)
end

function lxml.installsetup(what,document,setup,where)
    document = document or "*"
    local sd = setups[document]
    if not sd then sd = { } setups[document] = sd end
    for k=1,#sd do
        if sd[k] == setup then sd[k] = nil break end
    end
    if what == 1 then
        if trace_loading then
            commands.writestatus("lxml","prepending setup %s for %s",setup,document)
        end
        insert(sd,1,setup)
    elseif what == 2 then
        if trace_loading then
            commands.writestatus("lxml","appending setup %s for %s",setup,document)
        end
        insert(sd,setup)
    elseif what == 3 then
        if trace_loading then
            commands.writestatus("lxml","inserting setup %s for %s before %s",setup,document,where)
        end
        table.insert_before_value(sd,setup,where)
    elseif what == 4 then
        if trace_loading then
            commands.writestatus("lxml","inserting setup %s for %s after %s",setup,document,where)
        end
        table.insert_after_value(sd,setup,where)
    end
end

function lxml.flushsetups(id,...)
    local done, list = { }, { ... }
    for i=1,#list do
        local document = list[i]
        local sd = setups[document]
        if sd then
            for k=1,#sd do
                local v= sd[k]
                if not done[v] then
                    if trace_loading then
                        commands.writestatus("lxml","applying setup %02i = %s to %s",k,v,document)
                    end
                    texsprint(ctxcatcodes,"\\xmlsetup{",id,"}{",v,"}")
                    done[v] = true
                end
            end
        elseif trace_loading then
            commands.writestatus("lxml","no setups for %s",document)
        end
    end
end

function lxml.resetsetups(document)
    if trace_loading then
        commands.writestatus("lxml","resetting all setups for %s",document)
    end
    setups[document] = { }
end

function lxml.removesetup(document,setup)
    local s = setups[document]
    if s then
        for i=1,#s do
            if s[i] == setup then
                if trace_loading then
                    commands.writestatus("lxml","removing setup %s for %s",setup,document)
                end
                remove(t,i)
                break
            end
        end
    end
end

function lxml.setsetup(id,pattern,setup)
    if not setup or setup == "" or setup == "*" or setup == "-" or setup == "+" then
        local collected = lxmlparseapply(id,pattern)
        if collected then
            if trace_setups then
                for c=1, #collected do
                    local e = collected[c]
                    local ix = e.ix or 0
                    if setup == "-" then
                        e.command = false
                        logs.report("lxml","lpath matched (a) %5i: %s = %s -> skipped",c,ix,setup)
                    elseif setup == "+" then
                        e.command = true
                        logs.report("lxml","lpath matched (b) %5i: %s = %s -> text",c,ix,setup)
                    else
                        local tg = e.tg
                        if tg then -- to be sure
                            e.command = tg
                            local ns = e.rn or e.ns
                            if ns == "" then
                                logs.report("lxml","lpath matched (c) %5i: %s = %s -> %s",c,ix,tg,tg)
                            else
                                logs.report("lxml","lpath matched (d) %5i: %s = %s:%s -> %s",c,ix,ns,tg,tg)
                            end
                        end
                    end
                end
            else
                for c=1, #collected do
                    local e = collected[c]
                    if setup == "-" then
                        e.command = false
                    elseif setup == "+" then
                        e.command = true
                    else
                        e.command = e.tg
                    end
                end
            end
        elseif trace_setups then
            logs.report("lxml","no lpath matches for %s",pattern)
        end
    else
        local a, b = match(setup,"^(.+:)([%*%-])$")
        if a and b then
            local collected = lxmlparseapply(id,pattern)
            if collected then
                if trace_setups then
                    for c=1, #collected do
                        local e = collected[c]
                        local ns, tg, ix = e.rn or e.ns, e.tg, e.ix or 0
                        if b == "-" then
                            e.command = false
                            if ns == "" then
                                logs.report("lxml","lpath matched (e) %5i: %s = %s -> skipped",c,ix,tg)
                            else
                                logs.report("lxml","lpath matched (f) %5i: %s = %s:%s -> skipped",c,ix,ns,tg)
                            end
                        elseif b == "+" then
                            e.command = true
                            if ns == "" then
                                logs.report("lxml","lpath matched (g) %5i: %s = %s -> text",c,ix,tg)
                            else
                                logs.report("lxml","lpath matched (h) %5i: %s = %s:%s -> text",c,ix,ns,tg)
                            end
                        else
                            e.command = a .. tg
                            if ns == "" then
                                logs.report("lxml","lpath matched (i) %5i: %s = %s -> %s",c,ix,tg,e.command)
                            else
                                logs.report("lxml","lpath matched (j) %5i: %s = %s:%s -> %s",c,ix,ns,tg,e.command)
                            end
                        end
                    end
                else
                    for c=1, #collected do
                        local e = collected[c]
                        if b == "-" then
                            e.command = false
                        elseif b == "+" then
                            e.command = true
                        else
                            e.command = a .. e.tg
                        end
                    end
                end
            elseif trace_setups then
                logs.report("lxml","no lpath matches for %s",pattern)
            end
        else
            local collected = lxmlparseapply(id,pattern)
            if collected then
                if trace_setups then
                    for c=1, #collected do
                        local e = collected[c]
                        e.command = setup
                        local ns, tg, ix = e.rn or e.ns, e.tg, e.ix or 0
                        if ns == "" then
                            logs.report("lxml","lpath matched (k) %5i: %s = %s -> %s",c,ix,tg,setup)
                        else
                            logs.report("lxml","lpath matched (l) %5i: %s = %s:%s -> %s",c,ix,ns,tg,setup)
                        end
                    end
                else
                    for c=1, #collected do
                        collected[c].command = setup
                    end
                end
            elseif trace_setups then
                logs.report("lxml","no lpath matches for %s",pattern)
            end
        end
    end
end

-- finalizers

local finalizers = xml.finalizers.tex

local function first(collected)
    if collected then
        xmlsprint(collected[1])
    end
end

local function last(collected)
    if collected then
        xmlsprint(collected[#collected])
    end
end

local function all(collected)
    if collected then
        for c=1,#collected do
            xmlsprint(collected[c])
        end
    end
end

local function reverse(collected)
    if collected then
        for c=#collected,1,-1 do
            xmlsprint(collected[c])
        end
    end
end

local function count(collected)
    texwrite((collected and #collected) or 0)
end

local function position(collected,n)
    -- todo: if not n then == match
    if collected then
        n = tonumber(n) or 0
        if n < 0 then
            n = #collected + n + 1
        end
        if n > 0 then
            xmlsprint(collected[n])
        end
    end
end

local function match(collected) -- is match in preceding collected, never change, see bibxml
    texwrite((collected and collected[1].mi) or 0)
end

local function index(collected,n)
    if collected then
        n = tonumber(n) or 0
        if n < 0 then
            n = #collected + n + 1
        end
        if n > 0 then
            texwrite(collected[n].ni or 0)
        end
    end
end

local function command(collected,cmd)
    if collected then
        for c=1,#collected do
            local e = collected[c]
            local ix = e.ix
            if not ix then
                lxml.addindex(e.name,false,true)
                ix = e.ix
            end
            texsprint(ctxcatcodes,"\\xmlw{",cmd,"}{",e.name,"::",ix,"}")
        end
    end
end

local function attribute(collected,a,default)
    if collected and #collected > 0 then
        local at = collected[1].at
        local str = (at and at[a]) or default
        if str and str ~= "" then
            texsprint(notcatcodes,str)
        end
    elseif default then
        texsprint(notcatcodes,default)
    end
end

local function chainattribute(collected,arguments) -- todo: optional levels
    if collected then
        local e = collected[1]
        while e do
            local at = e.at
            if at then
                local a = at[arguments]
                if a then
                    texsprint(notcatcodes,a)
                end
            else
                break -- error
            end
            e = e.__p__
        end
    end
end

local function text(collected)
    if collected then
        local nc = #collected
        if nc == 1 then -- hardly any gain so this will go
            cprint(collected[1])
        else for c=1,nc do
            cprint(collected[c])
        end end
    end
end

local function ctxtext(collected)
    if collected then
        for c=1,#collected do
            texsprint(ctxcatcodes,collected[1].dt)
        end
    end
end

local lowerchars, upperchars = characters.lower, characters.upper

local function lower(collected)
    if collected then
        for c=1,#collected do
            texsprint(ctxcatcodes,lowerchars(collected[1].dt[1]))
        end
    end
end

local function upper(collected)
    if collected then
        for c=1,#collected do
            texsprint(ctxcatcodes,upperchars(collected[1].dt[1]))
        end
    end
end

local function number(collected)
    if collected then
        local n = 0
        for c=1,#collected do
            n = n + tonumber(collected[c].dt[1] or 0)
        end
        texwrite(n)
    end
end

local function concatrange(collected,start,stop,separator,lastseparator,textonly) -- test this on mml
    if collected then
        local nofcollected = #collected
        local separator = separator or ""
        local lastseparator = lastseparator or separator or ""
        start, stop = (start == "" and 1) or tonumber(start) or 1, (stop == "" and nofcollected) or tonumber(stop) or nofcollected
        if stop < 0 then stop = nofcollected + stop end -- -1 == last-1
        for i=start,stop do
            if textonly then
                xmlcprint(collected[i])
            else
                xmlsprint(collected[i])
            end
            if i == nofcollected then
                -- nothing
            elseif i == nofcollected-1 and lastseparator ~= "" then
                texsprint(ctxcatcodes,lastseparator)
            elseif separator ~= "" then
                texsprint(ctxcatcodes,separator)
            end
        end
    end
end

local function concat(collected,separator,lastseparator,textonly) -- test this on mml
    concatrange(collected,false,false,separator,lastseparator,textonly)
end

finalizers.first          = first
finalizers.last           = last
finalizers.all            = all
finalizers.reverse        = reverse
finalizers.count          = count
finalizers.command        = command
finalizers.attribute      = attribute
finalizers.text           = text
finalizers.lower          = lower
finalizers.upper          = upper
finalizers.ctxtext        = ctxtext
finalizers.context        = ctxtext
finalizers.position       = position
finalizers.match          = match
finalizers.index          = index
finalizers.concat         = concat
finalizers.concatrange    = concatrange
finalizers.chainattribute = chainattribute
finalizers.default        = all -- !!

local concat = table.concat

function finalizers.tag(collected)
    if collected then
        local c
        if n == 0 or not n then
            c = collected[1]
        elseif n > 1 then
            c = collected[n]
        else
            c = collected[#collected-n+1]
        end
        if c then
            texsprint(c.tg)
        end
    end
end

function finalizers.name(collected)
    if collected then
        local c
        if n == 0 or not n then
            c = collected[1]
        elseif n > 1 then
            c = collected[n]
        else
            c = collected[#collected-n+1]
        end
        if c then
            if c.ns == "" then
                texsprint(c.tg)
            else
                texsprint(c.ns,":",c.tg)
            end
        end
    end
end

function finalizers.tags(collected,nonamespace)
    if collected then
        for c=1,#collected do
            local e = collected[c]
            local ns, tg = e.ns, e.tg
            if nonamespace or ns == "" then
                texsprint(tg)
            else
                texsprint(ns,":",tg)
            end
        end
    end
end

--

local function verbatim(id,before,after)
    local root = get_id(id)
    if root then
        if before then texsprint(ctxcatcodes,before,"[",root.tg or "?","]") end
        lxml.toverbatim(xmltostring(root.dt))
        if after then texsprint(ctxcatcodes,after) end
    end
end
function lxml.inlineverbatim(id)
    verbatim(id,"\\startxmlinlineverbatim","\\stopxmlinlineverbatim")
end
function lxml.displayverbatim(id)
    verbatim(id,"\\startxmldisplayverbatim","\\stopxmldisplayverbatim")
end

lxml.verbatim = verbatim

-- helpers

function lxml.first(id,pattern)
    local collected = lxmlparseapply(id,pattern)
    if collected then
        first(collected)
    end
end

function lxml.last(id,pattern)
    local collected = lxmlparseapply(id,pattern)
    if collected then
        last(collected)
    end
end

function lxml.all(id,pattern)
    local collected = lxmlparseapply(id,pattern)
    if collected then
        all(collected)
    end
end

function lxml.count(id,pattern)
    -- always needs to produce a result so no test here
    count(lxmlparseapply(id,pattern))
end

function lxml.attribute(id,pattern,a,default)
    local collected = lxmlparseapply(id,pattern)
    if collected then
        attribute(collected,a,default)
    end
end

function lxml.raw(id,pattern) -- the content, untouched by commands
    local collected = (pattern and lxmlparseapply(id,pattern)) or get_id(id)
    if collected then
        texsprint(xmltostring(collected[1].dt))
    end
end

function lxml.context(id,pattern) -- the content, untouched by commands
    if not pattern then
        local collected = get_id(id)
    --  texsprint(ctxcatcodes,collected.dt[1])
        ctx_text(collected.dt[1])
    else
        local collected = lxmlparseapply(id,pattern) or get_id(id)
        if collected and #collected > 0 then
            texsprint(ctxcatcodes,collected[1].dt)
        end
    end
end

function lxml.text(id,pattern)
    local collected = (pattern and lxmlparseapply(id,pattern)) or get_id(id)
    if collected then
        text(collected)
    end
end

lxml.content = text

function lxml.position(id,pattern,n)
    local collected = lxmlparseapply(id,pattern)
    if collected then
        position(collected,n)
    end
end

function lxml.chainattribute(id,pattern,a,default)
    local collected = lxmlparseapply(id,pattern)
    if collected then
        chainattribute(collected,a,default)
    end
end

function lxml.concatrange(id,pattern,start,stop,separator,lastseparator,textonly) -- test this on mml
    concatrange(lxmlparseapply(id,pattern),start,stop,separator,lastseparator,textonly)
end

function lxml.concat(id,pattern,separator,lastseparator,textonly)
    concatrange(lxmlparseapply(id,pattern),false,false,separator,lastseparator,textonly)
end

function lxml.element(id,n)
    position(lxmlparseapply(id,"/*"),n)
end

lxml.index = lxml.position

function lxml.pos(id)
    local root = get_id(id)
    texwrite((root and root.ni) or 0)
end

function lxml.att(id,a,default)
    local root = get_id(id)
    if root then
        local at = root.at
        local str = (at and at[a]) or default
        if str and str ~= "" then
            texsprint(notcatcodes,str)
        end
    elseif default then
        texsprint(notcatcodes,default)
    end
end

function lxml.name(id) -- or remapped name? -> lxml.info, combine
    local r = get_id(id)
    local ns = r.rn or r.ns or ""
    if ns ~= "" then
        texsprint(ns,":",r.tg)
    else
        texsprint(r.tg)
    end
end

function lxml.match(id) -- or remapped name? -> lxml.info, combine
    texsprint(get_id(id).mi or 0)
end

function lxml.tag(id) -- tag vs name -> also in l-xml tag->name
    texsprint(get_id(id).tg or "")
end

function lxml.namespace(id) -- or remapped name?
    local root = get_id(id)
    texsprint(root.rn or root.ns or "")
end

function lxml.flush(id)
    id = get_id(id)
    local dt = id and id.dt
    if dt then
        xmlsprint(dt)
    end
end

function lxml.snippet(id,i)
    local e = get_id(id)
    if e then
        local edt = e.dt
        if edt then
            xmlsprint(edt[i])
        end
    end
end

function lxml.direct(id)
    xmlsprint(get_id(id))
end

function lxml.command(id,pattern,cmd)
    local i, p = get_id(id,true)
    local collected = lxmlparseapply(i,pattern)
    if collected then
        local rootname = p or i.name
        for c=1,#collected do
            local e = collected[c]
            local ix = e.ix
            if not ix then
                addindex(rootname,false,true)
                ix = e.ix
            end
            texsprint(ctxcatcodes,"\\xmlw{",cmd,"}{",rootname,"::",ix,"}")
        end
    end
end

-- loops

function lxml.collected(id,pattern,reverse)
    return xmlcollected(get_id(id),pattern,reverse)
end

function lxml.elements(id,pattern,reverse)
    return xmlelements(get_id(id),pattern,reverse)
end

-- obscure ones

lxml.info = lxml.name

-- testers

local found, empty = xml.found, xml.empty

local doif, doifnot, doifelse = commands.doif, commands.doifnot, commands.doifelse

function lxml.doif         (id,pattern) doif    (found(get_id(id),pattern)) end
function lxml.doifnot      (id,pattern) doifnot (found(get_id(id),pattern)) end
function lxml.doifelse     (id,pattern) doifelse(found(get_id(id),pattern)) end
function lxml.doiftext     (id,pattern) doif    (not empty(get_id(id),pattern)) end
function lxml.doifnottext  (id,pattern) doifnot (not empty(get_id(id),pattern)) end
function lxml.doifelsetext (id,pattern) doifelse(not empty(get_id(id),pattern)) end

-- special case: "*" and "" -> self else lpath lookup

--~ function lxml.doifelseempty(id,pattern) doifelse(isempty(get_id(id),pattern ~= "" and pattern ~= nil)) end -- not yet done, pattern

-- status info

statistics.register("xml load time", function()
    if noffiles > 0 or nofconverted > 0 then
        return format("%s seconds, %s files, %s converted", statistics.elapsedtime(xml), noffiles, nofconverted)
    else
        return nil
    end
end)

statistics.register("lxml preparation time", function()
    local calls, cached = xml.lpathcalls(), xml.lpathcached()
    if calls > 0 or cached > 0 then
        return format("%s seconds, %s nodes, %s lpath calls, %s cached calls",
            statistics.elapsedtime(lxml), nofindices, calls, cached)
    else
        return nil
    end
end)

statistics.register("lxml lpath profile", function()
    local p = xml.profiled
    if p and next(p) then
        local s = table.sortedkeys(p)
        local tested, matched, finalized = 0, 0, 0
        texio.write_nl("log","\nbegin of lxml profile\n")
        texio.write_nl("log","\n   tested    matched  finalized    pattern\n\n")
        for i=1,#s do
            local pattern = s[i]
            local pp = p[pattern]
            local t, m, f = pp.tested, pp.matched, pp.finalized
            tested, matched, finalized = tested + t, matched + m, finalized + f
            texio.write_nl("log",format("%9i  %9i  %9i    %s",t,m,f,pattern))
        end
        texio.write_nl("log","\nend of lxml profile\n")
        return format("%s patterns, %s tested, %s matched, %s finalized (see log for details)",#s,tested,matched,finalized)
    else
        return nil
    end
end)

-- misc

function lxml.nonspace(id,pattern) -- slow, todo loop
    xmltprint(xmlcollect(get_id(id),pattern,true))
end

function lxml.strip(id,pattern,nolines,anywhere)
    xml.strip(get_id(id),pattern,nolines,anywhere)
end

function lxml.stripped(id,pattern,nolines)
    local str = xmltext(get_id(id),pattern) or ""
    str = gsub(str,"^%s*(.-)%s*$","%1")
    if nolines then
        str = gsub(str,"%s+"," ")
    end
    xmlsprint(str)
end

function lxml.delete(id,pattern)
    xml.delete(get_id(id),pattern)
end