context from May 18 beta/cont-tmf.zip (11854476 bytes)

git-svn-id: svn://tug.org/texlive/trunk@34112 c570f23f-e606-0410-a88d-b1316a301751

1 files changed, 814 insertions, 462 deletions

diff --git a/Master/texmf-dist/tex/context/base/publ-ini.lua b/Master/texmf-dist/tex/context/base/publ-ini.lua
index e25c57e297d..d713f552cb5 100644
--- a/Master/texmf-dist/tex/context/base/publ-ini.lua
+++ b/Master/texmf-dist/tex/context/base/publ-ini.lua
@@ -6,10 +6,12 @@ if not modules then modules = { } end modules ['publ-ini'] = {
     license   = "see context related readme files"
 }
 
--- for the moment here
+-- we could store the destinations in the user list entries
+
+-- will move:
 
 local lpegmatch  = lpeg.match
-local P, C, Ct, Cs = lpeg.P, lpeg.C, lpeg.Ct, lpeg.Cs
+local P, R, C, Ct, Cs = lpeg.P, lpeg.R, lpeg.C, lpeg.Ct, lpeg.Cs
 
 local lpegmatch  = lpeg.match
 local pattern    = Cs((1 - P(1) * P(-1))^0 * (P(".")/"" + P(1)))
@@ -41,19 +43,26 @@ end
 
 -- use: for rest in gmatch(reference,"[^, ]+") do
 
-local next, rawget, type = next, rawget, type
+local next, rawget, type, tostring, tonumber = next, rawget, type, tostring, tonumber
 local match, gmatch, format, gsub = string.match, string.gmatch, string.format, string.gsub
-local concat, sort = table.concat, table.sort
+local concat, sort, tohash = table.concat, table.sort, table.tohash
 local utfsub = utf.sub
 local formatters = string.formatters
 local allocate = utilities.storage.allocate
 local settings_to_array, settings_to_set = utilities.parsers.settings_to_array, utilities.parsers.settings_to_set
 local sortedkeys, sortedhash = table.sortedkeys, table.sortedhash
+local setmetatableindex = table.setmetatableindex
 local lpegmatch = lpeg.match
 local P, C, Ct = lpeg.P, lpeg.C, lpeg.Ct
 
-local report         = logs.reporter("publications")
-local trace          = false  trackers.register("publications", function(v) trace = v end)
+local report           = logs.reporter("publications")
+local report_cite      = logs.reporter("publications","cite")
+local report_reference = logs.reporter("publications","reference")
+
+local trace            = false  trackers.register("publications",                 function(v) trace            = v end)
+local trace_cite       = false  trackers.register("publications.cite",            function(v) trace_cite       = v end)
+local trace_missing    = false  trackers.register("publications.cite.missing",    function(v) trace_missing    = v end)
+local trace_references = false  trackers.register("publications.cite.references", function(v) trace_references = v end)
 
 local datasets       = publications.datasets
 
@@ -70,7 +79,7 @@ local v_left         = variables.left
 local v_right        = variables.right
 local v_middle       = variables.middle
 local v_inbetween    = variables.inbetween
-
+local v_yes          = variables.yes
 local v_short        = variables.short
 local v_cite         = variables.cite
 local v_default      = variables.default
@@ -91,6 +100,8 @@ local sortcomparer   = sorters.comparers.basic -- (a,b)
 local sortstripper   = sorters.strip
 local sortsplitter   = sorters.splitters.utf
 
+local settings_to_array = utilities.parsers.settings_to_array
+
 local context                     = context
 
 local ctx_btxlistparameter        = context.btxlistparameter
@@ -105,8 +116,20 @@ local ctx_gobbleoneargument       = context.gobbleoneargument
 local ctx_btxdirectlink           = context.btxdirectlink
 local ctx_btxhandlelistentry      = context.btxhandlelistentry
 local ctx_btxchecklistentry       = context.btxchecklistentry
-local ctx_dodirectfullreference   = context.dodirectfullreference
+local ctx_btxchecklistcombi       = context.btxchecklistcombi
+----- ctx_dodirectfullreference   = context.dodirectfullreference
+local ctx_btxsetreference         = context.btxsetreference
 local ctx_directsetup             = context.directsetup
+local ctx_btxmissing              = context.btxmissing
+
+local ctx_btxsettag               = context.btxsettag
+local ctx_btxcitesetup            = context.btxcitesetup
+local ctx_btxsetfirst             = context.btxsetfirst
+local ctx_btxsetsecond            = context.btxsetsecond
+local ctx_btxsetinternal          = context.btxsetinternal
+local ctx_btxsetconcat            = context.btxsetconcat
+local ctx_btxstartsubcite         = context.btxstartsubcite
+local ctx_btxstopsubcite          = context.btxstopsubcite
 
 statistics.register("publications load time", function()
     local publicationsstats = publications.statistics
@@ -249,6 +272,197 @@ if not publications.authors then
     initializer() -- for now, runtime loaded
 end
 
+local usedentries = { }
+local initialized = false
+
+--  {
+--   ["metadata"]=1,
+--   ["references"]={
+--    ["block"]="bodypart",
+--    ["internal"]=2,
+--    ["realpage"]=1,
+--    ["section"]=0,
+--   },
+--   ["userdata"]={
+--    ["btxref"]="Cleveland1985",
+--    ["btxset"]="standard",
+--   },
+--  },
+
+setmetatableindex(usedentries,function(t,k)
+    if not initialized then
+        usedentries = { }
+        local internals = structures.references.internals
+        for i=1,#internals do
+            local entry = internals[i]
+            local metadata = entry.metadata
+            if metadata.kind == "full" then
+                local userdata = entry.userdata
+                if userdata then
+                    local set = userdata.btxset
+                    if set then
+                        local tag = userdata.btxref
+                        local s = usedentries[set]
+                        if s then
+                            local u = s[tag]
+                            if u then
+                                u[#u+1] = entry
+                            else
+                                u = { entry }
+                                s[tag] = u
+                            end
+                        else
+                            usedentries[set] = { [tag] = { entry } }
+                        end
+                    end
+                end
+            end
+        end
+        return usedentries[k]
+    end
+end)
+
+--     local subsets   = nil
+--     local block     = tex.count.btxblock
+--     local collected = references.collected
+--     local prefix    = nil -- todo: dataset ?
+--     if prefix and prefix ~= "" then
+--         subsets = { collected[prefix] or collected[""] }
+--     else
+--         local components = references.productdata.components
+--         local subset = collected[""]
+--         if subset then
+--             subsets = { subset }
+--         else
+--             subsets = { }
+--         end
+--         for i=1,#components do
+--             local subset = collected[components[i]]
+--             if subset then
+--                 subsets[#subsets+1] = subset
+--             end
+--         end
+--     end
+--     if #subsets == 0 then
+--         subsets = { collected[""] }
+--     end
+--     local list = type(reference) == "string" and settings_to_array(reference) or reference
+--     local todo = table.tohash(list)
+--     if #subsets > 0 then
+--         local result, nofresult, done = { }, 0, { }
+--         for i=1,#subsets do
+--             local subset = subsets[i]
+--             for i=1,#list do
+--                 local rest = list[i]
+--                 local blk, tag, found = block, nil, nil
+--                 if block then
+--                     tag = f_destination(dataset,blk,rest)
+--                     found = subset[tag]
+--                     if not found then
+--                         for i=block-1,1,-1 do
+--                             tag = f_destination(dataset,i,rest)
+--                             found = subset[tag]
+--                             if found then
+--                                 blk = i
+--                                 break
+--                             end
+--                         end
+--                     end
+--                 end
+--                 if not found then
+--                     blk = "*"
+--                     tag = f_destination(dataset,blk,rest)
+--                     found = subset[tag]
+--                 end
+--                 if found then
+--                     local entries = found.entries
+--                     if entries then
+--                         local current = tonumber(entries.text) -- todo: no ranges when no tonumber
+--                         if current and not done[current] then
+--                             nofresult = nofresult + 1
+--                             result[nofresult] = { blk, rest, current, found.references.internal }
+--                             done[current] = true
+--                         end
+--                     end
+--                 end
+--             end
+--         end
+
+local reported = { }
+
+local function findallused(dataset,reference,block,section)
+    local tags  = settings_to_array(reference)
+    local todo  = { }
+    local okay  = { } -- only if mark
+    local set   = usedentries[dataset]
+    local valid = datasets[dataset].luadata
+    if set then
+        for i=1,#tags do
+            local tag = tags[i]
+            if valid[tag] then
+                local entry = set[tag]
+                if entry then
+                    -- only once in a list
+                    if #entry == 1 then
+                        entry = entry[1]
+                    else
+                        -- find best match (todo)
+                        entry = entry[1] -- for now
+                    end
+                    okay[#okay+1] = entry
+                end
+                todo[tag] = true
+            elseif not reported[tag] then
+                reported[tag] = true
+                report_cite("non-existent entry %a in %a",tag,dataset)
+            end
+        end
+    else
+        for i=1,#tags do
+            local tag = tags[i]
+            if valid[tag] then
+                todo[tag] = true
+            elseif not reported[tag] then
+                reported[tag] = true
+                report_cite("non-existent entry %a in %a",tag,dataset)
+            end
+        end
+    end
+    return okay, todo, tags
+end
+
+local function flushcollected(flush,nofcollected)
+    if nofcollected > 0 then
+        flush(1,1)
+        if nofcollected > 2 then
+            for i=2,nofcollected-1 do
+                flush(i,2)
+            end
+            flush(nofcollected,3)
+        elseif nofcollected > 1 then
+            flush(nofcollected,4)
+        end
+    end
+end
+
+local function markcite(dataset,tag)
+    if trace_cite then
+        report_cite("mark, dataset: %s, tag: %s",dataset,tag)
+    end
+    ctx_btxdomarkcitation(dataset,tag)
+end
+
+local function flushmarked(dataset,list,todo)
+    if todo then
+        for i=1,#list do
+            local tag = list[i]
+            if todo[tag] then
+                markcite(dataset,tag)
+            end
+        end
+    end
+end
+
 -- basic access
 
 local function getfield(dataset,tag,name)
@@ -344,10 +558,10 @@ function publications.enhance(dataset) -- for the moment split runs (maybe publi
         if type(tags) == "table" then
             sort(tags)
             for i=1,#tags do
---                 details[tags[i]].short = short .. numbertochar(i)
-local detail = details[tags[i]]
-detail.short  = short
-detail.suffix = numbertochar(i)
+             -- details[tags[i]].short = short .. numbertochar(i)
+                local detail = details[tags[i]]
+                detail.short  = short
+                detail.suffix = numbertochar(i)
             end
         else
             details[tags].short = short
@@ -402,9 +616,9 @@ end
 
 -- rendering of fields (maybe multiple manipulators)
 
-local manipulation = utilities.parsers.manipulation
-local manipulators = utilities.parsers.manipulators
-
+-- local manipulation = utilities.parsers.manipulation
+-- local manipulators = utilities.parsers.manipulators
+--
 -- local function checked(field)
 --     local m, f = lpegmatch(manipulation,field)
 --     if m then
@@ -415,6 +629,7 @@ local manipulators = utilities.parsers.manipulators
 -- end
 
 local manipulation = Ct((C((1-P("->"))^1) * P("->"))^1) * C(P(1)^0)
+local manipulators = utilities.parsers.manipulators
 
 local function checked(field)
     local m, f = lpegmatch(manipulation,field)
@@ -443,7 +658,6 @@ function commands.btxflush(name,tag,field)
             local manipulator, field = checked(field)
             local value = fields[field]
             if type(value) == "string" then
-             -- context(manipulator and manipulator(value) or value)
                 context(manipulator and manipulated(manipulator,value) or value)
                 return
             end
@@ -451,7 +665,6 @@ function commands.btxflush(name,tag,field)
             if details then
                 local value = details[field]
                 if type(value) == "string" then
-                 -- context(manipulator and manipulator(value) or value)
                     context(manipulator and manipulated(manipulator,value) or value)
                     return
                 end
@@ -473,7 +686,6 @@ function commands.btxdetail(name,tag,field)
             local manipulator, field = checked(field)
             local value = details[field]
             if type(value) == "string" then
-             -- context(manipulator and manipulator(value) or value)
                 context(manipulator and manipulated(manipulator,value) or value)
             else
                 report("unknown detail %a of tag %a in dataset %a",field,tag,name)
@@ -494,7 +706,6 @@ function commands.btxfield(name,tag,field)
             local manipulator, field = checked(field)
             local value = fields[field]
             if type(value) == "string" then
-             -- context(manipulator and manipulator(value) or value)
                 context(manipulator and manipulated(manipulator,value) or value)
             else
                 report("unknown field %a of tag %a in dataset %a",field,tag,name)
@@ -550,44 +761,6 @@ end
 
 -- -- alternative approach: keep data at the tex end
 
-function publications.listconcat(t)
-    local n = #t
-    if n > 0 then
-        context(t[1])
-        if n > 1 then
-            if n > 2 then
-                for i=2,n-1 do
-                    ctx_btxlistparameter("sep")
-                    context(t[i])
-                end
-                ctx_btxlistparameter("finalsep")
-            else
-                ctx_btxlistparameter("lastsep")
-            end
-            context(t[n])
-        end
-    end
-end
-
-function publications.citeconcat(t)
-    local n = #t
-    if n > 0 then
-        context(t[1])
-        if n > 1 then
-            if n > 2 then
-                for i=2,n-1 do
-                    ctx_btxcitevariantparameter("sep")
-                    context(t[i])
-                end
-                ctx_btxcitevariantparameter("finalsep")
-            else
-                ctx_btxcitevariantparameter("lastsep")
-            end
-            context(t[n])
-        end
-    end
-end
-
 function publications.singularorplural(singular,plural)
     if lastconcatsize and lastconcatsize > 1 then
         context(plural)
@@ -596,106 +769,6 @@ function publications.singularorplural(singular,plural)
     end
 end
 
--- function commands.makebibauthorlist(settings) -- ?
---     if not settings then
---         return
---     end
---     local dataset = datasets[settings.dataset]
---     if not dataset or dataset == "" then
---         return
---     end
---     local tag = settings.tag
---     if not tag or tag == "" then
---         return
---     end
---     local asked = settings_to_array(tag)
---     if #asked == 0 then
---         return
---     end
---     local compress    = settings.compress
---     local interaction = settings.interactionn == v_start
---     local limit       = tonumber(settings.limit)
---     local found       = { }
---     local hash        = { }
---     local total       = 0
---     local luadata     = dataset.luadata
---     for i=1,#asked do
---         local tag  = asked[i]
---         local data = luadata[tag]
---         if data then
---             local author = data.a or "Xxxxxxxxxx"
---             local year   = data.y or "0000"
---             if not compress or not hash[author] then
---                 local t = {
---                     author = author,
---                     name   = name, -- first
---                     year   = { [year] = name },
---                 }
---                 total = total + 1
---                 found[total] = t
---                 hash[author] = t
---             else
---                 hash[author].year[year] = name
---             end
---         end
---     end
---     for i=1,total do
---         local data = found[i]
---         local author = data.author
---         local year = table.keys(data.year)
---         table.sort(year)
---         if interaction then
---             for i=1,#year do
---                 year[i] = formatters["\\bibmaybeinteractive{%s}{%s}"](data.year[year[i]],year[i])
---             end
---         end
---         ctx_setvalue("currentbibyear",concat(year,","))
---         if author == "" then
---             ctx_setvalue("currentbibauthor","")
---         else -- needs checking
---             local authors = settings_to_array(author) -- {{}{}},{{}{}}
---             local nofauthors = #authors
---             if nofauthors == 1 then
---                 if interaction then
---                     author = formatters["\\bibmaybeinteractive{%s}{%s}"](data.name,author)
---                 end
---                 ctx_setvalue("currentbibauthor",author)
---             else
---                 limit = limit or nofauthors
---                 if interaction then
---                     for i=1,#authors do
---                         authors[i] = formatters["\\bibmaybeinteractive{%s}{%s}"](data.name,authors[i])
---                     end
---                 end
---                 if limit == 1 then
---                     ctx_setvalue("currentbibauthor",authors[1] .. "\\bibalternative{otherstext}")
---                 elseif limit == 2 and nofauthors == 2 then
---                     ctx_setvalue("currentbibauthor",concat(authors,"\\bibalternative{andtext}"))
---                 else
---                     for i=1,limit-1 do
---                         authors[i] = authors[i] .. "\\bibalternative{namesep}"
---                     end
---                     if limit < nofauthors then
---                         authors[limit+1] = "\\bibalternative{otherstext}"
---                         ctx_setvalue("currentbibauthor",concat(authors,"",1,limit+1))
---                     else
---                         authors[limit-1] = authors[limit-1] .. "\\bibalternative{andtext}"
---                         ctx_setvalue("currentbibauthor",concat(authors))
---                     end
---                 end
---             end
---         end
---         -- the following use: currentbibauthor and currentbibyear
---         if i == 1 then
---             context.ixfirstcommand()
---         elseif i == total then
---             context.ixlastcommand()
---         else
---             context.ixsecondcommand()
---         end
---     end
--- end
-
 local patterns = { "publ-imp-%s.mkiv", "publ-imp-%s.tex" }
 
 local function failure(name)
@@ -731,7 +804,7 @@ local sections    = structures.sections
 
 local renderings = { } --- per dataset
 
-table.setmetatableindex(renderings,function(t,k)
+setmetatableindex(renderings,function(t,k)
     local v = {
         list         = { },
         done         = { },
@@ -788,8 +861,6 @@ local function validkeyword(dataset,tag,keyword)
     end
     local kw = dt.keyword
     if kw then
--- inspect(keyword)
--- inspect(kw)
         for k in next, keyword do
             if kw[k] then
                 return true
@@ -804,12 +875,10 @@ function lists.collectentries(specification)
         return
     end
     local rendering = renderings[dataset]
--- specification.names = "btx"
     local method = rendering.method
     if method == v_none then
         return
     end
--- method=v_local --------------------
     local result  = structures.lists.filter(specification)
     --
     local keyword = specification.keyword
@@ -948,16 +1017,32 @@ function lists.flushentries(dataset,sortvariant)
         list = sort(dataset,rendering,list) or list
     end
     for i=1,#list do
-        ctx_setvalue("currentbtxindex",i)
-        ctx_btxhandlelistentry(list[i][1]) -- we can pass i here too ... more efficient to avoid the setvalue
-    end
+--         ctx_setvalue("currentbtxindex",i) -- todo: helper
+--         -- todo: also flush combinations
+--         ctx_btxhandlelistentry(list[i][1]) -- we can pass i here too ... more efficient to avoid the setvalue
+        local tag = list[i][1]
+        local entry = datasets[dataset].luadata[tag]
+        if entry then
+            ctx_setvalue("currentbtxindex",i) -- todo: helper
+            local combined = entry.combined
+            if combined then
+                ctx_setvalue("currentbtxcombis",concat(combined,","))
+            else
+                ctx_setvalue("currentbtxcombis","")
+            end
+            ctx_btxhandlelistentry(tag) -- pas i instead
+        end
+     end
 end
 
 function lists.fetchentries(dataset)
     local list = renderings[dataset].list
     for i=1,#list do
-        ctx_setvalue("currentbtxindex",i)
-        ctx_btxchecklistentry(list[i][1])
+        local tag = list[i][1]
+        local entry = datasets[dataset].luadata[tag]
+        if entry then
+            ctx_btxchecklistentry(tag) -- integrate doifalreadyplaced here
+        end
     end
 end
 
@@ -987,158 +1072,38 @@ local function compare(a,b)
     return aa and bb and aa < bb
 end
 
--- maybe hash subsets
--- how efficient is this? old leftovers?
-
 -- rendering ?
 
+-- todo: nicer refs
+
 local f_reference   = formatters["r:%s:%s:%s"] -- dataset, instance (block), tag
 local f_destination = formatters["d:%s:%s:%s"] -- dataset, instance (block), tag
-
-function lists.resolve(dataset,reference) -- maybe already feed it split
-    -- needs checking (the prefix in relation to components)
-    local subsets   = nil
-    local block     = tex.count.btxblock
-    local collected = references.collected
-    local prefix    = nil -- todo: dataset ?
-    if prefix and prefix ~= "" then
-        subsets = { collected[prefix] or collected[""] }
-    else
-        local components = references.productdata.components
-        local subset = collected[""]
-        if subset then
-            subsets = { subset }
-        else
-            subsets = { }
-        end
-        for i=1,#components do
-            local subset = collected[components[i]]
-            if subset then
-                subsets[#subsets+1] = subset
-            end
-        end
-    end
--- inspect(subsets)
-    if #subsets > 0 then
-        local result, nofresult, done = { }, 0, { }
-        for i=1,#subsets do
-            local subset = subsets[i]
-            for rest in gmatch(reference,"[^, ]+") do
-                local blk, tag, found = block, nil, nil
-                if block then
-                    tag = f_destination(dataset,blk,rest)
-                    found = subset[tag]
-                    if not found then
-                        for i=block-1,1,-1 do
-                            tag = f_destination(dataset,blk,rest)
---                             tag = i .. ":" .. rest
-                            found = subset[tag]
-                            if found then
-                                blk = i
-                                break
-                            end
-                        end
-                    end
-                end
-                if not found then
-                    blk = "*"
-                    tag = f_destination(dataset,blk,rest)
-                    found = subset[tag]
-                end
-                if found then
-                    local current = tonumber(found.entries and found.entries.text) -- tonumber needed
-                    if current and not done[current] then
-                        nofresult = nofresult + 1
-                        result[nofresult] = { blk, rest, current }
-                        done[current] = true
-                    end
-                end
-            end
-        end
-        local first, last, firsti, lasti, firstr, lastr
-        local collected, nofcollected = { }, 0
-        for i=1,nofresult do
-            local r = result[i]
-            local current = r[3]
-            if not first then
-                first, last, firsti, lasti, firstr, lastr = current, current, i, i, r, r
-            elseif current == last + 1 then
-                last, lasti, lastr = current, i, r
-            else
-                if last > first + 1 then
-                    nofcollected = nofcollected + 1
-                    collected[nofcollected] = { firstr, lastr }
-                else
-                    nofcollected = nofcollected + 1
-                    collected[nofcollected] = firstr
-                    if last > first then
-                        nofcollected = nofcollected + 1
-                        collected[nofcollected] = lastr
-                    end
-                end
-                first, last, firsti, lasti, firstr, lastr = current, current, i, i, r, r
-            end
-        end
-        if first and last then
-            if last > first + 1 then
-                nofcollected = nofcollected + 1
-                collected[nofcollected] = { firstr, lastr }
-            else
-                nofcollected = nofcollected + 1
-                collected[nofcollected] = firstr
-                if last > first then
-                    nofcollected = nofcollected + 1
-                    collected[nofcollected] = lastr
-                end
-            end
-        end
-        if nofcollected > 0 then
--- inspect(reference)
--- inspect(result)
--- inspect(collected)
-            for i=1,nofcollected do
-                local c = collected[i]
-                if i == nofcollected then
-                    ctx_btxlistvariantparameter("lastpubsep")
-                elseif i > 1 then
-                    ctx_btxlistvariantparameter("pubsep")
-                end
-                if #c == 3 then -- a range (3 is first or last)
-                    ctx_btxdirectlink(f_reference(dataset,c[1],c[2]),c[3])
-                else
-                    local f, l = c[2], c[2]
-                    ctx_btxdirectlink(f_reference(dataset,f[1],f[2]),f[3])
-                    context.endash() -- to do
-                    ctx_btxdirectlink(f_reference(dataset,l[4],l[5]),l[6])
-                end
-            end
-        else
-            context("[btx error 1]")
-        end
-    else
-        context("[btx error 2]")
-    end
-end
+local f_listentry   = formatters["d:%s:%s:%s"] -- dataset, instance (block), tag
+local f_internal    = formatters["internal(%s)"] -- dataset, instance (block), tag
 
 local done = { }
 
 function commands.btxreference(dataset,block,tag,data)
-    local ref = f_reference(dataset,block,tag)
+    local ref = f_reference(dataset,block,tag) -- we just need a unique key
     if not done[ref] then
+        if trace_references then
+            report_reference("link: %s",ref)
+        end
         done[ref] = true
--- context("<%s>",data)
-        ctx_dodirectfullreference(ref,data)
+        ctx_btxsetreference(dataset,tag,ref,data)
     end
 end
 
 local done = { }
 
 function commands.btxdestination(dataset,block,tag,data)
-    local ref = f_destination(dataset,block,tag)
+    local ref = f_destination(dataset,block,tag) -- we just need a unique key
     if not done[ref] then
+        if trace_references then
+            report_reference("link: %s",ref)
+        end
         done[ref] = true
--- context("<<%s>>",data)
-        ctx_dodirectfullreference(ref,data)
+        ctx_btxsetreference(dataset,tag,ref,data)
     end
 end
 
@@ -1184,11 +1149,48 @@ local function sortedtags(dataset,list,sorttype)
     end
 end
 
--- todo: standard : current
+--     if sorttype and sorttype ~= "" then
+--         tags = sortedtags(dataset,tags,sorttype)
+--     end
 
 local prefixsplitter = lpeg.splitat("::")
 
-function commands.btxhandlecite(dataset,tag,mark,variant,sorttype,setup) -- variant for tracing
+function commands.btxhandlecite(specification)
+    local tag = specification.reference
+    if not tag or tag == "" then
+        return
+    end
+    --
+    local dataset   = specification.dataset or ""
+    local mark      = specification.markentry ~= false
+    local variant   = specification.variant or "num"
+    local sorttype  = specification.sorttype
+    local compress  = specification.compress == v_yes
+    --
+    local prefix, rest = lpegmatch(prefixsplitter,tag)
+    if rest then
+        dataset = prefix
+    else
+        rest = tag
+    end
+    local action = citevariants[variant] -- there is always fallback on default
+    if trace_cite then
+        report_cite("inject, dataset: %s, tag: %s, variant: %s, compressed",dataset or "-",rest,variant)
+    end
+    ctx_setvalue("currentbtxdataset",dataset)
+    action(dataset,rest,mark,compress,variant)
+end
+
+function commands.btxhandlenocite(specification)
+    local mark = specification.markentry ~= false
+    if not mark then
+        return
+    end
+    local tag = specification.reference
+    if not tag or tag == "" then
+        return
+    end
+    local dataset = specification.dataset or ""
     local prefix, rest = lpegmatch(prefixsplitter,tag)
     if rest then
         dataset = prefix
@@ -1197,190 +1199,551 @@ function commands.btxhandlecite(dataset,tag,mark,variant,sorttype,setup) -- vari
     end
     ctx_setvalue("currentbtxdataset",dataset)
     local tags = settings_to_array(rest)
-    if #tags > 0 then
-        if sorttype and sorttype ~= "" then
-            tags = sortedtags(dataset,tags,sorttype)
+    if trace_cite then
+        report_cite("mark, dataset: %s, tags: % | t",dataset or "-",tags)
+    end
+    for i=1,#tags do
+        markcite(dataset,tags[i])
+    end
+end
+
+-- function commands.btxcitevariant(dataset,block,tags,variant) -- uses? specification ?
+--     local action = citevariants[variant]
+--     if action then
+--         action(dataset,tags,variant)
+--     end
+-- end
+
+-- sorter
+
+local keysorter = function(a,b) return a.sortkey < b.sortkey end
+
+local function compresslist(source)
+    for i=1,#source do
+        if type(source[i].sortkey) ~= "number" then
+            return source
         end
-        ctx_btxcitevariantparameter(v_left)
-        for i=1,#tags do
-            local tag = tags[i]
-            ctx_setvalue("currentbtxtag",tag)
-            if i > 1 then
-                ctx_btxcitevariantparameter(v_middle)
-            end
-            if mark ~= false then
-                ctx_btxdomarkcitation(dataset,tag)
+    end
+    local first, last, firstr, lastr
+    local target, noftarget, tags = { }, 0, { }
+    sort(source,keysorter)
+    local function flushrange()
+        noftarget = noftarget + 1
+        if last > first + 1 then
+            target[noftarget] = {
+                first = firstr,
+                last  = lastr,
+                tags  = tags,
+            }
+        else
+            target[noftarget] = firstr
+            if last > first then
+                noftarget = noftarget + 1
+                target[noftarget] = lastr
             end
-            ctx_directsetup(setup) -- cite can become alternative
         end
-        ctx_btxcitevariantparameter(v_right)
-    else
-        -- error
+        tags = { }
     end
+    for i=1,#source do
+        local entry = source[i]
+        local current = entry.sortkey
+        if not first then
+            first, last, firstr, lastr = current, current, entry, entry
+        elseif current == last + 1 then
+            last, lastr = current, entry
+        else
+            flushrange()
+            first, last, firstr, lastr = current, current, entry, entry
+        end
+        tags[#tags+1] = entry.tag
+    end
+    if first and last then
+        flushrange()
+    end
+    return target
 end
 
-function commands.btxhandlenocite(dataset,tag,mark)
-    if mark ~= false then
-        local prefix, rest = lpegmatch(prefixsplitter,tag)
-        if rest then
-            dataset = prefix
+-- local source = {
+--     { tag = "one",   internal = 1, value = "foo", page = 1 },
+--     { tag = "two",   internal = 2, value = "bar", page = 2 },
+--     { tag = "three", internal = 3, value = "gnu", page = 3 },
+-- }
+--
+-- local target = compresslist(source)
+
+local numberonly = R("09")^1 / tonumber + P(1)^0
+local f_missing  = formatters["<%s>"]
+
+-- maybe also sparse (e.g. pages)
+
+local function processcite(dataset,reference,mark,compress,setup,getter,setter,compressor)
+    reference = publications.parenttag(dataset,reference)
+    local found, todo, list = findallused(dataset,reference)
+    if found and setup then
+        local source = { }
+        local badkey = false
+        for i=1,#found do
+            local entry    = found[i]
+            local tag      = entry.userdata.btxref
+            local internal = entry.references.internal
+            local data     = getter(dataset,tag,entry,internal)
+            if compress and not compressor then
+                local sortkey = data.sortkey
+                if sortkey then
+                    local key = lpegmatch(numberonly,sortkey)
+                    if key then
+                        data.sortkey = key
+                    else
+                        badkey = true
+                    end
+                else
+                    badkey = true
+                end
+            end
+            source[i] = data
+        end
+        if compress and not badkey then
+            local target = (compressor or compresslist)(source)
+            local function flush(i,state)
+                local entry = target[i]
+                local first = entry.first
+                if first then
+                    local tags = entry.tags
+                    if mark then
+                        for i=1,#tags do
+                            local tag = tags[i]
+                            markcite(dataset,tag)
+                            todo[tag] = false
+                        end
+                    end
+                    ctx_btxsettag(tags[1])
+                    local internal = first.internal
+                    if internal then
+                        ctx_btxsetinternal(internal)
+                    end
+                    if not setter(first,entry.last) then
+                        ctx_btxsetfirst(f_missing(first.tag))
+                    end
+                else
+                    local tag = entry.tag
+                    if mark then
+                        markcite(dataset,tag)
+                        todo[tag] = false
+                    end
+                    ctx_btxsettag(tag)
+                    local internal = entry.internal
+                    if internal then
+                        ctx_btxsetinternal(internal)
+                    end
+                    if not setter(entry) then
+                        ctx_btxsetfirst(f_missing(entry.tag))
+                    end
+                end
+                ctx_btxsetconcat(state)
+                ctx_btxcitesetup(setup)
+            end
+            flushcollected(flush,#target)
         else
-            rest = tag
+            local function flush(i,state)
+                local entry = source[i]
+                local tag   = entry.tag
+                if mark then
+                    markcite(dataset,tag)
+                    todo[tag] = false
+                end
+                ctx_btxsettag(tag)
+                local internal = entry.internal
+                if internal then
+                    ctx_btxsetinternal(internal)
+                end
+                ctx_btxsetconcat(state)
+                if not setter(entry) then
+                    ctx_btxsetfirst(f_missing(entry.tag))
+                end
+                ctx_btxcitesetup(setup)
+            end
+            flushcollected(flush,#source)
         end
-        ctx_setvalue("currentbtxdataset",dataset)
-        local tags = settings_to_array(rest)
-        for i=1,#tags do
-            ctx_btxdomarkcitation(dataset,tags[i])
+    end
+    if mark then
+        flushmarked(dataset,list,todo)
+    end
+end
+
+local function simplegetter(first,last,field)
+    local value = first[field]
+    if value then
+        ctx_btxsetfirst(value)
+        if last then
+            ctx_btxsetsecond(last[field])
         end
+        return true
     end
 end
 
-function commands.btxcitevariant(dataset,block,tags,variant)
-    local action = citevariants[variant] or citevariants.default
-    if action then
-        action(dataset,tags,variant)
+local setters = setmetatableindex({},function(t,k)
+    local v = function(dataset,tag,entry,internal)
+        local value = getfield(dataset,tag,k)
+        return {
+            tag      = tag,
+            internal = internal,
+            [k]      = value,
+            sortkey  = value,
+        }
+    end
+    t[k] = v
+    return v
+end)
+
+local getters = setmetatableindex({},function(t,k)
+    local v = function(first,last)
+        return simplegetter(first,last,k)
+    end
+    t[k] = v
+    return v
+end)
+
+-- default
+
+setmetatableindex(citevariants,function(t,k)
+    local v = t.default
+    t[k] = v
+    return v
+end)
+
+function citevariants.default(dataset,reference,mark,compress,variant,setup)
+    processcite(dataset,reference,mark,compress,setup or variant,setters[variant],getters[variant])
+end
+
+-- short
+
+local function setter(dataset,tag,entry,internal)
+    return {
+        tag      = tag,
+        internal = internal,
+        short    = getfield(dataset,tag,"short"),
+        suffix   = getfield(dataset,tag,"suffix"),
+    }
+end
+
+local function getter(first,last) -- last not used
+    local short = first.short
+    if short then
+        local suffix = first.suffix
+        if suffix then
+            ctx_btxsetfirst(short .. suffix)
+        else
+            ctx_btxsetfirst(short)
+        end
+        return true
     end
 end
 
-function citevariants.default(dataset,tags,variant)
-    local content = getfield(dataset,tags,variant)
-    if content then
-        context(content)
+function citevariants.short(dataset,reference,mark,compress)
+    processcite(dataset,reference,mark,false,"short",setter,getter)
+end
+
+-- pages (no compress)
+
+local function setter(dataset,tag,entry,internal)
+    return {
+        tag      = tag,
+        internal = internal,
+        pages    = getdetail(dataset,tag,"pages"),
+    }
+end
+
+local function getter(first,last)
+    local pages = first.pages
+    if pages then
+        if type(pages) == "table" then
+            ctx_btxsetfirst(pages[1])
+            ctx_btxsetsecond(pages[2])
+        else
+            ctx_btxsetfirst(pages)
+        end
+        return true
     end
 end
 
+function citevariants.page(dataset,reference,mark,compress)
+    processcite(dataset,reference,mark,compress,"page",setter,getter)
+end
+
+-- num
+
+local function setter(dataset,tag,entry,internal)
+    local text = entry.entries.text
+    return {
+        tag      = tag,
+        internal = internal,
+        num      = text,
+        sortkey  = text,
+    }
+end
+
+local function getter(first,last)
+    return simplegetter(first,last,"num")
+end
+
+function citevariants.num(dataset,reference,mark,compress)
+    processcite(dataset,reference,mark,compress,"num",setter,getter)
+end
+
+-- year
+
+local function setter(dataset,tag,entry,internal)
+    local year = getfield(dataset,tag,"year")
+    return {
+        tag      = tag,
+        internal = internal,
+        year     = year,
+        sortkey  = year,
+    }
+end
+
+local function getter(first,last)
+    return simplegetter(first,last,"year")
+end
+
+function citevariants.year(dataset,reference,mark,compress)
+    processcite(dataset,reference,mark,compress,"year",setter,getter)
+end
+
+-- index | serial
+
+local function setter(dataset,tag,entry,internal)
+    local index = getfield(dataset,tag,"index")
+    return {
+        tag      = tag,
+        internal = internal,
+        index    = index,
+        sortkey  = index,
+    }
+end
+
+local function getter(first,last)
+    return simplegetter(first,last,"index")
+end
+
+function citevariants.index(dataset,reference,mark,compress)
+    processcite(dataset,reference,mark,compress,"index",setter,getter)
+end
+
+function citevariants.serial(dataset,reference,mark,compress)
+    processcite(dataset,reference,mark,compress,"serial",setter,getter)
+end
+
+-- category | type
+
+local function setter(dataset,tag,entry,internal)
+    return {
+        tag      = tag,
+        internal = internal,
+        category = getfield(dataset,tag,"category"),
+    }
+end
+
+local function getter(first,last)
+    return simplegetter(first,last,"category")
+end
+
+function citevariants.category(dataset,reference,mark,compress)
+    processcite(dataset,reference,mark,compress,"category",setter,getter)
+end
+
+function citevariants.type(dataset,reference,mark,compress)
+    processcite(dataset,reference,mark,compress,"type",setter,getter)
+end
+
+-- key | tag
+
+local function setter(dataset,tag,entry,internal)
+    return {
+        tag      = tag,
+        internal = internal,
+    }
+end
+
+local function getter(first,last)
+    ctx_btxsetfirst(first.tag)
+    return true
+end
+
+function citevariants.key(dataset,reference,mark,compress) return
+    processcite(dataset,reference,mark,compress,"key",setter,getter)
+end
+
+function citevariants.tag(dataset,reference,mark,compress) return
+    processcite(dataset,reference,mark,compress,"tag",setter,getter)
+end
+
+-- author
+
+local function setter(dataset,tag,entry,internal)
+    return {
+        tag      = tag,
+        internal = internal,
+        author   = getfield(dataset,tag,"author"),
+    }
+end
+
+local function getter(first,last)
+    ctx_btxsetfirst(first.author) -- todo: formatted
+    return true
+end
+
+function citevariants.author(dataset,reference,mark,compress)
+    processcite(dataset,reference,mark,false,"author",setter,getter)
+end
+
 -- todo : sort
 -- todo : choose between publications or commands namespace
 -- todo : use details.author
 -- todo : sort details.author
+-- (name, name and name) .. how names? how sorted?
+-- todo: we loop at the tex end .. why not here
+-- \cite[{hh,afo},kvm]
 
-local function collectauthoryears(dataset,tags)
-    local luadata = datasets[dataset].luadata
-    local list    = settings_to_array(tags)
-    local found   = { }
+-- common
+
+local function authorcompressor(found,key)
     local result  = { }
-    local order   = { }
-    for i=1,#list do
-        local tag   = list[i]
-        local entry = luadata[tag]
-        if entry then
-            local year   = entry.year
-            local author = entry.author
-            if author and year then
-                local a = found[author]
-                if not a then
-                    a = { }
-                    found[author] = a
-                    order[#order+1] = author
-                end
-                local y = a[year]
-                if not y then
-                    y = { }
-                    a[year] = y
-                end
-                y[#y+1] = tag
-            end
+    local entries = { }
+    for i=1,#found do
+        local entry    = found[i]
+        local author   = entry.author
+        local aentries = entries[author]
+        -- will be just entry but for tracing ...
+        entry = { internal = entry.internal, author = author, [key] = entry[key], sortkey = entry.sortkey }
+        if aentries then
+            aentries[#aentries+1] = entry
+        else
+            entries[author] =  { entry }
         end
     end
-    -- found = { author = { year_1 = { e1, e2, e3 } } }
-    for i=1,#order do
-        local author = order[i]
-        local years  = found[author]
-        local yrs    = { }
-        for year, entries in next, years do
-            if subyears then
-             -- -- add letters to all entries of an author and if so shouldn't
-             -- -- we tag all years of an author as soon as we do this?
-             -- if #entries > 1 then
-             --     for i=1,#years do
-             --         local entry = years[i]
-             --         -- years[i] = year .. string.char(i + string.byte("0") - 1)
-             --     end
-             -- end
-            else
-                yrs[#yrs+1] = year
+    for i=1,#found do
+        local entry    = found[i]
+        local author   = entry.author
+        local aentries = entries[author]
+        if not aentries then
+            result[#result+1] = entry
+        elseif aentries == true then
+            -- already done
+        else
+            result[#result+1] = entry
+            entry.entries  = aentries
+            entries[author] = true
+        end
+    end
+    -- todo: add letters (should we then tag all?)
+    return result
+end
+
+local function authorconcat(target,key,setup)
+    local function flush(i,state)
+        local entry = target[i]
+        local first = entry.first
+        if first then
+            local internal = first.internal
+            if internal then
+                ctx_btxsetinternal(internal)
             end
+            ctx_btxsetfirst(first[key] or f_missing(first.tag))
+            ctx_btxsetsecond(entry.last[key])
+        else
+            local internal = entry.internal
+            if internal then
+                ctx_btxsetinternal(internal)
+            end
+            ctx_btxsetfirst(entry[key] or f_missing(entry.tag))
         end
-        result[i] = { author = author, years = yrs }
+        ctx_btxsetconcat(state)
+        ctx_btxcitesetup(setup)
     end
-    return result, order
+    ctx_btxstartsubcite(setup)
+    flushcollected(flush,#target)
+    ctx_btxstopsubcite()
 end
 
--- (name, name and name) .. how names? how sorted?
--- todo: we loop at the tex end .. why not here
--- \cite[{hh,afo},kvm]
+local function authorsingle(entry,key,setup)
+    ctx_btxstartsubcite(setup)
+    ctx_btxsetfirst(entry[key] or f_missing(entry.tag))
+    ctx_btxcitesetup(setup)
+    ctx_btxstopsubcite()
+end
 
--- maybe we will move this tex anyway
+local function authorgetter(first,last,key,setup) -- only first
+    ctx_btxsetfirst(first.author) -- todo: reformat
+    local entries = first.entries
+    if entries then
+        local c = compresslist(entries)
+        ctx_btxsetsecond(function() authorconcat(c,key,setup) end)
+    else
+        ctx_btxsetsecond(function() authorsingle(first,key,setup) end)
+    end
+    return true
+end
 
-function citevariants.author(dataset,tags)
-    local result, order = collectauthoryears(dataset,tags,method,what) -- we can have a collectauthors
-    publications.citeconcat(order)
+-- authornum
+
+local function setter(dataset,tag,entry,internal)
+    local text = entry.entries.text
+    return {
+        tag      = tag,
+        internal = internal,
+        author   = getfield(dataset,tag,"author"),
+        num      = text,
+        sortkey  = text and lpegmatch(numberonly,text)
+    }
 end
 
-local function authorandyear(dataset,tags,formatter)
-    local result, order = collectauthoryears(dataset,tags,method,what) -- we can have a collectauthors
-    for i=1,#result do
-        local r = result[i]
-        order[i] = formatter(r.author,r.years) -- reuse order
-    end
-    publications.citeconcat(order)
+local function getter(first,last)
+    authorgetter(first,last,"num","author:num")
+    return true
 end
 
-function citevariants.authoryear(dataset,tags)
-    authorandyear(dataset,tags,formatters["%s (%, t)"])
+local function compressor(found)
+    return authorcompressor(found,"num")
 end
 
-function citevariants.authoryears(dataset,tags)
-    authorandyear(dataset,tags,formatters["%s, %, t"])
+function citevariants.authornum(dataset,reference,mark,compress)
+    processcite(dataset,reference,mark,compress,"authornum",setter,getter,compressor)
 end
 
-function citevariants.authornum(dataset,tags)
-    local result, order = collectauthoryears(dataset,tags,method,what) -- we can have a collectauthors
-    publications.citeconcat(order)
-    ctx_btxcitevariantparameter(v_inbetween)
-    lists.resolve(dataset,tags) -- left/right ?
+-- authoryear | authoryears
+
+local function setter(dataset,tag,entry,internal)
+    local year = getfield(dataset,tag,"year")
+    return {
+        tag      = tag,
+        internal = internal,
+        author   = getfield(dataset,tag,"author"),
+        year     = year,
+        sortkey  = year and lpegmatch(numberonly,year)
+    }
 end
 
--- function citevariants.short(dataset,tags)
---     local short = getdetail(dataset,tags,"short")
---     if short then
---         context(short)
---     end
--- end
+local function getter(first,last)
+    authorgetter(first,last,"year","author:year")
+    return true
+end
 
-function citevariants.short(dataset,tags)
-    local short = getdetail(dataset,tags,"short")
-    local suffix = getdetail(dataset,tags,"suffix")
-    if suffix then
-        context(short .. suffix)
-    elseif short then
-        context(short)
-    end
+local function compressor(found)
+    return authorcompressor(found,"year")
 end
 
-function citevariants.page(dataset,tags)
-    local pages = getdetail(dataset,tags,"pages")
-    if not pages then
-        -- nothing
-    elseif type(pages) == "table" then
-        context(pages[1])
-        ctx_btxcitevariantparameter(v_inbetween)
-        context(pages[2])
-    else
-        context(pages)
-    end
+function citevariants.authoryear(dataset,reference,mark,compress)
+    processcite(dataset,reference,mark,compress,"authoryear",setter,getter,compressor)
 end
 
-function citevariants.num(dataset,tags)
---     ctx_btxdirectlink(f_destination(dataset,block,tags),listindex) -- not okay yet
-    lists.resolve(dataset,tags)
+local function getter(first,last)
+    authorgetter(first,last,"year","author:years")
+    return true
 end
 
-function citevariants.serial(dataset,tags) -- the traditional fieldname is "serial" and not "index"
-    local index = getfield(dataset,tags,"index")
-    if index then
-        context(index)
-    end
+function citevariants.authoryears(dataset,reference,mark,compress)
+    processcite(dataset,reference,mark,compress,"authoryears",setter,getter,compressor)
 end
 
 -- List variants
@@ -1388,41 +1751,30 @@ end
 local listvariants        = { }
 publications.listvariants = listvariants
 
--- function commands.btxhandlelist(dataset,block,tag,variant,setup)
---     if sorttype and sorttype ~= "" then
---         tags = sortedtags(dataset,tags,sorttype)
---     end
---     ctx_setvalue("currentbtxtag",tag)
---     ctx_btxlistvariantparameter(v_left)
---     ctx_directsetup(setup)
---     ctx_btxlistvariantparameter(v_right)
--- end
-
-function commands.btxlistvariant(dataset,block,tags,variant,listindex)
+function commands.btxlistvariant(dataset,block,tag,variant,listindex)
     local action = listvariants[variant] or listvariants.default
     if action then
-        action(dataset,block,tags,variant,tonumber(listindex) or 0)
+        action(dataset,block,tag,variant,tonumber(listindex) or 0)
     end
 end
 
-function listvariants.default(dataset,block,tags,variant)
+function listvariants.default(dataset,block,tag,variant)
     context("?")
 end
 
-function listvariants.num(dataset,block,tags,variant,listindex)
-    ctx_btxdirectlink(f_destination(dataset,block,tags),listindex) -- not okay yet
+function listvariants.num(dataset,block,tag,variant,listindex)
+    local lst = f_listentry(dataset,block,tag)
+    local ref = f_reference(dataset,block,tag)
+    if trace_references then
+        report_reference("list: %s",lst)
+    end
+    -- todo
+    ctx_btxdirectlink(ref,listindex) -- a goto
 end
 
--- function listvariants.short(dataset,block,tags,variant,listindex)
---     local short = getdetail(dataset,tags,variant,variant)
---     if short then
---         context(short)
---     end
--- end
-
-function listvariants.short(dataset,block,tags,variant,listindex)
-    local short  = getdetail(dataset,tags,"short","short")
-    local suffix = getdetail(dataset,tags,"suffix","suffix")
+function listvariants.short(dataset,block,tag,variant,listindex)
+    local short  = getdetail(dataset,tag,"short","short")
+    local suffix = getdetail(dataset,tag,"suffix","suffix")
     if suffix then
         context(short .. suffix)
     elseif short then