summaryrefslogtreecommitdiff
path: root/Master/texmf-dist/tex/context/base/bibl-bib.lua
diff options
context:
space:
mode:
Diffstat (limited to 'Master/texmf-dist/tex/context/base/bibl-bib.lua')
-rw-r--r--Master/texmf-dist/tex/context/base/bibl-bib.lua605
1 files changed, 567 insertions, 38 deletions
diff --git a/Master/texmf-dist/tex/context/base/bibl-bib.lua b/Master/texmf-dist/tex/context/base/bibl-bib.lua
index 028202ec22f..3c0dad2fa4e 100644
--- a/Master/texmf-dist/tex/context/base/bibl-bib.lua
+++ b/Master/texmf-dist/tex/context/base/bibl-bib.lua
@@ -12,8 +12,18 @@ bibtex files and converts them to xml so that the we access the content
in a convenient way. Actually handling the data takes place elsewhere.</p>
--ldx]]--
-local lower, format = string.lower, string.format
+local utf = unicode.utf8
+local lower, format, gsub, concat = string.lower, string.format, string.gsub, table.concat
local next = next
+local utfchar = utf.char
+local lpegmatch = lpeg.match
+local textoutf = characters and characters.tex.toutf
+local variables = interfaces and interfaces.variables
+
+local finalizers = xml.finalizers.tex
+local xmlfilter, xmltext = xml.filter, xml.text
+
+local trace_bibxml = false trackers.register("publications.bibxml", function(v) trace_bibtex = v end)
bibtex = bibtex or { }
@@ -21,10 +31,29 @@ bibtex.size = 0
bibtex.definitions = 0
bibtex.shortcuts = 0
+local defaultshortcuts = {
+ jan = "1",
+ feb = "2",
+ mar = "3",
+ apr = "4",
+ may = "5",
+ jun = "6",
+ jul = "7",
+ aug = "8",
+ sep = "9",
+ oct = "10",
+ nov = "11",
+ dec = "12",
+}
+
local shortcuts = { }
-local data = { }
+local data = { }
local entries
+-- Currently we expand shortcuts and for large ones (like the acknowledgements
+-- in tugboat.bib this is not that efficient. However, eventually strings get
+-- hashed again.
+
local function do_shortcut(tag,key,value)
bibtex.shortcuts = bibtex.shortcuts + 1
if lower(tag) == "@string" then
@@ -49,34 +78,33 @@ local function do_definition(tag,key,tab) -- maybe check entries here (saves mem
end
local function resolve(s)
- return shortcuts[s] or ""
-end
-
-local percent = lpeg.P("%")
-local start = lpeg.P("@")
-local comma = lpeg.P(",")
-local hash = lpeg.P("#")
-local escape = lpeg.P("\\")
-local single = lpeg.P("'")
-local double = lpeg.P('"')
-local left = lpeg.P('{')
-local right = lpeg.P('}')
+ return shortcuts[s] or defaultshortcuts[s] or s -- can be number
+end
+
+local P, R, S, C, Cc, Cs, Ct = lpeg.P, lpeg.R, lpeg.S, lpeg.C, lpeg.Cc, lpeg.Cs, lpeg.Ct
+
+local percent = P("%")
+local start = P("@")
+local comma = P(",")
+local hash = P("#")
+local escape = P("\\")
+local single = P("'")
+local double = P('"')
+local left = P('{')
+local right = P('}')
local both = left + right
-local lineending = lpeg.S("\n\r")
-local space = lpeg.S(" \t\n\r\f")
+local lineending = S("\n\r")
+local space = S(" \t\n\r\f")
local spacing = space^0
-local equal = lpeg.P("=")
+local equal = P("=")
local collapsed = (space^1)/ " "
local function add(a,b) if b then return a..b else return a end end
-local keyword = lpeg.C((lpeg.R("az","AZ","09") + lpeg.S("@_:-"))^1) -- lpeg.C((1-space)^1)
+local keyword = C((lpeg.R("az","AZ","09") + S("@_:-"))^1) -- lpeg.C((1-space)^1)
local s_quoted = ((escape*single) + collapsed + (1-single))^0
local d_quoted = ((escape*double) + collapsed + (1-double))^0
-local balanced = lpeg.P {
- [1] = ((escape * (left+right)) + (1 - (left+right)) + lpeg.V(2))^0,
- [2] = left * lpeg.V(1) * right
-}
+local balanced = lpeg.patterns.balanced
local s_value = (single/"") * s_quoted * (single/"")
local d_value = (double/"") * d_quoted * (double/"")
@@ -84,7 +112,7 @@ local b_value = (left /"") * balanced * (right /"")
local r_value = keyword/resolve
local somevalue = s_value + d_value + b_value + r_value
-local value = lpeg.Cs((somevalue * ((spacing * hash * spacing)/"" * somevalue)^0))
+local value = Cs((somevalue * ((spacing * hash * spacing)/"" * somevalue)^0))
local assignment = spacing * keyword * spacing * equal * spacing * value * spacing
local shortcut = keyword * spacing * left * spacing * (assignment * comma^0)^0 * spacing * right
@@ -101,14 +129,21 @@ function bibtex.convert(session,content)
data, shortcuts, entries = session.data, session.shortcuts, session.entries
-- session.size = session.size + #content
bibtex.size = bibtex.size + #content
- grammar:match(content or "")
+ session.size = session.size + #content
+ lpegmatch(grammar,content or "")
statistics.stoptiming(bibtex)
end
function bibtex.load(session,filename)
local filename = resolvers.find_file(filename,"bib")
if filename ~= "" then
- bibtex.convert(session,io.loaddata(filename) or "")
+ local data = io.loaddata(filename) or ""
+ if data == "" then
+ logs.report("publications","empty file '%s', no conversion to xml",filename)
+ elseif trace_bibxml then
+ logs.report("publications","converting file '%s' to xml",filename)
+ end
+ bibtex.convert(session,data)
end
end
@@ -119,37 +154,88 @@ function bibtex.new()
xml = xml.convert("<?xml version='1.0' standalone='yes'?>\n<bibtex></bibtex>"),
size = 0,
entries = nil,
+ loaded = false,
}
end
local escaped_pattern = xml.escaped_pattern
-function bibtex.toxml(session)
+local ihatethis = {
+ f = "\\f",
+ n = "\\n",
+ r = "\\r",
+ s = "\\s",
+ t = "\\t",
+ v = "\\v",
+ z = "\\z",
+}
+
+local command = P("\\")/"" * Cc("\\bibtexcommand{") * (R("az","AZ")^1) * Cc("}")
+local any = P(1)
+local done = P(-1)
+local one_l = P("{") / ""
+local one_r = P("}") / ""
+local two_l = P("{{") / ""
+local two_r = P("}}") / ""
+
+local filter = Cs(
+ two_l * (command + any - two_r - done)^0 * two_r * done +
+ one_l * (command + any - one_r - done)^0 * one_r * done +
+ (command + any )^0
+)
+
+function bibtex.toxml(session,options)
+ if session.loaded then
+ return
+ else
+ session.loaded = true
+ end
-- we can always speed this up if needed
-- format slows down things a bit but who cares
statistics.starttiming(bibtex)
local result = { }
+ local options = aux.settings_to_hash(options)
+ local convert = options.convert -- todo: interface
+ local strip = options.strip -- todo: interface
local entries = session.entries
result[#result+1] = format("<?xml version='1.0' standalone='yes'?>")
result[#result+1] = format("<bibtex>")
for id, categories in next, session.data do
- result[#result+1] = format(" <c n='%s'>",id)
+ id = lower(gsub(id,"^@",""))
for name, entry in next, categories do
if not entries or entries[name] then
- result[#result+1] = format(" <e n='%s'>",name)
+ result[#result+1] = format("<entry tag='%s' category='%s'>",lower(name),id)
for key, value in next, entry do
- value = escaped_pattern:match(value)
+ value = gsub(value,"\\(.)",ihatethis)
+ value = lpegmatch(escaped_pattern,value)
+
if value ~= "" then
- result[#result+1] = format(" <v n='%s'>%s</v>",key,value)
+ if convert then
+ value = textoutf(value,true)
+ end
+ if strip then
+ -- as there is no proper namespace in bibtex we need this
+ -- kind of hackery ... bibtex databases are quite unportable
+ value = lpegmatch(filter,value) or value
+ end
+ result[#result+1] = format(" <field name='%s'>%s</field>",key,value)
end
end
- result[#result+1] = format(" </e>")
+ result[#result+1] = format("</entry>")
end
end
- result[#result+1] = format(" </c>")
end
result[#result+1] = format("</bibtex>")
- session.xml = xml.convert(table.concat(result,"\n"))
+ result = concat(result,"\n")
+ -- alternatively we could use lxml.convert
+ session.xml = xml.convert(result, {
+ resolve_entities = true,
+ resolve_predefined_entities = true, -- in case we have escaped entities
+ -- unify_predefined_entities = true, -- &#038; -> &amp;
+ utfize_entities = true,
+ } )
+ session.data = nil
+ session.shortcuts = nil
statistics.stoptiming(bibtex)
end
@@ -204,30 +290,473 @@ end)
--~ print(table.serialize(session.shortcuts))
--~ print(xml.serialize(session.xml))
--- this will move:
+if not characters then dofile(resolvers.find_file("char-def.lua")) end
+
+local chardata = characters.data
+local concat = table.concat
+
+local P, Ct, lpegmatch = lpeg.P, lpeg.Ct, lpeg.match
+
+local space, comma = P(" "), P(",")
+
+local andsplitter = Ct(lpeg.splitat(space^1 * "and" * space^1))
+local commasplitter = Ct(lpeg.splitat(space^0 * comma * space^0))
+local spacesplitter = Ct(lpeg.splitat(space^1))
+local firstcharacter = lpeg.patterns.utf8byte
+
+function is_upper(str)
+ local first = lpegmatch(firstcharacter,str)
+ local okay = chardata[first]
+ return okay and okay.category == "lu"
+end
+
+local function splitauthors(str)
+ local authors = lpegmatch(andsplitter,str)
+ for i=1,#authors do
+ local firstnames, vons, surnames, initials, juniors, words
+ local author = authors[i]
+ local split = lpegmatch(commasplitter,author)
+ local n = #split
+ if n == 1 then
+ --~ First von Last
+ words = lpegmatch(spacesplitter,author)
+ firstnames, vons, surnames = { }, { }, { }
+ local i, n = 1, #words
+ while i <= n do
+ local w = words[i]
+ if is_upper(w) then
+ firstnames[#firstnames+1], i = w, i + 1
+ else
+ break
+ end
+ end
+ while i <= n do
+ local w = words[i]
+ if is_upper(w) then
+ break
+ else
+ vons[#vons+1], i = w, i + 1
+ end
+ end
+ while i <= n do
+ surnames[#surnames+1], i = words[i], i + 1
+ end
+ elseif n == 2 then
+ --~ von Last, First
+ words = lpegmatch(spacesplitter,split[2])
+ surnames = lpegmatch(spacesplitter,split[1])
+ firstnames, vons = { }, { }
+ local i, n = 1, #words
+ while i <= n do
+ local w = words[i]
+ if is_upper(w) then
+ firstnames[#firstnames+1], i = w, i + 1
+ else
+ break
+ end
+ end
+ while i <= n do
+ vons[#vons+1], i = words[i], i + 1
+ end
+ else
+ --~ von Last, Jr ,First
+ firstnames = lpegmatch(spacesplitter,split[1])
+ juniors = lpegmatch(spacesplitter,split[2])
+ surnames = lpegmatch(spacesplitter,split[3])
+ if n > 3 then
+ -- error
+ end
+ end
+ if #surnames == 0 then
+ surnames[1] = firstnames[#firstnames]
+ firstnames[#firstnames] = nil
+ end
+ if firstnames then
+ initials = { }
+ for i=1,#firstnames do
+ initials[i] = utfchar(lpegmatch(firstcharacter,firstnames[i]))
+ end
+ end
+ authors[i] = {
+ original = author,
+ firstnames = firstnames,
+ vons = vons,
+ surnames = surnames,
+ initials = initials,
+ juniors = juniors,
+ }
+ end
+ authors.original = str
+ return authors
+end
+
+local function the_initials(initials,symbol)
+ local t, symbol = { }, symbol or "."
+ for i=1,#initials do
+ t[i] = initials[i] .. symbol
+ end
+ return t
+end
+
+-- authors
+
+bibtex.authors = bibtex.authors or { }
+
+local authors = bibtex.authors
+
+local defaultsettings = {
+ firstnamesep = " ",
+ vonsep = " ",
+ surnamesep = " ",
+ juniorsep = " ",
+ surnamejuniorsep = ", ",
+ juniorjuniorsep = ", ",
+ surnamefirstnamesep = ", ",
+ surnameinitialsep = ", ",
+ namesep = ", ",
+ lastnamesep = " and ",
+ finalnamesep = " and ",
+}
+
+function authors.normal(author,settings)
+ local firstnames, vons, surnames, juniors = author.firstnames, author.vons, author.surnames, author.juniors
+ local result, settings = { }, settings or defaultsettings
+ if firstnames and #firstnames > 0 then
+ result[#result+1] = concat(firstnames," ")
+ result[#result+1] = settings.firstnamesep or defaultsettings.firstnamesep
+ end
+ if vons and #vons > 0 then
+ result[#result+1] = concat(vons," ")
+ result[#result+1] = settings.vonsep or defaultsettings.vonsep
+ end
+ if surnames then
+ result[#result+1] = concat(surnames," ")
+ end
+ if juniors and #juniors > 0 then
+ result[#result+1] = concat(juniors," ")
+ result[#result+1] = settings.surnamesep or defaultsettings.surnamesep
+ end
+ return concat(result)
+end
+
+function authors.normalshort(author,settings)
+ local firstnames, vons, surnames, juniors = author.firstnames, author.vons, author.surnames, author.juniors
+ local result, settings = { }, settings or defaultsettings
+ if firstnames and #firstnames > 0 then
+ result[#result+1] = concat(firstnames," ")
+ result[#result+1] = settings.firstnamesep or defaultsettings.firstnamesep
+ end
+ if vons and #vons > 0 then
+ result[#result+1] = concat(vons," ")
+ result[#result+1] = settings.vonsep or defaultsettings.vonsep
+ end
+ if surnames then
+ result[#result+1] = concat(surnames," ")
+ end
+ if juniors and #juniors > 0 then
+ result[#result+1] = concat(juniors," ")
+ result[#result+1] = settings.surnamejuniorsep or defaultsettings.surnamejuniorsep
+ end
+ return concat(result)
+end
+
+function authors.inverted(author,settings)
+ local firstnames, vons, surnames, juniors = author.firstnames, author.vons, author.surnames, author.juniors
+ local result, settings = { }, settings or defaultsettings
+ if vons and #vons > 0 then
+ result[#result+1] = concat(vons," ")
+ result[#result+1] = settings.vonsep or defaultsettings.vonsep
+ end
+ if surnames then
+ result[#result+1] = concat(surnames," ")
+ end
+ if juniors and #juniors > 0 then
+ result[#result+1] = settings.juniorjuniorsep or defaultsettings.juniorjuniorsep
+ result[#result+1] = concat(juniors," ")
+ end
+ if firstnames and #firstnames > 0 then
+ result[#result+1] = settings.surnamefirstnamesep or defaultsettings.surnamefirstnamesep
+ result[#result+1] = concat(firstnames," ")
+ end
+ return concat(result)
+end
+
+function authors.invertedshort(author,settings)
+ local vons, surnames, initials, juniors = author.vons, author.surnames, author.initials, author.juniors
+ local result, settings = { }, settings or defaultsettings
+ if vons and #vons > 0 then
+ result[#result+1] = concat(vons," ")
+ result[#result+1] = settings.vonsep or defaultsettings.vonsep
+ end
+ if surnames then
+ result[#result+1] = concat(surnames," ")
+ end
+ if juniors and #juniors > 0 then
+ result[#result+1] = settings.juniorjuniorsep or defaultsettings.juniorjuniorsep
+ result[#result+1] = concat(juniors," ")
+ end
+ if initials and #initials > 0 then
+ result[#result+1] = settings.surnameinitialsep or defaultsettings.surnameinitialsep
+ result[#result+1] = concat(the_initials(initials)," ")
+ end
+ return concat(result)
+end
+
+local lastconcatsize = 1
+
+local function bibtexconcat(t,settings)
+ local namesep = settings.namesep or defaultsettings.namesep or ", "
+ local lastnamesep = settings.lastnamesep or defaultsettings.lastnamesep or namesep
+ local finalnamesep = settings.finalnamesep or defaultsettings.finalnamesep or lastnamesep
+ local lastconcatsize = #t
+ if lastconcatsize > 2 then
+ local s = { }
+ for i=1,lastconcatsize-2 do
+ s[i] = t[i] .. namesep
+ end
+ s[lastconcatsize-1], s[lastconcatsize] = t[lastconcatsize-1] .. finalnamesep, t[lastconcatsize]
+ return concat(s)
+ elseif lastconcatsize > 1 then
+ return concat(t,lastnamesep)
+ elseif lastconcatsize > 0 then
+ return t[1]
+ else
+ return ""
+ end
+end
+
+function authors.concat(author,combiner,what,settings)
+ if type(combiner) == "string" then
+ combiner = authors[combiner or "normal"] or authors.normal
+ end
+ local split = splitauthors(author)
+ local setting = settings[what]
+ local etallimit, etaldisplay, etaltext = 1000, 1000, ""
+ if setting then
+ etallimit = settings.etallimit or 1000
+ etaldisplay = settings.etaldisplay or etallimit
+ etalltext = settings.etaltext or ""
+ end
+ local max = #split
+ if max > etallimit and etaldisplay < max then
+ max = etaldisplay
+ end
+ for i=1,max do
+ split[i] = combiner(split[i],settings)
+ end
+ local result = bibtexconcat(split,settings)
+ if max < #split then
+ return result
+ else
+ return result .. etaltext
+ end
+end
+
+function authors.short(author,year)
+ local result = { }
+ if author then
+ local authors = splitauthors(author)
+ for a=1,#authors do
+ local aa = authors[a]
+ local initials = aa.initials
+ for i=1,#initials do
+ result[#result+1] = initials[i]
+ end
+ local surnames = aa.surnames
+ for s=1,#surnames do
+ result[#result+1] = utfchar(lpegmatch(firstcharacter,surnames[s]))
+ end
+ end
+ end
+ if year then
+ result[#result+1] = year
+ end
+ return concat(result)
+end
+
+-- We can consider creating a hashtable key -> entry but I wonder if
+-- pays off.
+
+local function collectauthoryears(id,list)
+ list = aux.settings_to_hash(list)
+ id = lxml.get_id(id)
+ local found = { }
+ for e in xml.collected(id,"/bibtex/entry") do
+ if list[e.at.tag] then
+ local year = xmlfilter(e,"xml:///field[@name='year']/text()")
+ local author = xmlfilter(e,"xml:///field[@name='author']/text()")
+ if author and year then
+ local a = found[author]
+ if not a then
+ a = { }
+ found[author] = a
+ end
+ local y = a[year]
+ if not y then
+ y = { }
+ a[year] = y
+ end
+ y[#y+1] = e
+ end
+ end
+ end
+ -- found = { author = { year_1 = { e1, e2, e3 } } }
+ local done = { }
+ for author, years in next, found do
+ local yrs = { }
+ for year, entries in next, years do
+ if subyears then
+ -- -- add letters to all entries of an author and if so shouldn't
+ -- -- we tag all years of an author as soon as we do this?
+ -- if #entries > 1 then
+ -- for i=1,#years do
+ -- local entry = years[i]
+ -- -- years[i] = year .. string.char(i + string.byte("0") - 1)
+ -- end
+ -- end
+ else
+ yrs[#yrs+1] = year
+ end
+ end
+ done[author] = yrs
+ end
+ return done
+end
+
+local method, settings = "normal", { }
+
+function authors.setsettings(s)
+ settings = s or settings
+end
if commands then
+ local texsprint = tex and tex.sprint
+ local ctxcatcodes = tex and tex.ctxcatcodes
+
local sessions = { }
function commands.definebibtexsession(name)
sessions[name] = bibtex.new()
end
- function commands.preparebibtexsession(name)
- bibtex.toxml(sessions[name])
- lxml.register("bibtex:"..name,sessions[name].xml)
+
+ function commands.preparebibtexsession(name,xmlname,options)
+ bibtex.toxml(sessions[name],options)
+ lxml.register(xmlname,sessions[name].xml)
end
+
function commands.registerbibtexfile(name,filename)
bibtex.load(sessions[name],filename)
end
+
function commands.registerbibtexentry(name,entry)
local session = sessions[name]
local entries = session.entries
if not entries then
- session.entries = { [entry] = true }
+ session.entries = { [entry] = true } -- here we can keep more info
else
entries[entry] = true
end
end
+ -- commands.bibtexconcat = bibtexconcat
+
+ -- finalizers can be rather dumb as we have just text and no embedded xml
+
+ function finalizers.bibtexconcat(collected,method,what)
+ if collected then
+ local author = collected[1].dt[1] or ""
+ if author ~= "" then
+ texsprint(ctxcatcodes,authors.concat(author,method,what,settings))
+ end
+ end
+ end
+
+ function finalizers.bibtexshort(collected)
+ if collected then
+ local c = collected[1]
+ local year = xmlfilter(c,"xml://field[@name='year']/text()")
+ local author = xmlfilter(c,"xml://field[@name='author']/text()")
+ texsprint(ctxcatcodes,authors.short(author,year))
+ end
+ end
+
+ -- experiment:
+
+ --~ -- alternative approach: keep data at the tex end
+
+ --~ local function xbibtexconcat(t,sep,finalsep,lastsep)
+ --~ local n = #t
+ --~ if n > 0 then
+ --~ context(t[1])
+ --~ if n > 1 then
+ --~ if n > 2 then
+ --~ for i=2,n-1 do
+ --~ context.bibtexpublicationsparameter("sep")
+ --~ context(t[i])
+ --~ end
+ --~ context.bibtexpublicationsparameter("finalsep")
+ --~ else
+ --~ context.bibtexpublicationsparameter("lastsep")
+ --~ end
+ --~ context(t[n])
+ --~ end
+ --~ end
+ --~ end
+
+ -- todo : sort
+
+ -- todo: choose between bibtex or commands namespace
+
+ function bibtex.authorref(id,list)
+ local result = collectauthoryears(id,list,method,what)
+ for author, years in next, result do
+ texsprint(ctxcatcodes,authors.concat(author,method,what,settings))
+ end
+ end
+
+ function bibtex.authoryearref(id,list)
+ local result = collectauthoryears(id,list,method,what)
+ for author, years in next, result do
+ texsprint(ctxcatcodes,authors.concat(author,method,what,settings)," (",concat(years,", "),")")
+ end
+ end
+
+ function bibtex.authoryearsref(id,list)
+ local result = collectauthoryears(id,list,method,what)
+ for author, years in next, result do
+ texsprint(ctxcatcodes,"(",authors.concat(author,method,what,settings),", ",concat(years,", "),")")
+ end
+ end
+
+ function bibtex.singular_or_plural(singular,plural)
+ if lastconcatsize and lastconcatsize > 1 then
+ texsprint(ctxcatcodes,plural)
+ else
+ texsprint(ctxcatcodes,singular)
+ end
+ end
end
+
+
+--~ local function test(sample)
+--~ local authors = splitauthors(sample)
+--~ print(table.serialize(authors))
+--~ for i=1,#authors do
+--~ local author = authors[i]
+--~ print(normalauthor (author,settings))
+--~ print(normalshortauthor (author,settings))
+--~ print(invertedauthor (author,settings))
+--~ print(invertedshortauthor(author,settings))
+--~ end
+--~ print(concatauthors(sample,settings,normalauthor))
+--~ print(concatauthors(sample,settings,normalshortauthor))
+--~ print(concatauthors(sample,settings,invertedauthor))
+--~ print(concatauthors(sample,settings,invertedshortauthor))
+--~ end
+
+--~ local sample_a = "Hagen, Hans and Hoekwater, Taco Whoever T. Ex. and Henkel Hut, Hartmut Harald von der"
+--~ local sample_b = "Hans Hagen and Taco Whoever T. Ex. Hoekwater and Hartmut Harald von der Henkel Hut"
+
+--~ test(sample_a)
+--~ test(sample_b)