From 1772be123f1cfa713b25548f6fec135e7ab339a3 Mon Sep 17 00:00:00 2001 From: Norbert Preining Date: Fri, 19 Aug 2022 03:00:56 +0000 Subject: CTAN sync 202208190300 --- biblio/citation-style-language/citeproc-engine.lua | 2043 +++++++++++++++++--- 1 file changed, 1807 insertions(+), 236 deletions(-) (limited to 'biblio/citation-style-language/citeproc-engine.lua') diff --git a/biblio/citation-style-language/citeproc-engine.lua b/biblio/citation-style-language/citeproc-engine.lua index 33a5a08e08..2e9f4c4a61 100644 --- a/biblio/citation-style-language/citeproc-engine.lua +++ b/biblio/citation-style-language/citeproc-engine.lua @@ -8,17 +8,29 @@ local engine = {} local dom = require("luaxml-domobject") -local richtext = require("citeproc-richtext") -local element = require("citeproc-element") local nodes = require("citeproc-nodes") -local formats = require("citeproc-formats") +local Style = require("citeproc-node-style").Style +local Locale = require("citeproc-node-locale").Locale +local Context = require("citeproc-context").Context +local IrState = require("citeproc-context").IrState +local Rendered = require("citeproc-ir-node").Rendered +local YearSuffix = require("citeproc-ir-node").YearSuffix +-- local OutputFormat = require("citeproc-output").OutputFormat +local LatexWriter = require("citeproc-output").LatexWriter +local HtmlWriter = require("citeproc-output").HtmlWriter +local SortStringFormat = require("citeproc-output").SortStringFormat +local DisamStringFormat = require("citeproc-output").DisamStringFormat +local InlineElement = require("citeproc-output").InlineElement +local Micro = require("citeproc-output").Micro +local Formatted = require("citeproc-output").Formatted +local PlainText = require("citeproc-output").PlainText local util = require("citeproc-util") local CiteProc = {} -function CiteProc.new (sys, style, lang, force_lang) - if sys == nil then +function CiteProc.new(sys, style, lang, force_lang) + if not sys then error("\"citeprocSys\" required") end if sys.retrieveLocale == nil then @@ -28,213 +40,1579 @@ function CiteProc.new (sys, style, lang, force_lang) error("\"citeprocSys.retrieveItem\" required") end local o = {} + + o.style = Style:parse(style) + + o.sys = sys + o.locales = {} + o.system_locales = {} + + o.lang = o.style.default_locale + if not o.lang or force_lang then + o.lang = lang or "en-US" + end + + o.output_format = LatexWriter:new() + + o.opt = { + -- Similar to citeproc-js's development_extensions.wrap_url_and_doi + url_link = false, + doi_link = false, -- also applied to PMID and PMCID + title_link = false, + } + o.registry = { - citations = {}, -- A map - citation_strings = {}, -- A list - registry = {}, -- A map - reflist = {}, -- A list + citations_by_id = {}, -- A map + citation_list = {}, -- A list + citations_by_item_id = {}, -- A map from item id to a map of citations + registry = {}, -- A map of bibliographic meta data + reflist = {}, -- list of cited ids + uncited_list = {}, previous_citation = nil, requires_sorting = false, + longest_label = "", + maxoffset = 0, } - o.sys = sys - o.system_locales = {} + o.cite_first_note_numbers = {} + o.cite_last_note_numbers = {} + o.note_citations_map = {} + + o.tainted_item_ids = {} + + o.disam_irs = {} + -- { , , ... } + + o.cite_irs_by_output = {} + -- { + -- ["Roe, J"] = {}, + -- ["Doe, J"] = {, }, + -- ["Doe, John"] = {}, + -- ["Doe, Jack"] = {}, + -- } + + o.person_names = {} + o.person_names_by_output = {} + + setmetatable(o, { __index = CiteProc }) + return o +end + +function CiteProc:updateItems(ids) + self.registry.reflist = {} + self.registry.registry = {} + self.person_names = {} + self.person_names_by_output = {} + self.disam_irs = {} + self.cite_irs_by_output = {} + + local cite_items = {} + local loaded_ids = {} + + for _, id in ipairs(ids) do + table.insert(cite_items, {id = id}) + loaded_ids[id] = true + end + for _, id in ipairs(self.registry.uncited_list) do + if not loaded_ids[id] then + table.insert(cite_items, {id = id}) + loaded_ids[id] = true + end + end + + -- TODO: optimize this + self:makeCitationCluster(cite_items) + + self.registry.previous_citation = nil + self.cite_first_note_numbers = {} + self.cite_last_note_numbers = {} + self.note_citations_map = {} +end + +function CiteProc:updateUncitedItems(uncited_ids) + -- self.registry.reflist = {} + self.registry.registry = {} + self.registry.uncited_list = {} + self.person_names = {} + self.person_names_by_output = {} + self.disam_irs = {} + self.cite_irs_by_output = {} + + local cite_items = {} + local loaded_ids = {} + + for _, id in ipairs(self.registry.reflist) do + if not loaded_ids[id] then + table.insert(cite_items, {id = id}) + loaded_ids[id] = true + end + end + self.registry.reflist = {} + + for _, id in ipairs(uncited_ids) do + if not loaded_ids[id] then + table.insert(cite_items, {id = id}) + loaded_ids[id] = true + end + end + + loaded_ids = {} + for _, id in ipairs(uncited_ids) do + if not loaded_ids[id] then + table.insert(self.registry.uncited_list, id) + loaded_ids[id] = true + end + end + + -- TODO: optimize this + self:makeCitationCluster(cite_items) + + self.registry.previous_citation = nil + self.cite_first_note_numbers = {} + self.cite_last_note_numbers = {} + self.note_citations_map = {} +end + +function CiteProc:processCitationCluster(citation, citationsPre, citationsPost) + -- util.debug(citation) + -- util.debug(citationsPre) + -- Fix missing noteIndex: sort_CitationNumberPrimaryAscendingViaMacroCitation.txt + if not citation.properties then + citation.properties = {} + end + if not citation.properties.noteIndex then + citation.properties.noteIndex = 0 + end + + -- Registor citation + self.registry.citations_by_id[citation.citationID] = citation + + local citation_note_pairs = {} + util.extend(citation_note_pairs, citationsPre) + table.insert(citation_note_pairs, {citation.citationID, citation.properties.noteIndex}) + util.extend(citation_note_pairs, citationsPost) + -- util.debug(citation_note_pairs) + + local citations_by_id = {} + local citation_list = {} + for _, pair in ipairs(citation_note_pairs) do + local citation_id, note_number = table.unpack(pair) + local citation_ = self.registry.citations_by_id[citation_id] + if not citation_ then + util.error("Citation not in registry.") + end + citations_by_id[citation_.citationID] = citation_ + table.insert(citation_list, citation_) + end + self.registry.citations_by_id = citations_by_id + self.registry.citation_list = citation_list + + -- update self.registry.citations_by_item_id + local item_ids = {} + self.registry.citations_by_item_id = {} + for _, citation_ in ipairs(citation_list) do + for _, cite_item in ipairs(citation_.citationItems) do + if not self.registry.citations_by_item_id[cite_item.id] then + self.registry.citations_by_item_id[cite_item.id] = {} + table.insert(item_ids, cite_item.id) + end + self.registry.citations_by_item_id[cite_item.id][citation_.citationID] = true + end + end + self:updateItems(item_ids) + + local params = { + bibchange = false, + citation_errors = {}, + } + local output = {} + + local tainted_citation_ids = self:get_tainted_citaion_ids(citation_note_pairs) + -- util.debug(tainted_citation_ids) + + -- params.bibchange = #tainted_citation_ids > 0 + for citation_id, _ in pairs(tainted_citation_ids) do + local citation_ = self.registry.citations_by_id[citation_id] + + local citation_index = citation_.citation_index + local citation_str = self:build_citation_str(citation_) + table.insert(output, {citation_index, citation_str, citation_id}) + end + + -- util.debug(output) + return {params, output} +end + +-- A variant of processCitationCluster() for easy use with LaTeX. +-- It should be run after refreshing the registry (updateItems()) with all items +function CiteProc:process_citation(citation) + -- util.debug(citation) + -- util.debug(citationsPre) + -- Fix missing noteIndex: sort_CitationNumberPrimaryAscendingViaMacroCitation.txt + if not citation.properties then + citation.properties = {} + end + if not citation.properties.noteIndex then + citation.properties.noteIndex = 0 + end + + -- Registor citation + self.registry.citations_by_id[citation.citationID] = citation + + table.insert(self.registry.citation_list, citation) + + local citation_note_pairs = {} + for _, citation_ in ipairs(self.registry.citation_list) do + table.insert(citation_note_pairs, {citation_.citationID, citation_.properties.noteIndex}) + end + + -- update self.registry.citations_by_item_id + for _, cite_item in ipairs(citation.citationItems) do + if not self.registry.citations_by_item_id[cite_item.id] then + self.registry.citations_by_item_id[cite_item.id] = {} + end + self.registry.citations_by_item_id[cite_item.id][citation.citationID] = true + end + + -- self:updateItems(item_ids) + for i, cite_item in ipairs(citation.citationItems) do + cite_item.id = tostring(cite_item.id) + self:get_item(cite_item.id) + end + + local tainted_citation_ids = self:get_tainted_citaion_ids(citation_note_pairs) + local citation_str = self:build_citation_str(citation) + + return citation_str +end + + +function CiteProc:get_tainted_citaion_ids(citation_note_pairs) + local tainted_citation_ids = {} + + self.cite_first_note_numbers = {} + self.cite_last_note_numbers = {} + self.note_citations_map = {} + -- { + -- 1 = {"citation-1", "citation-2"}, + -- 2 = {"citation-2"}, + -- } + + local previous_citation + for citation_index, pair in ipairs(citation_note_pairs) do + local citation_id, note_number = table.unpack(pair) + -- util.debug(citation_id) + local citation = self.registry.citations_by_id[citation_id] + citation.properties.noteIndex = note_number + citation.citation_index = citation_index + + local tainted = false + + local previous_cite + for _, cite_item in ipairs(citation.citationItems) do + tainted = self:set_cite_item_position(cite_item, note_number, previous_cite, previous_citation) + self.cite_last_note_numbers[cite_item.id] = note_number + previous_cite = cite_item + end + + if tainted then + tainted_citation_ids[citation.citationID] = true + end + + if not self.note_citations_map[note_number] then + self.note_citations_map[note_number] = {} + end + table.insert(self.note_citations_map[note_number], citation.citationID) + previous_citation = citation + end + + -- Update tainted citation ids because of citation-number's change + -- The self.tainted_item_ids were added in the sort_bibliography() procedure. + for item_id, _ in pairs(self.tainted_item_ids) do + if self.registry.citations_by_item_id[item_id] then + for citation_id, _ in pairs(self.registry.citations_by_item_id[item_id]) do + tainted_citation_ids[citation_id] = true + end + end + end + + return tainted_citation_ids +end + +function CiteProc:set_cite_item_position(cite_item, note_number, previous_cite, previous_citation) + local position = util.position_map["first"] + + local first_reference_note_number = self.cite_first_note_numbers[cite_item.id] + if first_reference_note_number then + position = util.position_map["subsequent"] + else + self.cite_first_note_numbers[cite_item.id] = note_number + end + + local preceding_cite_item = self:get_preceding_cite_item(cite_item, previous_cite, previous_citation, note_number) + if preceding_cite_item then + position = self:_get_cite_position(cite_item, preceding_cite_item) + end + + local near_note = false + local last_note_number = self.cite_last_note_numbers[cite_item.id] + if last_note_number then + local note_distance = note_number - last_note_number + if note_distance <= self.style.citation.near_note_distance then + near_note = true + end + end + + local tainted = false + if cite_item.position_level ~= position then + tainted = true + cite_item.position_level = position + end + if cite_item["first-reference-note-number"] ~= first_reference_note_number then + tainted = true + cite_item["first-reference-note-number"] = first_reference_note_number + end + if cite_item.near_note ~= near_note then + tainted = true + cite_item.near_note = near_note + end + return tainted +end + +-- Find the preceding cite referencing the same item +function CiteProc:get_preceding_cite_item(cite_item, previous_cite, previous_citation, note_number) + if previous_cite then + -- a. the current cite immediately follows on another cite, within the same + -- citation, that references the same item + if cite_item.id == previous_cite.id then + return previous_cite + end + elseif previous_citation then + -- (hidden) The previous citation is the only one in the previous note. + -- See also + -- https://github.com/citation-style-language/documentation/issues/121 + -- position_IbidWithMultipleSoloCitesInBackref.txt + -- b. the current cite is the first cite in the citation, and the previous + -- citation consists of a single cite referencing the same item + local previous_note_number = previous_citation.properties.noteIndex + local num_previous_note_citations = #self.note_citations_map[previous_note_number] + if (previous_note_number == note_number - 1 and num_previous_note_citations == 1) + or previous_note_number == note_number then + if #previous_citation.citationItems == 1 then + previous_cite = previous_citation.citationItems[1] + if previous_cite.id == cite_item.id then + return previous_cite + end + end + end + end + return nil +end + +function CiteProc:_get_cite_position(item, preceding_cite) + if preceding_cite.locator then + if item.locator then + if item.locator == preceding_cite.locator and item.label == preceding_cite.label then + return util.position_map["ibid"] + else + return util.position_map["ibid-with-locator"] + end + else + return util.position_map["subsequent"] + end + else + if item.locator then + return util.position_map["ibid-with-locator"] + else + return util.position_map["ibid"] + end + end +end + +function CiteProc:build_citation_str(citation) + -- util.debug(citation.citationID) + local items = {} + for i, cite_item in ipairs(citation.citationItems) do + cite_item.id = tostring(cite_item.id) + -- util.debug(cite_item.id) + + -- Use "page" as locator label if missing + -- label_PluralWithAmpersand.txt + if cite_item.locator and not cite_item.label then + cite_item.label = "page" + end + + table.insert(items, cite_item) + end + + if self.registry.requires_sorting then + self:sort_bibliography() + end + + local citation_str = self:build_cluster(items) + return citation_str +end + +function CiteProc:build_cluster(citation_items) + local output_format = self.output_format + local irs = {} + citation_items = self:sorted_citation_items(citation_items) + for _, cite_item in ipairs(citation_items) do + local ir = self:build_fully_disambiguated_ir(cite_item, output_format) + table.insert(irs, ir) + end + + if self.style.citation.cite_grouping then + irs = self:group_cites(irs) + else + local citation_collapse = self.style.citation.collapse + if citation_collapse == "year" or citation_collapse == "year-suffix" or + citation_collapse == "year-suffix-ranged" then + irs = self:group_cites(irs) + end + end + + if self.style.citation.collapse then + self:collapse_cites(irs) + end + + -- Capitalize first + for i, ir in ipairs(irs) do + -- local layout_prefix + -- local layout_affixes = self.style.citation.layout.affixes + -- if layout_affixes then + -- layout_prefix = layout_affixes.prefix + -- end + local prefix = citation_items[i].prefix + if prefix then + if prefix and string.match(prefix, "[.!?]%s*$") and #util.split(util.strip(prefix)) > 1 then + ir:capitalize_first_term() + end + else + local delimiter = self.style.citation.layout.delimiter + if i == 1 or not delimiter or string.match(delimiter, "[.!?]%s*$") then + ir:capitalize_first_term() + end + end + end + + -- util.debug(irs) + + local citation_delimiter = self.style.citation.layout.delimiter + local citation_stream = {} + + local context = Context:new() + context.engine = self + context.style = self.style + context.area = self.style.citation + context.in_bibliography = false + context.locale = self:get_locale(self.lang) + context.name_inheritance = self.style.citation.name_inheritance + context.format = output_format + + local previous_ir + for i, ir in ipairs(irs) do + local cite_prefix = citation_items[i].prefix + local cite_suffix = citation_items[i].suffix + if not ir.collapse_suppressed then + local ir_inlines = ir:flatten(output_format) + if #ir_inlines > 0 then + -- Make sure ir_inlines has outputs contents. + -- collapse_AuthorCollapseNoDateSorted.txt + if previous_ir then + if previous_ir.own_delimiter then + table.insert(citation_stream, PlainText:new(previous_ir.own_delimiter)) + elseif citation_delimiter and not (cite_prefix and util.startswith(cite_prefix, ",")) then + table.insert(citation_stream, PlainText:new(citation_delimiter)) + end + end + + if cite_prefix then + table.insert(citation_stream, Micro:new(InlineElement:parse(cite_prefix, context))) + end + + -- util.debug(ir) + util.extend(citation_stream, ir_inlines) + previous_ir = ir + + if cite_suffix then + table.insert(citation_stream, Micro:new(InlineElement:parse(cite_suffix, context))) + end + end + end + end + -- util.debug(citation_stream) + + if context.area.layout.affixes then + local affixes = context.area.layout.affixes + if affixes.prefix then + table.insert(citation_stream, 1, PlainText:new(affixes.prefix)) + end + if affixes.suffix then + table.insert(citation_stream, PlainText:new(affixes.suffix)) + end + end + -- util.debug(citation_stream) + + if #citation_stream > 0 and context.area.layout.formatting then + citation_stream = {Formatted:new(citation_stream, context.area.layout.formatting)} + end + + if #citation_stream == 0 then + citation_stream = {PlainText:new("[CSL STYLE ERROR: reference with no printed form.]")} + end + + -- util.debug(citation_stream) + local str = output_format:output(citation_stream, context) + str = util.strip(str) + + return str +end + +function CiteProc:sorted_citation_items(items) + local citation_sort = self.style.citation.sort + if not citation_sort then + return items + end + + local state = IrState:new() + local context = Context:new() + context.engine = self + context.style = self.style + context.area = self.style.citation + context.in_bibliography = false + context.locale = self:get_locale(self.lang) + context.name_inheritance = self.style.citation.name_inheritance + context.format = SortStringFormat:new() + -- context.id = id + context.cite = nil + -- context.reference = self:get_item(id) + + items = citation_sort:sort(items, state, context) + return items +end + +function CiteProc:build_fully_disambiguated_ir(cite_item, output_format) + local cite_ir = self:build_ambiguous_ir(cite_item, output_format) + -- util.debug(cite_ir) + cite_ir = self:disambiguate_add_givenname(cite_ir) + cite_ir = self:disambiguate_add_names(cite_ir) + cite_ir = self:disambiguate_conditionals(cite_ir) + cite_ir = self:disambiguate_add_year_suffix(cite_ir) + return cite_ir +end + +function CiteProc:build_ambiguous_ir(cite_item, output_format) + local state = IrState:new(self.style) + cite_item.id = tostring(cite_item.id) + local context = Context:new() + context.engine = self + context.style = self.style + context.area = self.style.citation + context.locale = self:get_locale(self.lang) + context.name_inheritance = self.style.citation.name_inheritance + context.format = output_format + context.id = cite_item.id + context.cite = cite_item + -- context.reference = self:get_item(cite_item.id) + context.reference = self.registry.registry[cite_item.id] + + local ir + if context.reference then + ir = self.style.citation:build_ir(self, state, context) + else + -- The warning has been given in the retrieving procedure. + -- util.warning(string.format('Failed to find the entry "%s" in database.', cite_item.id)) + ir = Rendered:new({Formatted:new({PlainText:new(cite_item.id)}, {["font-weight"] = "bold"})}, self) + end + + ir.cite_item = cite_item + ir.reference = context.reference + ir.ir_index = #self.disam_irs + 1 + table.insert(self.disam_irs, ir) + ir.is_ambiguous = false + ir.disam_level = 0 + + -- Formattings like font-style are ignored for disambiguation. + local disam_format = DisamStringFormat:new() + local inlines = ir:flatten(disam_format) + local disam_str = disam_format:output(inlines, context) + ir.disam_str = disam_str + + if not self.cite_irs_by_output[disam_str] then + self.cite_irs_by_output[disam_str] = {} + end + + for ir_index, ir_ in pairs(self.cite_irs_by_output[disam_str]) do + if ir_.cite_item.id ~= cite_item.id then + ir.is_ambiguous = true + break + end + end + self.cite_irs_by_output[disam_str][ir.ir_index] = ir + + return ir +end + +function CiteProc:disambiguate_add_givenname(cite_ir) + if self.style.citation.disambiguate_add_givenname then + -- util.debug("disambiguate_add_givenname: " .. cite_ir.cite_item.id) + + local gn_disam_rule = self.style.citation.givenname_disambiguation_rule + if gn_disam_rule == "all-names" or gn_disam_rule == "all-names-with-initials" then + cite_ir = self:disambiguate_add_givenname_all_names(cite_ir) + elseif gn_disam_rule == "primary-name" or gn_disam_rule == "primary-name-with-initials" then + cite_ir = self:disambiguate_add_givenname_primary_name(cite_ir) + elseif gn_disam_rule == "by-cite" then + cite_ir = self:disambiguate_add_givenname_by_cite(cite_ir) + end + end + return cite_ir +end + +-- TODO: reorganize this code +function CiteProc:disambiguate_add_givenname_all_names(cite_ir) + -- util.debug("disambiguate_add_givenname_all_names: " .. cite_ir.cite_item.id) + if not cite_ir.person_name_irs or #cite_ir.person_name_irs == 0 then + return cite_ir + end + + -- util.debug(cite_ir.disam_str) + + for _, person_name_ir in ipairs(cite_ir.person_name_irs) do + local name_output = person_name_ir.name_output + -- util.debug(name_output) + + if not person_name_ir.person_name_index then + person_name_ir.person_name_index = #self.person_names + 1 + table.insert(self.person_names, person_name_ir) + end + + if not self.person_names_by_output[name_output] then + self.person_names_by_output[name_output] = {} + end + self.person_names_by_output[name_output][person_name_ir.person_name_index] = person_name_ir + + local ambiguous_name_irs = {} + local ambiguous_same_output_irs = {} + + for _, pn_ir in pairs(self.person_names_by_output[person_name_ir.name_output]) do + if pn_ir.full_name ~= person_name_ir.full_name then + table.insert(ambiguous_name_irs, pn_ir) + end + if pn_ir.name_output == person_name_ir.name_output then + table.insert(ambiguous_same_output_irs, pn_ir) + end + end + + -- util.debug(person_name_ir.name_output) + -- util.debug(person_name_ir.full_name) + -- util.debug(#ambiguous_name_irs) + -- util.debug(person_name_ir.disam_variants_index) + -- util.debug(person_name_ir.disam_variants) + + while person_name_ir.disam_variants_index < #person_name_ir.disam_variants do + if #ambiguous_name_irs == 0 then + break + end + + for _, pn_ir in ipairs(ambiguous_same_output_irs) do + -- expand one name + if pn_ir.disam_variants_index < #pn_ir.disam_variants then + pn_ir.disam_variants_index = pn_ir.disam_variants_index + 1 + pn_ir.name_output = pn_ir.disam_variants[pn_ir.disam_variants_index] + pn_ir.inlines = pn_ir.disam_inlines[pn_ir.name_output] + + if not self.person_names_by_output[pn_ir.name_output] then + self.person_names_by_output[pn_ir.name_output] = {} + end + self.person_names_by_output[pn_ir.name_output][pn_ir.person_name_index] = pn_ir + end + end + + -- util.debug(person_name_ir.name_output) + + -- update ambiguous_name_irs and ambiguous_same_output_irs + ambiguous_name_irs = {} + ambiguous_same_output_irs = {} + for _, pn_ir in pairs(self.person_names_by_output[person_name_ir.name_output]) do + if pn_ir.full_name ~= person_name_ir.full_name then + -- util.debug(pn_ir.full_name .. ": " .. pn_ir.name_output) + table.insert(ambiguous_name_irs, pn_ir) + end + if pn_ir.name_output == person_name_ir.name_output then + table.insert(ambiguous_same_output_irs, pn_ir) + end + end + + end + end + + -- update cite_ir output + local disam_format = DisamStringFormat:new() + local inlines = cite_ir:flatten(disam_format) + local disam_str = disam_format:output(inlines, context) + cite_ir.disam_str = disam_str + if not self.cite_irs_by_output[disam_str] then + self.cite_irs_by_output[disam_str] = {} + end + self.cite_irs_by_output[disam_str][cite_ir.ir_index] = cite_ir + + -- update ambiguous_cite_irs and ambiguous_same_output_irs + local ambiguous_cite_irs = {} + for ir_index, ir_ in pairs(self.cite_irs_by_output[cite_ir.disam_str]) do + -- util.debug(ir_.cite_item.id) + if ir_.cite_item.id ~= cite_ir.cite_item.id then + table.insert(ambiguous_cite_irs, ir_) + end + end + if #ambiguous_cite_irs == 0 then + cite_ir.is_ambiguous = false + end + + return cite_ir +end + +function CiteProc:disambiguate_add_givenname_primary_name(cite_ir) + if not cite_ir.person_name_irs or #cite_ir.person_name_irs == 0 then + return cite_ir + end + local person_name_ir = cite_ir.person_name_irs[1] + local name_output = person_name_ir.name_output + -- util.debug(name_output) + + if not person_name_ir.person_name_index then + person_name_ir.person_name_index = #self.person_names + 1 + table.insert(self.person_names, person_name_ir) + end + if not self.person_names_by_output[name_output] then + self.person_names_by_output[name_output] = {} + end + self.person_names_by_output[name_output][person_name_ir.person_name_index] = person_name_ir + + local ambiguous_name_irs = {} + local ambiguous_same_output_irs = {} + + for _, pn_ir in pairs(self.person_names_by_output[person_name_ir.name_output]) do + if pn_ir.full_name ~= person_name_ir.full_name then + table.insert(ambiguous_name_irs, pn_ir) + end + if pn_ir.name_output == person_name_ir.name_output then + table.insert(ambiguous_same_output_irs, pn_ir) + end + end + + for _, name_variant in ipairs(person_name_ir.disam_variants) do + if #ambiguous_name_irs == 0 then + break + end + + for _, pn_ir in ipairs(ambiguous_same_output_irs) do + -- expand one name + if pn_ir.disam_variants_index < #pn_ir.disam_variants then + pn_ir.disam_variants_index = pn_ir.disam_variants_index + 1 + pn_ir.name_output = pn_ir.disam_variants[pn_ir.disam_variants_index] + pn_ir.inlines = pn_ir.disam_inlines[pn_ir.name_output] + + if not self.person_names_by_output[pn_ir.name_output] then + self.person_names_by_output[pn_ir.name_output] = {} + end + self.person_names_by_output[pn_ir.name_output][person_name_ir.person_name_index] = person_name_ir + end + end + + -- update ambiguous_name_irs and ambiguous_same_output_irs + ambiguous_name_irs = {} + ambiguous_same_output_irs = {} + for _, pn_ir in pairs(self.person_names_by_output[person_name_ir.name_output]) do + if pn_ir.full_name ~= person_name_ir.full_name then + table.insert(ambiguous_name_irs, pn_ir) + end + if pn_ir.name_output == person_name_ir.name_output then + table.insert(ambiguous_same_output_irs, pn_ir) + end + end + end + + return cite_ir +end + +function CiteProc:disambiguate_add_givenname_by_cite(cite_ir) + if not cite_ir.is_ambiguous then + return cite_ir + end + if not cite_ir.person_name_irs or #cite_ir.person_name_irs == 0 then + return cite_ir + end + + for _, ir_ in ipairs(self.disam_irs) do + -- util.debug(ir_.cite_item.id) + -- util.debug(ir_.disam_str) + end + + local disam_format = DisamStringFormat:new() + + local ambiguous_cite_irs = {} + local ambiguous_same_output_irs = {} + + for ir_index, ir_ in pairs(self.cite_irs_by_output[cite_ir.disam_str]) do + if ir_.cite_item.id ~= cite_ir.cite_item.id then + table.insert(ambiguous_cite_irs, ir_) + end + if ir_.disam_str == cite_ir.disam_str then + table.insert(ambiguous_same_output_irs, ir_) + end + end + + for i, person_name_ir in ipairs(cite_ir.person_name_irs) do + -- util.debug(person_name_ir.name_output) + -- util.debug(person_name_ir.disam_variants) + if #ambiguous_cite_irs == 0 then + cite_ir.is_ambiguous = false + break + end + + -- util.debug(person_name_ir.disam_variants) + while person_name_ir.disam_variants_index < #person_name_ir.disam_variants do + -- util.debug(person_name_ir.name_output) + + local is_different_name = false + for _, ir_ in ipairs(ambiguous_cite_irs) do + if ir_.person_name_irs[i] then + if ir_.person_name_irs[i].full_name ~= person_name_ir.full_name then + -- util.debug(ir_.cite_item.id) + is_different_name = true + break + end + end + end + -- util.debug(is_different_name) + if not is_different_name then + break + end + + for _, ir_ in ipairs(ambiguous_same_output_irs) do + -- util.debug(ir_.cite_item.id) + local person_name_ir_ = ir_.person_name_irs[i] + if person_name_ir_ then + if person_name_ir_.disam_variants_index < #person_name_ir_.disam_variants then + person_name_ir_.disam_variants_index = person_name_ir_.disam_variants_index + 1 + local disam_variant = person_name_ir_.disam_variants[person_name_ir_.disam_variants_index] + person_name_ir_.name_output = disam_variant + -- util.debug(disam_variant) + person_name_ir_.inlines = person_name_ir_.disam_inlines[disam_variant] + -- Update cite ir output + local inlines = ir_:flatten(disam_format) + local disam_str = disam_format:output(inlines, context) + ir_.disam_str = disam_str + if not self.cite_irs_by_output[disam_str] then + self.cite_irs_by_output[disam_str] = {} + end + self.cite_irs_by_output[disam_str][ir_.ir_index] = ir_ + end + end + end + + -- update ambiguous_cite_irs and ambiguous_same_output_irs + ambiguous_cite_irs = {} + ambiguous_same_output_irs = {} + for ir_index, ir_ in pairs(self.cite_irs_by_output[cite_ir.disam_str]) do + -- util.debug(ir_.cite_item.id) + if ir_.cite_item.id ~= cite_ir.cite_item.id then + table.insert(ambiguous_cite_irs, ir_) + end + if ir_.disam_str == cite_ir.disam_str then + table.insert(ambiguous_same_output_irs, ir_) + end + end + + -- util.debug(#ambiguous_cite_irs) + + if #ambiguous_cite_irs == 0 then + cite_ir.is_ambiguous = false + return cite_ir + end + + end + end + + return cite_ir +end + +local function find_first_name_ir(ir) + if ir._type == "NameIr" then + return ir + end + if ir.children then + for _, child in ipairs(ir.children) do + local name_ir = find_first_name_ir(child) + if name_ir then + return name_ir + end + end + end + return nil +end + +function CiteProc:disambiguate_add_names(cite_ir) + if not self.style.citation.disambiguate_add_names then + return cite_ir + end + + if not cite_ir.name_ir then + cite_ir.name_ir = find_first_name_ir(cite_ir) + end + local name_ir = cite_ir.name_ir + + if not cite_ir.is_ambiguous then + return cite_ir + end + + if not name_ir or not name_ir.et_al_abbreviation then + return cite_ir + end + + -- util.debug("disambiguate_add_names: " .. cite_ir.cite_item.id) + + if name_ir then + -- util.debug(cite_ir.disam_str) + -- util.debug(cite_ir.name_ir.full_name_str) + -- util.debug(cite_ir.is_ambiguous) + end + + local disam_format = DisamStringFormat:new() + + while cite_ir.is_ambiguous do + if #cite_ir.name_ir.hidden_name_irs == 0 then + break + end + + local ambiguous_cite_irs = {} + local ambiguous_same_output_irs = {} + for _, ir_ in pairs(self.cite_irs_by_output[cite_ir.disam_str]) do + if ir_.cite_item.id ~= cite_ir.cite_item.id then + table.insert(ambiguous_cite_irs, ir_) + end + if ir_.disam_str == cite_ir.disam_str then + table.insert(ambiguous_same_output_irs, ir_) + end + end + + -- util.debug(#ambiguous_same_output_irs) + if #ambiguous_cite_irs == 0 then + cite_ir.is_ambiguous = false + break + end + + -- check if the cite can be (fully) disambiguated by adding names + local can_be_disambuguated = false + for _, ir_ in ipairs(ambiguous_cite_irs) do + if ir_.name_ir.full_name_str ~= cite_ir.name_ir.full_name_str then + can_be_disambuguated = true + break + end + end + -- util.debug(can_be_disambuguated) + if not can_be_disambuguated then + break + end + + for _, ir_ in ipairs(ambiguous_same_output_irs) do + local added_person_name_ir = ir_.name_ir.name_inheritance:expand_one_name(ir_.name_ir) + if added_person_name_ir then + -- util.debug("Updated: " .. ir_.cite_item.id) + table.insert(ir_.person_name_irs, added_person_name_ir) + + if not added_person_name_ir.person_name_index then + added_person_name_ir.person_name_index = #self.person_names + 1 + table.insert(self.person_names, added_person_name_ir) + end + local name_output = added_person_name_ir.name_output + if not self.person_names_by_output[name_output] then + self.person_names_by_output[name_output] = {} + end + self.person_names_by_output[name_output][added_person_name_ir.person_name_index] = added_person_name_ir + + -- Update ir output + local inlines = ir_:flatten(disam_format) + local disam_str = disam_format:output(inlines, context) + -- util.debug(disam_str) + ir_.disam_str = disam_str + if not self.cite_irs_by_output[disam_str] then + self.cite_irs_by_output[disam_str] = {} + end + self.cite_irs_by_output[disam_str][ir_.ir_index] = ir_ + end + end + + -- util.debug("disambiguate_add_givenname") + + if self.style.citation.disambiguate_add_givenname then + local gn_disam_rule = self.style.citation.givenname_disambiguation_rule + if gn_disam_rule == "all-names" or gn_disam_rule == "all-names-with-initials" then + cite_ir = self:disambiguate_add_givenname_all_names(cite_ir) + elseif gn_disam_rule == "by-cite" then + cite_ir = self:disambiguate_add_givenname_by_cite(cite_ir) + end + end + + cite_ir.is_ambiguous = self:check_ambiguity(cite_ir) + + for _, ir_ in ipairs(self.disam_irs) do + -- util.debug(ir_.cite_item.id .. ": " .. ir_.disam_str) + end + + end + + + return cite_ir +end - if type(style) == "string" then - o.csl = dom.parse(style) - else - o.csl = style +function CiteProc:collect_irs_with_disambiguate_branch(ir) + local irs_with_disambiguate_branch = {} + if ir.children then + for i, child_ir in ipairs(ir.children) do + if child_ir.disambiguate_branch_ir then + table.insert(irs_with_disambiguate_branch, child_ir) + elseif child_ir.children then + util.extend(irs_with_disambiguate_branch, + self:collect_irs_with_disambiguate_branch(child_ir)) + end + end end - o.csl:traverse_elements(CiteProc.set_base_class) - o.csl:root_node().engine = o - o.style = o.csl:get_path("style")[1] - o.style.lang = lang - o.csl:root_node().style = o.style + return irs_with_disambiguate_branch +end - o.style:set_lang(lang, force_lang) +function CiteProc:disambiguate_conditionals(cite_ir) + -- util.debug(cite_ir) - o.formatter = formats.latex - o.linking_enabled = false + cite_ir.irs_with_disambiguate_branch = self:collect_irs_with_disambiguate_branch(cite_ir) - setmetatable(o, { __index = CiteProc }) - return o + local disam_format = DisamStringFormat:new() + + while cite_ir.is_ambiguous do + if #cite_ir.irs_with_disambiguate_branch == 0 then + break + end + + -- util.debug(cite_ir.cite_item.id) + -- util.debug(cite_ir.disam_str) + + -- update ambiguous_same_output_irs + local ambiguous_same_output_irs = {} + for _, ir_ in pairs(self.cite_irs_by_output[cite_ir.disam_str]) do + if ir_.disam_str == cite_ir.disam_str then + table.insert(ambiguous_same_output_irs, ir_) + end + end + + for _, ir_ in ipairs(ambiguous_same_output_irs) do + if #ir_.irs_with_disambiguate_branch > 0 then + -- Disambiguation is incremental + -- disambiguate_IncrementalExtraText.txt + local condition_ir = ir_.irs_with_disambiguate_branch[1] + condition_ir.children[1] = condition_ir.disambiguate_branch_ir + condition_ir.group_var = condition_ir.disambiguate_branch_ir.group_var + table.remove(ir_.irs_with_disambiguate_branch, 1) + -- disambiguate_DisambiguateTrueReflectedInBibliography.txt + ir_.reference.disambiguate = true + + -- Update ir output + local inlines = ir_:flatten(disam_format) + local disam_str = disam_format:output(inlines, context) + -- util.debug("update: " .. ir_.cite_item.id .. ": " .. disam_str) + ir_.disam_str = disam_str + if not self.cite_irs_by_output[disam_str] then + self.cite_irs_by_output[disam_str] = {} + end + self.cite_irs_by_output[disam_str][ir_.ir_index] = ir_ + end + end + + cite_ir.is_ambiguous = self:check_ambiguity(cite_ir) + -- util.debug(cite_ir.is_ambiguous) + for _, ir_ in ipairs(self.disam_irs) do + -- util.debug(ir_.cite_item.id .. ": " .. ir_.disam_str) + end + + end + return cite_ir end -function CiteProc:updateItems (ids) - self.registry.reflist = {} - self.registry.registry = {} - for _, id in ipairs(ids) do - self:get_item(id) +function CiteProc:check_ambiguity(cite_ir) + for _, ir_ in pairs(self.cite_irs_by_output[cite_ir.disam_str]) do + if ir_.cite_item.id ~= cite_ir.cite_item.id then + return true + end end + return false end -function CiteProc:updateUncitedItems(ids) - for _, id in ipairs(ids) do - if not self.registry.registry[id] then - self:get_item(id) +function CiteProc:get_same_output_irs(cite_ir) + local ambiguous_same_output_irs = {} + for _, ir_ in pairs(self.cite_irs_by_output[cite_ir.disam_str]) do + if ir_.disam_str == cite_ir.disam_str then + table.insert(ambiguous_same_output_irs, ir_) end end - -- TODO: disambiguation + return ambiguous_same_output_irs end -function CiteProc:processCitationCluster(citation, citationsPre, citationsPost) - -- citation = { - -- citationID = "CITATION-3", - -- citationItems = { - -- { id = "ITEM-1" }, - -- { id = "ITEM-2" }, - -- }, - -- properties = { - -- noteIndex = 3, - -- }, - -- } - -- citationsPre = { - -- {"CITATION-1", 1}, - -- {"CITATION-2", 2}, - -- } - -- citationsPost = { - -- {"CITATION-4", 4}, - -- } - -- returns = { - -- { - -- bibchange = true, - -- citation_errors = {}, - -- }, - -- { - -- { 2, "[1,2]", "CITATION-3" } - -- } - -- } - self.registry.citations[citation.citationID] = citation +function CiteProc:disambiguate_add_year_suffix(cite_ir) + if not cite_ir.is_ambiguous or not self.style.citation.disambiguate_add_year_suffix then + return cite_ir + end - local items = {} + local same_output_irs = self:get_same_output_irs(cite_ir) - for _, cite_item in ipairs(citation.citationItems) do - cite_item.id = tostring(cite_item.id) - local position_first = (self.registry.registry[cite_item.id] == nil) - local item_data = self:get_item(cite_item.id) + table.sort(same_output_irs, function (a, b) + -- return a.ir_index < b.ir_index + return a.reference["citation-number"] < b.reference["citation-number"] + end) - if item_data then - -- Create a wrapper of the orignal item from registry so that - -- it may hold different `locator` or `position` values for cites. - local item = setmetatable({}, {__index = function (_, key) - if cite_item[key] then - return cite_item[key] - else - return item_data[key] + local year_suffix_number = 0 + -- util.debug(cite_ir) + + local disam_format = DisamStringFormat:new() + + for _, ir_ in ipairs(same_output_irs) do + ir_.reference.year_suffix_number = nil + end + + for _, ir_ in ipairs(same_output_irs) do + -- print(ir_.cite_item.id) + -- print(ir_.reference) + if not ir_.reference.year_suffix_number then + year_suffix_number = year_suffix_number + 1 + ir_.reference.year_suffix_number = year_suffix_number + ir_.reference["year-suffix"] = self:render_year_suffix(year_suffix_number) + end + + if not ir_.year_suffix_irs then + ir_.year_suffix_irs = self:collect_year_suffix_irs(ir_) + if #ir_.year_suffix_irs == 0 then + -- By default, the year-suffix is appended the first year rendered through cs:date + local year_ir = self:find_first_year_ir(ir_) + -- util.debug(year_ir) + if year_ir then + local year_suffix_ir = YearSuffix:new({}, self.style.citation) + table.insert(year_ir.children, year_suffix_ir) + table.insert(ir_.year_suffix_irs, year_suffix_ir) end - end}) + end + end + + for _, year_suffix_ir in ipairs(ir_.year_suffix_irs) do + year_suffix_ir.inlines = {PlainText:new(ir_.reference["year-suffix"])} + year_suffix_ir.year_suffix_number = ir_.reference.year_suffix_number + year_suffix_ir.group_var = "important" + end - if not item.position and position_first then - item.position = util.position_map["first"] + local inlines = ir_:flatten(disam_format) + local disam_str = disam_format:output(inlines, context) + -- util.debug("update: " .. ir_.cite_item.id .. ": " .. disam_str) + ir_.disam_str = disam_str + if not self.cite_irs_by_output[disam_str] then + self.cite_irs_by_output[disam_str] = {} + end + self.cite_irs_by_output[disam_str][ir_.ir_index] = ir_ + + end + + cite_ir.is_ambiguous = false + + return cite_ir +end + +function CiteProc:collect_year_suffix_irs(ir) + local year_suffix_irs = {} + if ir.children then + for i, child_ir in ipairs(ir.children) do + if child_ir._type == "YearSuffix" then + table.insert(year_suffix_irs, child_ir) + elseif child_ir.children then + util.extend(year_suffix_irs, + self:collect_year_suffix_irs(child_ir)) end + end + end + return year_suffix_irs +end - local first_reference_note_number = nil - for _, pre_citation in ipairs(citationsPre) do - pre_citation = self.registry.citations[pre_citation[1]] - for _, pre_cite_item in ipairs(pre_citation.citationItems) do - if pre_cite_item.id == cite_item.id then - first_reference_note_number = pre_citation.properties.noteIndex - end - break - end - if first_reference_note_number then - break - end +function CiteProc:find_first_year_ir(ir) + if ir.is_year then + return ir + end + if ir.children then + for _, child_ir in ipairs(ir.children) do + local year_ir = self:find_first_year_ir(child_ir) + if year_ir then + return year_ir + end + end + end + return nil +end + +function CiteProc:render_year_suffix(year_suffix_number) + if year_suffix_number <= 0 then + return nil + end + local year_suffix = "" + while year_suffix_number > 0 do + local i = (year_suffix_number - 1) % 26 + year_suffix = string.char(i + 97) .. year_suffix + year_suffix_number = (year_suffix_number - 1) // 26 + end + -- util.debug(year_suffix) + return year_suffix +end + +local function find_first(ir, check) + if check(ir) then + return ir + end + if ir.children then + for _, child in ipairs(ir.children) do + local target_ir = find_first(child, check) + if target_ir then + return target_ir + end + end + end + return nil +end + +function CiteProc:group_cites(irs) + local disam_format = DisamStringFormat:new() + for _, ir in ipairs(irs) do + local first_names_ir = ir.first_names_ir + if not first_names_ir then + first_names_ir = find_first(ir, function (ir_) + return ir_._element == "names" + end) + if first_names_ir then + local inlines = first_names_ir:flatten(disam_format) + first_names_ir.disam_str = disam_format:output(inlines, context) end - item["first-reference-note-number"] = first_reference_note_number + ir.first_names_ir = first_names_ir + end + end - table.insert(items, item) + local irs_by_name = {} + local name_list = {} + + for _, ir in ipairs(irs) do + local name_str = "" + if ir.first_names_ir then + name_str = ir.first_names_ir.disam_str + end + if not irs_by_name[name_str] then + irs_by_name[name_str] = {} + table.insert(name_list, name_str) end + table.insert(irs_by_name[name_str], ir) end - if #citationsPre > 0 then - local previous_citation_id = citationsPre[#citationsPre][1] - local previous_citation = self.registry.citations[previous_citation_id] - self.registry.previous_citation = previous_citation + local grouped = {} + for _, name_str in ipairs(name_list) do + local irs_with_same_name = irs_by_name[name_str] + for i, ir in ipairs(irs_with_same_name) do + if i < #irs_with_same_name then + ir.own_delimiter = self.style.citation.cite_group_delimiter + end + table.insert(grouped, ir) + end end + return grouped +end - if self.registry.requires_sorting then - self:sort_bibliography() +function CiteProc:collapse_cites(irs) + if self.style.citation.collapse == "citation-number" then + self:collapse_cites_by_citation_number(irs) + elseif self.style.citation.collapse == "year" then + self:collapse_cites_by_year(irs) + elseif self.style.citation.collapse == "year-suffix" then + self:collapse_cites_by_year_suffix(irs) + elseif self.style.citation.collapse == "year-suffix-ranged" then + self:collapse_cites_by_year_suffix_ranged(irs) end +end - local params = { - bibchange = false, - citation_errors = {}, - } +function CiteProc:collapse_cites_by_citation_number(irs) + local cite_groups = {} + local current_group = {} + local previous_citation_number + for i, ir in ipairs(irs) do + local citation_number + local only_citation_number_ir = self:get_only_citation_number(ir) + if only_citation_number_ir then + -- Other irs like locators are not rendered. + -- collapse_CitationNumberRangesWithAffixesGrouped.txt + citation_number = only_citation_number_ir.citation_number + end + if i == 1 then + table.insert(current_group, ir) + elseif citation_number and previous_citation_number and + previous_citation_number + 1 == citation_number then + table.insert(current_group, ir) + else + table.insert(cite_groups, current_group) + current_group = {ir} + end + previous_citation_number = citation_number + end + table.insert(cite_groups, current_group) - local citation_id_note_list = {} - for _, citation_id_note in ipairs(citationsPre) do - table.insert(citation_id_note_list, citation_id_note) + for _, cite_group in ipairs(cite_groups) do + if #cite_group >= 3 then + cite_group[1].own_delimiter = util.unicode["en dash"] + for i = 2, #cite_group - 1 do + cite_group[i].collapse_suppressed = true + end + cite_group[#cite_group].own_delimiter = self.style.citation.after_collapse_delimiter + end + end +end + +function CiteProc:get_only_citation_number(ir) + if ir.citation_number then + return ir end - local note_index = 0 - if citation.properties and citation.properties.noteIndex then - note_index = citation.properties.noteIndex + if not ir.children then + return nil end - table.insert(citation_id_note_list, {citation.citationID, note_index}) - for _, citation_id_note in ipairs(citationsPost) do - table.insert(citation_id_note_list, citation_id_note) + local only_citation_number_ir + for _, child in ipairs(ir.children) do + if child.group_var ~= "missing" then + local citation_number_ir = self:get_only_citation_number(child) + if citation_number_ir then + if only_citation_number_ir then + return nil + else + only_citation_number_ir = citation_number_ir + end + else + return false + end + end end + return only_citation_number_ir +end - local citation_id_cited = {} - for _, citation_id_note in ipairs(citation_id_note_list) do - citation_id_cited[citation_id_note[1]] = true +function CiteProc:collapse_cites_by_year(irs) + local cite_groups = {{}} + local previous_name_str + for i, ir in ipairs(irs) do + local name_str + if ir.first_names_ir then + name_str = ir.first_names_ir.disam_str + end + if i == 1 then + table.insert(cite_groups[#cite_groups], ir) + elseif name_str and name_str == previous_name_str then + -- ir.fist_names_ir was set in the cite grouping stage + -- TODO: and not previous cite suffix + table.insert(cite_groups[#cite_groups], ir) + else + table.insert(cite_groups, {ir}) + end + previous_name_str = name_str end - for citation_id, _ in pairs(self.registry.citations) do - if not citation_id_cited[citation_id] then - self.registry.citations[citation_id] = nil - self.registry.citation_strings[citation_id] = nil + + for _, cite_group in ipairs(cite_groups) do + if #cite_group > 1 then + for i, cite_ir in ipairs(cite_group) do + if i > 1 and cite_ir.first_names_ir then + cite_ir.first_names_ir.collapse_suppressed = true + end + if i == #cite_group then + cite_ir.own_delimiter = self.style.citation.after_collapse_delimiter + elseif i < #cite_group then + -- The delimiter depends on the citation > sort. + -- https://github.com/citation-style-language/test-suite/issues/39#issuecomment-687901688 + if cite_ir.cite_item.locator then + -- Special hack for + cite_ir.own_delimiter = self.style.citation.after_collapse_delimiter + elseif self.style.citation.cite_grouping then + if self.style.citation.sort then + cite_ir.own_delimiter = self.style.citation.cite_group_delimiter + else + cite_ir.own_delimiter = self.style.citation.layout.delimiter + end + else + if self.style.citation.sort then + cite_ir.own_delimiter = self.style.citation.cite_group_delimiter + else + -- disambiguate_YearCollapseWithInstitution.txt + -- disambiguate_InitializeWithButNoDisambiguation.txt ? + cite_ir.own_delimiter = self.style.citation.layout.delimiter + end + end + end + end end end +end - local output = {} +local function find_rendered_year_suffix(ir) + if ir._type == "YearSuffix" then + return ir + end + if ir.children then + for _, child in ipairs(ir.children) do + if child.group_var ~= "missing" then + local year_suffix = find_rendered_year_suffix(child) + if year_suffix then + return year_suffix + end + end + end + end + return nil +end - for i, citation_id_note in ipairs(citation_id_note_list) do - local citation_id = citation_id_note[1] - -- local note_index = citation_id_note[2] - if citation_id == citation.citationID then - local context = { - build = {}, - engine = self, - } - local citation_str = self.style:render_citation(items, context) - - self.registry.citation_strings[citation_id] = citation_str - table.insert(output, {i - 1, citation_str, citation_id}) +function CiteProc:collapse_cites_by_year_suffix(irs) + self:collapse_cites_by_year(irs) + -- Group by disam_str + -- The year-suffix is ommitted in DisamStringFormat + local cite_groups = {{}} + local previous_ir + local previous_year_suffix + for i, ir in ipairs(irs) do + local year_suffix = find_rendered_year_suffix(ir) + ir.rendered_year_suffix_ir = year_suffix + if i == 1 then + table.insert(cite_groups[#cite_groups], ir) + elseif year_suffix and previous_ir.disam_str == ir.disam_str and previous_year_suffix then + -- TODO: and not previous cite suffix + table.insert(cite_groups[#cite_groups], ir) else - -- TODO: correct note_index - -- TODO: update other citations after disambiguation - local citation_str = self.registry.citation_strings[citation_id] - if self.registry.citation_strings[citation_id] ~= citation_str then - params.bibchange = true - self.registry.citation_strings[citation_id] = citation_str - table.insert(output, {i - 1, citation_str, citation_id}) + table.insert(cite_groups, {ir}) + end + previous_ir = ir + previous_year_suffix = year_suffix + end + + for _, cite_group in ipairs(cite_groups) do + if #cite_group > 1 then + for i, cite_ir in ipairs(cite_group) do + if i > 1 then + -- cite_ir.children = {cite_ir.rendered_year_suffix_ir} + -- Set the collapse_suppressed flag rather than removing the child irs. + -- This leaves the disamb ir structure unchanged. + self:suppress_ir_except_child(cite_ir, cite_ir.rendered_year_suffix_ir) + end + if i < #cite_group then + if self.style.citation.cite_grouping then + -- In the current citeproc-js impplementation, explicitly set + -- cite-group-delimiter takes precedence over year-suffix-delimiter. + -- May be changed in the future. + -- https://github.com/citation-style-language/test-suite/issues/50 + cite_ir.own_delimiter = self.style.citation.cite_group_delimiter + else + cite_ir.own_delimiter = self.style.citation.year_suffix_delimiter + end + elseif i == #cite_group then + cite_ir.own_delimiter = self.style.citation.after_collapse_delimiter + end end end end +end - return {params, output} +function CiteProc:suppress_ir_except_child(ir, target) + if ir == target then + ir.collapse_suppressed = false + return false + end + ir.collapse_suppressed = true + if ir.children then + for _, child in ipairs(ir.children) do + if child.group_var ~= "missing" and not child.collapse_suppressed then + if not self:suppress_ir_except_child(child, target) then + ir.collapse_suppressed = false + end + end + end + end + return ir.collapse_suppressed +end + +function CiteProc:collapse_cites_by_year_suffix_ranged(irs) + self:collapse_cites_by_year_suffix(irs) + -- Group by disam_str + local cite_groups = {{}} + local previous_ir + local previous_year_suffix + for i, ir in ipairs(irs) do + local year_suffix_ir = find_rendered_year_suffix(ir) + ir.rendered_year_suffix_ir = year_suffix_ir + if i == 1 then + table.insert(cite_groups[#cite_groups], ir) + elseif year_suffix_ir and previous_ir.disam_str == ir.disam_str and previous_year_suffix and + year_suffix_ir.year_suffix_number == previous_year_suffix.year_suffix_number + 1 then + -- TODO: and not previous cite suffix + table.insert(cite_groups[#cite_groups], ir) + else + table.insert(cite_groups, {ir}) + end + previous_ir = ir + previous_year_suffix = year_suffix_ir + end + + for _, cite_group in ipairs(cite_groups) do + if #cite_group > 2 then + for i, cite_ir in ipairs(cite_group) do + if i == 1 then + cite_ir.own_delimiter = util.unicode["en dash"] + elseif i < #cite_group then + cite_ir.collapse_suppressed = true + end + end + end + end end -function CiteProc:makeCitationCluster (citation_items) +function CiteProc:makeCitationCluster(citation_items) local items = {} - for _, cite_item in ipairs(citation_items) do + for i, cite_item in ipairs(citation_items) do cite_item.id = tostring(cite_item.id) - local position_first = (self.registry.registry[cite_item.id] == nil) local item_data = self:get_item(cite_item.id) -- Create a wrapper of the orignal item from registry so that -- it may hold different `locator` or `position` values for cites. - local item = setmetatable({}, {__index = function (_, key) - if cite_item[key] then - return cite_item[key] - else - return item_data[key] + local item = setmetatable(cite_item, {__index = item_data}) + + -- Use "page" as locator label if missing + -- label_PluralWithAmpersand.txt + if item.locator and not item.label then + item.label = "page" + end + + item.position_level = util.position_map["first"] + if self.cite_first_note_numbers[cite_item.id] then + item.position_level = util.position_map["subsequent"] + else + self.cite_first_note_numbers[cite_item.id] = 0 + end + + local preceding_cite + if i == 1 then + local previous_citation = self.registry.previous_citation + if previous_citation then + if #previous_citation.citationItems == 1 and previous_citation.citationItems[1].id == item.id then + preceding_cite = previous_citation.citationItems[1] + end end - end}) + elseif citation_items[i - 1].id == item.id then + preceding_cite = citation_items[i - 1] + end - if not item.position and position_first then - item.position = util.position_map["first"] + if preceding_cite then + item.position_level = self:_get_cite_position(item, preceding_cite) end + table.insert(items, item) end @@ -242,134 +1620,252 @@ function CiteProc:makeCitationCluster (citation_items) self:sort_bibliography() end - local context = { - build = {}, - engine=self, - } - local res = self.style:render_citation(items, context) + local res = self:build_cluster(items) + + -- local context = { + -- build = {}, + -- engine=self, + -- } + -- local res = self.style:render_citation(items, context) + self.registry.previous_citation = { citationID = "pseudo-citation", citationItems = items, properties = { - noteIndex = 1, + noteIndex = 0, } } return res end function CiteProc:makeBibliography() - local items = {} + if not self.style.bibliography then + return {{}, {}} + end + + local output_format = self.output_format + + local res = {} + + self.registry.longest_label = "" + self.registry.maxoffset = 0 + + for _, id in ipairs(self:get_sorted_refs()) do + local ref = self.registry.registry[id] + + local state = IrState:new() + local context = Context:new() + context.engine = self + context.style = self.style + context.area = self.style.bibliography + context.in_bibliography = true + context.locale = self:get_locale(self.lang) + context.name_inheritance = self.style.bibliography.name_inheritance + context.format = output_format + context.id = id + context.cite = nil + context.reference = self:get_item(id) + + local ir = self.style.bibliography:build_ir(self, state, context) + -- util.debug(ir) + ir.reference = context.reference + + -- Add year-suffix + self:add_bibliography_year_suffix(ir) + + -- The layout output may be empty: sort_OmittedBibRefNonNumericStyle.txt + if ir then + local flat = ir:flatten(output_format) + local str = output_format:output_bibliography_entry(flat, context) + table.insert(res, str) + end + end + + local bib_start = self.output_format.markups["bibstart"] + local bib_end = self.output_format.markups["bibend"] + if type(bib_start) == "function" then + bib_start = bib_start(self) + end + if type(bib_end) == "function" then + bib_end = bib_end(self) + end + + local params = { + hangingindent = self.style.bibliography.hanging_indent, + ["second-field-align"] = self.style.bibliography.second_field_align ~= nil, + linespacing = self.style.bibliography.line_spacing, + entryspacing = self.style.bibliography.entry_spacing, + maxoffset = self.registry.maxoffset, + bibstart = bib_start, + bibend = bib_end, + entry_ids = util.clone(self.registry.reflist), + } + + return {params, res} +end +function CiteProc:get_sorted_refs() if self.registry.requires_sorting then self:sort_bibliography() end + return self.registry.reflist +end - for _, id in ipairs(self.registry.reflist) do - local item = self.registry.registry[id] - table.insert(items, item) +function CiteProc:add_bibliography_year_suffix(ir) + if not ir.reference.year_suffix_number then + return end - local context = { - build = {}, - engine=self, - } - local res = self.style:render_biblography(items, context) - return res + local year_suffix_number = ir.reference.year_suffix_number + + if not ir.year_suffix_irs then + ir.year_suffix_irs = self:collect_year_suffix_irs(ir) + if #ir.year_suffix_irs == 0 then + local year_ir = self:find_first_year_ir(ir) + -- util.debug(year_ir) + if year_ir then + local year_suffix_ir = YearSuffix:new({}, self.style.citation) + table.insert(year_ir.children, year_suffix_ir) + table.insert(ir.year_suffix_irs, year_suffix_ir) + end + end + end + + for _, year_suffix_ir in ipairs(ir.year_suffix_irs) do + year_suffix_ir.inlines = {PlainText:new(ir.reference["year-suffix"])} + year_suffix_ir.group_var = "important" + end + + end -function CiteProc:set_formatter(format) - self.formatter = formats[format] +function CiteProc:set_output_format(format) + if format == "latex" then + self.output_format = LatexWriter:new() + elseif format == "html" then + self.output_format = HtmlWriter:new() + end end function CiteProc:enable_linking() - self.linking_enabled = true + self.opt.url_link = true + self.opt.doi_link = true end function CiteProc:disable_linking() - self.linking_enabled = false + self.opt.url_link = false + self.opt.doi_link = false end -function CiteProc.set_base_class (node) - if node:is_element() then - local name = node:get_element_name() - local element_class = nodes[name] - if element_class then - element_class:set_base_class(node) - else - element.Element:set_base_class(node) +function CiteProc.create_element_tree(node) + local element_name = node:get_element_name() + local element_class = nodes[element_name] + local el = nil + if element_class then + el = element_class:from_node(node) + end + if el then + for i, child in ipairs(node:get_children()) do + if child:is_element() then + local child_element = CiteProc.create_element_tree(child) + if child_element then + if not el.children then + el.children = {} + end + table.insert(el.children, child_element) + end + end end end + return el end -function CiteProc:get_style_class() - return self.style:get_attribute("class") or "in-text" -end - -function CiteProc:get_item (id) +function CiteProc:get_item(id) local item = self.registry.registry[id] if not item then item = self:_retrieve_item(id) if not item then return nil end + item = self:process_extra_note(item) table.insert(self.registry.reflist, id) item["citation-number"] = #self.registry.reflist self.registry.registry[id] = item self.registry.requires_sorting = true end - local res = {} - setmetatable(res, {__index = item}) - return res + -- local res = {} + -- setmetatable(res, {__index = item}) + -- return res + return item end -function CiteProc:_retrieve_item (id) +function CiteProc:_retrieve_item(id) -- Retrieve, copy, and normalize local res = {} local item = self.sys.retrieveItem(id) if not item then - util.warning(string.format('Failed to retrieve item "%s"', id)) + util.warning(string.format('Failed to find entry "%s"', id)) return nil end item.id = tostring(item.id) for key, value in pairs(item) do - if key == "title" then - value = self.normalize_string(value) - end res[key] = value end - if res["page"] and not res["page-first"] then - local page_first = util.split(res["page"], "%s*[&,-]%s*")[1] - page_first = util.split(page_first, util.unicode["en dash"])[1] - res["page-first"] = page_first - end + -- if res["page"] and not res["page-first"] then + -- local page_first = util.split(res["page"], "%s*[&,-]%s*")[1] + -- page_first = util.split(page_first, util.unicode["en dash"])[1] + -- res["page-first"] = page_first + -- end return res end -function CiteProc.normalize_string (str) - if not str or str == "" then - return str - end - -- French punctuation spacing - if type(str) == "string" then - str = string.gsub(str, " ;", util.unicode["narrow no-break space"] .. ";") - str = string.gsub(str, " %?", util.unicode["narrow no-break space"] .. "?") - str = string.gsub(str, " !", util.unicode["narrow no-break space"] .. "!") - str = string.gsub(str, " »", util.unicode["narrow no-break space"] .. "»") - str = string.gsub(str, "« ", "«" .. util.unicode["narrow no-break space"]) +-- TODO: Nomalize all inputs +function CiteProc:process_extra_note(item) + if item.note then + local note_fields = {} + for _, line in ipairs(util.split(item.note, "%s*\r?\n%s*")) do + -- util.debug(line) + local splits = util.split(line, ":%s+", 1) + -- util.debug(splits) + if #splits == 2 then + local field, value = table.unpack(splits) + -- util.debug(field) + + local variable_type = util.variable_types[field] + if not item[field] or field == "type" or variable_type == "date" then + if variable_type == "number" then + item[field] = value + elseif variable_type == "date" then + item[field] = util.parse_iso_date(value) + elseif variable_type == "name" then + if not note_fields[field] then + note_fields[field] = {} + end + table.insert(note_fields[field], util.parse_extra_name(value)) + else + item[field] = value + end + end + end + end + for field, value in pairs(note_fields) do + item[field] = value + end end - -- local text = str - local text = richtext.new(str) - return text + return item end function CiteProc:sort_bibliography() -- Sort the items in registry according to the `sort` in `bibliography.` -- This will update the `citation-number` of each item. - local bibliography_sort = self.style:get_path("style bibliography sort")[1] + local bibliography_sort = nil + if self.style.bibliography and self.style.bibliography.sort then + bibliography_sort = self.style.bibliography.sort + end if not bibliography_sort then return end @@ -378,44 +1874,119 @@ function CiteProc:sort_bibliography() table.insert(items, self.registry.registry[id]) end - local context = { - engine = self, - style = self.style, - mode = "bibliography", - } - context = self.style:process_context(context) - context = self.style:get_path("style bibliography")[1]:process_context(context) + local state = IrState:new() + local context = Context:new() + context.engine = self + context.style = self.style + context.area = self.style.bibliography + context.in_bibliography = true + context.locale = self:get_locale(self.lang) + context.name_inheritance = self.style.bibliography.name_inheritance + context.format = SortStringFormat:new() + -- context.id = id + context.cite = nil + -- context.reference = self:get_item(id) - bibliography_sort:sort(items, context) + bibliography_sort:sort(items, state, context) self.registry.reflist = {} + self.tainted_item_ids = {} for i, item in ipairs(items) do + if item["citation-number"] ~= i then + self.tainted_item_ids[item.id] = true + end item["citation-number"] = i - table.insert(self.registry.reflist, item.id) + self.registry.reflist[i] = item.id end self.registry.requires_sorting = false end -function CiteProc:get_system_locale (lang) +function CiteProc:get_locale(lang) + if string.len(lang) == 2 then + lang = util.primary_dialects[lang] or lang + end + local locale = self.locales[lang] + if locale then + return locale + else + return self:get_merged_locales(lang) + end +end + +function CiteProc:get_merged_locales(lang) + local fall_back_locales = {} + + local language = string.sub(lang, 1, 2) + local primary_dialect = util.primary_dialects[language] + + -- 1. In-style cs:locale elements + -- i. `xml:lang` set to chosen dialect, “de-AT” + table.insert(fall_back_locales, self.style.locales[lang]) + + -- ii. `xml:lang` set to matching language, “de” (German) + if language and language ~= lang then + table.insert(fall_back_locales, self.style.locales[language]) + end + + -- iii. `xml:lang` not set + table.insert(fall_back_locales, self.style.locales["@generic"]) + + -- 2. Locale files + -- iv. `xml:lang` set to chosen dialect, “de-AT” + if lang then + table.insert(fall_back_locales, self:get_system_locale(lang)) + end + + -- v. `xml:lang` set to matching primary dialect, “de-DE” (Standard German) + -- (only applicable when the chosen locale is a secondary dialect) + if primary_dialect and primary_dialect ~= lang then + table.insert(fall_back_locales, self:get_system_locale(primary_dialect)) + end + + -- vi. `xml:lang` set to “en-US” (American English) + if lang ~= "en-US" and primary_dialect ~= "en-US" then + table.insert(fall_back_locales, self:get_system_locale("en-US")) + end + + -- Merge locales + + local locale = Locale:new() + for i = #fall_back_locales, 1, -1 do + local fall_back_locale = fall_back_locales[i] + locale:merge(fall_back_locale) + end + + self.locales[lang] = locale + return locale +end + +function CiteProc:get_system_locale(lang) local locale = self.system_locales[lang] - if not locale then - locale = self.sys.retrieveLocale(lang) - if not locale then - util.warning(string.format("Failed to retrieve locale \"%s\"", lang)) - return nil - end - if type(locale) == "string" then - locale = dom.parse(locale) - end - locale:traverse_elements(self.set_base_class) - locale = locale:get_path("locale")[1] - locale:root_node().engine = self - locale:root_node().style = self.style - self.system_locales[lang] = locale + if locale then + return locale + end + + local locale_str = self.sys.retrieveLocale(lang) + if not locale_str then + util.warning(string.format("Failed to retrieve locale \"%s\"", lang)) + return nil end + local locale_xml = dom.parse(locale_str) + local root_element = locale_xml:get_path("locale")[1] + locale = Locale:from_node(root_element) + self.system_locales[lang] = locale return locale end +function CiteProc:get_style_class() + if self.style and self.style.class then + return self.style.class + else + return nil + end +end + + engine.CiteProc = CiteProc return engine -- cgit v1.2.3