summaryrefslogtreecommitdiff
path: root/biblio/citation-style-language/citeproc-bibtex.lua
diff options
context:
space:
mode:
Diffstat (limited to 'biblio/citation-style-language/citeproc-bibtex.lua')
-rw-r--r--biblio/citation-style-language/citeproc-bibtex.lua713
1 files changed, 713 insertions, 0 deletions
diff --git a/biblio/citation-style-language/citeproc-bibtex.lua b/biblio/citation-style-language/citeproc-bibtex.lua
new file mode 100644
index 0000000000..645fb5384b
--- /dev/null
+++ b/biblio/citation-style-language/citeproc-bibtex.lua
@@ -0,0 +1,713 @@
+--
+-- Copyright (c) 2021-2022 Zeping Lee
+-- Released under the MIT license.
+-- Repository: https://github.com/zepinglee/citeproc-lua
+--
+
+--[[
+ A PEG-based implementation of a Bib(La)TeX dateabase (.bib) parser
+ References notes: tools/bibtex-parser-notes.md
+--]]
+
+local bibtex = {}
+
+local lpeg = require("lpeg")
+local unicode = require("unicode")
+local bibtex_data = require("citeproc-bibtex-data")
+local util = require("citeproc-util")
+
+local P = lpeg.P
+local R = lpeg.R
+local S = lpeg.S
+local C = lpeg.C
+local Cc = lpeg.Cc
+local Cf = lpeg.Cf
+local Cg = lpeg.Cg
+local Cmt = lpeg.Cmt
+local Cp = lpeg.Cp
+local Ct = lpeg.Ct
+local V = lpeg.V
+
+bibtex.strings = {} -- Stores all the @STRINGs
+for key, value in pairs(bibtex_data.macros) do
+ bibtex.strings[key] = value.value
+end
+
+
+-- Parse BibTeX content and convert to CSL-JSON
+function bibtex.parse(str)
+ local bibliography = bibtex.parse_bibtex(str, bibtex.strings)
+ local csl_json_items = bibtex.convert_csl_json(bibliography)
+ return csl_json_items
+end
+
+
+-- The case-insensitivity trick is taken from <http://boston.conman.org/2020/06/05.1>.
+local function ignore_case(str)
+ local char = R("AZ", "az") / function (c) return P(c:lower()) + P(c:upper()) end
+ + P(1) / function (c) return P(c) end
+ return Cf(char^1, function (a, b) return a * b end):match(str)
+end
+
+ local function normalize_white_space(str)
+ local res = string.gsub(str, "%s+", " ")
+ return res
+ end
+
+-- Based on the grammar described at <https://github.com/aclements/biblib>.
+function bibtex.get_bibtex_grammar()
+ local comment = (1 - P"@")^0
+ local space = S(" \t\r\n")^0
+ local comment_cmd = ignore_case("comment")
+ local balanced = P{ "{" * V(1)^0 * "}" + (1 - S"{}") }
+ local ident = (- R"09") * (R"\x20\x7F" - S" \t\"#%'(),={}")^1
+
+ local piece = (P"{" * (balanced^0 / normalize_white_space) * P"}")
+ + (P'"' * ((balanced - P'"')^0 / normalize_white_space) * P'"')
+ + C(R("09")^1)
+ + C(ident) / function (name)
+ return {
+ category = "string",
+ name = string.lower(name),
+ }
+ end
+ local value = Ct(piece * (space * P"#" * space * piece)^0)
+
+ local string_body = Cg(ident / string.lower, "name") * space * P"=" * space * Cg(value, "contents")
+ local string_cmd = Ct(Cg(Cc"string", "category") * ignore_case("string") * space * (
+ P"{" * space * string_body * space * P"}"
+ + P"(" * space * string_body * space * P")"
+ ))
+
+ local preamble_body = Cg(value, "contents")
+ local preamble_cmd = Ct(Cg(Cc"preamble", "category") * ignore_case("preamble") * space * (
+ P"{" * space * preamble_body * space * P"}"
+ + P"(" * space * preamble_body * space * P")"
+ ))
+
+ local key = (1 - S", \t}\r\n")^0
+ local key_paren = (1 - S", \t\r\n")^0
+ local field_value_pair = (ident / string.lower) * space * P"=" * space * value -- * record_success_position()
+
+ local entry_body = Cf(Ct"" * (P"," * space * Cg(field_value_pair))^0 * (P",")^-1, rawset)
+ local entry = Ct(Cg(Cc"entry", "category") * Cg(ident / string.lower, "type") * space * (
+ P"{" * space * Cg(key, "key") * space * Cg(entry_body, "fields")^-1 * space * (P"}")
+ + P"(" * space * Cg(key_paren, "key") * space * Cg(entry_body, "fields")^-1 * space * (P")")
+ ))
+
+ local command_or_entry = P"@" * space * (comment_cmd + preamble_cmd + string_cmd + entry)
+
+ -- The P(-1) causes nil parsing result in case of error.
+ local bibtex_grammar = Ct(comment * (command_or_entry * comment)^0) * P(-1)
+
+ return bibtex_grammar
+end
+
+
+bibtex.bibtex_grammar = bibtex.get_bibtex_grammar()
+
+
+-- Return a list of entries, strings, preamble
+function bibtex.parse_bibtex_objects(str)
+ return bibtex.bibtex_grammar:match(str)
+end
+
+
+local function concat_strings(pieces, strings)
+ local value = ""
+ for _, piece in ipairs(pieces) do
+ if type(piece) == "string" then
+ value = value .. piece
+ elseif type(piece) == "table" and piece.category == "string" then
+ local piece_str = strings[piece.name]
+ if piece_str then
+ value = value .. piece_str
+ else
+ util.warning(string.format('String name "%s" is undefined."', piece.name))
+ end
+ end
+ end
+ value = normalize_white_space(value)
+ return value
+end
+
+
+function bibtex.parse_bibtex(content, strings)
+ if strings then
+ strings = setmetatable({}, {__index = strings})
+ else
+ strings = setmetatable({}, {__index = bibtex.strings})
+ end
+ local bib_objects = bibtex.parse_bibtex_objects(content)
+
+ local res = {
+ entries = {},
+ strings = {},
+ preamble = "",
+ }
+ local exceptions = {}
+ for _, object in ipairs(bib_objects) do
+ if object.category == "entry" then
+ object.category = nil
+ for field, value in pairs(object.fields) do
+ local value_str = concat_strings(value, strings)
+ if field ~= "url" then
+ value_str = bibtex.to_unicode(value_str)
+ end
+ object.fields[field] = value_str
+ end
+ table.insert(res.entries, object)
+
+ elseif object.category == "string" then
+ local string_value = concat_strings(object.contents, strings)
+ strings[object.name] = string_value
+ res.strings[object.name] = string_value
+
+ elseif object.category == "preamble" then
+ res.preamble = res.preamble .. concat_strings(object.contents, strings)
+
+ -- elseif object.category == "comment" then
+ -- Is this really needed?
+
+ elseif object.category == "exception" then
+ -- TODO
+
+ end
+ end
+
+ return res
+end
+
+
+local function get_line_number(str, position)
+ local line_number = 1
+ for i = 1, position do
+ if string.sub(str, i, i) == "\n" then
+ line_number = line_number + 1
+ end
+ end
+ return line_number
+end
+
+
+function bibtex.get_latex_grammar()
+ local space = S(" \t\r\n")^0
+ local specials = P"\\$" / "$"
+ + P"\\%" / "%"
+ + P"\\&" / "&"
+ + P"\\#" / "#"
+ + P"\\_" / "_"
+ + P"\\{" / "{"
+ + P"\\}" / "}"
+ + P"~" / util.unicode["no-break space"]
+ local control_sequence = C(P"\\" * (R("AZ", "az")^1 + 1) * space) / function (cs)
+ return {
+ type = "control_sequence",
+ name = util.rstrip(cs),
+ raw = cs,
+ }
+ end
+ local math = P"$" * C((P"\\$" + 1 - S"$")^0) * P"$" / function (math_text)
+ return {
+ type = "math",
+ contents = math_text,
+ }
+ end
+ local ligatures = P"``" / util.unicode["left double quotation mark"]
+ + P"`" / util.unicode["left single quotation mark"]
+ + P"''" / util.unicode["right double quotation mark"]
+ + P"'" / util.unicode["right single quotation mark"]
+ + P"---" / util.unicode["em dash"]
+ + P"--" / util.unicode["en dash"]
+ local plain_text = C(1 - S"{}$\\")
+ local latex_grammar = P{
+ "latex_text";
+ latex_text = Ct((specials + control_sequence + math + ligatures + specials + V"group" + plain_text)^0),
+ group = P"{" * V"latex_text" * P"}" / function (group_contents)
+ return {
+ type = "group",
+ contents = group_contents,
+ }
+ end,
+ }
+ return latex_grammar
+end
+
+bibtex.latex_grammar = bibtex.get_latex_grammar()
+
+
+function bibtex.to_unicode(str)
+ local tokens = bibtex.latex_grammar:match(str)
+ local res = bibtex.unescape_tokens(tokens, 0)
+ return res
+end
+
+local format_commands_with_argment = {
+ ["\\textrm"] = {'<span class="nodecor">', '</span>'},
+ ["\\textbf"] = {'<b>', '</b>'},
+ ["\\textit"] = {'<i>', '</i>'},
+ ["\\textsl"] = {'<i>', '</i>'},
+ ["\\emph"] = {'<i>', '</i>'},
+ ["\\textsc"] = {'<span style="font-variant: small-caps;">', '</span>'},
+ ["\\textsuperscript"] = {'<sup>', '</sup>'},
+ ["\\textsubscript"] = {'<sub>', '</sub>'},
+}
+
+local format_commands_without_argment = {
+ ["\\rm"] = {'<span class="nodecor">', '</span>'},
+ ["\\upshape"] = {'<span class="nodecor">', '</span>'},
+ ["\\bf"] = {'<b>', '</b>'},
+ ["\\bfseries"] = {'<b>', '</b>'},
+ ["\\it"] = {'<i>', '</i>'},
+ ["\\itshape"] = {'<i>', '</i>'},
+ ["\\sl"] = {'<i>', '</i>'},
+ ["\\slshape"] = {'<i>', '</i>'},
+ ["\\em"] = {'<i>', '</i>'},
+ ["\\scshape"] = {'<span style="font-variant: small-caps;">', '</span>'},
+ ["\\sc"] = {'<span style="font-variant: small-caps;">', '</span>'},
+}
+
+
+function bibtex.unescape_tokens(tokens, brace_level)
+ local res = ""
+ local skip_token = false
+ for i, token in ipairs(tokens) do
+ if skip_token then
+ skip_token = false
+ else
+ if type(token) == "string" then
+ res = res .. token
+ elseif type(token) == "table" then
+ if token.type == "control_sequence" then
+ local code_point = bibtex_data.unicode_commands[token.name]
+ if code_point then
+ local unicode_char
+ if type(code_point) == "string" then
+ unicode_char = utf8.char(tonumber(code_point, 16))
+ elseif type(code_point) == "table" then
+ -- The command takes an argument (\"{o})
+ local arg
+ if i < #tokens then
+ local next_token = tokens[i + 1]
+ if type(next_token) == "string" then
+ arg = next_token
+ skip_token = true
+ elseif type(next_token) == "table" then
+ if next_token.type == "control_sequence" then
+ arg = next_token.name
+ skip_token = true
+ elseif next_token.type == "group" then
+ if #next_token.contents == 0 then
+ arg = "{}"
+ skip_token = true
+ elseif #next_token.contents == 1 then
+ next_token = next_token.contents[1]
+ if type(next_token) == "string" then
+ arg = next_token
+ skip_token = true
+ elseif type(next_token) == "table" then
+ arg = next_token.name
+ skip_token = true
+ end
+ end
+ end
+ end
+ end
+ if arg and code_point[arg] then
+ unicode_char = utf8.char(tonumber(code_point[arg], 16))
+ end
+ end
+ if unicode_char then
+ res = res .. unicode_char
+ else
+ res = res .. '<span class="nocase">' .. token.raw .. '</span>'
+ end
+
+ elseif format_commands_with_argment[token.name] then
+ -- Commands like `\textbf`
+ if i < #tokens then
+ local next_token = tokens[i + 1]
+ if type(next_token) == "table" and next_token.type == "group" then
+ local tags = format_commands_with_argment[token.name]
+ if brace_level == 0 then
+ res = res .. tags[1] .. '<span class="nocase">' .. bibtex.unescape_tokens(next_token.contents, brace_level + 1) .. '</span>' .. tags[2]
+ else
+ res = res .. tags[1] .. bibtex.unescape_tokens(next_token.contents, brace_level + 1) .. tags[2]
+ end
+ skip_token = true
+ else
+ res = res .. '<span class="nocase">' .. token.raw .. '</span>'
+ end
+ else
+ res = res .. '<span class="nocase">' .. token.raw .. '</span>'
+ end
+
+ else
+ res = res .. '<span class="nocase">' .. token.raw .. '</span>'
+
+ end
+
+ elseif token.type == "math" then
+ -- res = res .. "$" .. token .. "$"
+ res = res .. '<span class="nocase">' .. token.raw .. '</span>'
+
+ elseif token.type == "group" then
+ local first_token = token.contents[1]
+ if type(first_token) == "table" and first_token.type == "control_sequence" then
+ local tags = format_commands_without_argment[first_token.name]
+ if tags then
+ table.remove(token.contents, 1)
+ res = res .. tags[1] .. bibtex.unescape_tokens(token.contents, brace_level) .. tags[2]
+ else
+ res = res .. bibtex.unescape_tokens(token.contents, brace_level)
+ end
+ else
+ if brace_level == 0 then
+ res = res .. '<span class="nocase">' .. bibtex.unescape_tokens(token.contents, brace_level + 1) .. '</span>'
+ else
+ res = res .. '<span class="nocase">' .. bibtex.unescape_tokens(token.contents, brace_level + 1) .. '</span>'
+ end
+ end
+
+ end
+ end
+ end
+ end
+ return res
+end
+
+-- function bibtex.format_syntanx_error(str, position, file_path)
+-- local line_number = 1
+-- local line_start = 1
+-- local column_number = 0
+-- for i = 1, position do
+-- column_number = column_number + 1
+-- if string.sub(str, i, i) == "\n" then
+-- line_number = line_number + 1
+-- line_start = i + 1
+-- column_number = 0
+-- end
+-- end
+-- -- return line_number, column_number
+
+-- local res
+-- if file_path then
+-- res = string.format('File "%s", line %d\n', file_path, line_number)
+-- else
+-- res = string.format('Line %d\n', line_number)
+-- end
+-- local line_end = position
+-- while line_end < #str + 1 and string.sub(str, line_end, line_end) ~= "\n" do
+-- line_end = line_end + 1
+-- end
+-- res = res .. " " .. string.sub(str, line_start, line_end - 1) .. "\n"
+-- res = res .. "Syntax Error"
+-- return res
+-- end
+
+
+function bibtex.convert_csl_json(bibliography)
+ local csl_json_data = {}
+ for _, entry in ipairs(bibliography.entries) do
+ local item = { -- CSL-JSON item
+ id = entry.key,
+ type = "document",
+ }
+
+ -- CSL types
+ local type_data = bibtex_data.types[entry.type]
+ if type_data and type_data.csl then
+ item.type = type_data.csl
+ end
+
+ for field, value in pairs(entry.fields) do
+ local csl_field, csl_value = bibtex.convert_field(field, value)
+
+ if csl_field and csl_value and not item[csl_field] then
+ item[csl_field] = csl_value
+ end
+ end
+
+ bibtex.process_special_fields(item, entry.fields)
+
+ table.insert(csl_json_data, item)
+ end
+
+ return csl_json_data
+end
+
+
+function bibtex.convert_field(bib_field, value)
+ local field_data = bibtex_data.fields[bib_field]
+ if not field_data then
+ return nil, nil
+ end
+ local csl_field = field_data.csl
+ if not csl_field then
+ return nil, nil
+ end
+
+ local field_type = field_data.type
+ if field_type == "name" then
+ value = bibtex.parse_names(value)
+
+ elseif field_type == "date" then
+ value = bibtex.parse_date(value)
+
+ -- elseif bib_field == "title" or bib_field == "booktitle" then
+ -- value = bibtex.convert_sentence_case(value)
+
+ elseif bib_field == "volume" or bib_field == "pages" then
+ value = string.gsub(value, util.unicode["en dash"], "-")
+ end
+
+ return csl_field, value
+end
+
+
+function bibtex.parse_names(str)
+ -- "{International Federation of Library Association and Institutions}"
+ local names = {}
+ local brace_level = 0
+ local name = ""
+ local last_word = ""
+ for i = 1, #str do
+ local char = string.sub(str, i, i)
+ if char == " " then
+ if brace_level == 0 and last_word == "and" then
+ table.insert(names, name)
+ name = ""
+ else
+ if name ~= "" then
+ name = name .. " "
+ end
+ name = name .. last_word
+ end
+ last_word = ""
+ else
+ last_word = last_word .. char
+ if char == "{" then
+ brace_level = brace_level + 1
+ elseif char == "}" then
+ brace_level = brace_level - 1
+ end
+ end
+ end
+
+ if name ~= "" then
+ name = name .. " "
+ end
+ name = name .. last_word
+ table.insert(names, name)
+
+ for i, name in ipairs(names) do
+ names[i] = bibtex.parse_single_name(name)
+ end
+ return names
+end
+
+function bibtex.parse_single_name(str)
+ local literal = string.match(str, "^{(.*)}$")
+ if literal then
+ return {
+ literal = literal,
+ }
+ end
+
+ local name_parts = util.split(str, ",%s*")
+ if #name_parts > 1 then
+ return bibtex.parse_revesed_name(name_parts)
+ else
+ return bibtex.parse_non_revesed_name(str)
+ end
+end
+
+function bibtex.parse_revesed_name(name_parts)
+ local name = {}
+ local von, last, jr, first
+ if #name_parts == 2 then
+ first = name_parts[2]
+ elseif #name_parts >= 3 then
+ jr = name_parts[2]
+ first = name_parts[3]
+ end
+ if first and first ~= "" then
+ name.given = first
+ end
+ if jr and jr ~= "" then
+ name.suffix = jr
+ end
+
+ last = name_parts[1]
+ local words = util.split(last)
+ local index = #words - 1
+ while index > 0 and string.match(words[index], "^%L") do
+ index = index - 1
+ end
+ name.family = util.concat(util.slice(words, index + 1), " ")
+ if index >= 1 then
+ von = util.concat(util.slice(words, 1, index), " ")
+ name["non-dropping-particle"] = von
+ end
+ return name
+end
+
+function bibtex.parse_non_revesed_name(str)
+ local name = {}
+ local words = util.split(str)
+
+ local index = 1
+ -- TODO: case determination for pseudo-characters (e.g., "\bb{BB}")
+ while index < #words and string.match(words[index], "^%L") do
+ index = index + 1
+ end
+ if index > 1 then
+ name.given = util.concat(util.slice(words, 1, index - 1), " ")
+ end
+
+ local particle_start_index = index
+ index = #words - 1
+ while index >= particle_start_index and string.match(words[index], "^%L") do
+ index = index - 1
+ end
+ if index >= particle_start_index then
+ local particles = util.slice(words, particle_start_index, index)
+ -- TODO: distiguish dropping and non-dropping particles
+ name["non-dropping-particle"] = util.concat(particles, " ")
+ end
+ name.family = util.concat(util.slice(words, index + 1), " ")
+
+ return name
+end
+
+function bibtex.parse_date(str)
+ local date_range = util.split(str, "/")
+ if #date_range == 1 then
+ date_range = util.split(str, util.unicode["en dash"])
+ end
+
+ local literal = { literal = str }
+
+ if #date_range > 2 then
+ return literal
+ end
+
+ local date = {}
+ date["date-parts"] = {}
+ for _, date_part in ipairs(date_range) do
+ local date_ = bibtex.parse_single_date(date_part)
+ if not date_ then
+ return literal
+ end
+ table.insert(date["date-parts"], date_)
+ end
+ return date
+end
+
+function bibtex.parse_single_date(str)
+ local date = {}
+ for _, date_part in ipairs(util.split(str, "%-")) do
+ if not string.match(date_part, "^%d+$") then
+ return nil
+ end
+ table.insert(date, tonumber(date_part))
+ end
+ return date
+end
+
+
+-- -- TODO: check if the original title is in sentence case
+-- function bibtex.convert_sentence_case(str)
+-- local res = ""
+-- local to_lower = false
+-- local brace_level = 0
+-- for _, code_point in utf8.codes(str) do
+-- local char = utf8.char(code_point)
+-- if to_lower and brace_level == 0 then
+-- char = string.lower(char)
+-- end
+-- if string.match(char, "%S") then
+-- to_lower = true
+-- end
+-- if char == "{" then
+-- brace_level = brace_level + 1
+-- char = ""
+-- elseif char == "}" then
+-- brace_level = brace_level - 1
+-- char = ""
+-- elseif char == ":" then
+-- to_lower = false
+-- end
+-- res = res .. char
+-- end
+-- return res
+-- end
+
+
+
+function bibtex.process_special_fields(item, bib_fields)
+ -- Default entry type `document`
+ if item.type == "document" then
+ if item.URL then
+ item.type = "webpage"
+ else
+ item.type = "article"
+ end
+ end
+
+ -- event-title: for compatibility with CSL v1.0.1 and earlier versions
+ if item["event-title"] then
+ item.event = item["event-title"]
+ end
+
+ -- issued date
+ if bib_fields.year and not item.issued then
+ item.issued = bibtex.parse_date(bib_fields.year)
+ end
+ local month = bib_fields.month
+ if month and string.match(month, "^%d+$") then
+ if item.issued and item.issued["date-parts"] and
+ item.issued["date-parts"][1] and
+ item.issued["date-parts"][1][2] == nil then
+ item.issued["date-parts"][1][2] = tonumber(month)
+ end
+ end
+
+ -- language: convert `babel` language to ISO 639-1 language code
+ if not item.language and bib_fields.language then
+ item.language = bib_fields.language
+ end
+ if item.language then
+ local language_code = bibtex_data.language_code_map[item.language]
+ if language_code then
+ item.language = language_code
+ end
+ end
+ -- if not item.language then
+ -- if util.has_cjk_char(item.title) then
+ -- item.language = "zh"
+ -- end
+ -- end
+
+ -- number
+ if item.number then
+ if item.type == "article-journal" or item.type == "article-magazine" or item.type == "article-newspaper" or item.type == "periodical" then
+ if not item.issue then
+ item.issue = item.number
+ item.number = nil
+ end
+ elseif item["collection-title"] and not item["collection-number"] then
+ item["collection-number"] = item.number
+ item.number = nil
+ end
+ end
+
+ -- PMID
+ if bib_fields.eprint and string.lower(bib_fields.eprinttype) == "pubmed" and not item.PMID then
+ item.PMID = bib_fields.eprint
+ end
+
+end
+
+
+return bibtex