path: root/biblio/citation-style-language/citeproc-bib.lua

--
-- Copyright (c) 2021-2022 Zeping Lee
-- Released under the MIT license.
-- Repository: https://github.com/zepinglee/citeproc-lua
--

--[[
  A naive implementation of a Bib(La)TeX dateabase (.bib) parser
  References:
  - http://mirrors.ctan.org/biblio/bibtex/base/btxdoc.pdf
  - http://mirrors.ctan.org/info/bibtex/tamethebeast/ttb_en.pdf
  - https://github.com/brechtm/citeproc-py/blob/master/citeproc/source/bibtex/bibparse.py
  - http://maverick.inria.fr/~Xavier.Decoret/resources/xdkbibtex/bibtex_summary.html
  - https://github.com/pcooksey/bibtex-js/blob/master/src/bibtex_js.js
--]]

local bib = {}

require("lualibs")
local unicode = require("unicode")

local util = require("citeproc-util")


local path = "citeproc-bib-data.json"
if kpse then
  path = kpse.find_file(path)
end
if path then
  local contents = util.read_file(path)
  if not contents then
    error(string.format('Failed to find "%s"', path))
  end
  bib.bib_data = utilities.json.tolua(contents)
end

function bib.parse(contents)
  local items = {}
  for item_contents in string.gmatch(contents, "(@%w+%b{})") do
    local item = bib.parse_item(item_contents)
    table.insert(items, item)
  end
  return items
end

function bib.parse_item(contents)
  contents = string.gsub(contents, "%s*\r?\n%s*", " ")
  local bib_type, id
  bib_type, id, contents = string.match(contents, "^@(%w+){([^%s,]+),%s*(.*)}$")
  if not id then
    return nil
  end

  local item = {id = id, ["citation-key"] = id}

  bib_type = string.lower(bib_type)
  local type_data = bib.bib_data.types[bib_type]
  if type_data and type_data.csl then
    item.type = type_data.csl
  else
    item.type = "document"
  end

  local bib_fields = bib.parse_fields(contents)
  -- util.debug(bib_fields)

  for bib_field, value in pairs(bib_fields) do
    local csl_field, csl_value = bib.convert_field(bib_field, value)

    if csl_field and not item[csl_field] then
      item[csl_field] = csl_value
    end
  end

  bib.process_special_fields(item, bib_fields)

  return item
end

function bib.parse_fields(contents)
  local fields = {}
  local field_patterns = {
    "^(%w+)%s*=%s*(%b{}),?%s*(.-)$",
    '^(%w+)%s*=%s*"([^"]*)",?%s*(.-)$',
    "^(%w+)%s*=%s*(%w+),?%s*(.-)$",
  }

  while #contents > 0 do
    local field, value, rest
    -- This pattern may fail in the case of `title = {foo\}bar}`.
    for pattern_index, pattern in ipairs(field_patterns) do
      field, value, rest = string.match(contents, pattern)
      if value then
        if pattern_index == 1 then
          -- Strip braces "{}"
          value = string.sub(value, 2, -2)
        elseif pattern_index == 3 then
          if not string.match(value, "^%d+$") then
            local string_name = value
            local macro = bib.bib_data.macros[string_name]
            if macro then
              value = macro.value
            else
              util.warning(string.format('String name "%s" is undefined', string_name))
            end
          end
        end
        fields[field] = value
        contents = rest
        break
      end
    end
  end
  return fields
end

function bib.convert_field(bib_field, value)
  local field_data = bib.bib_data.fields[bib_field]
  if not field_data then
    return nil, nil
  end
  local csl_field = field_data.csl
  if not csl_field then
    return nil, nil
  end

  value = bib.unescape(bib_field, value)

  local field_type = field_data.type
  if field_type == "name" then
    value = bib.parse_names(value)
  elseif field_type == "date" then
    value = bib.parse_date(value)
  end

  if bib_field == "title" or bib_field == "booktitle" then
    -- TODO: check if the original title is in sentence case
    value = bib.convert_sentence_case(value)
  end

  if bib_field == "volume" or bib_field == "pages" then
    value = string.gsub(value, util.unicode["en dash"], "-")
  end

  return csl_field, value
end

function bib.unescape(field, str)
  str = string.gsub(str, "%-%-%-", util.unicode["em dash"])
  str = string.gsub(str, "%-%-", util.unicode["en dash"])
  str = string.gsub(str, "``", util.unicode["left double quotation mark"])
  str = string.gsub(str, "''", util.unicode["right double quotation mark"])
  str = string.gsub(str, "`", util.unicode["left single quotation mark"])
  str = string.gsub(str, "'", util.unicode["right single quotation mark"])
  -- TODO: unicode chars like \"{o}
  str = string.gsub(str, "\\#", "#")
  str = string.gsub(str, "\\%$", "$")
  str = string.gsub(str, "\\%%", "%")
  str = string.gsub(str, "\\&", "&")
  str = string.gsub(str, "\\{", "{")
  str = string.gsub(str, "\\}", "}")
  str = string.gsub(str, "\\_", "_")
  if field ~= "url" then
    str = string.gsub(str, "~", util.unicode["no-break space"])
  end
  str = string.gsub(str, "\\quad%s+", util.unicode["em space"])
  return str
end

function bib.convert_sentence_case(str)
  local res = ""
  local to_lower = false
  local brace_level = 0
  for _, code_point in utf8.codes(str) do
    local char = utf8.char(code_point)
    if to_lower and brace_level == 0 then
      char = unicode.utf8.lower(char)
    end
    if string.match(char, "%S") then
      to_lower = true
    end
    if char == "{" then
      brace_level = brace_level + 1
      char = ""
    elseif char == "}" then
      brace_level = brace_level - 1
      char = ""
    elseif char == ":" then
      to_lower = false
    end
    res = res .. char
  end
  return res
end

function bib.parse_names(str)
   -- "{International Federation of Library Association and Institutions}"
  local names = {}
  local brace_level = 0
  local name = ""
  local last_word = ""
  for i = 1, #str do
    local char = string.sub(str, i, i)
    if char == " " then
      if brace_level == 0 and last_word == "and" then
        table.insert(names, name)
        name = ""
      else
        if name ~= "" then
          name = name .. " "
        end
        name = name .. last_word
      end
      last_word = ""
    else
      last_word = last_word .. char
      if char == "{" then
        brace_level = brace_level + 1
      elseif char == "}" then
        brace_level = brace_level - 1
      end
    end
  end

  if name ~= "" then
    name = name .. " "
  end
  name = name .. last_word
  table.insert(names, name)

  for i, name in ipairs(names) do
    names[i] = bib.parse_single_name(name)
  end
  return names
end

function bib.parse_single_name(str)
  local literal = string.match(str, "^{(.*)}$")
  if literal then
    return {
      literal = literal,
    }
  end

  local name_parts = util.split(str, ",%s*")
  if #name_parts > 1 then
    return bib.parse_revesed_name(name_parts)
  else
    return bib.parse_non_revesed_name(str)
  end
end

function bib.parse_revesed_name(name_parts)
  local name = {}
  local von, last, jr, first
  if #name_parts == 2 then
    first = name_parts[2]
  elseif #name_parts >= 3 then
    jr = name_parts[2]
    first = name_parts[3]
  end
  if first and first ~= "" then
    name.given = first
  end
  if jr and jr ~= "" then
    name.suffix = jr
  end

  last = name_parts[1]
  local words = util.split(last)
  local index = #words - 1
  while index > 0 and string.match(words[index], "^%L") do
    index = index - 1
  end
  name.family = util.concat(util.slice(words, index + 1), " ")
  if index >= 1 then
    von = util.concat(util.slice(words, 1, index), " ")
    name["non-dropping-particle"] = von
  end
  return name
end

function bib.parse_non_revesed_name(str)
  local name = {}
  local words = util.split(str)

  local index = 1
  -- TODO: case determination for pseudo-characters (e.g., "\bb{BB}")
  while index < #words and string.match(words[index], "^%L") do
    index = index + 1
  end
  if index > 1 then
    name.given = util.concat(util.slice(words, 1, index - 1), " ")
  end

  local particle_start_index = index
  index = #words - 1
  while index >= particle_start_index and string.match(words[index], "^%L") do
    index = index - 1
  end
  if index >= particle_start_index then
    local particles = util.slice(words, particle_start_index, index)
    -- TODO: distiguish dropping and non-dropping particles
    name["non-dropping-particle"] = util.concat(particles, " ")
  end
  name.family = util.concat(util.slice(words, index + 1), " ")

  return name
end

function bib.parse_date(str)
  local date_range = util.split(str, "/")
  if #date_range == 1 then
    date_range = util.split(str, util.unicode["en dash"])
  end

  local literal = { literal = str }

  if #date_range > 2 then
    return literal
  end

  local date = {}
  date["date-parts"] = {}
  for _, date_part in ipairs(date_range) do
    local date_ = bib.parse_single_date(date_part)
    if not date_ then
      return literal
    end
    table.insert(date["date-parts"], date_)
  end
  return date
end

function bib.parse_single_date(str)
  local date = {}
  for _, date_part in ipairs(util.split(str, "%-")) do
    if not string.match(date_part, "^%d+$") then
      return nil
    end
    table.insert(date, tonumber(date_part))
  end
  return date
end


function bib.process_special_fields(item, bib_fields)
  -- Default entry type `document`
  if item.type == "document" then
    if item.URL then
      item.type = "webpage"
    else
      item.type = "article"
    end
  end

  -- event-title: for compatibility with CSL v1.0.1 and earlier versions
  if item["event-title"] then
    item.event = item["event-title"]
  end

  -- issued date
  if bib_fields.year and not item.issued then
    item.issued = bib.parse_date(bib_fields.year)
  end
  local month = bib_fields.month
  if month and string.match(month, "^%d+$") then
    if item.issued and item.issued["date-parts"] and
        item.issued["date-parts"][1] and
        item.issued["date-parts"][1][2] == nil then
      item.issued["date-parts"][1][2] = tonumber(month)
    end
  end

  -- language: convert `babel` language to ISO 639-1 language code
  if not item.language and bib_fields.language then
    item.language = bib_fields.language
  end
  if item.language then
    local language_code = bib.babel_locale_mapping[item.language]
    if language_code then
      item.language = language_code
    end
  end
  -- if not item.language then
  --   if util.has_cjk_char(item.title) then
  --     item.language = "zh"
  --   end
  -- end

  -- number
  if item.number then
    if item.type == "article-journal" or item.type == "article-magazine" or item.type == "article-newspaper" or item.type == "periodical" then
      if not item.issue then
        item.issue = item.number
        item.number = nil
      end
    elseif item["collection-title"] and not item["collection-number"] then
      item["collection-number"] = item.number
      item.number = nil
    end
  end

  -- PMID
  if bib_fields.eprint and string.lower(bib_fields.eprinttype) == "pubmed" and not item.PMID then
    item.PMID = bib_fields.eprint
  end

end


bib.babel_locale_mapping = {
  acadian         = "fr-CA",
  american        = "en-US",
  australian      = "en-AU",
  afrikaans       = "af-ZA",
  albanian        = "sq-AL",
  amharic         = "am-ET",
  arabic          = "ar",
  armenian        = "hy-AM",
  asturian        = "ast-ES",
  austrian        = "de-AT",
  bahasa          = "id-ID",
  bahasai         = "id-ID",
  bahasam         = "id-ID",
  basque          = "eu-ES",
  bengali         = "bn-BD",
  bgreek          = "el-GR",
  brazil          = "pt-BR",
  brazilian       = "pt-BR",
  breton          = "br-FR",
  british         = "en-GB",
  bulgarian       = "bg-BG",
  canadian        = "en-CA",
  canadien        = "fr-CA",
  catalan         = "ca-AD",
  coptic          = "cop",
  croatian        = "hr-HR",
  czech           = "cs-CZ",
  danish          = "da-DK",
  divehi          = "dv-MV",
  dutch           = "nl-NL",
  english         = "en-US",
  esperanto       = "eo-001",
  estonian        = "et-EE",
  ethiopia        = "am-ET",
  farsi           = "fa-IR",
  finnish         = "fi-FI",
  francais        = "fr-FR",
  french          = "fr-FR",
  frenchle        = "fr-FR",
  friulan         = "fur-IT",
  galician        = "gl-ES",
  german          = "de-DE",
  germanb         = "de-DE",
  greek           = "el-GR",
  hebrew          = "he-IL",
  hindi           = "hi-IN",
  ibygreek        = "el-CY",
  icelandic       = "is-IS",
  indon           = "id-ID",
  indonesia       = "id-ID",
  interlingua     = "ia-FR",
  irish           = "ga-IE",
  italian         = "it-IT",
  japanese        = "ja-JP",
  kannada         = "kn-IN",
  lao             = "lo-LA",
  latin           = "la-Latn",
  latvian         = "lv-LV",
  lithuanian      = "lt-LT",
  lowersorbian    = "dsb-DE",
  lsorbian        = "dsb-DE",
  magyar          = "hu-HU",
  malay           = "id-ID",
  malayalam       = "ml-IN",
  marathi         = "mr-IN",
  meyalu          = "id-ID",
  mongolian       = "mn-Cyrl",
  naustrian       = "de-AT",
  newzealand      = "en-NZ",
  ngerman         = "de-DE",
  nko             = "ha-NG",
  norsk           = "nb-NO",
  norwegian       = "nn-NO",
  nynorsk         = "nn-NO",
  occitan         = "oc-FR",
  piedmontese     = "pms-IT",
  pinyin          = "pny",
  polish          = "pl-PL",
  polutonikogreek = "el-GR",
  portuges        = "pt-PT",
  portuguese      = "pt-PT",
  romanian        = "ro-RO",
  romansh         = "rm-CH",
  russian         = "ru-RU",
  samin           = "se-NO",
  sanskrit        = "sa-IN",
  scottish        = "gd-GB",
  serbian         = "sr-Latn",
  serbianc        = "sr-Cyrl",
  slovak          = "sk-SK",
  slovene         = "sl-SI",
  slovenian       = "sl-SI",
  spanish         = "es-ES",
  swedish         = "sv-SE",
  swiss           = "de-CH",
  swissgerman     = "de-CH",
  nswissgerman    = "de-CH",
  syriac          = "syc",
  tamil           = "ta-IN",
  telugu          = "te-IN",
  thai            = "th-TH",
  thaicjk         = "th-TH",
  tibetan         = "bo-CN",
  turkish         = "tr-TR",
  turkmen         = "tk-TM",
  ukrainian       = "uk-UA",
  urdu            = "ur-IN",
  UKenglish       = "en-UK",
  uppersorbian    = "hsb-DE",
  USenglish       = "en-US",
  usorbian        = "hsb-DE",
  vietnamese      = "vi-VN",
  welsh           = "cy-GB",
}


return bib