summaryrefslogtreecommitdiff
path: root/biblio/citation-style-language/citeproc-bibtex-parser.lua
diff options
context:
space:
mode:
authorNorbert Preining <norbert@preining.info>2023-04-04 03:01:33 +0000
committerNorbert Preining <norbert@preining.info>2023-04-04 03:01:33 +0000
commite9f267fd4c8414d6146c5fee0dbe459613dd0f3b (patch)
tree83fb57a85aee532d8ccb344a7a808a446c31054d /biblio/citation-style-language/citeproc-bibtex-parser.lua
parent3b4a242af88203c2f3f551d46161bdd037c0d127 (diff)
CTAN sync 202304040301
Diffstat (limited to 'biblio/citation-style-language/citeproc-bibtex-parser.lua')
-rw-r--r--biblio/citation-style-language/citeproc-bibtex-parser.lua483
1 files changed, 483 insertions, 0 deletions
diff --git a/biblio/citation-style-language/citeproc-bibtex-parser.lua b/biblio/citation-style-language/citeproc-bibtex-parser.lua
new file mode 100644
index 0000000000..3932e7fe3f
--- /dev/null
+++ b/biblio/citation-style-language/citeproc-bibtex-parser.lua
@@ -0,0 +1,483 @@
+--
+-- Copyright (c) 2021-2023 Zeping Lee
+-- Released under the MIT license.
+-- Repository: https://github.com/zepinglee/citeproc-lua
+--
+
+--[[
+ A PEG-based implementation of a Bib(La)TeX database (.bib) parser
+ References notes: scripts/bibtex-parser-notes.md
+--]]
+
+
+-- @module bibtex_parser
+local bibtex_parser = {}
+
+local lpeg = require("lpeg")
+local unicode = require("unicode")
+local bibtex_data = require("citeproc-bibtex-data")
+local latex_parser = nil -- load as needed
+local util = require("citeproc-util")
+
+local P = lpeg.P
+local R = lpeg.R
+local S = lpeg.S
+local C = lpeg.C
+local Cc = lpeg.Cc
+local Cf = lpeg.Cf
+local Cg = lpeg.Cg
+local Cmt = lpeg.Cmt
+local Cp = lpeg.Cp
+local Ct = lpeg.Ct
+local V = lpeg.V
+
+
+-- Learned from <http://boston.conman.org/2020/06/05.1>.
+local function case_insensitive_pattern(str)
+ local char = R("AZ", "az") / function (c) return P(c:lower()) + P(c:upper()) end
+ + P(1) / function (c) return P(c) end
+ return Cf(char^1, function (a, b) return a * b end):match(str)
+end
+
+-- Merge multiple white spaces to one
+local function normalize_white_space(str)
+ local res = string.gsub(str, "%s+", " ")
+ return res
+end
+
+
+-- Based on the grammar described at <https://github.com/aclements/biblib>.
+local function get_bibtex_grammar()
+ local comment = (1 - P"@")^0
+ local space = S(" \t\r\n")^0
+ local comment_cmd = case_insensitive_pattern("comment")
+ local balanced = P{ "{" * V(1)^0 * "}" + (1 - S"{}") }
+ local ident = (- R"09") * (R"\x20\x7F" - S" \t\"#%'(),={}")^1
+
+ local piece = (P"{" * (balanced^0 / normalize_white_space) * P"}")
+ + (P'"' * ((balanced - P'"')^0 / normalize_white_space) * P'"')
+ + C(R("09")^1)
+ + C(ident) / function (name)
+ return {
+ category = "string",
+ name = string.lower(name),
+ }
+ end
+ local value = Ct(piece * (space * P"#" * space * piece)^0)
+
+ local string_body = Cg(ident / string.lower, "name") * space * P"=" * space * Cg(value, "contents")
+ local string_cmd = Ct(Cg(Cc"string", "category") * case_insensitive_pattern("string") * space * (
+ P"{" * space * string_body * space * P"}"
+ + P"(" * space * string_body * space * P")"
+ ))
+
+ local preamble_body = Cg(value, "contents")
+ local preamble_cmd = Ct(Cg(Cc"preamble", "category") * case_insensitive_pattern("preamble") * space * (
+ P"{" * space * preamble_body * space * P"}"
+ + P"(" * space * preamble_body * space * P")"
+ ))
+
+ local key = (1 - S", \t}\r\n")^0
+ local key_paren = (1 - S", \t\r\n")^0
+ local field_value_pair = (ident / string.lower) * space * P"=" * space * value -- * record_success_position()
+
+ local entry_body = Cf(Ct"" * (P"," * space * Cg(field_value_pair))^0 * (P",")^-1, rawset)
+ local entry = Ct(Cg(Cc"entry", "category") * Cg(ident / string.lower, "type") * space * (
+ P"{" * space * Cg(key, "key") * space * Cg(entry_body, "fields")^-1 * space * (P"}")
+ + P"(" * space * Cg(key_paren, "key") * space * Cg(entry_body, "fields")^-1 * space * (P")")
+ ))
+
+ local command_or_entry = P"@" * space * (comment_cmd + preamble_cmd + string_cmd + entry)
+
+ -- The P(-1) causes nil parsing result in case of error.
+ local bibtex_grammar = Ct(comment * (command_or_entry * comment)^0) * P(-1)
+
+ return bibtex_grammar
+end
+
+
+local function concat_strings(pieces, strings)
+ local value = ""
+ for _, piece in ipairs(pieces) do
+ if type(piece) == "string" then
+ value = value .. piece
+ elseif type(piece) == "table" and piece.category == "string" then
+ local piece_str = strings[piece.name]
+ if piece_str then
+ value = value .. piece_str
+ else
+ util.warning(string.format('String name "%s" is undefined."', piece.name))
+ end
+ end
+ end
+ value = normalize_white_space(value)
+ return value
+end
+
+
+-- The parser class
+---@class BibtexParser
+local BibtexParser = {
+ grammar = get_bibtex_grammar(),
+ -- config
+ strings = {},
+ options = {
+ -- allow_escaped_braces = true,
+ convert_to_unicode = false,
+ -- common_strings = false,
+ split_names = false,
+ split_name_parts = false,
+ },
+ -- variables
+ name_fields = {},
+}
+
+---Create a new BibtexParser instance.
+---@return BibtexParser
+function BibtexParser:new()
+ local obj = {}
+
+ obj.options = util.clone(self.options)
+
+ for field, info in ipairs(bibtex_data.fields) do
+ if info.type == "name" then
+ obj.name_fields[field] = true
+ end
+ end
+
+ setmetatable(obj, self)
+ self.__index = self
+ return obj
+end
+
+
+---@alias BibtexData table
+---@alias Exception table
+
+---comment
+---@return BibtexData?, Exception[]?
+function BibtexParser:parse(bib_str, strings)
+ if strings then
+ strings = setmetatable({}, {__index = strings})
+ else
+ strings = setmetatable({}, {__index = self.strings})
+ end
+ local bib_objects = self.grammar:match(bib_str)
+ if not bib_objects then
+ local error = {
+ type = "error",
+ message = "BibTeX parser error.",
+ }
+ return nil, {error}
+ end
+
+ local res = {
+ entries = {},
+ strings = {},
+ preamble = nil,
+ }
+ local exceptions = {}
+
+ for _, object in ipairs(bib_objects) do
+ if object.category == "entry" then
+ local entry = self:_make_entry(object, strings)
+ table.insert(res.entries, object)
+
+ elseif object.category == "string" then
+ local string_value = concat_strings(object.contents, strings)
+ strings[object.name] = string_value
+ res.strings[object.name] = string_value
+
+ elseif object.category == "preamble" then
+ local value = concat_strings(object.contents, strings)
+ if res.preamble then
+ res.preamble = res.preamble .. "\n" .. value
+ else
+ res.preamble = value
+ end
+
+ -- elseif object.category == "comment" then
+ -- Is this really needed?
+
+ elseif object.category == "exception" then
+ -- TODO
+
+ end
+ end
+
+ return res, exceptions
+end
+
+
+function BibtexParser:_make_entry(object, strings)
+ object.category = nil
+ for field, value in pairs(object.fields) do
+ value = concat_strings(value, strings)
+
+ if self.options.convert_to_unicode then
+ latex_parser = latex_parser or require("citeproc-latex-parser")
+ value = latex_parser.latex_to_unicode(value)
+ end
+
+ if self.name_fields[field] then
+ if self.options.split_names then
+ value = bibtex_parser.split_names(value)
+ if self.options.split_name_parts then
+ for i, name in ipairs(value) do
+ value[i] = bibtex_parser.split_name_parts(name)
+ end
+ end
+ end
+ end
+
+ object.fields[field] = value
+ end
+ return object
+end
+
+
+-- continuation byte
+local utf8_cont = lpeg.R("\128\191")
+
+local utf8_char = lpeg.R("\0\127")
+ + lpeg.R("\194\223") * utf8_cont
+ + lpeg.R("\224\239") * utf8_cont * utf8_cont
+ + lpeg.R("\240\244") * utf8_cont * utf8_cont * utf8_cont
+
+local space_char = S(" \t\r\n")
+local space = space_char^1
+local white_space = space_char^0
+local balanced = P { "{" * V(1) ^ 0 * "}" + (P "\\{" + P "\\}" + 1 - S "{}") }
+local utf8_balanced = P { "{" * V(1) ^ 0 * "}" + (P "\\{" + P "\\}" + utf8_char - S "{}") }
+
+-- The case-insensitivity trick is taken from <http://boston.conman.org/2020/06/05.1>.
+local function ignore_case(str)
+ local char = R("AZ", "az") / function (c) return P(c:lower()) + P(c:upper()) end
+ + P(1) / function (c) return P(c) end
+ return Cf(char^1, function (a, b) return a * b end):match(str)
+end
+
+
+---@alias NameDict table
+---@alias NameStr string
+
+---Split BibTeX names
+---@param str NameStr name field value
+---@return NameStr[]
+function bibtex_parser.split_names(str)
+ local delimiter_and = ignore_case("and") * (space + -1)
+ local name = (balanced - space * delimiter_and)^1
+ local names = Ct(((white_space * delimiter_and) + C(name))^0)
+ return names:match(str)
+end
+
+---Split BibTeX name parts
+---@param str NameStr single name string
+---@return NameDict
+function bibtex_parser.split_name_parts(str)
+ str = util.strip(str)
+ if string.match(str, ",$") then
+ util.warning(string.format('Name "%s" has has a comma at the end.', str))
+ str = string.gsub(str, ",$", '')
+ end
+
+ local comma = P","
+ local comma_part = (balanced - comma)^0
+ local comma_parts = Ct(C(comma_part) * (comma * white_space * C(comma_part))^0)
+ local parts = comma_parts:match(str)
+
+ local name = {}
+ if #parts == 1 then
+ name = bibtex_parser._split_first_von_last_parts(parts[1])
+ elseif #parts == 2 then
+ name = bibtex_parser._split_von_last_parts(parts[1])
+ if parts[2] ~= "" then
+ name.first = parts[2]
+ end
+ elseif #parts == 3 then
+ name = bibtex_parser._split_von_last_parts(parts[1])
+ if parts[2] ~= "" then
+ name.jr = parts[2]
+ end
+ if parts[3] ~= "" then
+ name.first = parts[3]
+ end
+ elseif #parts > 3 then
+ util.warning()
+ name = bibtex_parser._split_last_jr_fist_parts(util.slice(parts, 1, 3))
+ else
+ util.warning()
+ end
+
+ return name
+end
+
+local function is_upper_letter(char)
+ return unicode.utf8.upper(char) == char and unicode.utf8.lower(char) ~= char
+end
+
+local function is_lower_letter(char)
+ return unicode.utf8.lower(char) == char and unicode.utf8.upper(char) ~= char
+end
+
+local function is_lower_word(word)
+ -- Word is a list of tokens
+ for _, token in ipairs(word) do
+ if string.match(token, "^{") then
+ if string.match(token, "^{\\") then
+ -- Special characters, level 0
+ local token_content
+ if string.match(token, "^{\\%w+") then
+ token_content = string.gsub(token, "^{\\%w+%s*", "")
+ else
+ token_content = string.gsub(token, "^{\\.%s*", "")
+ end
+ token_content = string.gsub(token, "}%s*$", "")
+ for i = 1, #token_content do
+ local char = string.sub(token_content, i, i)
+ if is_lower_letter(char) then
+ return true
+ elseif is_upper_letter(char) then
+ return false
+ end
+ end
+ end
+ else
+ if is_lower_letter(token) then
+ return true
+ elseif is_upper_letter(token) then
+ return false
+ end
+ end
+ end
+ return false
+end
+
+local function join_words_and_seps(words, seps, start, stop)
+ local tokens = {}
+ for i = start, stop do
+ if i > start then
+ table.insert(tokens, seps[i])
+ end
+ for _, token in ipairs(words[i]) do
+ table.insert(tokens, token)
+ end
+ end
+ local res = table.concat(tokens, "")
+ if res == "" then
+ return nil
+ end
+ return res
+end
+
+function bibtex_parser._split_first_von_last_parts(str)
+ local word_sep = P"-" + P"~" + P(util.unicode['no-break space'])
+ local word_tokens = Ct(C(utf8_balanced - space_char - word_sep)^0)
+ local words_and_seps = Ct(word_tokens * (C(space + word_sep) * word_tokens)^0)
+ local pieces = words_and_seps:match(str)
+
+ local words = {}
+ local seps = { "" }
+
+ for _, piece in ipairs(pieces) do
+ if type(piece) == "table" then
+ table.insert(words, piece)
+ else
+ table.insert(seps, piece)
+ end
+ end
+
+ local von_start = 0
+ local von_stop = 0
+
+ local first_stop = #words - 1
+ local last_start = #words
+
+ -- for i, word_or_sep in ipairs(part) do
+ for i = 1, #words - 1 do
+ local word = words[i]
+ if is_lower_word(word) then
+ if von_start == 0 then
+ von_start = i
+ end
+ von_stop = i
+ end
+ end
+
+ -- util.debug(von_start)
+ -- util.debug(von_stop)
+
+ local name = {}
+
+ if von_stop > 0 then
+ name.von = join_words_and_seps(words, seps, von_start, von_stop)
+ last_start = von_stop + 1
+ first_stop = von_start - 1
+ end
+
+ name.last = join_words_and_seps(words, seps, last_start, #words)
+
+ if first_stop > 0 then
+ name.first = join_words_and_seps(words, seps, 1, first_stop)
+ end
+
+ return name
+end
+
+function bibtex_parser._split_von_last_parts(str)
+ local word_sep = P"-" + P"~" + P(util.unicode['no-break space'])
+ local word_tokens = Ct(C(utf8_balanced - space_char - word_sep)^0)
+ local words_and_seps = Ct(word_tokens * (C(space + word_sep) * word_tokens)^0)
+ local pieces = words_and_seps:match(str)
+
+ local words = {}
+ local seps = {""}
+
+ for _, piece in ipairs(pieces) do
+ if type(piece) == "table" then
+ table.insert(words, piece)
+ else
+ table.insert(seps, piece)
+ end
+ end
+
+ local von_stop = 0
+ local last_start = 1
+
+ -- for i, word_or_sep in ipairs(part) do
+ for i = #words - 1, 1, -1 do
+ local word = words[i]
+ if is_lower_word(word) then
+ von_stop = i
+ break
+ end
+ end
+
+ local name = {}
+
+ if von_stop > 0 then
+ name.von = join_words_and_seps(words, seps, 1, von_stop)
+ last_start = von_stop + 1
+ end
+ name.last = join_words_and_seps(words, seps, last_start, #words)
+
+ return name
+end
+
+
+bibtex_parser._default_parser = BibtexParser:new()
+
+
+---comment
+---@param bib_str string input string
+---@param strings table<string, string> strings
+---@return BibtexData?, Exception[]?
+function bibtex_parser.parse(bib_str, strings)
+ return bibtex_parser._default_parser:parse(bib_str, strings)
+end
+
+
+bibtex_parser.BibtexParser = BibtexParser
+
+
+return bibtex_parser