-- -- Copyright (c) 2021-2022 Zeping Lee -- Released under the MIT license. -- Repository: https://github.com/zepinglee/citeproc-lua -- --[[ A PEG-based implementation of a Bib(La)TeX dateabase (.bib) parser References notes: tools/bibtex-parser-notes.md --]] local bibtex = {} local lpeg = require("lpeg") local unicode = require("unicode") local bibtex_data = require("citeproc-bibtex-data") local util = require("citeproc-util") local P = lpeg.P local R = lpeg.R local S = lpeg.S local C = lpeg.C local Cc = lpeg.Cc local Cf = lpeg.Cf local Cg = lpeg.Cg local Cmt = lpeg.Cmt local Cp = lpeg.Cp local Ct = lpeg.Ct local V = lpeg.V bibtex.strings = {} -- Stores all the @STRINGs for key, value in pairs(bibtex_data.macros) do bibtex.strings[key] = value.value end -- Parse BibTeX content and convert to CSL-JSON function bibtex.parse(str) local bibliography = bibtex.parse_bibtex(str, bibtex.strings) local csl_json_items = bibtex.convert_csl_json(bibliography) return csl_json_items end -- The case-insensitivity trick is taken from . local function ignore_case(str) local char = R("AZ", "az") / function (c) return P(c:lower()) + P(c:upper()) end + P(1) / function (c) return P(c) end return Cf(char^1, function (a, b) return a * b end):match(str) end local function normalize_white_space(str) local res = string.gsub(str, "%s+", " ") return res end -- Based on the grammar described at . function bibtex.get_bibtex_grammar() local comment = (1 - P"@")^0 local space = S(" \t\r\n")^0 local comment_cmd = ignore_case("comment") local balanced = P{ "{" * V(1)^0 * "}" + (1 - S"{}") } local ident = (- R"09") * (R"\x20\x7F" - S" \t\"#%'(),={}")^1 local piece = (P"{" * (balanced^0 / normalize_white_space) * P"}") + (P'"' * ((balanced - P'"')^0 / normalize_white_space) * P'"') + C(R("09")^1) + C(ident) / function (name) return { category = "string", name = string.lower(name), } end local value = Ct(piece * (space * P"#" * space * piece)^0) local string_body = Cg(ident / string.lower, "name") * space * P"=" * space * Cg(value, "contents") local string_cmd = Ct(Cg(Cc"string", "category") * ignore_case("string") * space * ( P"{" * space * string_body * space * P"}" + P"(" * space * string_body * space * P")" )) local preamble_body = Cg(value, "contents") local preamble_cmd = Ct(Cg(Cc"preamble", "category") * ignore_case("preamble") * space * ( P"{" * space * preamble_body * space * P"}" + P"(" * space * preamble_body * space * P")" )) local key = (1 - S", \t}\r\n")^0 local key_paren = (1 - S", \t\r\n")^0 local field_value_pair = (ident / string.lower) * space * P"=" * space * value -- * record_success_position() local entry_body = Cf(Ct"" * (P"," * space * Cg(field_value_pair))^0 * (P",")^-1, rawset) local entry = Ct(Cg(Cc"entry", "category") * Cg(ident / string.lower, "type") * space * ( P"{" * space * Cg(key, "key") * space * Cg(entry_body, "fields")^-1 * space * (P"}") + P"(" * space * Cg(key_paren, "key") * space * Cg(entry_body, "fields")^-1 * space * (P")") )) local command_or_entry = P"@" * space * (comment_cmd + preamble_cmd + string_cmd + entry) -- The P(-1) causes nil parsing result in case of error. local bibtex_grammar = Ct(comment * (command_or_entry * comment)^0) * P(-1) return bibtex_grammar end bibtex.bibtex_grammar = bibtex.get_bibtex_grammar() -- Return a list of entries, strings, preamble function bibtex.parse_bibtex_objects(str) return bibtex.bibtex_grammar:match(str) end local function concat_strings(pieces, strings) local value = "" for _, piece in ipairs(pieces) do if type(piece) == "string" then value = value .. piece elseif type(piece) == "table" and piece.category == "string" then local piece_str = strings[piece.name] if piece_str then value = value .. piece_str else util.warning(string.format('String name "%s" is undefined."', piece.name)) end end end value = normalize_white_space(value) return value end function bibtex.parse_bibtex(content, strings) if strings then strings = setmetatable({}, {__index = strings}) else strings = setmetatable({}, {__index = bibtex.strings}) end local bib_objects = bibtex.parse_bibtex_objects(content) local res = { entries = {}, strings = {}, preamble = "", } local exceptions = {} for _, object in ipairs(bib_objects) do if object.category == "entry" then object.category = nil for field, value in pairs(object.fields) do local value_str = concat_strings(value, strings) if field ~= "url" then value_str = bibtex.to_unicode(value_str) end object.fields[field] = value_str end table.insert(res.entries, object) elseif object.category == "string" then local string_value = concat_strings(object.contents, strings) strings[object.name] = string_value res.strings[object.name] = string_value elseif object.category == "preamble" then res.preamble = res.preamble .. concat_strings(object.contents, strings) -- elseif object.category == "comment" then -- Is this really needed? elseif object.category == "exception" then -- TODO end end return res end local function get_line_number(str, position) local line_number = 1 for i = 1, position do if string.sub(str, i, i) == "\n" then line_number = line_number + 1 end end return line_number end function bibtex.get_latex_grammar() local space = S(" \t\r\n")^0 local specials = P"\\$" / "$" + P"\\%" / "%" + P"\\&" / "&" + P"\\#" / "#" + P"\\_" / "_" + P"\\{" / "{" + P"\\}" / "}" + P"~" / util.unicode["no-break space"] local control_sequence = C(P"\\" * (R("AZ", "az")^1 + 1) * space) / function (cs) return { type = "control_sequence", name = util.rstrip(cs), raw = cs, } end local math = P"$" * C((P"\\$" + 1 - S"$")^0) * P"$" / function (math_text) return { type = "math", contents = math_text, } end local ligatures = P"``" / util.unicode["left double quotation mark"] + P"`" / util.unicode["left single quotation mark"] + P"''" / util.unicode["right double quotation mark"] + P"'" / util.unicode["right single quotation mark"] + P"---" / util.unicode["em dash"] + P"--" / util.unicode["en dash"] local plain_text = C(1 - S"{}$\\") local latex_grammar = P{ "latex_text"; latex_text = Ct((specials + control_sequence + math + ligatures + specials + V"group" + plain_text)^0), group = P"{" * V"latex_text" * P"}" / function (group_contents) return { type = "group", contents = group_contents, } end, } return latex_grammar end bibtex.latex_grammar = bibtex.get_latex_grammar() function bibtex.to_unicode(str) local tokens = bibtex.latex_grammar:match(str) local res = bibtex.unescape_tokens(tokens, 0) return res end local format_commands_with_argment = { ["\\textrm"] = {'', ''}, ["\\textbf"] = {'', ''}, ["\\textit"] = {'', ''}, ["\\textsl"] = {'', ''}, ["\\emph"] = {'', ''}, ["\\textsc"] = {'', ''}, ["\\textsuperscript"] = {'', ''}, ["\\textsubscript"] = {'', ''}, } local format_commands_without_argment = { ["\\rm"] = {'', ''}, ["\\upshape"] = {'', ''}, ["\\bf"] = {'', ''}, ["\\bfseries"] = {'', ''}, ["\\it"] = {'', ''}, ["\\itshape"] = {'', ''}, ["\\sl"] = {'', ''}, ["\\slshape"] = {'', ''}, ["\\em"] = {'', ''}, ["\\scshape"] = {'', ''}, ["\\sc"] = {'', ''}, } function bibtex.unescape_tokens(tokens, brace_level) local res = "" local skip_token = false for i, token in ipairs(tokens) do if skip_token then skip_token = false else if type(token) == "string" then res = res .. token elseif type(token) == "table" then if token.type == "control_sequence" then local code_point = bibtex_data.unicode_commands[token.name] if code_point then local unicode_char if type(code_point) == "string" then unicode_char = utf8.char(tonumber(code_point, 16)) elseif type(code_point) == "table" then -- The command takes an argument (\"{o}) local arg if i < #tokens then local next_token = tokens[i + 1] if type(next_token) == "string" then arg = next_token skip_token = true elseif type(next_token) == "table" then if next_token.type == "control_sequence" then arg = next_token.name skip_token = true elseif next_token.type == "group" then if #next_token.contents == 0 then arg = "{}" skip_token = true elseif #next_token.contents == 1 then next_token = next_token.contents[1] if type(next_token) == "string" then arg = next_token skip_token = true elseif type(next_token) == "table" then arg = next_token.name skip_token = true end end end end end if arg and code_point[arg] then unicode_char = utf8.char(tonumber(code_point[arg], 16)) end end if unicode_char then res = res .. unicode_char else res = res .. '' .. token.raw .. '' end elseif format_commands_with_argment[token.name] then -- Commands like `\textbf` if i < #tokens then local next_token = tokens[i + 1] if type(next_token) == "table" and next_token.type == "group" then local tags = format_commands_with_argment[token.name] if brace_level == 0 then res = res .. tags[1] .. '' .. bibtex.unescape_tokens(next_token.contents, brace_level + 1) .. '' .. tags[2] else res = res .. tags[1] .. bibtex.unescape_tokens(next_token.contents, brace_level + 1) .. tags[2] end skip_token = true else res = res .. '' .. token.raw .. '' end else res = res .. '' .. token.raw .. '' end else res = res .. '' .. token.raw .. '' end elseif token.type == "math" then -- res = res .. "$" .. token .. "$" res = res .. '' .. token.raw .. '' elseif token.type == "group" then local first_token = token.contents[1] if type(first_token) == "table" and first_token.type == "control_sequence" then local tags = format_commands_without_argment[first_token.name] if tags then table.remove(token.contents, 1) res = res .. tags[1] .. bibtex.unescape_tokens(token.contents, brace_level) .. tags[2] else res = res .. bibtex.unescape_tokens(token.contents, brace_level) end else if brace_level == 0 then res = res .. '' .. bibtex.unescape_tokens(token.contents, brace_level + 1) .. '' else res = res .. '' .. bibtex.unescape_tokens(token.contents, brace_level + 1) .. '' end end end end end end return res end -- function bibtex.format_syntanx_error(str, position, file_path) -- local line_number = 1 -- local line_start = 1 -- local column_number = 0 -- for i = 1, position do -- column_number = column_number + 1 -- if string.sub(str, i, i) == "\n" then -- line_number = line_number + 1 -- line_start = i + 1 -- column_number = 0 -- end -- end -- -- return line_number, column_number -- local res -- if file_path then -- res = string.format('File "%s", line %d\n', file_path, line_number) -- else -- res = string.format('Line %d\n', line_number) -- end -- local line_end = position -- while line_end < #str + 1 and string.sub(str, line_end, line_end) ~= "\n" do -- line_end = line_end + 1 -- end -- res = res .. " " .. string.sub(str, line_start, line_end - 1) .. "\n" -- res = res .. "Syntax Error" -- return res -- end function bibtex.convert_csl_json(bibliography) local csl_json_data = {} for _, entry in ipairs(bibliography.entries) do local item = { -- CSL-JSON item id = entry.key, type = "document", } -- CSL types local type_data = bibtex_data.types[entry.type] if type_data and type_data.csl then item.type = type_data.csl end for field, value in pairs(entry.fields) do local csl_field, csl_value = bibtex.convert_field(field, value) if csl_field and csl_value and not item[csl_field] then item[csl_field] = csl_value end end bibtex.process_special_fields(item, entry.fields) table.insert(csl_json_data, item) end return csl_json_data end function bibtex.convert_field(bib_field, value) local field_data = bibtex_data.fields[bib_field] if not field_data then return nil, nil end local csl_field = field_data.csl if not csl_field then return nil, nil end local field_type = field_data.type if field_type == "name" then value = bibtex.parse_names(value) elseif field_type == "date" then value = bibtex.parse_date(value) -- elseif bib_field == "title" or bib_field == "booktitle" then -- value = bibtex.convert_sentence_case(value) elseif bib_field == "volume" or bib_field == "pages" then value = string.gsub(value, util.unicode["en dash"], "-") end return csl_field, value end function bibtex.parse_names(str) -- "{International Federation of Library Association and Institutions}" local names = {} local brace_level = 0 local name = "" local last_word = "" for i = 1, #str do local char = string.sub(str, i, i) if char == " " then if brace_level == 0 and last_word == "and" then table.insert(names, name) name = "" else if name ~= "" then name = name .. " " end name = name .. last_word end last_word = "" else last_word = last_word .. char if char == "{" then brace_level = brace_level + 1 elseif char == "}" then brace_level = brace_level - 1 end end end if name ~= "" then name = name .. " " end name = name .. last_word table.insert(names, name) for i, name in ipairs(names) do names[i] = bibtex.parse_single_name(name) end return names end function bibtex.parse_single_name(str) local literal = string.match(str, "^{(.*)}$") if literal then return { literal = literal, } end local name_parts = util.split(str, ",%s*") if #name_parts > 1 then return bibtex.parse_revesed_name(name_parts) else return bibtex.parse_non_revesed_name(str) end end function bibtex.parse_revesed_name(name_parts) local name = {} local von, last, jr, first if #name_parts == 2 then first = name_parts[2] elseif #name_parts >= 3 then jr = name_parts[2] first = name_parts[3] end if first and first ~= "" then name.given = first end if jr and jr ~= "" then name.suffix = jr end last = name_parts[1] local words = util.split(last) local index = #words - 1 while index > 0 and string.match(words[index], "^%L") do index = index - 1 end name.family = util.concat(util.slice(words, index + 1), " ") if index >= 1 then von = util.concat(util.slice(words, 1, index), " ") name["non-dropping-particle"] = von end return name end function bibtex.parse_non_revesed_name(str) local name = {} local words = util.split(str) local index = 1 -- TODO: case determination for pseudo-characters (e.g., "\bb{BB}") while index < #words and string.match(words[index], "^%L") do index = index + 1 end if index > 1 then name.given = util.concat(util.slice(words, 1, index - 1), " ") end local particle_start_index = index index = #words - 1 while index >= particle_start_index and string.match(words[index], "^%L") do index = index - 1 end if index >= particle_start_index then local particles = util.slice(words, particle_start_index, index) -- TODO: distiguish dropping and non-dropping particles name["non-dropping-particle"] = util.concat(particles, " ") end name.family = util.concat(util.slice(words, index + 1), " ") return name end function bibtex.parse_date(str) local date_range = util.split(str, "/") if #date_range == 1 then date_range = util.split(str, util.unicode["en dash"]) end local literal = { literal = str } if #date_range > 2 then return literal end local date = {} date["date-parts"] = {} for _, date_part in ipairs(date_range) do local date_ = bibtex.parse_single_date(date_part) if not date_ then return literal end table.insert(date["date-parts"], date_) end return date end function bibtex.parse_single_date(str) local date = {} for _, date_part in ipairs(util.split(str, "%-")) do if not string.match(date_part, "^%d+$") then return nil end table.insert(date, tonumber(date_part)) end return date end -- -- TODO: check if the original title is in sentence case -- function bibtex.convert_sentence_case(str) -- local res = "" -- local to_lower = false -- local brace_level = 0 -- for _, code_point in utf8.codes(str) do -- local char = utf8.char(code_point) -- if to_lower and brace_level == 0 then -- char = string.lower(char) -- end -- if string.match(char, "%S") then -- to_lower = true -- end -- if char == "{" then -- brace_level = brace_level + 1 -- char = "" -- elseif char == "}" then -- brace_level = brace_level - 1 -- char = "" -- elseif char == ":" then -- to_lower = false -- end -- res = res .. char -- end -- return res -- end function bibtex.process_special_fields(item, bib_fields) -- Default entry type `document` if item.type == "document" then if item.URL then item.type = "webpage" else item.type = "article" end end -- event-title: for compatibility with CSL v1.0.1 and earlier versions if item["event-title"] then item.event = item["event-title"] end -- issued date if bib_fields.year and not item.issued then item.issued = bibtex.parse_date(bib_fields.year) end local month = bib_fields.month if month and string.match(month, "^%d+$") then if item.issued and item.issued["date-parts"] and item.issued["date-parts"][1] and item.issued["date-parts"][1][2] == nil then item.issued["date-parts"][1][2] = tonumber(month) end end -- language: convert `babel` language to ISO 639-1 language code if not item.language and bib_fields.language then item.language = bib_fields.language end if item.language then local language_code = bibtex_data.language_code_map[item.language] if language_code then item.language = language_code end end -- if not item.language then -- if util.has_cjk_char(item.title) then -- item.language = "zh" -- end -- end -- number if item.number then if item.type == "article-journal" or item.type == "article-magazine" or item.type == "article-newspaper" or item.type == "periodical" then if not item.issue then item.issue = item.number item.number = nil end elseif item["collection-title"] and not item["collection-number"] then item["collection-number"] = item.number item.number = nil end end -- PMID if bib_fields.eprint and string.lower(bib_fields.eprinttype) == "pubmed" and not item.PMID then item.PMID = bib_fields.eprint end end return bibtex