diff options
Diffstat (limited to 'Master/texmf-dist/tex')
-rw-r--r-- | Master/texmf-dist/tex/generic/markdown/markdown.tex | 4 | ||||
-rw-r--r-- | Master/texmf-dist/tex/luatex/markdown/markdown.lua | 407 |
2 files changed, 278 insertions, 133 deletions
diff --git a/Master/texmf-dist/tex/generic/markdown/markdown.tex b/Master/texmf-dist/tex/generic/markdown/markdown.tex index 39d77c941de..22eb36e4fba 100644 --- a/Master/texmf-dist/tex/generic/markdown/markdown.tex +++ b/Master/texmf-dist/tex/generic/markdown/markdown.tex @@ -572,8 +572,8 @@ { boolean } { true } \ExplSyntaxOff -\def\markdownLastModified{2024-03-09}% -\def\markdownVersion{3.4.2-0-ga45cf0ed}% +\def\markdownLastModified{2024-04-04}% +\def\markdownVersion{3.4.3-0-ge2c6be1a}% \let\markdownBegin\relax \let\markdownEnd\relax \let\markdownInput\relax diff --git a/Master/texmf-dist/tex/luatex/markdown/markdown.lua b/Master/texmf-dist/tex/luatex/markdown/markdown.lua index 37448b4503a..1e520a30bf5 100644 --- a/Master/texmf-dist/tex/luatex/markdown/markdown.lua +++ b/Master/texmf-dist/tex/luatex/markdown/markdown.lua @@ -58,7 +58,7 @@ -- those in the standard .ins files. -- local metadata = { - version = "3.4.2-0-ga45cf0ed", + version = "3.4.3-0-ge2c6be1a", comment = "A module for the conversion from markdown to plain TeX", author = "John MacFarlane, Hans Hagen, Vít Starý Novotný", copyright = {"2009-2016 John MacFarlane, Hans Hagen", @@ -222,9 +222,6 @@ function util.encode_json_string(s) s = s:gsub([["]], [[\"]]) return [["]] .. s .. [["]] end -function util.lookup_files(f, options) - return kpse.lookup(f, options) -end function util.expand_tabs_in_line(s, tabstop) local tab = tabstop or 4 local corr = 0 @@ -2488,7 +2485,7 @@ function entities.char_entity(s) end local char_table = {} for _, code_point in ipairs(code_points) do - table.insert(char_table, unicode.utf8.char(code_point)) + table.insert(char_table, unicode.utf8.char(code_point)) end return table.concat(char_table) end @@ -2540,13 +2537,20 @@ function M.writer.new(options) function self.pack(name) return [[\input{]] .. name .. [[}\relax]] end + self.interblocksep_text = "\\markdownRendererInterblockSeparator\n{}" function self.interblocksep() if not self.is_writing then return "" end - return "\\markdownRendererInterblockSeparator\n{}" + return self.interblocksep_text end + self.paragraphsep_text = "\\markdownRendererParagraphSeparator\n{}" function self.paragraphsep() if not self.is_writing then return "" end - return "\\markdownRendererParagraphSeparator\n{}" + return self.paragraphsep_text + end + self.undosep_text = "\\markdownRendererUndoSeparator\n{}" + function self.undosep() + if not self.is_writing then return "" end + return self.undosep_text end self.soft_line_break = function() if self.flatten_inlines then return "\n" end @@ -2911,6 +2915,7 @@ function M.writer.new(options) if attributes["#" .. self.slice_end_identifier] ~= nil and self.slice_end_type == "^" then if self.is_writing then + table.insert(buf, self.undosep()) table.insert(buf, tear_down_attributes()) end self.is_writing = false @@ -2949,6 +2954,7 @@ function M.writer.new(options) if attributes["#" .. self.slice_end_identifier] ~= nil and self.slice_end_type == "$" then if self.is_writing then + table.insert(buf, self.undosep()) table.insert(buf, tear_down_attributes()) end self.is_writing = false @@ -2961,105 +2967,107 @@ function M.writer.new(options) end return buf end -local function create_auto_identifier(s) - local buffer = {} - local prev_space = false - local letter_found = false + local function create_auto_identifier(s) + local buffer = {} + local prev_space = false + local letter_found = false - for _, code in utf8.codes(uni_algos.normalize.NFC(s)) do - local char = utf8.char(code) + for _, code in utf8.codes(uni_algos.normalize.NFC(s)) do + local char = utf8.char(code) - -- Remove everything up to the first letter. - if not letter_found then - local is_letter = unicode.utf8.match(char, "%a") - if is_letter then - letter_found = true - else - goto continue + -- Remove everything up to the first letter. + if not letter_found then + local is_letter = unicode.utf8.match(char, "%a") + if is_letter then + letter_found = true + else + goto continue + end end - end - - -- Remove all non-alphanumeric characters, except underscores, hyphens, and periods. - if not unicode.utf8.match(char, "[%w_%-%.%s]") then - goto continue - end - -- Replace all spaces and newlines with hyphens. - if unicode.utf8.match(char, "[%s\n]") then - char = "-" - if prev_space then + -- Remove all non-alphanumeric characters, except underscores, hyphens, and periods. + if not unicode.utf8.match(char, "[%w_%-%.%s]") then goto continue + end + + -- Replace all spaces and newlines with hyphens. + if unicode.utf8.match(char, "[%s\n]") then + char = "-" + if prev_space then + goto continue + else + prev_space = true + end else - prev_space = true + -- Convert all alphabetic characters to lowercase. + char = unicode.utf8.lower(char) + prev_space = false end - else - -- Convert all alphabetic characters to lowercase. - char = unicode.utf8.lower(char) - prev_space = false - end - table.insert(buffer, char) + table.insert(buffer, char) - ::continue:: - end + ::continue:: + end + + if prev_space then + table.remove(buffer) + end - if prev_space then - table.remove(buffer) + local identifier = #buffer == 0 and "section" or table.concat(buffer, "") + return identifier end + local function create_gfm_auto_identifier(s) + local buffer = {} + local prev_space = false + local letter_found = false - local identifier = #buffer == 0 and "section" or table.concat(buffer, "") - return identifier -end -local function create_gfm_auto_identifier(s) - local buffer = {} - local prev_space = false - local letter_found = false + for _, code in utf8.codes(uni_algos.normalize.NFC(s)) do + local char = utf8.char(code) - for _, code in utf8.codes(uni_algos.normalize.NFC(s)) do - local char = utf8.char(code) + -- Remove everything up to the first non-space. + if not letter_found then + local is_letter = unicode.utf8.match(char, "%S") + if is_letter then + letter_found = true + else + goto continue + end + end - -- Remove everything up to the first non-space. - if not letter_found then - local is_letter = unicode.utf8.match(char, "%S") - if is_letter then - letter_found = true - else + -- Remove all non-alphanumeric characters, except underscores and hyphens. + if not unicode.utf8.match(char, "[%w_%-%s]") then + prev_space = false goto continue end - end - - -- Remove all non-alphanumeric characters, except underscores and hyphens. - if not unicode.utf8.match(char, "[%w_%-%s]") then - prev_space = false - goto continue - end - -- Replace all spaces and newlines with hyphens. - if unicode.utf8.match(char, "[%s\n]") then - char = "-" - if prev_space then - goto continue + -- Replace all spaces and newlines with hyphens. + if unicode.utf8.match(char, "[%s\n]") then + char = "-" + if prev_space then + goto continue + else + prev_space = true + end else - prev_space = true + -- Convert all alphabetic characters to lowercase. + char = unicode.utf8.lower(char) + prev_space = false end - else - -- Convert all alphabetic characters to lowercase. - char = unicode.utf8.lower(char) - prev_space = false - end - table.insert(buffer, char) + table.insert(buffer, char) - ::continue:: - end + ::continue:: + end - if prev_space then - table.remove(buffer) - end + if prev_space then + table.remove(buffer) + end - local identifier = #buffer == 0 and "section" or table.concat(buffer, "") - return identifier -end + local identifier = #buffer == 0 and "section" or table.concat(buffer, "") + return identifier + end + self.secbegin_text = "\\markdownRendererSectionBegin\n" + self.secend_text = "\n\\markdownRendererSectionEnd " function self.heading(s, level, attributes) local buf = {} local flat_text, inlines = table.unpack(s) @@ -3069,8 +3077,8 @@ end table.insert(buf, self.push_attributes("heading", nil, - "\\markdownRendererSectionBegin\n", - "\n\\markdownRendererSectionEnd ")) + self.secbegin_text, + self.secend_text)) end -- pop attributes for sections that have ended @@ -3091,8 +3099,8 @@ end -- push attributes for the new section local start_output = {} local end_output = {} - table.insert(start_output, "\\markdownRendererSectionBegin\n") - table.insert(end_output, "\n\\markdownRendererSectionEnd ") + table.insert(start_output, self.secbegin_text) + table.insert(end_output, self.secend_text) table.insert(buf, self.push_attributes("heading", normalized_attributes, @@ -3201,7 +3209,6 @@ parsers.letter = R("AZ","az") parsers.alphanumeric = R("AZ","az","09") parsers.keyword = parsers.letter * (parsers.alphanumeric + parsers.dash)^0 -parsers.internal_punctuation = S(":;,.?") parsers.doubleasterisks = P("**") parsers.doubleunderscores = P("__") @@ -3212,7 +3219,35 @@ parsers.any = P(1) parsers.succeed = P(true) parsers.fail = P(false) -parsers.escapable = S("!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~") +parsers.internal_punctuation = S(":;,.?") +parsers.ascii_punctuation = S("!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~") +parsers.punctuation = {} +(function() + local pathname = kpse.lookup("UnicodeData.txt") + local file = assert(io.open(pathname, "r"), + [[Could not open file "UnicodeData.txt"]]) + for line in file:lines() do + local codepoint, major_category = line:match("^(%x+);[^;]*;(%a)") + if major_category == "P" or major_category == "S" then + local code = unicode.utf8.char(tonumber(codepoint, 16)) + if parsers.punctuation[#code] == nil then + parsers.punctuation[#code] = parsers.fail + end + local code_parser = parsers.succeed + for i = 1, #code do + local byte = code:sub(i, i) + local byte_parser = S(byte) + code_parser = code_parser + * byte_parser + end + parsers.punctuation[#code] = parsers.punctuation[#code] + + code_parser + end + end + assert(file:close()) +end)() + +parsers.escapable = parsers.ascii_punctuation parsers.anyescaped = parsers.backslash / "" * parsers.escapable + parsers.any @@ -3463,7 +3498,7 @@ local function left_blank_starter(indent_table) return blank_starter_index end -local function traverse_indent(s, i, indent_table, is_optional, is_blank) +local function traverse_indent(s, i, indent_table, is_optional, is_blank, current_line_indents) local new_index = i local preceding_indentation = 0 @@ -3471,6 +3506,10 @@ local function traverse_indent(s, i, indent_table, is_optional, is_blank) local blank_starter = left_blank_starter(indent_table) + if current_line_indents == nil then + current_line_indents = {} + end + for index = 1,#indent_table.indents do local value = indent_table.indents[index] local pattern = decode_pattern(value.name) @@ -3480,10 +3519,10 @@ local function traverse_indent(s, i, indent_table, is_optional, is_blank) if new_indent_info == nil then local blankline_end = lpeg.match(Ct(parsers.blankline * Cg(Cp(), "pos")), s, new_index) if is_optional or not indent_table.ignore_blockquote_blank or not blankline_end then - return is_optional, new_index, current_trail + return is_optional, new_index, current_trail, current_line_indents end - return traverse_indent(s, tonumber(blankline_end.pos), indent_table, is_optional, is_blank) + return traverse_indent(s, tonumber(blankline_end.pos), indent_table, is_optional, is_blank, current_line_indents) end local raw_last_trail = new_indent_info[1] @@ -3502,7 +3541,7 @@ local function traverse_indent(s, i, indent_table, is_optional, is_blank) if next(current_trail) ~= nil then if not space_only and current_trail.is_code then - return is_optional, new_index, current_trail + return is_optional, new_index, current_trail, current_line_indents end if current_trail.internal_remainder ~= nil then raw_last_trail = current_trail.internal_remainder @@ -3534,7 +3573,7 @@ local function traverse_indent(s, i, indent_table, is_optional, is_blank) local sp = process_starter_spacing(total_indent_level, spacing_to_process, minimum, left_strip_length) if space_only and not sp.is_minimum then - return is_optional or (is_blank and blank_starter <= index), new_index, current_trail + return is_optional or (is_blank and blank_starter <= index), new_index, current_trail, current_line_indents end local indent_length = raw_last_trail_length + delimiter_length + sp.left_total_stripped @@ -3548,10 +3587,12 @@ local function traverse_indent(s, i, indent_table, is_optional, is_blank) current_trail = {is_code=sp.is_code, remainder=sp.remainder, internal_remainder=sp.minimum_remainder, total_length=sp.total_length, full_remainder=sp.full_remainder} + + current_line_indents[#current_line_indents + 1] = new_indent_info new_index = next_index end - return true, new_index, current_trail + return true, new_index, current_trail, current_line_indents end local function check_trail(expect_code, is_code) @@ -3609,9 +3650,11 @@ local function check_continuation_indentation(s, i, indent_table, is_optional, i return true end - local passes, new_index, current_trail = traverse_indent(s, i, indent_table, is_optional, is_blank) + local passes, new_index, current_trail, current_line_indents = + traverse_indent(s, i, indent_table, is_optional, is_blank) if passes then + indent_table.current_line_indents = current_line_indents indent_table = add_trail(indent_table, current_trail) return new_index, indent_table end @@ -4113,7 +4156,7 @@ parsers.html_comment = Cs( parsers.html_comment_start parsers.html_inline_comment = (parsers.html_comment_start / "") * -P(">") * -P("->") - * Cs((V("NoSoftLineBreakEndline") + parsers.any - P("--") + * Cs((V("NoSoftLineBreakEndline") + parsers.any - parsers.nested_breaking_blank - parsers.html_comment_end)^0) * (parsers.html_comment_end / "") @@ -4907,7 +4950,6 @@ function M.reader.new(writer, options) return lpeg.R("\240\244") * cont * cont * cont end end - local function check_unicode_type(s, i, start_pos, end_pos, chartype) local c local char_length @@ -4917,15 +4959,22 @@ function M.reader.new(writer, options) else char_length = pos + 1 end - c = lpeg.match({ C(utf8_by_byte_count(char_length)) },s,i+pos) - if (c ~= nil) and (unicode.utf8.match(c, chartype)) then - return i + + if (chartype == "punctuation") then + if lpeg.match(parsers.punctuation[char_length], s, i+pos) then + return i + end + else + c = lpeg.match({ C(utf8_by_byte_count(char_length)) },s,i+pos) + if (c ~= nil) and (unicode.utf8.match(c, chartype)) then + return i + end end end end local function check_preceding_unicode_punctuation(s, i) - return check_unicode_type(s, i, -4, -1, "%p") + return check_unicode_type(s, i, -4, -1, "punctuation") end local function check_preceding_unicode_whitespace(s, i) @@ -4933,7 +4982,7 @@ function M.reader.new(writer, options) end local function check_following_unicode_punctuation(s, i) - return check_unicode_type(s, i, 0, 3, "%p") + return check_unicode_type(s, i, 0, 3, "punctuation") end local function check_following_unicode_whitespace(s, i) @@ -6136,7 +6185,43 @@ end local output local function convert(input) local document = self.parser_functions.parse_blocks(input) - return util.rope_to_string(writer.document(document)) + local output = util.rope_to_string(writer.document(document)) + local undosep_start, undosep_end + local potential_secend_start, secend_start + local potential_sep_start, sep_start + while true do + -- find a `writer->undosep` + undosep_start, undosep_end = output:find(writer.undosep_text, 1, true) + if undosep_start == nil then break end + -- skip any preceding section ends + secend_start = undosep_start + while true do + potential_secend_start = secend_start - #writer.secend_text + if potential_secend_start < 1 + or output:sub(potential_secend_start, secend_start - 1) ~= writer.secend_text then + break + end + secend_start = potential_secend_start + end + -- find an immediately preceding block element / paragraph separator + sep_start = secend_start + potential_sep_start = sep_start - #writer.interblocksep_text + if potential_sep_start >= 1 + and output:sub(potential_sep_start, sep_start - 1) == writer.interblocksep_text then + sep_start = potential_sep_start + else + potential_sep_start = sep_start - #writer.paragraphsep_text + if potential_sep_start >= 1 + and output:sub(potential_sep_start, sep_start - 1) == writer.paragraphsep_text then + sep_start = potential_sep_start + end + end + -- remove `writer->undosep` and immediately preceding block element / paragraph separator + output = output:sub(1, sep_start - 1) + .. output:sub(secend_start, undosep_start - 1) + .. output:sub(undosep_end + 1) + end + return output end if options.eagerCache or options.finalizeCache then local name = util.cache(options.cacheDir, input, salt, convert, @@ -6254,11 +6339,11 @@ M.extensions.citations = function(citation_nbsps) + parsers.bracketed + parsers.inticks + parsers.autolink - + V("InlineHtml") + + V("InlineHtml") + V("Space") + V("Endline") + (parsers.anyescaped - (parsers.newline + parsers.rbracket + parsers.blankline^2)) - - (parsers.spnl * parsers.dash^-1 * parsers.at))^0) + - (parsers.spnl * parsers.dash^-1 * parsers.at))^1) local citation_body_postnote = Cs((parsers.alphanumeric^1 @@ -6270,18 +6355,28 @@ M.extensions.citations = function(citation_nbsps) + (parsers.anyescaped - (parsers.newline + parsers.rbracket + parsers.semicolon + parsers.blankline^2)) - - (parsers.spnl * parsers.rbracket))^0) + - (parsers.spnl * parsers.rbracket))^1) local citation_body_chunk - = citation_body_prenote - * parsers.spnlc * citation_name + = ( citation_body_prenote + * parsers.spnlc_sep + + Cc("") + * parsers.spnlc + ) + * citation_name * (parsers.internal_punctuation - parsers.semicolon)^-1 - * parsers.spnlc * citation_body_postnote + * ( parsers.spnlc + * citation_body_postnote + + Cc("") + * parsers.spnlc + ) local citation_body = citation_body_chunk - * (parsers.semicolon * parsers.spnlc - * citation_body_chunk)^0 + * ( parsers.semicolon + * parsers.spnlc + * citation_body_chunk + )^0 local citation_headless_body_postnote = Cs((parsers.alphanumeric^1 @@ -6297,8 +6392,10 @@ M.extensions.citations = function(citation_nbsps) local citation_headless_body = citation_headless_body_postnote - * (parsers.sp * parsers.semicolon * parsers.spnlc - * citation_body_chunk)^0 + * ( parsers.semicolon + * parsers.spnlc + * citation_body_chunk + )^0 local citations = function(text_cites, raw_cites) @@ -6359,7 +6456,7 @@ end M.extensions.content_blocks = function(language_map) local languages_json = (function() local base, prev, curr - for _, pathname in ipairs{util.lookup_files(language_map, { all=true })} do + for _, pathname in ipairs{kpse.lookup(language_map, { all=true })} do local file = io.open(pathname, "r") if not file then goto continue end local input = assert(file:read("*a")) @@ -7040,17 +7137,44 @@ M.extensions.fenced_divs = function(blank_before_div_fence) * parsers.colon^3 * parsers.optionalspace * (parsers.newline + parsers.eof) - self.initialize_named_group("div_level", "0") + self.initialize_named_group("fenced_div_level", "0") + self.initialize_named_group("fenced_div_num_opening_indents") + + local function increment_div_level() + local function push_indent_table(s, i, indent_table, -- luacheck: ignore s i + fenced_div_num_opening_indents, fenced_div_level) + fenced_div_level = tonumber(fenced_div_level) + 1 + local num_opening_indents = 0 + if indent_table.indents ~= nil then + num_opening_indents = #indent_table.indents + end + fenced_div_num_opening_indents[fenced_div_level] = num_opening_indents + return true, fenced_div_num_opening_indents + end - local function increment_div_level(increment) - local function update_div_level(s, i, current_level) -- luacheck: ignore s i - current_level = tonumber(current_level) - local next_level = tostring(current_level + increment) - return true, next_level + local function increment_level(s, i, fenced_div_level) -- luacheck: ignore s i + fenced_div_level = tonumber(fenced_div_level) + 1 + return true, tostring(fenced_div_level) end - return Cg( Cmt(Cb("div_level"), update_div_level) - , "div_level") + return Cg( Cmt( Cb("indent_info") + * Cb("fenced_div_num_opening_indents") + * Cb("fenced_div_level"), push_indent_table) + , "fenced_div_num_opening_indents") + * Cg( Cmt( Cb("fenced_div_level"), increment_level) + , "fenced_div_level") + end + + local function decrement_div_level() + local function pop_indent_table(s, i, fenced_div_indent_table, fenced_div_level) -- luacheck: ignore s i + fenced_div_level = tonumber(fenced_div_level) + fenced_div_indent_table[fenced_div_level] = nil + return true, tostring(fenced_div_level - 1) + end + + return Cg( Cmt( Cb("fenced_div_num_opening_indents") + * Cb("fenced_div_level"), pop_indent_table) + , "fenced_div_level") end local non_fenced_div_block = parsers.check_minimal_indent * V("Block") @@ -7087,11 +7211,12 @@ M.extensions.fenced_divs = function(blank_before_div_fence) return attr end / writer.div_begin - * increment_div_level(1) + * increment_div_level() * parsers.skipblanklines * Ct(content_loop) * parsers.minimally_indented_blank^0 - * parsers.check_minimal_indent_and_trail * fenced_div_end * increment_div_level(-1) + * parsers.check_minimal_indent_and_trail * fenced_div_end + * decrement_div_level() * (Cc("") / writer.div_end) self.insert_pattern("Block after Verbatim", @@ -7099,13 +7224,33 @@ M.extensions.fenced_divs = function(blank_before_div_fence) self.add_special_character(":") - local function check_div_level(s, i, current_level) -- luacheck: ignore s i - current_level = tonumber(current_level) - return current_level > 0 + local function is_inside_div() + local function check_div_level(s, i, fenced_div_level) -- luacheck: ignore s i + fenced_div_level = tonumber(fenced_div_level) + return fenced_div_level > 0 + end + + return Cmt(Cb("fenced_div_level"), check_div_level) + end + + local function check_indent() + local function compare_indent(s, i, indent_table, -- luacheck: ignore s i + fenced_div_num_opening_indents, fenced_div_level) + fenced_div_level = tonumber(fenced_div_level) + local num_current_indents = (indent_table.current_line_indents ~= nil and + #indent_table.current_line_indents) or 0 + local num_opening_indents = fenced_div_num_opening_indents[fenced_div_level] + return num_current_indents == num_opening_indents + end + + return Cmt( Cb("indent_info") + * Cb("fenced_div_num_opening_indents") + * Cb("fenced_div_level"), compare_indent) end - local is_inside_div = Cmt(Cb("div_level"), check_div_level) - local fencestart = is_inside_div * fenced_div_end + local fencestart = is_inside_div() + * fenced_div_end + * check_indent() if not blank_before_div_fence then self.update_rule("EndlineExceptions", function(previous_pattern) @@ -8116,7 +8261,7 @@ function M.new(options) end for _, user_extension_filename in ipairs(options.extensions) do local user_extension = (function(filename) - local pathname = util.lookup_files(filename) + local pathname = kpse.lookup(filename) local input_file = assert(io.open(pathname, "r"), [[Could not open user-defined syntax extension "]] .. pathname .. [[" for reading]]) |