From 835ae450df95b8a97500efebbe995d9950c4e2b2 Mon Sep 17 00:00:00 2001 From: Karl Berry Date: Mon, 29 Aug 2016 21:19:56 +0000 Subject: markdown (29aug16) git-svn-id: svn://tug.org/texlive/trunk@41950 c570f23f-e606-0410-a88d-b1316a301751 --- Master/texmf-dist/tex/luatex/markdown/markdown.lua | 344 +++++++++++++++------ 1 file changed, 257 insertions(+), 87 deletions(-) (limited to 'Master/texmf-dist/tex/luatex/markdown') diff --git a/Master/texmf-dist/tex/luatex/markdown/markdown.lua b/Master/texmf-dist/tex/luatex/markdown/markdown.lua index 8a0f237c1d7..6a194bbf7ea 100644 --- a/Master/texmf-dist/tex/luatex/markdown/markdown.lua +++ b/Master/texmf-dist/tex/luatex/markdown/markdown.lua @@ -57,24 +57,30 @@ -- should contain your own preambles for the files it generates, not -- those in the standard .ins files. -- -if not modules then modules = { } end modules ['markdown'] = { - version = "1.0.2", +local metadata = { + version = "2.1.0", comment = "A module for the conversion from markdown to plain TeX", author = "John MacFarlane, Hans Hagen, Vít Novotný", copyright = "2009-2016 John MacFarlane, Hans Hagen; 2016 Vít Novotný", license = "LPPL 1.3" } +if not modules then modules = { } end +modules['markdown'] = metadata local lpeg = require("lpeg") local unicode = require("unicode") local md5 = require("md5") local M = {} local defaultOptions = {} defaultOptions.blankBeforeBlockquote = false +defaultOptions.blankBeforeCodeFence = false defaultOptions.blankBeforeHeading = false defaultOptions.cacheDir = "." +defaultOptions.citationNbsps = true +defaultOptions.citations = false defaultOptions.definitionLists = false defaultOptions.hashEnumerators = false defaultOptions.hybrid = false +defaultOptions.fencedCode = false defaultOptions.footnotes = false defaultOptions.preserveTabs = false defaultOptions.smartEllipses = false @@ -222,6 +228,7 @@ function M.writer.new(options) return defaultOptions[key] end }) self.suffix = ".tex" self.space = " " + self.nbsp = "\\markdownRendererNbsp{}" function self.plain(s) return s end @@ -231,12 +238,11 @@ function M.writer.new(options) function self.pack(name) return [[\input"]] .. name .. [["\relax]] end - self.interblocksep = "\n\n" - self.containersep = "\n\n" + self.interblocksep = "\\markdownRendererInterblockSeparator\n{}" self.eof = [[\relax]] - self.linebreak = "\\markdownRendererLineBreak " + self.linebreak = "\\markdownRendererLineBreak\n{}" self.ellipsis = "\\markdownRendererEllipsis{}" - self.hrule = "\\markdownRendererHorizontalRule " + self.hrule = "\\markdownRendererHorizontalRule{}" local escaped_chars = { ["{"] = "\\markdownRendererLeftBrace{}", ["}"] = "\\markdownRendererRightBrace{}", @@ -282,7 +288,8 @@ function M.writer.new(options) "{",self.string(tit or ""),"}"} end local function ulitem(s) - return {"\\markdownRendererUlItem ",s} + return {"\\markdownRendererUlItem ",s, + "\\markdownRendererUlItemEnd "} end function self.bulletlist(items,tight) @@ -301,9 +308,11 @@ function M.writer.new(options) end local function olitem(s,num) if num ~= nil then - return {"\\markdownRendererOlItemWithNumber{",num,"}",s} + return {"\\markdownRendererOlItemWithNumber{",num,"}",s, + "\\markdownRendererOlItemEnd "} else - return {"\\markdownRendererOlItem ",s} + return {"\\markdownRendererOlItem ",s, + "\\markdownRendererOlItemEnd "} end end @@ -331,6 +340,7 @@ function M.writer.new(options) retVal[#retVal+1] = {"\\markdownRendererDlDefinitionBegin ",def, "\\markdownRendererDlDefinitionEnd "} end + retVal[#retVal+1] = "\\markdownRendererDlItemEnd " return retVal end @@ -339,13 +349,12 @@ function M.writer.new(options) for _,item in ipairs(items) do buffer[#buffer + 1] = dlitem(item.term, item.definitions) end - local contents = util.intersperse(buffer, self.containersep) if tight and options.tightLists then - return {"\\markdownRendererDlBeginTight\n\n", contents, - "\n\n\\markdownRendererDlEndTight\n"} + return {"\\markdownRendererDlBeginTight\n", buffer, + "\n\\markdownRendererDlEndTight"} else - return {"\\markdownRendererDlBegin\n\n", contents, - "\n\n\\markdownRendererDlEnd\n"} + return {"\\markdownRendererDlBegin\n", buffer, + "\n\\markdownRendererDlEnd"} end end function self.emphasis(s) @@ -362,6 +371,10 @@ function M.writer.new(options) local name = util.cache(options.cacheDir, s, nil, nil, ".verbatim") return {"\\markdownRendererInputVerbatim{",name,"}"} end + function self.fencedCode(i, s) + local name = util.cache(options.cacheDir, s, nil, nil, ".verbatim") + return {"\\markdownRendererInputFencedCode{",name,"}{",i,"}"} + end function self.heading(s,level) local cmd if level == 1 then @@ -384,6 +397,15 @@ function M.writer.new(options) function self.note(s) return {"\\markdownRendererFootnote{",s,"}"} end + function self.citations(text_cites, cites) + local buffer = {"\\markdownRenderer", text_cites and "TextCite" or "Cite", + "{", #cites, "}"} + for _,cite in ipairs(cites) do + buffer[#buffer+1] = {cite.suppress_author and "-" or "+", "{", + cite.prenote or "", "}{", cite.postnote or "", "}{", cite.name, "}"} + end + return buffer + end return self end @@ -410,41 +432,33 @@ function M.reader.new(writer, options) end end local syntax + local blocks_toplevel local blocks - local inlines + local inlines, inlines_no_link, inlines_nbsp - local parse_blocks = - function(str) - local res = lpeg.match(blocks, str) + local function create_parser(name, grammar) + return function(str) + local res = lpeg.match(grammar(), str) if res == nil then - error(format("parse_blocks failed on:\n%s", str:sub(1,20))) - else - return res - end - end - - local parse_inlines = - function(str) - local res = lpeg.match(inlines, str) - if res == nil then - error(format("parse_inlines failed on:\n%s", - str:sub(1,20))) + error(format("%s failed on:\n%s", name, str:sub(1,20))) else return res end end + end - local parse_inlines_no_link = - function(str) - local res = lpeg.match(inlines_no_link, str) - if res == nil then - error(format("parse_inlines_no_link failed on:\n%s", - str:sub(1,20))) - else - return res - end - end + local parse_blocks = create_parser("parse_blocks", + function() return blocks end) + local parse_blocks_toplevel = create_parser("parse_blocks_toplevel", + function() return blocks_toplevel end) + local parse_inlines = create_parser("parse_inlines", + function() return inlines end) + local parse_inlines_no_link = create_parser("parse_inlines_no_link", + function() return inlines_no_link end) + local parse_inlines_nbsp = create_parser("parse_inlines_nbsp", + function() return inlines_nbsp end) local percent = P("%") + local at = P("@") local asterisk = P("*") local dash = P("-") local plus = P("+") @@ -468,12 +482,14 @@ function M.reader.new(writer, options) local colon = P(":") local semicolon = P(";") local exclamation = P("!") + local tilde = P("~") local digit = R("09") local hexdigit = R("09","af","AF") local letter = R("AZ","az") local alphanumeric = R("AZ","az","09") local keyword = letter * alphanumeric^0 + local punctuation = S("!?,.:;") local doubleasterisks = P("**") local doubleunderscores = P("__") @@ -483,7 +499,8 @@ function M.reader.new(writer, options) local fail = any - 1 local always = P("") - local escapable = S("\\`*_{}[]()+_.!<>#-~:^") + local escapable = S("\\`*_{}[]()+_.!<>#-~:^@;") + local anyescaped = P("\\") / "" * escapable + any @@ -494,17 +511,14 @@ function M.reader.new(writer, options) local nonspacechar = any - spacing local tightblocksep = P("\001") - local specialchar - if options.smartEllipses then - specialchar = S("*_`&[]= #b and i + end + + local infostring = (linechar - (backtick + space^1 * (newline + eof)))^0 + + local fenceindent + local function fencehead(char) + return C(nonindentspace) / function(s) fenceindent = #s end + * Cg(char^3, "fencelength") + * optionalspace * C(infostring) * optionalspace + * (newline + eof) + end + + local function fencetail(char) + return nonindentspace + * Cmt(C(char^3) * Cb("fencelength"), + captures_geq_length) + * optionalspace * (newline + eof) + + eof + end + + local function fencedline(char) + return C(line - fencetail(char)) + / function(s) + return s:gsub("^" .. string.rep(" ?", + fenceindent), "") + end + end local leader = space^-3 -- in balanced brackets, parentheses, quotes: @@ -616,6 +663,38 @@ function M.reader.new(writer, options) local optionaltitle = spnl * title * spacechar^0 + Cc("") + local citation_name = (Cs(dash^-1) + * at * Cs((anyescaped + - (rbracket + spacing + punctuation + semicolon))^1)) + + local citation_body_prenote + = Cs((alphanumeric^1 + + bracketed + + inticks + + (anyescaped + - (rbracket + semicolon + blankline^2)) + - (spnl * dash^-1 * at))^0) + + local citation_body_postnote + = Cs((alphanumeric^1 + + bracketed + + inticks + + (anyescaped + - (rbracket + semicolon + blankline^2)) + - (spnl * rbracket))^0) + + local citation_body_chunk + = citation_body_prenote + * spnl * citation_name + * spnlor(punctuation - (semicolon + rbracket)) + * citation_body_postnote + + local citation_body = citation_body_chunk + * (semicolon * spnl * citation_body_chunk)^0 + + local citation_headless_body + = citation_body_postnote + * (sp * semicolon * spnl * citation_body_chunk)^0 local rawnotes = {} local function strip_first_char(s) @@ -627,7 +706,7 @@ function M.reader.new(writer, options) return function() local found = rawnotes[normalize_tag(ref)] if found then - return writer.note(parse_blocks(found)) + return writer.note(parse_blocks_toplevel(found)) else return {"[", parse_inlines("^" .. ref), "]"} end @@ -713,21 +792,10 @@ function M.reader.new(writer, options) end end end - local Inline = V("Inline") - - local Str = normalchar^1 / writer.string - - local Ellipsis = P("...") / writer.ellipsis - - local Smart = Ellipsis - - local Symbol = (specialchar - tightblocksep) / writer.string - - local Code = inticks / writer.code - local bqstart = more local headerstart = hash + (line * (equal^1 + dash^1) * optionalspace * newline) + local fencestart = fencehead(backtick) + fencehead(tilde) if options.blankBeforeBlockquote then bqstart = fail @@ -737,24 +805,41 @@ function M.reader.new(writer, options) headerstart = fail end + if not options.fencedCode or options.blankBeforeCodeFence then + fencestart = fail + end + local Inline = V("Inline") + + local Str = normalchar^1 / writer.string + + local Symbol = (specialchar - tightblocksep) / writer.string + local Ellipsis = P("...") / writer.ellipsis + + local Smart = Ellipsis + local Code = inticks / writer.code local Endline = newline * -( -- newline, but not before... blankline -- paragraph break + tightblocksep -- nested list + eof -- end of document + bqstart + headerstart + + fencestart ) * spacechar^0 / writer.space local Space = spacechar^2 * Endline / writer.linebreak + spacechar^1 * Endline^-1 * eof / "" + spacechar^1 * Endline^-1 * optionalspace / writer.space + local NonbreakingSpace + = spacechar^2 * Endline / writer.linebreak + + spacechar^1 * Endline^-1 * eof / "" + + spacechar^1 * Endline^-1 * optionalspace / writer.nbsp + -- parse many p between starter and ender local function between(p, starter, ender) local ender2 = B(nonspacechar) * ender return (starter * #nonspacechar * Ct(p * (p - ender2)^0) * ender2) end - local Strong = ( between(Inline, doubleasterisks, doubleasterisks) + between(Inline, doubleunderscores, doubleunderscores) ) / writer.strong @@ -762,7 +847,6 @@ function M.reader.new(writer, options) local Emph = ( between(Inline, asterisk, asterisk) + between(Inline, underscore, underscore) ) / writer.emphasis - local urlchar = anyescaped - newline - more local AutoLinkUrl = less @@ -792,7 +876,6 @@ function M.reader.new(writer, options) -- parse a link or image (direct or indirect) local Link = DirectLink + IndirectLink - local DirectImage = exclamation * (tag / parse_inlines) * spnl @@ -806,43 +889,89 @@ function M.reader.new(writer, options) indirect_image local Image = DirectImage + IndirectImage - -- avoid parsing long strings of * or _ as emph/strong local UlOrStarLine = asterisk^4 + underscore^4 / writer.string local EscapedChar = S("\\") * C(escapable) / writer.string + local function citations(text_cites, raw_cites) + local function normalize(str) + if str == "" then + str = nil + else + str = (options.citationNbsps and parse_inlines_nbsp or + parse_inlines)(str) + end + return str + end + + local cites = {} + for i = 1,#raw_cites,4 do + cites[#cites+1] = { + prenote = normalize(raw_cites[i]), + suppress_author = raw_cites[i+1] == "-", + name = writer.string(raw_cites[i+2]), + postnote = normalize(raw_cites[i+3]), + } + end + return writer.citations(text_cites, cites) + end + + local TextCitations = Ct(Cc("") + * citation_name + * ((spnl + * lbracket + * citation_headless_body + * rbracket) + Cc(""))) / + function(raw_cites) + return citations(true, raw_cites) + end + + local ParenthesizedCitations + = Ct(lbracket + * citation_body + * rbracket) / + function(raw_cites) + return citations(false, raw_cites) + end + + local Citations = TextCitations + ParenthesizedCitations local Block = V("Block") local Verbatim = Cs( (blanklines * ((indentedline - blankline))^1)^1 ) / expandtabs / writer.verbatim + local TildeFencedCode + = fencehead(tilde) + * Cs(fencedline(tilde)^0) + * fencetail(tilde) + + local BacktickFencedCode + = fencehead(backtick) + * Cs(fencedline(backtick)^0) + * fencetail(backtick) + + local FencedCode = (TildeFencedCode + BacktickFencedCode) + / function(infostring, code) + return writer.fencedCode( + writer.string(infostring), + expandtabs(code)) + end -- strip off leading > and indents, and run through blocks local Blockquote = Cs(( ((leader * more * space^-1)/"" * linechar^0 * newline)^1 * (-blankline * linechar^1 * newline)^0 - * blankline^0 - )^1) / parse_blocks / writer.blockquote + * (blankline^0 / "") + )^1) / parse_blocks_toplevel / writer.blockquote local function lineof(c) - return (leader * (P(c) * optionalspace)^3 * newline * blankline^1) + return (leader * (P(c) * optionalspace)^3 * (newline * blankline^1 + + newline^-1 * eof)) end - local HorizontalRule = ( lineof(asterisk) + lineof(dash) + lineof(underscore) ) / writer.hrule - - local Reference = define_reference_parser / register_link - - local Paragraph = nonindentspace * Ct(Inline^1) * newline - * ( blankline^1 - + #hash - + #(leader * more * space^-1) - ) - / writer.paragraph - - local Plain = nonindentspace * Ct(Inline^1) / writer.plain local starter = bullet + enumerator -- we use \001 as a separator between a tight list item and a @@ -913,7 +1042,7 @@ function M.reader.new(writer, options) local DefinitionListItemLoose = C(line) * skipblanklines * Ct((defstart * indented_blocks(dlchunk) / - parse_blocks)^1) + parse_blocks_toplevel)^1) * Cc(false) / definition_list_item @@ -928,10 +1057,29 @@ function M.reader.new(writer, options) * (skipblanklines * -DefinitionListItemLoose * Cc(true)) ) / writer.definitionlist + local Reference = define_reference_parser / register_link local Blank = blankline / "" + NoteBlock + Reference + (tightblocksep / "\n") + local Paragraph = nonindentspace * Ct(Inline^1) * newline + * ( blankline^1 + + #hash + + #(leader * more * space^-1) + ) + / writer.paragraph + + local ToplevelParagraph + = nonindentspace * Ct(Inline^1) * (newline + * ( blankline^1 + + #hash + + #(leader * more * space^-1) + + eof + ) + + eof ) + / writer.paragraph + + local Plain = nonindentspace * Ct(Inline^1) / writer.plain -- parse Atx heading start and return level local HeadingStart = #hash * C(hash^-6) * -hash / length @@ -955,6 +1103,8 @@ function M.reader.new(writer, options) * HeadingLevel * optionalspace * newline / writer.heading + + local Heading = AtxHeading + SetextHeading syntax = { "Blocks", @@ -970,6 +1120,7 @@ function M.reader.new(writer, options) Block = V("Blockquote") + V("Verbatim") + + V("FencedCode") + V("HorizontalRule") + V("BulletList") + V("OrderedList") @@ -980,10 +1131,11 @@ function M.reader.new(writer, options) Blockquote = Blockquote, Verbatim = Verbatim, + FencedCode = FencedCode, HorizontalRule = HorizontalRule, BulletList = BulletList, OrderedList = OrderedList, - Heading = AtxHeading + SetextHeading, + Heading = Heading, DefinitionList = DefinitionList, DisplayHtml = DisplayHtml, Paragraph = Paragraph, @@ -996,6 +1148,7 @@ function M.reader.new(writer, options) + V("Strong") + V("Emph") + V("NoteRef") + + V("Citations") + V("Link") + V("Image") + V("Code") @@ -1012,6 +1165,7 @@ function M.reader.new(writer, options) Strong = Strong, Emph = Emph, NoteRef = NoteRef, + Citations = Citations, Link = Link, Image = Image, Code = Code, @@ -1028,6 +1182,14 @@ function M.reader.new(writer, options) syntax.DefinitionList = fail end + if not options.fencedCode then + syntax.FencedCode = fail + end + + if not options.citations then + syntax.Citations = fail + end + if not options.footnotes then syntax.NoteRef = fail end @@ -1036,6 +1198,10 @@ function M.reader.new(writer, options) syntax.Smart = fail end + local blocks_toplevel_t = util.table_copy(syntax) + blocks_toplevel_t.Paragraph = ToplevelParagraph + blocks_toplevel = Ct(blocks_toplevel_t) + blocks = Ct(syntax) local inlines_t = util.table_copy(syntax) @@ -1043,9 +1209,13 @@ function M.reader.new(writer, options) inlines_t.Inlines = Inline^0 * (spacing^0 * eof / "") inlines = Ct(inlines_t) - inlines_no_link_t = util.table_copy(inlines_t) + local inlines_no_link_t = util.table_copy(inlines_t) inlines_no_link_t.Link = fail inlines_no_link = Ct(inlines_no_link_t) + + local inlines_nbsp_t = util.table_copy(inlines_t) + inlines_nbsp_t.Space = NonbreakingSpace + inlines_nbsp = Ct(inlines_nbsp_t) function self.convert(input) references = {} local opt_string = {} @@ -1056,9 +1226,9 @@ function M.reader.new(writer, options) end end table.sort(opt_string) - local salt = table.concat(opt_string, ",") + local salt = table.concat(opt_string, ",") .. "," .. metadata.version local name = util.cache(options.cacheDir, input, salt, function(input) - return util.rope_to_string(parse_blocks(input)) .. writer.eof + return util.rope_to_string(parse_blocks_toplevel(input)) .. writer.eof end, ".md" .. writer.suffix) return writer.pack(name) end -- cgit v1.2.3