markdown (29aug16)

git-svn-id: svn://tug.org/texlive/trunk@41950 c570f23f-e606-0410-a88d-b1316a301751

1 files changed, 257 insertions, 87 deletions

diff --git a/Master/texmf-dist/tex/luatex/markdown/markdown.lua b/Master/texmf-dist/tex/luatex/markdown/markdown.lua
index 8a0f237c1d7..6a194bbf7ea 100644
--- a/Master/texmf-dist/tex/luatex/markdown/markdown.lua
+++ b/Master/texmf-dist/tex/luatex/markdown/markdown.lua
@@ -57,24 +57,30 @@
 -- should contain your own preambles for the files it generates, not
 -- those in the standard .ins files.
 -- 
-if not modules then modules = { } end modules ['markdown'] = {
-    version   = "1.0.2",
+local metadata = {
+    version   = "2.1.0",
     comment   = "A module for the conversion from markdown to plain TeX",
     author    = "John MacFarlane, Hans Hagen, Vít Novotný",
     copyright = "2009-2016 John MacFarlane, Hans Hagen; 2016 Vít Novotný",
     license   = "LPPL 1.3"
 }
+if not modules then modules = { } end
+modules['markdown'] = metadata
 local lpeg = require("lpeg")
 local unicode = require("unicode")
 local md5 = require("md5")
 local M = {}
 local defaultOptions = {}
 defaultOptions.blankBeforeBlockquote = false
+defaultOptions.blankBeforeCodeFence = false
 defaultOptions.blankBeforeHeading = false
 defaultOptions.cacheDir = "."
+defaultOptions.citationNbsps = true
+defaultOptions.citations = false
 defaultOptions.definitionLists = false
 defaultOptions.hashEnumerators = false
 defaultOptions.hybrid = false
+defaultOptions.fencedCode = false
 defaultOptions.footnotes = false
 defaultOptions.preserveTabs = false
 defaultOptions.smartEllipses = false
@@ -222,6 +228,7 @@ function M.writer.new(options)
     return defaultOptions[key] end })
   self.suffix = ".tex"
   self.space = " "
+  self.nbsp = "\\markdownRendererNbsp{}"
   function self.plain(s)
     return s
   end
@@ -231,12 +238,11 @@ function M.writer.new(options)
   function self.pack(name)
     return [[\input"]] .. name .. [["\relax]]
   end
-  self.interblocksep = "\n\n"
-  self.containersep = "\n\n"
+  self.interblocksep = "\\markdownRendererInterblockSeparator\n{}"
   self.eof = [[\relax]]
-  self.linebreak = "\\markdownRendererLineBreak "
+  self.linebreak = "\\markdownRendererLineBreak\n{}"
   self.ellipsis = "\\markdownRendererEllipsis{}"
-  self.hrule = "\\markdownRendererHorizontalRule "
+  self.hrule = "\\markdownRendererHorizontalRule{}"
   local escaped_chars = {
      ["{"] = "\\markdownRendererLeftBrace{}",
      ["}"] = "\\markdownRendererRightBrace{}",
@@ -282,7 +288,8 @@ function M.writer.new(options)
                            "{",self.string(tit or ""),"}"}
   end
   local function ulitem(s)
-    return {"\\markdownRendererUlItem ",s}
+    return {"\\markdownRendererUlItem ",s,
+            "\\markdownRendererUlItemEnd "}
   end
 
   function self.bulletlist(items,tight)
@@ -301,9 +308,11 @@ function M.writer.new(options)
   end
   local function olitem(s,num)
     if num ~= nil then
-      return {"\\markdownRendererOlItemWithNumber{",num,"}",s}
+      return {"\\markdownRendererOlItemWithNumber{",num,"}",s,
+              "\\markdownRendererOlItemEnd "}
     else
-      return {"\\markdownRendererOlItem ",s}
+      return {"\\markdownRendererOlItem ",s,
+              "\\markdownRendererOlItemEnd "}
     end
   end
 
@@ -331,6 +340,7 @@ function M.writer.new(options)
       retVal[#retVal+1] = {"\\markdownRendererDlDefinitionBegin ",def,
                            "\\markdownRendererDlDefinitionEnd "}
     end
+    retVal[#retVal+1] = "\\markdownRendererDlItemEnd "
     return retVal
   end
 
@@ -339,13 +349,12 @@ function M.writer.new(options)
     for _,item in ipairs(items) do
       buffer[#buffer + 1] = dlitem(item.term, item.definitions)
     end
-    local contents = util.intersperse(buffer, self.containersep)
     if tight and options.tightLists then
-      return {"\\markdownRendererDlBeginTight\n\n", contents,
-        "\n\n\\markdownRendererDlEndTight\n"}
+      return {"\\markdownRendererDlBeginTight\n", buffer,
+        "\n\\markdownRendererDlEndTight"}
     else
-      return {"\\markdownRendererDlBegin\n\n", contents,
-        "\n\n\\markdownRendererDlEnd\n"}
+      return {"\\markdownRendererDlBegin\n", buffer,
+        "\n\\markdownRendererDlEnd"}
     end
   end
   function self.emphasis(s)
@@ -362,6 +371,10 @@ function M.writer.new(options)
     local name = util.cache(options.cacheDir, s, nil, nil, ".verbatim")
     return {"\\markdownRendererInputVerbatim{",name,"}"}
   end
+  function self.fencedCode(i, s)
+    local name = util.cache(options.cacheDir, s, nil, nil, ".verbatim")
+    return {"\\markdownRendererInputFencedCode{",name,"}{",i,"}"}
+  end
   function self.heading(s,level)
     local cmd
     if level == 1 then
@@ -384,6 +397,15 @@ function M.writer.new(options)
   function self.note(s)
     return {"\\markdownRendererFootnote{",s,"}"}
   end
+  function self.citations(text_cites, cites)
+    local buffer = {"\\markdownRenderer", text_cites and "TextCite" or "Cite",
+      "{", #cites, "}"}
+    for _,cite in ipairs(cites) do
+      buffer[#buffer+1] = {cite.suppress_author and "-" or "+", "{",
+        cite.prenote or "", "}{", cite.postnote or "", "}{", cite.name, "}"}
+    end
+    return buffer
+  end
 
   return self
 end
@@ -410,41 +432,33 @@ function M.reader.new(writer, options)
                  end
   end
   local syntax
+  local blocks_toplevel
   local blocks
-  local inlines
+  local inlines, inlines_no_link, inlines_nbsp
 
-  local parse_blocks =
-    function(str)
-      local res = lpeg.match(blocks, str)
+  local function create_parser(name, grammar)
+    return function(str)
+      local res = lpeg.match(grammar(), str)
       if res == nil then
-        error(format("parse_blocks failed on:\n%s", str:sub(1,20)))
-      else
-        return res
-      end
-    end
-
-  local parse_inlines =
-    function(str)
-      local res = lpeg.match(inlines, str)
-      if res == nil then
-        error(format("parse_inlines failed on:\n%s",
-        str:sub(1,20)))
+        error(format("%s failed on:\n%s", name, str:sub(1,20)))
       else
         return res
       end
     end
+  end
 
-  local parse_inlines_no_link =
-    function(str)
-      local res = lpeg.match(inlines_no_link, str)
-      if res == nil then
-        error(format("parse_inlines_no_link failed on:\n%s",
-        str:sub(1,20)))
-      else
-        return res
-      end
-    end
+  local parse_blocks = create_parser("parse_blocks",
+    function() return blocks end)
+  local parse_blocks_toplevel = create_parser("parse_blocks_toplevel",
+    function() return blocks_toplevel end)
+  local parse_inlines = create_parser("parse_inlines",
+    function() return inlines end)
+  local parse_inlines_no_link = create_parser("parse_inlines_no_link",
+    function() return inlines_no_link end)
+  local parse_inlines_nbsp = create_parser("parse_inlines_nbsp",
+    function() return inlines_nbsp end)
   local percent                = P("%")
+  local at                     = P("@")
   local asterisk               = P("*")
   local dash                   = P("-")
   local plus                   = P("+")
@@ -468,12 +482,14 @@ function M.reader.new(writer, options)
   local colon                  = P(":")
   local semicolon              = P(";")
   local exclamation            = P("!")
+  local tilde                  = P("~")
 
   local digit                  = R("09")
   local hexdigit               = R("09","af","AF")
   local letter                 = R("AZ","az")
   local alphanumeric           = R("AZ","az","09")
   local keyword                = letter * alphanumeric^0
+  local punctuation            = S("!?,.:;")
 
   local doubleasterisks        = P("**")
   local doubleunderscores      = P("__")
@@ -483,7 +499,8 @@ function M.reader.new(writer, options)
   local fail                   = any - 1
   local always                 = P("")
 
-  local escapable              = S("\\`*_{}[]()+_.!<>#-~:^")
+  local escapable              = S("\\`*_{}[]()+_.!<>#-~:^@;")
+
   local anyescaped             = P("\\") / "" * escapable
                                + any
 
@@ -494,17 +511,14 @@ function M.reader.new(writer, options)
   local nonspacechar           = any - spacing
   local tightblocksep          = P("\001")
 
-  local specialchar
-  if options.smartEllipses then
-    specialchar                = S("*_`&[]<!\\.")
-  else
-    specialchar                = S("*_`&[]<!\\")
-  end
+  local specialchar            = S("*_`&[]<!\\.@-")
 
   local normalchar             = any -
                                  (specialchar + spacing + tightblocksep)
   local optionalspace          = spacechar^0
-  local spaces                 = spacechar^1
+  local optionalspaceor = function(pattern)
+    return                      (spacechar + pattern)^0
+  end
   local eof                    = - any
   local nonindentspace         = space^-3 * - spacechar
   local indent                 = space^-3 * tab
@@ -518,6 +532,10 @@ function M.reader.new(writer, options)
   local optionallyindentedline = indent^-1 /"" * C(linechar^1 * newline^-1)
   local sp                     = spacing^0
   local spnl                   = optionalspace * (newline * optionalspace)^-1
+  local spnlor = function(pattern)
+    local optional = optionalspaceor(pattern)
+    return                       optional * (newline * optional)^-1
+  end
   local line                   = linechar^0 * newline
                                + linechar^1 * eof
   local nonemptyline           = line - blankline
@@ -529,7 +547,7 @@ function M.reader.new(writer, options)
   local function indented_blocks(bl)
     return Cs( bl
              * (blankline^1 * indent * -blankline * bl)^0
-             * blankline^1 )
+             * (blankline^1 + eof) )
   end
   local bulletchar = C(plus + asterisk + dash)
 
@@ -565,7 +583,36 @@ function M.reader.new(writer, options)
                     + (space - closeticks)
                     + (backtick^1 - closeticks)
 
-  local inticks     = openticks * space^-1 * C(intickschar^1) * closeticks
+  local inticks     = openticks * space^-1 * C(intickschar^0) * closeticks
+  local function captures_geq_length(s,i,a,b)
+    return #a >= #b and i
+  end
+
+  local infostring     = (linechar - (backtick + space^1 * (newline + eof)))^0
+
+  local fenceindent
+  local function fencehead(char)
+    return               C(nonindentspace) / function(s) fenceindent = #s end
+                       * Cg(char^3, "fencelength")
+                       * optionalspace * C(infostring) * optionalspace
+                       * (newline + eof)
+  end
+
+  local function fencetail(char)
+    return               nonindentspace
+                       * Cmt(C(char^3) * Cb("fencelength"),
+                             captures_geq_length)
+                       * optionalspace * (newline + eof)
+                       + eof
+  end
+
+  local function fencedline(char)
+    return               C(line - fencetail(char))
+                       / function(s)
+                             return s:gsub("^" .. string.rep(" ?",
+                                 fenceindent), "")
+                         end
+  end
   local leader        = space^-3
 
   -- in balanced brackets, parentheses, quotes:
@@ -616,6 +663,38 @@ function M.reader.new(writer, options)
 
   local optionaltitle = spnl * title * spacechar^0
                       + Cc("")
+  local citation_name = (Cs(dash^-1)
+                      * at * Cs((anyescaped
+                           - (rbracket + spacing + punctuation + semicolon))^1))
+
+  local citation_body_prenote
+                      = Cs((alphanumeric^1
+                           + bracketed
+                           + inticks
+                           + (anyescaped
+                               - (rbracket + semicolon + blankline^2))
+                           - (spnl * dash^-1 * at))^0)
+
+  local citation_body_postnote
+                      = Cs((alphanumeric^1
+                           + bracketed
+                           + inticks
+                           + (anyescaped
+                               - (rbracket + semicolon + blankline^2))
+                           - (spnl * rbracket))^0)
+
+  local citation_body_chunk
+                      = citation_body_prenote
+                      * spnl * citation_name
+                      * spnlor(punctuation - (semicolon + rbracket))
+                      * citation_body_postnote
+
+  local citation_body = citation_body_chunk
+                      * (semicolon * spnl * citation_body_chunk)^0
+
+  local citation_headless_body
+                      = citation_body_postnote
+                      * (sp * semicolon * spnl * citation_body_chunk)^0
   local rawnotes = {}
 
   local function strip_first_char(s)
@@ -627,7 +706,7 @@ function M.reader.new(writer, options)
     return function()
       local found = rawnotes[normalize_tag(ref)]
       if found then
-        return writer.note(parse_blocks(found))
+        return writer.note(parse_blocks_toplevel(found))
       else
         return {"[", parse_inlines("^" .. ref), "]"}
       end
@@ -713,21 +792,10 @@ function M.reader.new(writer, options)
       end
     end
   end
-  local Inline    = V("Inline")
-
-  local Str       = normalchar^1 / writer.string
-
-  local Ellipsis  = P("...") / writer.ellipsis
-
-  local Smart     = Ellipsis
-
-  local Symbol    = (specialchar - tightblocksep) / writer.string
-
-  local Code      = inticks / writer.code
-
   local bqstart      = more
   local headerstart  = hash
                      + (line * (equal^1 + dash^1) * optionalspace * newline)
+  local fencestart   = fencehead(backtick) + fencehead(tilde)
 
   if options.blankBeforeBlockquote then
     bqstart = fail
@@ -737,24 +805,41 @@ function M.reader.new(writer, options)
     headerstart = fail
   end
 
+  if not options.fencedCode or options.blankBeforeCodeFence then
+    fencestart = fail
+  end
+  local Inline    = V("Inline")
+
+  local Str       = normalchar^1 / writer.string
+
+  local Symbol    = (specialchar - tightblocksep) / writer.string
+  local Ellipsis  = P("...") / writer.ellipsis
+
+  local Smart     = Ellipsis
+  local Code      = inticks / writer.code
   local Endline   = newline * -( -- newline, but not before...
                         blankline -- paragraph break
                       + tightblocksep  -- nested list
                       + eof       -- end of document
                       + bqstart
                       + headerstart
+                      + fencestart
                     ) * spacechar^0 / writer.space
 
   local Space     = spacechar^2 * Endline / writer.linebreak
                   + spacechar^1 * Endline^-1 * eof / ""
                   + spacechar^1 * Endline^-1 * optionalspace / writer.space
 
+  local NonbreakingSpace
+                  = spacechar^2 * Endline / writer.linebreak
+                  + spacechar^1 * Endline^-1 * eof / ""
+                  + spacechar^1 * Endline^-1 * optionalspace / writer.nbsp
+
   -- parse many p between starter and ender
   local function between(p, starter, ender)
       local ender2 = B(nonspacechar) * ender
       return (starter * #nonspacechar * Ct(p * (p - ender2)^0) * ender2)
   end
-
   local Strong = ( between(Inline, doubleasterisks, doubleasterisks)
                  + between(Inline, doubleunderscores, doubleunderscores)
                  ) / writer.strong
@@ -762,7 +847,6 @@ function M.reader.new(writer, options)
   local Emph   = ( between(Inline, asterisk, asterisk)
                  + between(Inline, underscore, underscore)
                  ) / writer.emphasis
-
   local urlchar = anyescaped - newline - more
 
   local AutoLinkUrl   = less
@@ -792,7 +876,6 @@ function M.reader.new(writer, options)
 
   -- parse a link or image (direct or indirect)
   local Link          = DirectLink + IndirectLink
-
   local DirectImage   = exclamation
                       * (tag / parse_inlines)
                       * spnl
@@ -806,43 +889,89 @@ function M.reader.new(writer, options)
                          indirect_image
 
   local Image         = DirectImage + IndirectImage
-
   -- avoid parsing long strings of * or _ as emph/strong
   local UlOrStarLine  = asterisk^4 + underscore^4 / writer.string
 
   local EscapedChar   = S("\\") * C(escapable) / writer.string
+  local function citations(text_cites, raw_cites)
+      local function normalize(str)
+          if str == "" then
+              str = nil
+          else
+              str = (options.citationNbsps and parse_inlines_nbsp or
+                parse_inlines)(str)
+          end
+          return str
+      end
+
+      local cites = {}
+      for i = 1,#raw_cites,4 do
+          cites[#cites+1] = {
+              prenote = normalize(raw_cites[i]),
+              suppress_author = raw_cites[i+1] == "-",
+              name = writer.string(raw_cites[i+2]),
+              postnote = normalize(raw_cites[i+3]),
+          }
+      end
+      return writer.citations(text_cites, cites)
+  end
+
+  local TextCitations = Ct(Cc("")
+                      * citation_name
+                      * ((spnl
+                           * lbracket
+                           * citation_headless_body
+                           * rbracket) + Cc(""))) /
+                        function(raw_cites)
+                            return citations(true, raw_cites)
+                        end
+
+  local ParenthesizedCitations
+                      = Ct(lbracket
+                      * citation_body
+                      * rbracket) /
+                        function(raw_cites)
+                            return citations(false, raw_cites)
+                        end
+
+  local Citations     = TextCitations + ParenthesizedCitations
   local Block          = V("Block")
 
   local Verbatim       = Cs( (blanklines
                            * ((indentedline - blankline))^1)^1
                            ) / expandtabs / writer.verbatim
 
+  local TildeFencedCode
+                       = fencehead(tilde)
+                       * Cs(fencedline(tilde)^0)
+                       * fencetail(tilde)
+
+  local BacktickFencedCode
+                       = fencehead(backtick)
+                       * Cs(fencedline(backtick)^0)
+                       * fencetail(backtick)
+
+  local FencedCode     = (TildeFencedCode + BacktickFencedCode)
+                       / function(infostring, code)
+                             return writer.fencedCode(
+                                 writer.string(infostring),
+                                 expandtabs(code))
+                         end
   -- strip off leading > and indents, and run through blocks
   local Blockquote     = Cs((
             ((leader * more * space^-1)/"" * linechar^0 * newline)^1
           * (-blankline * linechar^1 * newline)^0
-          * blankline^0
-          )^1) / parse_blocks / writer.blockquote
+          * (blankline^0 / "")
+          )^1) / parse_blocks_toplevel / writer.blockquote
 
   local function lineof(c)
-      return (leader * (P(c) * optionalspace)^3 * newline * blankline^1)
+      return (leader * (P(c) * optionalspace)^3 * (newline * blankline^1
+          + newline^-1 * eof))
   end
-
   local HorizontalRule = ( lineof(asterisk)
                          + lineof(dash)
                          + lineof(underscore)
                          ) / writer.hrule
-
-  local Reference      = define_reference_parser / register_link
-
-  local Paragraph      = nonindentspace * Ct(Inline^1) * newline
-                       * ( blankline^1
-                         + #hash
-                         + #(leader * more * space^-1)
-                         )
-                       / writer.paragraph
-
-  local Plain          = nonindentspace * Ct(Inline^1) / writer.plain
   local starter = bullet + enumerator
 
   -- we use \001 as a separator between a tight list item and a
@@ -913,7 +1042,7 @@ function M.reader.new(writer, options)
   local DefinitionListItemLoose = C(line) * skipblanklines
                            * Ct((defstart *
                                  indented_blocks(dlchunk) /
-                                 parse_blocks)^1)
+                                 parse_blocks_toplevel)^1)
                            * Cc(false)
                            / definition_list_item
 
@@ -928,10 +1057,29 @@ function M.reader.new(writer, options)
                              * (skipblanklines *
                                 -DefinitionListItemLoose * Cc(true))
                           ) / writer.definitionlist
+  local Reference      = define_reference_parser / register_link
   local Blank          = blankline / ""
                        + NoteBlock
                        + Reference
                        + (tightblocksep / "\n")
+  local Paragraph      = nonindentspace * Ct(Inline^1) * newline
+                       * ( blankline^1
+                         + #hash
+                         + #(leader * more * space^-1)
+                         )
+                       / writer.paragraph
+
+  local ToplevelParagraph
+                       = nonindentspace * Ct(Inline^1) * (newline
+                       * ( blankline^1
+                         + #hash
+                         + #(leader * more * space^-1)
+                         + eof
+                         )
+                       + eof )
+                       / writer.paragraph
+
+  local Plain          = nonindentspace * Ct(Inline^1) / writer.plain
   -- parse Atx heading start and return level
   local HeadingStart = #hash * C(hash^-6) * -hash / length
 
@@ -955,6 +1103,8 @@ function M.reader.new(writer, options)
                      * HeadingLevel
                      * optionalspace * newline
                      / writer.heading
+
+  local Heading = AtxHeading + SetextHeading
   syntax =
     { "Blocks",
 
@@ -970,6 +1120,7 @@ function M.reader.new(writer, options)
 
       Block                 = V("Blockquote")
                             + V("Verbatim")
+                            + V("FencedCode")
                             + V("HorizontalRule")
                             + V("BulletList")
                             + V("OrderedList")
@@ -980,10 +1131,11 @@ function M.reader.new(writer, options)
 
       Blockquote            = Blockquote,
       Verbatim              = Verbatim,
+      FencedCode            = FencedCode,
       HorizontalRule        = HorizontalRule,
       BulletList            = BulletList,
       OrderedList           = OrderedList,
-      Heading               = AtxHeading + SetextHeading,
+      Heading               = Heading,
       DefinitionList        = DefinitionList,
       DisplayHtml           = DisplayHtml,
       Paragraph             = Paragraph,
@@ -996,6 +1148,7 @@ function M.reader.new(writer, options)
                             + V("Strong")
                             + V("Emph")
                             + V("NoteRef")
+                            + V("Citations")
                             + V("Link")
                             + V("Image")
                             + V("Code")
@@ -1012,6 +1165,7 @@ function M.reader.new(writer, options)
       Strong                = Strong,
       Emph                  = Emph,
       NoteRef               = NoteRef,
+      Citations             = Citations,
       Link                  = Link,
       Image                 = Image,
       Code                  = Code,
@@ -1028,6 +1182,14 @@ function M.reader.new(writer, options)
     syntax.DefinitionList = fail
   end
 
+  if not options.fencedCode then
+    syntax.FencedCode = fail
+  end
+
+  if not options.citations then
+    syntax.Citations = fail
+  end
+
   if not options.footnotes then
     syntax.NoteRef = fail
   end
@@ -1036,6 +1198,10 @@ function M.reader.new(writer, options)
     syntax.Smart = fail
   end
 
+  local blocks_toplevel_t = util.table_copy(syntax)
+  blocks_toplevel_t.Paragraph = ToplevelParagraph
+  blocks_toplevel = Ct(blocks_toplevel_t)
+
   blocks = Ct(syntax)
 
   local inlines_t = util.table_copy(syntax)
@@ -1043,9 +1209,13 @@ function M.reader.new(writer, options)
   inlines_t.Inlines = Inline^0 * (spacing^0 * eof / "")
   inlines = Ct(inlines_t)
 
-  inlines_no_link_t = util.table_copy(inlines_t)
+  local inlines_no_link_t = util.table_copy(inlines_t)
   inlines_no_link_t.Link = fail
   inlines_no_link = Ct(inlines_no_link_t)
+
+  local inlines_nbsp_t = util.table_copy(inlines_t)
+  inlines_nbsp_t.Space = NonbreakingSpace
+  inlines_nbsp = Ct(inlines_nbsp_t)
   function self.convert(input)
     references = {}
     local opt_string = {}
@@ -1056,9 +1226,9 @@ function M.reader.new(writer, options)
       end
     end
     table.sort(opt_string)
-    local salt = table.concat(opt_string, ",")
+    local salt = table.concat(opt_string, ",") .. "," .. metadata.version
     local name = util.cache(options.cacheDir, input, salt, function(input)
-        return util.rope_to_string(parse_blocks(input)) .. writer.eof
+        return util.rope_to_string(parse_blocks_toplevel(input)) .. writer.eof
       end, ".md" .. writer.suffix)
     return writer.pack(name)
   end