summaryrefslogtreecommitdiff
path: root/support/digestif/digestif
diff options
context:
space:
mode:
Diffstat (limited to 'support/digestif/digestif')
-rw-r--r--support/digestif/digestif/Cache.lua129
-rw-r--r--support/digestif/digestif/Manuscript.lua1523
-rw-r--r--support/digestif/digestif/ManuscriptBibTeX.lua38
-rw-r--r--support/digestif/digestif/ManuscriptConTeXt.lua604
-rw-r--r--support/digestif/digestif/ManuscriptDoctex.lua19
-rw-r--r--support/digestif/digestif/ManuscriptLaTeX.lua360
-rw-r--r--support/digestif/digestif/ManuscriptLatexProg.lua12
-rw-r--r--support/digestif/digestif/ManuscriptPlainTeX.lua56
-rw-r--r--support/digestif/digestif/ManuscriptTexinfo.lua25
-rw-r--r--support/digestif/digestif/Parser.lua227
-rw-r--r--support/digestif/digestif/Schema.lua189
-rw-r--r--support/digestif/digestif/bibtex.lua271
-rw-r--r--support/digestif/digestif/config.lua156
-rw-r--r--support/digestif/digestif/data.lua384
-rw-r--r--support/digestif/digestif/langserver.lua567
-rw-r--r--support/digestif/digestif/util.lua789
16 files changed, 5349 insertions, 0 deletions
diff --git a/support/digestif/digestif/Cache.lua b/support/digestif/digestif/Cache.lua
new file mode 100644
index 0000000000..d7e3e16f85
--- /dev/null
+++ b/support/digestif/digestif/Cache.lua
@@ -0,0 +1,129 @@
+-- A class to store transient file contents and manage the creation of
+-- Manuscript objects.
+
+local Manuscript = require "digestif.Manuscript"
+local lpeg = require "lpeg"
+local util = require "digestif.util"
+
+local S, C = lpeg.S, lpeg.C
+local sequence, gobble_until = util.sequence, util.gobble_until
+local path_split, path_join = util.path_split, util.path_join
+local path_normalize = util.path_normalize
+local weak_values = {__mode = "v"}
+local io_open = io.open
+
+local Cache = util.class()
+
+function Cache:__init(tbl)
+ self.store = setmetatable({}, weak_values)
+ self.persist = {} -- a place to keep an extra reference to things
+ -- that shouldn't vanish
+ if tbl then
+ for name, src in pairs(tbl) do
+ self:put(name, src)
+ end
+ end
+end
+
+-- Set the contents of a file in the cache.
+function Cache:put(filename, str)
+ local props = self.store[filename]
+ if not props then
+ props = {}
+ self.store[filename] = props
+ else
+ props.rootname = nil
+ end
+ props.contents = str
+ self.persist[filename] = props
+end
+
+-- Return the contents of a file, as stored with the put method, or
+-- attempt to read the file from disk. If neither succeeds, return
+-- nil. The second return value is a cookie. Values read from disk
+-- will be garbage collected when no reference to the cookie exists.
+-- Values stored by the put method are not subject to garbage
+-- collection until an explicit call to Cache.forget.
+--
+function Cache:__call(filename)
+ local props = self.store[filename]
+ if not props then
+ local file, str = io_open(filename), nil
+ if file then
+ str = file:read("*all")
+ file:close()
+ end
+ props = {contents = str}
+ self.store[filename] = props
+ end
+ return props.contents, props
+end
+
+-- Remove a file from the cache.
+function Cache:forget(filename)
+ self.store[filename] = nil
+ self.persist[filename] = nil
+end
+
+local space = S" \t\r"
+local magic_comment_patt = sequence(
+ (space^0 * "%")^1,
+ space^0 * "!" * util.case_fold("tex"),
+ space^1 * "root",
+ space^0 * "=",
+ space^0 * C(gobble_until(space^0 * "\n")))
+local search_magic_comment_patt = util.choice(
+ magic_comment_patt,
+ util.search("\n" * magic_comment_patt))
+
+-- Determine the root path of a document from magic comments.
+function Cache:rootname(filename)
+ local _, cookie = self(filename) -- warm up
+ local props = assert(self.store[filename])
+ local rootname = props.rootname
+ if rootname == nil then
+ local src = self(filename)
+ local val = search_magic_comment_patt:match(src:sub(1, 1000))
+ if val then
+ rootname = path_normalize(path_join(path_split(filename), val))
+ else
+ rootname = false
+ end
+ props.rootname = rootname
+ end
+ return rootname
+end
+
+-- Produce a Manuscript object. This method ensures that the result
+-- belongs to a tree of Manuscript objects representing a multi-file
+-- TeX document. If the rootname argument is omitted, try to infer it
+-- using magic comments.
+--
+-- This method reuses as much information as possible from the cache.
+--
+-- Arguments:
+-- filename (string): the path of the manuscript
+-- format (string): the TeX format
+-- rootname (string, optional): the root path of the TeX document
+--
+-- Returns:
+-- a Manuscript object
+--
+function Cache:manuscript(filename, format, rootname)
+ rootname = rootname or self:rootname(filename) or filename
+ local root = self.store[rootname] and self.store[rootname][format]
+ if not root or not root:is_current() then
+ root = Manuscript{
+ filename = rootname,
+ format = format,
+ files = self
+ }
+ self.store[rootname][format] = root
+ end
+ local script = root:find_manuscript(filename)
+ or self:manuscript(filename, format, filename) -- root does not refer back
+ self.store[filename][format] = script
+ return script
+end
+
+return Cache
diff --git a/support/digestif/digestif/Manuscript.lua b/support/digestif/digestif/Manuscript.lua
new file mode 100644
index 0000000000..d9101d8c07
--- /dev/null
+++ b/support/digestif/digestif/Manuscript.lua
@@ -0,0 +1,1523 @@
+-- Manuscript class
+
+local config = require "digestif.config"
+local util = require "digestif.util"
+
+local require_data = require "digestif.data".require
+local get_info = require "digestif.data".get_info
+local resolve_doc_items = require "digestif.data".resolve_doc_items
+local path_join, path_split = util.path_join, util.path_split
+local path_normalize = util.path_normalize
+local find_file = util.find_file
+local format_filename_template = util.format_filename_template
+
+local format = string.format
+local co_wrap, co_yield = coroutine.wrap, coroutine.yield
+local concat, sort = table.concat, table.sort
+local infty, min = math.huge, math.min
+local utf8_len, utf8_offset = utf8.len, utf8.offset
+local nested_get = util.nested_get
+local map_keys, update, extend = util.map_keys, util.update, util.extend
+local line_indices = util.line_indices
+local matcher, fuzzy_matcher = util.matcher, util.fuzzy_matcher
+
+local Manuscript = util.class()
+
+--* Constructor
+
+-- Only descendants of this class (representing various TeX formats)
+-- are ever instantiated. So we replace the constructor by a function
+-- defering to the subclass indicated by the format field of the
+-- argument.
+
+local formats = {
+ ["bibtex"] = "digestif.ManuscriptBibTeX",
+ ["context"] = "digestif.ManuscriptConTeXt",
+ ["latex"] = "digestif.ManuscriptLaTeX",
+ ["latex-prog"] = "digestif.ManuscriptLatexProg",
+ ["doctex"] = "digestif.ManuscriptDoctex",
+ ["plain"] = "digestif.ManuscriptPlainTeX",
+ ["texinfo"] = "digestif.ManuscriptTexinfo"
+}
+
+local function ManuscriptFactory(_, args)
+ local fmt = args.format
+ or args.parent and args.parent.format
+ or error "TeX format not specified"
+ return require(formats[fmt])(args)
+end
+getmetatable(Manuscript).__call = ManuscriptFactory
+
+local function infer_format(path)
+ local ext = path:sub(-4)
+ if ext == ".bib" then
+ return "bibtex"
+ elseif ext == ".sty" or ext == ".cls" then
+ return "latex-prog"
+ end
+end
+
+-- Create a new manuscript object. The argument is a table with the
+-- following keys:
+--
+-- filename: the manuscript's file name
+-- files: a function that returns file contents
+-- parent: a parent manuscript object (optional)
+-- format: the TeX format ("latex", "plain", etc.). This is
+-- actually only used by ManuscriptFactory
+--
+function Manuscript:__init(args)
+ local filename, parent, files = args.filename, args.parent, args.files
+ self.filename = filename
+ self.parent = parent
+ self.files = files
+ self.root = parent and parent.root or self
+ local src, cache_cookie = files(filename)
+ self.src, self.cache_cookie = src or "", cache_cookie
+ self.lines = line_indices(self.src)
+ local super = parent or self.__index
+ self.packages = setmetatable({}, {__index = super.packages})
+ self.commands = setmetatable({}, {__index = super.commands})
+ self.environments = setmetatable({}, {__index = super.environments})
+ -- We eagerly initialize most indexes here because the overhead of
+ -- going through the document is substantial. Only the indices used
+ -- by Manuscript:find_references are computed on demand.
+ self._children = {}
+ self.bib_index = {}
+ self.child_index = {}
+ self.section_index = {}
+ self.label_index = {}
+ if self.init_callbacks then
+ self:scan(self.init_callbacks)
+ end
+end
+
+function Manuscript:is_current()
+ local current_src, cookie = self.files(self.filename)
+ return (self.src == (current_src or "")) and (cookie or true)
+end
+
+-- Return a child manuscript, provided its name appear in the
+-- child_index. It inherits this manuscript's `files` function.
+-- Memoization is used to make this efficient.
+function Manuscript:child(name)
+ local child = self._children[name]
+ local is_current = child and child:is_current()
+ if not is_current then
+ if not child then
+ local ancestor = self
+ while ancestor do
+ if ancestor.filename == name then return end
+ ancestor = ancestor.parent
+ end
+ end
+ child = Manuscript{
+ filename = name,
+ parent = self,
+ format = infer_format(name),
+ files = self.files
+ }
+ self._children[name] = child
+ end
+ return child
+end
+
+--* Substrings
+
+-- Get a substring of the manuscript. The argument can be a pair of
+-- integers (inclusive indices, as in the Lua convention) or a table
+-- with fields pos (inclusive) and cont (exclusive).
+function Manuscript:substring(i, j)
+ if not i then return nil end
+ if type(i) == "table" then
+ j = i.cont - 1
+ i = i.pos
+ end
+ return self.src:sub(i, j)
+end
+
+-- Get a substring of the manuscript, trimmed. The argument follows
+-- the same convention as Manuscript:substring.
+function Manuscript:substring_trimmed(i, j)
+ return self.parser.trim(self:substring(i,j))
+end
+
+-- Get a substring of the manuscript, trimmed and without comments.
+-- The argument follows the same convention as Manuscript:substring.
+function Manuscript:substring_stripped(i, j)
+ local parser = self.parser
+ return parser.trim(parser.strip_comments(self:substring(i,j)))
+end
+
+-- Get a substring of the manuscript, without commments and reduced to
+-- one line. The argument follows the same convention as
+-- Manuscript:substring.
+function Manuscript:substring_clean(i, j)
+ local parser = self.parser
+ return parser.clean(parser.strip_comments(self:substring(i,j)))
+end
+
+--* Parsing commands, lists, and key-value lists.
+
+-- Parse a key-value list in a given manuscript range. The argument
+-- is a table with fields pos and cont. Returns a list of ranges,
+-- with additional fields "key" and "value" (if present). These are
+-- range tables as well.
+function Manuscript:parse_kvlist(range)
+ local s = self:substring(1, range.cont - 1) -- TODO: avoid creating a new string
+ return self.parser.parse_kvlist(s, range.pos)
+end
+
+-- Read the contents of a key-value list in the manuscript. The
+-- argument is a table with fields pos and cont. Returns a nested
+-- list of tables with fields "key" and "value" (if present),
+-- containing the corresponding text in the source.
+--
+-- TODO: add substring method as parameter
+function Manuscript:read_keys(range)
+ local tbl = self:parse_kvlist(range)
+ local r = {}
+ for i, v in ipairs(tbl) do
+ r[i] = {
+ key = self:substring_trimmed(v.key),
+ value = v.value and self:substring_trimmed(v.value)
+ }
+ end
+ return r
+end
+
+-- Read the contents of a list in the manuscript. Returns a list of
+-- strings.
+function Manuscript:read_list(i, j)
+ local parser = self.parser
+ local s = self:substring(i, j)
+ return parser.read_list(s)
+end
+
+-- Parse the arguments of a command.
+--
+-- Arguments:
+-- pos (number): A position in the source.
+-- cs (string, optional): The command name. If omitted, it is read
+-- from the manuscript.
+--
+-- Returns:
+-- A list of ranges.
+--
+function Manuscript:parse_command(pos, cs)
+ local parser = self.parser
+ if not cs then
+ cs = parser.csname:match(self.src, pos) or ""
+ end
+ local cmd = self.commands[cs]
+ local args = cmd and cmd.arguments
+ local cont = 1 + pos + #cs
+ if args then
+ local val = parser.parse_args(args, self.src, cont)
+ val.pos = pos
+ return val
+ else
+ return {pos = pos, cont = cont}
+ end
+end
+
+--* Find line numbers, paragraphs, etc.
+
+-- Find the line number (and its location) of a given position.
+-- Returns the line number (1-based) and that line's starting
+-- position.
+function Manuscript:line_number_at(pos)
+ local len = #self.src
+ local lines = self.lines
+ local j, l = 1, #lines -- top and bottom bounds for line search
+ if pos < 1 then pos = len + pos + 1 end
+ while pos < lines[l] do
+ local k = (j + l + 1) // 2
+ if pos < lines[k] then
+ l = k - 1
+ else
+ j = k
+ end
+ end -- now l = correct line, 1-based indexing
+ return l, lines[l]
+end
+
+-- Compute the line and column number (both 1-based) at the given
+-- position.
+--
+-- TODO: make len function a parameter
+function Manuscript:line_column_at(pos)
+ -- Out ranges are excluside on the right, Lua is inclusive, so we
+ -- may have pos == 1 + #self.src.
+ pos = min(pos, #self.src)
+ local l, line_pos = self:line_number_at(pos)
+ local c = utf8_len(self.src, line_pos, pos) or error("Invalid UTF-8")
+ return l, c
+end
+
+-- Compute the source position at the given line an column.
+--
+-- TODO: make offset function a parameter
+function Manuscript:position_at(line, col)
+ local line_pos = self.lines[line] or error("Position out of bounds")
+ return utf8_offset(self.src, col, line_pos) or error("Position out of bounds")
+end
+
+
+-- Find beginning of the paragraph containing the given position.
+function Manuscript:find_par(pos)
+ local src = self.src
+ local lines = self.lines
+ local l = self:line_number_at(pos)
+ for k = l, 1, -1 do
+ if self.parser.is_blank_line(src, lines[k]) then
+ return lines[k]
+ end
+ end
+ return 1
+end
+
+local preceding_command_callbacks = {}
+
+function preceding_command_callbacks.cs(self, pos, cs, end_pos)
+ if pos > end_pos then return nil end
+ local r = self:parse_command(pos, cs)
+ if r.cont <= end_pos then
+ local next_pos = self.parser.next_nonblank(self.src, r.cont)
+ if next_pos == end_pos then
+ return nil, pos, cs, r
+ end
+ end
+ return r.cont, end_pos
+end
+
+-- Find the preceding command, if any. If there is a command whose
+-- arguments end right before the given position, returns the
+-- position, command name, and parse data of the preceding command.
+function Manuscript:find_preceding_command(pos)
+ local par_pos = self:find_par(pos)
+ return self:scan(preceding_command_callbacks, par_pos, pos)
+end
+
+--* Indexes and document traversal
+
+function Manuscript:get_index(name)
+ name = name .. "_index"
+ local idx = self[name]
+ if not idx then
+ idx = {}
+ self[name] = idx
+ end
+ return idx
+end
+
+function Manuscript:index_pairs(name)
+ return ipairs(self:get_index(name))
+end
+
+local function compare_pos(x, y)
+ return x.pos < y.pos
+end
+
+local function traverse_indexes(script, indexes, max_depth)
+ local items = {}
+ for i = 1, #indexes do
+ local idx_name = indexes[i]
+ local idx = script[idx_name]
+ for j = 1, (idx and #idx or 0) do
+ items[#items+1] = idx[j]
+ items[idx[j]] = idx_name
+ end
+ end
+ if max_depth > 0 then
+ local child_index = script.child_index
+ for i = 1, #child_index do
+ local item = child_index[i]
+ local child = script:child(item.name)
+ if child then
+ items[#items+1] = {
+ pos = item.pos,
+ manuscript = child
+ }
+ end
+ end
+ end
+ sort(items, compare_pos)
+ for i = 1, #items do
+ local item = items[i]
+ local kind = items[item]
+ if kind then
+ co_yield(item, kind)
+ else
+ traverse_indexes(item.manuscript, indexes, max_depth - 1)
+ end
+ end
+end
+
+-- Iterator to transverse an index documentwise. This recursevely
+-- iterates over entries of the given indexes on self and its
+-- children, depth first. An index is a Manuscript field consisting a
+-- list of tables containing an entry "pos". At each iteration,
+-- yields one index entry and the name of the index to which it
+-- belongs.
+--
+-- Arguments:
+-- indexes: is the name of an index, as a string, or a list of
+-- such.
+-- max_depth: optional, leave at the default for the recursive
+-- behavior or set to 0 to disable it.
+--
+function Manuscript:traverse(indexes, max_depth)
+ if type(indexes) == "string" then indexes = {indexes} end
+ return co_wrap(function ()
+ return traverse_indexes(self, indexes, max_depth or 15)
+ end)
+end
+
+
+local function argument_items(script, sel, pos, cs)
+ local args = script.commands[cs].arguments
+ if not args then return end
+ local i
+ if type(sel) == "number" then
+ i = sel
+ elseif type(sel) == "string" then
+ for j = 1, #args do
+ if args[j].meta == sel then i = j; break end
+ end
+ else
+ i = sel(args)
+ end
+ if not i then return end
+ local ranges = script:parse_command(pos, cs)
+ local range = ranges[i]
+ if not range or range.omitted then return end
+ local arg = args[i]
+ if arg.list then
+ local items = script:parse_kvlist(range)
+ for j = 1, #items do
+ co_yield(items[j])
+ end
+ else
+ co_yield(range)
+ end
+end
+
+-- Iterator to look at arguments of a command. Yields the range of
+-- the relevant argument (if present), or succesive ranges
+-- corresponding to the argument's subitems, if the argument's `list`
+-- property is true.
+--
+-- Arguments:
+-- sel: determines which argument to look for; it it's a string, the
+-- first argument with that meta property is used; otherwise, sel
+-- should be a function that takes an `arguments` table and return
+-- the relevant index.
+-- pos: the position of the command to analyze
+-- cs: optional, the name of the command
+--
+function Manuscript:argument_items(sel, pos, cs)
+ return co_wrap(function() return argument_items(self, sel, pos, cs) end)
+end
+
+
+--* Manuscript scanning
+
+-- Scan the Manuscript, executing callbacks for each document element.
+--
+-- Each callback is a function taking at least two arguments (a
+-- Manuscript object and a source position) and returns at least one
+-- value, a position to continue scanning or nil to interrupt the
+-- process. When this happens, scan function returns the remaining
+-- return values of the callback. The remaining arguments and return
+-- values of a callback can be used to keep track of an internal
+-- state.
+--
+-- The callbacks argument is a table. Its keys correspond to either
+-- the "action" field of a command, or the "type" field of an item
+-- found by the parser ("cs", "mathshift" or "par").
+--
+function Manuscript:scan(callbacks, pos, ...)
+ local patt = self.parser.scan_patt(callbacks)
+ local match = patt.match
+ local commands = self.commands
+ local src = self.src
+ local function scan(pos0, ...)
+ if not pos0 then return ... end
+ local pos1, kind, detail, pos2 = match(patt, src, pos0)
+ local cmd = (kind == "cs") and commands[detail]
+ local cb = cmd and callbacks[cmd.action] or callbacks[kind]
+ if cb then
+ return scan(cb(self, pos1, detail, ...))
+ else
+ return scan(pos2, ...)
+ end
+ end
+ return scan(pos or 1, ...)
+end
+
+-- Copy entries from t to s, but only if not already present.
+local function copy_new(s, t)
+ for k, v in pairs(t) do
+ if not s[k] then s[k] = v end
+ end
+end
+
+-- Adds a package to the manuscript. This entails copying the command
+-- and environment definitions from the package tags to the
+-- manuscript. Returns true if package is (or already was) present,
+-- nil if the package tags weren't not found.
+function Manuscript:add_package(name)
+ if self.packages[name] then return true end
+ local pkg = require_data(name)
+ if not pkg then return end
+ self.packages[name] = pkg
+ local deps = pkg.dependencies or pkg.package and pkg.package.dependencies -- TODO: use only the former case
+ if deps then
+ for _, n in ipairs(deps) do
+ self:add_package(n)
+ end
+ end
+ -- Don't overwrite stuff from generated data files
+ local update_fn = pkg.generated and copy_new or update
+ if pkg.commands then
+ update_fn(self.commands, pkg.commands)
+ end
+ if pkg.environments then
+ update_fn(self.environments, pkg.environments)
+ end
+ return true
+end
+
+function Manuscript:find_manuscript(filename)
+ if self.filename == filename then return self end
+ local idx = self.child_index
+ for i = 1, #idx do
+ local script = self:child(idx[i].name)
+ local found = script and script:find_manuscript(filename)
+ if found then return found end
+ end
+ return nil
+end
+
+--* Getting the local context
+
+-- The context at a given manuscript postion is described by a linked
+-- list list of ranges, starting from the innermost, with additional
+-- annotations. More specifically, the following fields are possible
+-- in a context description table:
+--
+-- pos: the starting position
+-- cont: the ending postiion (exclusive)
+-- parent: a similar table, with range including the current range
+-- cs: if present, indicates that this range correspond to a command
+-- and its arguments. The value of this field is the command name.
+-- env: if present, indicates that this range correspond to an
+-- environment beginning and its arguments. The value of this field
+-- is the environment name.
+-- arg: if present, indicates that this range is a command argument.
+-- The value corresponds to the argument number. The parent is of
+-- "cs" or "env" type.
+-- key: if present, indicates this range is one item in a plain list
+-- or key-value list. The value of this field is the key text.
+-- value: if present, indicates this range is the value of a key in a
+-- key-value list. the parent is of "key" type.
+-- data: in each case above, data contains the corresponding data
+-- object, for instance the command description.
+--
+
+-- Scan the current paragraph, returning the context around the given
+-- position.
+function Manuscript:get_context(pos)
+ return self:scan(self.context_callbacks, self:find_par(pos), nil, pos)
+end
+
+local function local_scan_parse_keys(m, context, pos)
+ local items = m:parse_kvlist(context)
+ for _, it in ipairs(items) do -- are we inside a key/list item?
+ if it.pos and it.pos <= pos and pos <= it.cont then
+ local key = m:substring_trimmed(it.key)
+ context = {
+ key = key,
+ data = nested_get(context.data, "keys", key), -- or fetch context-dependent keys, say on a usepackage
+ pos = it.pos,
+ cont = it.cont,
+ parent = context
+ }
+ local v = it.value
+ if v and v.pos and v.pos <= pos and pos <= v.cont then -- are we inside the value of a key?
+ local value = m:substring_trimmed(v)
+ context = {
+ value = value,
+ data = nested_get(context.data, "values", value), -- what if "value" is command-like?
+ pos = v.pos,
+ cont = v.cont,
+ parent = context
+ }
+ end
+ break
+ end
+ end
+ return context
+end
+
+local function local_scan_parse_list(m, context, pos)
+ local items = m:parse_kvlist(context)
+ for i = 1, #items do -- are we inside a key/list item?
+ local it = items[i]
+ if it.pos and it.pos <= pos and pos <= it.cont then
+ context = {
+ item = i,
+ pos = it.pos,
+ cont = it.cont,
+ parent = context
+ }
+ end
+ end
+ return context
+end
+
+Manuscript.context_callbacks = {}
+
+function Manuscript.context_callbacks.cs(self, pos, cs, context, end_pos)
+ if pos > end_pos then return nil, context end -- stop parse
+ local r = self:parse_command(pos, cs)
+ if end_pos <= r.cont then
+ context = {
+ cs = cs,
+ data = self.commands[cs],
+ pos = pos,
+ cont = r.cont,
+ parent = context
+ }
+ elseif cs == "begin" then
+ local env_name = self:substring(r[1])
+ local env = self.environments[env_name]
+ local args = env and env.arguments
+ if not args then return r.cont, context, end_pos end
+ local q = self.parser.parse_args(args, self.src, r.cont)
+ if q.cont < end_pos then
+ return q.cont, context, end_pos -- end_pos is after current thing
+ end
+ context = {
+ env = env_name,
+ data = self.environments[env_name],
+ pos = pos,
+ cont = q.cont,
+ parent = context
+ }
+ r = q
+ else -- pos is after current thing
+ return r.cont, context, end_pos
+ end
+
+ for i, arg in ipairs(r) do -- are we inside an argument?
+ if arg.pos and arg.pos <= end_pos and end_pos <= arg.cont then
+ local data = nested_get(context.data, "arguments", i)
+ context = {
+ arg = i,
+ data = data,
+ pos = arg.pos,
+ cont = arg.cont,
+ parent = context
+ }
+ if data and data.keys then
+ context = local_scan_parse_keys(self, context, end_pos)
+ elseif data and data.list then
+ context = local_scan_parse_list(self, context, end_pos)
+ end
+ return context.pos, context, end_pos
+ end
+ end
+ return nil, context -- stop parse
+end
+
+function Manuscript.context_callbacks.tikzpath(m, pos, cs, context, end_pos)
+ if pos > end_pos then return nil, context end -- stop parse
+ local r = m:parse_command(pos, cs)
+ if end_pos <= r.cont then
+ context = {
+ cs = cs,
+ data = m.commands[cs],
+ pos = pos,
+ cont = r.cont,
+ parent = context
+ }
+ local p = r[1].pos
+ while p <= end_pos do
+ local q = m.parser.skip_to_bracketed(m.src, p)
+ if q and q.pos <= end_pos and end_pos <= q.cont then
+ context = {
+ arg = true,
+ data = {keys = require_data"tikz".keys.tikz},
+ pos = q.pos,
+ cont = q.cont,
+ parent = context
+ }
+ context = local_scan_parse_keys(m, context, end_pos)
+ end
+ p = q and q.cont or infty
+ end
+ end
+ return r.cont, context, end_pos
+end
+
+function Manuscript.context_callbacks.par (_, _, _, context)
+ return nil, context
+end
+
+--* Snippets and pretty-printing commands
+
+-- Pretty-print an argument list.
+--
+-- Arguments:
+-- args: A list of argument descriptors
+-- before: A piece of text inserted at the beginning of the
+-- formatted argument list.
+-- Returns:
+-- The formatted argument list (as a string) and a list of numbers
+-- (of length twice that of args) indicating the positions of each
+-- argument within that string.
+--
+function Manuscript:signature_args(args, before)
+ if not args then return before or "", {} end
+ local t, p, pos = {before}, {}, 1 + (before and #before or 0)
+ for i = 1, #args do
+ local arg, l, r = args[i]
+ if arg.literal then
+ l, r = "", ""
+ elseif arg.delimiters == false then
+ l, r = "‹", "›"
+ elseif arg.delimiters then
+ l, r = arg.delimiters[1], arg.delimiters[2]
+ if r == "\n" then r = "⤶" end
+ if l == "" then l, r = "‹", "›" .. r end
+ else
+ l, r = "{", "}"
+ end
+ local text = arg.literal or arg.meta or "#" .. i
+ t[#t+1] = l; pos = pos + #l; p[#p+1] = pos
+ t[#t+1] = text; pos = pos + #text; p[#p+1] = pos
+ t[#t+1] = r; pos = pos + #r
+ end
+ return concat(t), p
+end
+
+-- Pretty-print a command signature.
+--
+-- Arguments:
+-- cs: The command name.
+-- args: An argument list, or nil.
+--
+function Manuscript:signature_cmd(cs, args)
+ return self:signature_args(args, "\\" .. cs)
+end
+
+-- This is for plain TeX. Other formats should override this
+-- definition.
+Manuscript.signature_env = Manuscript.signature_cmd
+
+-- Make a snippet fragment from an argument list.
+--
+-- Arguments:
+-- args: An argument list
+-- i (optional, default 1): The number of the first placeholder in
+-- the snippet.
+--
+-- Returns:
+-- The formatted snippet, as a string, without the $0 placeholder.
+--
+function Manuscript:snippet_args(args, i)
+ if not args then return "" end
+ i = i or 1
+ local t = {}
+ for _, arg in ipairs(args) do
+ if arg.optional then
+ t[#t+1] = "${" .. i .. ":"
+ i = i + 1
+ end
+ if arg.literal then
+ t[#t+1] = arg.literal
+ else
+ local delims, l, r = arg.delimiters
+ local meta = arg.meta
+ if delims then
+ l, r = delims[1], delims[2]
+ if l == "" then meta = "‹" .. meta .. "›" end
+ elseif delims == false then
+ l, r = "", ""
+ meta = "‹" .. meta .. "›"
+ else -- delims == nil
+ l, r = "{", "}"
+ end
+ t[#t+1] = l .. "${" .. i .. (meta and ":" .. meta or "") .. "}" .. r
+ i = i + 1
+ end
+ if arg.optional then t[#t+1] = "}" end
+ end
+ return concat(t)
+end
+
+-- Make a snippet for a command.
+function Manuscript:snippet_cmd(cs, args)
+ local argsnippet = args and self:snippet_args(args) or ""
+ return cs .. argsnippet .. "$0"
+end
+
+-- Make a snippet for an environment.
+--
+-- This is the plain TeX version. It's intended to be overwritten by
+-- other classes.
+--
+function Manuscript:snippet_env(cs, args)
+ local argsnippet = args and self:snippet_args(args) or ""
+ return cs .. argsnippet .. "\n\t$0\n\\end" .. cs
+end
+
+
+--* Completion
+
+-- Calculate completions for the manuscript at the given position.
+-- Returns a a table containing a list of completion items (at
+-- numerical indices) and some addition information in the following
+-- fields.
+--
+-- pos: position where the matched text starts
+-- prefix: the matched text
+-- kind: whether the completions are for a command, a key, etc.
+--
+function Manuscript:complete(pos)
+ local ctx = self:get_context(pos - 1)
+ if ctx == nil then
+ return
+ elseif ctx.cs and pos == ctx.cont then
+ return self.completion_handlers.cs(self, ctx)
+ elseif ctx.arg then
+ local action = nested_get(ctx, "parent", "data", "action")
+ local handler = self.completion_handlers[action]
+ return handler and handler(self, ctx, pos)
+ elseif ctx.item then
+ local action = nested_get(ctx, "parent", "parent", "data", "action")
+ local handler = self.completion_handlers[action]
+ return handler and handler(self, ctx, pos)
+ elseif ctx.key then --and pos == ctx.pos + #ctx.key then
+ return self.completion_handlers.key(self, ctx, pos)
+ elseif ctx.value and pos == ctx.pos + #ctx.value then
+ return self.completion_handlers.value(self, ctx, pos)
+ end
+end
+
+Manuscript.completion_handlers = {}
+
+function Manuscript.completion_handlers.cs(self, ctx)
+ local commands, environments = self.commands, self.environments
+ local extra_snippets = config.extra_snippets
+ local prefix = ctx.cs
+ local ret = {
+ pos = ctx.pos + 1,
+ prefix = prefix,
+ kind = "command"
+ }
+ for cs in pairs(map_keys(self.parser.cs_matcher(prefix), commands)) do
+ local cmd = commands[cs]
+ local args = cmd.arguments
+ local snippet = extra_snippets[cs] or cmd.snippet
+ ret[#ret+1] = {
+ text = cs,
+ summary = cmd.summary,
+ annotation = args and self:signature_args(args) or cmd.symbol,
+ snippet = snippet or args and self:snippet_cmd(cs, args)
+ }
+ end
+ for env in pairs(map_keys(matcher(prefix), environments)) do
+ local cmd = environments[env]
+ local args = cmd.arguments
+ local snippet = extra_snippets[env] or cmd.snippet
+ local annotation = args and self:signature_args(args)
+ ret[#ret+1] = {
+ text = env,
+ summary = cmd.summary,
+ annotation = (annotation and annotation .. " " or "") .. "(environment)",
+ snippet = snippet or self:snippet_env(env, args)
+ }
+ end
+ table.sort(ret, function(x,y) return x.text < y.text end)
+ return ret
+end
+
+function Manuscript.completion_handlers.key(self, ctx, pos)
+ local prefix = self:substring(ctx.pos, pos - 1)
+ local len = #prefix
+ local r = {
+ pos = ctx.pos,
+ prefix = prefix,
+ kind = "key"
+ }
+ local keys = ctx.parent and ctx.parent.data and ctx.parent.data.keys
+ for text, key in pairs(keys or {}) do
+ if prefix == text:sub(1, len) then
+ r[#r+1] = {
+ text = text,
+ summary = key.summary,
+ annotation = key.meta and ("=" .. key.meta)
+ }
+ end
+ end
+ table.sort(r, function(x,y) return x.text < y.text end)
+ return r
+end
+
+function Manuscript.completion_handlers.value(self, ctx, pos)
+ local prefix = self:substring(ctx.pos, pos - 1)
+ local len = #prefix
+ local r = {
+ pos = ctx.pos,
+ prefix = prefix,
+ kind = "value"
+ }
+ local values = ctx.parent and ctx.parent.data and ctx.parent.data.values
+ for text, value in pairs(values or {}) do
+ if prefix == text:sub(1, len) then
+ r[#r+1] = {
+ text = text,
+ summary = value.summary
+ }
+ end
+ end
+ return r
+end
+
+function Manuscript.completion_handlers.begin(self, ctx, pos)
+ local environments = self.environments
+ local prefix = self:substring(ctx.pos, pos - 1)
+ local has_prefix = matcher(prefix)
+ local r = {
+ pos = ctx.pos,
+ prefix = prefix,
+ kind = "environment"
+ }
+ for env in pairs(map_keys(has_prefix, environments)) do
+ local cmd = environments[env]
+ r[#r+1] = {
+ text = env,
+ summary = cmd.summary,
+ }
+ end
+ return r
+end
+
+Manuscript.completion_handlers["end"] = Manuscript.completion_handlers.begin
+
+-- Get a short piece of text around a label. If there is a recognized
+-- command ending right before the label, the context starts there.
+--
+-- TODO: For now, the context is 60 bytes, but it should be smart and
+-- choose a lenght close to 100 characters but ending at a line end.
+-- It should also be Unicode-safe.
+--
+function Manuscript:label_context_short(item)
+ local pos, cs, _ = self:find_preceding_command(item.outer_pos)
+ local cmd = self.commands[cs]
+ if not cmd then
+ pos = self.parser.next_nonblank(self.src, item.outer_pos)
+ end
+ return self:substring_clean(pos, pos + 60)
+end
+
+function Manuscript.completion_handlers.ref(self, ctx, pos)
+ local prefix = self:substring(ctx.pos, pos - 1)
+ local has_prefix = matcher(prefix)
+ local fuzzy_match = config.fuzzy_ref and fuzzy_matcher(prefix)
+ local scores = {}
+ local r = {
+ pos = ctx.pos,
+ prefix = prefix,
+ kind = "label"
+ }
+ for label in self.root:traverse "label_index" do
+ local short_ctx = label.manuscript:label_context_short(label)
+ local score = has_prefix(label.name) and infty
+ or fuzzy_match and fuzzy_match(short_ctx)
+ if score then
+ r[#r+1] = {
+ text = label.name,
+ annotation = short_ctx,
+ summary = label.manuscript:label_context_long(label),
+ fuzzy_score = score < infty and score or nil
+ }
+ scores[r[#r]] = score
+ end
+ end
+ -- sort exact matches by order in the document, fuzzy ones by score
+ sort(r, function(a, b) return scores[a] > scores[b] end)
+ return r
+end
+
+function Manuscript.completion_handlers.cite(self, ctx, pos)
+ if nested_get(ctx, "data", "optional") then return end
+ local prefix = self:substring(ctx.pos, pos - 1)
+ local r = {
+ pos = ctx.pos,
+ prefix = prefix,
+ kind = "bibitem"
+ }
+ local scores = {}
+ local has_prefix = matcher(prefix)
+ local fuzzy_match = config.fuzzy_cite and fuzzy_matcher(prefix)
+ for item in self.root:traverse "bib_index" do
+ local score = has_prefix(item.name) and infty
+ or fuzzy_match and item.text and fuzzy_match(item.text)
+ if score then
+ scores[item.name] = score
+ r[#r+1] = {
+ text = item.name,
+ annotation = item.text,
+ fuzzy_score = score < infty and score or nil
+ }
+ end
+ end
+ -- sort exact matches by label, fuzzy matches by score
+ local cmp = function(a, b)
+ local na, nb = a.text, b.text
+ local sa, sb = scores[na], scores[nb]
+ if sa == sb then
+ return (na < nb)
+ else
+ return (sa > sb)
+ end
+ end
+ sort(r, cmp)
+ return r
+end
+
+--* Context help
+
+-- Get information about the thing at the given position.
+function Manuscript:describe(pos)
+ local ctx = self:get_context(pos)
+ if not ctx then return nil end
+ local action = ctx.arg and nested_get(ctx, "parent", "data", "action")
+ or ctx.item and nested_get(ctx, "parent", "parent", "data", "action")
+ local handlers = self.help_handlers
+ if handlers[action] then
+ return handlers[action](self, ctx)
+ elseif ctx.cs then
+ return handlers.cs(self, ctx)
+ elseif ctx.arg then
+ return handlers.arg(self, ctx)
+ -- elseif ctx.list then
+ elseif ctx.key then
+ return handlers.key(self, ctx)
+ else
+ return nil
+ end
+end
+
+Manuscript.help_handlers = {}
+
+function Manuscript.help_handlers.cite(self, ctx)
+ local name = self:substring(ctx)
+ for item in self.root:traverse "bib_index" do
+ if name == item.name then
+ local script, details = item.manuscript
+ if script.format == "bibtex" then
+ details = format(
+ [[
+`%s`: %s
+
+# Bibtex definition
+
+```bibtex
+%s
+```
+]],
+ item.name,
+ item.text,
+ script:substring(item)
+ )
+ end
+ return {
+ pos = ctx.pos,
+ cont = ctx.cont,
+ kind = "bibitem",
+ summary = item.name .. " " .. item.text,
+ details = details
+ }
+ end
+ end
+end
+
+function Manuscript:label_context_long(item)
+ local pos = self:find_preceding_command(item.outer_pos)
+ if not pos then pos = item.outer_pos end
+ local l = self:line_number_at(pos)
+ local lines = self.lines
+ local end_pos = lines[l + 10]
+ if end_pos then
+ return self:substring_trimmed(pos, end_pos - 1)
+ else
+ return self:substring_trimmed(pos, -1)
+ end
+end
+
+function Manuscript.help_handlers.ref(self, ctx)
+ local name = self:substring(ctx)
+ for item in self.root:traverse "label_index" do
+ if name == item.name then
+ local script = item.manuscript
+ local short_context = script:label_context_short(item)
+ local long_context = script:label_context_long(item)
+ local details = format(
+ [[
+`%s`: Refers to “%s...”
+
+# Label context
+
+```%s
+%s
+[...]
+```
+]],
+ item.name,
+ short_context,
+ script.format,
+ long_context
+ )
+ return {
+ pos = ctx.pos,
+ cont = ctx.cont,
+ kind = "label",
+ label = name,
+ summary = short_context,
+ details = details
+ }
+ end
+ end
+ return {
+ pos = ctx.pos,
+ cont = ctx.cont,
+ kind = "label",
+ label = name,
+ summary = "Unknown label"
+ }
+end
+
+function Manuscript.help_handlers.begin(self, ctx)
+ local env_name = self:substring(ctx)
+ local data = self.environments[env_name]
+ if not data then return nil end
+ local args = data.arguments
+ local sig_text, sig_pos = self:signature_env(env_name, args)
+ return {
+ pos = ctx.pos,
+ cont = ctx.cont,
+ kind = "environment",
+ label = sig_text,
+ label_positions = sig_pos,
+ summary = data.summary,
+ details = self:make_docstring("env", env_name, data),
+ data = data
+ }
+end
+
+Manuscript.help_handlers['end'] = function(self, ctx)
+ local env_name = self:substring(ctx)
+ local data = self.environments[env_name]
+ if not data then return nil end
+ local args = data.arguments
+ local sig_text, sig_pos = self:signature_env(env_name, args)
+ return {
+ pos = ctx.pos,
+ cont = ctx.cont,
+ kind = "environment",
+ label = sig_text,
+ label_positions = sig_pos,
+ summary = data.summary,
+ details = self:make_docstring("env", env_name, data),
+ data = data
+ }
+end
+
+function Manuscript.help_handlers.cs(self, ctx)
+ local data = ctx.data
+ if not data then return nil end
+ local args = data.arguments
+ local sig_text, sig_pos = self:signature_cmd(ctx.cs, args)
+ return {
+ pos = ctx.pos,
+ cont = ctx.cont,
+ kind = "command",
+ label = sig_text,
+ label_positions = sig_pos,
+ summary = data.summary,
+ details = self:make_docstring("cs", ctx.cs, data),
+ data = data
+ }
+end
+
+function Manuscript.help_handlers.arg(self, ctx)
+ if ctx.parent.cs then
+ return update(
+ self.help_handlers.cs(self, ctx.parent),
+ {pos = ctx.pos, cont = ctx.cont, arg = ctx.arg})
+ end
+end
+
+function Manuscript.help_handlers.key(self, ctx)
+ local key = ctx.key
+ local data = nested_get(ctx.parent, "data", "keys", key)
+ if not data then return nil end
+ return {
+ pos = ctx.pos,
+ cont = ctx.cont,
+ kind = "key",
+ label = key,
+ summary = data.summary,
+ details = self:make_docstring("key", key, data),
+ data = data
+ }
+end
+
+function Manuscript:make_docstring_header(kind, name, data)
+ local ret = name
+ if kind == "cs" then
+ ret = self:signature_cmd(name, data.arguments)
+ elseif kind == "env" then
+ ret = self:signature_env(name, data.arguments)
+ elseif kind == "key" and data.meta then
+ ret = ret .. " = " .. data.meta
+ end
+ if data.summary then
+ ret = "`" .. ret .. "`: " .. data.summary
+ else
+ ret = "`" .. ret .. "`"
+ end
+ if data.symbol then
+ ret = ret .. " (" .. data.symbol .. ")"
+ end
+ return ret
+end
+
+local function make_docstring_args(data)
+ local t = {"# Arguments\n"}
+ local args = data.arguments
+ if args then
+ for i = 1, #args do
+ local arg = args[i]
+ if arg.summary then
+ t[#t+1] = "- " .. (arg.meta or "#" .. i)
+ .. (arg.optional and " (optional): " or ": ")
+ .. arg.summary
+ end
+ end
+ t[#t+1] = ""
+ end
+ if #t > 2 then return concat(t, "\n") end
+end
+
+function Manuscript:make_docstring_variants(kind, name, data)
+ local variants = data.variants
+ if not variants then return end
+ local t = {"# Alternative forms\n"}
+ local fun = (kind == "env") and self.signature_env or self.signature_cmd
+ for i = 1, #variants do
+ t[#t+1] = "- `" .. fun(self, name, variants[i].arguments) .. "`"
+ end
+ t[#t+1] = ""
+ return concat(t, "\n")
+end
+
+local function make_docstring_details(data)
+ local details = data.details
+ local doc_field = data.documentation
+ if details then
+ return "# Details\n\n" .. details
+ elseif type(doc_field) == "string" and doc_field:match"^info:" then
+ local str, node, subnode = get_info(doc_field)
+ if str then
+ return format("# Info: (%s)%s\n\n```Info\n%s```", node, subnode, str)
+ end
+ else
+ return
+ end
+end
+
+function Manuscript:make_docstring_docs(kind, name, data)
+ local item_doc = data.documentation
+ local pkg = data.package
+ local pkg_doc = pkg and pkg.documentation
+ local t = {"# Documentation\n"}
+ if pkg and pkg.ctan_package then
+ if kind == "env" then
+ name = "{" .. name .. "}"
+ elseif kind == "cs" then
+ name = self:signature_cmd(name)
+ end
+ t[#t+1] = format(
+ "`%s` is defined in the [%s](https://www.ctan.org/pkg/%s) package.\n",
+ name, pkg.ctan_package, pkg.ctan_package
+ )
+ end
+ if item_doc then
+ extend(t, resolve_doc_items(item_doc))
+ end
+ if pkg_doc then
+ extend(t, resolve_doc_items(pkg_doc))
+ end
+ if #t > 1 then return concat(t, "\n") else return end
+end
+
+function Manuscript:make_docstring(kind, name, data)
+ local t = {self:make_docstring_header(kind, name, data), ""}
+ t[#t+1] = make_docstring_args(data)
+ t[#t+1] = self:make_docstring_variants(kind, name, data)
+ t[#t+1] = make_docstring_details(data)
+ t[#t+1] = self:make_docstring_docs(kind, name, data)
+ return concat(t, "\n")
+end
+
+--* Find definition
+
+-- Find the location where the thing at the given position is defined.
+function Manuscript:find_definition(pos)
+ local ctx = self:get_context(pos)
+ if not ctx then return nil end
+ local action = ctx.arg and nested_get(ctx, "parent", "data", "action")
+ or ctx.item and nested_get(ctx, "parent", "parent", "data", "action")
+ local handlers = self.find_definition_handlers
+ if handlers[action] then
+ return handlers[action](self, ctx)
+ elseif ctx.cs then
+ return handlers.cs(self, ctx)
+ else
+ return nil
+ end
+end
+
+Manuscript.find_definition_handlers = {}
+
+function Manuscript.find_definition_handlers.ref(self, ctx)
+ local name = self:substring(ctx)
+ for item in self.root:traverse "label_index" do
+ if name == item.name then
+ return {
+ pos = item.pos,
+ cont = item.cont,
+ manuscript = item.manuscript,
+ kind = "label"
+ }
+ end
+ end
+end
+
+function Manuscript.find_definition_handlers.cite(self, ctx)
+ local name = self:substring(ctx)
+ for item in self.root:traverse "bib_index" do
+ if name == item.name then
+ return {
+ pos = item.pos,
+ cont = item.cont,
+ manuscript = item.manuscript,
+ kind = "bibitem"
+ }
+ end
+ end
+end
+
+function Manuscript.find_definition_handlers.cs(self, ctx)
+ local name = ctx.cs
+ for item in self.root:traverse "newcommand_index" do
+ if name == item.name then
+ return {
+ pos = item.pos,
+ cont = item.cont,
+ manuscript = item.manuscript,
+ kind = "cs"
+ }
+ end
+ end
+end
+
+function Manuscript.find_definition_handlers.begin(self, ctx)
+ local name = self:substring(ctx)
+ for item in self.root:traverse "newenvironment_index" do
+ if name == item.name then
+ return {
+ pos = item.pos,
+ cont = item.cont,
+ manuscript = item.manuscript,
+ kind = "env"
+ }
+ end
+ end
+end
+
+Manuscript.find_definition_handlers["end"]
+ = Manuscript.find_definition_handlers.begin
+
+function Manuscript.find_definition_handlers.input(self, ctx)
+ local template
+ if ctx.arg then
+ template = nested_get(ctx, "parent", "data", "filename") or "?"
+ elseif ctx.item then
+ template = nested_get(ctx, "parent", "parent", "data", "filename") or "?"
+ else
+ return
+ end
+ local basename = format_filename_template(template, self:substring(ctx))
+ local filename = find_file(path_split(self.filename), basename)
+ if not filename then return end
+ local child = self:find_manuscript(path_normalize(filename))
+ if child then
+ return {
+ pos = 1,
+ cont = 1,
+ manuscript = child,
+ kind = "manuscript"
+ }
+ end
+end
+
+--* Find references
+
+function Manuscript:scan_references()
+ if not self.ref_index then
+ self.ref_index = {}
+ self.cite_index = {}
+ self:scan(self.scan_references_callbacks)
+ end
+ local idx = self.child_index
+ for i = 1, #idx do
+ local script = self:child(idx[i].name)
+ if script then script:scan_references() end
+ end
+end
+
+Manuscript.scan_references_callbacks = {}
+
+function Manuscript:scan_control_sequences()
+ if not self.cs_index then
+ self.cs_index = {}
+ self:scan(self.scan_cs_callbacks)
+ end
+ local idx = self.child_index
+ for i = 1, #idx do
+ local script = self:child(idx[i].name)
+ if script then script:scan_control_sequences() end
+ end
+end
+
+Manuscript.scan_cs_callbacks = {}
+
+function Manuscript.scan_cs_callbacks.cs(self, pos, cs)
+ local idx = self.cs_index
+ local cont = pos + 1 + #cs
+ idx[#idx + 1] = {
+ name = cs,
+ pos = pos,
+ cont = cont,
+ manuscript = self,
+ }
+ return cont
+end
+
+-- List all references to the thing at the given position.
+--
+-- Returns:
+-- A list of annotated ranges.
+--
+function Manuscript:find_references(pos)
+ local ctx = self:get_context(pos)
+ if not ctx then return nil end
+ local action = ctx.arg and nested_get(ctx, "parent", "data", "action")
+ or ctx.item and nested_get(ctx, "parent", "parent", "data", "action")
+ local handlers = self.find_references_handlers
+ if handlers[action] then
+ self.root:scan_references()
+ return handlers[action](self, ctx)
+ elseif ctx.cs then
+ self.root:scan_control_sequences()
+ return handlers.cs(self, ctx)
+ else
+ return nil
+ end
+end
+
+Manuscript.find_references_handlers = {}
+
+function Manuscript.find_references_handlers.cs(self, ctx)
+ local name = ctx.cs
+ local r = {}
+ for item in self.root:traverse "cs_index" do
+ if name == item.name then
+ r[#r + 1] = {
+ pos = item.pos,
+ cont = item.cont,
+ manuscript = item.manuscript,
+ kind = "cs"
+ }
+ end
+ end
+ return r
+end
+
+function Manuscript.find_references_handlers.ref(self, ctx)
+ local name = self:substring(ctx)
+ local r = {}
+ for item in self.root:traverse "ref_index" do
+ if name == item.name then
+ r[#r + 1] = {
+ pos = item.pos,
+ cont = item.cont,
+ manuscript = item.manuscript,
+ kind = "label"
+ }
+ end
+ end
+ return r
+end
+
+Manuscript.find_references_handlers.label =
+ Manuscript.find_references_handlers.ref
+
+function Manuscript.find_references_handlers.cite(self, ctx)
+ local name = self:substring(ctx)
+ local r = {}
+ for item in self.root:traverse "cite_index" do
+ if name == item.name then
+ r[#r + 1] = {
+ pos = item.pos,
+ cont = item.cont,
+ manuscript = item.manuscript,
+ kind = "bibitem"
+ }
+ end
+ end
+ return r
+end
+
+--* Outline
+
+-- Compute a table of contents for the document. If loc is false or
+-- omitted, include children of the manuscript; otherwise, restrict to
+-- the current manuscript.
+--
+function Manuscript:outline(loc)
+ local val = {}
+ for it in self:traverse("section_index", loc and 0) do
+ local lv = it.level or infty
+ local t = val
+ while t[#t] and (t[#t].level or -infty) < lv do t = t[#t] end
+ t[#t + 1] = {
+ name = it.name,
+ pos = it.pos,
+ cont = it.cont,
+ level = lv,
+ manuscript = it.manuscript,
+ kind = "section"
+ }
+ end
+ return val
+end
+
+return Manuscript
diff --git a/support/digestif/digestif/ManuscriptBibTeX.lua b/support/digestif/digestif/ManuscriptBibTeX.lua
new file mode 100644
index 0000000000..219f4552b4
--- /dev/null
+++ b/support/digestif/digestif/ManuscriptBibTeX.lua
@@ -0,0 +1,38 @@
+local lpeg = require "lpeg"
+local util = require "digestif.util"
+local Manuscript = require "digestif.Manuscript"
+local Parser = require "digestif.Parser"
+local bibtex = require "digestif.bibtex"
+
+local path_join, path_split = util.path_join, util.path_split
+local nested_get, nested_put = util.nested_get, util.nested_put
+local map, update, merge = util.map, util.update, util.merge
+
+local ManuscriptBibtex = util.class(Manuscript)
+
+ManuscriptBibtex.parser = Parser()
+ManuscriptBibtex.format = "bibtex"
+ManuscriptBibtex.packages = {}
+ManuscriptBibtex.commands = {}
+ManuscriptBibtex.environments = {}
+ManuscriptBibtex.init_callbacks = false -- Skip the normal init scan
+ManuscriptBibtex:add_package("plain") -- For basic command completion
+
+function ManuscriptBibtex:__init(args)
+ Manuscript.__init(self, args)
+ local bibitems = bibtex.parse(self.src)
+ local idx = {}
+ self.bib_index = idx
+ for i, item in ipairs(bibitems) do
+ idx[i] = {
+ name = item.id,
+ pos = item.pos,
+ cont = item.cont,
+ manuscript = self,
+ text = item:pretty_print(),
+ bibitem = item,
+ }
+ end
+end
+
+return ManuscriptBibtex
diff --git a/support/digestif/digestif/ManuscriptConTeXt.lua b/support/digestif/digestif/ManuscriptConTeXt.lua
new file mode 100644
index 0000000000..2243d3a9fc
--- /dev/null
+++ b/support/digestif/digestif/ManuscriptConTeXt.lua
@@ -0,0 +1,604 @@
+local util = require "digestif.util"
+local Manuscript = require "digestif.Manuscript"
+local Parser = require "digestif.Parser"
+
+local co_wrap, co_yield = coroutine.wrap, coroutine.yield
+local merge = util.merge
+local path_join, path_split = util.path_join, util.path_split
+local path_normalize = util.path_normalize
+local format_filename_template = util.format_filename_template
+local table_move, table_insert = table.move, table.insert
+
+--* Parsing ConTeXt interface files
+
+--** XML parser by Roberto Ierusalimschy
+--
+-- Cf. http://lua-users.org/wiki/LuaXml
+
+local function parseargs(s)
+ local arg = {}
+ string.gsub(s, "([%w:_-]+)=([\"'])(.-)%2", function (w, _, a)
+ arg[w] = a
+ end)
+ return arg
+end
+
+local function collect(s)
+ local stack = {}
+ local top = {}
+ table.insert(stack, top)
+ local ni,c,label,xarg, empty
+ local i, j = 1, 1
+ while true do
+ ni,j,c,label,xarg, empty = string.find(s, "<(%/?)([%w:]+)(.-)(%/?)>", i)
+ if not ni then break end
+ local text = string.sub(s, i, ni-1)
+ if not string.find(text, "^%s*$") then
+ table.insert(top, text)
+ end
+ if empty == "/" then -- empty element tag
+ table.insert(top, {label=label, xarg=parseargs(xarg), empty=1})
+ elseif c == "" then -- start tag
+ top = {label=label, xarg=parseargs(xarg)}
+ table.insert(stack, top) -- new level
+ else -- end tag
+ local toclose = table.remove(stack) -- remove top
+ top = stack[#stack]
+ if #stack < 1 then
+ error("nothing to close with "..label)
+ end
+ if toclose.label ~= label then
+ error("trying to close "..toclose.label.." with "..label)
+ end
+ table.insert(top, toclose)
+ end
+ i = j+1
+ end
+ local text = string.sub(s, i)
+ if not string.find(text, "^%s*$") then
+ table.insert(stack[#stack], text)
+ end
+ if #stack > 1 then
+ error("unclosed "..stack[#stack].label)
+ end
+ return stack[1]
+end
+
+local function child_with_label(node, label)
+ for _, n in ipairs(node) do
+ if n.label == label then return n end
+ end
+end
+
+local function children_with_label(node, label)
+ local function iter(n, l)
+ if n.label == l then
+ co_yield(n)
+ else
+ for _, m in ipairs(n) do
+ iter(m, l)
+ end
+ end
+ end
+ return co_wrap(function() return iter(node, label) end)
+end
+
+--** Parse ConTeXt interface (XML) files
+
+local braces = {"{", "}"}
+local brackets = {"[", "]"}
+local parenthesis = {"(", ")"}
+
+local function gen_tags(data)
+
+ local inheritances = {}
+ local inherit_type = {}
+ local inherit_order = {}
+
+ local function compute_meta(node)
+ local fromtag = node.label:gsub('^cd:', '')
+ local tbl = {}
+ local other = 0
+ for _, child in ipairs(node) do
+ local attribs = child.xarg
+ local val = attribs.type
+ if val and val:match"^cd:" then
+ tbl[#tbl+1]=val:sub(4)
+ else
+ other = other + 1
+ end
+ end
+ if other == 0 and #tbl == 1 then
+ return tbl[1]
+ else
+ return fromtag
+ end
+ end
+
+ local function compute_values(node)
+ local tbl = {}
+ for _, child in ipairs(node) do
+ if child.label == "cd:constant" then
+ local val = child.xarg.type
+ if not val:match"^cd:" then tbl[#tbl+1] = val end
+ elseif child.label == "cd:inherit" then
+ local val = child.xarg.name
+ if not inheritances[tbl] then inheritances[tbl] = {} end
+ local inh = inheritances[tbl]
+ inherit_type[tbl] = "values"
+ inherit_order[#inherit_order+1] = tbl
+ inh[#inh+1] = val
+ end
+ end
+ return #tbl > 0 and tbl or nil
+ end
+
+ local function compute_keys(node)
+ local tbl = {}
+ for _, child in ipairs(node) do
+ if child.label == "cd:parameter" then
+ local val = child.xarg.name
+ tbl[val] = {
+ values = compute_values(child),
+ meta = compute_meta(child)
+ }
+ elseif child.label == "cd:inherit" then
+ local val = child.xarg.name
+ if not inheritances[tbl] then inheritances[tbl] = {} end
+ local inh = inheritances[tbl]
+ inherit_type[tbl] = "keys"
+ inherit_order[#inherit_order+1] = tbl
+ inh[#inh+1] = val
+ end
+ end
+ return tbl
+ end
+
+ local function compute_argument(node)
+ local ret = {delimiters=brackets}
+ local attrs = node.xarg
+ if attrs.optional == 'yes' then ret.optional = true end
+ if attrs.list == 'yes' then ret.list = true end
+ if node.label == "cd:keywords" then
+ ret.meta=compute_meta(node)
+ ret.values = compute_values(node)
+ elseif node.label == "cd:assignments" then
+ ret.meta="assignments"
+ ret.keys = compute_keys(node)
+ elseif node.label == "cd:constant" then
+ ret.meta = node.xarg.type:gsub('^cd:', '')
+ elseif node.label == "cd:content" then
+ ret.meta="content"
+ ret.delimiters = braces
+ elseif node.label == "cd:csname" then
+ ret.meta="command"
+ ret.type="cs"
+ ret.delimiters = nil
+ elseif node.label == "cd:dimension" then
+ ret.meta = "dimension"
+ ret.type="dimen"
+ end
+ if attrs.delimiters == 'braces' then ret.delimiters = braces end
+ if attrs.delimiters == 'parenthesis' then ret.delimiters = parenthesis end
+ if attrs.delimiters == 'none' then ret.delimiters = false end
+ return ret
+ end
+
+ local function compute_arguments(node)
+ local tbl = {}
+ for _, child in ipairs(node) do
+ tbl[#tbl+1] = compute_argument(child)
+ end
+ return tbl
+ end
+
+ local function compute_instances(name, node)
+ local tbl = {}
+ if node == nil then return tbl end
+ for _, instance in ipairs(node) do
+ local inst_name = instance.xarg.value
+ if name ~= inst_name then
+ tbl[#tbl+1] = inst_name
+ end
+ end
+ return tbl
+ end
+
+ local command_list = {}
+ for node in children_with_label(data, "cd:command") do
+ local attribs = node.xarg
+ local arguments = child_with_label(node, "cd:arguments")
+ arguments = arguments and compute_arguments(arguments)
+ local cmd = {
+ cs = attribs.name,
+ environment = (attribs.type == "environment"),
+ --source = attribs.file,
+ --category = attribs.category,
+ arguments = arguments,
+ }
+ command_list[#command_list+1] = cmd
+
+ local instances = compute_instances(
+ attribs.name, child_with_label(node, "cd:instances"))
+ for _, name in ipairs(instances) do
+ command_list[#command_list+1] = merge(cmd, {cs = name})
+ end
+ end
+
+ -- FIXME: need to distinguish when doing
+ local cmds_and_envs = {}
+ for _, cmd in ipairs(command_list) do
+ local cs = cmd.cs
+ if cmd.link or not cmds_and_envs[cs] then
+ cmds_and_envs[cs] = cmd
+ end
+ end
+
+ for _, tbl in ipairs(inherit_order) do
+ local inh = inheritances[tbl]
+ --print("\n\nbefore", ser(tbl))
+ for _, cs in ipairs(inh) do
+ for _, arg in ipairs((cmds_and_envs[cs] or {}).arguments or {}) do
+ util.update(tbl, arg[inherit_type[tbl]] or {})
+ end
+ end
+ --print("\nafter", ser(tbl))
+ end
+
+ local tags = {
+ generated = true,
+ commands = {},
+ environments = {}
+
+ }
+
+ for _, cmd in ipairs(command_list) do
+ local cs = cmd.cs
+ local list = cmd.environment and tags.environments or tags.commands
+ local wikiname = cmd.environment and "start" .. cs or cs
+ cmd.documentation = {
+ [1] = {
+ summary = "“" .. wikiname .. "” on ConTeXt Garden",
+ uri = "https://wiki.contextgarden.net/Command/" .. wikiname
+ }
+ }
+ cmd.environment, cmd.cs = nil, nil
+ if not list[cs] then
+ list[cs] = cmd
+ else
+ local variants = list[cs].variants
+ if not variants then
+ variants = {}
+ list[cs].variants = variants
+ end
+ variants[#variants+1] = cmd
+ end
+ end
+
+ return tags
+end
+
+-- Generate tags from a ConTeXt interface (XML) file.
+--
+-- Arguments:
+-- file: The file name.
+-- pkg: An optional package description table.
+--
+-- Returns:
+-- The tags, or nil if file does not exist.
+--
+local function tags_from_xml(file, pkg)
+ local ok, str = util.find_file(file, nil, true)
+ if not ok then return end
+ local ok, data = pcall(collect, str)
+ if not ok then return end
+ return setmetatable(gen_tags(data), {__index = pkg})
+end
+
+-- Monkey-patch to isolate XML dependency in this file.
+(require "digestif.data").tags_from_xml = tags_from_xml
+
+--** Core ConTeXt tags
+
+local ctx_tags = (require "digestif.data".require_tags)("context-en.xml")
+
+if ctx_tags then
+
+ ctx_tags.ctan_package = "context"
+
+ ctx_tags.documentation = {
+ [1] = {
+ summary = "ConTeXt Mark IV: an excursion",
+ uri = "texmf:doc/context/documents/general/manuals/ma-cb-en.pdf"
+ },
+ [2] = {
+ summary = "ConTeXt documentation library",
+ uri = "https://wiki.contextgarden.net/Documentation"
+ }
+ }
+
+ local commands = ctx_tags.commands
+ local environments = ctx_tags.environments
+
+ -- Add details to sectioning commands
+ for _, tbl in ipairs{commands, environments} do
+ for i, cs in ipairs{
+ "part", "chapter", "section", "subsection", "subsubsection",
+ "subsubsubsection", "subsubsubsubsection"
+ } do
+ tbl[cs].section_level = i
+ tbl[cs].action = "section"
+ end
+ for i, cs in ipairs{
+ "title", "subject", "subsubject", "subsubsubject",
+ "subsubsubsubject", "subsubsubsubsubject"
+ } do
+ tbl[cs].section_level = i + 1
+ tbl[cs].action = "section"
+ end
+ end
+
+ -- Add details to file-inputting commands.
+ for cs, filename in pairs {
+ environment = "?.tex",
+ project = "?.tex",
+ component = "?.tex",
+ usemodule = "t-?.xml"
+ } do
+ if commands[cs] then
+ commands[cs].action = "input"
+ commands[cs].filename = filename
+ end
+ end
+
+ -- Add details to assorted commands.
+ for cs, action in pairs {
+ usebtxdataset = "input",
+ cite = "cite",
+ nocite = "cite",
+ define = "define",
+ definestartstop = "definestartstop",
+ defineitemgroup = "definestartstop",
+ definelist = "definestartstop",
+ citation = "cite",
+ nocitation = "cite",
+ pagereference = "label",
+ textreference = "label",
+ ["in"] = "ref",
+ at = "ref",
+ about = "ref"
+ } do
+ if commands[cs] then
+ commands[cs].action = action
+ end
+ end
+
+ -- Add start/stop commands
+ for env, cmd in pairs(environments) do
+ commands["start" .. env] = cmd
+ commands["stop" .. env] = {}
+ end
+
+end
+
+--* ManuscriptContext class
+
+local ManuscriptContext = util.class(Manuscript)
+
+ManuscriptContext.parser = Parser()
+ManuscriptContext.format = "context"
+ManuscriptContext.packages = {}
+ManuscriptContext.commands = {}
+ManuscriptContext.environments = {}
+ManuscriptContext.init_callbacks = {}
+ManuscriptContext.scan_references_callbacks = {}
+ManuscriptContext:add_package("context-en.xml")
+
+-- Make a snippet for an environment.
+function ManuscriptContext:snippet_env(cs, args)
+ local argsnippet = args and self:snippet_args(args) or ""
+ return "start" .. cs .. argsnippet .. "\n\t$0\n\\stop" .. cs
+end
+
+-- ConTeXt optional arguments can't always be distinguished by their
+-- delimiters, for instance \citation[optional][mandatory]. Here we
+-- patch Parser.parse_args to deal with this case, but just in the
+-- simplest case where the optional arguments are in the beginning of
+-- the argument list.
+local original_parse_args = ManuscriptContext.parser.parse_args
+
+local function new_parse_args(arglist, str, pos)
+ local val = original_parse_args(arglist, str, pos)
+ if val.incomplete and arglist[1].optional then
+ arglist = table_move(arglist, 2, #arglist, 1, {})
+ val = new_parse_args(arglist, str, pos)
+ table_insert(val, 1, {omitted = true})
+ end
+ return val
+end
+
+ManuscriptContext.parser.parse_args = new_parse_args
+
+--* Init scanning
+
+local to_args = {}
+for i = 1, 9 do
+ local t = merge(to_args[i-1] or {})
+ t[i] = {meta = "#" .. i}
+ to_args[i] = t
+end
+
+function ManuscriptContext.init_callbacks.define(self, pos, cs)
+ -- ugly! this function (and others below) parses the command twice
+ local cont = self:parse_command(pos, cs).cont
+ local csname, n_args
+ for r in self:argument_items("command", pos, cs) do
+ csname = self:substring_stripped(r):sub(2)
+ end
+ for r in self:argument_items("number", pos, cs) do
+ n_args = tonumber(self:substring_stripped(r))
+ end
+ if csname then
+ local idx = self:get_index "newcommand"
+ idx[#idx+1] = {
+ name = csname,
+ pos = pos,
+ cont = cont,
+ manuscript = self,
+ arguments = to_args[n_args]
+ }
+ if not self.commands[csname] then
+ self.commands[csname] = idx[#idx]
+ end
+ end
+ return cont
+end
+
+function ManuscriptContext.init_callbacks.definestartstop(self, pos, cs)
+ local cont = self:parse_command(pos, cs).cont
+ local name
+ for r in self:argument_items("name", pos, cs) do
+ name = self:substring_stripped(r)
+ end
+ if name then
+ local idx = self:get_index "newenvironment"
+ idx[#idx+1] = {
+ name = name,
+ pos = pos,
+ cont = cont,
+ manuscript = self,
+ }
+ if not self.environments[name] then
+ self.environments[name] = idx[#idx]
+ end
+ end
+ return cont
+end
+
+function ManuscriptContext.init_callbacks.label(self, pos, cs)
+ local cont = self:parse_command(pos, cs).cont
+ local idx = self.label_index
+ for r in self:argument_items("reference", pos, cs) do
+ idx[#idx + 1] = {
+ name = self:substring_stripped(r),
+ pos = r.pos,
+ cont = r.cont,
+ outer_pos = pos,
+ outer_cont = cont,
+ manuscript = self
+ }
+ end
+ return cont
+end
+
+function ManuscriptContext.init_callbacks.section(self, pos, cs)
+ local cont = self:parse_command(pos, cs).cont
+ for r in self:argument_items("assignments", pos, cs) do
+ local assigns = self:parse_kvlist(r)
+ for i = 1, #assigns do
+ local key = self:substring_stripped(assigns[i].key)
+ local val = assigns[i].value
+ if key == "reference" then
+ local idx = self.label_index
+ idx[#idx + 1] = {
+ name = self:substring_stripped(val),
+ pos = val.pos,
+ cont = val.cont,
+ outer_pos = pos,
+ outer_cont = cont,
+ manuscript = self
+ }
+ elseif key == "title" then
+ local idx = self.section_index
+ idx[#idx + 1] = {
+ name = self:substring_stripped(val),
+ level = self.commands[cs].section_level,
+ pos = val.pos,
+ cont = val.cont,
+ outer_pos = pos,
+ outer_cont = cont,
+ manuscript = self
+ }
+ end
+ end
+ end
+ for r in self:argument_items("reference", pos, cs) do
+ local idx = self.label_index
+ idx[#idx + 1] = {
+ name = self:substring_stripped(r),
+ pos = r.pos,
+ cont = r.cont,
+ outer_pos = pos,
+ outer_cont = cont,
+ manuscript = self
+ }
+ end
+ for r in self:argument_items("text", pos, cs) do
+ local idx = self.section_index
+ idx[#idx + 1] = {
+ name = self:substring_stripped(r),
+ level = self.commands[cs].section_level,
+ pos = r.pos,
+ cont = r.cont,
+ outer_pos = pos,
+ outer_cont = cont,
+ manuscript = self
+ }
+ end
+ return cont
+end
+
+function ManuscriptContext.init_callbacks.input(self, pos, cs)
+ local cont = self:parse_command(pos, cs).cont
+ local idx = self.child_index
+ local template = self.commands[cs].filename or "?"
+ for r in self:argument_items("file", pos, cs) do
+ local f = format_filename_template(template, self:substring_stripped(r))
+ local ok = self:add_package(f)
+ if not ok then
+ idx[#idx + 1] = {
+ name = path_normalize(path_join(path_split(self.filename), f)),
+ pos = r.pos,
+ cont = r.cont,
+ manuscript = self
+ }
+ end
+ end
+ return cont
+end
+
+--* Reference scanning
+
+function ManuscriptContext.scan_references_callbacks.ref(self, pos, cs)
+ local cont = self:parse_command(pos, cs).cont
+ local idx = self.ref_index
+ for s in self:argument_items("reference", pos, cs) do
+ idx[#idx + 1] = {
+ name = self:substring_stripped(s),
+ pos = s.pos,
+ cont = s.cont,
+ outer_pos = pos,
+ outer_cont = cont,
+ manuscript = self
+ }
+ end
+ return cont
+end
+
+function ManuscriptContext.scan_references_callbacks.cite(self, pos, cs)
+ local cont = self:parse_command(pos, cs).cont
+ local idx = self.cite_index
+ for s in self:argument_items("reference", pos, cs) do
+ idx[#idx + 1] = {
+ name = self:substring_stripped(s),
+ pos = s.pos,
+ cont = s.cont,
+ outer_pos = pos,
+ outer_cont = cont,
+ manuscript = self
+ }
+ end
+ return cont
+end
+
+return ManuscriptContext
diff --git a/support/digestif/digestif/ManuscriptDoctex.lua b/support/digestif/digestif/ManuscriptDoctex.lua
new file mode 100644
index 0000000000..c3f0c2b0ce
--- /dev/null
+++ b/support/digestif/digestif/ManuscriptDoctex.lua
@@ -0,0 +1,19 @@
+local ManuscriptLatex = require "digestif.ManuscriptLaTeX"
+local Parser = require "digestif.Parser"
+local lpeg = require "lpeg"
+local util = require "digestif.util"
+
+local ManuscriptDoctex = util.class(ManuscriptLatex)
+
+-- Consider both code (no prefix) and documentation (starting with a
+-- single %) as not being comments.
+local comment = (lpeg.B(1) - lpeg.B("\n")) * lpeg.P("%")
+
+ManuscriptDoctex.format = "doctex"
+ManuscriptDoctex.parser = Parser{
+ comment = comment,
+ letter = lpeg.R("az", "@Z")
+}
+ManuscriptLatex:add_package("latex.ltx")
+
+return ManuscriptDoctex
diff --git a/support/digestif/digestif/ManuscriptLaTeX.lua b/support/digestif/digestif/ManuscriptLaTeX.lua
new file mode 100644
index 0000000000..1366b0e146
--- /dev/null
+++ b/support/digestif/digestif/ManuscriptLaTeX.lua
@@ -0,0 +1,360 @@
+local lpeg = require "lpeg"
+local util = require "digestif.util"
+local Manuscript = require "digestif.Manuscript"
+local Parser = require "digestif.Parser"
+
+local path_join, path_split = util.path_join, util.path_split
+local path_normalize = util.path_normalize
+local nested_get, nested_put = util.nested_get, util.nested_put
+local map, update, merge = util.map, util.update, util.merge
+local format_filename_template = util.format_filename_template
+
+local ManuscriptLatex = util.class(Manuscript)
+
+ManuscriptLatex.parser = Parser()
+ManuscriptLatex.format = "latex"
+ManuscriptLatex.packages = {}
+ManuscriptLatex.commands = {}
+ManuscriptLatex.environments = {}
+ManuscriptLatex.init_callbacks = {}
+ManuscriptLatex.scan_references_callbacks = {}
+ManuscriptLatex:add_package("latex")
+
+--* Snippets
+
+-- Make a snippet for an environment.
+--
+-- Arguments:
+-- cs: The command name
+-- args: An argument list
+--
+-- Returns:
+-- A snippet string
+--
+function ManuscriptLatex:snippet_env(cs, args)
+ local argsnippet = args and self:snippet_args(args) or ""
+ return "begin{" .. cs .. "}" .. argsnippet .. "\n\t$0\n\\end{" .. cs .. "}"
+end
+
+-- Pretty-print an environment signature.
+function ManuscriptLatex:signature_env(cs, args)
+ return self:signature_args(args, "\\begin{" .. cs .. "}")
+end
+
+--* Helper functions
+
+local function first_mand(args)
+ for i, v in ipairs(args) do
+ if not v.optional then return i end
+ end
+end
+
+local function first_opt(args)
+ for i, v in ipairs(args) do
+ if v.optional then return i end
+ end
+end
+
+--* Global scanning
+
+--** Basic document elements
+
+function ManuscriptLatex.init_callbacks.input(self, pos, cs)
+ local cont = self:parse_command(pos, cs).cont
+ local idx = self.child_index
+ local template = self.commands[cs].filename or "?"
+ for r in self:argument_items(first_mand, pos, cs) do
+ local f = format_filename_template(template, self:substring_stripped(r))
+ local ok = self:add_package(f)
+ if not ok then
+ idx[#idx + 1] = {
+ name = path_normalize(path_join(path_split(self.filename), f)),
+ pos = r.pos,
+ cont = r.cont,
+ manuscript = self
+ }
+ end
+ end
+ return cont
+end
+
+function ManuscriptLatex.init_callbacks.label (m, pos, cs)
+ local idx = m.label_index
+ local args = m.commands[cs].arguments
+ local r = m:parse_command(pos, cs)
+ local i = first_mand(args)
+ if r[i] then
+ local l = m:substring_stripped(r[i])
+ idx[#idx + 1] = {
+ name = l,
+ pos = r[i].pos,
+ cont = r[i].cont,
+ outer_pos = r.pos,
+ outer_cont = r.cont,
+ manuscript = m
+ }
+ end
+ return r.cont
+end
+
+function ManuscriptLatex.init_callbacks.section(self, pos, cs)
+ local idx = self.section_index
+ for r in self:argument_items(first_mand, pos, cs) do
+ idx[#idx + 1] = {
+ name = self:substring_stripped(r),
+ level = self.commands[cs].section_level,
+ pos = r.pos,
+ cont = r.cont,
+ outer_pos = pos,
+ manuscript = self
+ }
+ end
+ return pos + #cs + 1
+end
+
+--** Bibliographic items
+
+function ManuscriptLatex.init_callbacks.bibitem (m, pos, cs)
+ local idx = m.bib_index
+ local args = m.commands[cs].arguments
+ local r = m:parse_command(pos, cs)
+ local i = first_mand(args)
+ if r[i] then
+ idx[#idx + 1] = {
+ name = m:substring_stripped(r[i]),
+ pos = r[i].pos,
+ cont = r[i].cont,
+ outer_pos = r.pos,
+ outer_cont = r.cont,
+ manuscript = m
+ }
+ end
+ return r.cont
+end
+
+function ManuscriptLatex.init_callbacks.amsrefs_bib(self, pos, cs)
+ local idx = self.bib_index
+ local r = self:parse_command(pos, cs)
+ if r.incomplete then return r.cont end
+ local keys = self:read_keys(r[3])
+ local authors, title, date = {}, "", "(n.d.)"
+ for i = 1, #keys do
+ local k, v = keys[i].key, keys[i].value
+ if k == "author" then
+ authors[#authors+1] = self.parser.clean(v:match("[^,]+", 2))
+ elseif k == "title" then
+ title = self.parser.clean(v:sub(2, -2))
+ elseif k == "date" then
+ date = self.parser.clean(v:match("(%d+)"))
+ end
+ end
+ idx[#idx + 1] = {
+ name = self:substring_stripped(r[1]),
+ text = string.format(
+ "%s %s; %s",
+ table.concat(authors, ", "),
+ date,
+ title),
+ pos = r[1].pos,
+ cont = r[1].cont,
+ outer_pos = r.pos,
+ outer_cont = r.cont,
+ manuscript = self
+ }
+ return r.cont
+end
+
+--** Command definitions, TeX style
+
+ManuscriptLatex.init_callbacks.def =
+ require "digestif.ManuscriptPlainTeX".init_callbacks.def
+
+--** Command and environment definitions, LaTeX style
+
+local to_args = {}
+for i = 1, 9 do
+ local t = merge(to_args[i-1] or {})
+ t[i] = {meta = "#" .. i}
+ to_args[i] = t
+end
+
+local function newcommand_args(number, default)
+ local args = to_args[number]
+ if default and args then
+ args = merge(args) -- make a copy
+ args[1] = {
+ meta = "#1",
+ optional = true,
+ delimiters = {"[", "]"},
+ details = "Default: “" .. default .. "”."
+ }
+ end
+ return args
+end
+
+function ManuscriptLatex.init_callbacks.newcommand(self, pos, cs)
+ local cont = self:parse_command(pos, cs).cont
+ local csname, nargs, optdefault
+ for r in self:argument_items(first_mand, pos, cs) do
+ csname = self:substring_stripped(r):sub(2)
+ end
+ for r in self:argument_items("number", pos, cs) do
+ nargs = tonumber(self:substring_stripped(r))
+ end
+ for r in self:argument_items("default", pos, cs) do
+ optdefault = self:substring_stripped(r)
+ end
+ if csname then
+ local idx = self:get_index "newcommand"
+ idx[#idx+1] = {
+ name = csname,
+ pos = pos,
+ cont = cont,
+ manuscript = self,
+ arguments = newcommand_args(nargs, optdefault)
+ }
+ if not self.commands[csname] then
+ self.commands[csname] = idx[#idx]
+ end
+ end
+ return cont
+end
+
+function ManuscriptLatex.init_callbacks.newenvironment(self, pos, cs)
+ local cont = self:parse_command(pos, cs).cont
+ local csname, nargs, optdefault
+ for r in self:argument_items(first_mand, pos, cs) do
+ csname = self:substring_stripped(r)
+ end
+ for r in self:argument_items("number", pos, cs) do
+ nargs = tonumber(self:substring_stripped(r))
+ end
+ for r in self:argument_items("default", pos, cs) do
+ optdefault = self:substring_stripped(r)
+ end
+ if csname then
+ local idx = self:get_index("newenvironment")
+ idx[#idx+1] = {
+ name = csname,
+ pos = pos,
+ cont = cont,
+ manuscript = self,
+ arguments = newcommand_args(nargs, optdefault)
+ }
+ if not self.environments[csname] then
+ self.environments[csname] = idx[#idx]
+ end
+ end
+ return cont
+end
+
+--** Command and environment definitions, xparse style
+
+local P, C, Cc, Cg, Ct = lpeg.P, lpeg.C, lpeg.Cc, lpeg.Cg, lpeg.Ct
+local Pgroup = util.between_balanced("{", "}")
+local Pdefault = Cg(Pgroup / "Default: “%1”", "details")
+
+local xparse_args = util.matcher(
+ Ct(util.many(Ct(util.sequence(
+ P" "^0,
+ (P"+" * Cg(Cc(true), "long"))^-1,
+ P"!"^-1,
+ util.choice(
+ "m",
+ "r" * Cg(Ct(C(1) * C(1)), "delimiters"),
+ "R" * Cg(Ct(C(1) * C(1)), "delimiters") * Pdefault,
+ "v" * Cg(Cc"verbatim", "type"),
+ "o" * Cg(Cc{"[", "]"}, "delimiters") * Cg(Cc(true), "optional"),
+ "O" * Cg(Cc{"[", "]"}, "delimiters") * Cg(Cc(true), "optional") * Pdefault,
+ "d" * Cg(Ct(C(1) * C(1)), "delimiters") * Cg(Cc(true), "optional"),
+ "D" * Cg(Ct(C(1) * C(1)), "delimiters") * Cg(Cc(true), "optional") * Pdefault,
+ "s" * Cg(Cc"*", "literal") * Cg(Cc"literal", "type") * Cg(Cc(true), "optional"),
+ "t" * Cg(C(1), "literal") * Cg(Cc"literal", "type") * Cg(Cc(true), "optional"),
+ "e" * Pgroup,
+ "E" * Pgroup * Pdefault))))))
+
+function ManuscriptLatex.init_callbacks.NewDocumentCommand(self, pos, cs)
+ local cont = self:parse_command(pos, cs).cont
+ local csname, arg_spec
+ for r in self:argument_items("command", pos, cs) do
+ csname = self:substring_stripped(r):sub(2)
+ end
+ for r in self:argument_items("arg spec", pos, cs) do
+ arg_spec = self:substring_stripped(r)
+ end
+ if csname then
+ local idx = self:get_index "newcommand"
+ idx[#idx+1] = {
+ name = csname,
+ pos = pos,
+ cont = cont,
+ manuscript = self,
+ arguments = xparse_args(arg_spec)
+ }
+ if not self.commands[csname] then
+ self.commands[csname] = idx[#idx]
+ end
+ end
+ return cont
+end
+
+function ManuscriptLatex.init_callbacks.NewDocumentEnvironment(self, pos, cs)
+ local cont = self:parse_command(pos, cs).cont
+ local csname, arg_spec
+ for r in self:argument_items("environment", pos, cs) do
+ csname = self:substring_stripped(r)
+ end
+ for r in self:argument_items("arg spec", pos, cs) do
+ arg_spec = self:substring_stripped(r)
+ end
+ if csname then
+ local idx = self:get_index "newenvironment"
+ idx[#idx+1] = {
+ name = csname,
+ pos = pos,
+ cont = cont,
+ manuscript = self,
+ arguments = xparse_args(arg_spec)
+ }
+ if not self.environments[csname] then
+ self.environments[csname] = idx[#idx]
+ end
+ end
+ return cont
+end
+
+--* Scan references callbacks
+
+function ManuscriptLatex.scan_references_callbacks.ref(self, pos, cs)
+ local cont = self:parse_command(pos, cs).cont
+ local idx = self.ref_index
+ for r in self:argument_items(first_mand, pos, cs) do
+ idx[#idx + 1] = {
+ name = self:substring_stripped(r),
+ pos = r.pos,
+ cont = r.cont,
+ outer_pos = pos,
+ outer_cont = cont,
+ manuscript = self
+ }
+ end
+ return cont
+end
+
+function ManuscriptLatex.scan_references_callbacks.cite(self, pos, cs)
+ local cont = self:parse_command(pos, cs).cont
+ local idx = self.cite_index
+ for r in self:argument_items(first_mand, pos, cs) do
+ idx[#idx + 1] = {
+ name = self:substring_stripped(r),
+ pos = r.pos,
+ cont = r.cont,
+ outer_pos = pos,
+ outer_cont = cont,
+ manuscript = self
+ }
+ end
+ return cont
+end
+
+return ManuscriptLatex
diff --git a/support/digestif/digestif/ManuscriptLatexProg.lua b/support/digestif/digestif/ManuscriptLatexProg.lua
new file mode 100644
index 0000000000..38746afcc8
--- /dev/null
+++ b/support/digestif/digestif/ManuscriptLatexProg.lua
@@ -0,0 +1,12 @@
+local ManuscriptLatex = require "digestif.ManuscriptLaTeX"
+local Parser = require "digestif.Parser"
+local lpeg = require "lpeg"
+local util = require "digestif.util"
+
+local ManuscriptLatexProg = util.class(ManuscriptLatex)
+
+ManuscriptLatexProg.format = "latex-prog"
+ManuscriptLatexProg.parser = Parser{letter = lpeg.R("az", "@Z")}
+ManuscriptLatex:add_package("latex.ltx")
+
+return ManuscriptLatexProg
diff --git a/support/digestif/digestif/ManuscriptPlainTeX.lua b/support/digestif/digestif/ManuscriptPlainTeX.lua
new file mode 100644
index 0000000000..1ac217985f
--- /dev/null
+++ b/support/digestif/digestif/ManuscriptPlainTeX.lua
@@ -0,0 +1,56 @@
+local lpeg = require "lpeg"
+local util = require "digestif.util"
+local Manuscript = require "digestif.Manuscript"
+local Parser = require "digestif.Parser"
+local ManuscriptPlain = util.class(Manuscript)
+
+local P, R = lpeg.P, lpeg.R
+local C, Ct, Cc, Cg = lpeg.C, lpeg.Ct, lpeg.Cc, lpeg.Cg
+
+-- In plain TeX, we can't distinguish between code and document files,
+-- so we pretend @ is always a letter.
+ManuscriptPlain.parser = Parser{letter = lpeg.R("az", "@Z")}
+ManuscriptPlain.format = "plain"
+ManuscriptPlain.packages = {}
+ManuscriptPlain.commands = {}
+ManuscriptPlain.environments = {}
+ManuscriptPlain.init_callbacks = {}
+ManuscriptPlain.scan_references_callbacks = {}
+ManuscriptPlain:add_package("plain")
+
+-- Convert a TeX parameter text to an arguments table.
+local param = P("#") * R("19")
+local before_params = Ct(Cg(C((1 - param)^1), "literal"))
+local one_param = Ct(
+ Cg(C(param), "meta") * Cg(Ct(Cc("") * C((1 - param)^1)), "delimiters")^-1
+)
+local parse_params = util.matcher(
+ Ct(before_params^-1 * one_param^0)
+)
+
+function ManuscriptPlain.init_callbacks.def(self, pos, cs)
+ local cont = self:parse_command(pos, cs).cont
+ local csname, params
+ for r in self:argument_items(1, pos, cs) do
+ csname = self:substring_stripped(r):sub(2)
+ end
+ for r in self:argument_items(2, pos, cs) do
+ params = self:substring_stripped(r)
+ end
+ if csname then
+ local idx = self:get_index "newcommand"
+ idx[#idx+1] = {
+ name = csname,
+ pos = pos,
+ cont = cont,
+ manuscript = self,
+ arguments = parse_params(params)
+ }
+ if not self.commands[csname] then
+ self.commands[csname] = idx[#idx]
+ end
+ end
+ return cont
+end
+
+return ManuscriptPlain
diff --git a/support/digestif/digestif/ManuscriptTexinfo.lua b/support/digestif/digestif/ManuscriptTexinfo.lua
new file mode 100644
index 0000000000..0e0c71667c
--- /dev/null
+++ b/support/digestif/digestif/ManuscriptTexinfo.lua
@@ -0,0 +1,25 @@
+local util = require "digestif.util"
+local Manuscript = require "digestif.Manuscript"
+local Parser = require "digestif.Parser"
+
+local ManuscriptTexinfo = util.class(Manuscript)
+
+ManuscriptTexinfo.parser = Parser({escape = "@"})
+ManuscriptTexinfo.format = "texinfo"
+ManuscriptTexinfo.packages = {}
+ManuscriptTexinfo.commands = {}
+ManuscriptTexinfo.environments = {}
+ManuscriptTexinfo.init_callbacks = {}
+ManuscriptTexinfo.scan_references_callbacks = {}
+ManuscriptTexinfo:add_package("texinfo")
+
+function ManuscriptTexinfo:snippet_env(cs, args)
+ local argsnippet = args and self:snippet_args(args) or ""
+ return cs .. argsnippet .. "\n\t$0\n@end " .. cs
+end
+
+function Manuscript:signature_cmd(cs, args)
+ return self:signature_args(args, "@" .. cs)
+end
+
+return ManuscriptTexinfo
diff --git a/support/digestif/digestif/Parser.lua b/support/digestif/digestif/Parser.lua
new file mode 100644
index 0000000000..e939a13c56
--- /dev/null
+++ b/support/digestif/digestif/Parser.lua
@@ -0,0 +1,227 @@
+-- Parser class
+
+local lpeg = require "lpeg"
+local util = require "digestif.util"
+
+local C = lpeg.C
+local Cc = lpeg.Cc
+local Cg = lpeg.Cg
+local Cs = lpeg.Cs
+local Ct = lpeg.Ct
+local P = lpeg.P
+local R = lpeg.R
+local S = lpeg.S
+local V = lpeg.V
+local match = lpeg.match
+
+local I = lpeg.Cp()
+local Ipos = Cg(I, "pos")
+local Icont = Cg(I, "cont")
+local Iouter_pos = Cg(I, "outer_pos")
+local Iouter_cont = Cg(I, "outer_cont")
+local Kcs = Cc("cs")
+local Kmathshift = Cc("mathshift")
+local Knil = Cc(nil)
+local Kpar = Cc("par")
+local Pend = P(-1)
+local is_incomplete = Cg(Cc(true), "incomplete")
+local is_omitted = Cg(Cc(true), "omitted")
+
+local choice = util.choice
+local gobble_until = util.gobble_until
+local many = util.many
+local matcher = util.matcher
+local search = util.search
+local sequence = util.sequence
+local trim = util.trim
+
+local default_catcodes = {
+ escape = P("\\"),
+ bgroup = P("{"),
+ egroup = P("}"),
+ mathshift = P("$"),
+ eol = P("\n"),
+ letter = R("az", "AZ"),
+ comment = P("%"),
+ char = P(1), -- are we utf-8 agnostic?
+ space = S(" \t"),
+ listsep = P(","),
+ valsep = P("=")
+}
+
+local Parser = util.class()
+
+function Parser:__init(catcodes)
+ catcodes = catcodes or default_catcodes
+
+--* Single characters
+ local escape = P(catcodes.escape or default_catcodes.escape)
+ local bgroup = P(catcodes.bgroup or default_catcodes.bgroup)
+ local egroup = P(catcodes.egroup or default_catcodes.egroup)
+ local mathshift = P(catcodes.mathshift or default_catcodes.mathshift)
+ local eol = P(catcodes.eol or default_catcodes.eol)
+ local letter = P(catcodes.letter or default_catcodes.letter)
+ local comment = P(catcodes.comment or default_catcodes.comment)
+ local char = P(catcodes.char or default_catcodes.char)
+ local space = P(catcodes.space or default_catcodes.space)
+ local listsep = P(catcodes.listsep or default_catcodes.listsep)
+ local valsep = P(catcodes.valsep or default_catcodes.valsep)
+
+--* Basic elements
+ local blank = space + eol
+ local cs = escape * (letter^1 + char)
+ local csname = escape * C(letter^1 + char)
+ local token = cs + char
+ local blank_line = eol * space^0 * eol
+ local single_eol = eol * space^0 * -eol
+ local comment_line = comment * gobble_until(eol, char)
+ local skip = (space + single_eol + comment_line)^0 -- simulates TeX state S
+ local skip_long = (space + eol + comment_line)^0
+ local par = blank_line * (space + eol + comment_line)^0
+ local next_par = search(blank_line, comment + char)
+ * (eol + space + comment_line)^0 * I * char -- need this char at the end?
+
+ -- These patterns match text delimited by braces. They succeed on a
+ -- incomplete subject string (with the additional incomplete = true
+ -- named capture). The second variant does not cross paragraph
+ -- boundaries.
+ local group_long = sequence(
+ Iouter_pos,
+ P{sequence(bgroup,
+ Ipos,
+ many(choice(comment_line,
+ V(1)/0,
+ token - egroup)),
+ Icont,
+ egroup + is_incomplete)},
+ Iouter_cont)
+ local group = sequence(
+ Iouter_pos,
+ P{sequence(bgroup,
+ Ipos,
+ many(choice(comment_line,
+ V(1)/0,
+ token - egroup)
+ - blank_line),
+ Icont,
+ egroup + is_incomplete)},
+ Iouter_cont)
+
+--* Trimming, cleaning, and cropping
+ local trimmer = blank^0 * C((blank^0 * (char - blank)^1)^0)
+ local cleaner = blank^0 * Cs(((blank^1 / " " + true) * (char - blank)^1)^0)
+ local comment_block = ((eol * space^0)^-1 * comment_line)^1 -- use the one below?
+ -- local comment_block = (comment_line * (Pend + eol * space^0))^0
+ local comment_stripper = Cs((comment_block / "" + char)^0)
+
+--* Parsing lists
+ local skim_long = comment_line + group_long/0 + token
+ local listsep_skip = listsep * skip_long
+
+ local list_item = Ct(Ipos * (skim_long - listsep)^1 * Icont)
+ local list_parser = skip_long * Ct((listsep_skip^0 * list_item)^0)
+
+ local list_reader = skip_long * Ct(
+ (listsep_skip^0 * C((skim_long - listsep)^1)
+ / trim(space + eol + comment_line, char))^0)
+
+ local key = Ct(Ipos * (skim_long - listsep - valsep)^1 * Icont)
+ local value = Ct(valsep * skip_long * Ipos * (skim_long - listsep)^0 * Icont)
+ local kvlist_item = Ct(Ipos * Cg(key, "key") * Cg(value, "value")^-1 * Icont)
+ local kvlist_parser = skip_long * Ct((listsep_skip^0 * kvlist_item)^0)
+
+--* Parsing command arguments
+ local patt_from_arg = function(arg)
+ local ret = skip
+ if arg.delimiters == false then
+ -- This is different from the “plain argument” case in that only
+ -- a single token is allowed.
+ ret = ret * Ipos * token * Icont
+ elseif arg.delimiters then
+ local l, r = arg.delimiters[1], arg.delimiters[2]
+ ret = sequence(
+ ret, Iouter_pos, l, Ipos,
+ many(
+ choice(comment_line,
+ group/0,
+ token)
+ - blank_line - r),
+ Icont,
+ r + is_incomplete,
+ Iouter_cont)
+ elseif arg.literal then
+ ret = ret * Ipos * P(arg.literal) * Icont
+ else -- plain argument
+ ret = ret * (group + Ipos * token * Icont)
+ end
+ if arg.optional then
+ ret = ret + is_omitted
+ end
+ return Ct(ret)
+ end
+
+ local patt_from_args = function(args)
+ local ret = Icont
+ for i = #args, 1, -1 do
+ ret = (patt_from_arg(args[i]) + is_incomplete) * ret
+ end
+ return Ct(Ipos * ret)
+ end
+ patt_from_args = util.memoize1(patt_from_args)
+
+ local parse_args = function(args, str, pos)
+ return match(patt_from_args(args), str, pos)
+ end
+
+--* Public patterns
+ self.group = group
+ self.group_long = group_long
+ self.next_par = next_par
+ self.comment_line = comment_line
+ self.cs = cs
+ self.csname = csname
+ self.blank_line = blank_line
+
+--* Public functions
+ self.parse_args = parse_args
+ self.is_blank_line = matcher(space^0 * eol)
+ self.next_nonblank = matcher(skip * I)
+ self.trim = matcher(trimmer) -- replace def of trimmer directly in here?
+ self.clean = matcher(cleaner)
+ self.strip_comments = matcher(comment_stripper)
+ self.skip_to_bracketed = matcher( -- for tikz paths
+ search(
+ patt_from_arg{delimiters = {"[", "]"}},
+ skim_long - blank_line)) -- also exclude ";"?
+
+ -- Parse a list, return a sequence of ranges
+ self.parse_list = matcher(list_parser)
+ -- Parse a list, return the item contents as strings
+ self.read_list = matcher(list_reader)
+ -- Parse a key-value list, return their contents as strings
+ self.parse_kvlist = matcher(kvlist_parser)
+
+ -- Match a normal control sequence name starting with prefix
+ function self.cs_matcher(prefix)
+ local patt = P(prefix) * letter^0 * Pend
+ return matcher(patt)
+ end
+
+ -- Build a pattern for Manuscript.scan, which looks ahead for one of
+ -- the elements of the set `things`. A match produces 4 captures: a
+ -- position before the item, the kind of item ("cs", "mathshift" or
+ -- "par"), a detail (e.g., the control sequence if kind is "cs"),
+ -- and a position after the item.
+ --
+ self.scan_patt = function(things)
+ local patt = Kcs * csname
+ if things.mathshift then
+ patt = patt + mathshift * Kmathshift * (mathshift * Cc"$$" + Cc"$")
+ end
+ if things.par then patt = patt + par * Kpar * Knil end
+ return search(I * patt * I, comment_line + char)
+ end
+
+end
+
+return Parser
diff --git a/support/digestif/digestif/Schema.lua b/support/digestif/digestif/Schema.lua
new file mode 100644
index 0000000000..b59a95157d
--- /dev/null
+++ b/support/digestif/digestif/Schema.lua
@@ -0,0 +1,189 @@
+--- A simple validation library, loosely inspired by JSON Schema
+
+local Schema = {}
+Schema.__index = Schema
+
+local function is_schema(tbl)
+ return getmetatable(tbl) == Schema
+end
+Schema.is_schema = is_schema
+
+local validators, initializers = {}, {}
+
+local function to_schema(tbl)
+ if is_schema(tbl) then return tbl end
+ for i = 1, #tbl do
+ tbl[i] = to_schema(tbl[i])
+ end
+ -- local fields, items = tbl.fields, tbl.items
+ -- if fields then
+ -- for k, v in pairs(fields) do
+ -- fields[k] = to_schema(v)
+ -- end
+ -- end
+ -- if items then
+ -- tbl.items = to_schema(items)
+ -- end
+ for k, fun in pairs(initializers) do
+ if tbl[k] then tbl[k] = fun(tbl[k]) end
+ end
+ return setmetatable(tbl, Schema)
+end
+setmetatable(Schema, {__call = function(_, tbl) return to_schema(tbl) end})
+
+local function msg_type(expect, got)
+ return ("Type: expected %q, got %q"):format(expect, got)
+end
+
+local function msg_where(msg, loc_type, loc_name)
+ return ("%s in %s %q"):format(msg, loc_type, loc_name)
+end
+
+initializers.items = to_schema
+initializers.keys = to_schema
+initializers.values = to_schema
+
+function initializers.enum(tbl)
+ local set = {}
+ for i = 1, #tbl do
+ set[tbl[i]] = true
+ end
+ return set
+end
+
+function initializers.fields(fields)
+ for k, v in pairs(fields) do
+ fields[k] = to_schema(v)
+ end
+ return fields
+end
+
+function validators.type(s, obj)
+ local ok = (type(obj) == s)
+ return ok, ok or msg_type(s, type(obj))
+end
+
+function validators.predicate(f, obj)
+ return f(obj)
+end
+
+function validators.keys(schema, obj)
+ if type(obj) ~= "table" then
+ return false, msg_type("table", type(obj))
+ end
+ for k in pairs(obj) do
+ local ok, msg = schema:validate(k)
+ if not ok then
+ return false, msg_where(msg, "key", k)
+ end
+ end
+ return true
+end
+
+function validators.values(schema, obj)
+ if type(obj) ~= "table" then
+ return false, msg_type("table", type(obj))
+ end
+ for k, v in pairs(obj) do
+ local ok, msg = schema:validate(v)
+ if not ok then
+ return false, msg_where(msg, "field", k)
+ end
+ end
+ return true
+end
+
+function validators.fields(fields, obj)
+ if type(obj) ~= "table" then
+ return false, msg_type("table", type(obj))
+ end
+ for k, schema in pairs(fields) do
+ local ok, msg = schema:validate(obj[k])
+ if not ok then
+ return false, msg_where(msg, "field", k)
+ end
+ end
+ return true
+end
+
+function validators.items(schema, obj)
+ if type(obj) ~= "table" then
+ return false, msg_type("table", type(obj))
+ end
+ for i, item in ipairs(obj) do
+ local ok, msg = schema:validate(item)
+ if not ok then
+ return false, msg_where(msg, "item", i)
+ end
+ end
+ return true
+end
+
+function validators.enum(set, obj)
+ local ok = set[obj]
+ return ok, ok or ('Enum: %q not allowed'):format(obj)
+end
+
+function Schema:validate(obj)
+ if self.optional and obj == nil then return true end
+ for k, v in pairs(self) do
+ local fun = validators[k]
+ if fun then
+ local ok, msg = fun(v, obj)
+ if not ok then return false, msg end
+ end
+ end
+ local len = #self
+ if len > 0 then
+ for i = 1, len do
+ if self[i]:validate(obj) then break end
+ if i == len then return false, "Alternatives: no match" end
+ end
+ end
+ return true
+end
+
+function Schema:assert(obj)
+ return assert(self:validate(obj))
+end
+
+function Schema:assert_fail(obj)
+ assert(not self:validate(obj))
+end
+
+local schema_of_schema = Schema {
+ fields = {
+ description = {
+ type = "string",
+ optional = true,
+ description = "A docstring for the schema"
+ },
+ fields = {
+ type = "table",
+ values = {predicate = is_schema}
+ },
+ enum = {
+ type = "table"
+ },
+ items = {
+ predicate = is_schema
+ },
+ optional = {
+ type = "boolean",
+ optional = true,
+ description = "Whether nil values are accepted"
+ },
+ predicate = {
+ type = "function",
+ optional = true,
+ description = "A function to test on the object"
+ },
+ type = {
+ type = "string",
+ optional = true,
+ description = "Check if the object is of a given type"
+ },
+ }
+}
+
+return Schema
diff --git a/support/digestif/digestif/bibtex.lua b/support/digestif/digestif/bibtex.lua
new file mode 100644
index 0000000000..f1430474e3
--- /dev/null
+++ b/support/digestif/digestif/bibtex.lua
@@ -0,0 +1,271 @@
+local lpeg = require "lpeg"
+local util = require "digestif.util"
+
+local B, P, R, S, V
+ = lpeg.B, lpeg.P, lpeg.R, lpeg.S, lpeg.V
+local C, Cc, Cp, Ct, Cmt, Cg
+ = lpeg.C, lpeg.Cc, lpeg.Cp, lpeg.Ct, lpeg.Cmt, lpeg.Cg
+local concat = table.concat
+local merge = util.merge
+local search, gobble_until, case_fold = util.search, util.gobble_until, util.case_fold
+local split, replace = util.split, util.replace
+
+local bibtex = {}
+
+local function ipairs_from(t, i)
+ return ipairs(t), t, i - 1
+end
+
+--* Parser
+
+local char = P(1)
+local at_sign = P"@"
+local newline = P"\n"
+local space = S" \r\n\t"
+local whitespace = space^0
+local comment = P"%" * (1 - newline)^0 * newline
+local junk = (comment + 1 - at_sign)^0
+local number = C(R"09"^1) * whitespace
+local name = C((R"az" + R"AZ" + R"09" + S"!$&*+-./:;<>?[]^_`|")^1) * whitespace
+local lbrace = P"{" * whitespace
+local rbrace = P"}" * whitespace
+local lparen = P"(" * whitespace
+local rparen = P")" * whitespace
+local equals = P"=" * whitespace
+local hash = P"#" * whitespace
+local comma = P"," * whitespace
+local quote = P'"' * whitespace
+local lit_string = C(case_fold "string") * whitespace
+local lit_comment = C(case_fold "comment") * whitespace
+local lit_preamble = C(case_fold "preamble") * whitespace
+
+local Cstart = Cg(Cp(), "start")
+local Cstop = Cg(Cp(), "stop")
+
+local curly_braced_string = util.between_balanced("{", "}")
+local round_braced_string = util.between_balanced("(", ")")
+local braced_string = (curly_braced_string + round_braced_string) * whitespace
+local quoted_string = '"' * C(gobble_until('"')) * '"' * whitespace
+local simple_value = quoted_string + braced_string + number + Ct(name)
+local value = simple_value * (hash * simple_value)^0
+local field = Ct(name * equals * value) + whitespace
+local fields = field * (comma * field)^0
+
+local token = curly_braced_string/0 + char
+local nonspace = token - space
+local author_sep = space * "and" * space
+local etal_marker = P("et al") * whitespace * P(-1)
+
+-- either curly or round braced
+local function braced(patt)
+ return lbrace * patt * rbrace + lparen * patt * rparen
+end
+
+local string_entry = at_sign * lit_string * braced(fields)
+local comment_entry = at_sign * lit_comment * braced_string
+local preamble_entry = at_sign * lit_preamble * braced(value)
+local regular_entry = at_sign * name * braced(name * comma * fields)
+local entry = string_entry + comment_entry + preamble_entry + regular_entry
+
+-- this pattern produces the parse tree
+-- TODO: catch premature end on invalid entry
+local all_entries = Ct((junk * Ct(Cstart * entry * Cstop))^0)
+
+--* Translate parse tree
+
+-- BibItem class
+local BibItem = {}
+local mt = {__index = BibItem}
+setmetatable(
+ BibItem, {
+ __call = function(_,t) return setmetatable(t, mt) end
+})
+
+-- replace user-defined strings and concatenate
+local function process_value(val, strings, i)
+ i = i or 2
+ local t = {}
+ for _, v in ipairs_from(val, i) do
+ if type(v) == "table" then
+ t[#t+1] = strings[v[1]] or ""
+ else
+ t[#t+1] = v
+ end
+ end
+ return concat(t)
+end
+
+local default_options = {
+ with_authors = true,
+ with_title = true
+}
+
+--- Parse a bibtex file.
+function bibtex.parse(src, options)
+ options = merge(default_options, options)
+ local entries = all_entries:match(src)
+ local strings = merge(options.strings)
+ local preambles = {}
+ local ids = {}
+ local items = {
+ strings = strings,
+ preambles = preambles,
+ ids = ids
+ }
+ for _, t in ipairs(entries) do
+ local entry_type = t[1]:lower()
+ if entry_type == "comment" then
+ -- pass
+ elseif entry_type == "preamble" then
+ preambles[#preambles + 1] = t[2]
+ elseif entry_type == "string" then
+ for _, u in ipairs_from(t, 2) do
+ local key = u[1]:lower()
+ local val = process_value(u, strings)
+ strings[key] = val
+ end
+ else
+ local id = t[2]
+ local fields = {}
+ for _, u in ipairs_from(t, 3) do
+ local key = u[1]:lower()
+ local val = process_value(u, strings)
+ fields[key] = val
+ end
+ local item = BibItem {
+ id = id,
+ type = entry_type,
+ fields = fields,
+ pos = t.start,
+ cont = t.stop
+ }
+ ids[id] = item
+ items[#items + 1] = item
+ end
+ end
+ return items
+end
+
+--* Deuglify strings
+
+local tex_symbols = {
+ oe = "œ",
+ OE = "Œ",
+ ae = "ӕ",
+ AE = "Ӕ",
+ aa = "å",
+ AA = "Å",
+ ss = "ß",
+ o = "ø",
+ O = "Ø",
+ l = "ł",
+ L = "Ł",
+ -- corner cases
+ i = "{i}", -- not "ı"
+ j = "j", -- not "ȷ"
+ [" "] = " ",
+}
+
+local tex_accents = util.map(
+ replace("◌", ""),
+ {
+ ['"'] = "◌̈",
+ ["'"] = "◌́",
+ ["."] = "◌̇",
+ ["="] = "◌̄",
+ ["^"] = "◌̂",
+ ["`"] = "◌̀",
+ ["~"] = "◌̃",
+ ["c"] = "◌̧",
+ ["d"] = "◌̣",
+ ["H"] = "◌̋",
+ ["u"] = "◌̆",
+ ["b"] = "◌̱",
+ ["v"] = "◌̌",
+ ["t"] = "◌͡"
+ }
+)
+
+local tex_letter = (R"AZ" + R"az")
+local tex_char_or_math = "$" * gobble_until("$") * "$" + char -- for deuglification purposes
+local tex_cs_patt = "\\" * C(tex_letter^1 + char)* whitespace
+local tex_accent_patt = tex_cs_patt * (curly_braced_string + C(char))
+local function repl_accents_fun(cs, arg)
+ local acc = tex_accents[cs]
+ if not acc then
+ return -- must return 0 values!
+ else
+ return arg .. acc
+ end
+end
+
+local detexify_symbols = replace(tex_cs_patt, tex_symbols, tex_char_or_math)
+local detexify_accents = replace(tex_accent_patt, repl_accents_fun, tex_char_or_math)
+local debracify = replace(curly_braced_string, 1, tex_char_or_math)
+local detitlify = replace(B(space) * C(tex_letter), string.lower, tex_char_or_math)
+local trim = util.trim(space)
+local clean = util.clean(space)
+
+local function deuglify_name (s)
+ return
+ clean(
+ debracify(
+ detexify_accents(
+ detexify_symbols(s))))
+end
+
+local function deuglify_title (s)
+ return
+ clean(
+ debracify(
+ detitlify(
+ detexify_accents(
+ detexify_symbols(s)))))
+end
+
+--* Pretty-printing
+
+local split_authors = split(author_sep, token)
+local split_name = split(comma, token)
+local split_last = search(Cp() * C(nonspace^1) * whitespace * P(-1))
+
+function BibItem:authors()
+ local t = {}
+ local author = self.fields.author
+ if not author then return {} end
+ for _, name in ipairs(split_authors(author)) do
+ local u = {}
+ local parts = split_name(name)
+ if #parts == 3 then
+ u.first = parts[3]
+ u.last = parts[1]
+ u.suffix = parts[2]
+ elseif #parts == 2 then
+ u.first = parts[2]
+ u.last = parts[1]
+ else
+ local p, l = split_last:match(name)
+ if p then
+ u.first = name:sub(1, p - 1)
+ u.last = l
+ else
+ u.last = name
+ end
+ end
+ t[#t + 1] = u
+ end
+ return t
+end
+
+function BibItem:pretty_print()
+ local t, a = {}, {}
+ for _, name in ipairs(self:authors()) do
+ a[#a + 1] = deuglify_name(name.last)
+ end
+ t[#t + 1] = concat(a, ", ")
+ t[#t + 1] = (self.fields.year or self.fields.date or "(n.d.)") .. ";"
+ t[#t + 1] = deuglify_title(self.fields.title or "")
+ return concat(t, " ")
+end
+
+return bibtex
diff --git a/support/digestif/digestif/config.lua b/support/digestif/digestif/config.lua
new file mode 100644
index 0000000000..8b901ba93d
--- /dev/null
+++ b/support/digestif/digestif/config.lua
@@ -0,0 +1,156 @@
+local util = require "digestif.util"
+local format = string.format
+
+local config = {}
+
+config.version = "0.5"
+local pre_version = os.getenv("DIGESTIF_PRERELEASE")
+if pre_version then
+ config.version = config.version .. "-" .. pre_version
+end
+
+if util.is_command("kpsewhich") then
+ local pipe = io.popen("kpsewhich -var-brace-value=TEXMF")
+ local output = pipe:read("l")
+ local ok, exitt, exitc = pipe:close()
+ if ok and exitt == "exit" and exitc == 0 then
+ config.texmf_dirs = util.imap(
+ function (s) return s:gsub("^!!", "") end,
+ util.path_list_split(output)
+ )
+ elseif config.verbose then
+ util.log("Error running kpsewhich (%s %d)", exitt, exitc)
+ end
+else -- TODO: What should be the default?
+ config.texmf_dirs = {
+ "/usr/local/share/texmf",
+ "/usr/share/texmf",
+ "/usr/share/texlive/texmf-local",
+ "/usr/share/texlive/texmf-dist",
+ }
+end
+
+config.data_dirs = {} -- TODO: What should be the default?
+
+-- Location of a complete texmf distribution, used for instance to
+-- find documentation not installed locally. Passed to format with
+-- one argument, the percent-encoded name of a file.
+config.external_texmf = "https://www.tug.org/texlive/Contents/live/texmf-dist/%s"
+
+config.provide_snippets = false
+
+-- Table mapping command names to the snippet to be used, overriding
+-- the default.
+config.extra_snippets = {}
+
+-- This allows the user to assign custom actions to generated command
+-- tags.
+config.extra_actions = {
+ eqref = "ref"
+}
+
+config.fuzzy_cite = true
+config.fuzzy_ref = true
+config.info_command = util.is_command("info")
+
+-- For candidates of these kinds, include the annotation in the
+-- candidate label. The values of this table are a string which is
+-- formatted with two arguments, the candidate text and annotation.
+config.lsp_long_candidates = {
+ label = "%-12s %s",
+ bibitem = "%-12s %s",
+}
+
+--* Loading user settings
+
+local function is_table(key_type, val_type)
+ return function(obj)
+ if type(obj) ~= "table" then return false end
+ for key, val in pairs(obj) do
+ if type(key) ~= key_type then return false end
+ if type(val) ~= val_type then return false end
+ end
+ return true
+ end
+end
+
+local validators = {
+ data_dirs = is_table("number", "string"),
+ extra_actions = is_table("string", "string"),
+ extra_snippets = is_table("string", "string"),
+ fuzzy_cite = "boolean",
+ fuzzy_ref = "boolean",
+ info_command = "string",
+ lsp_long_candidates = is_table("string", "string"),
+ provide_snippets = "boolean",
+ texmf_dirs = is_table("number", "string"),
+ tlpdb_path = "string",
+ verbose = "boolean",
+}
+
+-- Set config entries found in `tbl`.
+function config.load_from_table(tbl)
+ -- Set verbose option before all others
+ local verbose = tbl.verbose or config.verbose
+ for key, val in pairs(tbl) do
+ local validator, ok = validators[key]
+ if type(validator) == "string" then
+ ok = type(val) == validator
+ elseif type(validator) == "function" then
+ ok = validator(val)
+ end
+ if ok then
+ config[key] = val
+ if verbose then
+ local msg = "Setting configuration option %s = %s"
+ util.log(msg, key, util.inspect(val))
+ end
+ else
+ local msg = "Invalid configuration option: %s = %s"
+ error(format(msg, key, util.inspect(val)))
+ end
+ end
+end
+
+function config.load_from_file(file)
+ local tbl = {}
+ local ok, err = loadfile(file, "t", tbl)
+ if ok then ok, err = pcall(ok) end
+ if not ok then
+ local msg = "Error loading configuration from %s: %s"
+ error(msg, file, err)
+ end
+ config.load_from_table(tbl)
+end
+
+function config.load_from_env()
+ local DIGESTIF_DATA = os.getenv("DIGESTIF_DATA")
+ if DIGESTIF_DATA then
+ config.data_dirs = util.path_list_split(DIGESTIF_DATA)
+ end
+
+ local DIGESTIF_TEXMF = os.getenv("DIGESTIF_TEXMF")
+ if DIGESTIF_TEXMF then
+ config.texmf_dirs = util.path_list_split(DIGESTIF_TEXMF)
+ end
+
+ local DIGESTIF_TLPDB = os.getenv("DIGESTIF_TLPDB")
+ if DIGESTIF_TLPDB then
+ config.tlpdb_path = util.path_list_split(DIGESTIF_TLPDB)
+ end
+end
+
+function config.check_data(dir)
+ if not dir then
+ for _, dir in ipairs(config.data_dirs) do
+ if config.check_data(dir) then return true end
+ end
+ end
+ local f = io.open(util.path_join(dir, "primitives.tags"))
+ if f then
+ f:close()
+ return true
+ end
+end
+
+return config
diff --git a/support/digestif/digestif/data.lua b/support/digestif/digestif/data.lua
new file mode 100644
index 0000000000..621048f4a9
--- /dev/null
+++ b/support/digestif/digestif/data.lua
@@ -0,0 +1,384 @@
+local lpeg = require "lpeg"
+local util = require "digestif.util"
+local config = require "digestif.config"
+
+local format, strfind = string.format, string.find
+local concat, unpack = table.concat, table.unpack
+local popen = io.popen
+local P, C, Cg, Ct = lpeg.P, lpeg.C, lpeg.Cg, lpeg.Ct
+local match = lpeg.match
+local many, sequence = util.many, util.sequence
+local gobble, search = util.gobble, util.search
+local nested_get, extend = util.nested_get, util.extend
+local find_file = util.find_file
+local parse_uri, make_uri = util.parse_uri, util.make_uri
+local log = util.log
+local path_split, path_join = util.path_split, util.path_join
+
+local data = {}
+local loaded_tags = {}
+
+--* CTAN data
+
+-- function ctan_package(name)
+--
+-- Return a little tags table with the package's details on CTAN
+-- (package description, ctan link, documentation). No command
+-- information.
+--
+-- function ctan_package_of(file)
+--
+-- The name of the CTAN package to which file belongs.
+--
+local ctan_package, ctan_package_of -- to be defined
+
+local tlpdb_path = config.tlpdb_path
+ and find_file(config.tlpdb_path)
+ or find_file(config.texmf_dirs, "../tlpkg/texlive.tlpdb")
+
+if tlpdb_path then
+
+ local _, tlpdb_text = find_file(tlpdb_path, nil, true)
+
+ if config.verbose then
+ log("Reading TLPDB from '%s'", tlpdb_path)
+ end
+
+ local Peol = P"\n"
+ local concat_lines = function(...) return concat({...}, "\n") end
+
+ local within_item = 1 - Peol * Peol
+ local within_files = (1 - Peol) + Peol * P" "
+ local gobble_to_eol = gobble("\n")
+
+ -- Pattern to turn an entry in the TLPDB file into a table with
+ -- entries name, summary, details, documentation, runfiles.
+ local tlpdb_item_patt = Ct(
+ sequence(
+ Cg( -- collect the name
+ P"name " * C(gobble_to_eol),
+ "ctan_package"),
+ Cg( -- find a shortdesc or give up
+ search(
+ Peol * "shortdesc " * C(gobble_to_eol),
+ within_item),
+ "summary"),
+ Cg( -- find a longdesc or give up
+ search(
+ (Peol * "longdesc " * C(gobble_to_eol))^1 / concat_lines,
+ within_item),
+ "details"),
+ Cg( -- find docfiles section
+ many(-1, -- or continue
+ sequence(
+ search(Peol * "docfiles " * gobble_to_eol, within_item),
+ Ct(many( -- collect several docfile entries in a list
+ search( -- looking for the interesting ones only
+ Ct( -- parse one docfile entry
+ sequence( -- collect URI, but only if details field exist
+ Peol * " RELOC/",
+ Cg(gobble(" ", 1 - Peol) / "texmf:%0", "uri"),
+ Cg(P" details=\"" * C(gobble"\""), "summary"))),
+ within_files))))),
+ "documentation"),
+ Cg( -- find runfiles section or give up
+ sequence(
+ search(Peol * "runfiles " * gobble_to_eol, within_item),
+ Ct(many( -- collect the runfile entries is a list
+ sequence( -- parse one runfile entry
+ Peol * " ", -- discard the file line marker
+ many(search("/", 1 - Peol)), -- discard folder part
+ C(gobble_to_eol))))), -- collect base name
+ "runfiles")))
+
+ local tlpdb_items = Ct(many(search(Peol * tlpdb_item_patt))):match(tlpdb_text)
+
+ function ctan_package(name)
+ for i = 1, #tlpdb_items do
+ local item = tlpdb_items[i]
+ if item.name == name then
+ return item
+ end
+ end
+ end
+ ctan_package = util.memoize1(ctan_package)
+
+ function ctan_package_of(file)
+ for i = 1, #tlpdb_items do
+ local item = tlpdb_items[i]
+ local runfiles = item.runfiles
+ for j = 1, #runfiles do
+ if runfiles[j] == file then
+ return item
+ end
+ end
+ end
+ end
+ ctan_package_of = util.memoize1(ctan_package_of)
+
+else
+
+ ctan_package = function() end
+ ctan_package_of = function() end
+ if config.verbose then log("TLPDB not found") end
+
+end
+
+--* kpathsea emulation
+
+-- function kpsewhich(name)
+--
+-- Like the kpsewhich command, return the full path of tex input file
+-- name. We use luatex's kpse bindings if available, and parse the
+-- system's ls-R files otherwise.
+--
+local kpsewhich
+
+if kpse then -- we're on luatex
+
+ local kpse_obj = kpse.new("luatex")
+
+ function kpsewhich(name)
+ return kpse_obj:find_file(name, "tex")
+ end
+
+else -- on plain lua, we look for ls-R files
+
+ local texmf_files = {}
+ local texmf_dirs = config.texmf_dirs
+ local dir_patt = P"./" * C(gobble(":" * P(-1)))
+ for i = 1, #texmf_dirs do
+ local texmf_dir = texmf_dirs[i]
+ local listing_path = find_file(texmf_dir, "ls-R")
+ if listing_path then
+ local current_dir
+ for line in io.lines(listing_path) do
+ local subdir = match(dir_patt, line)
+ if subdir then
+ current_dir = path_join(texmf_dir, subdir)
+ elseif current_dir and not texmf_files[line] then
+ texmf_files[line] = path_join(current_dir, line)
+ end
+ end
+ end
+ end
+
+ function kpsewhich(name)
+ return texmf_files[name]
+ end
+
+end
+
+--* Generate tags from the user's TeX installation
+
+local function infer_format(path)
+ local ext = path:sub(-4)
+ if ext == ".bib" then
+ return "bibtex"
+ elseif ext == ".sty" or ext == ".cls" or ext == ".ltx" then
+ return "latex-prog"
+ elseif ext == ".xml" and path:match("%Acontext%A") then
+ return "context-xml"
+ elseif ext == ".tex" then
+ return "latex-prog"
+ end
+end
+
+local function tags_from_manuscript(script, ctan_data)
+ local commands, environments, dependencies = {}, {}, {}
+ local tags = {
+ generated = true,
+ dependencies = dependencies,
+ commands = commands,
+ environments = environments
+ }
+ if ctan_data then
+ setmetatable(tags, {__index = ctan_data})
+ end
+ for pkg in pairs(script.packages) do
+ dependencies[#dependencies+1] = pkg
+ end
+ table.sort(dependencies)
+ for _, it in script:index_pairs("newcommand") do
+ commands[it.name] = {
+ arguments = it.arguments,
+ }
+ end
+ for _, it in script:index_pairs("newenvironment") do
+ environments[it.name] = {
+ arguments = it.arguments,
+ }
+ end
+ return tags
+end
+
+-- Generate tags from TeX source code (or ConTeXt interface XML file,
+-- if applicable). The argument is a file name found by kpsewhich.
+--
+local function generate_tags(name)
+ local path = kpsewhich(name)
+ path = path and find_file(config.texmf_dirs, path)
+ if not path then return end
+ local texformat = infer_format(path)
+ if not texformat then return end
+ if config.verbose then log("Generating tags: %s", path) end
+ local pkg = ctan_package_of(name)
+ if texformat == "context-xml" then
+ -- The function below is defined and monkey-patched in
+ -- ManuscriptConTeXt, to keep the XML dependency separated.
+ if not data.tags_from_xml then require "digestif.ManuscriptConTeXt" end
+ return data.tags_from_xml(path, pkg)
+ else
+ loaded_tags[name] = {} -- TODO: this is to avoid loops, find a better way
+ local cache = require "digestif.Cache"()
+ local script = cache:manuscript(path, texformat)
+ return tags_from_manuscript(script, pkg)
+ end
+end
+data.generate_tags = generate_tags
+
+--* Load tags
+
+-- function require_tags(name)
+--
+-- Return tags table for input file name. Either loads from a tags
+-- file in the data directory, or generate from source on the fly.
+--
+local require_tags
+
+local parse_ref = util.matcher(
+ sequence(
+ P"$ref:",
+ C(gobble"#") * P"#",
+ Ct(many(P"/" * C(gobble"/")))))
+p=parse_ref
+local function resolve_refs(tbl, seen)
+ seen = seen or {}
+ for k, v in pairs(tbl) do
+ if type(v) == "string" then
+ local loc, path = parse_ref(v)
+ if loc then
+ tbl[k] = nested_get(require_tags(loc), unpack(path))
+ end
+ elseif type(v) == "table" and not seen[v] then
+ seen[v] = true
+ resolve_refs(v, seen)
+ end
+ end
+end
+
+-- Load a tags file from the data directory.
+local function load_tags(name)
+ if strfind(name, "..", 1, true) then return end -- bad file name
+ local path, str = find_file(config.data_dirs, name .. ".tags", true)
+ if not path then return end
+ if config.verbose then log("Loading tags: %s", path) end
+ local tags = {}
+ local ok, message = load(str, path, "t", tags)
+ if ok then ok, message = pcall(ok) end
+ if not ok and config.verbose then
+ log("Error loading %s.tags: %s", name, message)
+ return -- TODO: should throw an error?
+ end
+ local pkg = ctan_package(tags.ctan_package) or ctan_package_of(name)
+ setmetatable(tags, {__index = pkg})
+ return tags
+end
+
+require_tags = function(name)
+ local tags = loaded_tags[name]
+ if not tags then
+ tags = load_tags(name) or generate_tags(name)
+ if tags then
+ loaded_tags[name] = tags
+ resolve_refs(tags)
+ local extra_actions = config.extra_actions or {}
+ for _, kind in ipairs{"commands", "environments"} do
+ for cs, cmd in pairs(tags[kind] or {}) do
+ if not cmd.package then cmd.package = tags end
+ if extra_actions[cs] then cmd.action = extra_actions[cs] end
+ end
+ end
+ end
+ end
+ return tags
+end
+data.require = require_tags
+data.require_tags = require_tags
+
+-- Load all data files, and return them in a table. This is intended
+-- for debugging and testing only, and depends on luafilesystem.
+local function load_all_tags()
+ local t = {}
+ local ok, lfs = pcall(require, "lfs")
+ assert(ok, "Function data.load_all() need the luafilesystem library.")
+ for _, data_dir in ipairs(config.data_dirs) do
+ for path in lfs.dir(data_dir) do
+ local pkg = path:match("(.*)%.tags")
+ if pkg then
+ assert(load_tags(pkg), "Error loading data file " .. path)
+ t[pkg] = require_tags(pkg) or error("Error processing data file " .. path)
+ end
+ end
+ end
+ return t
+end
+data.load_all = load_all_tags
+
+--* User-readable documentation
+
+local function resolve_uri(uri)
+ local scheme, _, location, _, fragment = parse_uri(uri)
+ if scheme == "info" then
+ return uri
+ elseif scheme == "texmf" then
+ local path = find_file(config.texmf_dirs, location)
+ if path then
+ return make_uri("file", "", path, nil, fragment)
+ else
+ return format(
+ config.external_texmf, make_uri(nil, nil, location, nil, fragment)
+ )
+ end
+ else
+ return uri
+ end
+end
+
+-- Given a list of documentation items, return a markdown-formatted
+-- string of links to these documents.
+local function resolve_doc_items(items)
+ if type(items) == "string" then items = {items} end
+ local t = {}
+ for _, item in ipairs(items) do
+ if type(item) == "string" then
+ t[#t+1] = format("- <%s>", resolve_uri(item))
+ else
+ t[#t+1] = format("- [%s](%s)", item.summary, resolve_uri(item.uri))
+ end
+ end
+ return t
+end
+data.resolve_doc_items = resolve_doc_items
+
+-- Call info, return the relevant part of the info node.
+local function get_info(uri)
+ if config.info_command then
+ local scheme, _, path, _, fragment = parse_uri(uri)
+ if scheme ~= "info" then return end
+ local cmd = format("%s '(%s)%s'", config.info_command, path, fragment)
+ local pipe = popen(cmd)
+ local str = pipe:read("a")
+ local ok, exitt, exitc = pipe:close()
+ if ok and exitt == "exit" and exitc == 0 then
+ str = str:gsub(".-\n", "", 2) -- discard header line
+ str = str:gsub("\nFile: " .. path .. ".info.*", "\n") -- discard child nodes
+ return str, path, fragment
+ elseif config.verbose then
+ log("Error running info (%s %d)", exitt, exitc)
+ end
+ end
+end
+data.get_info=util.memoize(get_info)
+
+return data
diff --git a/support/digestif/digestif/langserver.lua b/support/digestif/digestif/langserver.lua
new file mode 100644
index 0000000000..55bd8c8dd7
--- /dev/null
+++ b/support/digestif/digestif/langserver.lua
@@ -0,0 +1,567 @@
+local config = require "digestif.config"
+local util = require "digestif.util"
+
+local floor = math.floor
+local format = string.format
+local imap, nested_get, lines = util.imap, util.nested_get, util.lines
+local parse_uri, make_uri = util.parse_uri, util.make_uri
+local log = util.log
+
+-- Use cjson if available, otherwise fall back to `digestif.util`
+-- implementation.
+local null, json_decode, json_encode
+if pcall(require, "cjson") then
+ local cjson = require "cjson"
+ cjson.encode_empty_table_as_object(false)
+ null = cjson.null
+ json_decode, json_encode = cjson.decode, cjson.encode
+else
+ null = util.json_null
+ json_decode, json_encode = util.json_decode, util.json_encode
+end
+
+--* Convert LSP API objects to/from internal representations
+
+-- This will be a digestif.Cache object. Its initialization is
+-- deferred to `initialized` method.
+local cache = setmetatable({}, {
+ __index = function () error "Server not initialized!" end
+})
+
+-- A place to store the file name and texformat of open documents.
+local open_documents = setmetatable({}, {
+ __index = function(_, k)
+ error(format("Trying to access unopened document %s", k))
+ end
+})
+
+local function from_DocumentUri(str)
+ local scheme, auth, path, query, fragment = parse_uri(str)
+ if scheme ~= "file" or (auth and auth ~= "") or query or fragment then
+ error("Invalid or unsupported URI: " .. str)
+ end
+ if util.os_type == "windows" and path:find("^/%a:") then
+ path = path:sub(2)
+ end
+ return path
+end
+
+local function to_DocumentUri(str)
+ if util.os_type == "windows" then
+ str = str:gsub("[/\\]", "/")
+ if str:find("^%a:") then str = "/" .. str end
+ end
+ return make_uri("file", "", str)
+end
+
+-- p0 is the position of a line l0, provided as a hint for the search.
+-- Return a position in bytes, and a new hint p0, l0.
+local function from_Position(str, position, p0, l0)
+ local l, c = position.line + 1, position.character + 1
+ if l0 and l0 > l then p0, l0 = nil, nil end
+ for n, i in lines(str, p0, l0) do
+ if n == l then
+ return utf8.offset(str, c, i), i, l
+ end
+ end
+end
+
+local function from_Range(str, range, p0, l0)
+ local pos, p1, l1 = from_Position(str, range.start, p0, l0) -- inclusive
+ local cont = from_Position(str, range['end'], p1, l1) -- exclusive
+ if not (pos and cont) then error("Position out of bounds") end
+ return pos, cont, p1, l1
+end
+
+local function from_TextDocumentIdentifier(arg)
+ local filename = from_DocumentUri(arg.uri)
+ local texformat = open_documents[filename]
+ local script = cache:manuscript(filename, texformat)
+ return script
+end
+
+local function from_TextDocumentPositionParams(arg)
+ local filename = from_DocumentUri(arg.textDocument.uri)
+ local texformat = open_documents[filename]
+ local script = cache:manuscript(filename, texformat)
+ local l, c = arg.position.line + 1, arg.position.character + 1
+ return script, script:position_at(l, c)
+end
+
+local function to_Range(item)
+ local script = item.manuscript
+ local l1, c1 = script:line_column_at(item.pos)
+ local l2, c2 = script:line_column_at(item.cont)
+ return {
+ start = {line = l1 - 1, character = c1 - 1},
+ ["end"] = {line = l2 - 1, character = c2 - 1},
+ }
+end
+
+local function to_Location(item)
+ return {
+ uri = to_DocumentUri(item.manuscript.filename),
+ range = to_Range(item)
+ }
+end
+
+local function to_MarkupContent(str)
+ return {kind = "markdown", value = str}
+end
+
+local function to_TextEdit(script, pos, old, new)
+ local l, c_start = script:line_column_at(pos)
+ local c_end = c_start + utf8.len(old)
+ return {
+ range = {
+ start = {line = l - 1, character = c_start - 1},
+ ["end"] = {line = l - 1, character = c_end - 1},
+ },
+ newText = new
+ }
+end
+
+-- An essentially random assignment of symbol kinds, since LSP doesn't
+-- support custom kinds.
+local to_SymbolKind = {
+ section = 5,
+ section_index = 5,
+ label_index = 7,
+ bib_index = 20,
+ newcommand_index = 12,
+ newenvironment_index = 12
+}
+
+local function to_SymbolInformation(item, index_name)
+ return {
+ name = item.name,
+ kind = to_SymbolKind[index_name],
+ location = to_Location(item)
+ }
+end
+
+local function to_DocumentSymbol(outline)
+ return {
+ name = outline.name,
+ kind = to_SymbolKind[outline.kind],
+ range = to_Range(outline),
+ selectionRange = to_Range(outline),
+ children = outline[1] and imap(to_DocumentSymbol, outline)
+ }
+end
+
+local languageId_translation_table = {
+ bibtex = "bibtex",
+ context = "context",
+ doctex = "doctex",
+ latex = "latex",
+ plain = "plain",
+ plaintex = "plain",
+ ["plain-tex"] = "plain",
+ tex = "latex", -- this is for vim; maybe "tex" should mean "tex file, undecided format"
+ texinfo = "texinfo"
+}
+
+local function languageId_translate(id, filename)
+ local ext = filename:sub(-4)
+ local texformat = languageId_translation_table[id]
+ if not texformat then
+ error(("Invalid LSP language id %q"):format(id))
+ end
+ if texformat == "latex" and (ext == ".sty" or ext == ".cls") then
+ return "latex-prog"
+ end
+ -- TODO: Handle .code.tex files from PGF, .mkiv files from ConTeXt, etc.
+ return texformat
+end
+
+--* LSP methods
+
+local methods = {}
+
+methods["initialize"] = function(params)
+ config.provide_snippets = nested_get(params.capabilities,
+ "textDocument", "completion", "completionItem", "snippetSupport")
+ if params.initializationOptions then
+ config.load_from_table(params.initializationOptions)
+ end
+ return {
+ capabilities = {
+ textDocumentSync = {
+ openClose = true,
+ change = 2
+ },
+ completionProvider = {
+ triggerCharacters = {"\\", "{", "[", ",", "="},
+ },
+ signatureHelpProvider = {
+ triggerCharacters = {"{", "[", "="},
+ },
+ hoverProvider = true,
+ definitionProvider = true,
+ referencesProvider = true,
+ documentSymbolProvider = true,
+ workspaceSymbolProvider = true
+ },
+ serverInfo = {
+ name = "Digestif",
+ version = config.version
+ }
+ }
+end
+
+methods["initialized"] = function()
+ cache = require "digestif.Cache"()
+end
+
+methods["shutdown"] = function() return null end
+methods["exit"] = function() os.exit() end
+methods["textDocument/willSave"] = function() end
+methods["textDocument/didSave"] = function() end
+
+methods["workspace/didChangeConfiguration"] = function(params)
+ local settings = params.settings.digestif
+ if type(settings) ~= "table" then return end
+ config.load_from_table(settings)
+end
+
+methods["textDocument/didOpen"] = function(params)
+ local filename = from_DocumentUri(params.textDocument.uri)
+ local texformat = languageId_translate(params.textDocument.languageId, filename)
+ open_documents[filename] = texformat
+ cache:put(filename, params.textDocument.text)
+end
+
+methods["textDocument/didChange"] = function(params)
+ local filename = from_DocumentUri(params.textDocument.uri)
+ local p0, l0, src = 1, 1, cache(filename)
+ for _, change in ipairs(params.contentChanges) do
+ if change.range then
+ local pos, cont
+ pos, cont, p0, l0 = from_Range(src, change.range, p0, l0)
+ src = src:sub(1, pos - 1) .. change.text .. src:sub(cont)
+ else
+ src = change.text
+ end
+ end
+ cache:put(filename, src)
+end
+
+methods["textDocument/didClose"] = function(params)
+ local filename = from_DocumentUri(params.textDocument.uri)
+ open_documents[filename] = nil
+ cache:forget(filename)
+end
+
+methods["textDocument/signatureHelp"] = function(params)
+ local script, pos = from_TextDocumentPositionParams(params)
+ local help = script:describe(pos)
+ if not help or not help.arg then return null end
+ local parameters, label_positions = {}, help.label_positions or {}
+ for i, arg in ipairs(nested_get(help, "data", "arguments") or {}) do
+ parameters[i] = {
+ label = {label_positions[2*i-1] - 1, label_positions[2*i] - 1},
+ documentation = arg.summary
+ }
+ end
+ return {
+ signatures = {
+ [1] = {
+ label = help.label,
+ documentation = help.summary,
+ parameters = parameters,
+ activeParameter = help.arg - 1
+ }
+ },
+ activeSignature = 0,
+ activeParameter = help.arg - 1
+ }
+end
+
+methods["textDocument/hover"] = function(params)
+ local script, pos = from_TextDocumentPositionParams(params)
+ local help = script:describe(pos)
+ if (not help) or help.arg then return null end
+ local contents = help.details or help.summary or "???"
+ return {contents = to_MarkupContent(contents)}
+end
+
+methods["textDocument/completion"] = function(params)
+ local script, pos = from_TextDocumentPositionParams(params)
+ local candidates = script:complete(pos)
+ if not candidates then return null end
+ local long_format = config.lsp_long_candidates
+ and config.lsp_long_candidates[candidates.kind]
+ local items = {}
+ for i, cand in ipairs(candidates) do
+ local snippet = config.provide_snippets and cand.snippet
+ local fuzzy_score = cand.fuzzy_score or nil
+ local annotation = cand.annotation
+ local long_label = long_format and annotation
+ and format(long_format, cand.text, annotation)
+ items[i] = {
+ label = long_label or cand.text,
+ sortText = fuzzy_score and format("~%03d", floor(1000 * (1 - fuzzy_score))),
+ documentation = cand.summary,
+ detail = not long_label and cand.annotation or nil,
+ insertTextFormat = snippet and 2 or 1,
+ textEdit = to_TextEdit(
+ script,
+ candidates.pos,
+ candidates.prefix,
+ snippet or cand.text
+ )
+ }
+ end
+ return items
+end
+
+methods["textDocument/definition"] = function(params)
+ local script, pos = from_TextDocumentPositionParams(params)
+ local definition = script:find_definition(pos)
+ return definition and to_Location(definition) or null
+end
+
+methods["textDocument/references"] = function(params)
+ local script, pos = from_TextDocumentPositionParams(params)
+ local result = {}
+ if params.context and params.context.includeDeclaration then
+ local definition = script:find_definition(pos)
+ if definition then
+ result[#result + 1] = to_Location(definition)
+ end
+ end
+ local references = script:find_references(pos)
+ if references then
+ for _, ref in ipairs(references) do
+ result[#result + 1] = to_Location(ref)
+ end
+ end
+ if #result > 0 then
+ return result
+ else
+ return null
+ end
+end
+
+methods["textDocument/documentSymbol"] = function(params)
+ local script = from_TextDocumentIdentifier(params.textDocument)
+ local outline = script:outline(true) -- local only
+ return imap(to_DocumentSymbol, outline)
+end
+
+methods["workspace/symbol"] = function(params)
+ local query, t = params.query, {}
+
+ -- Find all root documents and sort them
+ local root_documents, sorted = {}, {}
+ for filename, texformat in pairs(open_documents) do
+ local script = cache:manuscript(filename, texformat)
+ root_documents[script.root.filename] = script.root
+ end
+
+ for filename in pairs(root_documents) do
+ sorted[#sorted+1] = filename
+ end
+ table.sort(sorted)
+
+ -- Gather all entries in all indexes
+ for _, filename in ipairs(sorted) do
+ local script = root_documents[filename]
+ for item, index_name in script:traverse {
+ "section_index",
+ "label_index",
+ "bib_index",
+ "newcommand_index",
+ "newenvironment_index"
+ } do
+ if item.name:find(query, 1, true) then
+ t[#t+1] = to_SymbolInformation(item, index_name)
+ end
+ end
+ end
+ return t
+end
+
+--* RPC functions
+
+local function log_error(err)
+ if config.verbose then
+ log("Error: %s", err)
+ log(debug.traceback())
+ end
+ return err
+end
+
+local crlf = util.os_type == "windows" and "\n" or "\r\n"
+
+local function write_msg(msg)
+ io.write("Content-Length: ", #msg, crlf, crlf, msg)
+ io.flush()
+end
+
+local function read_msg()
+ local headers, msg = {}, nil
+ for h in io.lines() do
+ if h == "" or h == "\r" then break end
+ local k, v = string.match(h, "^([%a%-]+): (.*)")
+ if k then headers[k] = v end
+ end
+ local len = tonumber(headers["Content-Length"])
+ if len then msg = io.read(len) end
+ return msg, headers
+end
+
+local function rpc_send(id, result, error_code)
+ write_msg(
+ json_encode({
+ jsonrpc = "2.0",
+ id = id,
+ result = not error_code and result or nil,
+ error = error_code and {code = error_code, message = result}
+ }))
+end
+
+local function rpc_receive()
+ local msg = read_msg()
+ local ok, request = xpcall(json_decode, log_error, msg)
+ if not ok then
+ rpc_send(null, request, -32700)
+ os.exit(false)
+ end
+ return request.id, request.method, request.params
+end
+
+--* The main loop
+
+local is_optional = util.matcher("$/")
+
+local function process_request()
+ local clock = config.verbose and os.clock()
+ local id, method_name, params = rpc_receive()
+ local method = methods[method_name]
+ if method then
+ local ok, result = xpcall(method, log_error, params)
+ if ok then
+ if id then rpc_send(id, result) end
+ else
+ rpc_send(id, result, 1)
+ end
+ elseif not is_optional(method_name) then
+ rpc_send(id, "Unknown method " .. method_name, -32601)
+ end
+ if clock then
+ log("Request: %4s %-40s %6.2f ms",
+ id or "*", method_name, 1000 * (os.clock() - clock))
+ end
+end
+
+local function generate(path)
+ local generate_tags = require "digestif.data".generate_tags
+ local tags = generate_tags(path)
+ if not tags then
+ io.stderr:write(
+ format("Error: can't find '%s' or can't generate tags from it.\n", path)
+ )
+ os.exit(false)
+ end
+ local _, basename = util.path_split(path)
+ local file = io.open(basename .. ".tags", "w")
+ for _, item in ipairs(
+ {"generated", "dependencies", "documentation", "commands", "environments"})
+ do
+ if tags[item] then
+ file:write(item, " = ", util.inspect(tags[item]), "\n")
+ end
+ end
+ local i, j = 0, 0
+ for _ in pairs(tags.commands) do i = i + 1 end
+ for _ in pairs(tags.environments) do j = j + 1 end
+ io.stderr:write(
+ format("Generated %15s.tags with %3i commands and %3i environments.\n",
+ basename, i, j)
+ )
+end
+
+local usage = [[
+Usage: digestif [--version] [-h] [-v] [-g FILES]
+]]
+
+local help = [[
+Digestif is a language server for TeX
+
+Optional arguments:
+ -g, --generate FILES Generate data file stubs for FILES
+ -h, --help Display this message and exit
+ -v, --verbose Enable log output to stderr
+ --version Show version information
+
+Environment variables:
+ DIGESTIF_DATA Paths to look for data files
+ DIGESTIF_TEXMF Paths to look for TeX files
+ DIGESTIF_TLPDB Path to the TeXLive package database file
+
+ If your TeX distribution or Digestif are installed in a non-standard
+ location, you may need to set some of the above variables.
+]]
+
+local function main(arg)
+ -- Set up default config.data_dirs, if needed
+ config.load_from_env()
+ if #config.data_dirs == 0 then
+ for _, dir in ipairs{
+ util.path_split(debug.getinfo(1).source:match("^@(.*)")),
+ util.path_split(arg[0]),
+ nil
+ } do
+ local f = io.open(util.path_join(dir, "../data/primitives.tags"))
+ if f then
+ f:close()
+ config.data_dirs = {util.path_join(dir, "../data")}
+ break
+ end
+ end
+ end
+
+ -- Read CLI args
+ while arg[1] do
+ local switch = table.remove(arg, 1)
+ if switch == "-v" or switch == "--verbose" then
+ config.verbose = true
+ elseif switch == "-g" or switch == "--generate" then
+ for i = 1, #arg do generate(arg[i]) end
+ os.exit()
+ elseif switch == "-h" or switch == "--help" then
+ io.write(usage)
+ io.write(help)
+ os.exit()
+ elseif switch == "--version" then
+ io.write(format("Digestif %s\n", config.version))
+ os.exit()
+ else
+ io.stderr:write(usage)
+ io.stderr:write(format("Invalid option: %s\n", switch))
+ os.exit(false)
+ end
+ end
+
+ -- Check if config.data_dirs was set up correctly
+ if not util.find_file(config.data_dirs, "primitives.tags") then
+ io.stderr:write(
+ "Error: could not find data files at the following locations:\n\t"
+ .. table.concat(config.data_dirs, "\n\t")
+ .. "\nSet the DIGESTIF_DATA environment variable to fix this.\n"
+ )
+ os.exit(false)
+ end
+
+ -- Main language server loop
+ if config.verbose then log("Digestif started!") end
+ while true do process_request() end
+end
+
+return {
+ main = main,
+ methods = methods
+}
+
diff --git a/support/digestif/digestif/util.lua b/support/digestif/digestif/util.lua
new file mode 100644
index 0000000000..2e042ebd70
--- /dev/null
+++ b/support/digestif/digestif/util.lua
@@ -0,0 +1,789 @@
+-- Assorted utility functions
+
+local lpeg = require "lpeg"
+local zip = pcall(require, "zip") and require "zip"
+
+local io, os = io, os
+local co_yield, co_wrap = coroutine.yield, coroutine.wrap
+local strupper, strfind, strsub, strrep = string.upper, string.find, string.sub, string.rep
+local strchar, strbyte, utf8_char = string.char, string.byte, utf8.char
+local format, gsub = string.format, string.gsub
+local pack, unpack, concat = table.pack, table.unpack, table.concat
+local move, sort = table.move, table.sort
+local pairs, getmetatable, setmetatable = pairs, getmetatable, setmetatable
+local P, V, R, S, I, B = lpeg.P, lpeg.V, lpeg.R, lpeg.S, lpeg.Cp(), lpeg.B
+local C, Cs, Cf, Ct, Cc, Cg = lpeg.C, lpeg.Cs, lpeg.Cf, lpeg.Ct, lpeg.Cc, lpeg.Cg
+local match, locale_table = lpeg.match, lpeg.locale()
+
+local util = {}
+
+--* Table manipulation
+
+local function map(f, t)
+ local r = {}
+ for k, v in pairs(t) do
+ r[k] = f(v)
+ end
+ return r
+end
+util.map = map
+
+local function imap(f, t)
+ local r = {}
+ for i = 1, #t do
+ r[i] = f(t[i])
+ end
+ return r
+end
+util.imap = imap
+
+-- Return a table with entries (k, f(k)), where k ranges over the keys
+-- of t and all its __index parents.
+local function map_keys(f, t)
+ local mt = getmetatable(t)
+ local p = mt and mt.__index
+ local r = p and map_keys(f, p) or {}
+ for k in pairs(t) do
+ r[k] = f(k)
+ end
+ return r
+end
+util.map_keys = map_keys
+
+local function foldl1(f, t)
+ local v = t[1]
+ for i = 2, t.n or #t do
+ v = f(v, t[i])
+ end
+ return v
+end
+util.foldl1 = foldl1
+
+local function nested_get(t, ...)
+ local arg = pack(...)
+ for i = 1, #arg do
+ if t then t = t[arg[i]] else return nil end
+ end
+ return t
+end
+util.nested_get = nested_get
+
+local function nested_put(v, t, ...)
+ local arg = pack(...)
+ local k = arg[1]
+ for i = 2, arg.n do
+ local u = t[k]
+ if u == nil then u = {}; t[k] = u end
+ t, k = u, arg[i]
+ end
+ t[k] = v
+ return v
+end
+util.nested_put = nested_put
+
+-- Copy all entries of s onto t
+local function update(t, s)
+ if s then
+ for k, v in pairs(s) do
+ t[k] = v
+ end
+ end
+ return t
+end
+util.update = update
+
+-- Return a new table containing all entries of the given tables,
+-- priority given to the latter ones.
+local function merge(...)
+ return foldl1(update, pack({}, ...))
+end
+util.merge = merge
+
+-- Copy entries of the second list to the end of the first.
+local function extend(s, t)
+ return move(t, 1, #t, #s+1, s)
+end
+util.extend = extend
+
+-- Iterate over a table of tables.
+local function triples(t)
+ return co_wrap(
+ function()
+ for i, u in pairs(t) do
+ for j, v in pairs(u) do
+ co_yield(i, j, v)
+ end
+ end
+ end
+ )
+end
+util.triples = triples
+
+--* Cool combinators and friendly functions for LPeg
+
+--** Simple things for better legibility of complicated constructions
+
+local lpeg_add = getmetatable(P(true)).__add
+local lpeg_mul = getmetatable(P(true)).__mul
+local lpeg_pow = getmetatable(P(true)).__pow
+
+local char = P(1) -- in most cases, this works with utf8 too
+local uchar = R("\0\127") + R("\194\244") * R("\128\191")^-3
+local hex = R("09", "AF", "af")
+local alnum = R("09", "AZ", "az")
+local alpha = R("AZ", "az")
+local eol = P("\n")
+
+local function choice(...)
+ return foldl1(lpeg_add, pack(...))
+end
+util.choice = choice
+
+local function sequence(...)
+ return foldl1(lpeg_mul, pack(...))
+end
+util.sequence = sequence
+
+local function many(times, patt)
+ if patt then
+ return lpeg_pow(patt, times)
+ else
+ return lpeg_pow(times, 0)
+ end
+end
+util.many = many
+
+-- Return a function to match against a pattern
+local function matcher(patt)
+ patt = P(patt)
+ return function(s, i) return match(patt, s, i) end
+end
+util.matcher = matcher
+
+-- Return a function to perfom replacement. Like string.gsub, but
+-- partially evaluated
+local function replace(patt, repl, token)
+ token = token and P(token) or char
+ patt = Cs((P(patt) / repl + token)^0)
+ return function(s, i) return match(patt, s, i) end
+end
+util.replace = replace
+
+util.lpeg_escape = replace("%", "%%%%")
+
+--** General purpose combinators
+
+local function search(patt, token)
+ token = token and P(token) or char
+ return P{P(patt) + token * V(1)}
+end
+util.search = search
+
+local function gobble(patt, token)
+ token = token and P(token) or char
+ return (token - P(patt))^0
+end
+util.gobble = gobble
+util.gobble_until = gobble
+
+local function between_balanced(l, r, token) --nicer name?
+ l, r = P(l), P(r)
+ token = token and P(token) or char
+ return P{l * C(((token - l - r) + V(1)/0)^0) * r}
+end
+util.between_balanced = between_balanced
+
+-- function case_fold(str)
+--
+-- Return a pattern that matches the given string, ignoring case.
+-- Uppercase characters in the input still match only uppercase.
+-- Indexing case_fold as a table does the same, but for individual
+-- characters only.
+local case_fold = {}
+for i = strbyte"a", strbyte"z" do
+ local c = strchar(i)
+ case_fold[c] = S(c .. strupper(c))
+end
+local cf_patt = Cf((C(char) / case_fold)^1, lpeg_mul)
+setmetatable(case_fold, {
+ __index = function(_, c) return P(c) end,
+ __call = function(_, s) return match(cf_patt, s) end
+})
+util.case_fold = case_fold
+
+--** String functions, with a tendency towards currying
+
+-- Split a string at the given separators. `nulls` determines wheter
+-- empty sequences are returned.
+local function split(sep, token, nulls)
+ sep = sep and P(sep) or locale_table.space
+ token = token and P(token) or char
+ local patt
+ if nulls then
+ local item = C((token - sep)^0)
+ patt = Ct(item * (sep * item)^0)
+ else
+ patt = Ct((sep^0 * C((token - sep)^1))^0)
+ end
+ return function (s, i) return match(patt, s, i) end
+end
+util.split = split
+
+-- Remove spaces from the ends of subject.
+local function trim(space, token)
+ space = space and P(space) or locale_table.space
+ token = token and P(token) or char
+ local patt = space^0 * C((space^0 * (token - space)^1)^0)
+ return function(s, i) return match(patt, s, i) end
+end
+util.trim = trim
+
+-- Trim and remove repeated spaces inside subject.
+local function clean(space, token)
+ space = space and P(space) or locale_table.space
+ token = token and P(token) or char
+ local patt = space^0 * Cs(((space^1 / " " + true) * (token - space)^1)^0)
+ return function(s, i) return match(patt, s, i) end
+end
+util.clean = clean
+
+-- Return a list of new lines in the subject string.
+util.line_indices = matcher(Ct(I * search(eol * I)^0))
+
+local utf8_sync_patt = R("\128\191")^-3 * I + I
+
+-- Like string.sub, but don't break up UTF-8 codepoints. May return
+-- a string slightly longer or shorter than j - i + 1 bytes.
+local function strsub8(s, i, j)
+ i = match(utf8_sync_patt, s, i)
+ j = j and match(utf8_sync_patt, s, j + 1) - 1
+ return strsub(s, i, j)
+end
+util.strsub8 = strsub8
+
+--** Fuzzy matching
+
+local fuzzy_build_patt = Cf(
+ (C(uchar) / function(c) return search(I * case_fold[c]) end)^1,
+ lpeg_mul)
+
+-- Return a function that fuzzy-matches against a string.
+-- Higher values of the "penalty parameter" p0 reduce the relative
+-- penalty for long gaps. This is a made-up scoring algorithm, there
+-- must be better ones.
+--
+-- Arguments:
+--
+-- - str: the string to match against
+-- - p0: penalty parameter for computing scores, default is 2
+--
+local function fuzzy_matcher(str, p0)
+ if str == "" then return function() return 1 end end
+ p0 = p0 or 2
+ local best_score = #str / (p0 + 1) -- Score of a prefix match
+ local search_patt = Ct(match(fuzzy_build_patt, str))
+ return function(s, i)
+ local score, old_pos, matches = 0, 0, match(search_patt, s, i)
+ if not matches then return end
+ for j = 1, #matches do
+ local pos = matches[j]
+ score = score + 1 / (p0 + pos - old_pos)
+ old_pos = pos
+ end
+ return score / best_score
+ end
+end
+util.fuzzy_matcher = fuzzy_matcher
+
+--** Iterators
+
+local line_patt = I * (search(I * eol) * I + P(true))
+
+local function lines(s, i, n)
+ return co_wrap(function()
+ local n, i, j, k = n or 1, match(line_patt, s, i)
+ while k do
+ co_yield(n, i, j - 1)
+ n, i, j, k = n + 1, match(line_patt, s, k)
+ end
+ co_yield(n, i or 1, #s)
+ end)
+end
+util.lines = lines
+
+--* Classes
+
+local function create_object (c, ...)
+ local obj = setmetatable({}, c)
+ c.__init(obj, ...)
+ return obj
+end
+
+local function class(parent)
+ local c = {}
+ local mt = {
+ __call = create_object,
+ __index = parent
+ }
+ c.__index = c
+ return setmetatable(c, mt)
+end
+util.class = class
+
+--* Memoization
+
+local weak_keys, nil_marker, value_marker = {__mode = "k"}, {}, {}
+
+-- Memoize a function of one argument with one return value. Nil as
+-- an argument is equivalent to false, and nil as return value is not
+-- memoized.
+local function memoize1(fun)
+ local values = setmetatable({}, weak_keys)
+ return function(arg)
+ arg = arg or false
+ local val = values[arg]
+ if val == nil then
+ val = fun(arg)
+ values[arg] = val
+ end
+ return val
+ end
+end
+util.memoize1 = memoize1
+
+-- Return a memoizing version of a function.
+local function memoize(fun)
+ local values = setmetatable({}, weak_keys)
+ return function(...)
+ local arg, val = pack(...), values
+ for i = 1, arg.n do
+ local a = arg[i]
+ if a == nil then a = nil_marker end
+ local v = val[a]
+ if v == nil then
+ v = setmetatable({}, weak_keys)
+ val[a] = v
+ end
+ val = v
+ end
+ local v = val[value_marker]
+ if v == nil then
+ v = pack(fun(...))
+ val[value_marker] = v
+ end
+ return unpack(v, 1, v.n)
+ end
+end
+util.memoize = memoize
+
+--* OS utilities
+
+if package.config:sub(1, 1) == "\\" then
+ util.os_type = "windows"
+else
+ util.os_type = "posix"
+end
+
+local is_command_cmd = util.os_type == "windows"
+ and "WHERE /Q %q"
+ or ">/dev/null command -v %q"
+
+-- Return `name` if an executable with that name exists, nil
+-- otherwise.
+local function is_command(name)
+ local ok = os.execute(format(is_command_cmd, name))
+ return ok and name or nil
+end
+util.is_command = is_command
+
+--* Path and file manipulation
+
+local dir_sep, dir_sep_patt, path_is_abs_patt
+
+if util.os_type == "windows" then
+ dir_sep = "/"
+ dir_sep_patt = S"/\\"
+ path_is_abs_patt = (alpha * P":")^-1 * dir_sep_patt
+else
+ dir_sep = "/"
+ dir_sep_patt = P"/"
+ path_is_abs_patt = dir_sep_patt
+end
+
+-- Concatenate two paths. If the second is absolute, the first one is
+-- ignored.
+local function path_join(p, q)
+ if match(path_is_abs_patt, q) or p == "" then
+ return q
+ else
+ local sep = match(dir_sep_patt, p, #p) and "" or dir_sep
+ return p .. sep .. q
+ end
+end
+util.path_join = path_join
+
+-- Split a path into directory and file parts.
+local path_split_patt = sequence(
+ C(sequence(
+ many(dir_sep_patt),
+ gobble(
+ sequence(
+ dir_sep_patt^0,
+ many(1 - dir_sep_patt),
+ P(-1))))),
+ dir_sep_patt^0,
+ C(many(char))
+)
+
+local function path_split(p)
+ return match(path_split_patt, p)
+end
+util.path_split = path_split
+
+local path_norm_double_sep = Cs(search((dir_sep_patt ^ 2) / dir_sep) * P(1)^0)
+local path_norm_dot_patt = Cs(
+ search(((B(dir_sep_patt) + B(-1)) * P"." * dir_sep_patt) / "") * P(1)^0)
+local path_norm_dotdot_patt = Cs(
+ search(((1 - dir_sep_patt)^1 * dir_sep_patt * P".." * dir_sep_patt) / "") * P(1)^0)
+
+-- Normalize a path name, removing repeated separators and "./" and "../"
+local function path_normalize(p)
+ local q = match(path_norm_double_sep, p)
+ or match(path_norm_dot_patt, p)
+ or match(path_norm_dotdot_patt, p)
+ if q then
+ return path_normalize(q)
+ elseif util.os_type == "windows" then
+ return p:gsub("\\", "/")
+ else
+ return p
+ end
+end
+util.path_normalize = path_normalize
+
+if util.os_type == "windows" then
+ util.path_list_split = split";"
+else
+ util.path_list_split = split":"
+end
+
+local function format_filename_template(template, name)
+ name = gsub(name, "%%", "%%%%")
+ return gsub(template, "?", name)
+end
+util.format_filename_template = format_filename_template
+
+-- Look for a file in several locations. `path` is a string or a list
+-- of strings. The optional `name` is joined to each element.
+-- Returns the first file name that exists on disk. If `read` is
+-- true, give the file contents as second return value.
+local function find_file(path, name, read)
+ if type(path) == "table" then
+ for i = 1, #path do
+ local p, s = find_file(path[i], name, read)
+ if p then return p, s end
+ end
+ return
+ end
+ local file
+ local zipfile = zip and name and zip.open(path)
+ if zipfile then
+ path = path .. "#" .. name
+ file = zipfile:open(name)
+ zipfile:close()
+ else
+ if name then path = path_join(path, name) end
+ file = io.open(path)
+ end
+ if file then
+ local str = read and file:read("*a")
+ file:close()
+ return path, str
+ end
+end
+util.find_file = find_file
+
+--* URI manipulation
+
+local percent_decode = replace(
+ P"%" * C(hex * hex),
+ function(s) return strchar(tonumber(s, 16)) end
+)
+
+local percent_encode = replace(
+ char - (alnum + S"-./:=_~"),
+ function(s) return format("%%%X", strbyte(s)) end
+)
+
+local uri_patt = sequence(
+ C(alpha^1) * P":",
+ (P"//" * C(gobble("/"))) + Cc(nil),
+ C(gobble(S"?#")) / percent_decode,
+ (P"?" * C(gobble("#")) / percent_decode) + Cc(nil),
+ (P"#" * C(char^0) / percent_decode) + Cc(nil)
+)
+util.parse_uri = matcher(uri_patt)
+
+local function make_uri(scheme, authority, path, query, fragment)
+ local t = {}
+ if scheme then t[#t+1] = scheme; t[#t+1] = ":" end
+ if authority then t[#t+1] = "//"; t[#t+1] = authority end
+ if path then t[#t+1] = percent_encode(path) end
+ if query then t[#t+1] = "?"; t[#t+1] = percent_encode(query) end
+ if fragment then t[#t+1] = "#"; t[#t+1] = percent_encode(fragment) end
+ return concat(t)
+end
+util.make_uri = make_uri
+
+--* JSON
+
+util.json_null = setmetatable({}, {__json = "null"})
+
+--** Decoding
+
+do
+ local ws = S" \n\t\r"^0
+ local quote = P"\""
+
+ local function decode_number(s)
+ return tonumber(s) or error("Error parsing “" .. s .. "” as a number")
+ end
+
+ local char_or_escape = choice(
+ P(1) - P"\\",
+ P"\\n" / "\n",
+ P"\\\"" / "\"",
+ P"\\\\" / "\\",
+ P"\\/" / "/",
+ P"\\r" / "\r",
+ P"\\t" / "\t",
+ P"\\b" / "\b",
+ P"\\f" / "\f",
+ P"\\u" * C(S"Dd" * S"89ABab" * hex * hex)
+ * P"\\u" * C(S"Dd" * R("CF","cf") * hex * hex)
+ / function(high, low)
+ high = tonumber(high, 16)
+ low = tonumber(low, 16)
+ return utf8_char(
+ (high - 0xD800) * 2^10 + low - 0xDC00 + 0x10000)
+ end,
+ P"\\u" * C(hex * hex * hex * hex)
+ / function(s) return utf8_char(tonumber(s, 16)) end
+ )
+
+ local json_patt = P{
+ [1] = ws * V"element" * P(-1),
+ ["true"] = P"true" * Cc(true) * ws,
+ ["false"] = P"false" * Cc(false) * ws,
+ null = P"null" * Cc(util.json_null) * ws,
+ number = (R"09" + S"-+.") * (1 - S",]}")^0 / decode_number,
+ string = quote * Cs(gobble(quote, char_or_escape)) * quote * ws,
+ element = V"string" + V"number" + V"true" + V"false" + V"null" + V"array" + V"object",
+ elements = V"element" * (P"," * ws * V"element")^0,
+ array = P"[" * ws * Ct(V"elements"^-1) * P"]" * ws,
+ member = Cg(V"string" * P":" * ws * V"element"),
+ members = V"member" * (P"," * ws * V"member")^0,
+ object = P"{" * ws * Cf(Ct(true) * V"members"^-1, rawset) * P"}" * ws
+ }
+
+ function util.json_decode(str)
+ return match(json_patt, str) or error "Error decoding json"
+ end
+end
+
+--** Encoding
+
+do
+ local control_chars = {}
+ for i = 0, 31 do
+ control_chars[strchar(i)] = format("\\u%04x", i)
+ end
+
+ local encode_string = matcher(
+ Cs(many(choice(
+ P(1) - R"\0\31" - S"\"\\",
+ P"\n" / "\\n",
+ P"\"" / "\\\"",
+ P"\\" / "\\\\",
+ P"\r" / "\\r",
+ P"\t" / "\\t",
+ P"\b" / "\\b",
+ P"\f" / "\\f",
+ R"\0\31" / control_chars
+ ))))
+
+ local fix_decimal
+ local decimal_sep = tostring(5.5):gsub("5", "")
+ if decimal_sep == "." then
+ fix_decimal = function(x) return x end
+ else
+ fix_decimal = replace(decimal_sep, ".")
+ end
+
+ local inf = math.huge
+
+ local function encode_number(v)
+ if -inf < v and v < inf then
+ return fix_decimal(tostring(v))
+ else
+ return "null"
+ end
+ end
+
+ local function do_encode(obj, t, n)
+ local obj_type = type(obj)
+ if obj_type == "string" then
+ t[n] = "\""
+ t[n + 1] = encode_string(obj)
+ t[n + 2] = "\""
+ return n + 3
+ elseif obj_type == "number" then
+ t[n] = encode_number(obj)
+ return n + 1
+ elseif obj_type == "table" then
+ local v = obj[1]
+ if v ~= nil then
+ t[n] = "["
+ n = do_encode(v, t, n + 1)
+ for i = 2, #obj do
+ t[n] = ","
+ n = do_encode(obj[i], t, n + 1)
+ end
+ t[n] = "]"
+ return n + 1
+ end
+ local k, v = next(obj)
+ if k ~= nil then
+ t[n] = "{\""
+ t[n + 1] = encode_string(k)
+ t[n + 2] = "\":"
+ n = do_encode(v, t, n + 3)
+ for k, v in next, obj, k do
+ t[n] = ",\""
+ t[n + 1] = encode_string(k)
+ t[n + 2] = "\":"
+ n = do_encode(v, t, n + 3)
+ end
+ t[n] = "}"
+ return n + 1
+ else
+ local mt = getmetatable(obj)
+ t[n] = mt and mt.__json or "[]"
+ return n + 1
+ end
+ elseif obj_type == "boolean" then
+ t[n] = obj and "true" or "false"
+ return n + 1
+ else
+ error("Error encoding json, found object of type " .. type)
+ end
+ end
+
+ function util.json_encode(obj)
+ local t = {}
+ do_encode(obj, t, 1)
+ return concat(t)
+ end
+end
+
+--* Inspect and serialize Lua values
+
+-- function inspect(obj, depth)
+--
+-- Return a string representation of `obj`, with at most `depth`
+-- layers of nested tables. If `obj` consists solely of scalars,
+-- strings and tables and does not exceed the maximum nesting, the
+-- return value is valid Lua code.
+--
+local inspect
+
+local is_lua_identifier = util.matcher(
+ C(R("AZ", "az", "__") * R("09", "AZ", "az", "__")^0) * P(-1)
+ / function(s) return load("local " .. s) and s end
+)
+
+local function lua_encode_key(obj)
+ if type(obj) == "string" then
+ if is_lua_identifier(obj) then
+ return obj
+ else
+ return "[" .. format("%q", obj) .. "]"
+ end
+ else
+ return "[" .. tostring(obj) .. "]"
+ end
+end
+
+local function lua_encode_string(obj)
+ if strfind(obj, "\n", 1, true) then
+ local delim = ""
+ while strfind(obj, "]" .. delim .. "]", 1, true) do
+ delim = delim .. "="
+ end
+ return "[" .. delim .. "[\n" .. obj .. "]" .. delim .. "]"
+ else
+ return format("%q", obj)
+ end
+end
+
+local function lua_encode_table(obj, depth, d)
+ local t = {}
+ local array_keys, hash_keys = {}, {}
+ for i = 1, #obj do
+ array_keys[i] = true
+ t[#t+1] = inspect(obj[i], depth, d+1)
+ end
+ for k in pairs(obj) do
+ if not array_keys[k] then
+ hash_keys[#hash_keys+1] = k
+ end
+ end
+ sort(
+ hash_keys,
+ function(v, w) return tostring(v) < tostring(w) end
+ )
+ for i = 1, #hash_keys do
+ local k = hash_keys[i]
+ local v = obj[k]
+ t[#t+1] = lua_encode_key(k) .. " = " .. inspect(v, depth, d+1)
+ end
+ local short = concat(t, ", ")
+ if 2*d + #short > 70 or strfind(short, "\n", 1, true) then
+ local sep = strrep(" ", d)
+ return "{\n " .. sep .. concat(t, ",\n " .. sep) .. "\n" .. sep .. "}"
+ else
+ return "{" .. short .. "}"
+ end
+end
+
+inspect = function(obj, depth, d)
+ depth, d = depth or 10, d or 0
+ if type(obj) == "table" and d < depth then
+ return lua_encode_table(obj, depth, d)
+ elseif type(obj) == "string" then
+ if d < depth or #obj < 20 then
+ return lua_encode_string(obj)
+ else
+ return format("string: \"%s...\"", strsub(obj, 1, 9))
+ end
+ else
+ return tostring(obj)
+ end
+end
+util.inspect = inspect
+
+--* Logging
+
+local function log(msg, ...)
+ if select("#", ...) > 0 then msg = format(msg, ...) end
+ io.stderr:write(os.date("%H:%M:%S "), msg, "\n")
+ io.stderr:flush()
+end
+util.log = log
+
+local function log_objects(...)
+ return log(concat(map(inspect, {...}), "\n"))
+end
+util.log_objects = log_objects
+
+return util