diff options
Diffstat (limited to 'support/digestif/digestif')
-rw-r--r-- | support/digestif/digestif/Cache.lua | 129 | ||||
-rw-r--r-- | support/digestif/digestif/Manuscript.lua | 1523 | ||||
-rw-r--r-- | support/digestif/digestif/ManuscriptBibTeX.lua | 38 | ||||
-rw-r--r-- | support/digestif/digestif/ManuscriptConTeXt.lua | 604 | ||||
-rw-r--r-- | support/digestif/digestif/ManuscriptDoctex.lua | 19 | ||||
-rw-r--r-- | support/digestif/digestif/ManuscriptLaTeX.lua | 360 | ||||
-rw-r--r-- | support/digestif/digestif/ManuscriptLatexProg.lua | 12 | ||||
-rw-r--r-- | support/digestif/digestif/ManuscriptPlainTeX.lua | 56 | ||||
-rw-r--r-- | support/digestif/digestif/ManuscriptTexinfo.lua | 25 | ||||
-rw-r--r-- | support/digestif/digestif/Parser.lua | 227 | ||||
-rw-r--r-- | support/digestif/digestif/Schema.lua | 189 | ||||
-rw-r--r-- | support/digestif/digestif/bibtex.lua | 271 | ||||
-rw-r--r-- | support/digestif/digestif/config.lua | 156 | ||||
-rw-r--r-- | support/digestif/digestif/data.lua | 384 | ||||
-rw-r--r-- | support/digestif/digestif/langserver.lua | 567 | ||||
-rw-r--r-- | support/digestif/digestif/util.lua | 789 |
16 files changed, 5349 insertions, 0 deletions
diff --git a/support/digestif/digestif/Cache.lua b/support/digestif/digestif/Cache.lua new file mode 100644 index 0000000000..d7e3e16f85 --- /dev/null +++ b/support/digestif/digestif/Cache.lua @@ -0,0 +1,129 @@ +-- A class to store transient file contents and manage the creation of +-- Manuscript objects. + +local Manuscript = require "digestif.Manuscript" +local lpeg = require "lpeg" +local util = require "digestif.util" + +local S, C = lpeg.S, lpeg.C +local sequence, gobble_until = util.sequence, util.gobble_until +local path_split, path_join = util.path_split, util.path_join +local path_normalize = util.path_normalize +local weak_values = {__mode = "v"} +local io_open = io.open + +local Cache = util.class() + +function Cache:__init(tbl) + self.store = setmetatable({}, weak_values) + self.persist = {} -- a place to keep an extra reference to things + -- that shouldn't vanish + if tbl then + for name, src in pairs(tbl) do + self:put(name, src) + end + end +end + +-- Set the contents of a file in the cache. +function Cache:put(filename, str) + local props = self.store[filename] + if not props then + props = {} + self.store[filename] = props + else + props.rootname = nil + end + props.contents = str + self.persist[filename] = props +end + +-- Return the contents of a file, as stored with the put method, or +-- attempt to read the file from disk. If neither succeeds, return +-- nil. The second return value is a cookie. Values read from disk +-- will be garbage collected when no reference to the cookie exists. +-- Values stored by the put method are not subject to garbage +-- collection until an explicit call to Cache.forget. +-- +function Cache:__call(filename) + local props = self.store[filename] + if not props then + local file, str = io_open(filename), nil + if file then + str = file:read("*all") + file:close() + end + props = {contents = str} + self.store[filename] = props + end + return props.contents, props +end + +-- Remove a file from the cache. +function Cache:forget(filename) + self.store[filename] = nil + self.persist[filename] = nil +end + +local space = S" \t\r" +local magic_comment_patt = sequence( + (space^0 * "%")^1, + space^0 * "!" * util.case_fold("tex"), + space^1 * "root", + space^0 * "=", + space^0 * C(gobble_until(space^0 * "\n"))) +local search_magic_comment_patt = util.choice( + magic_comment_patt, + util.search("\n" * magic_comment_patt)) + +-- Determine the root path of a document from magic comments. +function Cache:rootname(filename) + local _, cookie = self(filename) -- warm up + local props = assert(self.store[filename]) + local rootname = props.rootname + if rootname == nil then + local src = self(filename) + local val = search_magic_comment_patt:match(src:sub(1, 1000)) + if val then + rootname = path_normalize(path_join(path_split(filename), val)) + else + rootname = false + end + props.rootname = rootname + end + return rootname +end + +-- Produce a Manuscript object. This method ensures that the result +-- belongs to a tree of Manuscript objects representing a multi-file +-- TeX document. If the rootname argument is omitted, try to infer it +-- using magic comments. +-- +-- This method reuses as much information as possible from the cache. +-- +-- Arguments: +-- filename (string): the path of the manuscript +-- format (string): the TeX format +-- rootname (string, optional): the root path of the TeX document +-- +-- Returns: +-- a Manuscript object +-- +function Cache:manuscript(filename, format, rootname) + rootname = rootname or self:rootname(filename) or filename + local root = self.store[rootname] and self.store[rootname][format] + if not root or not root:is_current() then + root = Manuscript{ + filename = rootname, + format = format, + files = self + } + self.store[rootname][format] = root + end + local script = root:find_manuscript(filename) + or self:manuscript(filename, format, filename) -- root does not refer back + self.store[filename][format] = script + return script +end + +return Cache diff --git a/support/digestif/digestif/Manuscript.lua b/support/digestif/digestif/Manuscript.lua new file mode 100644 index 0000000000..d9101d8c07 --- /dev/null +++ b/support/digestif/digestif/Manuscript.lua @@ -0,0 +1,1523 @@ +-- Manuscript class + +local config = require "digestif.config" +local util = require "digestif.util" + +local require_data = require "digestif.data".require +local get_info = require "digestif.data".get_info +local resolve_doc_items = require "digestif.data".resolve_doc_items +local path_join, path_split = util.path_join, util.path_split +local path_normalize = util.path_normalize +local find_file = util.find_file +local format_filename_template = util.format_filename_template + +local format = string.format +local co_wrap, co_yield = coroutine.wrap, coroutine.yield +local concat, sort = table.concat, table.sort +local infty, min = math.huge, math.min +local utf8_len, utf8_offset = utf8.len, utf8.offset +local nested_get = util.nested_get +local map_keys, update, extend = util.map_keys, util.update, util.extend +local line_indices = util.line_indices +local matcher, fuzzy_matcher = util.matcher, util.fuzzy_matcher + +local Manuscript = util.class() + +--* Constructor + +-- Only descendants of this class (representing various TeX formats) +-- are ever instantiated. So we replace the constructor by a function +-- defering to the subclass indicated by the format field of the +-- argument. + +local formats = { + ["bibtex"] = "digestif.ManuscriptBibTeX", + ["context"] = "digestif.ManuscriptConTeXt", + ["latex"] = "digestif.ManuscriptLaTeX", + ["latex-prog"] = "digestif.ManuscriptLatexProg", + ["doctex"] = "digestif.ManuscriptDoctex", + ["plain"] = "digestif.ManuscriptPlainTeX", + ["texinfo"] = "digestif.ManuscriptTexinfo" +} + +local function ManuscriptFactory(_, args) + local fmt = args.format + or args.parent and args.parent.format + or error "TeX format not specified" + return require(formats[fmt])(args) +end +getmetatable(Manuscript).__call = ManuscriptFactory + +local function infer_format(path) + local ext = path:sub(-4) + if ext == ".bib" then + return "bibtex" + elseif ext == ".sty" or ext == ".cls" then + return "latex-prog" + end +end + +-- Create a new manuscript object. The argument is a table with the +-- following keys: +-- +-- filename: the manuscript's file name +-- files: a function that returns file contents +-- parent: a parent manuscript object (optional) +-- format: the TeX format ("latex", "plain", etc.). This is +-- actually only used by ManuscriptFactory +-- +function Manuscript:__init(args) + local filename, parent, files = args.filename, args.parent, args.files + self.filename = filename + self.parent = parent + self.files = files + self.root = parent and parent.root or self + local src, cache_cookie = files(filename) + self.src, self.cache_cookie = src or "", cache_cookie + self.lines = line_indices(self.src) + local super = parent or self.__index + self.packages = setmetatable({}, {__index = super.packages}) + self.commands = setmetatable({}, {__index = super.commands}) + self.environments = setmetatable({}, {__index = super.environments}) + -- We eagerly initialize most indexes here because the overhead of + -- going through the document is substantial. Only the indices used + -- by Manuscript:find_references are computed on demand. + self._children = {} + self.bib_index = {} + self.child_index = {} + self.section_index = {} + self.label_index = {} + if self.init_callbacks then + self:scan(self.init_callbacks) + end +end + +function Manuscript:is_current() + local current_src, cookie = self.files(self.filename) + return (self.src == (current_src or "")) and (cookie or true) +end + +-- Return a child manuscript, provided its name appear in the +-- child_index. It inherits this manuscript's `files` function. +-- Memoization is used to make this efficient. +function Manuscript:child(name) + local child = self._children[name] + local is_current = child and child:is_current() + if not is_current then + if not child then + local ancestor = self + while ancestor do + if ancestor.filename == name then return end + ancestor = ancestor.parent + end + end + child = Manuscript{ + filename = name, + parent = self, + format = infer_format(name), + files = self.files + } + self._children[name] = child + end + return child +end + +--* Substrings + +-- Get a substring of the manuscript. The argument can be a pair of +-- integers (inclusive indices, as in the Lua convention) or a table +-- with fields pos (inclusive) and cont (exclusive). +function Manuscript:substring(i, j) + if not i then return nil end + if type(i) == "table" then + j = i.cont - 1 + i = i.pos + end + return self.src:sub(i, j) +end + +-- Get a substring of the manuscript, trimmed. The argument follows +-- the same convention as Manuscript:substring. +function Manuscript:substring_trimmed(i, j) + return self.parser.trim(self:substring(i,j)) +end + +-- Get a substring of the manuscript, trimmed and without comments. +-- The argument follows the same convention as Manuscript:substring. +function Manuscript:substring_stripped(i, j) + local parser = self.parser + return parser.trim(parser.strip_comments(self:substring(i,j))) +end + +-- Get a substring of the manuscript, without commments and reduced to +-- one line. The argument follows the same convention as +-- Manuscript:substring. +function Manuscript:substring_clean(i, j) + local parser = self.parser + return parser.clean(parser.strip_comments(self:substring(i,j))) +end + +--* Parsing commands, lists, and key-value lists. + +-- Parse a key-value list in a given manuscript range. The argument +-- is a table with fields pos and cont. Returns a list of ranges, +-- with additional fields "key" and "value" (if present). These are +-- range tables as well. +function Manuscript:parse_kvlist(range) + local s = self:substring(1, range.cont - 1) -- TODO: avoid creating a new string + return self.parser.parse_kvlist(s, range.pos) +end + +-- Read the contents of a key-value list in the manuscript. The +-- argument is a table with fields pos and cont. Returns a nested +-- list of tables with fields "key" and "value" (if present), +-- containing the corresponding text in the source. +-- +-- TODO: add substring method as parameter +function Manuscript:read_keys(range) + local tbl = self:parse_kvlist(range) + local r = {} + for i, v in ipairs(tbl) do + r[i] = { + key = self:substring_trimmed(v.key), + value = v.value and self:substring_trimmed(v.value) + } + end + return r +end + +-- Read the contents of a list in the manuscript. Returns a list of +-- strings. +function Manuscript:read_list(i, j) + local parser = self.parser + local s = self:substring(i, j) + return parser.read_list(s) +end + +-- Parse the arguments of a command. +-- +-- Arguments: +-- pos (number): A position in the source. +-- cs (string, optional): The command name. If omitted, it is read +-- from the manuscript. +-- +-- Returns: +-- A list of ranges. +-- +function Manuscript:parse_command(pos, cs) + local parser = self.parser + if not cs then + cs = parser.csname:match(self.src, pos) or "" + end + local cmd = self.commands[cs] + local args = cmd and cmd.arguments + local cont = 1 + pos + #cs + if args then + local val = parser.parse_args(args, self.src, cont) + val.pos = pos + return val + else + return {pos = pos, cont = cont} + end +end + +--* Find line numbers, paragraphs, etc. + +-- Find the line number (and its location) of a given position. +-- Returns the line number (1-based) and that line's starting +-- position. +function Manuscript:line_number_at(pos) + local len = #self.src + local lines = self.lines + local j, l = 1, #lines -- top and bottom bounds for line search + if pos < 1 then pos = len + pos + 1 end + while pos < lines[l] do + local k = (j + l + 1) // 2 + if pos < lines[k] then + l = k - 1 + else + j = k + end + end -- now l = correct line, 1-based indexing + return l, lines[l] +end + +-- Compute the line and column number (both 1-based) at the given +-- position. +-- +-- TODO: make len function a parameter +function Manuscript:line_column_at(pos) + -- Out ranges are excluside on the right, Lua is inclusive, so we + -- may have pos == 1 + #self.src. + pos = min(pos, #self.src) + local l, line_pos = self:line_number_at(pos) + local c = utf8_len(self.src, line_pos, pos) or error("Invalid UTF-8") + return l, c +end + +-- Compute the source position at the given line an column. +-- +-- TODO: make offset function a parameter +function Manuscript:position_at(line, col) + local line_pos = self.lines[line] or error("Position out of bounds") + return utf8_offset(self.src, col, line_pos) or error("Position out of bounds") +end + + +-- Find beginning of the paragraph containing the given position. +function Manuscript:find_par(pos) + local src = self.src + local lines = self.lines + local l = self:line_number_at(pos) + for k = l, 1, -1 do + if self.parser.is_blank_line(src, lines[k]) then + return lines[k] + end + end + return 1 +end + +local preceding_command_callbacks = {} + +function preceding_command_callbacks.cs(self, pos, cs, end_pos) + if pos > end_pos then return nil end + local r = self:parse_command(pos, cs) + if r.cont <= end_pos then + local next_pos = self.parser.next_nonblank(self.src, r.cont) + if next_pos == end_pos then + return nil, pos, cs, r + end + end + return r.cont, end_pos +end + +-- Find the preceding command, if any. If there is a command whose +-- arguments end right before the given position, returns the +-- position, command name, and parse data of the preceding command. +function Manuscript:find_preceding_command(pos) + local par_pos = self:find_par(pos) + return self:scan(preceding_command_callbacks, par_pos, pos) +end + +--* Indexes and document traversal + +function Manuscript:get_index(name) + name = name .. "_index" + local idx = self[name] + if not idx then + idx = {} + self[name] = idx + end + return idx +end + +function Manuscript:index_pairs(name) + return ipairs(self:get_index(name)) +end + +local function compare_pos(x, y) + return x.pos < y.pos +end + +local function traverse_indexes(script, indexes, max_depth) + local items = {} + for i = 1, #indexes do + local idx_name = indexes[i] + local idx = script[idx_name] + for j = 1, (idx and #idx or 0) do + items[#items+1] = idx[j] + items[idx[j]] = idx_name + end + end + if max_depth > 0 then + local child_index = script.child_index + for i = 1, #child_index do + local item = child_index[i] + local child = script:child(item.name) + if child then + items[#items+1] = { + pos = item.pos, + manuscript = child + } + end + end + end + sort(items, compare_pos) + for i = 1, #items do + local item = items[i] + local kind = items[item] + if kind then + co_yield(item, kind) + else + traverse_indexes(item.manuscript, indexes, max_depth - 1) + end + end +end + +-- Iterator to transverse an index documentwise. This recursevely +-- iterates over entries of the given indexes on self and its +-- children, depth first. An index is a Manuscript field consisting a +-- list of tables containing an entry "pos". At each iteration, +-- yields one index entry and the name of the index to which it +-- belongs. +-- +-- Arguments: +-- indexes: is the name of an index, as a string, or a list of +-- such. +-- max_depth: optional, leave at the default for the recursive +-- behavior or set to 0 to disable it. +-- +function Manuscript:traverse(indexes, max_depth) + if type(indexes) == "string" then indexes = {indexes} end + return co_wrap(function () + return traverse_indexes(self, indexes, max_depth or 15) + end) +end + + +local function argument_items(script, sel, pos, cs) + local args = script.commands[cs].arguments + if not args then return end + local i + if type(sel) == "number" then + i = sel + elseif type(sel) == "string" then + for j = 1, #args do + if args[j].meta == sel then i = j; break end + end + else + i = sel(args) + end + if not i then return end + local ranges = script:parse_command(pos, cs) + local range = ranges[i] + if not range or range.omitted then return end + local arg = args[i] + if arg.list then + local items = script:parse_kvlist(range) + for j = 1, #items do + co_yield(items[j]) + end + else + co_yield(range) + end +end + +-- Iterator to look at arguments of a command. Yields the range of +-- the relevant argument (if present), or succesive ranges +-- corresponding to the argument's subitems, if the argument's `list` +-- property is true. +-- +-- Arguments: +-- sel: determines which argument to look for; it it's a string, the +-- first argument with that meta property is used; otherwise, sel +-- should be a function that takes an `arguments` table and return +-- the relevant index. +-- pos: the position of the command to analyze +-- cs: optional, the name of the command +-- +function Manuscript:argument_items(sel, pos, cs) + return co_wrap(function() return argument_items(self, sel, pos, cs) end) +end + + +--* Manuscript scanning + +-- Scan the Manuscript, executing callbacks for each document element. +-- +-- Each callback is a function taking at least two arguments (a +-- Manuscript object and a source position) and returns at least one +-- value, a position to continue scanning or nil to interrupt the +-- process. When this happens, scan function returns the remaining +-- return values of the callback. The remaining arguments and return +-- values of a callback can be used to keep track of an internal +-- state. +-- +-- The callbacks argument is a table. Its keys correspond to either +-- the "action" field of a command, or the "type" field of an item +-- found by the parser ("cs", "mathshift" or "par"). +-- +function Manuscript:scan(callbacks, pos, ...) + local patt = self.parser.scan_patt(callbacks) + local match = patt.match + local commands = self.commands + local src = self.src + local function scan(pos0, ...) + if not pos0 then return ... end + local pos1, kind, detail, pos2 = match(patt, src, pos0) + local cmd = (kind == "cs") and commands[detail] + local cb = cmd and callbacks[cmd.action] or callbacks[kind] + if cb then + return scan(cb(self, pos1, detail, ...)) + else + return scan(pos2, ...) + end + end + return scan(pos or 1, ...) +end + +-- Copy entries from t to s, but only if not already present. +local function copy_new(s, t) + for k, v in pairs(t) do + if not s[k] then s[k] = v end + end +end + +-- Adds a package to the manuscript. This entails copying the command +-- and environment definitions from the package tags to the +-- manuscript. Returns true if package is (or already was) present, +-- nil if the package tags weren't not found. +function Manuscript:add_package(name) + if self.packages[name] then return true end + local pkg = require_data(name) + if not pkg then return end + self.packages[name] = pkg + local deps = pkg.dependencies or pkg.package and pkg.package.dependencies -- TODO: use only the former case + if deps then + for _, n in ipairs(deps) do + self:add_package(n) + end + end + -- Don't overwrite stuff from generated data files + local update_fn = pkg.generated and copy_new or update + if pkg.commands then + update_fn(self.commands, pkg.commands) + end + if pkg.environments then + update_fn(self.environments, pkg.environments) + end + return true +end + +function Manuscript:find_manuscript(filename) + if self.filename == filename then return self end + local idx = self.child_index + for i = 1, #idx do + local script = self:child(idx[i].name) + local found = script and script:find_manuscript(filename) + if found then return found end + end + return nil +end + +--* Getting the local context + +-- The context at a given manuscript postion is described by a linked +-- list list of ranges, starting from the innermost, with additional +-- annotations. More specifically, the following fields are possible +-- in a context description table: +-- +-- pos: the starting position +-- cont: the ending postiion (exclusive) +-- parent: a similar table, with range including the current range +-- cs: if present, indicates that this range correspond to a command +-- and its arguments. The value of this field is the command name. +-- env: if present, indicates that this range correspond to an +-- environment beginning and its arguments. The value of this field +-- is the environment name. +-- arg: if present, indicates that this range is a command argument. +-- The value corresponds to the argument number. The parent is of +-- "cs" or "env" type. +-- key: if present, indicates this range is one item in a plain list +-- or key-value list. The value of this field is the key text. +-- value: if present, indicates this range is the value of a key in a +-- key-value list. the parent is of "key" type. +-- data: in each case above, data contains the corresponding data +-- object, for instance the command description. +-- + +-- Scan the current paragraph, returning the context around the given +-- position. +function Manuscript:get_context(pos) + return self:scan(self.context_callbacks, self:find_par(pos), nil, pos) +end + +local function local_scan_parse_keys(m, context, pos) + local items = m:parse_kvlist(context) + for _, it in ipairs(items) do -- are we inside a key/list item? + if it.pos and it.pos <= pos and pos <= it.cont then + local key = m:substring_trimmed(it.key) + context = { + key = key, + data = nested_get(context.data, "keys", key), -- or fetch context-dependent keys, say on a usepackage + pos = it.pos, + cont = it.cont, + parent = context + } + local v = it.value + if v and v.pos and v.pos <= pos and pos <= v.cont then -- are we inside the value of a key? + local value = m:substring_trimmed(v) + context = { + value = value, + data = nested_get(context.data, "values", value), -- what if "value" is command-like? + pos = v.pos, + cont = v.cont, + parent = context + } + end + break + end + end + return context +end + +local function local_scan_parse_list(m, context, pos) + local items = m:parse_kvlist(context) + for i = 1, #items do -- are we inside a key/list item? + local it = items[i] + if it.pos and it.pos <= pos and pos <= it.cont then + context = { + item = i, + pos = it.pos, + cont = it.cont, + parent = context + } + end + end + return context +end + +Manuscript.context_callbacks = {} + +function Manuscript.context_callbacks.cs(self, pos, cs, context, end_pos) + if pos > end_pos then return nil, context end -- stop parse + local r = self:parse_command(pos, cs) + if end_pos <= r.cont then + context = { + cs = cs, + data = self.commands[cs], + pos = pos, + cont = r.cont, + parent = context + } + elseif cs == "begin" then + local env_name = self:substring(r[1]) + local env = self.environments[env_name] + local args = env and env.arguments + if not args then return r.cont, context, end_pos end + local q = self.parser.parse_args(args, self.src, r.cont) + if q.cont < end_pos then + return q.cont, context, end_pos -- end_pos is after current thing + end + context = { + env = env_name, + data = self.environments[env_name], + pos = pos, + cont = q.cont, + parent = context + } + r = q + else -- pos is after current thing + return r.cont, context, end_pos + end + + for i, arg in ipairs(r) do -- are we inside an argument? + if arg.pos and arg.pos <= end_pos and end_pos <= arg.cont then + local data = nested_get(context.data, "arguments", i) + context = { + arg = i, + data = data, + pos = arg.pos, + cont = arg.cont, + parent = context + } + if data and data.keys then + context = local_scan_parse_keys(self, context, end_pos) + elseif data and data.list then + context = local_scan_parse_list(self, context, end_pos) + end + return context.pos, context, end_pos + end + end + return nil, context -- stop parse +end + +function Manuscript.context_callbacks.tikzpath(m, pos, cs, context, end_pos) + if pos > end_pos then return nil, context end -- stop parse + local r = m:parse_command(pos, cs) + if end_pos <= r.cont then + context = { + cs = cs, + data = m.commands[cs], + pos = pos, + cont = r.cont, + parent = context + } + local p = r[1].pos + while p <= end_pos do + local q = m.parser.skip_to_bracketed(m.src, p) + if q and q.pos <= end_pos and end_pos <= q.cont then + context = { + arg = true, + data = {keys = require_data"tikz".keys.tikz}, + pos = q.pos, + cont = q.cont, + parent = context + } + context = local_scan_parse_keys(m, context, end_pos) + end + p = q and q.cont or infty + end + end + return r.cont, context, end_pos +end + +function Manuscript.context_callbacks.par (_, _, _, context) + return nil, context +end + +--* Snippets and pretty-printing commands + +-- Pretty-print an argument list. +-- +-- Arguments: +-- args: A list of argument descriptors +-- before: A piece of text inserted at the beginning of the +-- formatted argument list. +-- Returns: +-- The formatted argument list (as a string) and a list of numbers +-- (of length twice that of args) indicating the positions of each +-- argument within that string. +-- +function Manuscript:signature_args(args, before) + if not args then return before or "", {} end + local t, p, pos = {before}, {}, 1 + (before and #before or 0) + for i = 1, #args do + local arg, l, r = args[i] + if arg.literal then + l, r = "", "" + elseif arg.delimiters == false then + l, r = "‹", "›" + elseif arg.delimiters then + l, r = arg.delimiters[1], arg.delimiters[2] + if r == "\n" then r = "⤶" end + if l == "" then l, r = "‹", "›" .. r end + else + l, r = "{", "}" + end + local text = arg.literal or arg.meta or "#" .. i + t[#t+1] = l; pos = pos + #l; p[#p+1] = pos + t[#t+1] = text; pos = pos + #text; p[#p+1] = pos + t[#t+1] = r; pos = pos + #r + end + return concat(t), p +end + +-- Pretty-print a command signature. +-- +-- Arguments: +-- cs: The command name. +-- args: An argument list, or nil. +-- +function Manuscript:signature_cmd(cs, args) + return self:signature_args(args, "\\" .. cs) +end + +-- This is for plain TeX. Other formats should override this +-- definition. +Manuscript.signature_env = Manuscript.signature_cmd + +-- Make a snippet fragment from an argument list. +-- +-- Arguments: +-- args: An argument list +-- i (optional, default 1): The number of the first placeholder in +-- the snippet. +-- +-- Returns: +-- The formatted snippet, as a string, without the $0 placeholder. +-- +function Manuscript:snippet_args(args, i) + if not args then return "" end + i = i or 1 + local t = {} + for _, arg in ipairs(args) do + if arg.optional then + t[#t+1] = "${" .. i .. ":" + i = i + 1 + end + if arg.literal then + t[#t+1] = arg.literal + else + local delims, l, r = arg.delimiters + local meta = arg.meta + if delims then + l, r = delims[1], delims[2] + if l == "" then meta = "‹" .. meta .. "›" end + elseif delims == false then + l, r = "", "" + meta = "‹" .. meta .. "›" + else -- delims == nil + l, r = "{", "}" + end + t[#t+1] = l .. "${" .. i .. (meta and ":" .. meta or "") .. "}" .. r + i = i + 1 + end + if arg.optional then t[#t+1] = "}" end + end + return concat(t) +end + +-- Make a snippet for a command. +function Manuscript:snippet_cmd(cs, args) + local argsnippet = args and self:snippet_args(args) or "" + return cs .. argsnippet .. "$0" +end + +-- Make a snippet for an environment. +-- +-- This is the plain TeX version. It's intended to be overwritten by +-- other classes. +-- +function Manuscript:snippet_env(cs, args) + local argsnippet = args and self:snippet_args(args) or "" + return cs .. argsnippet .. "\n\t$0\n\\end" .. cs +end + + +--* Completion + +-- Calculate completions for the manuscript at the given position. +-- Returns a a table containing a list of completion items (at +-- numerical indices) and some addition information in the following +-- fields. +-- +-- pos: position where the matched text starts +-- prefix: the matched text +-- kind: whether the completions are for a command, a key, etc. +-- +function Manuscript:complete(pos) + local ctx = self:get_context(pos - 1) + if ctx == nil then + return + elseif ctx.cs and pos == ctx.cont then + return self.completion_handlers.cs(self, ctx) + elseif ctx.arg then + local action = nested_get(ctx, "parent", "data", "action") + local handler = self.completion_handlers[action] + return handler and handler(self, ctx, pos) + elseif ctx.item then + local action = nested_get(ctx, "parent", "parent", "data", "action") + local handler = self.completion_handlers[action] + return handler and handler(self, ctx, pos) + elseif ctx.key then --and pos == ctx.pos + #ctx.key then + return self.completion_handlers.key(self, ctx, pos) + elseif ctx.value and pos == ctx.pos + #ctx.value then + return self.completion_handlers.value(self, ctx, pos) + end +end + +Manuscript.completion_handlers = {} + +function Manuscript.completion_handlers.cs(self, ctx) + local commands, environments = self.commands, self.environments + local extra_snippets = config.extra_snippets + local prefix = ctx.cs + local ret = { + pos = ctx.pos + 1, + prefix = prefix, + kind = "command" + } + for cs in pairs(map_keys(self.parser.cs_matcher(prefix), commands)) do + local cmd = commands[cs] + local args = cmd.arguments + local snippet = extra_snippets[cs] or cmd.snippet + ret[#ret+1] = { + text = cs, + summary = cmd.summary, + annotation = args and self:signature_args(args) or cmd.symbol, + snippet = snippet or args and self:snippet_cmd(cs, args) + } + end + for env in pairs(map_keys(matcher(prefix), environments)) do + local cmd = environments[env] + local args = cmd.arguments + local snippet = extra_snippets[env] or cmd.snippet + local annotation = args and self:signature_args(args) + ret[#ret+1] = { + text = env, + summary = cmd.summary, + annotation = (annotation and annotation .. " " or "") .. "(environment)", + snippet = snippet or self:snippet_env(env, args) + } + end + table.sort(ret, function(x,y) return x.text < y.text end) + return ret +end + +function Manuscript.completion_handlers.key(self, ctx, pos) + local prefix = self:substring(ctx.pos, pos - 1) + local len = #prefix + local r = { + pos = ctx.pos, + prefix = prefix, + kind = "key" + } + local keys = ctx.parent and ctx.parent.data and ctx.parent.data.keys + for text, key in pairs(keys or {}) do + if prefix == text:sub(1, len) then + r[#r+1] = { + text = text, + summary = key.summary, + annotation = key.meta and ("=" .. key.meta) + } + end + end + table.sort(r, function(x,y) return x.text < y.text end) + return r +end + +function Manuscript.completion_handlers.value(self, ctx, pos) + local prefix = self:substring(ctx.pos, pos - 1) + local len = #prefix + local r = { + pos = ctx.pos, + prefix = prefix, + kind = "value" + } + local values = ctx.parent and ctx.parent.data and ctx.parent.data.values + for text, value in pairs(values or {}) do + if prefix == text:sub(1, len) then + r[#r+1] = { + text = text, + summary = value.summary + } + end + end + return r +end + +function Manuscript.completion_handlers.begin(self, ctx, pos) + local environments = self.environments + local prefix = self:substring(ctx.pos, pos - 1) + local has_prefix = matcher(prefix) + local r = { + pos = ctx.pos, + prefix = prefix, + kind = "environment" + } + for env in pairs(map_keys(has_prefix, environments)) do + local cmd = environments[env] + r[#r+1] = { + text = env, + summary = cmd.summary, + } + end + return r +end + +Manuscript.completion_handlers["end"] = Manuscript.completion_handlers.begin + +-- Get a short piece of text around a label. If there is a recognized +-- command ending right before the label, the context starts there. +-- +-- TODO: For now, the context is 60 bytes, but it should be smart and +-- choose a lenght close to 100 characters but ending at a line end. +-- It should also be Unicode-safe. +-- +function Manuscript:label_context_short(item) + local pos, cs, _ = self:find_preceding_command(item.outer_pos) + local cmd = self.commands[cs] + if not cmd then + pos = self.parser.next_nonblank(self.src, item.outer_pos) + end + return self:substring_clean(pos, pos + 60) +end + +function Manuscript.completion_handlers.ref(self, ctx, pos) + local prefix = self:substring(ctx.pos, pos - 1) + local has_prefix = matcher(prefix) + local fuzzy_match = config.fuzzy_ref and fuzzy_matcher(prefix) + local scores = {} + local r = { + pos = ctx.pos, + prefix = prefix, + kind = "label" + } + for label in self.root:traverse "label_index" do + local short_ctx = label.manuscript:label_context_short(label) + local score = has_prefix(label.name) and infty + or fuzzy_match and fuzzy_match(short_ctx) + if score then + r[#r+1] = { + text = label.name, + annotation = short_ctx, + summary = label.manuscript:label_context_long(label), + fuzzy_score = score < infty and score or nil + } + scores[r[#r]] = score + end + end + -- sort exact matches by order in the document, fuzzy ones by score + sort(r, function(a, b) return scores[a] > scores[b] end) + return r +end + +function Manuscript.completion_handlers.cite(self, ctx, pos) + if nested_get(ctx, "data", "optional") then return end + local prefix = self:substring(ctx.pos, pos - 1) + local r = { + pos = ctx.pos, + prefix = prefix, + kind = "bibitem" + } + local scores = {} + local has_prefix = matcher(prefix) + local fuzzy_match = config.fuzzy_cite and fuzzy_matcher(prefix) + for item in self.root:traverse "bib_index" do + local score = has_prefix(item.name) and infty + or fuzzy_match and item.text and fuzzy_match(item.text) + if score then + scores[item.name] = score + r[#r+1] = { + text = item.name, + annotation = item.text, + fuzzy_score = score < infty and score or nil + } + end + end + -- sort exact matches by label, fuzzy matches by score + local cmp = function(a, b) + local na, nb = a.text, b.text + local sa, sb = scores[na], scores[nb] + if sa == sb then + return (na < nb) + else + return (sa > sb) + end + end + sort(r, cmp) + return r +end + +--* Context help + +-- Get information about the thing at the given position. +function Manuscript:describe(pos) + local ctx = self:get_context(pos) + if not ctx then return nil end + local action = ctx.arg and nested_get(ctx, "parent", "data", "action") + or ctx.item and nested_get(ctx, "parent", "parent", "data", "action") + local handlers = self.help_handlers + if handlers[action] then + return handlers[action](self, ctx) + elseif ctx.cs then + return handlers.cs(self, ctx) + elseif ctx.arg then + return handlers.arg(self, ctx) + -- elseif ctx.list then + elseif ctx.key then + return handlers.key(self, ctx) + else + return nil + end +end + +Manuscript.help_handlers = {} + +function Manuscript.help_handlers.cite(self, ctx) + local name = self:substring(ctx) + for item in self.root:traverse "bib_index" do + if name == item.name then + local script, details = item.manuscript + if script.format == "bibtex" then + details = format( + [[ +`%s`: %s + +# Bibtex definition + +```bibtex +%s +``` +]], + item.name, + item.text, + script:substring(item) + ) + end + return { + pos = ctx.pos, + cont = ctx.cont, + kind = "bibitem", + summary = item.name .. " " .. item.text, + details = details + } + end + end +end + +function Manuscript:label_context_long(item) + local pos = self:find_preceding_command(item.outer_pos) + if not pos then pos = item.outer_pos end + local l = self:line_number_at(pos) + local lines = self.lines + local end_pos = lines[l + 10] + if end_pos then + return self:substring_trimmed(pos, end_pos - 1) + else + return self:substring_trimmed(pos, -1) + end +end + +function Manuscript.help_handlers.ref(self, ctx) + local name = self:substring(ctx) + for item in self.root:traverse "label_index" do + if name == item.name then + local script = item.manuscript + local short_context = script:label_context_short(item) + local long_context = script:label_context_long(item) + local details = format( + [[ +`%s`: Refers to “%s...” + +# Label context + +```%s +%s +[...] +``` +]], + item.name, + short_context, + script.format, + long_context + ) + return { + pos = ctx.pos, + cont = ctx.cont, + kind = "label", + label = name, + summary = short_context, + details = details + } + end + end + return { + pos = ctx.pos, + cont = ctx.cont, + kind = "label", + label = name, + summary = "Unknown label" + } +end + +function Manuscript.help_handlers.begin(self, ctx) + local env_name = self:substring(ctx) + local data = self.environments[env_name] + if not data then return nil end + local args = data.arguments + local sig_text, sig_pos = self:signature_env(env_name, args) + return { + pos = ctx.pos, + cont = ctx.cont, + kind = "environment", + label = sig_text, + label_positions = sig_pos, + summary = data.summary, + details = self:make_docstring("env", env_name, data), + data = data + } +end + +Manuscript.help_handlers['end'] = function(self, ctx) + local env_name = self:substring(ctx) + local data = self.environments[env_name] + if not data then return nil end + local args = data.arguments + local sig_text, sig_pos = self:signature_env(env_name, args) + return { + pos = ctx.pos, + cont = ctx.cont, + kind = "environment", + label = sig_text, + label_positions = sig_pos, + summary = data.summary, + details = self:make_docstring("env", env_name, data), + data = data + } +end + +function Manuscript.help_handlers.cs(self, ctx) + local data = ctx.data + if not data then return nil end + local args = data.arguments + local sig_text, sig_pos = self:signature_cmd(ctx.cs, args) + return { + pos = ctx.pos, + cont = ctx.cont, + kind = "command", + label = sig_text, + label_positions = sig_pos, + summary = data.summary, + details = self:make_docstring("cs", ctx.cs, data), + data = data + } +end + +function Manuscript.help_handlers.arg(self, ctx) + if ctx.parent.cs then + return update( + self.help_handlers.cs(self, ctx.parent), + {pos = ctx.pos, cont = ctx.cont, arg = ctx.arg}) + end +end + +function Manuscript.help_handlers.key(self, ctx) + local key = ctx.key + local data = nested_get(ctx.parent, "data", "keys", key) + if not data then return nil end + return { + pos = ctx.pos, + cont = ctx.cont, + kind = "key", + label = key, + summary = data.summary, + details = self:make_docstring("key", key, data), + data = data + } +end + +function Manuscript:make_docstring_header(kind, name, data) + local ret = name + if kind == "cs" then + ret = self:signature_cmd(name, data.arguments) + elseif kind == "env" then + ret = self:signature_env(name, data.arguments) + elseif kind == "key" and data.meta then + ret = ret .. " = " .. data.meta + end + if data.summary then + ret = "`" .. ret .. "`: " .. data.summary + else + ret = "`" .. ret .. "`" + end + if data.symbol then + ret = ret .. " (" .. data.symbol .. ")" + end + return ret +end + +local function make_docstring_args(data) + local t = {"# Arguments\n"} + local args = data.arguments + if args then + for i = 1, #args do + local arg = args[i] + if arg.summary then + t[#t+1] = "- " .. (arg.meta or "#" .. i) + .. (arg.optional and " (optional): " or ": ") + .. arg.summary + end + end + t[#t+1] = "" + end + if #t > 2 then return concat(t, "\n") end +end + +function Manuscript:make_docstring_variants(kind, name, data) + local variants = data.variants + if not variants then return end + local t = {"# Alternative forms\n"} + local fun = (kind == "env") and self.signature_env or self.signature_cmd + for i = 1, #variants do + t[#t+1] = "- `" .. fun(self, name, variants[i].arguments) .. "`" + end + t[#t+1] = "" + return concat(t, "\n") +end + +local function make_docstring_details(data) + local details = data.details + local doc_field = data.documentation + if details then + return "# Details\n\n" .. details + elseif type(doc_field) == "string" and doc_field:match"^info:" then + local str, node, subnode = get_info(doc_field) + if str then + return format("# Info: (%s)%s\n\n```Info\n%s```", node, subnode, str) + end + else + return + end +end + +function Manuscript:make_docstring_docs(kind, name, data) + local item_doc = data.documentation + local pkg = data.package + local pkg_doc = pkg and pkg.documentation + local t = {"# Documentation\n"} + if pkg and pkg.ctan_package then + if kind == "env" then + name = "{" .. name .. "}" + elseif kind == "cs" then + name = self:signature_cmd(name) + end + t[#t+1] = format( + "`%s` is defined in the [%s](https://www.ctan.org/pkg/%s) package.\n", + name, pkg.ctan_package, pkg.ctan_package + ) + end + if item_doc then + extend(t, resolve_doc_items(item_doc)) + end + if pkg_doc then + extend(t, resolve_doc_items(pkg_doc)) + end + if #t > 1 then return concat(t, "\n") else return end +end + +function Manuscript:make_docstring(kind, name, data) + local t = {self:make_docstring_header(kind, name, data), ""} + t[#t+1] = make_docstring_args(data) + t[#t+1] = self:make_docstring_variants(kind, name, data) + t[#t+1] = make_docstring_details(data) + t[#t+1] = self:make_docstring_docs(kind, name, data) + return concat(t, "\n") +end + +--* Find definition + +-- Find the location where the thing at the given position is defined. +function Manuscript:find_definition(pos) + local ctx = self:get_context(pos) + if not ctx then return nil end + local action = ctx.arg and nested_get(ctx, "parent", "data", "action") + or ctx.item and nested_get(ctx, "parent", "parent", "data", "action") + local handlers = self.find_definition_handlers + if handlers[action] then + return handlers[action](self, ctx) + elseif ctx.cs then + return handlers.cs(self, ctx) + else + return nil + end +end + +Manuscript.find_definition_handlers = {} + +function Manuscript.find_definition_handlers.ref(self, ctx) + local name = self:substring(ctx) + for item in self.root:traverse "label_index" do + if name == item.name then + return { + pos = item.pos, + cont = item.cont, + manuscript = item.manuscript, + kind = "label" + } + end + end +end + +function Manuscript.find_definition_handlers.cite(self, ctx) + local name = self:substring(ctx) + for item in self.root:traverse "bib_index" do + if name == item.name then + return { + pos = item.pos, + cont = item.cont, + manuscript = item.manuscript, + kind = "bibitem" + } + end + end +end + +function Manuscript.find_definition_handlers.cs(self, ctx) + local name = ctx.cs + for item in self.root:traverse "newcommand_index" do + if name == item.name then + return { + pos = item.pos, + cont = item.cont, + manuscript = item.manuscript, + kind = "cs" + } + end + end +end + +function Manuscript.find_definition_handlers.begin(self, ctx) + local name = self:substring(ctx) + for item in self.root:traverse "newenvironment_index" do + if name == item.name then + return { + pos = item.pos, + cont = item.cont, + manuscript = item.manuscript, + kind = "env" + } + end + end +end + +Manuscript.find_definition_handlers["end"] + = Manuscript.find_definition_handlers.begin + +function Manuscript.find_definition_handlers.input(self, ctx) + local template + if ctx.arg then + template = nested_get(ctx, "parent", "data", "filename") or "?" + elseif ctx.item then + template = nested_get(ctx, "parent", "parent", "data", "filename") or "?" + else + return + end + local basename = format_filename_template(template, self:substring(ctx)) + local filename = find_file(path_split(self.filename), basename) + if not filename then return end + local child = self:find_manuscript(path_normalize(filename)) + if child then + return { + pos = 1, + cont = 1, + manuscript = child, + kind = "manuscript" + } + end +end + +--* Find references + +function Manuscript:scan_references() + if not self.ref_index then + self.ref_index = {} + self.cite_index = {} + self:scan(self.scan_references_callbacks) + end + local idx = self.child_index + for i = 1, #idx do + local script = self:child(idx[i].name) + if script then script:scan_references() end + end +end + +Manuscript.scan_references_callbacks = {} + +function Manuscript:scan_control_sequences() + if not self.cs_index then + self.cs_index = {} + self:scan(self.scan_cs_callbacks) + end + local idx = self.child_index + for i = 1, #idx do + local script = self:child(idx[i].name) + if script then script:scan_control_sequences() end + end +end + +Manuscript.scan_cs_callbacks = {} + +function Manuscript.scan_cs_callbacks.cs(self, pos, cs) + local idx = self.cs_index + local cont = pos + 1 + #cs + idx[#idx + 1] = { + name = cs, + pos = pos, + cont = cont, + manuscript = self, + } + return cont +end + +-- List all references to the thing at the given position. +-- +-- Returns: +-- A list of annotated ranges. +-- +function Manuscript:find_references(pos) + local ctx = self:get_context(pos) + if not ctx then return nil end + local action = ctx.arg and nested_get(ctx, "parent", "data", "action") + or ctx.item and nested_get(ctx, "parent", "parent", "data", "action") + local handlers = self.find_references_handlers + if handlers[action] then + self.root:scan_references() + return handlers[action](self, ctx) + elseif ctx.cs then + self.root:scan_control_sequences() + return handlers.cs(self, ctx) + else + return nil + end +end + +Manuscript.find_references_handlers = {} + +function Manuscript.find_references_handlers.cs(self, ctx) + local name = ctx.cs + local r = {} + for item in self.root:traverse "cs_index" do + if name == item.name then + r[#r + 1] = { + pos = item.pos, + cont = item.cont, + manuscript = item.manuscript, + kind = "cs" + } + end + end + return r +end + +function Manuscript.find_references_handlers.ref(self, ctx) + local name = self:substring(ctx) + local r = {} + for item in self.root:traverse "ref_index" do + if name == item.name then + r[#r + 1] = { + pos = item.pos, + cont = item.cont, + manuscript = item.manuscript, + kind = "label" + } + end + end + return r +end + +Manuscript.find_references_handlers.label = + Manuscript.find_references_handlers.ref + +function Manuscript.find_references_handlers.cite(self, ctx) + local name = self:substring(ctx) + local r = {} + for item in self.root:traverse "cite_index" do + if name == item.name then + r[#r + 1] = { + pos = item.pos, + cont = item.cont, + manuscript = item.manuscript, + kind = "bibitem" + } + end + end + return r +end + +--* Outline + +-- Compute a table of contents for the document. If loc is false or +-- omitted, include children of the manuscript; otherwise, restrict to +-- the current manuscript. +-- +function Manuscript:outline(loc) + local val = {} + for it in self:traverse("section_index", loc and 0) do + local lv = it.level or infty + local t = val + while t[#t] and (t[#t].level or -infty) < lv do t = t[#t] end + t[#t + 1] = { + name = it.name, + pos = it.pos, + cont = it.cont, + level = lv, + manuscript = it.manuscript, + kind = "section" + } + end + return val +end + +return Manuscript diff --git a/support/digestif/digestif/ManuscriptBibTeX.lua b/support/digestif/digestif/ManuscriptBibTeX.lua new file mode 100644 index 0000000000..219f4552b4 --- /dev/null +++ b/support/digestif/digestif/ManuscriptBibTeX.lua @@ -0,0 +1,38 @@ +local lpeg = require "lpeg" +local util = require "digestif.util" +local Manuscript = require "digestif.Manuscript" +local Parser = require "digestif.Parser" +local bibtex = require "digestif.bibtex" + +local path_join, path_split = util.path_join, util.path_split +local nested_get, nested_put = util.nested_get, util.nested_put +local map, update, merge = util.map, util.update, util.merge + +local ManuscriptBibtex = util.class(Manuscript) + +ManuscriptBibtex.parser = Parser() +ManuscriptBibtex.format = "bibtex" +ManuscriptBibtex.packages = {} +ManuscriptBibtex.commands = {} +ManuscriptBibtex.environments = {} +ManuscriptBibtex.init_callbacks = false -- Skip the normal init scan +ManuscriptBibtex:add_package("plain") -- For basic command completion + +function ManuscriptBibtex:__init(args) + Manuscript.__init(self, args) + local bibitems = bibtex.parse(self.src) + local idx = {} + self.bib_index = idx + for i, item in ipairs(bibitems) do + idx[i] = { + name = item.id, + pos = item.pos, + cont = item.cont, + manuscript = self, + text = item:pretty_print(), + bibitem = item, + } + end +end + +return ManuscriptBibtex diff --git a/support/digestif/digestif/ManuscriptConTeXt.lua b/support/digestif/digestif/ManuscriptConTeXt.lua new file mode 100644 index 0000000000..2243d3a9fc --- /dev/null +++ b/support/digestif/digestif/ManuscriptConTeXt.lua @@ -0,0 +1,604 @@ +local util = require "digestif.util" +local Manuscript = require "digestif.Manuscript" +local Parser = require "digestif.Parser" + +local co_wrap, co_yield = coroutine.wrap, coroutine.yield +local merge = util.merge +local path_join, path_split = util.path_join, util.path_split +local path_normalize = util.path_normalize +local format_filename_template = util.format_filename_template +local table_move, table_insert = table.move, table.insert + +--* Parsing ConTeXt interface files + +--** XML parser by Roberto Ierusalimschy +-- +-- Cf. http://lua-users.org/wiki/LuaXml + +local function parseargs(s) + local arg = {} + string.gsub(s, "([%w:_-]+)=([\"'])(.-)%2", function (w, _, a) + arg[w] = a + end) + return arg +end + +local function collect(s) + local stack = {} + local top = {} + table.insert(stack, top) + local ni,c,label,xarg, empty + local i, j = 1, 1 + while true do + ni,j,c,label,xarg, empty = string.find(s, "<(%/?)([%w:]+)(.-)(%/?)>", i) + if not ni then break end + local text = string.sub(s, i, ni-1) + if not string.find(text, "^%s*$") then + table.insert(top, text) + end + if empty == "/" then -- empty element tag + table.insert(top, {label=label, xarg=parseargs(xarg), empty=1}) + elseif c == "" then -- start tag + top = {label=label, xarg=parseargs(xarg)} + table.insert(stack, top) -- new level + else -- end tag + local toclose = table.remove(stack) -- remove top + top = stack[#stack] + if #stack < 1 then + error("nothing to close with "..label) + end + if toclose.label ~= label then + error("trying to close "..toclose.label.." with "..label) + end + table.insert(top, toclose) + end + i = j+1 + end + local text = string.sub(s, i) + if not string.find(text, "^%s*$") then + table.insert(stack[#stack], text) + end + if #stack > 1 then + error("unclosed "..stack[#stack].label) + end + return stack[1] +end + +local function child_with_label(node, label) + for _, n in ipairs(node) do + if n.label == label then return n end + end +end + +local function children_with_label(node, label) + local function iter(n, l) + if n.label == l then + co_yield(n) + else + for _, m in ipairs(n) do + iter(m, l) + end + end + end + return co_wrap(function() return iter(node, label) end) +end + +--** Parse ConTeXt interface (XML) files + +local braces = {"{", "}"} +local brackets = {"[", "]"} +local parenthesis = {"(", ")"} + +local function gen_tags(data) + + local inheritances = {} + local inherit_type = {} + local inherit_order = {} + + local function compute_meta(node) + local fromtag = node.label:gsub('^cd:', '') + local tbl = {} + local other = 0 + for _, child in ipairs(node) do + local attribs = child.xarg + local val = attribs.type + if val and val:match"^cd:" then + tbl[#tbl+1]=val:sub(4) + else + other = other + 1 + end + end + if other == 0 and #tbl == 1 then + return tbl[1] + else + return fromtag + end + end + + local function compute_values(node) + local tbl = {} + for _, child in ipairs(node) do + if child.label == "cd:constant" then + local val = child.xarg.type + if not val:match"^cd:" then tbl[#tbl+1] = val end + elseif child.label == "cd:inherit" then + local val = child.xarg.name + if not inheritances[tbl] then inheritances[tbl] = {} end + local inh = inheritances[tbl] + inherit_type[tbl] = "values" + inherit_order[#inherit_order+1] = tbl + inh[#inh+1] = val + end + end + return #tbl > 0 and tbl or nil + end + + local function compute_keys(node) + local tbl = {} + for _, child in ipairs(node) do + if child.label == "cd:parameter" then + local val = child.xarg.name + tbl[val] = { + values = compute_values(child), + meta = compute_meta(child) + } + elseif child.label == "cd:inherit" then + local val = child.xarg.name + if not inheritances[tbl] then inheritances[tbl] = {} end + local inh = inheritances[tbl] + inherit_type[tbl] = "keys" + inherit_order[#inherit_order+1] = tbl + inh[#inh+1] = val + end + end + return tbl + end + + local function compute_argument(node) + local ret = {delimiters=brackets} + local attrs = node.xarg + if attrs.optional == 'yes' then ret.optional = true end + if attrs.list == 'yes' then ret.list = true end + if node.label == "cd:keywords" then + ret.meta=compute_meta(node) + ret.values = compute_values(node) + elseif node.label == "cd:assignments" then + ret.meta="assignments" + ret.keys = compute_keys(node) + elseif node.label == "cd:constant" then + ret.meta = node.xarg.type:gsub('^cd:', '') + elseif node.label == "cd:content" then + ret.meta="content" + ret.delimiters = braces + elseif node.label == "cd:csname" then + ret.meta="command" + ret.type="cs" + ret.delimiters = nil + elseif node.label == "cd:dimension" then + ret.meta = "dimension" + ret.type="dimen" + end + if attrs.delimiters == 'braces' then ret.delimiters = braces end + if attrs.delimiters == 'parenthesis' then ret.delimiters = parenthesis end + if attrs.delimiters == 'none' then ret.delimiters = false end + return ret + end + + local function compute_arguments(node) + local tbl = {} + for _, child in ipairs(node) do + tbl[#tbl+1] = compute_argument(child) + end + return tbl + end + + local function compute_instances(name, node) + local tbl = {} + if node == nil then return tbl end + for _, instance in ipairs(node) do + local inst_name = instance.xarg.value + if name ~= inst_name then + tbl[#tbl+1] = inst_name + end + end + return tbl + end + + local command_list = {} + for node in children_with_label(data, "cd:command") do + local attribs = node.xarg + local arguments = child_with_label(node, "cd:arguments") + arguments = arguments and compute_arguments(arguments) + local cmd = { + cs = attribs.name, + environment = (attribs.type == "environment"), + --source = attribs.file, + --category = attribs.category, + arguments = arguments, + } + command_list[#command_list+1] = cmd + + local instances = compute_instances( + attribs.name, child_with_label(node, "cd:instances")) + for _, name in ipairs(instances) do + command_list[#command_list+1] = merge(cmd, {cs = name}) + end + end + + -- FIXME: need to distinguish when doing + local cmds_and_envs = {} + for _, cmd in ipairs(command_list) do + local cs = cmd.cs + if cmd.link or not cmds_and_envs[cs] then + cmds_and_envs[cs] = cmd + end + end + + for _, tbl in ipairs(inherit_order) do + local inh = inheritances[tbl] + --print("\n\nbefore", ser(tbl)) + for _, cs in ipairs(inh) do + for _, arg in ipairs((cmds_and_envs[cs] or {}).arguments or {}) do + util.update(tbl, arg[inherit_type[tbl]] or {}) + end + end + --print("\nafter", ser(tbl)) + end + + local tags = { + generated = true, + commands = {}, + environments = {} + + } + + for _, cmd in ipairs(command_list) do + local cs = cmd.cs + local list = cmd.environment and tags.environments or tags.commands + local wikiname = cmd.environment and "start" .. cs or cs + cmd.documentation = { + [1] = { + summary = "“" .. wikiname .. "” on ConTeXt Garden", + uri = "https://wiki.contextgarden.net/Command/" .. wikiname + } + } + cmd.environment, cmd.cs = nil, nil + if not list[cs] then + list[cs] = cmd + else + local variants = list[cs].variants + if not variants then + variants = {} + list[cs].variants = variants + end + variants[#variants+1] = cmd + end + end + + return tags +end + +-- Generate tags from a ConTeXt interface (XML) file. +-- +-- Arguments: +-- file: The file name. +-- pkg: An optional package description table. +-- +-- Returns: +-- The tags, or nil if file does not exist. +-- +local function tags_from_xml(file, pkg) + local ok, str = util.find_file(file, nil, true) + if not ok then return end + local ok, data = pcall(collect, str) + if not ok then return end + return setmetatable(gen_tags(data), {__index = pkg}) +end + +-- Monkey-patch to isolate XML dependency in this file. +(require "digestif.data").tags_from_xml = tags_from_xml + +--** Core ConTeXt tags + +local ctx_tags = (require "digestif.data".require_tags)("context-en.xml") + +if ctx_tags then + + ctx_tags.ctan_package = "context" + + ctx_tags.documentation = { + [1] = { + summary = "ConTeXt Mark IV: an excursion", + uri = "texmf:doc/context/documents/general/manuals/ma-cb-en.pdf" + }, + [2] = { + summary = "ConTeXt documentation library", + uri = "https://wiki.contextgarden.net/Documentation" + } + } + + local commands = ctx_tags.commands + local environments = ctx_tags.environments + + -- Add details to sectioning commands + for _, tbl in ipairs{commands, environments} do + for i, cs in ipairs{ + "part", "chapter", "section", "subsection", "subsubsection", + "subsubsubsection", "subsubsubsubsection" + } do + tbl[cs].section_level = i + tbl[cs].action = "section" + end + for i, cs in ipairs{ + "title", "subject", "subsubject", "subsubsubject", + "subsubsubsubject", "subsubsubsubsubject" + } do + tbl[cs].section_level = i + 1 + tbl[cs].action = "section" + end + end + + -- Add details to file-inputting commands. + for cs, filename in pairs { + environment = "?.tex", + project = "?.tex", + component = "?.tex", + usemodule = "t-?.xml" + } do + if commands[cs] then + commands[cs].action = "input" + commands[cs].filename = filename + end + end + + -- Add details to assorted commands. + for cs, action in pairs { + usebtxdataset = "input", + cite = "cite", + nocite = "cite", + define = "define", + definestartstop = "definestartstop", + defineitemgroup = "definestartstop", + definelist = "definestartstop", + citation = "cite", + nocitation = "cite", + pagereference = "label", + textreference = "label", + ["in"] = "ref", + at = "ref", + about = "ref" + } do + if commands[cs] then + commands[cs].action = action + end + end + + -- Add start/stop commands + for env, cmd in pairs(environments) do + commands["start" .. env] = cmd + commands["stop" .. env] = {} + end + +end + +--* ManuscriptContext class + +local ManuscriptContext = util.class(Manuscript) + +ManuscriptContext.parser = Parser() +ManuscriptContext.format = "context" +ManuscriptContext.packages = {} +ManuscriptContext.commands = {} +ManuscriptContext.environments = {} +ManuscriptContext.init_callbacks = {} +ManuscriptContext.scan_references_callbacks = {} +ManuscriptContext:add_package("context-en.xml") + +-- Make a snippet for an environment. +function ManuscriptContext:snippet_env(cs, args) + local argsnippet = args and self:snippet_args(args) or "" + return "start" .. cs .. argsnippet .. "\n\t$0\n\\stop" .. cs +end + +-- ConTeXt optional arguments can't always be distinguished by their +-- delimiters, for instance \citation[optional][mandatory]. Here we +-- patch Parser.parse_args to deal with this case, but just in the +-- simplest case where the optional arguments are in the beginning of +-- the argument list. +local original_parse_args = ManuscriptContext.parser.parse_args + +local function new_parse_args(arglist, str, pos) + local val = original_parse_args(arglist, str, pos) + if val.incomplete and arglist[1].optional then + arglist = table_move(arglist, 2, #arglist, 1, {}) + val = new_parse_args(arglist, str, pos) + table_insert(val, 1, {omitted = true}) + end + return val +end + +ManuscriptContext.parser.parse_args = new_parse_args + +--* Init scanning + +local to_args = {} +for i = 1, 9 do + local t = merge(to_args[i-1] or {}) + t[i] = {meta = "#" .. i} + to_args[i] = t +end + +function ManuscriptContext.init_callbacks.define(self, pos, cs) + -- ugly! this function (and others below) parses the command twice + local cont = self:parse_command(pos, cs).cont + local csname, n_args + for r in self:argument_items("command", pos, cs) do + csname = self:substring_stripped(r):sub(2) + end + for r in self:argument_items("number", pos, cs) do + n_args = tonumber(self:substring_stripped(r)) + end + if csname then + local idx = self:get_index "newcommand" + idx[#idx+1] = { + name = csname, + pos = pos, + cont = cont, + manuscript = self, + arguments = to_args[n_args] + } + if not self.commands[csname] then + self.commands[csname] = idx[#idx] + end + end + return cont +end + +function ManuscriptContext.init_callbacks.definestartstop(self, pos, cs) + local cont = self:parse_command(pos, cs).cont + local name + for r in self:argument_items("name", pos, cs) do + name = self:substring_stripped(r) + end + if name then + local idx = self:get_index "newenvironment" + idx[#idx+1] = { + name = name, + pos = pos, + cont = cont, + manuscript = self, + } + if not self.environments[name] then + self.environments[name] = idx[#idx] + end + end + return cont +end + +function ManuscriptContext.init_callbacks.label(self, pos, cs) + local cont = self:parse_command(pos, cs).cont + local idx = self.label_index + for r in self:argument_items("reference", pos, cs) do + idx[#idx + 1] = { + name = self:substring_stripped(r), + pos = r.pos, + cont = r.cont, + outer_pos = pos, + outer_cont = cont, + manuscript = self + } + end + return cont +end + +function ManuscriptContext.init_callbacks.section(self, pos, cs) + local cont = self:parse_command(pos, cs).cont + for r in self:argument_items("assignments", pos, cs) do + local assigns = self:parse_kvlist(r) + for i = 1, #assigns do + local key = self:substring_stripped(assigns[i].key) + local val = assigns[i].value + if key == "reference" then + local idx = self.label_index + idx[#idx + 1] = { + name = self:substring_stripped(val), + pos = val.pos, + cont = val.cont, + outer_pos = pos, + outer_cont = cont, + manuscript = self + } + elseif key == "title" then + local idx = self.section_index + idx[#idx + 1] = { + name = self:substring_stripped(val), + level = self.commands[cs].section_level, + pos = val.pos, + cont = val.cont, + outer_pos = pos, + outer_cont = cont, + manuscript = self + } + end + end + end + for r in self:argument_items("reference", pos, cs) do + local idx = self.label_index + idx[#idx + 1] = { + name = self:substring_stripped(r), + pos = r.pos, + cont = r.cont, + outer_pos = pos, + outer_cont = cont, + manuscript = self + } + end + for r in self:argument_items("text", pos, cs) do + local idx = self.section_index + idx[#idx + 1] = { + name = self:substring_stripped(r), + level = self.commands[cs].section_level, + pos = r.pos, + cont = r.cont, + outer_pos = pos, + outer_cont = cont, + manuscript = self + } + end + return cont +end + +function ManuscriptContext.init_callbacks.input(self, pos, cs) + local cont = self:parse_command(pos, cs).cont + local idx = self.child_index + local template = self.commands[cs].filename or "?" + for r in self:argument_items("file", pos, cs) do + local f = format_filename_template(template, self:substring_stripped(r)) + local ok = self:add_package(f) + if not ok then + idx[#idx + 1] = { + name = path_normalize(path_join(path_split(self.filename), f)), + pos = r.pos, + cont = r.cont, + manuscript = self + } + end + end + return cont +end + +--* Reference scanning + +function ManuscriptContext.scan_references_callbacks.ref(self, pos, cs) + local cont = self:parse_command(pos, cs).cont + local idx = self.ref_index + for s in self:argument_items("reference", pos, cs) do + idx[#idx + 1] = { + name = self:substring_stripped(s), + pos = s.pos, + cont = s.cont, + outer_pos = pos, + outer_cont = cont, + manuscript = self + } + end + return cont +end + +function ManuscriptContext.scan_references_callbacks.cite(self, pos, cs) + local cont = self:parse_command(pos, cs).cont + local idx = self.cite_index + for s in self:argument_items("reference", pos, cs) do + idx[#idx + 1] = { + name = self:substring_stripped(s), + pos = s.pos, + cont = s.cont, + outer_pos = pos, + outer_cont = cont, + manuscript = self + } + end + return cont +end + +return ManuscriptContext diff --git a/support/digestif/digestif/ManuscriptDoctex.lua b/support/digestif/digestif/ManuscriptDoctex.lua new file mode 100644 index 0000000000..c3f0c2b0ce --- /dev/null +++ b/support/digestif/digestif/ManuscriptDoctex.lua @@ -0,0 +1,19 @@ +local ManuscriptLatex = require "digestif.ManuscriptLaTeX" +local Parser = require "digestif.Parser" +local lpeg = require "lpeg" +local util = require "digestif.util" + +local ManuscriptDoctex = util.class(ManuscriptLatex) + +-- Consider both code (no prefix) and documentation (starting with a +-- single %) as not being comments. +local comment = (lpeg.B(1) - lpeg.B("\n")) * lpeg.P("%") + +ManuscriptDoctex.format = "doctex" +ManuscriptDoctex.parser = Parser{ + comment = comment, + letter = lpeg.R("az", "@Z") +} +ManuscriptLatex:add_package("latex.ltx") + +return ManuscriptDoctex diff --git a/support/digestif/digestif/ManuscriptLaTeX.lua b/support/digestif/digestif/ManuscriptLaTeX.lua new file mode 100644 index 0000000000..1366b0e146 --- /dev/null +++ b/support/digestif/digestif/ManuscriptLaTeX.lua @@ -0,0 +1,360 @@ +local lpeg = require "lpeg" +local util = require "digestif.util" +local Manuscript = require "digestif.Manuscript" +local Parser = require "digestif.Parser" + +local path_join, path_split = util.path_join, util.path_split +local path_normalize = util.path_normalize +local nested_get, nested_put = util.nested_get, util.nested_put +local map, update, merge = util.map, util.update, util.merge +local format_filename_template = util.format_filename_template + +local ManuscriptLatex = util.class(Manuscript) + +ManuscriptLatex.parser = Parser() +ManuscriptLatex.format = "latex" +ManuscriptLatex.packages = {} +ManuscriptLatex.commands = {} +ManuscriptLatex.environments = {} +ManuscriptLatex.init_callbacks = {} +ManuscriptLatex.scan_references_callbacks = {} +ManuscriptLatex:add_package("latex") + +--* Snippets + +-- Make a snippet for an environment. +-- +-- Arguments: +-- cs: The command name +-- args: An argument list +-- +-- Returns: +-- A snippet string +-- +function ManuscriptLatex:snippet_env(cs, args) + local argsnippet = args and self:snippet_args(args) or "" + return "begin{" .. cs .. "}" .. argsnippet .. "\n\t$0\n\\end{" .. cs .. "}" +end + +-- Pretty-print an environment signature. +function ManuscriptLatex:signature_env(cs, args) + return self:signature_args(args, "\\begin{" .. cs .. "}") +end + +--* Helper functions + +local function first_mand(args) + for i, v in ipairs(args) do + if not v.optional then return i end + end +end + +local function first_opt(args) + for i, v in ipairs(args) do + if v.optional then return i end + end +end + +--* Global scanning + +--** Basic document elements + +function ManuscriptLatex.init_callbacks.input(self, pos, cs) + local cont = self:parse_command(pos, cs).cont + local idx = self.child_index + local template = self.commands[cs].filename or "?" + for r in self:argument_items(first_mand, pos, cs) do + local f = format_filename_template(template, self:substring_stripped(r)) + local ok = self:add_package(f) + if not ok then + idx[#idx + 1] = { + name = path_normalize(path_join(path_split(self.filename), f)), + pos = r.pos, + cont = r.cont, + manuscript = self + } + end + end + return cont +end + +function ManuscriptLatex.init_callbacks.label (m, pos, cs) + local idx = m.label_index + local args = m.commands[cs].arguments + local r = m:parse_command(pos, cs) + local i = first_mand(args) + if r[i] then + local l = m:substring_stripped(r[i]) + idx[#idx + 1] = { + name = l, + pos = r[i].pos, + cont = r[i].cont, + outer_pos = r.pos, + outer_cont = r.cont, + manuscript = m + } + end + return r.cont +end + +function ManuscriptLatex.init_callbacks.section(self, pos, cs) + local idx = self.section_index + for r in self:argument_items(first_mand, pos, cs) do + idx[#idx + 1] = { + name = self:substring_stripped(r), + level = self.commands[cs].section_level, + pos = r.pos, + cont = r.cont, + outer_pos = pos, + manuscript = self + } + end + return pos + #cs + 1 +end + +--** Bibliographic items + +function ManuscriptLatex.init_callbacks.bibitem (m, pos, cs) + local idx = m.bib_index + local args = m.commands[cs].arguments + local r = m:parse_command(pos, cs) + local i = first_mand(args) + if r[i] then + idx[#idx + 1] = { + name = m:substring_stripped(r[i]), + pos = r[i].pos, + cont = r[i].cont, + outer_pos = r.pos, + outer_cont = r.cont, + manuscript = m + } + end + return r.cont +end + +function ManuscriptLatex.init_callbacks.amsrefs_bib(self, pos, cs) + local idx = self.bib_index + local r = self:parse_command(pos, cs) + if r.incomplete then return r.cont end + local keys = self:read_keys(r[3]) + local authors, title, date = {}, "", "(n.d.)" + for i = 1, #keys do + local k, v = keys[i].key, keys[i].value + if k == "author" then + authors[#authors+1] = self.parser.clean(v:match("[^,]+", 2)) + elseif k == "title" then + title = self.parser.clean(v:sub(2, -2)) + elseif k == "date" then + date = self.parser.clean(v:match("(%d+)")) + end + end + idx[#idx + 1] = { + name = self:substring_stripped(r[1]), + text = string.format( + "%s %s; %s", + table.concat(authors, ", "), + date, + title), + pos = r[1].pos, + cont = r[1].cont, + outer_pos = r.pos, + outer_cont = r.cont, + manuscript = self + } + return r.cont +end + +--** Command definitions, TeX style + +ManuscriptLatex.init_callbacks.def = + require "digestif.ManuscriptPlainTeX".init_callbacks.def + +--** Command and environment definitions, LaTeX style + +local to_args = {} +for i = 1, 9 do + local t = merge(to_args[i-1] or {}) + t[i] = {meta = "#" .. i} + to_args[i] = t +end + +local function newcommand_args(number, default) + local args = to_args[number] + if default and args then + args = merge(args) -- make a copy + args[1] = { + meta = "#1", + optional = true, + delimiters = {"[", "]"}, + details = "Default: “" .. default .. "”." + } + end + return args +end + +function ManuscriptLatex.init_callbacks.newcommand(self, pos, cs) + local cont = self:parse_command(pos, cs).cont + local csname, nargs, optdefault + for r in self:argument_items(first_mand, pos, cs) do + csname = self:substring_stripped(r):sub(2) + end + for r in self:argument_items("number", pos, cs) do + nargs = tonumber(self:substring_stripped(r)) + end + for r in self:argument_items("default", pos, cs) do + optdefault = self:substring_stripped(r) + end + if csname then + local idx = self:get_index "newcommand" + idx[#idx+1] = { + name = csname, + pos = pos, + cont = cont, + manuscript = self, + arguments = newcommand_args(nargs, optdefault) + } + if not self.commands[csname] then + self.commands[csname] = idx[#idx] + end + end + return cont +end + +function ManuscriptLatex.init_callbacks.newenvironment(self, pos, cs) + local cont = self:parse_command(pos, cs).cont + local csname, nargs, optdefault + for r in self:argument_items(first_mand, pos, cs) do + csname = self:substring_stripped(r) + end + for r in self:argument_items("number", pos, cs) do + nargs = tonumber(self:substring_stripped(r)) + end + for r in self:argument_items("default", pos, cs) do + optdefault = self:substring_stripped(r) + end + if csname then + local idx = self:get_index("newenvironment") + idx[#idx+1] = { + name = csname, + pos = pos, + cont = cont, + manuscript = self, + arguments = newcommand_args(nargs, optdefault) + } + if not self.environments[csname] then + self.environments[csname] = idx[#idx] + end + end + return cont +end + +--** Command and environment definitions, xparse style + +local P, C, Cc, Cg, Ct = lpeg.P, lpeg.C, lpeg.Cc, lpeg.Cg, lpeg.Ct +local Pgroup = util.between_balanced("{", "}") +local Pdefault = Cg(Pgroup / "Default: “%1”", "details") + +local xparse_args = util.matcher( + Ct(util.many(Ct(util.sequence( + P" "^0, + (P"+" * Cg(Cc(true), "long"))^-1, + P"!"^-1, + util.choice( + "m", + "r" * Cg(Ct(C(1) * C(1)), "delimiters"), + "R" * Cg(Ct(C(1) * C(1)), "delimiters") * Pdefault, + "v" * Cg(Cc"verbatim", "type"), + "o" * Cg(Cc{"[", "]"}, "delimiters") * Cg(Cc(true), "optional"), + "O" * Cg(Cc{"[", "]"}, "delimiters") * Cg(Cc(true), "optional") * Pdefault, + "d" * Cg(Ct(C(1) * C(1)), "delimiters") * Cg(Cc(true), "optional"), + "D" * Cg(Ct(C(1) * C(1)), "delimiters") * Cg(Cc(true), "optional") * Pdefault, + "s" * Cg(Cc"*", "literal") * Cg(Cc"literal", "type") * Cg(Cc(true), "optional"), + "t" * Cg(C(1), "literal") * Cg(Cc"literal", "type") * Cg(Cc(true), "optional"), + "e" * Pgroup, + "E" * Pgroup * Pdefault)))))) + +function ManuscriptLatex.init_callbacks.NewDocumentCommand(self, pos, cs) + local cont = self:parse_command(pos, cs).cont + local csname, arg_spec + for r in self:argument_items("command", pos, cs) do + csname = self:substring_stripped(r):sub(2) + end + for r in self:argument_items("arg spec", pos, cs) do + arg_spec = self:substring_stripped(r) + end + if csname then + local idx = self:get_index "newcommand" + idx[#idx+1] = { + name = csname, + pos = pos, + cont = cont, + manuscript = self, + arguments = xparse_args(arg_spec) + } + if not self.commands[csname] then + self.commands[csname] = idx[#idx] + end + end + return cont +end + +function ManuscriptLatex.init_callbacks.NewDocumentEnvironment(self, pos, cs) + local cont = self:parse_command(pos, cs).cont + local csname, arg_spec + for r in self:argument_items("environment", pos, cs) do + csname = self:substring_stripped(r) + end + for r in self:argument_items("arg spec", pos, cs) do + arg_spec = self:substring_stripped(r) + end + if csname then + local idx = self:get_index "newenvironment" + idx[#idx+1] = { + name = csname, + pos = pos, + cont = cont, + manuscript = self, + arguments = xparse_args(arg_spec) + } + if not self.environments[csname] then + self.environments[csname] = idx[#idx] + end + end + return cont +end + +--* Scan references callbacks + +function ManuscriptLatex.scan_references_callbacks.ref(self, pos, cs) + local cont = self:parse_command(pos, cs).cont + local idx = self.ref_index + for r in self:argument_items(first_mand, pos, cs) do + idx[#idx + 1] = { + name = self:substring_stripped(r), + pos = r.pos, + cont = r.cont, + outer_pos = pos, + outer_cont = cont, + manuscript = self + } + end + return cont +end + +function ManuscriptLatex.scan_references_callbacks.cite(self, pos, cs) + local cont = self:parse_command(pos, cs).cont + local idx = self.cite_index + for r in self:argument_items(first_mand, pos, cs) do + idx[#idx + 1] = { + name = self:substring_stripped(r), + pos = r.pos, + cont = r.cont, + outer_pos = pos, + outer_cont = cont, + manuscript = self + } + end + return cont +end + +return ManuscriptLatex diff --git a/support/digestif/digestif/ManuscriptLatexProg.lua b/support/digestif/digestif/ManuscriptLatexProg.lua new file mode 100644 index 0000000000..38746afcc8 --- /dev/null +++ b/support/digestif/digestif/ManuscriptLatexProg.lua @@ -0,0 +1,12 @@ +local ManuscriptLatex = require "digestif.ManuscriptLaTeX" +local Parser = require "digestif.Parser" +local lpeg = require "lpeg" +local util = require "digestif.util" + +local ManuscriptLatexProg = util.class(ManuscriptLatex) + +ManuscriptLatexProg.format = "latex-prog" +ManuscriptLatexProg.parser = Parser{letter = lpeg.R("az", "@Z")} +ManuscriptLatex:add_package("latex.ltx") + +return ManuscriptLatexProg diff --git a/support/digestif/digestif/ManuscriptPlainTeX.lua b/support/digestif/digestif/ManuscriptPlainTeX.lua new file mode 100644 index 0000000000..1ac217985f --- /dev/null +++ b/support/digestif/digestif/ManuscriptPlainTeX.lua @@ -0,0 +1,56 @@ +local lpeg = require "lpeg" +local util = require "digestif.util" +local Manuscript = require "digestif.Manuscript" +local Parser = require "digestif.Parser" +local ManuscriptPlain = util.class(Manuscript) + +local P, R = lpeg.P, lpeg.R +local C, Ct, Cc, Cg = lpeg.C, lpeg.Ct, lpeg.Cc, lpeg.Cg + +-- In plain TeX, we can't distinguish between code and document files, +-- so we pretend @ is always a letter. +ManuscriptPlain.parser = Parser{letter = lpeg.R("az", "@Z")} +ManuscriptPlain.format = "plain" +ManuscriptPlain.packages = {} +ManuscriptPlain.commands = {} +ManuscriptPlain.environments = {} +ManuscriptPlain.init_callbacks = {} +ManuscriptPlain.scan_references_callbacks = {} +ManuscriptPlain:add_package("plain") + +-- Convert a TeX parameter text to an arguments table. +local param = P("#") * R("19") +local before_params = Ct(Cg(C((1 - param)^1), "literal")) +local one_param = Ct( + Cg(C(param), "meta") * Cg(Ct(Cc("") * C((1 - param)^1)), "delimiters")^-1 +) +local parse_params = util.matcher( + Ct(before_params^-1 * one_param^0) +) + +function ManuscriptPlain.init_callbacks.def(self, pos, cs) + local cont = self:parse_command(pos, cs).cont + local csname, params + for r in self:argument_items(1, pos, cs) do + csname = self:substring_stripped(r):sub(2) + end + for r in self:argument_items(2, pos, cs) do + params = self:substring_stripped(r) + end + if csname then + local idx = self:get_index "newcommand" + idx[#idx+1] = { + name = csname, + pos = pos, + cont = cont, + manuscript = self, + arguments = parse_params(params) + } + if not self.commands[csname] then + self.commands[csname] = idx[#idx] + end + end + return cont +end + +return ManuscriptPlain diff --git a/support/digestif/digestif/ManuscriptTexinfo.lua b/support/digestif/digestif/ManuscriptTexinfo.lua new file mode 100644 index 0000000000..0e0c71667c --- /dev/null +++ b/support/digestif/digestif/ManuscriptTexinfo.lua @@ -0,0 +1,25 @@ +local util = require "digestif.util" +local Manuscript = require "digestif.Manuscript" +local Parser = require "digestif.Parser" + +local ManuscriptTexinfo = util.class(Manuscript) + +ManuscriptTexinfo.parser = Parser({escape = "@"}) +ManuscriptTexinfo.format = "texinfo" +ManuscriptTexinfo.packages = {} +ManuscriptTexinfo.commands = {} +ManuscriptTexinfo.environments = {} +ManuscriptTexinfo.init_callbacks = {} +ManuscriptTexinfo.scan_references_callbacks = {} +ManuscriptTexinfo:add_package("texinfo") + +function ManuscriptTexinfo:snippet_env(cs, args) + local argsnippet = args and self:snippet_args(args) or "" + return cs .. argsnippet .. "\n\t$0\n@end " .. cs +end + +function Manuscript:signature_cmd(cs, args) + return self:signature_args(args, "@" .. cs) +end + +return ManuscriptTexinfo diff --git a/support/digestif/digestif/Parser.lua b/support/digestif/digestif/Parser.lua new file mode 100644 index 0000000000..e939a13c56 --- /dev/null +++ b/support/digestif/digestif/Parser.lua @@ -0,0 +1,227 @@ +-- Parser class + +local lpeg = require "lpeg" +local util = require "digestif.util" + +local C = lpeg.C +local Cc = lpeg.Cc +local Cg = lpeg.Cg +local Cs = lpeg.Cs +local Ct = lpeg.Ct +local P = lpeg.P +local R = lpeg.R +local S = lpeg.S +local V = lpeg.V +local match = lpeg.match + +local I = lpeg.Cp() +local Ipos = Cg(I, "pos") +local Icont = Cg(I, "cont") +local Iouter_pos = Cg(I, "outer_pos") +local Iouter_cont = Cg(I, "outer_cont") +local Kcs = Cc("cs") +local Kmathshift = Cc("mathshift") +local Knil = Cc(nil) +local Kpar = Cc("par") +local Pend = P(-1) +local is_incomplete = Cg(Cc(true), "incomplete") +local is_omitted = Cg(Cc(true), "omitted") + +local choice = util.choice +local gobble_until = util.gobble_until +local many = util.many +local matcher = util.matcher +local search = util.search +local sequence = util.sequence +local trim = util.trim + +local default_catcodes = { + escape = P("\\"), + bgroup = P("{"), + egroup = P("}"), + mathshift = P("$"), + eol = P("\n"), + letter = R("az", "AZ"), + comment = P("%"), + char = P(1), -- are we utf-8 agnostic? + space = S(" \t"), + listsep = P(","), + valsep = P("=") +} + +local Parser = util.class() + +function Parser:__init(catcodes) + catcodes = catcodes or default_catcodes + +--* Single characters + local escape = P(catcodes.escape or default_catcodes.escape) + local bgroup = P(catcodes.bgroup or default_catcodes.bgroup) + local egroup = P(catcodes.egroup or default_catcodes.egroup) + local mathshift = P(catcodes.mathshift or default_catcodes.mathshift) + local eol = P(catcodes.eol or default_catcodes.eol) + local letter = P(catcodes.letter or default_catcodes.letter) + local comment = P(catcodes.comment or default_catcodes.comment) + local char = P(catcodes.char or default_catcodes.char) + local space = P(catcodes.space or default_catcodes.space) + local listsep = P(catcodes.listsep or default_catcodes.listsep) + local valsep = P(catcodes.valsep or default_catcodes.valsep) + +--* Basic elements + local blank = space + eol + local cs = escape * (letter^1 + char) + local csname = escape * C(letter^1 + char) + local token = cs + char + local blank_line = eol * space^0 * eol + local single_eol = eol * space^0 * -eol + local comment_line = comment * gobble_until(eol, char) + local skip = (space + single_eol + comment_line)^0 -- simulates TeX state S + local skip_long = (space + eol + comment_line)^0 + local par = blank_line * (space + eol + comment_line)^0 + local next_par = search(blank_line, comment + char) + * (eol + space + comment_line)^0 * I * char -- need this char at the end? + + -- These patterns match text delimited by braces. They succeed on a + -- incomplete subject string (with the additional incomplete = true + -- named capture). The second variant does not cross paragraph + -- boundaries. + local group_long = sequence( + Iouter_pos, + P{sequence(bgroup, + Ipos, + many(choice(comment_line, + V(1)/0, + token - egroup)), + Icont, + egroup + is_incomplete)}, + Iouter_cont) + local group = sequence( + Iouter_pos, + P{sequence(bgroup, + Ipos, + many(choice(comment_line, + V(1)/0, + token - egroup) + - blank_line), + Icont, + egroup + is_incomplete)}, + Iouter_cont) + +--* Trimming, cleaning, and cropping + local trimmer = blank^0 * C((blank^0 * (char - blank)^1)^0) + local cleaner = blank^0 * Cs(((blank^1 / " " + true) * (char - blank)^1)^0) + local comment_block = ((eol * space^0)^-1 * comment_line)^1 -- use the one below? + -- local comment_block = (comment_line * (Pend + eol * space^0))^0 + local comment_stripper = Cs((comment_block / "" + char)^0) + +--* Parsing lists + local skim_long = comment_line + group_long/0 + token + local listsep_skip = listsep * skip_long + + local list_item = Ct(Ipos * (skim_long - listsep)^1 * Icont) + local list_parser = skip_long * Ct((listsep_skip^0 * list_item)^0) + + local list_reader = skip_long * Ct( + (listsep_skip^0 * C((skim_long - listsep)^1) + / trim(space + eol + comment_line, char))^0) + + local key = Ct(Ipos * (skim_long - listsep - valsep)^1 * Icont) + local value = Ct(valsep * skip_long * Ipos * (skim_long - listsep)^0 * Icont) + local kvlist_item = Ct(Ipos * Cg(key, "key") * Cg(value, "value")^-1 * Icont) + local kvlist_parser = skip_long * Ct((listsep_skip^0 * kvlist_item)^0) + +--* Parsing command arguments + local patt_from_arg = function(arg) + local ret = skip + if arg.delimiters == false then + -- This is different from the “plain argument” case in that only + -- a single token is allowed. + ret = ret * Ipos * token * Icont + elseif arg.delimiters then + local l, r = arg.delimiters[1], arg.delimiters[2] + ret = sequence( + ret, Iouter_pos, l, Ipos, + many( + choice(comment_line, + group/0, + token) + - blank_line - r), + Icont, + r + is_incomplete, + Iouter_cont) + elseif arg.literal then + ret = ret * Ipos * P(arg.literal) * Icont + else -- plain argument + ret = ret * (group + Ipos * token * Icont) + end + if arg.optional then + ret = ret + is_omitted + end + return Ct(ret) + end + + local patt_from_args = function(args) + local ret = Icont + for i = #args, 1, -1 do + ret = (patt_from_arg(args[i]) + is_incomplete) * ret + end + return Ct(Ipos * ret) + end + patt_from_args = util.memoize1(patt_from_args) + + local parse_args = function(args, str, pos) + return match(patt_from_args(args), str, pos) + end + +--* Public patterns + self.group = group + self.group_long = group_long + self.next_par = next_par + self.comment_line = comment_line + self.cs = cs + self.csname = csname + self.blank_line = blank_line + +--* Public functions + self.parse_args = parse_args + self.is_blank_line = matcher(space^0 * eol) + self.next_nonblank = matcher(skip * I) + self.trim = matcher(trimmer) -- replace def of trimmer directly in here? + self.clean = matcher(cleaner) + self.strip_comments = matcher(comment_stripper) + self.skip_to_bracketed = matcher( -- for tikz paths + search( + patt_from_arg{delimiters = {"[", "]"}}, + skim_long - blank_line)) -- also exclude ";"? + + -- Parse a list, return a sequence of ranges + self.parse_list = matcher(list_parser) + -- Parse a list, return the item contents as strings + self.read_list = matcher(list_reader) + -- Parse a key-value list, return their contents as strings + self.parse_kvlist = matcher(kvlist_parser) + + -- Match a normal control sequence name starting with prefix + function self.cs_matcher(prefix) + local patt = P(prefix) * letter^0 * Pend + return matcher(patt) + end + + -- Build a pattern for Manuscript.scan, which looks ahead for one of + -- the elements of the set `things`. A match produces 4 captures: a + -- position before the item, the kind of item ("cs", "mathshift" or + -- "par"), a detail (e.g., the control sequence if kind is "cs"), + -- and a position after the item. + -- + self.scan_patt = function(things) + local patt = Kcs * csname + if things.mathshift then + patt = patt + mathshift * Kmathshift * (mathshift * Cc"$$" + Cc"$") + end + if things.par then patt = patt + par * Kpar * Knil end + return search(I * patt * I, comment_line + char) + end + +end + +return Parser diff --git a/support/digestif/digestif/Schema.lua b/support/digestif/digestif/Schema.lua new file mode 100644 index 0000000000..b59a95157d --- /dev/null +++ b/support/digestif/digestif/Schema.lua @@ -0,0 +1,189 @@ +--- A simple validation library, loosely inspired by JSON Schema + +local Schema = {} +Schema.__index = Schema + +local function is_schema(tbl) + return getmetatable(tbl) == Schema +end +Schema.is_schema = is_schema + +local validators, initializers = {}, {} + +local function to_schema(tbl) + if is_schema(tbl) then return tbl end + for i = 1, #tbl do + tbl[i] = to_schema(tbl[i]) + end + -- local fields, items = tbl.fields, tbl.items + -- if fields then + -- for k, v in pairs(fields) do + -- fields[k] = to_schema(v) + -- end + -- end + -- if items then + -- tbl.items = to_schema(items) + -- end + for k, fun in pairs(initializers) do + if tbl[k] then tbl[k] = fun(tbl[k]) end + end + return setmetatable(tbl, Schema) +end +setmetatable(Schema, {__call = function(_, tbl) return to_schema(tbl) end}) + +local function msg_type(expect, got) + return ("Type: expected %q, got %q"):format(expect, got) +end + +local function msg_where(msg, loc_type, loc_name) + return ("%s in %s %q"):format(msg, loc_type, loc_name) +end + +initializers.items = to_schema +initializers.keys = to_schema +initializers.values = to_schema + +function initializers.enum(tbl) + local set = {} + for i = 1, #tbl do + set[tbl[i]] = true + end + return set +end + +function initializers.fields(fields) + for k, v in pairs(fields) do + fields[k] = to_schema(v) + end + return fields +end + +function validators.type(s, obj) + local ok = (type(obj) == s) + return ok, ok or msg_type(s, type(obj)) +end + +function validators.predicate(f, obj) + return f(obj) +end + +function validators.keys(schema, obj) + if type(obj) ~= "table" then + return false, msg_type("table", type(obj)) + end + for k in pairs(obj) do + local ok, msg = schema:validate(k) + if not ok then + return false, msg_where(msg, "key", k) + end + end + return true +end + +function validators.values(schema, obj) + if type(obj) ~= "table" then + return false, msg_type("table", type(obj)) + end + for k, v in pairs(obj) do + local ok, msg = schema:validate(v) + if not ok then + return false, msg_where(msg, "field", k) + end + end + return true +end + +function validators.fields(fields, obj) + if type(obj) ~= "table" then + return false, msg_type("table", type(obj)) + end + for k, schema in pairs(fields) do + local ok, msg = schema:validate(obj[k]) + if not ok then + return false, msg_where(msg, "field", k) + end + end + return true +end + +function validators.items(schema, obj) + if type(obj) ~= "table" then + return false, msg_type("table", type(obj)) + end + for i, item in ipairs(obj) do + local ok, msg = schema:validate(item) + if not ok then + return false, msg_where(msg, "item", i) + end + end + return true +end + +function validators.enum(set, obj) + local ok = set[obj] + return ok, ok or ('Enum: %q not allowed'):format(obj) +end + +function Schema:validate(obj) + if self.optional and obj == nil then return true end + for k, v in pairs(self) do + local fun = validators[k] + if fun then + local ok, msg = fun(v, obj) + if not ok then return false, msg end + end + end + local len = #self + if len > 0 then + for i = 1, len do + if self[i]:validate(obj) then break end + if i == len then return false, "Alternatives: no match" end + end + end + return true +end + +function Schema:assert(obj) + return assert(self:validate(obj)) +end + +function Schema:assert_fail(obj) + assert(not self:validate(obj)) +end + +local schema_of_schema = Schema { + fields = { + description = { + type = "string", + optional = true, + description = "A docstring for the schema" + }, + fields = { + type = "table", + values = {predicate = is_schema} + }, + enum = { + type = "table" + }, + items = { + predicate = is_schema + }, + optional = { + type = "boolean", + optional = true, + description = "Whether nil values are accepted" + }, + predicate = { + type = "function", + optional = true, + description = "A function to test on the object" + }, + type = { + type = "string", + optional = true, + description = "Check if the object is of a given type" + }, + } +} + +return Schema diff --git a/support/digestif/digestif/bibtex.lua b/support/digestif/digestif/bibtex.lua new file mode 100644 index 0000000000..f1430474e3 --- /dev/null +++ b/support/digestif/digestif/bibtex.lua @@ -0,0 +1,271 @@ +local lpeg = require "lpeg" +local util = require "digestif.util" + +local B, P, R, S, V + = lpeg.B, lpeg.P, lpeg.R, lpeg.S, lpeg.V +local C, Cc, Cp, Ct, Cmt, Cg + = lpeg.C, lpeg.Cc, lpeg.Cp, lpeg.Ct, lpeg.Cmt, lpeg.Cg +local concat = table.concat +local merge = util.merge +local search, gobble_until, case_fold = util.search, util.gobble_until, util.case_fold +local split, replace = util.split, util.replace + +local bibtex = {} + +local function ipairs_from(t, i) + return ipairs(t), t, i - 1 +end + +--* Parser + +local char = P(1) +local at_sign = P"@" +local newline = P"\n" +local space = S" \r\n\t" +local whitespace = space^0 +local comment = P"%" * (1 - newline)^0 * newline +local junk = (comment + 1 - at_sign)^0 +local number = C(R"09"^1) * whitespace +local name = C((R"az" + R"AZ" + R"09" + S"!$&*+-./:;<>?[]^_`|")^1) * whitespace +local lbrace = P"{" * whitespace +local rbrace = P"}" * whitespace +local lparen = P"(" * whitespace +local rparen = P")" * whitespace +local equals = P"=" * whitespace +local hash = P"#" * whitespace +local comma = P"," * whitespace +local quote = P'"' * whitespace +local lit_string = C(case_fold "string") * whitespace +local lit_comment = C(case_fold "comment") * whitespace +local lit_preamble = C(case_fold "preamble") * whitespace + +local Cstart = Cg(Cp(), "start") +local Cstop = Cg(Cp(), "stop") + +local curly_braced_string = util.between_balanced("{", "}") +local round_braced_string = util.between_balanced("(", ")") +local braced_string = (curly_braced_string + round_braced_string) * whitespace +local quoted_string = '"' * C(gobble_until('"')) * '"' * whitespace +local simple_value = quoted_string + braced_string + number + Ct(name) +local value = simple_value * (hash * simple_value)^0 +local field = Ct(name * equals * value) + whitespace +local fields = field * (comma * field)^0 + +local token = curly_braced_string/0 + char +local nonspace = token - space +local author_sep = space * "and" * space +local etal_marker = P("et al") * whitespace * P(-1) + +-- either curly or round braced +local function braced(patt) + return lbrace * patt * rbrace + lparen * patt * rparen +end + +local string_entry = at_sign * lit_string * braced(fields) +local comment_entry = at_sign * lit_comment * braced_string +local preamble_entry = at_sign * lit_preamble * braced(value) +local regular_entry = at_sign * name * braced(name * comma * fields) +local entry = string_entry + comment_entry + preamble_entry + regular_entry + +-- this pattern produces the parse tree +-- TODO: catch premature end on invalid entry +local all_entries = Ct((junk * Ct(Cstart * entry * Cstop))^0) + +--* Translate parse tree + +-- BibItem class +local BibItem = {} +local mt = {__index = BibItem} +setmetatable( + BibItem, { + __call = function(_,t) return setmetatable(t, mt) end +}) + +-- replace user-defined strings and concatenate +local function process_value(val, strings, i) + i = i or 2 + local t = {} + for _, v in ipairs_from(val, i) do + if type(v) == "table" then + t[#t+1] = strings[v[1]] or "" + else + t[#t+1] = v + end + end + return concat(t) +end + +local default_options = { + with_authors = true, + with_title = true +} + +--- Parse a bibtex file. +function bibtex.parse(src, options) + options = merge(default_options, options) + local entries = all_entries:match(src) + local strings = merge(options.strings) + local preambles = {} + local ids = {} + local items = { + strings = strings, + preambles = preambles, + ids = ids + } + for _, t in ipairs(entries) do + local entry_type = t[1]:lower() + if entry_type == "comment" then + -- pass + elseif entry_type == "preamble" then + preambles[#preambles + 1] = t[2] + elseif entry_type == "string" then + for _, u in ipairs_from(t, 2) do + local key = u[1]:lower() + local val = process_value(u, strings) + strings[key] = val + end + else + local id = t[2] + local fields = {} + for _, u in ipairs_from(t, 3) do + local key = u[1]:lower() + local val = process_value(u, strings) + fields[key] = val + end + local item = BibItem { + id = id, + type = entry_type, + fields = fields, + pos = t.start, + cont = t.stop + } + ids[id] = item + items[#items + 1] = item + end + end + return items +end + +--* Deuglify strings + +local tex_symbols = { + oe = "œ", + OE = "Œ", + ae = "ӕ", + AE = "Ӕ", + aa = "å", + AA = "Å", + ss = "ß", + o = "ø", + O = "Ø", + l = "ł", + L = "Ł", + -- corner cases + i = "{i}", -- not "ı" + j = "j", -- not "ȷ" + [" "] = " ", +} + +local tex_accents = util.map( + replace("◌", ""), + { + ['"'] = "◌̈", + ["'"] = "◌́", + ["."] = "◌̇", + ["="] = "◌̄", + ["^"] = "◌̂", + ["`"] = "◌̀", + ["~"] = "◌̃", + ["c"] = "◌̧", + ["d"] = "◌̣", + ["H"] = "◌̋", + ["u"] = "◌̆", + ["b"] = "◌̱", + ["v"] = "◌̌", + ["t"] = "◌͡" + } +) + +local tex_letter = (R"AZ" + R"az") +local tex_char_or_math = "$" * gobble_until("$") * "$" + char -- for deuglification purposes +local tex_cs_patt = "\\" * C(tex_letter^1 + char)* whitespace +local tex_accent_patt = tex_cs_patt * (curly_braced_string + C(char)) +local function repl_accents_fun(cs, arg) + local acc = tex_accents[cs] + if not acc then + return -- must return 0 values! + else + return arg .. acc + end +end + +local detexify_symbols = replace(tex_cs_patt, tex_symbols, tex_char_or_math) +local detexify_accents = replace(tex_accent_patt, repl_accents_fun, tex_char_or_math) +local debracify = replace(curly_braced_string, 1, tex_char_or_math) +local detitlify = replace(B(space) * C(tex_letter), string.lower, tex_char_or_math) +local trim = util.trim(space) +local clean = util.clean(space) + +local function deuglify_name (s) + return + clean( + debracify( + detexify_accents( + detexify_symbols(s)))) +end + +local function deuglify_title (s) + return + clean( + debracify( + detitlify( + detexify_accents( + detexify_symbols(s))))) +end + +--* Pretty-printing + +local split_authors = split(author_sep, token) +local split_name = split(comma, token) +local split_last = search(Cp() * C(nonspace^1) * whitespace * P(-1)) + +function BibItem:authors() + local t = {} + local author = self.fields.author + if not author then return {} end + for _, name in ipairs(split_authors(author)) do + local u = {} + local parts = split_name(name) + if #parts == 3 then + u.first = parts[3] + u.last = parts[1] + u.suffix = parts[2] + elseif #parts == 2 then + u.first = parts[2] + u.last = parts[1] + else + local p, l = split_last:match(name) + if p then + u.first = name:sub(1, p - 1) + u.last = l + else + u.last = name + end + end + t[#t + 1] = u + end + return t +end + +function BibItem:pretty_print() + local t, a = {}, {} + for _, name in ipairs(self:authors()) do + a[#a + 1] = deuglify_name(name.last) + end + t[#t + 1] = concat(a, ", ") + t[#t + 1] = (self.fields.year or self.fields.date or "(n.d.)") .. ";" + t[#t + 1] = deuglify_title(self.fields.title or "") + return concat(t, " ") +end + +return bibtex diff --git a/support/digestif/digestif/config.lua b/support/digestif/digestif/config.lua new file mode 100644 index 0000000000..8b901ba93d --- /dev/null +++ b/support/digestif/digestif/config.lua @@ -0,0 +1,156 @@ +local util = require "digestif.util" +local format = string.format + +local config = {} + +config.version = "0.5" +local pre_version = os.getenv("DIGESTIF_PRERELEASE") +if pre_version then + config.version = config.version .. "-" .. pre_version +end + +if util.is_command("kpsewhich") then + local pipe = io.popen("kpsewhich -var-brace-value=TEXMF") + local output = pipe:read("l") + local ok, exitt, exitc = pipe:close() + if ok and exitt == "exit" and exitc == 0 then + config.texmf_dirs = util.imap( + function (s) return s:gsub("^!!", "") end, + util.path_list_split(output) + ) + elseif config.verbose then + util.log("Error running kpsewhich (%s %d)", exitt, exitc) + end +else -- TODO: What should be the default? + config.texmf_dirs = { + "/usr/local/share/texmf", + "/usr/share/texmf", + "/usr/share/texlive/texmf-local", + "/usr/share/texlive/texmf-dist", + } +end + +config.data_dirs = {} -- TODO: What should be the default? + +-- Location of a complete texmf distribution, used for instance to +-- find documentation not installed locally. Passed to format with +-- one argument, the percent-encoded name of a file. +config.external_texmf = "https://www.tug.org/texlive/Contents/live/texmf-dist/%s" + +config.provide_snippets = false + +-- Table mapping command names to the snippet to be used, overriding +-- the default. +config.extra_snippets = {} + +-- This allows the user to assign custom actions to generated command +-- tags. +config.extra_actions = { + eqref = "ref" +} + +config.fuzzy_cite = true +config.fuzzy_ref = true +config.info_command = util.is_command("info") + +-- For candidates of these kinds, include the annotation in the +-- candidate label. The values of this table are a string which is +-- formatted with two arguments, the candidate text and annotation. +config.lsp_long_candidates = { + label = "%-12s %s", + bibitem = "%-12s %s", +} + +--* Loading user settings + +local function is_table(key_type, val_type) + return function(obj) + if type(obj) ~= "table" then return false end + for key, val in pairs(obj) do + if type(key) ~= key_type then return false end + if type(val) ~= val_type then return false end + end + return true + end +end + +local validators = { + data_dirs = is_table("number", "string"), + extra_actions = is_table("string", "string"), + extra_snippets = is_table("string", "string"), + fuzzy_cite = "boolean", + fuzzy_ref = "boolean", + info_command = "string", + lsp_long_candidates = is_table("string", "string"), + provide_snippets = "boolean", + texmf_dirs = is_table("number", "string"), + tlpdb_path = "string", + verbose = "boolean", +} + +-- Set config entries found in `tbl`. +function config.load_from_table(tbl) + -- Set verbose option before all others + local verbose = tbl.verbose or config.verbose + for key, val in pairs(tbl) do + local validator, ok = validators[key] + if type(validator) == "string" then + ok = type(val) == validator + elseif type(validator) == "function" then + ok = validator(val) + end + if ok then + config[key] = val + if verbose then + local msg = "Setting configuration option %s = %s" + util.log(msg, key, util.inspect(val)) + end + else + local msg = "Invalid configuration option: %s = %s" + error(format(msg, key, util.inspect(val))) + end + end +end + +function config.load_from_file(file) + local tbl = {} + local ok, err = loadfile(file, "t", tbl) + if ok then ok, err = pcall(ok) end + if not ok then + local msg = "Error loading configuration from %s: %s" + error(msg, file, err) + end + config.load_from_table(tbl) +end + +function config.load_from_env() + local DIGESTIF_DATA = os.getenv("DIGESTIF_DATA") + if DIGESTIF_DATA then + config.data_dirs = util.path_list_split(DIGESTIF_DATA) + end + + local DIGESTIF_TEXMF = os.getenv("DIGESTIF_TEXMF") + if DIGESTIF_TEXMF then + config.texmf_dirs = util.path_list_split(DIGESTIF_TEXMF) + end + + local DIGESTIF_TLPDB = os.getenv("DIGESTIF_TLPDB") + if DIGESTIF_TLPDB then + config.tlpdb_path = util.path_list_split(DIGESTIF_TLPDB) + end +end + +function config.check_data(dir) + if not dir then + for _, dir in ipairs(config.data_dirs) do + if config.check_data(dir) then return true end + end + end + local f = io.open(util.path_join(dir, "primitives.tags")) + if f then + f:close() + return true + end +end + +return config diff --git a/support/digestif/digestif/data.lua b/support/digestif/digestif/data.lua new file mode 100644 index 0000000000..621048f4a9 --- /dev/null +++ b/support/digestif/digestif/data.lua @@ -0,0 +1,384 @@ +local lpeg = require "lpeg" +local util = require "digestif.util" +local config = require "digestif.config" + +local format, strfind = string.format, string.find +local concat, unpack = table.concat, table.unpack +local popen = io.popen +local P, C, Cg, Ct = lpeg.P, lpeg.C, lpeg.Cg, lpeg.Ct +local match = lpeg.match +local many, sequence = util.many, util.sequence +local gobble, search = util.gobble, util.search +local nested_get, extend = util.nested_get, util.extend +local find_file = util.find_file +local parse_uri, make_uri = util.parse_uri, util.make_uri +local log = util.log +local path_split, path_join = util.path_split, util.path_join + +local data = {} +local loaded_tags = {} + +--* CTAN data + +-- function ctan_package(name) +-- +-- Return a little tags table with the package's details on CTAN +-- (package description, ctan link, documentation). No command +-- information. +-- +-- function ctan_package_of(file) +-- +-- The name of the CTAN package to which file belongs. +-- +local ctan_package, ctan_package_of -- to be defined + +local tlpdb_path = config.tlpdb_path + and find_file(config.tlpdb_path) + or find_file(config.texmf_dirs, "../tlpkg/texlive.tlpdb") + +if tlpdb_path then + + local _, tlpdb_text = find_file(tlpdb_path, nil, true) + + if config.verbose then + log("Reading TLPDB from '%s'", tlpdb_path) + end + + local Peol = P"\n" + local concat_lines = function(...) return concat({...}, "\n") end + + local within_item = 1 - Peol * Peol + local within_files = (1 - Peol) + Peol * P" " + local gobble_to_eol = gobble("\n") + + -- Pattern to turn an entry in the TLPDB file into a table with + -- entries name, summary, details, documentation, runfiles. + local tlpdb_item_patt = Ct( + sequence( + Cg( -- collect the name + P"name " * C(gobble_to_eol), + "ctan_package"), + Cg( -- find a shortdesc or give up + search( + Peol * "shortdesc " * C(gobble_to_eol), + within_item), + "summary"), + Cg( -- find a longdesc or give up + search( + (Peol * "longdesc " * C(gobble_to_eol))^1 / concat_lines, + within_item), + "details"), + Cg( -- find docfiles section + many(-1, -- or continue + sequence( + search(Peol * "docfiles " * gobble_to_eol, within_item), + Ct(many( -- collect several docfile entries in a list + search( -- looking for the interesting ones only + Ct( -- parse one docfile entry + sequence( -- collect URI, but only if details field exist + Peol * " RELOC/", + Cg(gobble(" ", 1 - Peol) / "texmf:%0", "uri"), + Cg(P" details=\"" * C(gobble"\""), "summary"))), + within_files))))), + "documentation"), + Cg( -- find runfiles section or give up + sequence( + search(Peol * "runfiles " * gobble_to_eol, within_item), + Ct(many( -- collect the runfile entries is a list + sequence( -- parse one runfile entry + Peol * " ", -- discard the file line marker + many(search("/", 1 - Peol)), -- discard folder part + C(gobble_to_eol))))), -- collect base name + "runfiles"))) + + local tlpdb_items = Ct(many(search(Peol * tlpdb_item_patt))):match(tlpdb_text) + + function ctan_package(name) + for i = 1, #tlpdb_items do + local item = tlpdb_items[i] + if item.name == name then + return item + end + end + end + ctan_package = util.memoize1(ctan_package) + + function ctan_package_of(file) + for i = 1, #tlpdb_items do + local item = tlpdb_items[i] + local runfiles = item.runfiles + for j = 1, #runfiles do + if runfiles[j] == file then + return item + end + end + end + end + ctan_package_of = util.memoize1(ctan_package_of) + +else + + ctan_package = function() end + ctan_package_of = function() end + if config.verbose then log("TLPDB not found") end + +end + +--* kpathsea emulation + +-- function kpsewhich(name) +-- +-- Like the kpsewhich command, return the full path of tex input file +-- name. We use luatex's kpse bindings if available, and parse the +-- system's ls-R files otherwise. +-- +local kpsewhich + +if kpse then -- we're on luatex + + local kpse_obj = kpse.new("luatex") + + function kpsewhich(name) + return kpse_obj:find_file(name, "tex") + end + +else -- on plain lua, we look for ls-R files + + local texmf_files = {} + local texmf_dirs = config.texmf_dirs + local dir_patt = P"./" * C(gobble(":" * P(-1))) + for i = 1, #texmf_dirs do + local texmf_dir = texmf_dirs[i] + local listing_path = find_file(texmf_dir, "ls-R") + if listing_path then + local current_dir + for line in io.lines(listing_path) do + local subdir = match(dir_patt, line) + if subdir then + current_dir = path_join(texmf_dir, subdir) + elseif current_dir and not texmf_files[line] then + texmf_files[line] = path_join(current_dir, line) + end + end + end + end + + function kpsewhich(name) + return texmf_files[name] + end + +end + +--* Generate tags from the user's TeX installation + +local function infer_format(path) + local ext = path:sub(-4) + if ext == ".bib" then + return "bibtex" + elseif ext == ".sty" or ext == ".cls" or ext == ".ltx" then + return "latex-prog" + elseif ext == ".xml" and path:match("%Acontext%A") then + return "context-xml" + elseif ext == ".tex" then + return "latex-prog" + end +end + +local function tags_from_manuscript(script, ctan_data) + local commands, environments, dependencies = {}, {}, {} + local tags = { + generated = true, + dependencies = dependencies, + commands = commands, + environments = environments + } + if ctan_data then + setmetatable(tags, {__index = ctan_data}) + end + for pkg in pairs(script.packages) do + dependencies[#dependencies+1] = pkg + end + table.sort(dependencies) + for _, it in script:index_pairs("newcommand") do + commands[it.name] = { + arguments = it.arguments, + } + end + for _, it in script:index_pairs("newenvironment") do + environments[it.name] = { + arguments = it.arguments, + } + end + return tags +end + +-- Generate tags from TeX source code (or ConTeXt interface XML file, +-- if applicable). The argument is a file name found by kpsewhich. +-- +local function generate_tags(name) + local path = kpsewhich(name) + path = path and find_file(config.texmf_dirs, path) + if not path then return end + local texformat = infer_format(path) + if not texformat then return end + if config.verbose then log("Generating tags: %s", path) end + local pkg = ctan_package_of(name) + if texformat == "context-xml" then + -- The function below is defined and monkey-patched in + -- ManuscriptConTeXt, to keep the XML dependency separated. + if not data.tags_from_xml then require "digestif.ManuscriptConTeXt" end + return data.tags_from_xml(path, pkg) + else + loaded_tags[name] = {} -- TODO: this is to avoid loops, find a better way + local cache = require "digestif.Cache"() + local script = cache:manuscript(path, texformat) + return tags_from_manuscript(script, pkg) + end +end +data.generate_tags = generate_tags + +--* Load tags + +-- function require_tags(name) +-- +-- Return tags table for input file name. Either loads from a tags +-- file in the data directory, or generate from source on the fly. +-- +local require_tags + +local parse_ref = util.matcher( + sequence( + P"$ref:", + C(gobble"#") * P"#", + Ct(many(P"/" * C(gobble"/"))))) +p=parse_ref +local function resolve_refs(tbl, seen) + seen = seen or {} + for k, v in pairs(tbl) do + if type(v) == "string" then + local loc, path = parse_ref(v) + if loc then + tbl[k] = nested_get(require_tags(loc), unpack(path)) + end + elseif type(v) == "table" and not seen[v] then + seen[v] = true + resolve_refs(v, seen) + end + end +end + +-- Load a tags file from the data directory. +local function load_tags(name) + if strfind(name, "..", 1, true) then return end -- bad file name + local path, str = find_file(config.data_dirs, name .. ".tags", true) + if not path then return end + if config.verbose then log("Loading tags: %s", path) end + local tags = {} + local ok, message = load(str, path, "t", tags) + if ok then ok, message = pcall(ok) end + if not ok and config.verbose then + log("Error loading %s.tags: %s", name, message) + return -- TODO: should throw an error? + end + local pkg = ctan_package(tags.ctan_package) or ctan_package_of(name) + setmetatable(tags, {__index = pkg}) + return tags +end + +require_tags = function(name) + local tags = loaded_tags[name] + if not tags then + tags = load_tags(name) or generate_tags(name) + if tags then + loaded_tags[name] = tags + resolve_refs(tags) + local extra_actions = config.extra_actions or {} + for _, kind in ipairs{"commands", "environments"} do + for cs, cmd in pairs(tags[kind] or {}) do + if not cmd.package then cmd.package = tags end + if extra_actions[cs] then cmd.action = extra_actions[cs] end + end + end + end + end + return tags +end +data.require = require_tags +data.require_tags = require_tags + +-- Load all data files, and return them in a table. This is intended +-- for debugging and testing only, and depends on luafilesystem. +local function load_all_tags() + local t = {} + local ok, lfs = pcall(require, "lfs") + assert(ok, "Function data.load_all() need the luafilesystem library.") + for _, data_dir in ipairs(config.data_dirs) do + for path in lfs.dir(data_dir) do + local pkg = path:match("(.*)%.tags") + if pkg then + assert(load_tags(pkg), "Error loading data file " .. path) + t[pkg] = require_tags(pkg) or error("Error processing data file " .. path) + end + end + end + return t +end +data.load_all = load_all_tags + +--* User-readable documentation + +local function resolve_uri(uri) + local scheme, _, location, _, fragment = parse_uri(uri) + if scheme == "info" then + return uri + elseif scheme == "texmf" then + local path = find_file(config.texmf_dirs, location) + if path then + return make_uri("file", "", path, nil, fragment) + else + return format( + config.external_texmf, make_uri(nil, nil, location, nil, fragment) + ) + end + else + return uri + end +end + +-- Given a list of documentation items, return a markdown-formatted +-- string of links to these documents. +local function resolve_doc_items(items) + if type(items) == "string" then items = {items} end + local t = {} + for _, item in ipairs(items) do + if type(item) == "string" then + t[#t+1] = format("- <%s>", resolve_uri(item)) + else + t[#t+1] = format("- [%s](%s)", item.summary, resolve_uri(item.uri)) + end + end + return t +end +data.resolve_doc_items = resolve_doc_items + +-- Call info, return the relevant part of the info node. +local function get_info(uri) + if config.info_command then + local scheme, _, path, _, fragment = parse_uri(uri) + if scheme ~= "info" then return end + local cmd = format("%s '(%s)%s'", config.info_command, path, fragment) + local pipe = popen(cmd) + local str = pipe:read("a") + local ok, exitt, exitc = pipe:close() + if ok and exitt == "exit" and exitc == 0 then + str = str:gsub(".-\n", "", 2) -- discard header line + str = str:gsub("\nFile: " .. path .. ".info.*", "\n") -- discard child nodes + return str, path, fragment + elseif config.verbose then + log("Error running info (%s %d)", exitt, exitc) + end + end +end +data.get_info=util.memoize(get_info) + +return data diff --git a/support/digestif/digestif/langserver.lua b/support/digestif/digestif/langserver.lua new file mode 100644 index 0000000000..55bd8c8dd7 --- /dev/null +++ b/support/digestif/digestif/langserver.lua @@ -0,0 +1,567 @@ +local config = require "digestif.config" +local util = require "digestif.util" + +local floor = math.floor +local format = string.format +local imap, nested_get, lines = util.imap, util.nested_get, util.lines +local parse_uri, make_uri = util.parse_uri, util.make_uri +local log = util.log + +-- Use cjson if available, otherwise fall back to `digestif.util` +-- implementation. +local null, json_decode, json_encode +if pcall(require, "cjson") then + local cjson = require "cjson" + cjson.encode_empty_table_as_object(false) + null = cjson.null + json_decode, json_encode = cjson.decode, cjson.encode +else + null = util.json_null + json_decode, json_encode = util.json_decode, util.json_encode +end + +--* Convert LSP API objects to/from internal representations + +-- This will be a digestif.Cache object. Its initialization is +-- deferred to `initialized` method. +local cache = setmetatable({}, { + __index = function () error "Server not initialized!" end +}) + +-- A place to store the file name and texformat of open documents. +local open_documents = setmetatable({}, { + __index = function(_, k) + error(format("Trying to access unopened document %s", k)) + end +}) + +local function from_DocumentUri(str) + local scheme, auth, path, query, fragment = parse_uri(str) + if scheme ~= "file" or (auth and auth ~= "") or query or fragment then + error("Invalid or unsupported URI: " .. str) + end + if util.os_type == "windows" and path:find("^/%a:") then + path = path:sub(2) + end + return path +end + +local function to_DocumentUri(str) + if util.os_type == "windows" then + str = str:gsub("[/\\]", "/") + if str:find("^%a:") then str = "/" .. str end + end + return make_uri("file", "", str) +end + +-- p0 is the position of a line l0, provided as a hint for the search. +-- Return a position in bytes, and a new hint p0, l0. +local function from_Position(str, position, p0, l0) + local l, c = position.line + 1, position.character + 1 + if l0 and l0 > l then p0, l0 = nil, nil end + for n, i in lines(str, p0, l0) do + if n == l then + return utf8.offset(str, c, i), i, l + end + end +end + +local function from_Range(str, range, p0, l0) + local pos, p1, l1 = from_Position(str, range.start, p0, l0) -- inclusive + local cont = from_Position(str, range['end'], p1, l1) -- exclusive + if not (pos and cont) then error("Position out of bounds") end + return pos, cont, p1, l1 +end + +local function from_TextDocumentIdentifier(arg) + local filename = from_DocumentUri(arg.uri) + local texformat = open_documents[filename] + local script = cache:manuscript(filename, texformat) + return script +end + +local function from_TextDocumentPositionParams(arg) + local filename = from_DocumentUri(arg.textDocument.uri) + local texformat = open_documents[filename] + local script = cache:manuscript(filename, texformat) + local l, c = arg.position.line + 1, arg.position.character + 1 + return script, script:position_at(l, c) +end + +local function to_Range(item) + local script = item.manuscript + local l1, c1 = script:line_column_at(item.pos) + local l2, c2 = script:line_column_at(item.cont) + return { + start = {line = l1 - 1, character = c1 - 1}, + ["end"] = {line = l2 - 1, character = c2 - 1}, + } +end + +local function to_Location(item) + return { + uri = to_DocumentUri(item.manuscript.filename), + range = to_Range(item) + } +end + +local function to_MarkupContent(str) + return {kind = "markdown", value = str} +end + +local function to_TextEdit(script, pos, old, new) + local l, c_start = script:line_column_at(pos) + local c_end = c_start + utf8.len(old) + return { + range = { + start = {line = l - 1, character = c_start - 1}, + ["end"] = {line = l - 1, character = c_end - 1}, + }, + newText = new + } +end + +-- An essentially random assignment of symbol kinds, since LSP doesn't +-- support custom kinds. +local to_SymbolKind = { + section = 5, + section_index = 5, + label_index = 7, + bib_index = 20, + newcommand_index = 12, + newenvironment_index = 12 +} + +local function to_SymbolInformation(item, index_name) + return { + name = item.name, + kind = to_SymbolKind[index_name], + location = to_Location(item) + } +end + +local function to_DocumentSymbol(outline) + return { + name = outline.name, + kind = to_SymbolKind[outline.kind], + range = to_Range(outline), + selectionRange = to_Range(outline), + children = outline[1] and imap(to_DocumentSymbol, outline) + } +end + +local languageId_translation_table = { + bibtex = "bibtex", + context = "context", + doctex = "doctex", + latex = "latex", + plain = "plain", + plaintex = "plain", + ["plain-tex"] = "plain", + tex = "latex", -- this is for vim; maybe "tex" should mean "tex file, undecided format" + texinfo = "texinfo" +} + +local function languageId_translate(id, filename) + local ext = filename:sub(-4) + local texformat = languageId_translation_table[id] + if not texformat then + error(("Invalid LSP language id %q"):format(id)) + end + if texformat == "latex" and (ext == ".sty" or ext == ".cls") then + return "latex-prog" + end + -- TODO: Handle .code.tex files from PGF, .mkiv files from ConTeXt, etc. + return texformat +end + +--* LSP methods + +local methods = {} + +methods["initialize"] = function(params) + config.provide_snippets = nested_get(params.capabilities, + "textDocument", "completion", "completionItem", "snippetSupport") + if params.initializationOptions then + config.load_from_table(params.initializationOptions) + end + return { + capabilities = { + textDocumentSync = { + openClose = true, + change = 2 + }, + completionProvider = { + triggerCharacters = {"\\", "{", "[", ",", "="}, + }, + signatureHelpProvider = { + triggerCharacters = {"{", "[", "="}, + }, + hoverProvider = true, + definitionProvider = true, + referencesProvider = true, + documentSymbolProvider = true, + workspaceSymbolProvider = true + }, + serverInfo = { + name = "Digestif", + version = config.version + } + } +end + +methods["initialized"] = function() + cache = require "digestif.Cache"() +end + +methods["shutdown"] = function() return null end +methods["exit"] = function() os.exit() end +methods["textDocument/willSave"] = function() end +methods["textDocument/didSave"] = function() end + +methods["workspace/didChangeConfiguration"] = function(params) + local settings = params.settings.digestif + if type(settings) ~= "table" then return end + config.load_from_table(settings) +end + +methods["textDocument/didOpen"] = function(params) + local filename = from_DocumentUri(params.textDocument.uri) + local texformat = languageId_translate(params.textDocument.languageId, filename) + open_documents[filename] = texformat + cache:put(filename, params.textDocument.text) +end + +methods["textDocument/didChange"] = function(params) + local filename = from_DocumentUri(params.textDocument.uri) + local p0, l0, src = 1, 1, cache(filename) + for _, change in ipairs(params.contentChanges) do + if change.range then + local pos, cont + pos, cont, p0, l0 = from_Range(src, change.range, p0, l0) + src = src:sub(1, pos - 1) .. change.text .. src:sub(cont) + else + src = change.text + end + end + cache:put(filename, src) +end + +methods["textDocument/didClose"] = function(params) + local filename = from_DocumentUri(params.textDocument.uri) + open_documents[filename] = nil + cache:forget(filename) +end + +methods["textDocument/signatureHelp"] = function(params) + local script, pos = from_TextDocumentPositionParams(params) + local help = script:describe(pos) + if not help or not help.arg then return null end + local parameters, label_positions = {}, help.label_positions or {} + for i, arg in ipairs(nested_get(help, "data", "arguments") or {}) do + parameters[i] = { + label = {label_positions[2*i-1] - 1, label_positions[2*i] - 1}, + documentation = arg.summary + } + end + return { + signatures = { + [1] = { + label = help.label, + documentation = help.summary, + parameters = parameters, + activeParameter = help.arg - 1 + } + }, + activeSignature = 0, + activeParameter = help.arg - 1 + } +end + +methods["textDocument/hover"] = function(params) + local script, pos = from_TextDocumentPositionParams(params) + local help = script:describe(pos) + if (not help) or help.arg then return null end + local contents = help.details or help.summary or "???" + return {contents = to_MarkupContent(contents)} +end + +methods["textDocument/completion"] = function(params) + local script, pos = from_TextDocumentPositionParams(params) + local candidates = script:complete(pos) + if not candidates then return null end + local long_format = config.lsp_long_candidates + and config.lsp_long_candidates[candidates.kind] + local items = {} + for i, cand in ipairs(candidates) do + local snippet = config.provide_snippets and cand.snippet + local fuzzy_score = cand.fuzzy_score or nil + local annotation = cand.annotation + local long_label = long_format and annotation + and format(long_format, cand.text, annotation) + items[i] = { + label = long_label or cand.text, + sortText = fuzzy_score and format("~%03d", floor(1000 * (1 - fuzzy_score))), + documentation = cand.summary, + detail = not long_label and cand.annotation or nil, + insertTextFormat = snippet and 2 or 1, + textEdit = to_TextEdit( + script, + candidates.pos, + candidates.prefix, + snippet or cand.text + ) + } + end + return items +end + +methods["textDocument/definition"] = function(params) + local script, pos = from_TextDocumentPositionParams(params) + local definition = script:find_definition(pos) + return definition and to_Location(definition) or null +end + +methods["textDocument/references"] = function(params) + local script, pos = from_TextDocumentPositionParams(params) + local result = {} + if params.context and params.context.includeDeclaration then + local definition = script:find_definition(pos) + if definition then + result[#result + 1] = to_Location(definition) + end + end + local references = script:find_references(pos) + if references then + for _, ref in ipairs(references) do + result[#result + 1] = to_Location(ref) + end + end + if #result > 0 then + return result + else + return null + end +end + +methods["textDocument/documentSymbol"] = function(params) + local script = from_TextDocumentIdentifier(params.textDocument) + local outline = script:outline(true) -- local only + return imap(to_DocumentSymbol, outline) +end + +methods["workspace/symbol"] = function(params) + local query, t = params.query, {} + + -- Find all root documents and sort them + local root_documents, sorted = {}, {} + for filename, texformat in pairs(open_documents) do + local script = cache:manuscript(filename, texformat) + root_documents[script.root.filename] = script.root + end + + for filename in pairs(root_documents) do + sorted[#sorted+1] = filename + end + table.sort(sorted) + + -- Gather all entries in all indexes + for _, filename in ipairs(sorted) do + local script = root_documents[filename] + for item, index_name in script:traverse { + "section_index", + "label_index", + "bib_index", + "newcommand_index", + "newenvironment_index" + } do + if item.name:find(query, 1, true) then + t[#t+1] = to_SymbolInformation(item, index_name) + end + end + end + return t +end + +--* RPC functions + +local function log_error(err) + if config.verbose then + log("Error: %s", err) + log(debug.traceback()) + end + return err +end + +local crlf = util.os_type == "windows" and "\n" or "\r\n" + +local function write_msg(msg) + io.write("Content-Length: ", #msg, crlf, crlf, msg) + io.flush() +end + +local function read_msg() + local headers, msg = {}, nil + for h in io.lines() do + if h == "" or h == "\r" then break end + local k, v = string.match(h, "^([%a%-]+): (.*)") + if k then headers[k] = v end + end + local len = tonumber(headers["Content-Length"]) + if len then msg = io.read(len) end + return msg, headers +end + +local function rpc_send(id, result, error_code) + write_msg( + json_encode({ + jsonrpc = "2.0", + id = id, + result = not error_code and result or nil, + error = error_code and {code = error_code, message = result} + })) +end + +local function rpc_receive() + local msg = read_msg() + local ok, request = xpcall(json_decode, log_error, msg) + if not ok then + rpc_send(null, request, -32700) + os.exit(false) + end + return request.id, request.method, request.params +end + +--* The main loop + +local is_optional = util.matcher("$/") + +local function process_request() + local clock = config.verbose and os.clock() + local id, method_name, params = rpc_receive() + local method = methods[method_name] + if method then + local ok, result = xpcall(method, log_error, params) + if ok then + if id then rpc_send(id, result) end + else + rpc_send(id, result, 1) + end + elseif not is_optional(method_name) then + rpc_send(id, "Unknown method " .. method_name, -32601) + end + if clock then + log("Request: %4s %-40s %6.2f ms", + id or "*", method_name, 1000 * (os.clock() - clock)) + end +end + +local function generate(path) + local generate_tags = require "digestif.data".generate_tags + local tags = generate_tags(path) + if not tags then + io.stderr:write( + format("Error: can't find '%s' or can't generate tags from it.\n", path) + ) + os.exit(false) + end + local _, basename = util.path_split(path) + local file = io.open(basename .. ".tags", "w") + for _, item in ipairs( + {"generated", "dependencies", "documentation", "commands", "environments"}) + do + if tags[item] then + file:write(item, " = ", util.inspect(tags[item]), "\n") + end + end + local i, j = 0, 0 + for _ in pairs(tags.commands) do i = i + 1 end + for _ in pairs(tags.environments) do j = j + 1 end + io.stderr:write( + format("Generated %15s.tags with %3i commands and %3i environments.\n", + basename, i, j) + ) +end + +local usage = [[ +Usage: digestif [--version] [-h] [-v] [-g FILES] +]] + +local help = [[ +Digestif is a language server for TeX + +Optional arguments: + -g, --generate FILES Generate data file stubs for FILES + -h, --help Display this message and exit + -v, --verbose Enable log output to stderr + --version Show version information + +Environment variables: + DIGESTIF_DATA Paths to look for data files + DIGESTIF_TEXMF Paths to look for TeX files + DIGESTIF_TLPDB Path to the TeXLive package database file + + If your TeX distribution or Digestif are installed in a non-standard + location, you may need to set some of the above variables. +]] + +local function main(arg) + -- Set up default config.data_dirs, if needed + config.load_from_env() + if #config.data_dirs == 0 then + for _, dir in ipairs{ + util.path_split(debug.getinfo(1).source:match("^@(.*)")), + util.path_split(arg[0]), + nil + } do + local f = io.open(util.path_join(dir, "../data/primitives.tags")) + if f then + f:close() + config.data_dirs = {util.path_join(dir, "../data")} + break + end + end + end + + -- Read CLI args + while arg[1] do + local switch = table.remove(arg, 1) + if switch == "-v" or switch == "--verbose" then + config.verbose = true + elseif switch == "-g" or switch == "--generate" then + for i = 1, #arg do generate(arg[i]) end + os.exit() + elseif switch == "-h" or switch == "--help" then + io.write(usage) + io.write(help) + os.exit() + elseif switch == "--version" then + io.write(format("Digestif %s\n", config.version)) + os.exit() + else + io.stderr:write(usage) + io.stderr:write(format("Invalid option: %s\n", switch)) + os.exit(false) + end + end + + -- Check if config.data_dirs was set up correctly + if not util.find_file(config.data_dirs, "primitives.tags") then + io.stderr:write( + "Error: could not find data files at the following locations:\n\t" + .. table.concat(config.data_dirs, "\n\t") + .. "\nSet the DIGESTIF_DATA environment variable to fix this.\n" + ) + os.exit(false) + end + + -- Main language server loop + if config.verbose then log("Digestif started!") end + while true do process_request() end +end + +return { + main = main, + methods = methods +} + diff --git a/support/digestif/digestif/util.lua b/support/digestif/digestif/util.lua new file mode 100644 index 0000000000..2e042ebd70 --- /dev/null +++ b/support/digestif/digestif/util.lua @@ -0,0 +1,789 @@ +-- Assorted utility functions + +local lpeg = require "lpeg" +local zip = pcall(require, "zip") and require "zip" + +local io, os = io, os +local co_yield, co_wrap = coroutine.yield, coroutine.wrap +local strupper, strfind, strsub, strrep = string.upper, string.find, string.sub, string.rep +local strchar, strbyte, utf8_char = string.char, string.byte, utf8.char +local format, gsub = string.format, string.gsub +local pack, unpack, concat = table.pack, table.unpack, table.concat +local move, sort = table.move, table.sort +local pairs, getmetatable, setmetatable = pairs, getmetatable, setmetatable +local P, V, R, S, I, B = lpeg.P, lpeg.V, lpeg.R, lpeg.S, lpeg.Cp(), lpeg.B +local C, Cs, Cf, Ct, Cc, Cg = lpeg.C, lpeg.Cs, lpeg.Cf, lpeg.Ct, lpeg.Cc, lpeg.Cg +local match, locale_table = lpeg.match, lpeg.locale() + +local util = {} + +--* Table manipulation + +local function map(f, t) + local r = {} + for k, v in pairs(t) do + r[k] = f(v) + end + return r +end +util.map = map + +local function imap(f, t) + local r = {} + for i = 1, #t do + r[i] = f(t[i]) + end + return r +end +util.imap = imap + +-- Return a table with entries (k, f(k)), where k ranges over the keys +-- of t and all its __index parents. +local function map_keys(f, t) + local mt = getmetatable(t) + local p = mt and mt.__index + local r = p and map_keys(f, p) or {} + for k in pairs(t) do + r[k] = f(k) + end + return r +end +util.map_keys = map_keys + +local function foldl1(f, t) + local v = t[1] + for i = 2, t.n or #t do + v = f(v, t[i]) + end + return v +end +util.foldl1 = foldl1 + +local function nested_get(t, ...) + local arg = pack(...) + for i = 1, #arg do + if t then t = t[arg[i]] else return nil end + end + return t +end +util.nested_get = nested_get + +local function nested_put(v, t, ...) + local arg = pack(...) + local k = arg[1] + for i = 2, arg.n do + local u = t[k] + if u == nil then u = {}; t[k] = u end + t, k = u, arg[i] + end + t[k] = v + return v +end +util.nested_put = nested_put + +-- Copy all entries of s onto t +local function update(t, s) + if s then + for k, v in pairs(s) do + t[k] = v + end + end + return t +end +util.update = update + +-- Return a new table containing all entries of the given tables, +-- priority given to the latter ones. +local function merge(...) + return foldl1(update, pack({}, ...)) +end +util.merge = merge + +-- Copy entries of the second list to the end of the first. +local function extend(s, t) + return move(t, 1, #t, #s+1, s) +end +util.extend = extend + +-- Iterate over a table of tables. +local function triples(t) + return co_wrap( + function() + for i, u in pairs(t) do + for j, v in pairs(u) do + co_yield(i, j, v) + end + end + end + ) +end +util.triples = triples + +--* Cool combinators and friendly functions for LPeg + +--** Simple things for better legibility of complicated constructions + +local lpeg_add = getmetatable(P(true)).__add +local lpeg_mul = getmetatable(P(true)).__mul +local lpeg_pow = getmetatable(P(true)).__pow + +local char = P(1) -- in most cases, this works with utf8 too +local uchar = R("\0\127") + R("\194\244") * R("\128\191")^-3 +local hex = R("09", "AF", "af") +local alnum = R("09", "AZ", "az") +local alpha = R("AZ", "az") +local eol = P("\n") + +local function choice(...) + return foldl1(lpeg_add, pack(...)) +end +util.choice = choice + +local function sequence(...) + return foldl1(lpeg_mul, pack(...)) +end +util.sequence = sequence + +local function many(times, patt) + if patt then + return lpeg_pow(patt, times) + else + return lpeg_pow(times, 0) + end +end +util.many = many + +-- Return a function to match against a pattern +local function matcher(patt) + patt = P(patt) + return function(s, i) return match(patt, s, i) end +end +util.matcher = matcher + +-- Return a function to perfom replacement. Like string.gsub, but +-- partially evaluated +local function replace(patt, repl, token) + token = token and P(token) or char + patt = Cs((P(patt) / repl + token)^0) + return function(s, i) return match(patt, s, i) end +end +util.replace = replace + +util.lpeg_escape = replace("%", "%%%%") + +--** General purpose combinators + +local function search(patt, token) + token = token and P(token) or char + return P{P(patt) + token * V(1)} +end +util.search = search + +local function gobble(patt, token) + token = token and P(token) or char + return (token - P(patt))^0 +end +util.gobble = gobble +util.gobble_until = gobble + +local function between_balanced(l, r, token) --nicer name? + l, r = P(l), P(r) + token = token and P(token) or char + return P{l * C(((token - l - r) + V(1)/0)^0) * r} +end +util.between_balanced = between_balanced + +-- function case_fold(str) +-- +-- Return a pattern that matches the given string, ignoring case. +-- Uppercase characters in the input still match only uppercase. +-- Indexing case_fold as a table does the same, but for individual +-- characters only. +local case_fold = {} +for i = strbyte"a", strbyte"z" do + local c = strchar(i) + case_fold[c] = S(c .. strupper(c)) +end +local cf_patt = Cf((C(char) / case_fold)^1, lpeg_mul) +setmetatable(case_fold, { + __index = function(_, c) return P(c) end, + __call = function(_, s) return match(cf_patt, s) end +}) +util.case_fold = case_fold + +--** String functions, with a tendency towards currying + +-- Split a string at the given separators. `nulls` determines wheter +-- empty sequences are returned. +local function split(sep, token, nulls) + sep = sep and P(sep) or locale_table.space + token = token and P(token) or char + local patt + if nulls then + local item = C((token - sep)^0) + patt = Ct(item * (sep * item)^0) + else + patt = Ct((sep^0 * C((token - sep)^1))^0) + end + return function (s, i) return match(patt, s, i) end +end +util.split = split + +-- Remove spaces from the ends of subject. +local function trim(space, token) + space = space and P(space) or locale_table.space + token = token and P(token) or char + local patt = space^0 * C((space^0 * (token - space)^1)^0) + return function(s, i) return match(patt, s, i) end +end +util.trim = trim + +-- Trim and remove repeated spaces inside subject. +local function clean(space, token) + space = space and P(space) or locale_table.space + token = token and P(token) or char + local patt = space^0 * Cs(((space^1 / " " + true) * (token - space)^1)^0) + return function(s, i) return match(patt, s, i) end +end +util.clean = clean + +-- Return a list of new lines in the subject string. +util.line_indices = matcher(Ct(I * search(eol * I)^0)) + +local utf8_sync_patt = R("\128\191")^-3 * I + I + +-- Like string.sub, but don't break up UTF-8 codepoints. May return +-- a string slightly longer or shorter than j - i + 1 bytes. +local function strsub8(s, i, j) + i = match(utf8_sync_patt, s, i) + j = j and match(utf8_sync_patt, s, j + 1) - 1 + return strsub(s, i, j) +end +util.strsub8 = strsub8 + +--** Fuzzy matching + +local fuzzy_build_patt = Cf( + (C(uchar) / function(c) return search(I * case_fold[c]) end)^1, + lpeg_mul) + +-- Return a function that fuzzy-matches against a string. +-- Higher values of the "penalty parameter" p0 reduce the relative +-- penalty for long gaps. This is a made-up scoring algorithm, there +-- must be better ones. +-- +-- Arguments: +-- +-- - str: the string to match against +-- - p0: penalty parameter for computing scores, default is 2 +-- +local function fuzzy_matcher(str, p0) + if str == "" then return function() return 1 end end + p0 = p0 or 2 + local best_score = #str / (p0 + 1) -- Score of a prefix match + local search_patt = Ct(match(fuzzy_build_patt, str)) + return function(s, i) + local score, old_pos, matches = 0, 0, match(search_patt, s, i) + if not matches then return end + for j = 1, #matches do + local pos = matches[j] + score = score + 1 / (p0 + pos - old_pos) + old_pos = pos + end + return score / best_score + end +end +util.fuzzy_matcher = fuzzy_matcher + +--** Iterators + +local line_patt = I * (search(I * eol) * I + P(true)) + +local function lines(s, i, n) + return co_wrap(function() + local n, i, j, k = n or 1, match(line_patt, s, i) + while k do + co_yield(n, i, j - 1) + n, i, j, k = n + 1, match(line_patt, s, k) + end + co_yield(n, i or 1, #s) + end) +end +util.lines = lines + +--* Classes + +local function create_object (c, ...) + local obj = setmetatable({}, c) + c.__init(obj, ...) + return obj +end + +local function class(parent) + local c = {} + local mt = { + __call = create_object, + __index = parent + } + c.__index = c + return setmetatable(c, mt) +end +util.class = class + +--* Memoization + +local weak_keys, nil_marker, value_marker = {__mode = "k"}, {}, {} + +-- Memoize a function of one argument with one return value. Nil as +-- an argument is equivalent to false, and nil as return value is not +-- memoized. +local function memoize1(fun) + local values = setmetatable({}, weak_keys) + return function(arg) + arg = arg or false + local val = values[arg] + if val == nil then + val = fun(arg) + values[arg] = val + end + return val + end +end +util.memoize1 = memoize1 + +-- Return a memoizing version of a function. +local function memoize(fun) + local values = setmetatable({}, weak_keys) + return function(...) + local arg, val = pack(...), values + for i = 1, arg.n do + local a = arg[i] + if a == nil then a = nil_marker end + local v = val[a] + if v == nil then + v = setmetatable({}, weak_keys) + val[a] = v + end + val = v + end + local v = val[value_marker] + if v == nil then + v = pack(fun(...)) + val[value_marker] = v + end + return unpack(v, 1, v.n) + end +end +util.memoize = memoize + +--* OS utilities + +if package.config:sub(1, 1) == "\\" then + util.os_type = "windows" +else + util.os_type = "posix" +end + +local is_command_cmd = util.os_type == "windows" + and "WHERE /Q %q" + or ">/dev/null command -v %q" + +-- Return `name` if an executable with that name exists, nil +-- otherwise. +local function is_command(name) + local ok = os.execute(format(is_command_cmd, name)) + return ok and name or nil +end +util.is_command = is_command + +--* Path and file manipulation + +local dir_sep, dir_sep_patt, path_is_abs_patt + +if util.os_type == "windows" then + dir_sep = "/" + dir_sep_patt = S"/\\" + path_is_abs_patt = (alpha * P":")^-1 * dir_sep_patt +else + dir_sep = "/" + dir_sep_patt = P"/" + path_is_abs_patt = dir_sep_patt +end + +-- Concatenate two paths. If the second is absolute, the first one is +-- ignored. +local function path_join(p, q) + if match(path_is_abs_patt, q) or p == "" then + return q + else + local sep = match(dir_sep_patt, p, #p) and "" or dir_sep + return p .. sep .. q + end +end +util.path_join = path_join + +-- Split a path into directory and file parts. +local path_split_patt = sequence( + C(sequence( + many(dir_sep_patt), + gobble( + sequence( + dir_sep_patt^0, + many(1 - dir_sep_patt), + P(-1))))), + dir_sep_patt^0, + C(many(char)) +) + +local function path_split(p) + return match(path_split_patt, p) +end +util.path_split = path_split + +local path_norm_double_sep = Cs(search((dir_sep_patt ^ 2) / dir_sep) * P(1)^0) +local path_norm_dot_patt = Cs( + search(((B(dir_sep_patt) + B(-1)) * P"." * dir_sep_patt) / "") * P(1)^0) +local path_norm_dotdot_patt = Cs( + search(((1 - dir_sep_patt)^1 * dir_sep_patt * P".." * dir_sep_patt) / "") * P(1)^0) + +-- Normalize a path name, removing repeated separators and "./" and "../" +local function path_normalize(p) + local q = match(path_norm_double_sep, p) + or match(path_norm_dot_patt, p) + or match(path_norm_dotdot_patt, p) + if q then + return path_normalize(q) + elseif util.os_type == "windows" then + return p:gsub("\\", "/") + else + return p + end +end +util.path_normalize = path_normalize + +if util.os_type == "windows" then + util.path_list_split = split";" +else + util.path_list_split = split":" +end + +local function format_filename_template(template, name) + name = gsub(name, "%%", "%%%%") + return gsub(template, "?", name) +end +util.format_filename_template = format_filename_template + +-- Look for a file in several locations. `path` is a string or a list +-- of strings. The optional `name` is joined to each element. +-- Returns the first file name that exists on disk. If `read` is +-- true, give the file contents as second return value. +local function find_file(path, name, read) + if type(path) == "table" then + for i = 1, #path do + local p, s = find_file(path[i], name, read) + if p then return p, s end + end + return + end + local file + local zipfile = zip and name and zip.open(path) + if zipfile then + path = path .. "#" .. name + file = zipfile:open(name) + zipfile:close() + else + if name then path = path_join(path, name) end + file = io.open(path) + end + if file then + local str = read and file:read("*a") + file:close() + return path, str + end +end +util.find_file = find_file + +--* URI manipulation + +local percent_decode = replace( + P"%" * C(hex * hex), + function(s) return strchar(tonumber(s, 16)) end +) + +local percent_encode = replace( + char - (alnum + S"-./:=_~"), + function(s) return format("%%%X", strbyte(s)) end +) + +local uri_patt = sequence( + C(alpha^1) * P":", + (P"//" * C(gobble("/"))) + Cc(nil), + C(gobble(S"?#")) / percent_decode, + (P"?" * C(gobble("#")) / percent_decode) + Cc(nil), + (P"#" * C(char^0) / percent_decode) + Cc(nil) +) +util.parse_uri = matcher(uri_patt) + +local function make_uri(scheme, authority, path, query, fragment) + local t = {} + if scheme then t[#t+1] = scheme; t[#t+1] = ":" end + if authority then t[#t+1] = "//"; t[#t+1] = authority end + if path then t[#t+1] = percent_encode(path) end + if query then t[#t+1] = "?"; t[#t+1] = percent_encode(query) end + if fragment then t[#t+1] = "#"; t[#t+1] = percent_encode(fragment) end + return concat(t) +end +util.make_uri = make_uri + +--* JSON + +util.json_null = setmetatable({}, {__json = "null"}) + +--** Decoding + +do + local ws = S" \n\t\r"^0 + local quote = P"\"" + + local function decode_number(s) + return tonumber(s) or error("Error parsing “" .. s .. "” as a number") + end + + local char_or_escape = choice( + P(1) - P"\\", + P"\\n" / "\n", + P"\\\"" / "\"", + P"\\\\" / "\\", + P"\\/" / "/", + P"\\r" / "\r", + P"\\t" / "\t", + P"\\b" / "\b", + P"\\f" / "\f", + P"\\u" * C(S"Dd" * S"89ABab" * hex * hex) + * P"\\u" * C(S"Dd" * R("CF","cf") * hex * hex) + / function(high, low) + high = tonumber(high, 16) + low = tonumber(low, 16) + return utf8_char( + (high - 0xD800) * 2^10 + low - 0xDC00 + 0x10000) + end, + P"\\u" * C(hex * hex * hex * hex) + / function(s) return utf8_char(tonumber(s, 16)) end + ) + + local json_patt = P{ + [1] = ws * V"element" * P(-1), + ["true"] = P"true" * Cc(true) * ws, + ["false"] = P"false" * Cc(false) * ws, + null = P"null" * Cc(util.json_null) * ws, + number = (R"09" + S"-+.") * (1 - S",]}")^0 / decode_number, + string = quote * Cs(gobble(quote, char_or_escape)) * quote * ws, + element = V"string" + V"number" + V"true" + V"false" + V"null" + V"array" + V"object", + elements = V"element" * (P"," * ws * V"element")^0, + array = P"[" * ws * Ct(V"elements"^-1) * P"]" * ws, + member = Cg(V"string" * P":" * ws * V"element"), + members = V"member" * (P"," * ws * V"member")^0, + object = P"{" * ws * Cf(Ct(true) * V"members"^-1, rawset) * P"}" * ws + } + + function util.json_decode(str) + return match(json_patt, str) or error "Error decoding json" + end +end + +--** Encoding + +do + local control_chars = {} + for i = 0, 31 do + control_chars[strchar(i)] = format("\\u%04x", i) + end + + local encode_string = matcher( + Cs(many(choice( + P(1) - R"\0\31" - S"\"\\", + P"\n" / "\\n", + P"\"" / "\\\"", + P"\\" / "\\\\", + P"\r" / "\\r", + P"\t" / "\\t", + P"\b" / "\\b", + P"\f" / "\\f", + R"\0\31" / control_chars + )))) + + local fix_decimal + local decimal_sep = tostring(5.5):gsub("5", "") + if decimal_sep == "." then + fix_decimal = function(x) return x end + else + fix_decimal = replace(decimal_sep, ".") + end + + local inf = math.huge + + local function encode_number(v) + if -inf < v and v < inf then + return fix_decimal(tostring(v)) + else + return "null" + end + end + + local function do_encode(obj, t, n) + local obj_type = type(obj) + if obj_type == "string" then + t[n] = "\"" + t[n + 1] = encode_string(obj) + t[n + 2] = "\"" + return n + 3 + elseif obj_type == "number" then + t[n] = encode_number(obj) + return n + 1 + elseif obj_type == "table" then + local v = obj[1] + if v ~= nil then + t[n] = "[" + n = do_encode(v, t, n + 1) + for i = 2, #obj do + t[n] = "," + n = do_encode(obj[i], t, n + 1) + end + t[n] = "]" + return n + 1 + end + local k, v = next(obj) + if k ~= nil then + t[n] = "{\"" + t[n + 1] = encode_string(k) + t[n + 2] = "\":" + n = do_encode(v, t, n + 3) + for k, v in next, obj, k do + t[n] = ",\"" + t[n + 1] = encode_string(k) + t[n + 2] = "\":" + n = do_encode(v, t, n + 3) + end + t[n] = "}" + return n + 1 + else + local mt = getmetatable(obj) + t[n] = mt and mt.__json or "[]" + return n + 1 + end + elseif obj_type == "boolean" then + t[n] = obj and "true" or "false" + return n + 1 + else + error("Error encoding json, found object of type " .. type) + end + end + + function util.json_encode(obj) + local t = {} + do_encode(obj, t, 1) + return concat(t) + end +end + +--* Inspect and serialize Lua values + +-- function inspect(obj, depth) +-- +-- Return a string representation of `obj`, with at most `depth` +-- layers of nested tables. If `obj` consists solely of scalars, +-- strings and tables and does not exceed the maximum nesting, the +-- return value is valid Lua code. +-- +local inspect + +local is_lua_identifier = util.matcher( + C(R("AZ", "az", "__") * R("09", "AZ", "az", "__")^0) * P(-1) + / function(s) return load("local " .. s) and s end +) + +local function lua_encode_key(obj) + if type(obj) == "string" then + if is_lua_identifier(obj) then + return obj + else + return "[" .. format("%q", obj) .. "]" + end + else + return "[" .. tostring(obj) .. "]" + end +end + +local function lua_encode_string(obj) + if strfind(obj, "\n", 1, true) then + local delim = "" + while strfind(obj, "]" .. delim .. "]", 1, true) do + delim = delim .. "=" + end + return "[" .. delim .. "[\n" .. obj .. "]" .. delim .. "]" + else + return format("%q", obj) + end +end + +local function lua_encode_table(obj, depth, d) + local t = {} + local array_keys, hash_keys = {}, {} + for i = 1, #obj do + array_keys[i] = true + t[#t+1] = inspect(obj[i], depth, d+1) + end + for k in pairs(obj) do + if not array_keys[k] then + hash_keys[#hash_keys+1] = k + end + end + sort( + hash_keys, + function(v, w) return tostring(v) < tostring(w) end + ) + for i = 1, #hash_keys do + local k = hash_keys[i] + local v = obj[k] + t[#t+1] = lua_encode_key(k) .. " = " .. inspect(v, depth, d+1) + end + local short = concat(t, ", ") + if 2*d + #short > 70 or strfind(short, "\n", 1, true) then + local sep = strrep(" ", d) + return "{\n " .. sep .. concat(t, ",\n " .. sep) .. "\n" .. sep .. "}" + else + return "{" .. short .. "}" + end +end + +inspect = function(obj, depth, d) + depth, d = depth or 10, d or 0 + if type(obj) == "table" and d < depth then + return lua_encode_table(obj, depth, d) + elseif type(obj) == "string" then + if d < depth or #obj < 20 then + return lua_encode_string(obj) + else + return format("string: \"%s...\"", strsub(obj, 1, 9)) + end + else + return tostring(obj) + end +end +util.inspect = inspect + +--* Logging + +local function log(msg, ...) + if select("#", ...) > 0 then msg = format(msg, ...) end + io.stderr:write(os.date("%H:%M:%S "), msg, "\n") + io.stderr:flush() +end +util.log = log + +local function log_objects(...) + return log(concat(map(inspect, {...}), "\n")) +end +util.log_objects = log_objects + +return util |