diff options
Diffstat (limited to 'Master/texmf/scripts/texdoc/search.tlu')
-rw-r--r-- | Master/texmf/scripts/texdoc/search.tlu | 565 |
1 files changed, 422 insertions, 143 deletions
diff --git a/Master/texmf/scripts/texdoc/search.tlu b/Master/texmf/scripts/texdoc/search.tlu index 2679d01d1d2..e336d01ed12 100644 --- a/Master/texmf/scripts/texdoc/search.tlu +++ b/Master/texmf/scripts/texdoc/search.tlu @@ -5,28 +5,34 @@ Distributed under the terms of the GNU GPL version 3 or later. See texdoc.tlu for details. --]] +-- Load a private environment for this submodule (see texdoc.tlu). local L = {} load_env(L, { - 'export_symbols', - 'os', 'string', 'table', 'lfs', 'kpse', 'io', - 'ipairs', 'assert', 'error', 'tostring', 'setmetatable', - 'deb_print', 'err_print', 'win32_hook', 'parse_zip', - 'get_patterns', 'sort_doclist', 'docfile_quality', - 'config', 'C', + 'os', 'lfs', 'kpse', 'io', 'setmetatable', 'print', + 'dofile', 'loadfile', 'setfenv', 'rawget', 'rawset', + 'win32_hook', 'parse_zip', 'path_parent', + 'get_patterns', 'sort_doclist', 'docfile_quality', 'ext_pos', + 'config', }) +-- Warning: every single function here assumes init_databases() has been called. + -- shared by all functions in this file -local s_doclist +local s_doclist -- the Doclist objet to be populated by various functions +local s_meta -- {[normname] = meta, ...} (populated by init_tlp_database) +local vanilla -- is this a vanilla TL or a re-package one without tlpdb? ----------------------- docfile and doclist objects ------------------------ --- doclist = { --- [1] = docfile1, [2] = docfiles2, ..., --- inv = { realpath1 = index1, ... } --- } --- --- The inv subtable is such that for all i --- doclist.inv(doclist[i].realpath) == i +--[[ +doclist = { + [1] = docfile1, [2] = docfiles2, ..., + inv = { realpath1 = index1, ... } +} + +The inv subtable is such that for all i +doclist.inv(doclist[i].realpath) == i +--]] local Doclist = {} Doclist.__index = Doclist @@ -40,6 +46,7 @@ end -- add a docfile to a list function Doclist:add(df) + if not df.realpath then return end -- useful if vanilla == false local index = self.inv[df.realpath] if index then self[index]:mergein(df) @@ -55,85 +62,149 @@ function Doclist:stop() self.inv = nil end --- docfile = { --- name = filename (used for scoring only) --- tree = code of the tree, see below --- matches = {pattern1, pattern2, ...} --- } --- if tree > 1, this is the index of the tree in TEXDOCS --- if tree = 0, then name is relative to TLROOT (file found from tlpdb only) --- tree = - 1 if and only if file is a sty file. Here name is absolute. +--[[ +docfile = { + -- name and tree are mendatory + name = filename (used for scoring only) + tree = code of the tree, see below + -- at least one of the following fields should exist + matches = {pattern1, pattern2, ...} or {} + runtodoc = true if there is a runfile -> docfile association + tlptodoc = true if there is a tlp name -> docfile association + -- those are virtual members, see below + realpath = full path + normname = nomrmalised (path removed up to the 'doc' component) + shortname = short name used for scoring + basename = basename + lang = language tag from the catalogue metadata + details = details tag from the catalogue metadata + quality = 'good', 'bad', or 'killed' depending on score + ext_pos = position of the extension in ext_list + -- set for elements of a list as a side effect of sort_doclist() + score = score +} +if tree > 1, this is the index of the tree in TEXDOCS +if tree = 0, then name is relative to TLROOT +tree = - 1 if and only if file is a sty file. Here name is absolute. +--]] +-- Docfile objects inherit members from Docfile +-- and have virtual members implemented by get_<member>() methods +-- for best cache performance, getters should not return nil local Docfile = {} -Docfile.__index = Docfile +function Docfile:__index(key) + if Docfile[key] then return Docfile[key] end + local getter = Docfile['get_'..key] + if getter then + rawset(self, key, getter(self)) + return rawget(self, key) + end +end -- create a new docfile objet using initilisation info --- fields : name (relative to tree), tree, pattern +-- required fields: name, tree function Docfile:new(info) - df = {} + local df = {} setmetatable(df, self) - -- get realpath, tree and prepare name - df.tree = info.tree - if info.tree > 0 then - df.realpath = texdocs_tree_to_path(info.tree, info.name) - elseif info.tree == 0 then - error('Unimplemented') - df.realpath = get_tlroot()..'/'..info.name - info.name = string.gsub(info.name, '^texmf(-dist)?/doc/', '', 1) - elseif info.tree == -1 then - df.realpath = info.name - else - error('Internal error: bad tree number') - end - -- remove first component of name if at least two directory levels - if info.tree > -1 then - local name = string.match(info.name, '^..-/(.+/.+)$') - if name then - df.name = '/'..name + for k, v in pairs(info) do + if k == 'pattern' then + df.matches = { info.pattern } else - df.name = info.name + df[k] = v end - else - df.name = info.name - end - -- initialise the list of matches - if info.pattern then - df.matches = { info.pattern } - else - df.matches = {} end return df end --- merge a second docfile objet into self +-- merge a second docfile objet in, assuming it represents the same file function Docfile:mergein(df) - if df.tree > self.tree then - self.name = df.name - self.tree = df.tree + for k, v in pairs(df) do + if k == 'matches' then + for _, m in ipairs(df.matches or {}) do + table.insert(self.matches, m) + end + else + self[k] = v + end end - for _, m in ipairs(df.matches) do - table.insert(self.matches, m) +end + +-- return the full path to the file +function Docfile:get_realpath() + if self.tree > 0 then + return texdocs_tree_to_path(self.tree, self.name) + elseif self.tree == 0 then + if vanilla then + return get_tlroot()..'/'..self.name + else + return kpse.find_file(self.normname, 'TeX system documentation') + end + else + return self.name end end +-- normalise a name from the tlpdb (use for s_meta indexes) +function reloc_tlpdb_path(name) + return string.gsub(name, '^texmf[^/]*/doc/', '', 1) +end + +-- return normalised name +function Docfile:get_normname() + return (self.tree == 0) and reloc_tlpdb_path(self.name) or self.name +end + +-- retrieve the lang from meta +function Docfile:get_lang() + local meta = s_meta[self.normname] + return meta and (meta.lang or false) or false +end + +-- retrieve the details from meta +function Docfile:get_details() + local meta = s_meta[self.normname] + return meta and (meta.details or false) or false +end + +-- return the short name used for scoring +function Docfile:get_shortname() + if self.tree == -1 then return self.name end + -- remove first component of name if at least two directory levels + return string.match(self.normname, '^..-/(.+/.+)$') or self.normname +end + +-- return the base name +function Docfile:get_basename() + return string.gsub(self.name, '.*/', '', 1) +end + +-- for interface consistency, matches should always be a table, never nil +function Docfile:get_matches() + return {} +end + +-- from score.tlu +Docfile.get_quality = docfile_quality + -- from score.tlu -Docfile.quality = docfile_quality +function Docfile:get_ext_pos() + return ext_pos(self.basename) +end ------------------- get results from TEXDOCS (à la kpse) ------------------- +--------------------- manage TEXDOCS trees à la kpse ---------------------- do -- scope of doc_roots local doc_roots --- doc_roots is a Lua version of kpse's TEXDOCS --- structure of the doc_roots variable: --- doc_roots[i] = { --- path = <path>, --- index_mandatory = <does path begin with !! in TEXDOCS?> --- recursion_allowed = <does path ends with // in TEXDOCS?>, --- } +--[[ +doc_roots[i] = { + path = initial path, + db = { [file1] = basename1, [file2] = basename2, ... }, +} +--]] --- set the doc_roots list from kpse's $TEXDOCS -function get_texdocs () +-- populate the doc_roots filename databases +function init_texdocs_database() doc_roots = {} local sep = (os.type == 'windows') and ';' or ':' local kpse_texdocs = kpse.expand_var("$TEXDOCS") @@ -142,56 +213,47 @@ function get_texdocs () local max = #raw_doc_roots + 1 for j, dir in ipairs(raw_doc_roots) do local i = max - j - local dr = {} local n + local path, db -- get path, !! and // values dir, n = string.gsub (dir, '//$', '') - dr.recursion_allowed = (n == 1) - dr.path, n = string.gsub (dir, '^!!', '') - dr.index_mandatory = (n == 1) + local recursion_allowed = (n == 1) + local path, n = string.gsub (dir, '^!!', '') + local index_mandatory = (n == 1) deb_print('texdocs', string.format( 'texdocs[%d] = %s (index_mandatory=%s, recursion_allowed=%s)', - i, dr.path, - tostring(dr.index_mandatory), - tostring(dr.recursion_allowed))) + i, path, tostring(index_mandatory), tostring(recursion_allowed))) -- decide if we should use a ls-R index, the filesystem, or do nothing - local root, shift = lsr_root(dr.path) - if root and shift and dr.recursion_allowed then - dr.lsr = root - dr.lsr_shift = shift + local root, shift = lsr_root(path) + if root and shift and recursion_allowed then deb_print('texdocs', string.format( 'texdocs[%d] using index: %s (shift=%s)', i, root, shift)) - elseif not dr.index_mandatory and lfs.isdir(dr.path) then - dr.searchfs = true + db = init_lsr_db(root, shift) + elseif not index_mandatory and lfs.isdir(path) then deb_print('texdocs', string.format( 'texdocs[%d] using filesystem search', i)) + db = init_tree_db(path, recursion_allowed) end -- register this in docroots - doc_roots[i] = dr + doc_roots[i] = { path = path, db = db } end end -- return the real path from a texdocs tree number + relative path function texdocs_tree_to_path(tree, rel) - if doc_roots == nil then get_texdocs() end return win32_hook(doc_roots[tree].path..'/'..rel) end -- find docfiles in texdocs directories function get_doclist_texdocs(patlist) - if doc_roots == nil then get_texdocs() end for code, dr in ipairs(doc_roots) do - if dr.lsr then - scan_lsr(patlist, code, dr.lsr, dr.lsr_shift) - elseif dr.searchfs then - scan_tree(patlist, code, dr.path, '', dr.recursion_allowed) - end + if dr.db then scan_db(patlist, code, dr.db) end end end end -- scope of doc_roots --- find a ls-R file in a parent directory an return it or nil +-- find a ls-R file in a parent directory and return it or nil function lsr_root (path) if not lfs.isdir (path) then return end local root, shift = path, '' @@ -207,57 +269,66 @@ function lsr_root (path) end end --- scan a tree without ls-R file -function scan_tree (patlist, code, base, cwd, recurse) - for file in lfs.dir(base..'/'..cwd) do - if file ~= '.' and file ~= '..' then - local f = (cwd == '') and file or cwd..'/'..file - if lfs.isdir(base..'/'..f) then - if recurse then scan_tree(patlist, code, base, f, recurse) end - else - local df = process_file(patlist, file, f, code, true) - if df then s_doclist:add(df) end - end - end - end -end - --- scan a ls-R file -function scan_lsr(patlist, code, cwd, shift) - local is_dir = {} -- is_dir[path] = true iff path is a dir - local results = Doclist:new() - local isdoc = false +-- build a db from a ls-R file +function init_lsr_db(root, shift) + -- open the file + local lsr = assert(io.open(root..'/ls-R', 'r')) + local _ = lsr:read('*line') -- throw away first line (comment) + -- scan it + local db = {} + local maybe_dir, isdoc = true, false local current_dir local l = #shift - local lsr = assert(io.open(cwd..'/ls-R', 'r')) - local _ = lsr:read('*line') -- throw away first line (comment) - local maybe_dir = true -- next line may be a directory while true do local line = lsr:read('*line') while line == '' do line, maybe_dir = lsr:read('*line'), true end - if line == nil then break end -- EOF - local dir_line = maybe_dir and string.match (line, '^%./(.*):$') + if line == nil then break end -- EOF + local dir_line = maybe_dir and string.match(line, '^%./(.*):$') if dir_line then maybe_dir = false -- next line may not be a dir - if string.sub (dir_line, 1, l) == shift then + if string.sub(dir_line, 1, l) == shift then isdoc = true - current_dir = string.sub (dir_line, l+1) - is_dir[current_dir] = true + current_dir = string.sub(dir_line, l+1) + db[current_dir] = nil elseif isdoc then break -- we're exiting the ./doc (or shift) dir, so it's over end elseif isdoc then - local df = process_file(patlist, line, current_dir..'/'..line, code) - if df then results:add(df) end + local file = current_dir..'/'..line + if check_ext(line) then db[file] = line end end end lsr:close() - -- add non-directories to the list - for _, df in ipairs(results) do - if not is_dir[df.name] then - s_doclist:add(df) + return db +end + +-- build a db for a tree without ls-R index +function init_tree_db(base, recurse) + local db = {} + local function init_tree_db_rec(dir) + for file in lfs.dir(base..'/'..dir) do + if file ~= '.' and file ~= '..' then + local f = (dir == '') and file or dir..'/'..file + if lfs.isdir(base..'/'..f) then + if recurse then init_tree_db_rec(f) end + else + if check_ext(file) then db[f] = file end + end + end end end + init_tree_db_rec('') + return db +end + +-------------------- select results from TEXDOCS trees --------------------- + +-- scan a database +function scan_db(patlist, code, lsr_db) + for file, basename in pairs(lsr_db) do + local df = process_file(patlist, basename, file, code) + if df then s_doclist:add(df) end + end end -- says if file has a 'good' extenstion according to ext_list @@ -285,27 +356,20 @@ end -- return a docfile object if file "matches", nil ortherwise function process_file(patlist, file, pathfile, code) - deb_print('kpse', 'Processing file: '..pathfile) local docfile local pattern for _, pattern in ipairs(patlist) do if string.find(string.lower(pathfile), string.lower(pattern.name), 1, config.mode ~= 'regex') then - local good_ext = check_ext(file) - deb_print('kpse', string.format( - "File '%s' matches '%s'; good_ext=%s", - pathfile, pattern.name, tostring(good_ext))) - if good_ext then - local info = { - name = pathfile, - tree = code, - pattern = pattern, - } - if docfile then - docfile:mergein(Docfile:new(info)) - else - docfile = Docfile:new(info) - end + local info = { + name = pathfile, + tree = code, + pattern = pattern, + } + if docfile then + docfile:mergein(Docfile:new(info)) + else + docfile = Docfile:new(info) end end end @@ -329,17 +393,231 @@ function get_doclist_sty(patlist) end end +-------------------------------- use tlpdb --------------------------------- + +-- tlpdb mean TeX Live Package DataBase and tlp means TeX Live Package + +-- find the TeX Live root +function get_tlroot() + local tlroot = kpse.var_value('SELFAUTOPARENT') + get_tlroot = function() return tlroot end + return tlroot +end + +-- return true if cache exists and is newer than original, false otherwise +function good_cache(cache, ori) + local cache_date = lfs.attributes(cache, 'modification') + if not cache_date then return false end + local ori_date = assert(lfs.attributes(ori, 'modification')) + return cache_date > ori_date +end + +-- make sure a given directory exists, or return nil plus an error string +function mkdir_p(dir) + if lfs.isdir(dir) then return true end + local parent = path_parent(dir) + if parent then + local ok, msg = mkdir_p(parent) + if not ok then return nil, msg end + end + return lfs.mkdir(dir) +end + +-- get tlpinfo tables initialised by whatever mean +function init_tlp_database() + local TEXMFVAR = kpse.var_value('TEXMFVAR') + local cache_file = TEXMFVAR..'/'..C.cache_name + local texlive_tlpdb = get_tlroot()..'/tlpkg/texlive.tlpdb' + vanilla = lfs.isfile(texlive_tlpdb) + if vanilla then + if good_cache(cache_file, texlive_tlpdb) then + deb_print('tlpdb', 'Using cached data from '..cache_file) + get_tlpinfo_from_cache(cache_file) + else + deb_print('tlpdb', 'Getting data from tlpdb file '..texlive_tlpdb) + get_tlpinfo_from_tlpdb(texlive_tlpdb) + deb_print('tlpdb', 'Writing data in cache file '..cache_file) + local ok, msg = mkdir_p(path_parent(cache_file)) + if not ok then + err_print('warning', + 'Failed to create cache file in '..cache_file..':') + err_print('warning', msg) + else + print_out_tlpinfo(cache_file) + end + end + else + deb_print('tlpdb', 'Using shipped tlpdb data.') + get_tlpinfo_from_dist() + end +end + +do -- begin scope of tlpinfo tables +local tlp_from_runfile -- { [runfile_basename] = {tlp1 = true, ...}, ... } +local tlp_doclist -- { [tlp_name] = { relname1, relname2, ...}, ... } + +-- populate tlpinfo tables using the given texlive.tlpdb +function get_tlpinfo_from_tlpdb(filename) + s_meta, tlp_from_runfile, tlp_doclist = {}, {}, {} + get_meta_hack() + local curr_tlp + local state = 'none' + for line in io.lines(filename) do + if state == 'none' and string.find(line, '^name ') then + -- begin a new package + curr_tlp = string.lower(string.sub(line, 6, -1)) + tlp_doclist[curr_tlp] = {} + elseif state == 'docfiles' then + if not string.find(line, '^ ') then + state = 'none' + else + local file = string.match(line, '^ ([^ ]*)') + local meta = string.match(line, '^ [^ ]* (.+)') + if check_ext(file) then + -- we've got a docfile here, add it + table.insert(tlp_doclist[curr_tlp], file) + if meta then + local details = string.match(meta, 'details="([^"]+)"') + local lang = string.match(meta, 'language="([^"]+)"') + s_meta[reloc_tlpdb_path(file)] = { + details = details, + lang = lang, + } + end + end + end + elseif state == 'runfiles' then + if not string.find(line, '^ ') then + state = 'none' + else + -- check for interesting runfiles + local e = string.sub(line, -4, -1) + if e == '.tex' or e == '.sty' or e == '.cls' then + local f = string.match(line, '.*/(.*)%.') + tlp_from_runfile[f] = tlp_from_runfile[f] or {} + tlp_from_runfile[f][curr_tlp] = true + end + end + end + -- update state + if string.find(line, '^docfiles ') then + state = 'docfiles' + elseif string.find(line, '^runfiles ') then + state = 'runfiles' + end + end + remove_useless_tlp() +end + +-- remove entries for tlp without any docfile +function remove_useless_tlp() + for tlp, doclist in pairs(tlp_doclist) do + if #doclist == 0 then tlp_doclist[tlp] = nil end + end + for runfile, tlp_set in pairs(tlp_from_runfile) do + for tlp in pairs(tlp_set) do + if not tlp_doclist[tlp] then + tlp_from_runfile[runfile][tlp] = nil + end + end + end +end + +-- print out data from tlpdb in dofile()-able form +function print_out_tlpinfo(filename) + local fh = assert(io.open(filename, 'w')) + local function printf(s, ...) fh:write(string.format(s, ...)) end + -- s_meta + printf('local s_meta = {\n') + for k, v in pairs(s_meta) do + printf(' [%q] = {', k) + for i, j in pairs(v) do printf('[%q] = %q, ', i, j) end + printf('},\n') + end + printf('}\n') + -- tlp_from_runfile + printf('local tlp_from_runfile = {\n') + for k, v in pairs(tlp_from_runfile) do + printf(' [%q] = {', k) + for f in pairs(v) do printf('[%q]=true,', f) end + printf('},\n') + end + printf('}\n') + -- tlp_doclist + printf('local tlp_doclist = {\n') + for k, v in pairs(tlp_doclist) do + printf(' [%q] = {\n', k) + for _, f in ipairs(v) do printf(' %q,\n', f) end + printf(' },\n') + end + printf('}\n') + printf('return s_meta, tlp_from_runfile, tlp_doclist\n') + fh:close() +end + +-- get pre-hashed tlpdb info from a cache file +function get_tlpinfo_from_cache(filename) + s_meta, tlp_from_runfile, tlp_doclist = dofile(filename) +end + +-- get pre-hashed meta from a file +function get_meta_hack() + local f = assert(kpse.find_file(C.data_meta_name, 'texmfscripts')) + s_meta = dofile(f) +end + +-- get pre-hashed tlpdb info from a pseudo-cache file +function get_tlpinfo_from_dist() + local f = assert(kpse.find_file(C.data_tlpdb_name, 'texmfscripts')) + s_meta, tlp_from_runfile, tlp_doclist = dofile(f) +end + +-- get docfiles for pattern using specific tlpdb information +function get_doclist_tlpdb(pattern) + -- runfile to tlp to docfile + if tlp_from_runfile[pattern] then + for tlp in pairs(tlp_from_runfile[pattern]) do + for _, file in ipairs(tlp_doclist[tlp]) do + s_doclist:add(Docfile:new{ + name = file, + tree = 0, + runtodoc = true, + }) + end + end + end + -- tlp name to docfile + if tlp_doclist[pattern] then + for _, file in ipairs(tlp_doclist[pattern]) do + s_doclist:add(Docfile:new{ + name = file, + tree = 0, + tlptodoc = true, + }) + end + end +end + +end --scope of tlpinfo table + ------------------------------ main function ------------------------------- +-- initialise the various databases (must be called first) +function init_databases() + init_texdocs_database() + init_tlp_database() +end + -- find docfiles according to pattern -function get_doclist(pattern) +function get_doclist(pattern, no_alias) -- get patterns (inc. aliases) - local normal, sty = normal_vs_sty(get_patterns(pattern)) + local normal, sty = normal_vs_sty(get_patterns(pattern, no_alias)) -- initialise result list s_doclist = Doclist:new() -- get results get_doclist_sty(sty) get_doclist_texdocs(normal) + get_doclist_tlpdb(pattern) -- finally, sort results sort_doclist(s_doclist, pattern) return s_doclist @@ -361,5 +639,6 @@ end -- finally export a few symbols export_symbols(L, { + 'init_databases', 'get_doclist', }) |