-- search.tlu: file searching functions for texdoc. -- -- Manuel Pégourié-Gonnard, GPLv3, see texdoclib.tlu for details -- Warning: every single function here assumes init_databases() has been called. -- shared by all functions in this file local s_doclist -- the Doclist objet to be populated by various functions local s_meta -- {[normname] = meta, ...} (populated by init_tlp_database) local vanilla -- is this a vanilla TL or a re-package one without tlpdb? ----------------------- docfile and doclist objects ------------------------ --[[ doclist = { [1] = docfile1, [2] = docfiles2, ..., inv = { realpath1 = index1, ... } } The inv subtable is such that for all i doclist.inv(doclist[i].realpath:lower()) == i Paths are lowercased in order to avoid duplicates on windows. --]] local Doclist = {} Doclist.__index = Doclist -- create a new list of docfiles function Doclist:new() local dl = { inv = {} } setmetatable(dl, self) return dl end -- add a docfile to a list function Doclist:add(df) if not df.realpath then return end -- useful if vanilla == false local index = self.inv[df.realpath:lower()] if index then self[index]:mergein(df) else local newindex = #self + 1 self[newindex] = df self.inv[df.realpath:lower()] = newindex end end -- stops a doclist function Doclist:stop() self.inv = nil end --[[ docfile = { -- name and tree are mendatory name = filename (used for scoring only) tree = code of the tree, see below -- at least one of the following fields should exist matches = {pattern1, pattern2, ...} or {} runtodoc = true if there is a runfile -> docfile association tlptodoc = true if there is a tlp name -> docfile association -- those are virtual members, see below realpath = full path normname = nomrmalised (path removed up to the 'doc' component) shortname = short name used for scoring basename = basename lang = language tag from the catalogue metadata details = details tag from the catalogue metadata quality = 'good', 'bad', or 'killed' depending on score ext_pos = position of the extension in ext_list -- set for elements of a list as a side effect of sort_doclist() score = score } if tree > 1, this is the index of the tree in TEXDOCS if tree = 0, then name is relative to TLROOT tree = - 1 if and only if file is a sty file. Here name is absolute. --]] -- Docfile objects inherit members from Docfile -- and have virtual members implemented by get_() methods -- for best cache performance, getters should not return nil local Docfile = {} function Docfile:__index(key) if Docfile[key] then return Docfile[key] end local getter = Docfile['get_'..key] if getter then rawset(self, key, getter(self)) return rawget(self, key) end end -- create a new docfile objet using initilisation info -- required fields: name, tree function Docfile:new(info) local df = {} setmetatable(df, self) for k, v in pairs(info) do if k == 'pattern' then df.matches = { info.pattern } else df[k] = v end end return df end -- merge a second docfile objet in, assuming it represents the same file function Docfile:mergein(df) for k, v in pairs(df) do if k == 'matches' then for _, m in ipairs(df.matches or {}) do table.insert(self.matches, m) end else self[k] = v end end end -- return the full path to the file function Docfile:get_realpath() if self.tree > 0 then return texdocs_tree_to_path(self.tree, self.name) elseif self.tree == 0 then if vanilla then return get_tlroot()..'/'..self.name else return kpse.find_file(self.normname, 'TeX system documentation') end else return self.name end end -- normalise a name from the tlpdb (use for s_meta indexes) function reloc_tlpdb_path(name) return string.gsub(name, '^texmf[^/]*/doc/', '', 1) end -- return normalised name function Docfile:get_normname() return (self.tree == 0) and reloc_tlpdb_path(self.name) or self.name end -- retrieve the lang from meta function Docfile:get_lang() local meta = s_meta[self.normname] return meta and (meta.lang or false) or false end -- retrieve the details from meta function Docfile:get_details() local meta = s_meta[self.normname] return meta and (meta.details or false) or false end -- return the short name used for scoring function Docfile:get_shortname() if self.tree == -1 then return self.name end -- remove first component of name if at least two directory levels return string.match(self.normname, '^..-/(.+/.+)$') or self.normname end -- return the base name function Docfile:get_basename() return string.gsub(self.name, '.*/', '', 1) end -- for interface consistency, matches should always be a table, never nil function Docfile:get_matches() return {} end -- from score.tlu Docfile.get_quality = docfile_quality -- from score.tlu function Docfile:get_ext_pos() return ext_pos(self.basename) end --------------------- manage TEXDOCS trees à la kpse ---------------------- do -- scope of doc_roots local doc_roots --[[ doc_roots[i] = { path = initial path, db = { [file1] = basename1, [file2] = basename2, ... }, } --]] -- populate the doc_roots filename databases function init_texdocs_database() doc_roots = {} local sep = (os.type == 'windows') and ';' or ':' local kpse_texdocs = kpse.expand_var("$TEXDOCS") -- expand the path and turn it into a lua list local raw_doc_roots = string.explode(kpse.expand_braces(kpse_texdocs), sep) local max = #raw_doc_roots + 1 for j, dir in ipairs(raw_doc_roots) do local i = max - j local n local path, db -- get path, !! and // values dir, n = string.gsub (dir, '//$', '') local recursion_allowed = (n == 1) local path, n = string.gsub (dir, '^!!', '') local index_mandatory = (n == 1) deb_print('texdocs', string.format( 'texdocs[%d] = %s (index_mandatory=%s, recursion_allowed=%s)', i, path, tostring(index_mandatory), tostring(recursion_allowed))) -- decide if we should use a ls-R index, the filesystem, or do nothing local root, shift = lsr_root(path) if root and shift and recursion_allowed then deb_print('texdocs', string.format( 'texdocs[%d] using index: %s (shift=%s)', i, root, shift)) db = init_lsr_db(root, shift) elseif not index_mandatory and lfs.isdir(path) then deb_print('texdocs', string.format( 'texdocs[%d] using filesystem search', i)) db = init_tree_db(path, recursion_allowed) end -- register this in docroots doc_roots[i] = { path = path, db = db } end end -- return the real path from a texdocs tree number + relative path function texdocs_tree_to_path(tree, rel) return doc_roots[tree].path..'/'..rel end -- find docfiles in texdocs directories function get_doclist_texdocs(patlist) for code, dr in ipairs(doc_roots) do if dr.db then scan_db(patlist, code, dr.db) end end end end -- scope of doc_roots -- find a ls-R file in a parent directory and return it or nil function lsr_root (path) if not lfs.isdir (path) then return end local root, shift = path, '' if string.sub(root, -1) == '/' then root = string.sub(root, 1, -2) end while string.find(root, '/', 1, true) do if lfs.isfile(root..'/ls-R') then return root, shift end local last_comp = string.match(root, '^.*/(.*)$') -- /!\ cannot put last_comp in a regex: can contain special char root = string.sub(root, 1, - (#last_comp + 2)) shift = last_comp..'/'..shift end end -- build a db from a ls-R file function init_lsr_db(root, shift) -- open the file local lsr = assert(io.open(root..'/ls-R', 'r')) local _ = lsr:read('*line') -- throw away first line (comment) -- scan it local db = {} local maybe_dir, isdoc = true, false local current_dir local l = #shift while true do local line = lsr:read('*line') while line == '' do line, maybe_dir = lsr:read('*line'), true end if line == nil then break end -- EOF local dir_line = maybe_dir and string.match(line, '^%./(.*):$') if dir_line then maybe_dir = false -- next line may not be a dir if string.sub(dir_line..'/', 1, l) == shift then isdoc = true current_dir = string.sub(dir_line, l+1) db[current_dir] = nil elseif isdoc then break -- we're exiting the ./doc (or shift) dir, so it's over end elseif isdoc then local file = (current_dir == '') and line or current_dir..'/'..line if check_ext(line) then db[file] = line end end end lsr:close() return db end -- build a db for a tree without ls-R index function init_tree_db(base, recurse) local db = {} local function init_tree_db_rec(dir) for file in lfs.dir(base..'/'..dir) do if file ~= '.' and file ~= '..' then local f = (dir == '') and file or dir..'/'..file if lfs.isdir(base..'/'..f) then if recurse then init_tree_db_rec(f) end else if check_ext(file) then db[f] = file end end end end end init_tree_db_rec('') return db end -------------------- select results from TEXDOCS trees --------------------- -- scan a database function scan_db(patlist, code, lsr_db) for file, basename in pairs(lsr_db) do local df = process_file(patlist, basename, file, code) if df then s_doclist:add(df) end end end -- says if file has a known extenstion according to ext_list -- (or known basename according to basename_list) function check_ext(file) file = string.lower(file) -- remove zipext if applicable file = parse_zip(file) -- then do the normal thing for _, e in ipairs(config.ext_list) do if e == '*' then return true elseif (e == '') then if not string.find(file, '.', 1, true) then return true end else local dot_e = '.'..e if string.sub(file, -string.len(dot_e)) == dot_e then return true end end end -- is the basename good? for _, b in ipairs(config.basename_list) do if file:find('^'..b..'$') or file:find('^'..b..'%.') then return true end end return false end -- say if a file (with its path) matches a pattern function matches(pattern, file) if pattern.original then return string.find(file:lower(), pattern.name:lower(), 1, true) else return is_exact(file, pattern.name) end end -- return a docfile object if file "matches", nil ortherwise function process_file(patlist, file, pathfile, code) local docfile local pattern for _, pattern in ipairs(patlist) do if matches(pattern, pathfile) then local info = { name = pathfile, tree = code, pattern = pattern, } if docfile then docfile:mergein(Docfile:new(info)) else docfile = Docfile:new(info) end end end return docfile end ---------------------------- look for sty files ---------------------------- -- add doclist entries for sty files in patlist function get_doclist_sty(patlist) for _, pat in ipairs(patlist) do local file = kpse.find_file(pat.name) if file then local df = Docfile:new({ name = file, tree = -1, pattern = pat, }) s_doclist:add(df) end end end -------------------------------- use tlpdb --------------------------------- -- tlpdb mean TeX Live Package DataBase and tlp means TeX Live Package -- find the TeX Live root function get_tlroot() local tlroot = kpse.var_value('SELFAUTOPARENT') get_tlroot = function() return tlroot end return tlroot end -- return true if cache exists and is newer than original, false otherwise function good_cache(cache, ori) local cache_date = lfs.attributes(cache, 'modification') if not cache_date then return false end local ori_date = assert(lfs.attributes(ori, 'modification')) return cache_date > ori_date end -- make sure a given directory exists, or return nil plus an error string function mkdir_p(dir) if lfs.isdir(dir) then return true end local parent = path_parent(dir) if parent then local ok, msg = mkdir_p(parent) if not ok then return nil, msg end end return lfs.mkdir(dir) end -- get tlpinfo tables initialised by whatever mean function init_tlp_database() local TEXMFVAR = kpse.var_value('TEXMFVAR') local cache_file = TEXMFVAR..'/'..C.cache_name local texlive_tlpdb = get_tlroot()..'/tlpkg/texlive.tlpdb' vanilla = lfs.isfile(texlive_tlpdb) if vanilla then if good_cache(cache_file, texlive_tlpdb) then deb_print('tlpdb', 'Using cached data from '..cache_file) get_tlpinfo_from_cache(cache_file) else deb_print('tlpdb', 'Getting data from tlpdb file '..texlive_tlpdb) get_tlpinfo_from_tlpdb(texlive_tlpdb) deb_print('tlpdb', 'Writing data in cache file '..cache_file) local ok, msg = mkdir_p(path_parent(cache_file)) if not ok then err_print('warning', 'Failed to create cache file in '..cache_file..':') err_print('warning', msg) else print_out_tlpinfo(cache_file) end end else deb_print('tlpdb', 'Using shipped tlpdb data.') get_tlpinfo_from_dist() end end do -- begin scope of tlpinfo tables local tlp_from_runfile -- { [runfile_basename] = {tlp1 = true, ...}, ... } local tlp_doclist -- { [tlp_name] = { relname1, relname2, ...}, ... } -- populate tlpinfo tables using the given texlive.tlpdb function get_tlpinfo_from_tlpdb(filename) s_meta, tlp_from_runfile, tlp_doclist = {}, {}, {} local curr_tlp local state = 'none' for line in io.lines(filename) do if state == 'none' and string.find(line, '^name ') then -- begin a new package curr_tlp = string.lower(string.sub(line, 6, -1)) tlp_doclist[curr_tlp] = {} elseif state == 'docfiles' then if not string.find(line, '^ ') then state = 'none' else local file = string.match(line, '^ ([^ ]*)') local meta = string.match(line, '^ [^ ]* (.+)') local basename = string.match(file, '([^/]*)$') if check_ext(basename) then -- we've got a docfile here, add it table.insert(tlp_doclist[curr_tlp], file) if meta then local details = string.match(meta, 'details="([^"]+)"') local lang = string.match(meta, 'language="([^"]+)"') s_meta[reloc_tlpdb_path(file)] = { details = details, lang = lang, } end end end elseif state == 'runfiles' then if not string.find(line, '^ ') then state = 'none' else -- check for interesting runfiles local e = string.sub(line, -4, -1) if e == '.tex' or e == '.sty' or e == '.cls' then local f = string.match(line, '.*/(.*)%.') tlp_from_runfile[f] = tlp_from_runfile[f] or {} tlp_from_runfile[f][curr_tlp] = true end end end -- update state if string.find(line, '^docfiles ') then state = 'docfiles' elseif string.find(line, '^runfiles ') then state = 'runfiles' end end remove_useless_tlp() end -- remove entries for tlp without any docfile function remove_useless_tlp() for tlp, doclist in pairs(tlp_doclist) do if #doclist == 0 then tlp_doclist[tlp] = nil end end for runfile, tlp_set in pairs(tlp_from_runfile) do for tlp in pairs(tlp_set) do if not tlp_doclist[tlp] then tlp_from_runfile[runfile][tlp] = nil end end end end -- print out data from tlpdb in dofile()-able form function print_out_tlpinfo(filename) local fh = assert(io.open(filename, 'w')) local function printf(s, ...) fh:write(string.format(s, ...)) end -- s_meta printf('local s_meta = {\n') for k, v in pairs(s_meta) do printf(' [%q] = {', k) for i, j in pairs(v) do printf('[%q] = %q, ', i, j) end printf('},\n') end printf('}\n') -- tlp_from_runfile printf('local tlp_from_runfile = {\n') for k, v in pairs(tlp_from_runfile) do printf(' [%q] = {', k) for f in pairs(v) do printf('[%q]=true,', f) end printf('},\n') end printf('}\n') -- tlp_doclist printf('local tlp_doclist = {\n') for k, v in pairs(tlp_doclist) do printf(' [%q] = {\n', k) for _, f in ipairs(v) do printf(' %q,\n', f) end printf(' },\n') end printf('}\n') printf('return s_meta, tlp_from_runfile, tlp_doclist\n') fh:close() end -- get pre-hashed tlpdb info from a cache file function get_tlpinfo_from_cache(filename) s_meta, tlp_from_runfile, tlp_doclist = dofile(filename) end -- get pre-hashed tlpdb info from a pseudo-cache file function get_tlpinfo_from_dist() local f = assert(kpse.find_file(C.data_tlpdb_name, 'texmfscripts')) s_meta, tlp_from_runfile, tlp_doclist = dofile(f) end -- get docfiles for pattern using specific tlpdb information function get_doclist_tlpdb(pattern) -- runfile to tlp to docfile if tlp_from_runfile[pattern] then for tlp in pairs(tlp_from_runfile[pattern]) do for _, file in ipairs(tlp_doclist[tlp]) do s_doclist:add(Docfile:new{ name = file, tree = 0, runtodoc = true, }) end end end -- tlp name to docfile if tlp_doclist[pattern] then for _, file in ipairs(tlp_doclist[pattern]) do s_doclist:add(Docfile:new{ name = file, tree = 0, tlptodoc = true, }) end end end end --scope of tlpinfo table ------------------------------ main function ------------------------------- -- initialise the various databases (must be called first) function init_databases() init_texdocs_database() init_tlp_database() end -- find docfiles according to pattern function get_doclist(pattern, no_alias) -- get patterns (inc. aliases) local normal, sty = normal_vs_sty(get_patterns(pattern, no_alias)) -- initialise result list s_doclist = Doclist:new() -- get results get_doclist_sty(sty) get_doclist_texdocs(normal) get_doclist_tlpdb(pattern) -- finally, sort results sort_doclist(s_doclist, pattern) return s_doclist end -- separate sty patterns from the rest function normal_vs_sty(list) local normal, sty = {}, {} for _, p in ipairs(list) do if string.match(string.lower(p.name), '%.([^/.]*)$') == 'sty' then table.insert(sty, p) else table.insert(normal, p) end end return normal, sty end return { init_databases = init_databases, get_doclist = get_doclist, }