diff options
author | Karl Berry <karl@freefriends.org> | 2014-01-01 22:57:02 +0000 |
---|---|---|
committer | Karl Berry <karl@freefriends.org> | 2014-01-01 22:57:02 +0000 |
commit | 0479a789f43c32b8fbc9d74bb82c615932a3c5b1 (patch) | |
tree | b1c9d139f32df9676c239bdad84408cf55124729 /Master/texmf-dist/tex/luatex/luaotfload/luaotfload-database.lua | |
parent | 999dd6fa37bd291385b20ccf85f980eccfd2e683 (diff) |
luaotfload (1jan14)
git-svn-id: svn://tug.org/texlive/trunk@32538 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Master/texmf-dist/tex/luatex/luaotfload/luaotfload-database.lua')
-rw-r--r-- | Master/texmf-dist/tex/luatex/luaotfload/luaotfload-database.lua | 2998 |
1 files changed, 2143 insertions, 855 deletions
diff --git a/Master/texmf-dist/tex/luatex/luaotfload/luaotfload-database.lua b/Master/texmf-dist/tex/luatex/luaotfload/luaotfload-database.lua index 1b75458088a..b47fc8b1f58 100644 --- a/Master/texmf-dist/tex/luatex/luaotfload/luaotfload-database.lua +++ b/Master/texmf-dist/tex/luatex/luaotfload/luaotfload-database.lua @@ -1,15 +1,40 @@ if not modules then modules = { } end modules ['luaotfload-database'] = { - version = "2.3b", + version = "2.4", comment = "companion to luaotfload.lua", author = "Khaled Hosny, Elie Roux, Philipp Gesang", copyright = "Luaotfload Development Team", license = "GNU GPL v2" } ---- TODO: if the specification is an absolute filename with a font not in the ---- database, add the font to the database and load it. There is a small ---- difficulty with the filenames of the TEXMF tree that are referenced as ---- relative paths... +--[[doc-- + + Some statistics: + + a) TL 2012, mkluatexfontdb --force + b) v2.4, luaotfload-tool --update --force + c) v2.4, luaotfload-tool --update --force --formats=+afm,pfa,pfb + d) Context, mtxrun --script fonts --reload --force + + (Keep in mind that Context does index fewer fonts since it + considers only the contents of the minimals tree, not the + tex live one!) + + time (m:s) peak VmSize (kB) + a 1:19 386 018 + b 0:37 715 797 + c 2:27 1 017 674 + d 0:44 1 082 313 + + Most of the increase in memory consumption from version 1.x to 2.2+ + can be attributed to the move from single-pass to a multi-pass + approach to building the index: Information is first gathered from + all reachable fonts and only afterwards processed, classified and + discarded. Also, there is a good deal of additional stuff kept in + the database now: two extra tables for file names and font families + have been added, making font lookups more efficient while improving + maintainability of the code. + +--doc]]-- local lpeg = require "lpeg" @@ -20,93 +45,104 @@ local C, Cc, Cf, Cg, Cs, Ct = lpeg.C, lpeg.Cc, lpeg.Cf, lpeg.Cg, lpeg.Cs, lpeg.Ct --- Luatex builtins -local load = load -local next = next -local pcall = pcall -local require = require -local tonumber = tonumber -local unpack = table.unpack - -local fontloaderinfo = fontloader.info -local fontloaderopen = fontloader.open -local iolines = io.lines -local ioopen = io.open -local kpseexpand_path = kpse.expand_path -local kpseexpand_var = kpse.expand_var -local kpsefind_file = kpse.find_file -local kpselookup = kpse.lookup -local kpsereadable_file = kpse.readable_file -local lfsattributes = lfs.attributes -local lfschdir = lfs.chdir -local lfscurrentdir = lfs.currentdir -local lfsdir = lfs.dir -local mathabs = math.abs -local mathmin = math.min -local osremove = os.remove -local stringfind = string.find -local stringformat = string.format -local stringgmatch = string.gmatch -local stringgsub = string.gsub -local stringlower = string.lower -local stringsub = string.sub -local stringupper = string.upper -local tableconcat = table.concat -local tablesort = table.sort -local texiowrite_nl = texio.write_nl -local utf8gsub = unicode.utf8.gsub -local utf8lower = unicode.utf8.lower +local load = load +local next = next +local require = require +local tonumber = tonumber +local unpack = table.unpack + +local fontloaderinfo = fontloader.info +local fontloaderclose = fontloader.close +local fontloaderopen = fontloader.open +----- fontloaderto_table = fontloader.to_table +local gzipopen = gzip.open +local iolines = io.lines +local ioopen = io.open +local iopopen = io.popen +local kpseexpand_path = kpse.expand_path +local kpseexpand_var = kpse.expand_var +local kpsefind_file = kpse.find_file +local kpselookup = kpse.lookup +local kpsereadable_file = kpse.readable_file +local lfsattributes = lfs.attributes +local lfschdir = lfs.chdir +local lfscurrentdir = lfs.currentdir +local lfsdir = lfs.dir +local mathabs = math.abs +local mathmin = math.min +local osgettimeofday = os.gettimeofday +local osremove = os.remove +local stringfind = string.find +local stringformat = string.format +local stringgmatch = string.gmatch +local stringgsub = string.gsub +local stringlower = string.lower +local stringsub = string.sub +local stringupper = string.upper +local tableconcat = table.concat +local tablesort = table.sort +local utf8gsub = unicode.utf8.gsub +local utf8lower = unicode.utf8.lower +local utf8length = unicode.utf8.length +local zlibcompress = zlib.compress --- these come from Lualibs/Context -local getwritablepath = caches.getwritablepath -local filebasename = file.basename -local filecollapsepath = file.collapsepath or file.collapse_path -local filedirname = file.dirname -local fileextname = file.extname -local fileiswritable = file.iswritable -local filejoin = file.join -local filenameonly = file.nameonly -local filereplacesuffix = file.replacesuffix -local filesplitpath = file.splitpath or file.split_path -local lfsisdir = lfs.isdir -local lfsisfile = lfs.isfile -local lfsmkdirs = lfs.mkdirs -local stringis_empty = string.is_empty -local stringsplit = string.split -local stringstrip = string.strip -local tableappend = table.append -local tablecopy = table.copy -local tabletofile = table.tofile -local tabletohash = table.tohash - +local filebasename = file.basename +local filecollapsepath = file.collapsepath or file.collapse_path +local filedirname = file.dirname +local fileextname = file.extname +local fileiswritable = file.iswritable +local filejoin = file.join +local filenameonly = file.nameonly +local filereplacesuffix = file.replacesuffix +local filesplitpath = file.splitpath or file.split_path +local filesuffix = file.suffix +local getwritablepath = caches.getwritablepath +local lfsisdir = lfs.isdir +local lfsisfile = lfs.isfile +local lfsmkdirs = lfs.mkdirs +local lpegsplitat = lpeg.splitat +local stringis_empty = string.is_empty +local stringsplit = string.split +local stringstrip = string.strip +local tableappend = table.append +local tablecopy = table.copy +local tablefastcopy = table.fastcopy +local tabletofile = table.tofile +local tabletohash = table.tohash +local tableserialize = table.serialize +local runasscript = caches == nil --- the font loader namespace is “fonts”, same as in Context --- we need to put some fallbacks into place for when running --- as a script -fonts = fonts or { } -fonts.names = fonts.names or { } -fonts.definers = fonts.definers or { } - -local names = fonts.names - -config = config or { } -config.luaotfload = config.luaotfload or { } -config.luaotfload.resolver = config.luaotfload.resolver or "normal" -if config.luaotfload.update_live ~= false then - --- this option allows for disabling updates - --- during a TeX run - config.luaotfload.update_live = true -end - -names.version = 2.207 -names.data = nil --- contains the loaded database -names.lookups = nil --- contains the lookup cache - -names.path = { index = { }, lookups = { } } -names.path.globals = { - prefix = "", --- writable_path/names_dir - names_dir = config.luaotfload.names_dir or "names", - index_file = config.luaotfload.index_file - or "luaotfload-names.lua", - lookups_file = "luaotfload-lookup-cache.lua", +fonts = fonts or { } +fonts.names = fonts.names or { } +fonts.definers = fonts.definers or { } + +local luaotfloadconfig = config.luaotfload --- always present +luaotfloadconfig.resolver = luaotfloadconfig.resolver or "normal" +luaotfloadconfig.formats = luaotfloadconfig.formats or "otf,ttf,ttc,dfont" +luaotfloadconfig.strip = luaotfloadconfig.strip == true + +--- this option allows for disabling updates +--- during a TeX run +luaotfloadconfig.update_live = luaotfloadconfig.update_live ~= false +luaotfloadconfig.compress = luaotfloadconfig.compress ~= false + +local names = fonts.names +local name_index = nil --> upvalue for names.data +local lookup_cache = nil --> for names.lookups +names.version = 2.4 +names.data = nil --- contains the loaded database +names.lookups = nil --- contains the lookup cache + +names.path = { index = { }, lookups = { } } +names.path.globals = { + prefix = "", --- writable_path/names_dir + names_dir = luaotfloadconfig.names_dir or "names", + index_file = luaotfloadconfig.index_file + or "luaotfload-names.lua", + lookups_file = "luaotfload-lookup-cache.lua", } --- string -> (string * string) @@ -115,7 +151,61 @@ local make_luanames = function (path) filereplacesuffix(path, "luc") end -local report = logs.names_report +local report = logs.names_report +local report_status = logs.names_status +local report_status_start = logs.names_status_start +local report_status_stop = logs.names_status_stop + +--- The “termwidth” value is only considered when printing +--- short status messages, e.g. when building the database +--- online. +if not luaotfloadconfig.termwidth then + local tw = 79 + if not ( os.type == "windows" --- Assume broken terminal. + or os.getenv "TERM" == "dumb") + then + local p = iopopen "tput cols" + if p then + result = tonumber (p:read "*all") + p:close () + if result then + tw = result + else + report ("log", 2, "db", "tput returned non-number.") + end + else + report ("log", 2, "db", "Shell escape disabled or tput executable missing.") + report ("log", 2, "db", "Assuming 79 cols terminal width.") + end + end + luaotfloadconfig.termwidth = tw +end + +names.patterns = { } +local patterns = names.patterns + +local trailingslashes = P"/"^1 * P(-1) +local stripslashes = C((1 - trailingslashes)^0) +patterns.stripslashes = stripslashes + +local comma = P"," +local noncomma = 1-comma +local splitcomma = Ct((C(noncomma^1) + comma)^1) +patterns.splitcomma = splitcomma + +local format_precedence = { + "otf", "ttc", "ttf", + "dfont", "afm", "pfb", + "pfa", +} + +local location_precedence = { + "local", "system", "texmf", +} + +local set_location_precedence = function (precedence) + location_precedence = precedence +end --[[doc-- We use the functions in the cache.* namespace that come with the @@ -130,7 +220,7 @@ local report = logs.names_report created by different user. --doc]]-- -if caches then +if not runasscript then local globals = names.path.globals local names_dir = globals.names_dir @@ -139,16 +229,17 @@ if caches then luaotfload.error ("Impossible to find a suitable writeable cache...") else + prefix = lpegmatch (stripslashes, prefix) report ("log", 0, "db", - "root cache directory is " .. prefix) + "Root cache directory is %s.", prefix) end globals.prefix = prefix - local lookups = names.path.lookups + local lookup_path = names.path.lookups local index = names.path.index local lookups_file = filejoin (prefix, globals.lookups_file) local index_file = filejoin (prefix, globals.index_file) - lookups.lua, lookups.luc = make_luanames (lookups_file) + lookup_path.lua, lookup_path.luc = make_luanames (lookups_file) index.lua, index.luc = make_luanames (index_file) else --- running as script, inject some dummies caches = { } @@ -160,14 +251,42 @@ end Auxiliary functions --doc]]-- +--- fontnames contain all kinds of garbage; as a precaution we +--- lowercase and strip them of non alphanumerical characters + --- string -> string -local sanitize_string = function (str) + +local invalidchars = "[^%a%d]" + +local sanitize_fontname = function (str) if str ~= nil then - return utf8gsub(utf8lower(str), "[^%a%d]", "") + str = utf8gsub (utf8lower (str), invalidchars, "") + return str end return nil end +local sanitize_fontnames = function (rawnames) + local result = { } + for category, namedata in next, rawnames do + + if type (namedata) == "string" then + result [category] = utf8gsub (utf8lower (namedata), + invalidchars, + "") + else + local target = { } + for field, name in next, namedata do + target [field] = utf8gsub (utf8lower (name), + invalidchars, + "") + end + result [category] = target + end + end + return result +end + local find_files_indeed find_files_indeed = function (acc, dirs, filter) if not next (dirs) then --- done @@ -219,16 +338,44 @@ end This is a sketch of the luaotfload db: type dbobj = { - mappings : fontentry list; - status : filestatus; - version : float; - // new in v2.3; these supersede the basenames / barenames - // hashes from v2.2 - filenames : filemap; + families : familytable; + files : filemap; + status : filestatus; + mappings : fontentry list; + meta : metadata; + names : namedata; // TODO: check for relevance after db is finalized + } + and familytable = { + local : (format, familyentry) hash; // specified with include dir + texmf : (format, familyentry) hash; + system : (format, familyentry) hash; + } + and familyentry = { + regular : sizes; + italic : sizes; + bold : sizes; + bolditalic : sizes; + } + and sizes = { + default : int; // points into mappings or names + optical : (int, int) list; // design size -> index entry + } + and metadata = { + formats : string list; // { "otf", "ttf", "ttc", "dfont" } + statistics : TODO; + version : float; } and filemap = { - base : (string, int) hash; // basename -> idx - bare : (string, int) hash; // barename -> idx + base : { + local : (string, int) hash; // basename -> idx + system : (string, int) hash; + texmf : (string, int) hash; + }; + bare : { + local : (string, (string, int) hash) hash; // location -> (barename -> idx) + system : (string, (string, int) hash) hash; + texmf : (string, (string, int) hash) hash; + }; full : (int, string) hash; // idx -> full path } and fontentry = { @@ -239,6 +386,7 @@ This is a sketch of the luaotfload db: fullname : string; // <- metadata sanitized : { family : string; + fontstyle_name : string; // <- new in 2.4 fontname : string; // <- metadata fullname : string; // <- namedata.names metafamily : string; @@ -250,13 +398,14 @@ This is a sketch of the luaotfload db: size : int list; slant : int; subfont : int; - texmf : bool; + location : local | system | texmf; weight : int; width : int; - units_per_em : int; // mainly 1000, but also 2048 or 256 + units_per_em : int; // mainly 1000, but also 2048 or 256 } - and filestatus = (fullname, - { index : int list; timestamp : int }) dict + and filestatus = (string, // fullname + { index : int list; // pointer into mappings + timestamp : int; }) dict beware that this is a reconstruction and may be incomplete. @@ -295,42 +444,95 @@ mtx-fonts has in names.tma: --doc]]-- -local fontnames_init = function (keep_cache) --- returns dbobj +local initialize_namedata = function (formats) --- returns dbobj return { - mappings = { }, - status = { }, --- filenames = { }, -- created later - version = names.version, + --families = { }, + status = { }, -- was: status; map abspath -> mapping + mappings = { }, -- TODO: check if still necessary after rewrite + names = { }, +-- files = { }, -- created later + meta = { + formats = formats, + statistics = { }, + version = names.version, + }, } end +--[[doc-- + + Since Luaotfload does not depend on the lualibs anymore we + have to put our own small wrappers for the gzip library in + place. + + load_gzipped -- Read and decompress and entire gzipped file. + Returns the uncompressed content as a string. + +--doc]]-- + +local load_gzipped = function (filename) + local gh = gzipopen (filename,"rb") + if gh then + local data = gh:read "*all" + gh:close () + return data + end +end + +--[[doc-- + + save_gzipped -- Compress and write a string to file. The return + value is the number of bytes written. Zlib parameters are: best + compression and default strategy. + +--doc]]-- + +local save_gzipped = function (filename, data) + local gh = gzipopen (filename, "wb9") + if gh then + gh:write (data) + local bytes = gh:seek () + gh:close () + return bytes + end +end + --- When loading a lua file we try its binary complement first, which --- is assumed to be located at an identical path, carrying the suffix --- .luc. --- string -> (string * table) local load_lua_file = function (path) - local foundname = filereplacesuffix(path, "luc") + local foundname = filereplacesuffix (path, "luc") + local code = nil - local fh = ioopen(foundname, "rb") -- try bin first + local fh = ioopen (foundname, "rb") -- try bin first if fh then local chunk = fh:read"*all" fh:close() - code = load(chunk, "b") + code = load (chunk, "b") end if not code then --- fall back to text file - foundname = filereplacesuffix(path, "lua") + foundname = filereplacesuffix (path, "lua") fh = ioopen(foundname, "rb") if fh then local chunk = fh:read"*all" fh:close() - code = load(chunk, "t") + code = load (chunk, "t") + end + end + + if not code then --- probe gzipped file + foundname = filereplacesuffix (path, "lua.gz") + local chunk = load_gzipped (foundname) + if chunk then + code = load (chunk, "t") end end if not code then return nil, nil end - return foundname, code() + return foundname, code () end --- define locals in scope @@ -338,127 +540,106 @@ local crude_file_lookup local crude_file_lookup_verbose local find_closest local flush_lookup_cache -local font_fullinfo +local ot_fullinfo +local t1_fullinfo local load_names local load_lookups local read_blacklist local read_fonts_conf local reload_db -local resolve +local resolve_name local resolve_cached local resolve_fullpath local save_names local save_lookups local update_names +local get_font_filter +local set_font_filter --- state of the database -local fonts_loaded = false local fonts_reloaded = false +local fonts_read = 0 --- limit output when approximate font matching (luaotfload-tool -F) local fuzzy_limit = 1 --- display closest only --- bool? -> dbobj load_names = function (dry_run) - local starttime = os.gettimeofday() - local foundname, data = load_lua_file(names.path.index.lua) + local starttime = osgettimeofday () + local foundname, data = load_lua_file (names.path.index.lua) if data then - report("both", 2, "db", - "Font names database loaded", "%s", foundname) - report("info", 3, "db", "Loading took %0.f ms", - 1000*(os.gettimeofday()-starttime)) + report ("both", 2, "db", + "Font names database loaded", "%s", foundname) + report ("info", 3, "db", "Loading took %0.f ms.", + 1000 * (osgettimeofday () - starttime)) - local db_version, nms_version = data.version, names.version + local db_version, nms_version = data.meta.version, names.version if db_version ~= nms_version then - report("both", 0, "db", - [[Version mismatch; expected %4.3f, got %4.3f]], - nms_version, db_version) + report ("both", 0, "db", + [[Version mismatch; expected %4.3f, got %4.3f.]], + nms_version, db_version) if not fonts_reloaded then - report("both", 0, "db", [[Force rebuild]]) - return update_names({ }, true, false) + report ("both", 0, "db", [[Force rebuild.]]) + data = update_names ({ }, true, false) + if not data then + report ("both", 0, "db", + "Database creation unsuccessful.") + end end end else - report("both", 0, "db", - [[Font names database not found, generating new one.]]) - report("both", 0, "db", - [[This can take several minutes; please be patient.]]) - data = update_names(fontnames_init(false), nil, dry_run) - local success = save_names(data) - if not success then - report("both", 0, "db", "Database creation unsuccessful.") + report ("both", 0, "db", + [[Font names database not found, generating new one.]]) + report ("both", 0, "db", + [[This can take several minutes; please be patient.]]) + data = update_names (initialize_namedata (get_font_filter ()), + nil, dry_run) + if not data then + report ("both", 0, "db", "Database creation unsuccessful.") end end - fonts_loaded = true return data end ---- unit -> dbobj +--- unit -> unit load_lookups = function ( ) local foundname, data = load_lua_file(names.path.lookups.lua) if data then report("both", 3, "cache", - "Lookup cache loaded (%s)", foundname) + "Lookup cache loaded from %s.", foundname) else report("both", 1, "cache", "No lookup cache, creating empty.") data = { } end - return data + lookup_cache = data end -local style_synonyms = { set = { } } -do - local combine = function (ta, tb) - local result = { } - for i=1, #ta do - for j=1, #tb do - result[#result+1] = ta[i] .. tb[j] - end - end - return result - end - - --- read this: http://blogs.adobe.com/typblography/2008/05/indesign_font_conflicts.html - --- tl;dr: font style synonyms are unreliable. - --- - --- Context matches font names against lists of known identifiers - --- for weight, style, width, and variant, so that including - --- the family name there are five dimensions for choosing a - --- match. The sad thing is, while this is a decent heuristic it - --- makes no sense to imitate it in luaotfload because the user - --- interface must fit into the much more limited Xetex scheme that - --- distinguishes between merely four style categories (variants): - --- “regular”, “italic”, “bold”, and “bolditalic”. As a result, - --- some of the styles are lumped together although they can differ - --- significantly (like “medium” and “bold”). - - --- Xetex (XeTeXFontMgr.cpp) appears to recognize only “plain”, - --- “normal”, and “roman” as synonyms for “regular”. - local list = { - regular = { "normal", "roman", - "plain", "book", - "light", "extralight", - "ultralight", }, - bold = { "demi", "demibold", - "semibold", "boldregular", - "medium", "mediumbold", - "ultrabold", "extrabold", - "heavy", "black", - "bold", }, - italic = { "regularitalic", "normalitalic", - "oblique", "slanted", - "italic", }, - } +local regular_synonym = { + book = "r", + normal = "r", + plain = "r", + regular = "r", + roman = "r", +} - list.bolditalic = combine(list.bold, list.italic) - style_synonyms.list = list +local italic_synonym = { + oblique = true, + slanted = true, + italic = true, +} - for category, synonyms in next, style_synonyms.list do - style_synonyms.set[category] = tabletohash(synonyms, true) - end -end +local style_category = { + regular = "r", + bold = "b", + bolditalic = "bi", + italic = "i", + r = "regular", + b = "bold", + bi = "bolditalic", + i = "italic", +} local type1_formats = { "tfm", "ofm", } @@ -471,12 +652,12 @@ local verbose_lookup = function (data, kind, filename) found = data.full[found] if found == nil then --> texmf report("info", 0, "db", - "Crude file lookup: req=%s; hit=%s => kpse", + "Crude file lookup: req=%s; hit=%s => kpse.", filename, kind) found = dummy_findfile(filename) else report("info", 0, "db", - "Crude file lookup: req=%s; hit=%s; ret=%s", + "Crude file lookup: req=%s; hit=%s; ret=%s.", filename, kind, found) end return found @@ -484,60 +665,88 @@ local verbose_lookup = function (data, kind, filename) return false end ---- string -> (string | string * string) +--- string -> (string * string * bool) crude_file_lookup_verbose = function (filename) - if not names.data then names.data = load_names() end - local data = names.data - local mappings = data.mappings - local filenames = data.filenames + if not name_index then name_index = load_names() end + local mappings = name_index.mappings + local files = name_index.files local found --- look up in db first ... - found = verbose_lookup(filenames, "bare", filename) + found = verbose_lookup(files, "bare", filename) if found then - return found + return found, nil, true end - found = verbose_lookup(filenames, "base", filename) + found = verbose_lookup(files, "base", filename) if found then - return found + return found, nil, true end --- ofm and tfm, returns pair for i=1, #type1_formats do local format = type1_formats[i] if resolvers.findfile(filename, format) then - return file.addsuffix(filename, format), format + return file.addsuffix(filename, format), format, true end end - return filename, nil + return filename, nil, false end ---- string -> (string | string * string) -crude_file_lookup = function (filename) - if not names.data then names.data = load_names() end - local data = names.data - local mappings = data.mappings - local filenames = data.filenames +local lookup_filename = function (filename) + if not name_index then name_index = load_names () end + local files = name_index.files + local basedata = files.base + local baredata = files.bare + for i = 1, #location_precedence do + local location = location_precedence [i] + local basenames = basedata [location] + local barenames = baredata [location] + local idx + if basenames ~= nil then + idx = basenames [filename] + if idx then + goto done + end + end + if barenames ~= nil then + for j = 1, #format_precedence do + local format = format_precedence [j] + local filemap = barenames [format] + if filemap then + idx = barenames [format] [filename] + if idx then + break + end + end + end + end +::done:: + if idx then + return files.full [idx] + end + end +end - local found +--- string -> (string * string * bool) +crude_file_lookup = function (filename) + local found = lookup_filename (filename) - found = filenames.base[filename] - or filenames.bare[filename] + if not found then + found = dummy_findfile(filename) + end if found then - found = filenames.full[found] - if found == nil then - found = dummy_findfile(filename) - end - return found or filename + return found, nil, true end + for i=1, #type1_formats do local format = type1_formats[i] if resolvers.findfile(filename, format) then - return file.addsuffix(filename, format), format + return file.addsuffix(filename, format), format, true end end - return filename, nil + + return filename, nil, false end --[[doc-- @@ -545,15 +754,19 @@ Existence of the resolved file name is verified differently depending on whether the index entry has a texmf flag set. --doc]]-- -local get_font_file = function (fullnames, entry) +local get_font_file = function (index) + local entry = name_index.mappings [index] + if not entry then + return false + end local basename = entry.basename - if entry.texmf == true then + if entry.location == "texmf" then if kpselookup(basename) then return true, basename, entry.subfont end - else - local fullname = fullnames[entry.index] - if lfsisfile(fullname) then + else --- system, local + local fullname = name_index.files.full [index] + if lfsisfile (fullname) then return true, basename, entry.subfont end end @@ -562,28 +775,18 @@ end --[[doc-- We need to verify if the result of a cached lookup actually exists in -the texmf or filesystem. +the texmf or filesystem. Again, due to the schizoprenic nature of the +font managment we have to check both the system path and the texmf. --doc]]-- local verify_font_file = function (basename) - if not names.data then names.data = load_names() end - local filenames = names.data.filenames - local idx = filenames.base[basename] - if not idx then - return false - end - - --- firstly, check filesystem - local fullname = filenames.full[idx] - if fullname and lfsisfile(fullname) then + local path = resolve_fullpath (basename) + if path and lfsisfile(path) then return true end - - --- secondly, locate via kpathsea if kpsefind_file(basename) then return true end - return false end @@ -618,8 +821,8 @@ Idk what the “spec” resolver is for. optsize, size spec: name, sub resolved, sub, name, forced -* name: contains both the name resolver from luatex-fonts and resolve() - below +* name: contains both the name resolver from luatex-fonts and + resolve_name() below From my reading of font-def.lua, what a resolver does is basically rewrite the “name” field of the specification record @@ -648,46 +851,48 @@ local hash_request = function (specification) end --- 'a -> 'a -> table -> (string * int|boolean * boolean) -resolve_cached = function (_, _, specification) - if not names.lookups then names.lookups = load_lookups() end +resolve_cached = function (specification) + if not lookup_cache then load_lookups () end local request = hash_request(specification) report("both", 4, "cache", "Looking for %q in cache ...", request) - local found = names.lookups[request] + local found = lookup_cache [request] --- case 1) cache positive ---------------------------------------- if found then --- replay fields from cache hit report("info", 4, "cache", "Found!") local basename = found[1] --- check the presence of the file in case it’s been removed - local success = verify_font_file(basename) + local success = verify_font_file (basename) if success == true then return basename, found[2], true end - report("both", 4, "cache", "Cached file not found; resolving again") + report("both", 4, "cache", "Cached file not found; resolving again.") else - report("both", 4, "cache", "Not cached; resolving") + report("both", 4, "cache", "Not cached; resolving.") end --- case 2) cache negative ---------------------------------------- --- first we resolve normally ... - local filename, subfont, success = resolve(nil, nil, specification) - if not success then return filename, subfont, false end + local filename, subfont = resolve_name (specification) + if not filename then + return nil, nil + end --- ... then we add the fields to the cache ... ... local entry = { filename, subfont } - report("both", 4, "cache", "New entry: %s", request) - names.lookups[request] = entry + report("both", 4, "cache", "New entry: %s.", request) + lookup_cache [request] = entry --- obviously, the updated cache needs to be stored. --- TODO this should trigger a save only once the --- document is compiled (finish_pdffile callback?) - report("both", 5, "cache", "Saving updated cache") - local success = save_lookups() + report("both", 5, "cache", "Saving updated cache.") + local success = save_lookups () if not success then --- sad, but not critical - report("both", 0, "cache", "Could not write to cache") + report("both", 0, "cache", "Error writing cache.") end - return filename, subfont, true + return filename, subfont end --- this used to be inlined; with the lookup cache we don’t @@ -695,17 +900,19 @@ end --- “found” is the match accumulator local add_to_match = function (found, size, face) - local optsize, dsnsize, maxsize, minsize - if #face.size > 0 then - optsize = face.size - dsnsize = optsize[1] and optsize[1] / 10 - -- can be nil - maxsize = optsize[2] and optsize[2] / 10 or dsnsize - minsize = optsize[3] and optsize[3] / 10 or dsnsize - end local continue = true - if optsize then - if dsnsize == size or (size > minsize and size <= maxsize) then + + local optsize = face.size + + if optsize and next (optsize) then + local dsnsize, maxsize, minsize + dsnsize = optsize[1] + maxsize = optsize[2] + minsize = optsize[3] + + if size ~= nil + and (dsnsize == size or (size > minsize and size <= maxsize)) + then found[1] = face continue = false ---> break else @@ -715,244 +922,287 @@ local add_to_match = function (found, size, face) found[1] = face continue = false ---> break end + return found, continue end +local choose_closest = function (distances) + local closest = 2^51 + local match + for i = 1, #distances do + local d, index = unpack (distances [i]) + if d < closest then + closest = d + match = index + end + end + return match +end + --[[doc-- -Luatex-fonts, the font-loader package luaotfload imports, comes with -basic file location facilities (see luatex-fonts-syn.lua). -However, not only does the builtin functionality rely on Context’s font -name database, it is also too limited to be of more than basic use. -For this reason, luaotfload supplies its own resolvers that accesses -the font database created by the luaotfload-tool script. + choose_size -- Pick a font face of appropriate size from the list + of family members with matching style. There are three categories: + + 1. exact matches: if there is a face whose design size equals + the asked size, it is returned immediately and no further + candidates are inspected. + + 2. range matches: of all faces in whose design range the + requested size falls the one whose center the requested + size is closest to is returned. + + 3. out-of-range matches: of all other faces (i. e. whose range + is above or below the asked size) the one is chosen whose + boundary (upper or lower) is closest to the requested size. + + 4. default matches: if no design size or a design size of zero + is requested, the face with the default size is returned. --doc]]-- +--- int * int * int * int list -> int -> int +local choose_size = function (sizes, askedsize) + local mappings = name_index.mappings + local match = sizes.default + local exact + local inrange = { } --- distance * index list + local norange = { } --- distance * index list + local fontname, subfont + if askedsize ~= 0 then + --- firstly, look for an exactly matching design size or + --- matching range + for i = 1, #sizes do + local dsnsize, high, low, index = unpack (sizes [i]) + if dsnsize == askedsize then + --- exact match, this is what we were looking for + exact = index + goto skip + elseif askedsize < low then + --- below range, add to the norange table + local d = low - askedsize + norange [#norange + 1] = { d, index } + elseif askedsize > high then + --- beyond range, add to the norange table + local d = askedsize - high + norange [#norange + 1] = { d, index } + else + --- range match + local d = ((low + high) / 2) - askedsize + if d < 0 then + d = -d + end + inrange [#inrange + 1] = { d, index } + end + end + end +::skip:: + if exact then + match = exact + elseif #inrange > 0 then + match = choose_closest (inrange) + elseif #norange > 0 then + match = choose_closest (norange) + end + return match +end ---- ---- the request specification has the fields: ---- ---- · features: table ---- · normal: set of { ccmp clig itlc kern liga locl mark mkmk rlig } ---- · ??? ---- · forced: string ---- · lookup: "name" ---- · method: string ---- · name: string ---- · resolved: string ---- · size: int ---- · specification: string (== <lookup> ":" <name>) ---- · sub: string ---- ---- the first return value of “resolve” is the file name of the ---- requested font (string) ---- the second is of type bool or string and indicates the subfont of a ---- ttc ---- ---- 'a -> 'a -> table -> (string * string | bool * bool) ---- ---- note by phg: I added a third return value that indicates a ---- successful lookup as this cannot be inferred from the other ---- values. ---- - -resolve = function (_, _, specification) -- the 1st two parameters are used by ConTeXt - if not fonts_loaded then names.data = load_names() end - local data = names.data - - local name = sanitize_string(specification.name) - local style = sanitize_string(specification.style) or "regular" - - local askedsize - if specification.optsize then - askedsize = tonumber(specification.optsize) - elseif specification.size then - askedsize = specification.size / 65536 - end - - if type(data) ~= "table" then - --- this catches a case where load_names() doesn’t - --- return a database object, which can happen only - --- in case there is valid Lua code in the database, - --- but it’s not a table, e.g. it contains an integer. - if not fonts_reloaded then - return reload_db("invalid database; not a table", - resolve, nil, nil, specification - ) +--[[doc-- + + resolve_familyname -- Query the families table for an entry + matching the specification. + The parameters “name” and “style” are pre-sanitized. + +--doc]]-- +--- spec -> string -> string -> int -> string * int +local resolve_familyname = function (specification, name, style, askedsize) + local families = name_index.families + local mappings = name_index.mappings + local candidates = nil + --- arrow code alert + for i = 1, #location_precedence do + local location = location_precedence [i] + local locgroup = families [location] + for j = 1, #format_precedence do + local format = format_precedence [j] + local fmtgroup = locgroup [format] + if fmtgroup then + local familygroup = fmtgroup [name] + if familygroup then + local stylegroup = familygroup [style] + if stylegroup then --- suitable match + candidates = stylegroup + goto done + end + end + end end - --- unsucessfully reloaded; bail - return specification.name, false, false end + if true then + return nil, nil + end +::done:: + index = choose_size (candidates, askedsize) + local success, resolved, subfont = get_font_file (index) + if not success then + return nil, nil + end + report ("info", 2, "db", "Match found: %s(%d).", + resolved, subfont or 0) + return resolved, subfont +end - if not data.mappings then - if not fonts_reloaded then - return reload_db("invalid database; missing font mapping", - resolve, nil, nil, specification) - end - return specification.name, false, false - end - - local exact = { } --> collect exact style matches - local synonymous = { } --> collect matching style synonyms - local fallback --> e.g. non-matching style (fontspec is anal about this) - local candidates = { } --> secondary results, incomplete matches - - local synonym_set = style_synonyms.set - for n, face in next, data.mappings do - local family, subfamily, fullname, prefmodifiers - local psname, fontname, pfullname, metafamily - - local facenames = face.sanitized - if facenames then - family = facenames.family - subfamily = facenames.subfamily - prefmodifiers = facenames.prefmodifiers or facenames.subfamily - fullname = facenames.fullname - psname = facenames.psname - fontname = facenames.fontname - pfullname = facenames.pfullname - metafamily = facenames.metafamily - end - fontname = fontname or sanitize_string(face.fontname) - pfullname = pfullname or sanitize_string(face.fullname) - - if name == family - or name == metafamily +local resolve_fontname = function (specification, name, style) + local mappings = name_index.mappings + local fallback = nil + local lastresort = nil + style = style_category [style] + for i = 1, #mappings do + local face = mappings [i] + local prefmodifiers = face.prefmodifiers + local subfamily = face.subfamily + if face.fontname == name + or face.fullname == name + or face.psname == name then - if style == prefmodifiers then - local continue - exact, continue = add_to_match(exact, askedsize, face) - if continue == false then break end - elseif style == subfamily then - exact = add_to_match(exact, askedsize, face) - elseif synonym_set[style] and synonym_set[style][prefmodifiers] - or synonym_set.regular[prefmodifiers] - then - --- treat synonyms for prefmodifiers as first-class - --- (needed to prioritize DejaVu Book over Condensed) - exact = add_to_match(exact, askedsize, face) - elseif name == fullname - or name == pfullname - or name == fontname - or name == psname + return face.basename, face.subfont + elseif face.familyname == name then + if prefmodifiers == style + or subfamily == style then - synonymous = add_to_match(synonymous, askedsize, face) - elseif synonym_set[style] and synonym_set[style][subfamily] - or synonym_set.regular[subfamily] - then - synonymous = add_to_match(synonymous, askedsize, face) - elseif prefmodifiers == "regular" - or subfamily == "regular" then fallback = face - else --- mark as last straw but continue - candidates[#candidates+1] = face - end - else - if name == fullname - or name == pfullname - or name == fontname - or name == psname then - local continue - exact, continue = add_to_match(exact, askedsize, face) - if continue == false then break end + elseif regular_synonym [prefmodifiers] + or regular_synonym [subfamily] + then + lastresort = face end + elseif face.metafamily == name + and (regular_synonym [prefmodifiers] + or regular_synonym [subfamily]) + then + lastresort = face end end - - local found - if next(exact) then - found = exact - else - found = synonymous + if fallback then + return fallback.basename, fallback.subfont + end + if lastresort then + return lastresort.basename, lastresort.subfont end + return nil, nil +end - --- this is a monster - if #found == 1 then - --- “found” is really synonymous with “registered in the db”. - local entry = found[1] - local success, filename, subfont - = get_font_file(data.filenames.full, entry) - if success == true then - report("log", 0, "resolve", - "Font family='%s', subfamily='%s' found: %s", - name, style, filename - ) - return filename, subfont, true - end - elseif #found > 1 then - -- we found matching font(s) but not in the requested optical - -- sizes, so we loop through the matches to find the one with - -- least difference from the requested size. - local closest - local least = math.huge -- initial value is infinity - for i,face in next, found do - local dsnsize = face.size[1]/10 - local difference = mathabs(dsnsize - askedsize) - if difference < least then - closest = face - least = difference - end - end - local success, filename, subfont - = get_font_file(data.filenames.full, closest) - if success == true then - report("log", 0, "resolve", - "Font family='%s', subfamily='%s' found: %s", - name, style, filename - ) - return filename, subfont, true - end - elseif fallback then - local success, filename, subfont - = get_font_file(data.filenames.full, fallback) - if success == true then - report("log", 0, "resolve", - "No exact match for request %s; using fallback", - specification.specification - ) - report("log", 0, "resolve", - "Font family='%s', subfamily='%s' found: %s", - name, style, filename - ) - return filename, subfont, true - end - elseif next(candidates) then - --- pick the first candidate encountered - local entry = candidates[1] - local success, filename, subfont - = get_font_file(data.filenames.full, entry) - if success == true then - report("log", 0, "resolve", - "Font family='%s', subfamily='%s' found: %s", - name, style, filename - ) - return filename, subfont, true +--[[doc-- + + resolve_name -- Perform a name: lookup. This first queries the + font families table and, if there is no match for the spec, the + font names table. + The return value is a pair consisting of the file name and the + subfont index if appropriate.. + + the request specification has the fields: + + · features: table + · normal: set of { ccmp clig itlc kern liga locl mark mkmk rlig } + · ??? + · forced: string + · lookup: "name" + · method: string + · name: string + · resolved: string + · size: int + · specification: string (== <lookup> ":" <name>) + · sub: string + + The “size” field deserves special attention: if its value is + negative, then it actually specifies a scalefactor of the + design size of the requested font. This happens e.g. if a font is + requested without an explicit “at size”. If the font is part of a + larger collection with different design sizes, this complicates + matters a bit: Normally, the resolver prefers fonts that have a + design size as close as possible to the requested size. If no + size specified, then the design size is implied. But which design + size should that be? Xetex appears to pick the “normal” (unmarked) + size: with Adobe fonts this would be the one that is neither + “caption” nor “subhead” nor “display” &c ... For fonts by Adobe this + seems to be the one that does not receive a “prefmodifiers” field. + (IOW Adobe uses the “prefmodifiers” field to encode the design size + in more or less human readable format.) However, this is not true + of LM and EB Garamond. As this matters only where there are + multiple design sizes to a given font/style combination, we put a + workaround in place that chooses that unmarked version. + + The first return value of “resolve_name” is the file name of the + requested font (string). It can be passed to the fullname resolver + get_font_file(). + The second value is either “false” or an integer indicating the + subfont index in a TTC. + +--doc]]-- + +--- table -> string * (int | bool) +resolve_name = function (specification) + local resolved, subfont + if not name_index then name_index = load_names () end + local name = sanitize_fontname (specification.name) + local style = sanitize_fontname (specification.style) or "r" + local askedsize = specification.optsize + + if askedsize then + askedsize = tonumber (askedsize) + else + askedsize = specification.size + if askedsize and askedsize >= 0 then + askedsize = askedsize / 65536 + else + askedsize = 0 end end - --- no font found so far - if not fonts_reloaded then - --- last straw: try reloading the database - return reload_db( - "unresolved font name: '" .. name .. "'", - resolve, nil, nil, specification - ) + resolved, subfont = resolve_familyname (specification, + name, + style, + askedsize) + if not resolved then + resolved, subfont = resolve_fontname (specification, + name, + style) + end + if not resolved then + resolved = specification.name, false end - --- else, default to requested name - return specification.name, false, false -end --- resolve() + if not resolved then + if not fonts_reloaded then + return reload_db ("Font not found.", + resolve_name, + specification) + end + end + return resolved, subfont +end resolve_fullpath = function (fontname, ext) --- getfilename() - if not fonts_loaded then - names.data = load_names() - end - local filenames = names.data.filenames - local idx = filenames.base[fontname] - or filenames.bare[fontname] - if idx then - return filenames.full[idx] + if not name_index then name_index = load_names () end + local files = name_index.files + local basedata = files.base + local baredata = files.bare + for i = 1, #location_precedence do + local location = location_precedence [i] + local basenames = basedata [location] + local idx + if basenames ~= nil then + idx = basenames [fontname] + end + if ext then + local barenames = baredata [location] [ext] + if not idx and barenames ~= nil then + idx = barenames [fontname] + end + end + if idx then + return files.full [idx] + end end return "" end @@ -962,14 +1212,23 @@ end --- string -> ('a -> 'a) -> 'a list -> 'a reload_db = function (why, caller, ...) - report("both", 1, "db", "Reload initiated; reason: %q", why) - names.data = update_names(names.data, false, false) - local success = save_names() - if success then + local namedata = name_index + local formats = tableconcat (namedata.meta.formats, ",") + + report ("both", 1, "db", + "Reload initiated (formats: %s); reason: %q.", + formats, why) + + set_font_filter (formats) + namedata = update_names (namedata, false, false) + + if namedata then fonts_reloaded = true - return caller(...) + name_index = namedata + return caller (...) end - report("both", 0, "db", "Database update unsuccessful.") + + report ("both", 0, "db", "Database update unsuccessful.") end --- string -> string -> int @@ -1002,54 +1261,50 @@ end --- string -> int -> bool find_closest = function (name, limit) - local name = sanitize_string(name) + local name = sanitize_fontname (name) limit = limit or fuzzy_limit - if not fonts_loaded then names.data = load_names() end - - local data = names.data - - if type(data) ~= "table" then + if not name_index then name_index = load_names () end + if not name_index or type (name_index) ~= "table" then if not fonts_reloaded then return reload_db("no database", find_closest, name) end return false end + local by_distance = { } --- (int, string list) dict local distances = { } --- int list local cached = { } --- (string, int) dict - local mappings = data.mappings + local mappings = name_index.mappings local n_fonts = #mappings for n = 1, n_fonts do local current = mappings[n] - local cnames = current.sanitized --[[ This is simplistic but surpisingly fast. - Matching is performed against the “family” name - of a db record. We then store its “fullname” at - it edit distance. + Matching is performed against the “fullname” field + of a db record in preprocessed form. We then store the + raw “fullname” at its edit distance. We should probably do some weighting over all the font name categories as well as whatever agrep does. --]] - if cnames then - local fullname, family = cnames.fullname, cnames.family + local fullname = current.plainname + local sfullname = current.fullname + local dist = cached[sfullname]--- maybe already calculated - local dist = cached[family]--- maybe already calculated - if not dist then - dist = iterative_levenshtein(name, family) - cached[family] = dist - end - local namelst = by_distance[dist] - if not namelst then --- first entry - namelst = { fullname } - distances[#distances+1] = dist - else --- append - namelst[#namelst+1] = fullname - end - by_distance[dist] = namelst + if not dist then + dist = iterative_levenshtein(name, sfullname) + cached[sfullname] = dist + end + local namelst = by_distance[dist] + if not namelst then --- first entry + namelst = { fullname } + distances[#distances+1] = dist + else --- append + namelst[#namelst+1] = fullname end + by_distance[dist] = namelst end --- print the matches according to their distance @@ -1058,15 +1313,15 @@ find_closest = function (name, limit) tablesort(distances) limit = mathmin(n_distances, limit) report(false, 1, "query", - "Displaying %d distance levels", limit) + "Displaying %d distance levels.", limit) for i = 1, limit do local dist = distances[i] local namelst = by_distance[dist] report(false, 0, "query", - "Distance from \"" .. name .. "\": " .. dist - .. "\n " .. tableconcat(namelst, "\n ") - ) + "Distance from \"%s\": %s\n " + .. tableconcat (namelst, "\n "), + name, dist) end return true @@ -1074,182 +1329,478 @@ find_closest = function (name, limit) return false end --- find_closest() -local sanitize_names = function (names) - local res = { } - for idx, name in next, names do - res[idx] = sanitize_string(name) - end - return res -end - --[[doc-- -The data inside an Opentype font file can be quite heterogeneous. -Thus in order to get the relevant information, parts of the original -table as returned by the font file reader need to be relocated. + + load_font_file -- Safely open a font file. See + <http://www.ntg.nl/pipermail/ntg-context/2013/075885.html> + regarding the omission of ``fontloader.close()``. + --doc]]-- ---- string -> int -> bool -> string -> fontentry -font_fullinfo = function (filename, subfont, texmf, basename) - local tfmdata = { } - local rawfont = fontloaderopen(filename, subfont) + +local load_font_file = function (filename, subfont) + local rawfont, _msg = fontloaderopen (filename, subfont) if not rawfont then - report("log", 1, "error", "Failed to open %s", filename) + report ("log", 1, "db", "ERROR: failed to open %s.", filename) return end - local metadata = fontloader.to_table(rawfont) - fontloader.close(rawfont) - collectgarbage("collect") + return rawfont +end + +--- rawdata -> (int * int * int | bool) + +local get_size_info = function (metadata) + local design_size = metadata.design_size + local design_range_top = metadata.design_range_top + local design_range_bottom = metadata.design_range_bottom + + local fallback_size = design_size ~= 0 and design_size + or design_range_bottom ~= 0 and design_range_bottom + or design_range_top ~= 0 and design_range_top + + if fallback_size then + design_size = (design_size or fallback_size) / 10 + design_range_top = (design_range_top or fallback_size) / 10 + design_range_bottom = (design_range_bottom or fallback_size) / 10 + return { + design_size, design_range_top, design_range_bottom, + } + end + + return false +end + +local get_english_names = function (names, basename) + + local english_names + + if names then + --inspect(names) + for _, raw_namedata in next, names do + if raw_namedata.lang == "English (US)" then + english_names = raw_namedata.names + end + end + else + -- no names table, probably a broken font + report("log", 1, "db", + "Broken font %s rejected due to missing names table.", + basename) + return nil + end + + return english_names +end + +local organize_namedata = function (metadata, + english_names, + basename, + info) + local fontnames = { + --- see + --- https://developer.apple.com/fonts/TTRefMan/RM06/Chap6name.html + --- http://www.microsoft.com/typography/OTSPEC/name.htm#NameIDs + english = { + --- where a “compatfull” field is given, the value of “fullname” is + --- either identical or differs by separating the style + --- with a hyphen and omitting spaces. (According to the + --- spec, “compatfull” is “Macintosh only”.) + --- Of the three “fullname” fields, this one appears to be the one + --- with the entire name given in a legible, + --- non-abbreviated fashion, for most fonts at any rate. + --- However, in some fonts (e.g. CMU) all three fields are + --- identical. + fullname = --[[ 18 ]] english_names.compatfull + or --[[ 4 ]] english_names.fullname, + --- we keep both the “preferred family” and the “family” + --- values around since both are valid but can turn out + --- quite differently, e.g. with Latin Modern: + --- preffamily: “Latin Modern Sans”, + --- family: “LM Sans 10” + preffamily = --[[ 16 ]] english_names.preffamilyname, + family = --[[ 1 ]] english_names.family, + prefmodifiers = --[[ 17 ]] english_names.prefmodifiers, + subfamily = --[[ 2 ]] english_names.subfamily, + psname = --[[ 6 ]] english_names.postscriptname, + }, + + metadata = { + fullname = metadata.fullname, + fontname = metadata.fontname, + familyname = metadata.familyname, + }, + + info = { + fullname = info.fullname, + familyname = info.familyname, + fontname = info.fontname, + }, + } + -- see http://www.microsoft.com/typography/OTSPEC/features_pt.htm#size if metadata.fontstyle_name then + --- not present in all fonts, often differs from the preferred + --- subfamily as well as subfamily fields, e.g. with + --- LMSans10-BoldOblique: + --- subfamily: “Bold Italic” + --- prefmodifiers: “10 Bold Oblique” + --- fontstyle_name: “Bold Oblique” for _, name in next, metadata.fontstyle_name do if name.lang == 1033 then --- I hate magic numbers - tfmdata.fontstyle_name = name.name + fontnames.fontstyle_name = name.name end end end - if metadata.names then - for _, namedata in next, metadata.names do - if namedata.lang == "English (US)" then - local names = { - --- see - --- https://developer.apple.com/fonts/TTRefMan/RM06/Chap6name.html - fullname = namedata.names.compatfull - or namedata.names.fullname, - family = namedata.names.preffamilyname - or namedata.names.family, - prefmodifiers = namedata.names.prefmodifiers, - subfamily = tfmdata.fontstyle_name - or namedata.names.subfamily, - psname = namedata.names.postscriptname, - pfullname = metadata.fullname, - fontname = metadata.fontname, - metafamily = metadata.familyname, - } --- tfmdata.names = names - tfmdata.sanitized = sanitize_names(names) - end - end - else - -- no names table, propably a broken font - report("log", 1, "db", "Broken font rejected", "%s", basefile) - return + + return { + sanitized = sanitize_fontnames (fontnames), + fontname = metadata.fontname, + fullname = metadata.fullname, + familyname = metadata.familyname, + } + +end + + +local dashsplitter = lpegsplitat "-" + +local split_fontname = function (fontname) + --- sometimes the style hides in the latter part of the + --- fontname, separated by a dash, e.g. “Iwona-Regular”, + --- “GFSSolomos-Regular” + local splitted = { lpegmatch (dashsplitter, fontname) } + if next (splitted) then + return sanitize_fontname (splitted [#splitted]) end - tfmdata.fontname = metadata.fontname - tfmdata.fullname = metadata.fullname - tfmdata.familyname = metadata.familyname - tfmdata.weight = metadata.pfminfo.weight - tfmdata.width = metadata.pfminfo.width - tfmdata.slant = metadata.italicangle - --- this is for querying - tfmdata.units_per_em = metadata.units_per_em - tfmdata.version = metadata.version - -- don't waste the space with zero values - tfmdata.size = { - metadata.design_size ~= 0 and metadata.design_size or nil, - metadata.design_range_top ~= 0 and metadata.design_range_top or nil, - metadata.design_range_bottom ~= 0 and metadata.design_range_bottom or nil, +end + +local organize_styledata = function (fontname, + metadata, + english_names, + info) + local pfminfo = metadata.pfminfo + local names = metadata.names + + return { + -- see http://www.microsoft.com/typography/OTSPEC/features_pt.htm#size + size = get_size_info (metadata), + weight = { + pfminfo.weight, -- integer (multiple of 100?) + sanitize_fontname (info.weight), -- style name + }, + split = split_fontname (fontname), + width = pfminfo.width, + italicangle = metadata.italicangle, +-- italicangle = { +-- metadata.italicangle, -- float +-- info.italicangle, -- truncated to integer point size? +-- }, + --- this is for querying, see www.ntg.nl/maps/40/07.pdf for details + units_per_em = metadata.units_per_em, + version = metadata.version, } +end - --- file location data (used to be filename field) - tfmdata.filename = filename --> sys - tfmdata.basename = basename --> texmf - tfmdata.texmf = texmf or false - tfmdata.subfont = subfont +--[[doc-- +The data inside an Opentype font file can be quite heterogeneous. +Thus in order to get the relevant information, parts of the original +table as returned by the font file reader need to be relocated. +--doc]]-- + +--- string -> int -> bool -> string -> fontentry - return tfmdata +ot_fullinfo = function (filename, + subfont, + location, + basename, + format, + info) + + local metadata = load_font_file (filename, subfont) + if not metadata then + return nil + end + + local english_names = get_english_names (metadata.names, basename) + local namedata = organize_namedata (metadata, + english_names, + basename, + info) + local style = organize_styledata (namedata.fontname, + metadata, + english_names, + info) + + local res = { + file = { base = basename, + full = filename, + subfont = subfont, + location = location or "system" }, + format = format, + names = namedata, + style = style, + version = metadata.version, + } + --- Closing the file manually is a tad faster and more memory + --- efficient than having it closed by the gc + fontloaderclose (metadata) + return res end ---- we return true if the fond is new or re-indexed ---- string -> dbobj -> dbobj -> bool -local load_font = function (fullname, fontnames, newfontnames, texmf) - if not fullname then +--[[doc-- + + Type1 font inspector. In comparison with OTF, PFB’s contain a good + deal less name fields which makes it tricky in some parts to find a + meaningful representation for the database. + + Good read: http://www.adobe.com/devnet/font/pdfs/5004.AFM_Spec.pdf + +--doc]]-- + +--- string -> int -> bool -> string -> fontentry + +t1_fullinfo = function (filename, _subfont, location, basename, format) + local sanitized + local metadata = load_font_file (filename) + local fontname = metadata.fontname + local fullname = metadata.fullname + local familyname = metadata.familyname + local italicangle = metadata.italicangle + local splitstyle = split_fontname (fontname) + local style = "" + local weight + + sanitized = sanitize_fontnames ({ + fontname = fontname, + psname = fullname, + pfullname = fullname, + metafamily = family, + familyname = familyname, + weight = metadata.weight, --- string identifier + prefmodifiers = style, + }) + + weight = sanitized.weight + + if weight == "bold" then + style = weight + end + + if italicangle ~= 0 then + style = style .. "italic" + end + + return { + basename = basename, + fullpath = filename, + subfont = false, + location = location or "system", + format = format, + fullname = sanitized.fullname, + fontname = sanitized.fontname, + familyname = sanitized.familyname, + plainname = fullname, + psname = sanitized.fontname, + version = metadata.version, + size = false, + splitstyle = splitstyle, + fontstyle_name = style ~= "" and style or weight, + weight = { metadata.pfminfo.weight, + weight }, + italicangle = italicangle, + } +end + +local loaders = { + dfont = ot_fullinfo, + otf = ot_fullinfo, + ttc = ot_fullinfo, + ttf = ot_fullinfo, + + pfb = t1_fullinfo, + pfa = t1_fullinfo, +} + +--- not side-effect free! + +local compare_timestamps = function (fullname, + currentstatus, + currententrystatus, + currentmappings, + targetstatus, + targetentrystatus, + targetmappings) + + local currenttimestamp = currententrystatus + and currententrystatus.timestamp + local targettimestamp = lfsattributes (fullname, "modification") + + if targetentrystatus ~= nil + and targetentrystatus.timestamp == targettimestamp then + report ("log", 3, "db", "Font %q already read.", fullname) return false end - local newmappings = newfontnames.mappings - local newstatus = newfontnames.status --- by full path + targetentrystatus.timestamp = targettimestamp + targetentrystatus.index = targetentrystatus.index or { } + + if currenttimestamp == targettimestamp + and not targetentrystatus.index [1] + then + --- copy old namedata into new + + for _, currentindex in next, currententrystatus.index do + + local targetindex = #targetentrystatus.index + local fullinfo = currentmappings [currentindex] + local location = #targetmappings + 1 + + targetmappings [location] = fullinfo + targetentrystatus.index [targetindex + 1] = location + end + + report ("log", 3, "db", "Font %q already indexed.", fullname) + + return false + end + + return true +end + +local insert_fullinfo = function (fullname, + basename, + n_font, + loader, + format, + location, + targetmappings, + targetentrystatus, + info) + + local subfont + if n_font ~= false then + subfont = n_font - 1 + else + subfont = false + n_font = 1 + end + + local fullinfo = loader (fullname, subfont, + location, basename, + format, info) + + if not fullinfo then + return false + end + + local index = targetentrystatus.index [n_font] + + if not index then + index = #targetmappings + 1 + end + + targetmappings [index] = fullinfo + targetentrystatus.index [n_font] = index + + return true +end + - local mappings = fontnames.mappings - local status = fontnames.status - local basename = filebasename(fullname) - local barename = filenameonly(fullname) +--- we return true if the font is new or re-indexed +--- string -> dbobj -> dbobj -> bool + +local read_font_names = function (fullname, + currentnames, + targetnames, + location) + + local targetmappings = targetnames.mappings + local targetstatus = targetnames.status --- by full path + local targetentrystatus = targetstatus [fullname] + + if targetentrystatus == nil then + targetentrystatus = { } + targetstatus [fullname] = targetentrystatus + end - local entryname = fullname - if texmf == true then + local currentmappings = currentnames.mappings + local currentstatus = currentnames.status + local currententrystatus = currentstatus [fullname] + + local basename = filebasename (fullname) + local barename = filenameonly (fullname) + local entryname = fullname + + if location == "texmf" then entryname = basename end - if names.blacklist[fullname] or names.blacklist[basename] - then + --- 1) skip if blacklisted + + if names.blacklist[fullname] or names.blacklist[basename] then report("log", 2, "db", - "Ignoring blacklisted font %q", fullname) + "Ignoring blacklisted font %q.", fullname) return false end - local new_timestamp, current_timestamp - current_timestamp = status[fullname] - and status[fullname].timestamp - new_timestamp = lfsattributes(fullname, "modification") + --- 2) skip if known with same timestamp - local newentrystatus = newstatus[fullname] - --- newentrystatus: nil | false | table - if newentrystatus and newentrystatus.timestamp == new_timestamp then - -- already indexed this run + if not compare_timestamps (fullname, + currentstatus, + currententrystatus, + currentmappings, + targetstatus, + targetentrystatus, + targetmappings) + then return false end - newstatus[fullname] = newentrystatus or { } - local newentrystatus = newstatus[fullname] - newentrystatus.timestamp = new_timestamp - newentrystatus.index = newentrystatus.index or { } + --- 3) new font; choose a loader, abort if unknown - if current_timestamp == new_timestamp - and not newentrystatus.index[1] - then - for _, v in next, status[fullname].index do - local index = #newentrystatus.index - local fullinfo = mappings[v] - local location = #newmappings + 1 - newmappings[location] = fullinfo --- keep - newentrystatus.index[index+1] = location --- is this actually used anywhere? - end - report("log", 2, "db", "Font %q already indexed", basename) + local format = stringlower (filesuffix (basename)) + local loader = loaders [format] --- ot_fullinfo, t1_fullinfo + + if not loader then + report ("both", 0, "db", + "Unknown format: %q, skipping.", format) return false end - local info = fontloaderinfo(fullname) - if info then - if type(info) == "table" and #info > 1 then --- ttc - for n_font = 1, #info do - local fullinfo = font_fullinfo(fullname, n_font-1, texmf, basename) - if not fullinfo then - return false - end - local location = #newmappings+1 - local index = newentrystatus.index[n_font] - if not index then index = location end + --- 4) get basic info, abort if fontloader can’t read it - newmappings[index] = fullinfo - newentrystatus.index[n_font] = index - end - else - local fullinfo = font_fullinfo(fullname, false, texmf, basename) - if not fullinfo then - return false - end - local location = #newmappings+1 - local index = newentrystatus.index[1] - if not index then index = location end + local info = fontloaderinfo (fullname) + + if not info then + report ("log", 1, "db", + "Failed to read basic information from %q", basename) + return false + end - newmappings[index] = fullinfo - newentrystatus.index[1] = index + + --- 5) check for subfonts and process each of them + + if type (info) == "table" and #info > 1 then --- ttc + + local success = false --- true if at least one subfont got read + + for n_font = 1, #info do + if insert_fullinfo (fullname, basename, n_font, + loader, format, location, + targetmappings, targetentrystatus, + info) + then + success = true + end end - else --- missing info - report("log", 1, "db", "Failed to load %q", basename) - return false + return success end - return true + + return insert_fullinfo (fullname, basename, false, + loader, format, location, + targetmappings, targetentrystatus, + info) end local path_normalize @@ -1327,7 +1878,7 @@ local create_blacklist = function (blacklist, whitelist) local result = { } local dirs = { } - report("info", 2, "db", "Blacklisting %q files and directories", + report("info", 2, "db", "Blacklisting %d files and directories.", #blacklist) for i=1, #blacklist do local entry = blacklist[i] @@ -1338,7 +1889,7 @@ local create_blacklist = function (blacklist, whitelist) end end - report("info", 2, "db", "Whitelisting %q files", #whitelist) + report("info", 2, "db", "Whitelisting %d files.", #whitelist) for i=1, #whitelist do result[whitelist[i]] = nil end @@ -1387,7 +1938,7 @@ read_blacklist = function () line = stringsub(line, 1, cmt - 1) end line = stringstrip(line) - report("log", 2, "db", "Blacklisted file %q", line) + report("log", 2, "db", "Blacklisted file %q.", line) blacklist[#blacklist+1] = line end end @@ -1396,22 +1947,73 @@ read_blacklist = function () names.blacklist = create_blacklist(blacklist, whitelist) end -local font_extensions = { "otf", "ttf", "ttc", "dfont" } -local font_extensions_set = tabletohash (font_extensions) -local p_font_extensions +local p_font_filter + do - local extns - --tablesort (font_extensions) --- safeguard - for i=#font_extensions, 1, -1 do - local e = font_extensions[i] - if not extns then - extns = P(e) - else - extns = extns + P(e) + local current_formats = { } + + local extension_pattern = function (list) + local pat + for i=#list, 1, -1 do + local e = list[i] + if not pat then + pat = P(e) + else + pat = pat + P(e) + end end + pat = pat * P(-1) + return (1 - pat)^1 * pat end - extns = extns * P(-1) - p_font_extensions = (1 - extns)^1 * extns + + --- small helper to adjust the font filter pattern (--formats + --- option) + + set_font_filter = function (formats) + + if not formats or type (formats) ~= "string" then + return + end + + if stringsub (formats, 1, 1) == "+" then -- add + formats = lpegmatch (splitcomma, stringsub (formats, 2)) + if formats then + current_formats = tableappend (current_formats, formats) + end + elseif stringsub (formats, 1, 1) == "-" then -- add + formats = lpegmatch (splitcomma, stringsub (formats, 2)) + if formats then + local newformats = { } + for i = 1, #current_formats do + local fmt = current_formats[i] + local include = true + for j = 1, #formats do + if current_formats[i] == formats[j] then + include = false + goto skip + end + end + newformats[#newformats+1] = fmt + ::skip:: + end + current_formats = newformats + end + else -- set + formats = lpegmatch (splitcomma, formats) + if formats then + current_formats = formats + end + end + + p_font_filter = extension_pattern (current_formats) + end + + get_font_filter = function (formats) + return tablefastcopy (current_formats) + end + + --- initialize + set_font_filter (luaotfloadconfig.formats) end local process_dir_tree @@ -1438,9 +2040,22 @@ process_dir_tree = function (acc, dirs) then dirs[#dirs+1] = fullpath elseif lfsisfile (fullpath) then - if lpegmatch (p_font_extensions, stringlower(ent)) then - newfiles[#newfiles+1] = fullpath + ent = stringlower (ent) + + if lpegmatch (p_font_filter, ent) then + if filesuffix (ent) == "afm" then + --- fontloader.open() will load the afm + --- iff both files are in the same directory + local pfbpath = filereplacesuffix + (fullpath, "pfb") + if lfsisfile (pfbpath) then + newfiles[#newfiles+1] = pfbpath + end + else + newfiles[#newfiles+1] = fullpath + end end + end end end @@ -1450,88 +2065,197 @@ process_dir_tree = function (acc, dirs) return process_dir_tree (acc, dirs) end ---- string -> string list -local find_font_files = function (root) +local process_dir = function (dir) + local pwd = lfscurrentdir () + if lfschdir (dir) then + lfschdir (pwd) + + local files = { } + local blacklist = names.blacklist + for ent in lfsdir (dir) do + if ent ~= "." and ent ~= ".." and not blacklist[ent] then + local fullpath = dir .. "/" .. ent + if lfsisfile (fullpath) then + ent = stringlower (ent) + if lpegmatch (p_font_filter, ent) + then + if filesuffix (ent) == "afm" then + --- fontloader.open() will load the afm + --- iff both files are in the same + --- directory + local pfbpath = filereplacesuffix + (fullpath, "pfb") + if lfsisfile (pfbpath) then + files[#files+1] = pfbpath + end + else + files[#files+1] = fullpath + end + end + end + end + end + return files + end + return { } +end + +--- string -> bool -> string list +local find_font_files = function (root, recurse) if lfsisdir (root) then - return process_dir_tree ({}, { root }) + if recurse == true then + return process_dir_tree ({}, { root }) + else --- kpathsea already delivered the necessary subdirs + return process_dir (root) + end end end +--- truncate_string -- Cut the first part of a string to fit it +--- into a given terminal width. The parameter “restrict” (int) +--- indicates the number of characters already consumed on the +--- line. +local truncate_string = function (str, restrict) + local tw = luaotfloadconfig.termwidth + local wd = tw - restrict + local len = utf8length (str) + if wd - len < 0 then + --- combined length exceeds terminal, + str = ".." .. stringsub(str, len - wd + 2) + end + return str +end + --[[doc-- scan_dir() scans a directory and populates the list of fonts with all the fonts it finds. - · dirname : name of the directory to scan - · fontnames : current font db object - · newnames : font db object to fill - · dry_run : don’t touch anything + · dirname : name of the directory to scan + · currentnames : current font db object + · targetnames : font db object to fill + · dry_run : don’t touch anything --doc]]-- --- string -> dbobj -> dbobj -> bool -> bool -> (int * int) -local scan_dir = function (dirname, fontnames, newfontnames, - dry_run, texmf) + +local scan_dir = function (dirname, currentnames, targetnames, + dry_run, location) if lpegmatch (p_blacklist, dirname) then - report ("both", 3, "db", - "Skipping blacklisted directory %s", dirname) + report ("both", 4, "db", + "Skipping blacklisted directory %s.", dirname) --- ignore return 0, 0 end - local found = find_font_files (dirname) + local found = find_font_files (dirname, location ~= "texmf") if not found then - report ("both", 3, "db", + report ("both", 4, "db", "No such directory: %q; skipping.", dirname) return 0, 0 end - report ("both", 3, "db", "Scanning directory %s", dirname) + report ("both", 4, "db", "Scanning directory %s.", dirname) + + local n_new = 0 --- total of fonts collected + local n_found = #found + local max_fonts = luaotfloadconfig.max_fonts - local n_new = 0 --- total of fonts collected - local n_found = #found - report ("both", 4, "db", "%d font files detected", n_found) + report ("both", 4, "db", "%d font files detected.", n_found) for j=1, n_found do + if max_fonts and fonts_read >= max_fonts then + break + end + local fullname = found[j] fullname = path_normalize(fullname) local new + if dry_run == true then - report ("both", 1, "db", - "Would have been loading %q", fullname) + local truncated = truncate_string (fullname, 43) + report ("log", 2, "db", + "Would have been loading %s.", fullname) + report_status ("term", "db", + "Would have been loading %s", truncated) else - report ("both", 4, "db", - "Loading font %q", fullname) - local new = load_font (fullname, fontnames, - newfontnames, texmf) + local truncated = truncate_string (fullname, 32) + report ("log", 2, "db", "Loading font %s.", fullname) + report_status ("term", "db", "Loading font %s", truncated) + local new = read_font_names (fullname, currentnames, + targetnames, texmf) if new == true then + fonts_read = fonts_read + 1 n_new = n_new + 1 end end end - - report("both", 4, "db", "%d fonts found in '%s'", n_found, dirname) + report ("both", 4, "db", "Done. %d fonts indexed in %q.", + n_found, dirname) return n_found, n_new end +--- string list -> string list +local filter_out_pwd = function (dirs) + local result = { } + local pwd = path_normalize (lpegmatch (stripslashes, + lfscurrentdir ())) + for i = 1, #dirs do + --- better safe than sorry + local dir = path_normalize (lpegmatch (stripslashes, dirs[i])) + if not (dir == "." or dir == pwd) then + result[#result+1] = dir + end + end + return result +end + +local path_separator = ostype == "windows" and ";" or ":" + +--[[doc-- + scan_texmf_fonts() scans all fonts in the texmf tree through the + kpathsea variables OPENTYPEFONTS and TTFONTS of texmf.cnf. + The current working directory comes as “.” (texlive) or absolute + path (miktex) and will always be filtered out. +--doc]]-- + --- dbobj -> dbobj -> bool? -> (int * int) -local scan_texmf_fonts = function (fontnames, newfontnames, dry_run) - local n_scanned, n_new = 0, 0 - --[[ - This function scans all fonts in the texmf tree, through kpathsea - variables OPENTYPEFONTS and TTFONTS of texmf.cnf - ]] - if stringis_empty(kpseexpand_path("$OSFONTDIR")) then - report("info", 2, "db", "Scanning TEXMF fonts...") + +local scan_texmf_fonts = function (currentnames, targetnames, dry_run) + + local n_scanned, n_new, fontdirs = 0, 0 + local osfontdir = kpseexpand_path "$OSFONTDIR" + + if stringis_empty (osfontdir) then + report ("info", 1, "db", "Scanning TEXMF fonts...") else - report("info", 2, "db", "Scanning TEXMF and OS fonts...") + report ("info", 1, "db", "Scanning TEXMF and OS fonts...") + if logs.get_loglevel () > 3 then + local osdirs = filesplitpath (osfontdir) + report ("info", 0, "db", + "$OSFONTDIR has %d entries:", #osdirs) + for i = 1, #osdirs do + report ("info", 0, "db", "[%d] %s", i, osdirs[i]) + end + end end - local fontdirs = stringgsub(kpseexpand_path("$OPENTYPEFONTS"), "^%.", "") - fontdirs = fontdirs .. stringgsub(kpseexpand_path("$TTFONTS"), "^%.", "") - if not stringis_empty(fontdirs) then - for _,d in next, filesplitpath(fontdirs) do - local found, new = scan_dir(d, fontnames, newfontnames, dry_run, true) + + fontdirs = kpseexpand_path "$OPENTYPEFONTS" + fontdirs = fontdirs .. path_separator .. kpseexpand_path "$TTFONTS" + fontdirs = fontdirs .. path_separator .. kpseexpand_path "$T1FONTS" + + if not stringis_empty (fontdirs) then + local tasks = filter_out_pwd (filesplitpath (fontdirs)) + report ("info", 3, "db", + "Initiating scan of %d directories.", #tasks) + report_status_start (2, 4) + for _, d in next, tasks do + local found, new = scan_dir (d, currentnames, targetnames, + dry_run, "texmf") n_scanned = n_scanned + found n_new = n_new + new end + report_status_stop ("term", "db", "Scanned %d files, %d new.", n_scanned, n_new) end + return n_scanned, n_new end @@ -1631,7 +2355,7 @@ do --- closure for read_fonts_conf() local fonts_conf_scanner = function (path) local fh = ioopen(path, "r") if not fh then - report("both", 3, "db", "Cannot open fontconfig file %s", path) + report("both", 3, "db", "Cannot open fontconfig file %s.", path) return end local raw = fh:read"*all" @@ -1639,7 +2363,7 @@ do --- closure for read_fonts_conf() local confdata = lpegmatch(p_cheapxml, raw) if not confdata then - report("both", 3, "db", "Cannot scan fontconfig file %s", path) + report("both", 3, "db", "Cannot scan fontconfig file %s.", path) return end return confdata @@ -1690,6 +2414,10 @@ do --- closure for read_fonts_conf() --- We exclude paths with texmf in them, as they should be --- found anyway; also duplicates are ignored by checking --- if they are elements of dirs_done. + --- + --- FIXME does this mean we cannot access paths from + --- distributions (e.g. Context minimals) installed + --- separately? if not (stringfind(path, "texmf") or dirs_done[path]) then acc[#acc+1] = path dirs_done[path] = true @@ -1786,226 +2514,782 @@ local function get_os_dirs () return {} end ---- dbobj -> dbobj -> bool? -> (int * int) -local scan_os_fonts = function (fontnames, newfontnames, dry_run) - local n_scanned, n_new = 0, 0 - --[[ - This function scans the OS fonts through - - fontcache for Unix (reads the fonts.conf file and scans the +--[[doc-- + + scan_os_fonts() scans the OS fonts through + - fontconfig for Unix (reads the fonts.conf file[s] and scans the directories) - a static set of directories for Windows and MacOSX - ]] - report("info", 2, "db", "Scanning OS fonts...") - report("info", 3, "db", "Searching in static system directories...") - for _, d in next, get_os_dirs() do - local found, new = scan_dir(d, fontnames, newfontnames, dry_run) + + **NB**: If $OSFONTDIR is nonempty, as it appears to be by default + on Windows setups, the system fonts will have already been + processed while scanning the TEXMF. Thus, this function is + never called. + +--doc]]-- + +--- dbobj -> dbobj -> bool? -> (int * int) +local scan_os_fonts = function (currentnames, + targetnames, + dry_run) + + local n_scanned, n_new = 0, 0 + report ("info", 1, "db", "Scanning OS fonts...") + report ("info", 2, "db", + "Searching in static system directories...") + + report_status_start (2, 4) + for _, d in next, get_os_dirs () do + local found, new = scan_dir (d, currentnames, + targetnames, dry_run) n_scanned = n_scanned + found n_new = n_new + new end + report_status_stop ("term", "db", "Scanned %d files, %d new.", n_scanned, n_new) + return n_scanned, n_new end --- unit -> (bool, lookup_cache) flush_lookup_cache = function () - if not names.lookups then names.lookups = load_lookups() end - names.lookups = { } + lookup_cache = { } collectgarbage "collect" - return true, names.lookups + return true, lookup_cache end ---- dbobj -> dbobj -local gen_fast_lookups = function (fontnames) - report("both", 2, "db", "Creating filename map") - local mappings = fontnames.mappings + +--- fontentry list -> filemap + +local generate_filedata = function (mappings) + + report ("both", 2, "db", "Creating filename map.") + local nmappings = #mappings - --- this is needlessly complicated due to texmf priorization - local filenames = { - bare = { }, - base = { }, + + local files = { + bare = { + ["local"] = { }, + system = { }, --- mapped to mapping format -> index in full + texmf = { }, --- mapped to mapping format -> “true” + }, + base = { + ["local"] = { }, + system = { }, --- mapped to index in “full” + texmf = { }, --- set; all values are “true” + }, full = { }, --- non-texmf } - local texmf, sys = { }, { } -- quintuple list + local base = files.base + local bare = files.bare + local full = files.full + + local conflicts = { + basenames = 0, + barenames = 0, + } - for idx = 1, nmappings do - local entry = mappings[idx] - local filename = entry.filename - local basename = entry.basename - local bare = filenameonly(filename) - local subfont = entry.subfont + for index = 1, nmappings do + local entry = mappings [index] + + local filedata = entry.file + local format + local location + local fullpath + local basename + local barename + local subfont + + if filedata then --- new entry + format = entry.format --- otf, afm, ... + location = filedata.location --- texmf, system, ... + fullpath = filedata.full + basename = filedata.base + barename = filenameonly (fullpath) + subfont = filedata.subfont + else + format = entry.format --- otf, afm, ... + location = entry.location --- texmf, system, ... + fullpath = entry.fullpath + basename = entry.basename + barename = filenameonly (fullpath) + subfont = entry.subfont + end - entry.index = idx ---- unfortunately, the sys/texmf schism prevents us from ---- doing away the full name, so we cannot avoid the ---- substantial duplication --- entry.filename = nil + entry.index = index - if entry.texmf == true then - texmf[#texmf+1] = { idx, basename, bare, true, nil } + --- 1) add to basename table + + local inbase = base [location] --- no format since the suffix is known + + if inbase then + local present = inbase [basename] + if present then + report ("both", 4, "db", + "Conflicting basename: %q already indexed \z + in category %s, ignoring.", + barename, location) + conflicts.basenames = conflicts.basenames + 1 + + --- track conflicts per font + local conflictdata = entry.conflicts + + if not conflictdata then + entry.conflicts = { basename = present } + else -- some conflicts already detected + conflictdata.basename = present + end + + else + inbase [basename] = index + end else - sys[#sys+1] = { idx, basename, bare, false, filename } + inbase = { basename = index } + base [location] = inbase end + + --- 2) add to barename table + + local inbare = bare [location] [format] + + if inbare then + local present = inbare [barename] + if present then + report ("both", 4, "db", + "Conflicting barename: %q already indexed \z + in category %s/%s, ignoring.", + barename, location, format) + conflicts.barenames = conflicts.barenames + 1 + + --- track conflicts per font + local conflictdata = entry.conflicts + + if not conflictdata then + entry.conflicts = { barename = present } + else -- some conflicts already detected + conflictdata.barename = present + end + + else + inbare [barename] = index + end + else + inbare = { [barename] = index } + bare [location] [format] = inbare + end + + --- 3) add to fullpath map + + full [index] = fullpath end - local addmap = function (lst) - --- this will overwrite existing entries - for i=1, #lst do - local idx, base, bare, intexmf, full = unpack(lst[i]) + return files +end + +local pick_style +local check_regular - local known = filenames.base[base] or filenames.bare[bare] - if known then --- known - report("both", 3, "db", - "Font file %q already indexed (%d)", - base, idx) - report("both", 3, "db", "> old location: %s", - (filenames.full[known] or "texmf")) - report("both", 3, "db", "> new location: %s", - (intexmf and "texmf" or full)) +do + local splitfontname = lpeg.splitat "-" + + local choose_exact = function (field) + if italic_synonym [field] then + return "i" + elseif field == "bold" then + return "b" + elseif field == "bolditalic" or field == "boldoblique" then + return "bi" + end + + return false + end + + pick_style = function (fontstyle_name, + prefmodifiers, + subfamily, + splitstyle) + local style + if fontstyle_name then + style = choose_exact (fontstyle_name) + end + if not style then + if prefmodifiers then + style = choose_exact (prefmodifiers) + elseif subfamily then + style = choose_exact (subfamily) + end + end +-- if not style and splitstyle then +-- style = choose_exact (splitstyle) +-- end + return style + end + + --- we use only exact matches here since there are constructs + --- like “regularitalic” (Cabin, Bodoni Old Fashion) + + check_regular = function (fontstyle_name, + prefmodifiers, + subfamily, + splitstyle) + + if fontstyle_name then + return regular_synonym [fontstyle_name] + elseif prefmodifiers then + return regular_synonym [prefmodifiers] + elseif subfamily then + return regular_synonym [subfamily] + elseif splitstyle then + return regular_synonym [splitstyle] + end + + return nil + end +end + +local pull_values = function (entry) + local file = entry.file + local names = entry.names + local style = entry.style + local sanitized = names.sanitized + local english = sanitized.english + local info = sanitized.info + local metadata = sanitized.metadata + + --- pull file info ... + entry.basename = file.base + entry.fullpath = file.full + entry.location = file.location + entry.subfont = file.subfont + + --- pull name info ... + entry.psname = english.psname + entry.fontname = info.fontname + entry.fullname = english.fullname or info.fullname + entry.prefmodifiers = english.prefmodifiers + local metafamily = metadata.familyname + local familyname = english.preffamily or english.family + entry.familyname = familyname + if familyname ~= metafamily then + entry.metafamily = metadata.familyname + end + entry.fontstyle_name = sanitized.fontstyle_name + entry.plainname = names.fullname + entry.subfamily = english.subfamily + + --- pull style info ... + entry.italicangle = style.italicangle + entry.size = style.size + entry.splitstyle = style.split + entry.weight = style.weight + + if luaotfloadconfig.strip == true then + entry.file = nil + entry.names = nil + entry.style = nil + end +end + +local add_family = function (name, subtable, modifier, entry) + local familytable = subtable [name] + if not familytable then + familytable = { } + subtable [name] = familytable + end + + --- the style table is treated as an unordered list + local styletable = familytable [modifier] + if not styletable then + styletable = { } + familytable [modifier] = styletable + end + + if not entry.prefmodifiers then --- default size for this style/family combo + styletable.default = entry.index + end + + local size = entry.size --- dsnsize * high * low + if size then + styletable [#styletable + 1] = { + size [1], + size [2], + size [3], + entry.index, + } + else + styletable.default = entry.index + end +end + +local collect_families = function (mappings) + + report ("info", 2, "db", "Analyzing families, sizes, and styles.") + + local families = { + ["local"] = { }, + system = { }, + texmf = { }, + } + + for i = 1, #mappings do + + local entry = mappings [i] + + if entry.file then + pull_values (entry) + end + + local location = entry.location + local format = entry.format + + local subtable = families [location] [format] + if not subtable then + subtable = { } + families [location] [format] = subtable + end + + local familyname = entry.familyname + local metafamily = entry.metafamily + local fontstyle_name = entry.fontstyle_name + local prefmodifiers = entry.prefmodifiers + local subfamily = entry.subfamily + + local weight = entry.weight + local italicangle = entry.italicangle + local splitstyle = entry.splitstyle + + local modifier = pick_style (fontstyle_name, + prefmodifiers, + subfamily, + splitstyle) + + if not modifier then --- regular, exact only + modifier = check_regular (fontstyle_name, + prefmodifiers, + subfamily, + splitstyle) + end + + if modifier then + add_family (familyname, subtable, modifier, entry) + --- registering the metafamilies is unreliable within the + --- same table as identifiers might interfere with an + --- unmarked style that lacks a metafamily, e.g. + --- + --- iwona condensed regular -> + --- family: iwonacond + --- metafamily: iwona + --- iwona regular -> + --- family: iwona + --- metafamily: ø + --- + --- Both would be registered as under the same family, + --- i.e. “iwona”, and depending on the loading order + --- the query “name:iwona” can resolve to the condensed + --- version instead of the actual unmarked one. The only + --- way around this would be to introduce a separate + --- table for metafamilies and do fallback queries on it. + --- At the moment this is not pressing enough to justify + --- further increasing the index size, maybe if need + --- arises from the user side. +-- if metafamily and metafamily ~= familyname then +-- add_family (metafamily, subtable, modifier, entry) +-- end + end + end + + collectgarbage "collect" + return families +end + +local cmp_sizes = function (a, b) + return a [1] < b [1] +end + +local order_design_sizes = function (families) + + report ("info", 2, "db", "Ordering design sizes.") + + for location, data in next, families do + for format, data in next, data do + for familyname, data in next, data do + for style, data in next, data do + tablesort (data, cmp_sizes) + end end + end + end + + return families +end + +local retrieve_namedata = function (currentnames, + targetnames, + dry_run, + n_rawnames, + n_newnames) + + local rawnames, new = scan_texmf_fonts (currentnames, + targetnames, + dry_run) + + n_rawnames = n_rawnames + rawnames + n_newnames = n_newnames + new + + rawnames, new = scan_os_fonts (currentnames, targetnames, dry_run) - filenames.bare[bare] = idx - filenames.base[base] = idx - if intexmf == true then - filenames.full[idx] = nil + n_rawnames = n_rawnames + rawnames + n_newnames = n_newnames + new + + return n_rawnames, n_newnames +end + + +--- dbobj -> stats + +local collect_statistics = function (mappings) + local sum_dsnsize, n_dsnsize = 0, 0 + + local fullname, family, families = { }, { }, { } + local subfamily, prefmodifiers, fontstyle_name = { }, { }, { } + + local addtohash = function (hash, item) + if item then + local times = hash [item] + if times then + hash [item] = times + 1 + else + hash [item] = 1 + end + end + end + + local appendtohash = function (hash, key, value) + if key and value then + local entry = hash [key] + if entry then + entry [#entry + 1] = value else - filenames.full[idx] = full + hash [key] = { value } end end end - if config.luaotfload.prioritize == "texmf" then - report("both", 2, "db", "Preferring texmf fonts") - addmap(sys) - addmap(texmf) - else --- sys - addmap(texmf) - addmap(sys) + local addtoset = function (hash, key, value) + if key and value then + local set = hash [key] + if set then + set [value] = true + else + hash [key] = { [value] = true } + end + end end - fontnames.filenames = filenames - texmf, sys = nil, nil - collectgarbage "collect" - return fontnames + local setsize = function (set) + local n = 0 + for _, _ in next, set do + n = n + 1 + end + return n + end + + local hashsum = function (hash) + local n = 0 + for _, m in next, hash do + n = n + m + end + return n + end + + for _, entry in next, mappings do + local style = entry.style + local names = entry.names.sanitized + local englishnames = names.english + + addtohash (fullname, englishnames.fullname) + addtohash (family, englishnames.family) + addtohash (subfamily, englishnames.subfamily) + addtohash (prefmodifiers, englishnames.prefmodifiers) + addtohash (fontstyle_name, names.fontstyle_name) + + addtoset (families, englishnames.family, englishnames.fullname) + + local sizeinfo = entry.style.size + if sizeinfo then + sum_dsnsize = sum_dsnsize + sizeinfo [1] + n_dsnsize = n_dsnsize + 1 + end + end + + --inspect (families) + + local n_fullname = setsize (fullname) + local n_family = setsize (family) + + if logs.get_loglevel () > 1 then + local pprint_top = function (hash, n, set) + + local freqs = { } + local items = { } + + for item, value in next, hash do + if set then + freq = setsize (value) + else + freq = value + end + local ifreq = items [freq] + if ifreq then + ifreq [#ifreq + 1] = item + else + items [freq] = { item } + freqs [#freqs + 1] = freq + end + end + + tablesort (freqs) + + local from = #freqs + local to = from - (n - 1) + if to < 1 then + to = 1 + end + + for i = from, to, -1 do + local freq = freqs [i] + local itemlist = items [freq] + + if type (itemlist) == "table" then + itemlist = tableconcat (itemlist, ", ") + end + + report ("both", 0, "db", + " · %4d × %s.", + freq, itemlist) + end + end + + report ("both", 0, "", "~~~~ font index statistics ~~~~") + report ("both", 0, "db", + " · Collected %d fonts (%d names) in %d families.", + #mappings, n_fullname, n_family) + pprint_top (families, 4, true) + + report ("both", 0, "db", + " · %d different “subfamily” kinds.", + setsize (subfamily)) + pprint_top (subfamily, 4) + + report ("both", 0, "db", + " · %d different “prefmodifiers” kinds.", + setsize (prefmodifiers)) + pprint_top (prefmodifiers, 4) + + report ("both", 0, "db", + " · %d different “fontstyle_name” kinds.", + setsize (fontstyle_name)) + pprint_top (fontstyle_name, 4) + end + + local mean_dsnsize = 0 + if n_dsnsize > 0 then + mean_dsnsize = sum_dsnsize / n_dsnsize + end + + return { + mean_dsnsize = mean_dsnsize, + names = { + fullname = n_fullname, + families = n_family, + }, +-- style = { +-- subfamily = subfamily, +-- prefmodifiers = prefmodifiers, +-- fontstyle_name = fontstyle_name, +-- }, + } end --- force: dictate rebuild from scratch --- dry_dun: don’t write to the db, just scan dirs --- dbobj? -> bool? -> bool? -> dbobj -update_names = function (fontnames, force, dry_run) - if config.luaotfload.update_live == false then - report("info", 2, "db", - "skipping database update") +update_names = function (currentnames, force, dry_run) + + local targetnames + + if luaotfloadconfig.update_live == false then + report ("info", 2, "db", + "Skipping database update.") --- skip all db updates - return fontnames or names.data + return currentnames or name_index end - local starttime = os.gettimeofday() - local n_scanned, n_new = 0, 0 + + local starttime = osgettimeofday () + local n_rawnames, n_newnames = 0, 0 + --[[ The main function, scans everything - - “newfontnames” is the final table to return + - “targetnames” is the final table to return - force is whether we rebuild it from scratch or not ]] - report("both", 2, "db", "Updating the font names database" - .. (force and " forcefully" or "")) - - if force then - fontnames = fontnames_init(false) + report("both", 1, "db", "Updating the font names database" + .. (force and " forcefully." or ".")) + + --- pass 1 get raw data: read font files (normal case) or reuse + --- information present in index + + if luaotfloadconfig.skip_read == true then + --- the difference to a “dry run” is that we don’t search + --- for font files entirely. we also ignore the “force” + --- parameter since it concerns only the font files. + report ("info", 2, "db", + "Ignoring font files, reusing old data.") + currentnames = load_names (false) + targetnames = currentnames else - if not fontnames then - fontnames = load_names(dry_run) - end - if fontnames.version ~= names.version then - report("both", 1, "db", "No font names database or old " - .. "one found; generating new one") - fontnames = fontnames_init(true) + if force then + currentnames = initialize_namedata (get_font_filter ()) + else + if not currentnames then + currentnames = load_names (dry_run) + end + if currentnames.meta.version ~= names.version then + report ("both", 1, "db", "No font names database or old " + .. "one found; generating new one.") + currentnames = initialize_namedata (get_font_filter ()) + end end - end - local newfontnames = fontnames_init(true) - read_blacklist() - local scanned, new - scanned, new = scan_texmf_fonts(fontnames, newfontnames, dry_run) - n_scanned = n_scanned + scanned - n_new = n_new + new + targetnames = initialize_namedata (get_font_filter ()) - scanned, new = scan_os_fonts(fontnames, newfontnames, dry_run) - n_scanned = n_scanned + scanned - n_new = n_new + new + read_blacklist () + + local n_raw, n_new= retrieve_namedata (currentnames, + targetnames, + dry_run, + n_rawnames, + n_newnames) + report ("info", 3, "db", + "Scanned %d font files; %d new entries.", + n_rawnames, n_newnames) + end + + --- pass 2 (optional): collect some stats about the raw font info + if luaotfloadconfig.statistics == true then + targetnames.meta.statistics = collect_statistics + (targetnames.mappings) + end --- we always generate the file lookup tables because --- non-texmf entries are redirected there and the mapping --- needs to be 100% consistent - newfontnames = gen_fast_lookups(newfontnames) - --- stats: - --- before rewrite | after rewrite - --- partial: 804 ms | 701 ms - --- forced: 45384 ms | 44714 ms - report("info", 3, "db", - "Scanned %d font files; %d new entries.", n_scanned, n_new) - report("info", 3, "db", - "Rebuilt in %0.f ms", 1000*(os.gettimeofday()-starttime)) - return newfontnames -end + --- pass 3: build filename table + targetnames.files = generate_filedata (targetnames.mappings) + + --- pass 4: build family lookup table + targetnames.families = collect_families (targetnames.mappings) + + --- pass 5: order design size tables + targetnames.families = order_design_sizes (targetnames.families) ---- The lookup cache is an experimental feature of version 2.2; ---- instead of incorporating it into the database it gets its own ---- file. As we update it after every single addition this saves us ---- quite some time. + + report ("info", 3, "db", + "Rebuilt in %0.f ms.", + 1000 * (osgettimeofday () - starttime)) + name_index = targetnames + + if dry_run ~= true then + + save_names () + + local success, _lookups = flush_lookup_cache () + if success then + local success = save_lookups () + if success then + logs.names_report ("info", 2, "cache", + "Lookup cache emptied.") + return targetnames + end + end + end + return targetnames +end --- unit -> bool save_lookups = function ( ) - local lookups = names.lookups local path = names.path.lookups local luaname, lucname = path.lua, path.luc if fileiswritable (luaname) and fileiswritable (lucname) then - tabletofile (luaname, lookups, true) + tabletofile (luaname, lookup_cache, true) osremove (lucname) - caches.compile (lookups, luaname, lucname) + caches.compile (lookup_cache, luaname, lucname) --- double check ... if lfsisfile (luaname) and lfsisfile (lucname) then - report ("both", 3, "cache", "Lookup cache saved") + report ("both", 3, "cache", "Lookup cache saved.") return true end - report ("info", 0, "cache", "Could not compile lookup cache") + report ("info", 0, "cache", "Could not compile lookup cache.") return false end - report ("info", 0, "cache", "Lookup cache file not writable") + report ("info", 0, "cache", "Lookup cache file not writable.") if not fileiswritable (luaname) then - report ("info", 0, "cache", "Failed to write %s", luaname) + report ("info", 0, "cache", "Failed to write %s.", luaname) end if not fileiswritable (lucname) then - report ("info", 0, "cache", "Failed to write %s", lucname) + report ("info", 0, "cache", "Failed to write %s.", lucname) end return false end --- save_names() is usually called without the argument --- dbobj? -> bool -save_names = function (fontnames) - if not fontnames then fontnames = names.data end +save_names = function (currentnames) + if not currentnames then + currentnames = name_index + end local path = names.path.index local luaname, lucname = path.lua, path.luc if fileiswritable (luaname) and fileiswritable (lucname) then - tabletofile (luaname, fontnames, true) osremove (lucname) - caches.compile (fontnames, luaname, lucname) - if lfsisfile (luaname) and lfsisfile (lucname) then - report ("info", 1, "db", "Font index saved") - report ("info", 3, "db", "Text: " .. luaname) - report ("info", 3, "db", "Byte: " .. lucname) + local gzname = luaname .. ".gz" + if luaotfloadconfig.compress then + local serialized = tableserialize (currentnames, true) + save_gzipped (gzname, serialized) + caches.compile (currentnames, "", lucname) + else + tabletofile (luaname, currentnames, true) + caches.compile (currentnames, luaname, lucname) + end + report ("info", 2, "db", "Font index saved at ...") + local success = false + if lfsisfile (luaname) then + report ("info", 2, "db", "Text: " .. luaname) + success = true + end + if lfsisfile (gzname) then + report ("info", 2, "db", "Gzip: " .. gzname) + success = true + end + if lfsisfile (lucname) then + report ("info", 2, "db", "Byte: " .. lucname) + success = true + end + if success then return true + else + report ("info", 0, "db", "Could not compile font index.") + return false end - report ("info", 0, "db", "Could not compile font index") - return false end report ("info", 0, "db", "Index file not writable") if not fileiswritable (luaname) then - report ("info", 0, "db", "Failed to write %s", luaname) + report ("info", 0, "db", "Failed to write %s.", luaname) end if not fileiswritable (lucname) then - report ("info", 0, "db", "Failed to write %s", lucname) + report ("info", 0, "db", "Failed to write %s.", lucname) end return false end @@ -2031,24 +3315,24 @@ end --- string -> string -> string list -> bool -> bool local purge_from_cache = function (category, path, list, all) - report("info", 2, "cache", "Luaotfload cache: %s %s", + report("info", 1, "cache", "Luaotfload cache: %s %s", (all and "erase" or "purge"), category) - report("info", 2, "cache", "location: %s",path) + report("info", 1, "cache", "location: %s",path) local n = 0 for i=1,#list do local filename = list[i] if stringfind(filename,"luatex%-cache") then -- safeguard if all then - report("info", 5, "cache", "removing %s", filename) + report("info", 5, "cache", "Removing %s.", filename) osremove(filename) n = n + 1 else - local suffix = file.suffix(filename) + local suffix = filesuffix(filename) if suffix == "lua" then local checkname = file.replacesuffix( filename, "lua", "luc") - if lfs.isfile(checkname) then - report("info", 5, "cache", "Removing %s", filename) + if lfsisfile(checkname) then + report("info", 5, "cache", "Removing %s.", filename) osremove(filename) n = n + 1 end @@ -2056,7 +3340,7 @@ local purge_from_cache = function (category, path, list, all) end end end - report("info", 2, "cache", "Removed lua files : %i", n) + report("info", 1, "cache", "Removed lua files : %i", n) return true end @@ -2073,7 +3357,7 @@ local collect_cache collect_cache = function (path, all, n, luanames, local filename = all[n] if filename then - local suffix = file.suffix(filename) + local suffix = filesuffix(filename) if suffix == "lua" then luanames[#luanames+1] = filename elseif suffix == "luc" then @@ -2089,7 +3373,7 @@ end local getwritablecachepath = function ( ) --- fonts.handlers.otf doesn’t exist outside a Luatex run, --- so we have to improvise - local writable = getwritablepath (config.luaotfload.cache_dir) + local writable = getwritablepath (luaotfloadconfig.cache_dir) if writable then return writable end @@ -2097,7 +3381,7 @@ end local getreadablecachepaths = function ( ) local readables = caches.getreadablepaths - (config.luaotfload.cache_dir) + (luaotfloadconfig.cache_dir) local result = { } if readables then for i=1, #readables do @@ -2145,7 +3429,7 @@ local show_cache = function ( ) separator () print_cache ("writable path", writable_path, luanames, lucnames, rest) - texiowrite_nl"" + texio.write_nl"" for i=1,#readable_paths do local readable_path = readable_paths[i] if readable_path ~= writable_path then @@ -2163,16 +3447,19 @@ end ----------------------------------------------------------------------- names.scan_dir = scan_dir +names.set_font_filter = set_font_filter names.flush_lookup_cache = flush_lookup_cache names.save_lookups = save_lookups names.load = load_names +names.data = function () return name_index end names.save = save_names names.update = update_names names.crude_file_lookup = crude_file_lookup names.crude_file_lookup_verbose = crude_file_lookup_verbose names.read_blacklist = read_blacklist -names.sanitize_string = sanitize_string +names.sanitize_fontname = sanitize_fontname names.getfilename = resolve_fullpath +names.set_location_precedence = set_location_precedence --- font cache names.purge_cache = purge_cache @@ -2180,15 +3467,16 @@ names.erase_cache = erase_cache names.show_cache = show_cache --- replace the resolver from luatex-fonts -if config.luaotfload.resolver == "cached" then - report("both", 2, "cache", "caching of name: lookups active") - names.resolve = resolve_cached - names.resolvespec = resolve_cached +if luaotfloadconfig.resolver == "cached" then + report("both", 2, "cache", "Caching of name: lookups active.") + names.resolvespec = resolve_cached + names.resolve_name = resolve_cached else - names.resolve = resolve - names.resolvespec = resolve + names.resolvespec = resolve_name + names.resolve_name = resolve_name end -names.find_closest = find_closest + +names.find_closest = find_closest -- for testing purpose names.read_fonts_conf = read_fonts_conf |