From 3a7fd63f467d90781d764fbded8b377343d25522 Mon Sep 17 00:00:00 2001 From: Karl Berry Date: Fri, 5 Jul 2013 23:54:33 +0000 Subject: luaotfload (4jul13) git-svn-id: svn://tug.org/texlive/trunk@31106 c570f23f-e606-0410-a88d-b1316a301751 --- .../scripts/luaotfload/luaotfload-tool.lua | 540 ++++++++++++++++++--- Master/texmf-dist/scripts/luaotfload/mkcharacters | 7 +- Master/texmf-dist/scripts/luaotfload/mkglyphlist | 2 +- 3 files changed, 481 insertions(+), 68 deletions(-) (limited to 'Master/texmf-dist/scripts/luaotfload') diff --git a/Master/texmf-dist/scripts/luaotfload/luaotfload-tool.lua b/Master/texmf-dist/scripts/luaotfload/luaotfload-tool.lua index 721e3cff175..a353b377930 100755 --- a/Master/texmf-dist/scripts/luaotfload/luaotfload-tool.lua +++ b/Master/texmf-dist/scripts/luaotfload/luaotfload-tool.lua @@ -4,9 +4,9 @@ -- DESCRIPTION: database functionality -- REQUIREMENTS: luaotfload 2.2 -- AUTHOR: Khaled Hosny, Élie Roux, Philipp Gesang --- VERSION: 2.2 +-- VERSION: 2.3 -- LICENSE: GPL v2 --- CREATED: 2013-05-06 13:37:12+0200 +-- MODIFIED: 2013-06-02 19:23:54+0200 ----------------------------------------------------------------------- --[[doc-- @@ -43,11 +43,15 @@ if _G.getfenv then return require(oldscript) end +local stringexplode = string.explode local stringformat = string.format -local texiowrite_nl = texio.write_nl local stringlower = string.lower +local stringrep = string.rep +local tableconcat = table.concat +local texiowrite_nl = texio.write_nl +local texiowrite = texio.write -local C, Ct, P = lpeg.C, lpeg.Ct, lpeg.P +local C, Ct, P, S = lpeg.C, lpeg.Ct, lpeg.P, lpeg.S local lpegmatch = lpeg.match local loader_file = "luatexbase.loader.lua" @@ -76,9 +80,13 @@ After support for querying the database was added, the latter appeared to be the more appropriate. --doc]]-- -config = config or { } -local config = config -config.luaotfload = config.luaotfload or { } +config = config or { } +local config = config +config.luaotfload = config.luaotfload or { } +config.luaotfload.names_dir = config.luaotfload.names_dir or "names" +config.luaotfload.cache_dir = config.luaotfload.cache_dir or "fonts" +config.luaotfload.names_file = config.luaotfload.names_file + or "luaotfload-names.lua" do -- we don’t have file.basename and the likes yet, so inline parser ftw local slash = P"/" @@ -103,20 +111,7 @@ config.lualibs.verbose = false config.lualibs.prefer_merged = true config.lualibs.load_extended = false -require"lualibs" - ---- prepare directories: the cache function in Luatex-Fonts ---- checks for writable directory only on startup, so everything ---- has to be laid out before we load basics-gen - -local cachepath = kpse.expand_var"$TEXMFVAR" -if not lfs.isdir(cachepath) then - dir.mkdirs(cachepath) - if not lfs.isdir(cachepath) then - texiowrite(stringformat( - "ERROR could not create directory %s", cachepath)) - end -end +require "lualibs" --[[doc-- \fileent{luatex-basics-gen.lua} calls functions from the @@ -135,11 +130,16 @@ require"luaotfload-override.lua" --- this populates the logs.* namespace require"luaotfload-database" require"alt_getopt" -local version = "2.2" -- same version number as luaotfload +local version = "2.3" -- same version number as luaotfload local names = fonts.names -local db_src_out = names.path.dir.."/"..names.path.basename -local db_bin_out = file.replacesuffix(db_src_out, "luc") +local sanitize_string = names.sanitize_string + +--local db_src_out = names.path.dir.."/"..names.path.basename +local names_plain = file.join + (caches.getwritablepath (config.luaotfload.names_dir), + config.luaotfload.names_file) +local names_bin = file.replacesuffix (names_plain, "luc") local help_messages = { ["luaotfload-tool"] = [[ @@ -172,16 +172,22 @@ This tool is part of the luaotfload package. Valid options are: -f --force force re-indexing all fonts -l --flush-lookups empty lookup cache of font requests -D --dry-run skip loading of fonts, just scan + -p --prefer-texmf prefer fonts in the TEXMF over system fonts --find="font name" query the database for a font name -F --fuzzy look for approximate matches if --find fails --limit=n limit display of fuzzy matches to (default: n = 1) - -i --info display font metadata + + -i --info display basic font metadata + -I --inspect display detailed font metadata + -w --warnings display warnings generated by the + fontloader library --list= output list of entries by field --list=: restrict to entries with = --fields=,,…, which fields to print with --list + -b --show-blacklist show blacklisted files The font database will be saved to %s @@ -193,6 +199,9 @@ The font database will be saved to --cache= operate on font cache, where is “show”, “purge”, or “erase” +The font cache will be written to + %s + ]], mkluatexfontdb = [[ @@ -221,7 +230,12 @@ The font database will be saved to local help_msg = function ( ) local template = help_messages[config.luaotfload.self] or help_messages["luaotfload-tool"] - texiowrite_nl(stringformat(template, config.luaotfload.self, db_src_out, db_bin_out)) + texiowrite_nl(stringformat(template, + config.luaotfload.self, + names_plain, + names_bin, + caches.getwritablepath + (config.luaotfload.cache_dir))) end local version_msg = function ( ) @@ -230,34 +244,386 @@ local version_msg = function ( ) config.luaotfload.self, version, names.version)) end + +--- makeshift formatting + +local head_adornchars = { + [1] = "*", [2] = "=", [3] = "~", [4] = "-", [5] = "·", +} + +local textwidth = 80 +local wd_leftcolumn = math.floor(textwidth * .25) +local key_fmt = stringformat([[%%%ds]], wd_leftcolumn) +local val_fmt = [[%s]] +local fieldseparator = ":" +local info_fmt = key_fmt .. fieldseparator .. " " .. val_fmt + +local currentdepth = 0 +local counterstack = { } -- counters per level +local counterformat = "%d" + +local format_counter = function (stack) + local acc = { } + for lvl=1, #stack do + acc[#acc+1] = stringformat(counterformat, stack[lvl]) + end + return tableconcat(acc, ".") +end + +local print_heading = function (title, level) + local structuredata + if currentdepth == level then -- top is current + counterstack[#counterstack] = counterstack[#counterstack] + 1 + elseif currentdepth < level then -- push new + counterstack[#counterstack+1] = 1 + else -- pop + local diff = currentdepth - level + while diff > 0 do + counterstack[#counterstack] = nil + diff = diff - 1 + end + counterstack[#counterstack] = counterstack[#counterstack] + 1 + end + currentdepth = level + + texiowrite_nl "" + if not level or level > #head_adornchars then + level = #head_adornchars + end + local adornchar = head_adornchars[level] + + local counter = format_counter(counterstack) + + local s = adornchar .. adornchar .. " " + .. counter .. " " + .. title .. " " + texiowrite_nl (s .. stringrep(adornchar, textwidth-utf.len(s))) +end + local show_info_items = function (fontinfo) - local items = table.sortedkeys(fontinfo) + local items = table.sortedkeys(fontinfo) + print_heading(fontinfo.fullname, 1) + texiowrite_nl "" for n = 1, #items do local item = items[n] texiowrite_nl(stringformat( - [[ %11s: %s]], item, fontinfo[item])) + info_fmt, item, fontinfo[item])) + end + texiowrite_nl "" +end + +local p_eol = S"\n\r"^1 +local p_space = S" \t\v"^0 +local p_line = p_space * C((1 - p_eol)^1)^-1 +local p_lines = Ct(p_line * (p_eol^1 * p_line^-1)^0) + +local show_fontloader_warnings = function (ws) + local nws = #ws + print_heading(stringformat( + [[the fontloader emitted %d warnings]], + nws), 2) + texiowrite_nl "" + for i=1, nws do + local w = ws[i] + texiowrite_nl (stringformat("%d:", i)) + local lines = lpegmatch(p_lines, w) + for i=1, #lines do + local line = lines[i] + texiowrite_nl(" · " .. line) + end + texiowrite_nl "" + end +end + +local p_spacechar = S" \n\r\t\v" +local p_wordchar = (1 - p_spacechar) +local p_whitespace = p_spacechar^1 +local p_word = C(p_wordchar^1) +local p_words = Ct(p_word * (p_whitespace * p_word)^0) + +--- string -> int -> string list +local reflow = function (text, width) + local words + if type(text) == "string" then + words = lpegmatch(p_words, text) + if #words < 2 then + return { text } + end + else + words = text + if #words < 2 then + return words + end + end + + local space = " " + local utflen = utf.len + local reflowed = { } + + local first = words[1] + local linelen = #first + local line = { first } + + for i=2, #words do + local word = words[i] + local lword = utflen(word) + linelen = linelen + lword + 1 + if linelen > width then + reflowed[#reflowed+1] = tableconcat(line) + linelen = #word + line = { word } + else + line[#line+1] = space + line[#line+1] = word + end + end + reflowed[#reflowed+1] = tableconcat(line) + return reflowed +end + +--- string -> 'a -> string list +local print_field = function (key, val) + val = tostring(val) + local lhs = stringformat(key_fmt, key) .. fieldseparator .. " " + local wd_lhs = #lhs + local lines = reflow(val, textwidth - wd_lhs) + + texiowrite_nl(lhs) + texiowrite(lines[1]) + if #lines > 1 then + local indent = stringrep(" ", wd_lhs) + for i=2, #lines do + texiowrite_nl(indent) + texiowrite (lines[i]) + end + end +end + +local display_names = function (names) + print_heading("Font Metadata", 2) + for i=1, #names do + local lang, namedata = names[i].lang, names[i].names + print_heading(stringformat("Language: %s ", i, lang), 3) + texiowrite_nl "" + if namedata then + for field, value in next, namedata do + print_field(field, value) + end + end + end +end + +--- see luafflib.c +local general_fields = { + --- second: l -> literal | n -> length | d -> date + { "fullname", "l", "font name" }, + { "version", "l", "font version" }, + { "creationtime", "d", "creation time" }, + { "modificationtime", "d", "modification time" }, + { "subfonts", "n", "number of subfonts" }, + { "glyphcnt", "l", "number of glyphs" }, + { "weight", "l", "weight indicator" }, + { "design_size", "l", "design size" }, + { "design_range_bottom", "l", "design size min" }, + { "design_range_top", "l", "design size max" }, + { "fontstyle_id", "l", "font style id" }, + { "fontstyle_name", "l", "font style name" }, + { "strokewidth", "l", "stroke width" }, + { "units_per_em", "l", "units per em" }, + { "ascent", "l", "ascender height" }, + { "descent", "l", "descender height" }, + { "comments", "l", "comments" }, + { "os2_version", "l", "os2 version" }, + { "sfd_version", "l", "sfd version" }, +} + +local display_general = function (fullinfo) + texiowrite_nl "" + print_heading("General Information", 2) + texiowrite_nl "" + for i=1, #general_fields do + local field = general_fields[i] + local key, mode, desc = unpack(field) + local val + if mode == "l" then + val = fullinfo[key] + elseif mode == "n" then + local v = fullinfo[key] + if v then + val = #fullinfo[key] + end + elseif mode == "d" then + val = os.date("%F %T", fullinfo[key]) + end + if not val then + val = "" + end + print_field(desc, val) + end +end + +local print_features = function (features) + for tag, data in next, features do + print_heading(tag, 4) + for script, languages in next, data do + local field = stringformat(key_fmt, script).. fieldseparator .. " " + local wd_field = #field + --inspect(languages.list) + local lines = reflow(languages.list, textwidth - wd_field) + local indent = stringrep(" ", wd_field) + texiowrite_nl(field) + texiowrite(lines[1]) + if #lines > 1 then + for i=1, #lines do + texiowrite_nl(indent .. lines[i]) + end + end + end + end +end + +local extract_feature_info = function (set) + local collected = { } + for i=1, #set do + local features = set[i].features + if features then + for j=1, #features do + local feature = features[j] + local scripts = feature.scripts + local tagname = stringlower(feature.tag) + local entry = collected[tagname] or { } + + for k=1, #scripts do + local script = scripts[k] + local scriptname = stringlower(script.script) + local c_script = entry[scriptname] or { + list = { }, + set = { }, + } + local list, set = c_script.list, c_script.set + + for l=1, #script.langs do + local langname = stringlower(script.langs[l]) + if not set[langname] then + list[#list+1] = langname + set[langname] = true + end + end + entry[scriptname] = c_script + end + collected[tagname] = entry + end + end end + return collected end -local show_font_info = function (filename) - local fullname = resolvers.findfile(filename) +local display_feature_set = function (set) + local collected = extract_feature_info(set) + print_features(collected) +end + +local display_features = function (gsub, gpos) + texiowrite_nl "" + print_heading("Features", 2) + print_heading("GSUB Features", 3) + display_feature_set(gsub) + print_heading("GPOS Features", 3) + display_feature_set(gpos) +end + +local show_full_info = function (path, subfont, warnings) + local rawinfo, warn = fontloader.open(path, subfont) + if warnings then + show_fontloader_warnings(warn) + end + if not rawinfo then + texiowrite_nl(stringformat([[cannot open font %s]], path)) + return + end + local fontdata = { } + local fullinfo = fontloader.to_table(rawinfo) + local fields = fontloader.fields(rawinfo) + fontloader.close(rawinfo) + display_names(fullinfo.names) + display_general(fullinfo) + display_features(fullinfo.gsub, fullinfo.gpos) +end + +--- Subfonts returned by fontloader.info() do not correspond +--- to the actual indices required by fontloader.open(), so +--- we try and locate the correct one by matching the request +--- against the full name. + +local subfont_by_name +subfont_by_name = function (lst, askedname, n) + if not n then + return subfont_by_name (lst, askedname, 1) + end + + local font = lst[n] + if font then + if sanitize_string(font.fullname) == askedname then + return font + end + return subfont_by_name (lst, askedname, n+1) + end + return false +end + +--[[doc-- +The font info knows two levels of detail: + + a) basic information returned by fontloader.info(); and + b) detailed information that is a subset of the font table + returned by fontloader.open(). +--doc]]-- + +local show_font_info = function (basename, askedname, detail, warnings) + local filenames = names.data.filenames + local index = filenames.base[basename] + local fullname = filenames.full[index] + askedname = sanitize_string(askedname) + if not fullname then -- texmf + fullname = resolvers.findfile(basename) + end if fullname then - local fontinfo = fontloader.info(fullname) - local nfonts = #fontinfo + local shortinfo = fontloader.info(fullname) + local nfonts = #shortinfo if nfonts > 0 then -- true type collection - logs.names_report(true, 1, "resolve", - [[%s is a font collection]], filename) - for n = 1, nfonts do + local subfont + if askedname then + logs.names_report(true, 1, "resolve", + [[%s is part of the font collection %s]], + askedname, basename) + subfont = subfont_by_name(shortinfo, askedname) + end + if subfont then + show_info_items(subfont) + if detail == true then + show_full_info(fullname, subfont, warnings) + end + else -- list all subfonts logs.names_report(true, 1, "resolve", - [[showing info for font no. %d]], n) - show_info_items(fontinfo[n]) + [[%s is a font collection]], basename) + for subfont = 1, nfonts do + logs.names_report(true, 1, "resolve", + [[Showing info for font no. %d]], n) + show_info_items(shortinfo[subfont]) + if detail == true then + show_full_info(fullname, subfont, warnings) + end + end end else - show_info_items(fontinfo) + show_info_items(shortinfo) + if detail == true then + show_full_info(fullname, subfont, warnings) + end end else logs.names_report(true, 1, "resolve", - "font %s not found", filename) + "Font %s not found", filename) end end @@ -268,7 +634,7 @@ set. --]]-- local action_sequence = { - "loglevel", "help", "version", "cache", + "loglevel", "help", "version", "blacklist", "cache", "flush", "generate", "list", "query", } local action_pending = table.tohash(action_sequence, false) @@ -281,8 +647,8 @@ local actions = { } --- (jobspec -> (bool * bool)) list actions.loglevel = function (job) logs.set_loglevel(job.log_level) logs.names_report("info", 3, "util", - "setting log level", "%d", job.log_level) - logs.names_report("log", 0, "util", "lua=%s", _VERSION) + "Setting log level", "%d", job.log_level) + logs.names_report("log", 0, "util", "Lua=%s", _VERSION) return true, true end @@ -296,6 +662,15 @@ actions.help = function (job) return true, false end +actions.blacklist = function (job) + names.read_blacklist() + local n = 0 + for n, entry in next, table.sortedkeys(fonts.names.blacklist) do + texiowrite_nl(stringformat("(%d %s)", n, entry)) + end + return true, false +end + actions.generate = function (job) local fontnames, savedname fontnames = names.update(fontnames, job.force_reload, job.dry_run) @@ -353,24 +728,24 @@ actions.query = function (job) fonts.names.resolve(nil, nil, tmpspec) if success then - logs.names_report(false, 1, + logs.names_report(false, 0, "resolve", "Font “%s” found!", query) if subfont then - logs.names_report(false, 1, "resolve", + logs.names_report(false, 0, "resolve", "Resolved file name “%s”, subfont nr. “%s”", foundname, subfont) else - logs.names_report(false, 1, - "resolve", "Resolved file name “%s”", foundname) + logs.names_report(false, 0, "resolve", + "Resolved file name “%s”", foundname) end if job.show_info then - show_font_info(foundname) + show_font_info(foundname, query, job.full_info, job.warnings) end else - logs.names_report(false, 1, + logs.names_report(false, 0, "resolve", "Cannot find “%s”.", query) if job.fuzzy == true then - logs.names_report(false, 1, + logs.names_report(false, 0, "resolve", "Looking for close matches, this may take a while ...") local success = fonts.names.find_closest(query, job.fuzzy_limit) end @@ -401,10 +776,6 @@ local comma = P"," local noncomma = 1-comma local split_comma = Ct((C(noncomma^1) + comma)^1) -local texiowrite_nl = texio.write_nl -local tableconcat = table.concat -local stringexplode = string.explode - local separator = "\t" --- could be “,” for csv local format_fields format_fields = function (fields, acc, n) @@ -459,7 +830,7 @@ actions.list = function (job) local nmappings = #mappings if criterion == "*" then - logs.names_report(false, 1, "list", "all %d entries", nmappings) + logs.names_report(false, 1, "list", "All %d entries", nmappings) for i=1, nmappings do local entry = mappings[i] local fields = get_fields(entry, asked_fields) @@ -474,12 +845,12 @@ actions.list = function (job) criterion = criterion[1] asked_fields = set_primary_field(asked_fields, criterion) - logs.names_report(false, 1, "list", "by %s", criterion) + logs.names_report(false, 1, "list", "By %s", criterion) --- firstly, build a list of fonts to operate on local targets = { } if asked_value then --- only those whose value matches - logs.names_report(false, 2, "list", "restricting to value %s", asked_value) + logs.names_report(false, 2, "list", "Restricting to value %s", asked_value) for i=1, nmappings do local entry = mappings[i] if entry[criterion] @@ -530,6 +901,22 @@ actions.list = function (job) return true, true end +--- stuff to be carried out prior to exit + +local finalizers = { } + +--- returns false if at least one of the actions failed, mainly +--- for closing io channels +local finalize = function () + local success = true + for _, fun in next, finalizers do + if type (fun) == "function" then + if fun () == false then success = false end + end + end + return success +end + --[[-- Command-line processing. mkluatexfontdb.lua relies on the script alt_getopt to process argv and @@ -543,9 +930,11 @@ alt_getopt. local process_cmdline = function ( ) -- unit -> jobspec local result = { -- jobspec force_reload = nil, + full_info = false, + warnings = false, criterion = "", query = "", - log_level = 1, --- 2 is approx. the old behavior + log_level = 0, --- 2 is approx. the old behavior } local long_options = { @@ -559,16 +948,20 @@ local process_cmdline = function ( ) -- unit -> jobspec fuzzy = "F", help = "h", info = "i", + inspect = "I", limit = 1, list = 1, log = 1, + ["prefer-texmf"] = "p", quiet = "q", + ["show-blacklist"] = "b", update = "u", verbose = 1 , version = "V", + warnings = "w", } - local short_options = "DfFilquvVh" + local short_options = "bDfFiIlpquvVhw" local options, _, optarg = alt_getopt.get_ordered_opts (arg, short_options, long_options) @@ -596,12 +989,18 @@ local process_cmdline = function ( ) -- unit -> jobspec elseif v == "verbose" then local lvl = optarg[n] if lvl then - result.log_level = tonumber(lvl) + lvl = tonumber(lvl) + result.log_level = lvl + if lvl > 2 then + result.warnings = true + end end + elseif v == "w" then + result.warnings = true elseif v == "log" then local str = optarg[n] if str then - logs.set_logout(str) + finalizers = logs.set_logout(str, finalizers) end elseif v == "find" then action_pending["query"] = true @@ -615,6 +1014,9 @@ local process_cmdline = function ( ) -- unit -> jobspec end elseif v == "i" then result.show_info = true + elseif v == "I" then + result.show_info = true + result.full_info = true elseif v == "alias" then config.luaotfload.self = optarg[n] elseif v == "l" then @@ -629,6 +1031,10 @@ local process_cmdline = function ( ) -- unit -> jobspec result.cache = optarg[n] elseif v == "D" then result.dry_run = true + elseif v == "p" then + config.luaotfload.prioritize = "texmf" + elseif v == "b" then + action_pending["blacklist"] = true end end @@ -651,7 +1057,7 @@ local main = function ( ) -- unit -> int local actionname = action_sequence[i] local exit = false if action_pending[actionname] then - logs.names_report("log", 3, "util", "preparing for task", + logs.names_report("log", 3, "util", "Preparing for task", "%s", actionname) local action = actions[actionname] @@ -659,22 +1065,26 @@ local main = function ( ) -- unit -> int if not success then logs.names_report(false, 0, "util", - "could not finish task", "%s", actionname) + "Could not finish task", "%s", actionname) retval = -1 exit = true elseif not continue then logs.names_report(false, 3, "util", - "task completed, exiting", "%s", actionname) + "Task completed, exiting", "%s", actionname) exit = true else logs.names_report(false, 3, "util", - "task completed successfully", "%s", actionname) + "Task completed successfully", "%s", actionname) end end if exit then break end end - texiowrite_nl"" + if finalize () == false then + retval = -1 + end + + --texiowrite_nl"" return retval end diff --git a/Master/texmf-dist/scripts/luaotfload/mkcharacters b/Master/texmf-dist/scripts/luaotfload/mkcharacters index a627ea9049c..a1c420481c3 100755 --- a/Master/texmf-dist/scripts/luaotfload/mkcharacters +++ b/Master/texmf-dist/scripts/luaotfload/mkcharacters @@ -5,7 +5,7 @@ -- DESCRIPTION: import parts of char-def.lua -- REQUIREMENTS: lua, ConTeXt, the lualibs package -- AUTHOR: Philipp Gesang (Phg), --- VERSION: 1.0 +-- VERSION: 1.1 -- CREATED: 2013-05-17 12:41:39+0200 ----------------------------------------------------------------------- -- we create a stripped-down version of char-def.lua @@ -16,7 +16,10 @@ ----------------------------------------------------------------------- local charfile = "./luaotfload-characters.lua" local chardef = "~phg/base/char-def.lua" -local import = { "direction", "mirror", } --> πολυγλωσσία/uax9 +local import = { + "direction", "mirror", --> πολυγλωσσία/uax9 + "category", --> https://gist.github.com/phi-gamma/5812290 +} ----------------------------------------------------------------------- -- includes diff --git a/Master/texmf-dist/scripts/luaotfload/mkglyphlist b/Master/texmf-dist/scripts/luaotfload/mkglyphlist index 94aac39d7a1..8b89935dc4e 100755 --- a/Master/texmf-dist/scripts/luaotfload/mkglyphlist +++ b/Master/texmf-dist/scripts/luaotfload/mkglyphlist @@ -96,7 +96,7 @@ end local file_header = [==[ if not modules then modules = { } end modules ["font-age"] = { - version = 2.200, + version = 2.300, comment = "part of the luaotfload package", author = "luaotfload team / mkglyphlist", copyright = "derived from %s", -- cgit v1.2.3