diff options
author | Norbert Preining <norbert@preining.info> | 2019-05-30 11:46:25 +0900 |
---|---|---|
committer | Norbert Preining <norbert@preining.info> | 2019-05-30 11:46:25 +0900 |
commit | 116e75910dd7de0682f578cae0194588286f5079 (patch) | |
tree | 19cc108cdc264f90273b85480c06b8fbaa2e5687 /texmf-dist/tex/generic | |
parent | bed92c2e7b43679063fc2053728bb4a35b3b3b3d (diff) |
add harftex
Diffstat (limited to 'texmf-dist/tex/generic')
-rw-r--r-- | texmf-dist/tex/generic/harftex/harf-base.lua | 12 | ||||
-rw-r--r-- | texmf-dist/tex/generic/harftex/harf-load.lua | 309 | ||||
-rw-r--r-- | texmf-dist/tex/generic/harftex/harf-luaotfload.lua | 232 | ||||
-rw-r--r-- | texmf-dist/tex/generic/harftex/harf-node.lua | 914 | ||||
-rw-r--r-- | texmf-dist/tex/generic/harftex/harf-plain.lua | 5 | ||||
-rw-r--r-- | texmf-dist/tex/generic/harftex/harf.lua | 16 | ||||
-rw-r--r-- | texmf-dist/tex/generic/harftex/harfload.sty | 14 |
7 files changed, 1502 insertions, 0 deletions
diff --git a/texmf-dist/tex/generic/harftex/harf-base.lua b/texmf-dist/tex/generic/harftex/harf-base.lua new file mode 100644 index 00000000..4b5931fd --- /dev/null +++ b/texmf-dist/tex/generic/harftex/harf-base.lua @@ -0,0 +1,12 @@ +local hb = require("luaharfbuzz") + +-- LuaTeX’s TFM structure indexes glyphs by character codes, so we fake it by +-- adding the maximum possible Unicode code point to the glyph id. This way +-- we have a simple mapping for all glyphs without interfering with any valid +-- Unicode code point. +-- +-- LuaTeX use the first 256 characters above maximum Unicode character for +-- escaping raw bytes, so skip that as well. +hb.CH_GID_PREFIX = 0x110000 + 256 + +return hb diff --git a/texmf-dist/tex/generic/harftex/harf-load.lua b/texmf-dist/tex/generic/harftex/harf-load.lua new file mode 100644 index 00000000..c467cedd --- /dev/null +++ b/texmf-dist/tex/generic/harftex/harf-load.lua @@ -0,0 +1,309 @@ +local hb = require("harf-base") + +local hbfonts = hb.fonts +local hbfonts = hbfonts or {} + +local cfftag = hb.Tag.new("CFF ") +local cff2tag = hb.Tag.new("CFF2") +local os2tag = hb.Tag.new("OS/2") +local posttag = hb.Tag.new("post") +local glyftag = hb.Tag.new("glyf") + +local function trim(str) + return str:gsub("^%s*(.-)%s*$", "%1") +end + +local function split(str, sep) + if str then + local result = string.explode(str, sep.."+") + for i, s in next, result do + result[i] = trim(result[i]) + end + return result + end +end + +local function parse(str, size) + local name, options = str:match("%s*(.*)%s*:%s*(.*)%s*") + local spec = { + specification = str, + size = size, + variants = {}, features = {}, options = {}, + } + + name = trim(name or str) + + local filename = name:match("%[(.*)%]") + if filename then + -- [file] + -- [file:index] + filename = string.explode(filename, ":+") + spec.file = filename[1] + spec.index = tonumber(filename[2]) or 0 + else + -- name + -- name/variants + local fontname, variants = name:match("(.-)%s*/%s*(.*)") + spec.name = fontname or name + spec.variants = split(variants, "/") + end + if options then + options = split(options, ";+") + for _, opt in next, options do + if opt:find("[+-]") == 1 then + local feature = hb.Feature.new(opt) + spec.features[#spec.features + 1] = feature + elseif opt ~= "" then + local key, val = opt:match("(.*)%s*=%s*(.*)") + if key == "language" then val = hb.Language.new(val) end + spec.options[key or opt] = val or true + end + end + end + return spec +end + +local function loadfont(spec) + local path, index = spec.path, spec.index + if not path then + return nil + end + + local key = string.format("%s:%d", path, index) + local data = hbfonts[key] + if data then + return data + end + + local hbface = hb.Face.new(path, index) + local tags = hbface and hbface:get_table_tags() + -- If the face has no table tags then it isn’t a valid SFNT font that + -- HarfBuzz can handle. + if tags then + local hbfont = hb.Font.new(hbface) + local upem = hbface:get_upem() + + -- All LuaTeX seem to care about in font type is whether it has CFF table + -- or not, so we check for that here. + local fonttype = nil + local hasos2 = false + local haspost = false + for i = 1, #tags do + local tag = tags[i] + if tag == cfftag or tag == cff2tag then + fonttype = "opentype" + elseif tag == glyftag then + fonttype = "truetype" + elseif tag == os2tag then + hasos2 = true + elseif tag == posttag then + haspost = true + end + end + + local fontextents = hbfont:get_h_extents() + local ascender = fontextents and fontextents.ascender or upem * .8 + local descender = fontextents and fontextents.descender or upem * .2 + + local gid = hbfont:get_nominal_glyph(0x0020) + local space = gid and hbfont:get_glyph_h_advance(gid) or upem / 2 + + local xheight, capheight = nil, nil + if hasos2 then + local os2 = hbface:get_table(os2tag) + local length = os2:get_length() + local data = os2:get_data() + if length >= 96 and string.unpack(">H", data) > 1 then + -- We don’t need much of the table, so we read from hard-coded offsets. + xheight = string.unpack(">H", data, 87) + capheight = string.unpack(">H", data, 89) + end + end + + xheight = xheight or ascender / 2 + capheight = capheight or ascender + + local slant = 0 + if haspost then + local post = hbface:get_table(posttag) + local length = post:get_length() + local data = post:get_data() + if length >= 32 and string.unpack(">i4", data) <= 0x00030000 then + local italicangle = string.unpack(">i4", data, 5) / 2^16 + if italicangle ~= 0 then + slant = -math.tan(italicangle * math.pi / 180) * 65536.0 + end + end + end + + -- Load CPAL palettes if avialable in the font. + local palettes = nil + if hbface:ot_color_has_palettes() and hbface:ot_color_has_layers() then + local count = hbface:ot_color_palette_get_count() + palettes = {} + for i = 1, count do + palettes[#palettes + 1] = hbface:ot_color_palette_get_colors(i) + end + end + + -- Load glyph metrics for all glyphs in the font. We used to do this on + -- demand to make loading fonts faster, but hit many limitations inside + -- LuaTeX (mainly with shared backend fonts, where LuaTeX would assume all + -- fonts it decides to share load the same set of glyphs). + -- + -- Getting glyph advances is fast enough, but glyph extents are slower + -- especially in CFF fonts. We might want to have an option to ignore exact + -- glyph extents and use font ascender and descender if this proved to be + -- too slow. + local glyphcount = hbface:get_glyph_count() + local glyphs = {} + for gid = 0, glyphcount - 1 do + local width = hbfont:get_glyph_h_advance(gid) + local height, depth, italic = nil, nil, nil + local extents = hbfont:get_glyph_extents(gid) + if extents then + height = extents.y_bearing + depth = extents.y_bearing + extents.height + if extents.x_bearing < 0 then + italic = -extents.x_bearing + end + end + glyphs[gid] = { + width = width, + height = height or ascender, + depth = -(depth or descender), + italic = italic or 0, + } + end + + local unicodes = hbface:collect_unicodes() + local characters = {} + for _, uni in next, unicodes do + characters[uni] = hbfont:get_nominal_glyph(uni) + end + + data = { + face = hbface, + font = hbfont, + upem = upem, + fonttype = fonttype, + space = space, + xheight = xheight, + capheight = capheight, + slant = slant, + glyphs = glyphs, + unicodes = characters, + psname = hbface:get_name(hb.ot.NAME_ID_POSTSCRIPT_NAME), + fullname = hbface:get_name(hb.ot.NAME_ID_FULL_NAME), + palettes = palettes, + haspng = hbface:ot_color_has_png(), + loaded = {}, -- Cached loaded glyph data. + } + + hbfonts[key] = data + return data + end +end + +local function scalefont(data, spec) + local size = spec.size + local options = spec.options + local hbfont = data.font + local upem = data.upem + local space = data.space + local capheight = data.capheight + + if size < 0 then + size = -655.36 * size + end + + -- We shape in font units (at UPEM) and then scale output with the desired + -- sfont size. + local scale = size / upem + hbfont:set_scale(upem, upem) + + -- Populate font’s characters table. + local glyphs = data.glyphs + local characters = {} + for gid, glyph in next, glyphs do + characters[hb.CH_GID_PREFIX + gid] = { + index = gid, + width = glyph.width * scale, + height = glyph.height * scale, + depth = glyph.depth * scale, + italic = glyph.italic * scale, + } + end + + local unicodes = data.unicodes + for uni, gid in next, unicodes do + characters[uni] = characters[hb.CH_GID_PREFIX + gid] + end + + -- Select font palette, we support `palette=index` option, and load the first + -- one otherwise. + local palettes = data.palettes + local palette = palettes and palettes[tonumber(options.palette) or 1] + + local letterspace = tonumber(options.letterspace or 0) / 100 * upem + space = space + letterspace + + local slantfactor = tonumber(options.slant or 0) + + return { + name = spec.specification, + filename = spec.path, + designsize = size, + psname = data.psname, + fullname = data.fullname, + index = spec.index, + size = size, + type = "real", + embedding = "subset", + tounicode = 1, + nomath = true, + format = data.fonttype, + slant = slantfactor * 1000, + characters = characters, + parameters = { + slant = data.slant, + space = space * scale, + space_stretch = space * scale / 2, + space_shrink = space * scale / 3, + x_height = data.xheight * scale, + quad = size, + extra_space = space * scale / 3, + [8] = capheight * scale, -- for XeTeX compatibility. + }, + hb = { + scale = scale, + spec = spec, + palette = palette, + shared = data, + letterspace = letterspace, + }, + } +end + +local function define_font(name, size) + local spec = type(name) == "string" and parse(name, size) or name + if spec.file then + spec.path = kpse.find_file(spec.file, "truetype fonts") or + kpse.find_file(spec.file, "opentype fonts") + else + -- XXX support font names + end + + if spec.specification == "" then return nil end + + local tfmdata = nil + local hbdata = loadfont(spec) + if hbdata then + tfmdata = scalefont(hbdata, spec) + else + tfmdata = font.read_tfm(spec.specification, spec.size) + end + return tfmdata +end + +return define_font diff --git a/texmf-dist/tex/generic/harftex/harf-luaotfload.lua b/texmf-dist/tex/generic/harftex/harf-luaotfload.lua new file mode 100644 index 00000000..45a007f5 --- /dev/null +++ b/texmf-dist/tex/generic/harftex/harf-luaotfload.lua @@ -0,0 +1,232 @@ +if not pcall(require, "luaharfbuzz") then + luatexbase.module_error("harf", "'luaharfbuzz' module is required.") +end + +local harf = require("harf") + +local define_font = harf.callbacks.define_font + +-- Change luaotfload’s default of preferring system fonts. +fonts.names.set_location_precedence { + "local", "texmf", "system" +} + +local readers = { + opentype = fonts.readers.opentype, + otf = fonts.readers.otf, + ttf = fonts.readers.ttf, + ttc = fonts.readers.ttc, +} + +local function harf_reader(spec) + local features = {} + local options = {} + local rawfeatures = spec.features and spec.features.raw or {} + + local mode = rawfeatures.mode + if mode and mode ~= "harf" then + return readers[spec.forced](spec) + end + + -- Rewrite luaotfload specification to look like what we expect. + local specification = { + features = features, + options = options, + path = spec.resolved or spec.name, + index = spec.sub and spec.sub - 1 or 0, + size = spec.size, + specification = spec.specification, + } + + for key, val in next, rawfeatures do + if key == "language" then val = harf.Language.new(val) end + if key == "colr" then key = "palette" end + if key:len() == 4 then + -- 4-letter options are likely font features, but not always, so we do + -- some checks below. We put non feature options in the `options` dict. + if val == true or val == false then + val = (val and '+' or '-')..key + features[#features + 1] = harf.Feature.new(val) + elseif tonumber(val) then + val = '+'..key..'='..tonumber(val) - 1 + features[#features + 1] = harf.Feature.new(val) + else + options[key] = val + end + else + options[key] = val + end + end + return define_font(specification) +end + +-- Register font readers. We override the default ones to always use HarfBuzz +-- if no mode is explicitly set or when `mode=harf` is used, otherwise we +-- fallback to the old readers. Fonts we load will be shaped by the callbacks +-- we register below. +fonts.readers.harf = harf_reader +fonts.readers.opentype = harf_reader +fonts.readers.otf = harf_reader +fonts.readers.ttf = harf_reader +fonts.readers.ttc = harf_reader + +local GSUBtag = harf.Tag.new("GSUB") +local GPOStag = harf.Tag.new("GPOS") +local dflttag = harf.Tag.new("dflt") + +local aux = luaotfload.aux + +local aux_provides_script = aux.provides_script +aux.provides_script = function(fontid, script) + local fontdata = font.getfont(fontid) + local hbdata = fontdata and fontdata.hb + if hbdata then + local hbshared = hbdata.shared + local hbface = hbshared.face + + local script = harf.Tag.new(script) + for _, tag in next, { GSUBtag, GPOStag } do + local scripts = hbface:ot_layout_get_script_tags(tag) or {} + for i = 1, #scripts do + if script == scripts[i] then return true end + end + end + return false + end + return aux_provides_script(fontid, script) +end + +local aux_provides_language = aux.provides_language +aux.provides_language = function(fontid, script, language) + local fontdata = font.getfont(fontid) + local hbdata = fontdata and fontdata.hb + if hbdata then + local hbshared = hbdata.shared + local hbface = hbshared.face + + local script = harf.Tag.new(script) + -- fontspec seems to incorrectly use “DFLT” for language instead of “dflt”. + local language = harf.Tag.new(language == "DFLT" and "dflt" or language) + + for _, tag in next, { GSUBtag, GPOStag } do + local scripts = hbface:ot_layout_get_script_tags(tag) or {} + for i = 1, #scripts do + if script == scripts[i] then + if language == dflttag then + -- By definition “dflt” language is always present. + return true + else + local languages = hbface:ot_layout_get_language_tags(tag, i - 1) or {} + for j = 1, #languages do + if language == languages[j] then return true end + end + end + end + end + end + return false + end + return aux_provides_language(fontid, script, language) +end + +local aux_provides_feature = aux.provides_feature +aux.provides_feature = function(fontid, script, language, feature) + local fontdata = font.getfont(fontid) + local hbdata = fontdata and fontdata.hb + if hbdata then + local hbshared = hbdata.shared + local hbface = hbshared.face + + local script = harf.Tag.new(script) + -- fontspec seems to incorrectly use “DFLT” for language instead of “dflt”. + local language = harf.Tag.new(language == "DFLT" and "dflt" or language) + local feature = harf.Tag.new(feature) + + for _, tag in next, { GSUBtag, GPOStag } do + local _, script_idx = hbface:ot_layout_find_script(tag, script) + local _, language_idx = hbface:ot_layout_find_language(tag, script_idx, language) + if hbface:ot_layout_find_feature(tag, script_idx, language_idx, feature) then + return true + end + end + return false + end + return aux_provides_feature(fontid, script, language, feature) +end + +local aux_font_has_glyph = aux.font_has_glyph +aux.font_has_glyph = function(fontid, codepoint) + local fontdata = font.getfont(fontid) + local hbdata = fontdata and fontdata.hb + if hbdata then + local hbshared = hbdata.shared + local unicodes = hbshared.unicodes + return unicodes[codepoint] ~= nil + end + return aux_font_has_glyph(fontid, codepoint) +end + +local aux_slot_of_name = aux.slot_of_name +aux.slot_of_name = function(fontid, glyphname, unsafe) + local fontdata = font.getfont(fontid) + local hbdata = fontdata and fontdata.hb + if hbdata then + local hbshared = hbdata.shared + local hbfont = hbshared.font + + local gid = hbfont:get_glyph_from_name(glyphname) + if gid ~= nil then + return gid + harf.CH_GID_PREFIX + end + return nil + end + return aux_slot_of_name(fontid, glyphname, unsafe) +end + +local aux_name_of_slot = aux.name_of_slot +aux.name_of_slot = function(fontid, codepoint) + local fontdata = font.getfont(fontid) + local hbdata = fontdata and fontdata.hb + if hbdata then + local hbshared = hbdata.shared + local hbfont = hbshared.font + local characters = fontdata.characters + local character = characters[codepoint] + + if character then + local gid = characters[codepoint].index + return hbfont:get_glyph_name(gid) + end + return nil + end + return aux_name_of_slot(fontid, codepoint) +end + +-- luatexbase does not know how to handle `wrapup_run` callback, teach it. +luatexbase.callbacktypes.wrapup_run = 1 -- simple +luatexbase.callbacktypes.get_char_tounicode = 1 -- simple +luatexbase.callbacktypes.get_glyph_string = 1 -- simple + +local base_callback_descriptions = luatexbase.callback_descriptions +local base_add_to_callback = luatexbase.add_to_callback +local base_remove_from_callback = luatexbase.remove_from_callback + +-- Remove all existing functions from given callback, insert ours, then +-- reinsert the removed ones, so ours takes a priority. +local function add_to_callback(name, func) + local saved_callbacks = {}, ff, dd + for k, v in next, base_callback_descriptions(name) do + saved_callbacks[k] = { base_remove_from_callback(name, v) } + end + base_add_to_callback(name, func, "Harf "..name.." callback") + for _, v in next, saved_callbacks do + base_add_to_callback(name, v[1], v[2]) + end +end + +-- Register all Harf callbacks, except `define_font` which is handled above. +for name, func in next, harf.callbacks do + if name ~= "define_font" then + add_to_callback(name, func) + end +end diff --git a/texmf-dist/tex/generic/harftex/harf-node.lua b/texmf-dist/tex/generic/harftex/harf-node.lua new file mode 100644 index 00000000..fb4bfabe --- /dev/null +++ b/texmf-dist/tex/generic/harftex/harf-node.lua @@ -0,0 +1,914 @@ +local hb = require("harf-base") + +local direct = node.direct +local tonode = direct.tonode +local todirect = direct.todirect +local traverse = direct.traverse +local traverseid = direct.traverse_id +local insertbefore = direct.insert_before +local insertafter = direct.insert_after +local protectglyph = direct.protect_glyph +local newnode = direct.new +local copynode = direct.copy +local removenode = direct.remove +local copynodelist = direct.copy_list +local isglyph = direct.is_glyph + +local getattrs = direct.getattributelist +local setattrs = direct.setattributelist +local getchar = direct.getchar +local setchar = direct.setchar +local getdir = direct.getdir +local setdir = direct.setdir +local getdata = direct.getdata +local setdata = direct.setdata +local getfont = direct.getfont +local setfont = direct.setfont +local getfield = direct.getfield +local setfield = direct.setfield +local getid = direct.getid +local getkern = direct.getkern +local setkern = direct.setkern +local getnext = direct.getnext +local setnext = direct.setnext +local getoffsets = direct.getoffsets +local setoffsets = direct.setoffsets +local getproperty = direct.getproperty +local setproperty = direct.setproperty +local getprev = direct.getprev +local setprev = direct.setprev +local getsubtype = direct.getsubtype +local setsubtype = direct.setsubtype +local getwidth = direct.getwidth +local setwidth = direct.setwidth + +local getpre = function (n) return getfield(n, "pre") end +local setpre = function (n, v) setfield(n, "pre", v) end +local getpost = function (n) return getfield(n, "post") end +local setpost = function (n, v) setfield(n, "post", v) end +local getrep = function (n) return getfield(n, "replace") end +local setrep = function (n, v) setfield(n, "replace", v) end + +local discid = node.id("disc") +local glueid = node.id("glue") +local glyphid = node.id("glyph") +local dirid = node.id("dir") +local kernid = node.id("kern") +local localparid = node.id("local_par") + +local spaceskip = 13 +local directmode = 2 +local fontkern = 0 +local italiccorrection = 3 + +local getscript = hb.unicode.script +local sc_common = hb.Script.new("Zyyy") +local sc_inherited = hb.Script.new("Zinh") +local sc_unknown = hb.Script.new("Zzzz") +local sc_latn = hb.Script.new("Latn") +local dir_ltr = hb.Direction.new("ltr") +local dir_rtl = hb.Direction.new("rtl") +local lang_invalid = hb.Language.new() +local fl_unsafe = hb.Buffer.GLYPH_FLAG_UNSAFE_TO_BREAK + +local p_startactual = "startactualtext" +local p_endactual = "endactualtext" +local p_color = "color" +local p_string = "string" + +local format = string.format + +-- Simple table copying function. +local function copytable(old) + local new = {} + for k, v in next, old do + if type(v) == "table" then v = copytable(v) end + new[k] = v + end + return new +end + +-- Set and get properties from our private `harf` subtable. +local function setprop(n, prop, value) + local props = getproperty(n) + if not props then + props = {} + setproperty(n, props) + end + props.harf = props.harf or {} + props.harf[prop] = value +end + +-- Copy node properties and attributes. +local function copyprops(src, dst) + local props = getproperty(src) + local attrs = getattrs(src) + if props then + setproperty(dst, copytable(props)) + end + if attrs then + setattrs(dst, copynodelist(attrs)) + end +end + +-- Convert list of integers to UTF-16 hex string used in PDF. +local function to_utf16_hex(uni) + if uni < 0x10000 then + return format("%04X", uni) + else + uni = uni - 0x10000 + local hi = 0xD800 + (uni // 0x400) + local lo = 0xDC00 + (uni % 0x400) + return format("%04X%04X", hi, lo) + end +end + +local paired_open = { + [0x0028] = 0x0029, [0x003c] = 0x003e, [0x005b] = 0x005d, [0x007b] = 0x007d, + [0x00ab] = 0x00bb, [0x2018] = 0x2019, [0x201c] = 0x201d, [0x2039] = 0x203a, + [0x3008] = 0x3009, [0x300a] = 0x300b, [0x300c] = 0x300d, [0x300e] = 0x300f, + [0x3010] = 0x3011, [0x3014] = 0x3015, [0x3016] = 0x3017, [0x3018] = 0x3019, + [0x301a] = 0x301b, +} + +local paired_close = { + [0x0029] = 0x0028, [0x003e] = 0x003c, [0x005d] = 0x005b, [0x007d] = 0x007b, + [0x00bb] = 0x00ab, [0x2019] = 0x2018, [0x201d] = 0x201c, [0x203a] = 0x2039, + [0x3009] = 0x3008, [0x300b] = 0x300a, [0x300d] = 0x300c, [0x300f] = 0x300e, + [0x3011] = 0x3010, [0x3015] = 0x3014, [0x3017] = 0x3016, [0x3019] = 0x3018, + [0x301b] = 0x301a, +} + +local process + +local function itemize(head, direction) + -- Collect character properties (font, direction, script) and resolve common + -- and inherited scripts. Pre-requisite for itemization into smaller runs. + local props, nodes, codes = {}, {}, {} + local dirstack, pairstack = {}, {} + local currdir = direction or "TLT" + local currfontid = nil + + for n in direct.traverse(head) do + local id = getid(n) + local code = 0xFFFC -- OBJECT REPLACEMENT CHARACTER + local script = sc_common + local skip = false + + if id == glyphid then + currfontid = getfont(n) + if getsubtype(n) > 255 then + skip = true + else + code = getchar(n) + script = getscript(code) + end + elseif id == glueid and getsubtype(n) == spaceskip then + code = 0x0020 -- SPACE + elseif id == discid then + code = 0x00AD -- SOFT HYPHEN + elseif id == dirid then + local dir = getdir(n) + if dir:sub(1, 1) == "+" then + -- Push the current direction to the stack. + table.insert(dirstack, currdir) + currdir = dir:sub(2) + else + assert(currdir == dir:sub(2)) + -- Pop the last direction from the stack. + currdir = table.remove(dirstack) + end + elseif id == localparid then + currdir = getdir(n) + end + + local fontdata = currfontid and font.getfont(currfontid) + local hbdata = fontdata and fontdata.hb + if not hbdata then skip = true end + + -- Resolve common and inherited scripts. Inherited takes the script of the + -- previous character. Common almost the same, but we tray to make paired + -- characters (e.g. parentheses) to take the same script. + if #props > 0 and (script == sc_common or script == sc_inherited) then + script = props[#props].script + -- Paired punctuation characters + if paired_open[code] then + table.insert(pairstack, { code, script }) + elseif paired_close[code] then + while #pairstack > 0 do + local c = table.remove(pairstack) + if c[1] == paired_close[code] then + script = c[2] + break + end + end + end + end + + -- If script is not resolved yet, and the font has a "script" option, use + -- it. + if (script == sc_common or script == sc_inherited) and hbdata then + local spec = hbdata.spec + local features = spec.features + local options = spec.options + script = options.script and hb.Script.new(options.script) or script + end + + codes[#codes + 1] = code + nodes[#nodes + 1] = n + props[#props + 1] = { + font = currfontid, + -- XXX handle RTT and LTL. + dir = currdir == "TRT" and dir_rtl or dir_ltr, + script = script, + skip = skip, + } + end + + for i = #props - 1, 1, -1 do + -- If script is not resolved yet, use that of the next character. + if props[i].script == sc_common or props[i].script == sc_inherited then + props[i].script = props[i + 1].script + end + end + + -- Split into a list of runs, each has the same font, direction and script. + -- TODO: itemize by language as well. + local runs = {} + local currfontid, currdir, currscript, currskip = nil, nil, nil, nil + for i, prop in next, props do + local fontid = prop.font + local dir = prop.dir + local script = prop.script + local skip = prop.skip + + -- Start a new run if there is a change in properties. + if fontid ~= currfontid or + dir ~= currdir or + script ~= currscript or + skip ~= currskip then + runs[#runs + 1] = { + start = i, + len = 0, + font = fontid, + dir = dir, + script = script, + skip = skip, + nodes = nodes, + codes = codes, + } + end + + runs[#runs].len = runs[#runs].len + 1 + + currfontid = fontid + currdir = dir + currscript = script + currskip = skip + end + + return runs +end + +-- Find how many characters are part of this glyph. +-- +-- The first return value is the number of characters, with 0 meaning it is +-- inside a multi-glyph cluster +-- +-- The second return value is the number of glyph in this cluster. +-- +local function chars_in_glyph(i, glyphs, stop) + local nchars, nglyphs = 0, 0 + local cluster = glyphs[i].cluster + + -- Glyph is not the first in cluster + if glyphs[i - 1] and glyphs[i - 1].cluster == cluster then + return 0, 0 + end + + -- Find the last glyph in this cluster. + while glyphs[i + nglyphs] and glyphs[i + nglyphs].cluster == cluster do + nglyphs = nglyphs + 1 + end + + -- The number of characters is the diff between the next cluster in this one. + if glyphs[i + nglyphs] then + nchars = glyphs[i + nglyphs].cluster - cluster + else + -- This glyph cluster in the last in the run. + nchars = stop - cluster - 1 + end + + return nchars, nglyphs +end + +-- Check if it is safe to break before this glyph. +local function unsafetobreak(glyph, nodes) + return glyph + and glyph.flags + and glyph.flags & fl_unsafe + -- LuaTeX’s discretionary nodes can’t contain glue, so stop at first glue + -- as well. This is incorrect, but I don’t have a better idea. + and getid(nodes[glyph.cluster + 1]) ~= glueid +end + +local shape + +-- Make s a sub run, used by discretionary nodes. +local function makesub(run, start, stop, nodelist) + local nodes = run.nodes + local codes = run.codes + local start = start + local stop = stop + local subnodes, subcodes = {}, {} + for i = start, stop do + if getid(nodes[i]) ~= discid then + subnodes[#subnodes + 1] = copynode(nodes[i]) + subcodes[#subcodes + 1] = codes[i] + end + end + -- Prepend any existing nodes to the list. + for n in traverse(nodelist) do + subnodes[#subnodes + 1] = n + subcodes[#subcodes + 1] = getchar(n) + end + local subrun = { + start = 1, + len = #subnodes, + font = run.font, + script = run.script, + dir = run.dir, + fordisc = true, + nodes = subnodes, + codes = subcodes, + } + return { glyphs = shape(subrun), run = subrun } +end + +-- Main shaping function that calls HarfBuzz, and does some post-processing of +-- the output. +shape = function(run) + local nodes = run.nodes + local codes = run.codes + local offset = run.start + local len = run.len + local fontid = run.font + local dir = run.dir + local script = run.script + local lang = run.lang + local fordisc = run.fordisc + + local fontdata = font.getfont(fontid) + local hbdata = fontdata.hb + local palette = hbdata.palette + local spec = hbdata.spec + local features = spec.features + local options = spec.options + local hbshared = hbdata.shared + local hbfont = hbshared.font + local hbface = hbshared.face + + local lang = lang or options.language or lang_invalid + local shapers = options.shaper and { options.shaper } or {} + + local buf = hb.Buffer.new() + buf:set_direction(dir) + buf:set_script(script) + buf:set_language(lang) + buf:set_cluster_level(buf.CLUSTER_LEVEL_MONOTONE_CHARACTERS) + buf:add_codepoints(codes, offset - 1, len) + + if hb.shape_full(hbfont, buf, features, shapers) then + -- LuaTeX wants the glyphs in logical order, so reverse RTL buffers. + if dir:is_backward() then buf:reverse() end + + local glyphs = buf:get_glyphs() + + -- If the font has COLR/CPAL tables, decompose each glyph to its color + -- layers and set the color from the palette. + if palette then + for i, glyph in next, glyphs do + local gid = glyph.codepoint + local layers = hbface:ot_color_glyph_get_layers(gid) + if layers then + -- Remove this glyph, we will use its layers. + table.remove(glyphs, i) + for j, layer in next, layers do + -- All glyphs but the last use 0 advance so that the layers + -- overlap. + local xadavance, yadvance = nil, nil + if dir:is_backward() then + x_advance = j == 1 and glyph.x_advance or 0 + y_advance = j == 1 and glyph.y_advance or 0 + else + x_advance = j == #layers and glyph.x_advance or 0 + y_advance = j == #layers and glyph.y_advance or 0 + end + table.insert(glyphs, i + j - 1, { + codepoint = layer.glyph, + cluster = glyph.cluster, + x_advance = x_advance, + y_advance = y_advance, + x_offset = glyph.x_offset, + y_offset = glyph.y_offset, + flags = glyph.flags, + -- color_index has a special value, 0x10000, that mean use text + -- color, we don’t check for it here explicitly since we will + -- get nil anyway. + color = palette[layer.color_index], + }) + end + end + end + end + + for i, glyph in next, glyphs do + local nodeindex = glyph.cluster + 1 + local nchars, nglyphs = chars_in_glyph(i, glyphs, offset + len) + glyph.nchars, glyph.nglyphs = nchars, nglyphs + + -- Calculate the Unicode code points of this glyph. If nchars is zero + -- then this is a glyph inside a complex cluster and will be handled with + -- the start of its cluster. + if nchars > 0 then + local hex = "" + local str = "" + for j = 0, nchars - 1 do + local id = getid(nodes[nodeindex + j]) + if id == glyphid or id == glueid then + local code = codes[nodeindex + j] + hex = hex..to_utf16_hex(code) + str = str..utf8.char(code) + end + end + glyph.tounicode = hex + glyph.string = str + end + + -- Find if we have a discretionary inside a ligature, if nchars less than + -- two then either this is not a ligature or there is no discretionary + -- involved. + if nchars > 2 and not fordisc then + local discindex = nil + for j = nodeindex, nodeindex + nchars - 1 do + if codes[j] == 0x00AD then + discindex = j + break + end + end + if discindex then + -- Discretionary found. + local disc = nodes[discindex] + local startindex, stopindex = nil, nil + local startglyph, stopglyph = nil, nil + + -- Find the previous glyph that is safe to break at. + startglyph = i + while unsafetobreak(glyphs[startglyph], nodes) do + startglyph = startglyph - 1 + end + -- Get the corresponding character index. + startindex = glyphs[startglyph].cluster + 1 + + -- Find the next glyph that is safe to break at. + stopglyph = i + nglyphs + while unsafetobreak(glyphs[stopglyph], nodes) do + stopglyph = stopglyph + 1 + end + -- We also want the last char in the previous glyph, so no +1 below. + stopindex = glyphs[stopglyph].cluster + -- We break up to stop glyph but not including it, so the -1 below. + stopglyph = stopglyph - 1 + + -- Mark these glyph for skipping since they will be replaced by the + -- discretionary fields. + for j = startglyph, stopglyph do + glyphs[j].skip = true + end + + local pre, post, rep = getpre(disc), getpost(disc), getrep(disc) + glyph.disc = disc + glyph.replace = makesub(run, startindex, stopindex, rep) + glyph.pre = makesub(run, startindex, discindex - 1, pre) + glyph.post = makesub(run, discindex + 1, stopindex, post) + end + end + end + return glyphs + end + + return {} +end + +local function color_to_rgba(color) + local r = color.red / 255 + local g = color.green / 255 + local b = color.blue / 255 + local a = color.alpha / 255 + if a ~= 1 then + -- XXX: alpha + return format('%s %s %s rg', r, g, b) + else + return format('%s %s %s rg', r, g, b) + end +end + +-- Cache of color glyph PNG data for bookkeeping, only because I couldn’t +-- figure how to make LuaTeX load the image from the binary data directly. +local pngcache = {} +local function cachedpng(data) + local hash = md5.sumhexa(data) + local path = pngcache[hash] + if not path then + path = os.tmpname() + local file = io.open(path, "wb") + file:write(data) + file:close() + pngcache[hash] = path + end + return path +end + +-- Convert glyphs to nodes and collect font characters. +local function tonodes(head, current, run, glyphs, color) + local nodes = run.nodes + local dir = run.dir + local fontid = run.font + local fontdata = font.getfont(fontid) + local characters = fontdata.characters + local hbdata = fontdata.hb + local hbshared = hbdata.shared + local hbfont = hbshared.font + local fontglyphs = hbshared.glyphs + local rtl = dir:is_backward() + + local tracinglostchars = tex.tracinglostchars + local tracingonline = tex.tracingonline + + local scale = hbdata.scale + local letterspace = hbdata.letterspace + + local haspng = hbshared.haspng + local fonttype = hbshared.fonttype + + for i, glyph in next, glyphs do + local index = glyph.cluster + 1 + local gid = glyph.codepoint + local char = hb.CH_GID_PREFIX + gid + local n = nodes[index] + local id = getid(n) + local nchars, nglyphs = glyph.nchars, glyph.nglyphs + + -- If this glyph is part of a complex cluster, then copy the node as + -- more than one glyph will use it. + if nglyphs < 1 or nglyphs > 1 then + n = copynode(nodes[index]) + end + + if color then + setprop(n, p_color, color) + end + + if glyph.disc then + -- For discretionary the glyph itself is skipped and a discretionary node + -- is output in place of it. + local disc = glyph.disc + local rep, pre, post = glyph.replace, glyph.pre, glyph.post + + setrep(disc, tonodes(nil, nil, rep.run, rep.glyphs, color)) + setpre(disc, tonodes(nil, nil, pre.run, pre.glyphs, color)) + setpost(disc, tonodes(nil, nil, post.run, post.glyphs, color)) + + head, current = insertafter(head, current, disc) + elseif not glyph.skip then + if glyph.color then + setprop(n, p_color, color_to_rgba(glyph.color)) + end + + if id == glyphid then + local fontglyph = fontglyphs[gid] + + local pngblob = fontglyph.png + if haspng and not pngblob then + pngblob = hbfont:ot_color_glyph_get_png(gid) + fontglyph.png = pngblob + end + local character = characters[char] + if pngblob then + -- Color bitmap font, extract the PNG data and insert it in the node + -- list. + local data = pngblob:get_data() + local path = cachedpng(data) + + local image = img.node { + filename = path, + width = character.width, + height = character.height, + depth = character.depth, + } + head, current = insertafter(head, current, todirect(image)) + else + if haspng and not fonttype then + -- If this is a color bitmap font with no glyph outlines (like Noto + -- Color Emoji) and we end up here then the glyph is not supported + -- by the font. LuaTeX does not now how to embed such fonts, so we + -- don’t want them to reach the backend as it will cause a fatal + -- error. We use `nullfont` instead. + -- That is a hack, but I think it is good enough for now. + setfont(n, 0) + else + local oldcharacter = characters[getchar(n)] + -- If the glyph index of current font chars is the same as shaped + -- glyph, keep the node char unchanged. Helps with primitives that + -- take characters as input but actually work on glyphs, like + -- `\rpcode`. + if not oldcharacter or character.index ~= oldcharacter.index then + setchar(n, char) + end + end + local xoffset = (rtl and -glyph.x_offset or glyph.x_offset) * scale + local yoffset = glyph.y_offset * scale + setoffsets(n, xoffset, yoffset) + protectglyph(n) + head, current = insertafter(head, current, n) + + local x_advance = glyph.x_advance + letterspace + local width = fontglyph.width + if width ~= x_advance then + -- LuaTeX always uses the glyph width from the font, so we need to + -- insert a kern node if the x advance is different. + local kern = newnode(kernid) + setkern(kern, (x_advance - width) * scale) + copyprops(n, kern) + if rtl then + head = insertbefore(head, current, kern) + else + head, current = insertafter(head, current, kern) + end + end + + -- HarfTeX will use this string when printing a glyph node e.g. in + -- overfull messages, otherwise it will be trying to print our + -- invalid pseudo Unicode code points. + -- If the string is empty it means this glyph is part of a larger + -- cluster and we don’t to print anything for it as the first glyph + -- in the cluster will have the string of the whole cluster. + setprop(n, p_string, glyph.string or "") + + -- Handle PDF text extraction: + -- * Find how many characters in this cluster and how many glyphs, + -- * If there is more than 0 characters + -- * One glyph: one to one or one to many mapping, can be + -- represented by font’s /ToUnicode + -- * More than one: many to one or many to many mapping, can be + -- represented by /ActualText spans. + -- * If there are zero characters, then this glyph is part of complex + -- cluster that will be covered by an /ActualText span. + local tounicode = glyph.tounicode + if tounicode then + if nglyphs == 1 and not fontglyph.tounicode then + fontglyph.tounicode = tounicode + elseif tounicode ~= fontglyph.tounicode then + setprop(n, p_startactual, tounicode) + glyphs[i + nglyphs - 1].endactual = true + end + end + if glyph.endactual then + setprop(n, p_endactual, true) + end + end + elseif id == glueid and getsubtype(n) == spaceskip then + -- If the glyph advance is different from the font space, then a + -- substitution or positioning was applied to the space glyph changing + -- it from the default, so reset the glue using the new advance. + -- We are intentionally not comparing with the existing glue width as + -- spacing after the period is larger by default in TeX. + local width = (glyph.x_advance + letterspace) * scale + if fontdata.parameters.space ~= width then + setwidth(n, width) + setfield(n, "stretch", width / 2) + setfield(n, "shrink", width / 3) + end + head, current = insertafter(head, current, n) + elseif id == kernid and getsubtype(n) == italiccorrection then + -- If this is an italic correction node and the previous node is a + -- glyph, update its kern value with the glyph’s italic correction. + -- I’d have expected the engine to do this, but apparently it doesn’t. + -- May be it is checking for the italic correction before we have had + -- loaded the glyph? + local prevchar, prevfontid = isglyph(current) + if prevchar > 0 then + local prevfontdata = font.getfont(prevfontid) + local prevcharacters = prevfontdata and prevfontdata.characters + local italic = prevcharacters and prevcharacters[prevchar].italic + if italic then + setkern(n, italic) + end + end + head, current = insertafter(head, current, n) + elseif id == discid then + assert(nglyphs == 1) + -- The simple case of a discretionary that is not part of a complex + -- cluster. We only need to make sure kerning before the hyphenation + -- point is dropped when a line break is inserted here. + -- + -- TODO: nothing as simple as it sounds, we need to handle this like + -- the other discretionary handling, otherwise the discretionary + -- contents do not interact with the surrounding (e.g. no ligatures or + -- kerning) as it should. + if current and getid(current) == kernid and getsubtype(current) == fontkern then + setprev(current, nil) + setnext(current, nil) + setfield(n, "replace", current) + head, current = removenode(head, current) + end + local pre, post, rep = getpre(n), getpost(n), getrep(n) + setfield(n, "pre", process(pre, direction)) + setfield(n, "post", process(post, direction)) + setfield(n, "replace", process(rep, direction)) + + head, current = insertafter(head, current, n) + else + head, current = insertafter(head, current, n) + end + end + end + + return head, current +end + +local function validate_color(s) + local r = tonumber(s:sub(1, 2), 16) + local g = tonumber(s:sub(3, 4), 16) + local b = tonumber(s:sub(5, 6), 16) + if not (r and g and b) then return end + if #s == 8 then + local a = tonumber(s:sub(7, 8), 16) + if not a then return end + end + return s +end + +local function hex_to_rgba(s) + if not validate_color(s) then return end + local r = tonumber(s:sub(1, 2), 16) / 255 + local g = tonumber(s:sub(3, 4), 16) / 255 + local b = tonumber(s:sub(5, 6), 16) / 255 + if #s == 8 then + local a = tonumber(s:sub(7, 8), 16) / 255 + -- XXX: alpha + return format('%s %s %s rg', r, g, b) + else + return format('%s %s %s rg', r, g, b) + end +end + +local function shape_run(head, current, run) + if not run.skip then + -- Font loaded with our loader and an HarfBuzz face is present, do our + -- shaping. + local fontid = run.font + local fontdata = font.getfont(fontid) + local options = fontdata.hb.spec.options + local color = options and options.color and hex_to_rgba(options.color) + + local glyphs = shape(run) + head, current = tonodes(head, current, run, glyphs, color) + else + -- Not shaping, insert the original node list of of this run. + local nodes = run.nodes + local offset = run.start + local len = run.len + for i = offset, offset + len - 1 do + head, current = insertafter(head, current, nodes[i]) + end + end + + return head, current +end + +process = function(head, direction) + local newhead, current = nil, nil + local runs = itemize(head, direction) + + for _, run in next, runs do + newhead, current = shape_run(newhead, current, run) + end + + return newhead or head +end + +local function process_nodes(head, groupcode, size, packtype, direction) + local head = todirect(head) + + -- Check if any fonts are loaded by us and then process the whole node list, + -- we will take care of skipping fonts we did not load later, otherwise + -- return unmodified head. + for n in traverseid(glyphid, head) do + local fontid = getfont(n) + local fontdata = font.getfont(fontid) + local hbdata = fontdata and fontdata.hb + if hbdata then + head = process(head, direction) + break + end + end + + -- Nothing to do; no glyphs or no HarfBuzz fonts. + return tonode(head) +end + +local function pdfdirect(data) + local n = newnode("whatsit", "pdf_literal") + setfield(n, "mode", directmode) + setdata(n, data) + return n +end + +local function post_process(head, currentcolor) + for n in traverse(head) do + local props = getproperty(n) + local harfprops = props and props.harf + + local startactual, endactual, color + if harfprops then + startactual = harfprops[p_startactual] + endactual = harfprops[p_endactual] + color = harfprops[p_color] + end + + if currentcolor and currentcolor ~= color then + -- Pop current color. + head = insertbefore(head, n, pdfdirect("0 g")) + end + + if currentcolor ~= color then + -- Push new color. + head = insertbefore(head, n, pdfdirect(color)) + currentcolor = color + end + + if startactual then + local actualtext = "/Span<</ActualText<FEFF"..startactual..">>>BDC" + head = insertbefore(head, n, pdfdirect(actualtext)) + end + + if endactual then + head = insertafter(head, n, pdfdirect("EMC")) + end + + local replace = getfield(n, "replace") + if replace then + setfield(n, "replace", post_process(replace, currentcolor)) + end + + local subhead = getfield(n, "head") + if subhead then + setfield(n, "head", post_process(subhead, currentcolor)) + end + end + return head +end + +local function post_process_nodes(head, groupcode) + return tonode(post_process(todirect(head))) +end + +local function run_cleanup() + -- Remove temporary PNG files that we created, if any. + for _, path in next, pngcache do + os.remove(path) + end +end + +local function get_tounicode(fontid, c) + local fontdata = font.getfont(fontid) + local hbdata = fontdata.hb + if hbdata then + local hbshared = hbdata.shared + local fontglyphs = hbshared.glyphs + local hbfont = hbshared.font + + local gid = c - hb.CH_GID_PREFIX + if c < hb.CH_GID_PREFIX then + gid = hbfont:get_nominal_glyph(c) + end + + return fontglyphs[gid].tounicode + end +end + +local function get_glyph_string(n) + local n = todirect(n) + local props = getproperty(n) + props = props and props.harf + return props and props[p_string] or nil +end + +return { + process = process_nodes, + post_process = post_process_nodes, + cleanup = run_cleanup, + get_tounicode = get_tounicode, + get_glyph_string = get_glyph_string, +} diff --git a/texmf-dist/tex/generic/harftex/harf-plain.lua b/texmf-dist/tex/generic/harftex/harf-plain.lua new file mode 100644 index 00000000..a79a174c --- /dev/null +++ b/texmf-dist/tex/generic/harftex/harf-plain.lua @@ -0,0 +1,5 @@ +local harf = require("harf") + +for name, func in next, harf.callbacks do + callback.register(name, func) +end diff --git a/texmf-dist/tex/generic/harftex/harf.lua b/texmf-dist/tex/generic/harftex/harf.lua new file mode 100644 index 00000000..08d57d64 --- /dev/null +++ b/texmf-dist/tex/generic/harftex/harf.lua @@ -0,0 +1,16 @@ +local harf = require("harf-base") + +local define_font = require("harf-load") +local harf_node = require("harf-node") + +harf.callbacks = { + define_font = define_font, + pre_linebreak_filter = harf_node.process, + hpack_filter = harf_node.process, + pre_output_filter = harf_node.post_process, + wrapup_run = harf_node.cleanup, + get_char_tounicode = harf_node.get_tounicode, + get_glyph_string = harf_node.get_glyph_string, +} + +return harf diff --git a/texmf-dist/tex/generic/harftex/harfload.sty b/texmf-dist/tex/generic/harftex/harfload.sty new file mode 100644 index 00000000..ae85ab31 --- /dev/null +++ b/texmf-dist/tex/generic/harftex/harfload.sty @@ -0,0 +1,14 @@ +\csname ifharfloadloaded\endcsname +\let\ifharfloadloaded\endinput +\ifdefined\ProvidesPackage + \RequirePackage{luaotfload} + \ProvidesPackage{harfload}[2019/04/26 v0.1.0 Unicode text layout system] +\else + \input luaotfload.sty +\fi +\directlua{ + if not xpcall(function() require('harf-luaotfload.lua') end, + texio.write_nl) then + texio.write_nl('Error in harfload - loading aborted') + end +} |