-- merged file : luatex-fonts-merged.lua -- parent file : luatex-fonts.lua -- merge date : 05/23/12 23:31:34 do -- begin closure to overcome local limits and interference if not modules then modules = { } end modules ['l-string'] = { version = 1.001, comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } local string = string local sub, gsub, find, match, gmatch, format, char, byte, rep, lower = string.sub, string.gsub, string.find, string.match, string.gmatch, string.format, string.char, string.byte, string.rep, string.lower local lpegmatch, S, C, Ct = lpeg.match, lpeg.S, lpeg.C, lpeg.Ct -- some functions may disappear as they are not used anywhere if not string.split then -- this will be overloaded by a faster lpeg variant function string.split(str,pattern) local t = { } if #str > 0 then local n = 1 for s in gmatch(str..pattern,"(.-)"..pattern) do t[n] = s n = n + 1 end end return t end end function string.unquoted(str) return (gsub(str,"^([\"\'])(.*)%1$","%2")) end --~ function stringunquoted(str) --~ if find(str,"^[\'\"]") then --~ return sub(str,2,-2) --~ else --~ return str --~ end --~ end function string.quoted(str) return format("%q",str) -- always " end function string.count(str,pattern) -- variant 3 local n = 0 for _ in gmatch(str,pattern) do -- not for utf n = n + 1 end return n end function string.limit(str,n,sentinel) -- not utf proof if #str > n then sentinel = sentinel or "..." return sub(str,1,(n-#sentinel)) .. sentinel else return str end end local space = S(" \t\v\n") local nospace = 1 - space local stripper = space^0 * C((space^0 * nospace^1)^0) -- roberto's code function string.strip(str) return lpegmatch(stripper,str) or "" end function string.is_empty(str) return not find(str,"%S") end local patterns_escapes = { ["%"] = "%%", ["."] = "%.", ["+"] = "%+", ["-"] = "%-", ["*"] = "%*", ["["] = "%[", ["]"] = "%]", ["("] = "%(", [")"] = "%)", -- ["{"] = "%{", ["}"] = "%}" -- ["^"] = "%^", ["$"] = "%$", } local simple_escapes = { ["-"] = "%-", ["."] = "%.", ["?"] = ".", ["*"] = ".*", } function string.escapedpattern(str,simple) return (gsub(str,".",simple and simple_escapes or patterns_escapes)) end function string.topattern(str,lowercase,strict) if str == "" then return ".*" else str = gsub(str,".",simple_escapes) if lowercase then str = lower(str) end if strict then return "^" .. str .. "$" else return str end end end -- obsolete names: string.quote = string.quoted string.unquote = string.unquoted end -- closure do -- begin closure to overcome local limits and interference if not modules then modules = { } end modules ['l-table'] = { version = 1.001, comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } local type, next, tostring, tonumber, ipairs, table, string = type, next, tostring, tonumber, ipairs, table, string local concat, sort, insert, remove = table.concat, table.sort, table.insert, table.remove local format, find, gsub, lower, dump, match = string.format, string.find, string.gsub, string.lower, string.dump, string.match local getmetatable, setmetatable = getmetatable, setmetatable local getinfo = debug.getinfo -- Starting with version 5.2 Lua no longer provide ipairs, which makes -- sense. As we already used the for loop and # in most places the -- impact on ConTeXt was not that large; the remaining ipairs already -- have been replaced. In a similar fashion we also hardly used pairs. -- -- Just in case, we provide the fallbacks as discussed in Programming -- in Lua (http://www.lua.org/pil/7.3.html): if not ipairs then -- for k, v in ipairs(t) do ... end -- for k=1,#t do local v = t[k] ... end local function iterate(a,i) i = i + 1 local v = a[i] if v ~= nil then return i, v --, nil end end function ipairs(a) return iterate, a, 0 end end if not pairs then -- for k, v in pairs(t) do ... end -- for k, v in next, t do ... end function pairs(t) return next, t -- , nil end end -- Also, unpack has been moved to the table table, and for compatiility -- reasons we provide both now. if not table.unpack then table.unpack = _G.unpack elseif not unpack then _G.unpack = table.unpack end -- extra functions, some might go (when not used) function table.strip(tab) local lst, l = { }, 0 for i=1,#tab do local s = gsub(tab[i],"^%s*(.-)%s*$","%1") if s == "" then -- skip this one else l = l + 1 lst[l] = s end end return lst end function table.keys(t) local keys, k = { }, 0 for key, _ in next, t do k = k + 1 keys[k] = key end return keys end local function compare(a,b) local ta, tb = type(a), type(b) -- needed, else 11 < 2 if ta == tb then return a < b else return tostring(a) < tostring(b) end end local function sortedkeys(tab) local srt, category, s = { }, 0, 0 -- 0=unknown 1=string, 2=number 3=mixed for key,_ in next, tab do s = s + 1 srt[s] = key if category == 3 then -- no further check else local tkey = type(key) if tkey == "string" then category = (category == 2 and 3) or 1 elseif tkey == "number" then category = (category == 1 and 3) or 2 else category = 3 end end end if category == 0 or category == 3 then sort(srt,compare) else sort(srt) end return srt end local function sortedhashkeys(tab) -- fast one local srt, s = { }, 0 for key,_ in next, tab do if key then s= s + 1 srt[s] = key end end sort(srt) return srt end table.sortedkeys = sortedkeys table.sortedhashkeys = sortedhashkeys local function nothing() end local function sortedhash(t) if t then local n, s = 0, sortedkeys(t) -- the robust one local function kv(s) n = n + 1 local k = s[n] return k, t[k] end return kv, s else return nothing end end table.sortedhash = sortedhash table.sortedpairs = sortedhash function table.append(t, list) local n = #t for i=1,#list do n = n + 1 t[n] = list[i] end return t end function table.prepend(t, list) local nl = #list local nt = nl + #t for i=#t,1,-1 do t[nt] = t[i] nt = nt - 1 end for i=1,#list do t[i] = list[i] end return t end function table.merge(t, ...) -- first one is target t = t or { } local lst = { ... } for i=1,#lst do for k, v in next, lst[i] do t[k] = v end end return t end function table.merged(...) local tmp, lst = { }, { ... } for i=1,#lst do for k, v in next, lst[i] do tmp[k] = v end end return tmp end function table.imerge(t, ...) local lst, nt = { ... }, #t for i=1,#lst do local nst = lst[i] for j=1,#nst do nt = nt + 1 t[nt] = nst[j] end end return t end function table.imerged(...) local tmp, ntmp, lst = { }, 0, {...} for i=1,#lst do local nst = lst[i] for j=1,#nst do ntmp = ntmp + 1 tmp[ntmp] = nst[j] end end return tmp end local function fastcopy(old,metatabletoo) -- fast one if old then local new = { } for k,v in next, old do if type(v) == "table" then new[k] = fastcopy(v,metatabletoo) -- was just table.copy else new[k] = v end end if metatabletoo then -- optional second arg local mt = getmetatable(old) if mt then setmetatable(new,mt) end end return new else return { } end end -- todo : copy without metatable local function copy(t, tables) -- taken from lua wiki, slightly adapted tables = tables or { } local tcopy = {} if not tables[t] then tables[t] = tcopy end for i,v in next, t do -- brrr, what happens with sparse indexed if type(i) == "table" then if tables[i] then i = tables[i] else i = copy(i, tables) end end if type(v) ~= "table" then tcopy[i] = v elseif tables[v] then tcopy[i] = tables[v] else tcopy[i] = copy(v, tables) end end local mt = getmetatable(t) if mt then setmetatable(tcopy,mt) end return tcopy end table.fastcopy = fastcopy table.copy = copy function table.derive(parent) local child = { } if parent then setmetatable(child,{ __index = parent }) end return child end function table.tohash(t,value) local h = { } if t then if value == nil then value = true end for _, v in next, t do -- no ipairs here h[v] = value end end return h end function table.fromhash(t) local hsh, h = { }, 0 for k, v in next, t do -- no ipairs here if v then h = h + 1 hsh[h] = k end end return hsh end local noquotes, hexify, handle, reduce, compact, inline, functions local reserved = table.tohash { -- intercept a language inconvenience: no reserved words as key 'and', 'break', 'do', 'else', 'elseif', 'end', 'false', 'for', 'function', 'if', 'in', 'local', 'nil', 'not', 'or', 'repeat', 'return', 'then', 'true', 'until', 'while', } local function simple_table(t) if #t > 0 then local n = 0 for _,v in next, t do n = n + 1 end if n == #t then local tt, nt = { }, 0 for i=1,#t do local v = t[i] local tv = type(v) if tv == "number" then nt = nt + 1 if hexify then tt[nt] = format("0x%04X",v) else tt[nt] = tostring(v) -- tostring not needed end elseif tv == "boolean" then nt = nt + 1 tt[nt] = tostring(v) elseif tv == "string" then nt = nt + 1 tt[nt] = format("%q",v) else tt = nil break end end return tt end end return nil end -- Because this is a core function of mkiv I moved some function calls -- inline. -- -- twice as fast in a test: -- -- local propername = lpeg.P(lpeg.R("AZ","az","__") * lpeg.R("09","AZ","az", "__")^0 * lpeg.P(-1) ) -- problem: there no good number_to_string converter with the best resolution local function dummy() end local function do_serialize(root,name,depth,level,indexed) if level > 0 then depth = depth .. " " if indexed then handle(format("%s{",depth)) else local tn = type(name) if tn == "number" then -- or find(k,"^%d+$") then if hexify then handle(format("%s[0x%04X]={",depth,name)) else handle(format("%s[%s]={",depth,name)) end elseif tn == "string" then if noquotes and not reserved[name] and find(name,"^%a[%w%_]*$") then handle(format("%s%s={",depth,name)) else handle(format("%s[%q]={",depth,name)) end elseif tn == "boolean" then handle(format("%s[%s]={",depth,tostring(name))) else handle(format("%s{",depth)) end end end -- we could check for k (index) being number (cardinal) if root and next(root) then local first, last = nil, 0 -- #root cannot be trusted here (will be ok in 5.2 when ipairs is gone) if compact then -- NOT: for k=1,#root do (we need to quit at nil) for k,v in ipairs(root) do -- can we use next? if not first then first = k end last = last + 1 end end local sk = sortedkeys(root) for i=1,#sk do local k = sk[i] local v = root[k] --~ if v == root then -- circular --~ else local t, tk = type(v), type(k) if compact and first and tk == "number" and k >= first and k <= last then if t == "number" then if hexify then handle(format("%s 0x%04X,",depth,v)) else handle(format("%s %s,",depth,v)) -- %.99g end elseif t == "string" then if reduce and tonumber(v) then handle(format("%s %s,",depth,v)) else handle(format("%s %q,",depth,v)) end elseif t == "table" then if not next(v) then handle(format("%s {},",depth)) elseif inline then -- and #t > 0 local st = simple_table(v) if st then handle(format("%s { %s },",depth,concat(st,", "))) else do_serialize(v,k,depth,level+1,true) end else do_serialize(v,k,depth,level+1,true) end elseif t == "boolean" then handle(format("%s %s,",depth,tostring(v))) elseif t == "function" then if functions then handle(format('%s loadstring(%q),',depth,dump(v))) else handle(format('%s "function",',depth)) end else handle(format("%s %q,",depth,tostring(v))) end elseif k == "__p__" then -- parent if false then handle(format("%s __p__=nil,",depth)) end elseif t == "number" then if tk == "number" then -- or find(k,"^%d+$") then if hexify then handle(format("%s [0x%04X]=0x%04X,",depth,k,v)) else handle(format("%s [%s]=%s,",depth,k,v)) -- %.99g end elseif tk == "boolean" then if hexify then handle(format("%s [%s]=0x%04X,",depth,tostring(k),v)) else handle(format("%s [%s]=%s,",depth,tostring(k),v)) -- %.99g end elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then if hexify then handle(format("%s %s=0x%04X,",depth,k,v)) else handle(format("%s %s=%s,",depth,k,v)) -- %.99g end else if hexify then handle(format("%s [%q]=0x%04X,",depth,k,v)) else handle(format("%s [%q]=%s,",depth,k,v)) -- %.99g end end elseif t == "string" then if reduce and tonumber(v) then if tk == "number" then -- or find(k,"^%d+$") then if hexify then handle(format("%s [0x%04X]=%s,",depth,k,v)) else handle(format("%s [%s]=%s,",depth,k,v)) end elseif tk == "boolean" then handle(format("%s [%s]=%s,",depth,tostring(k),v)) elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then handle(format("%s %s=%s,",depth,k,v)) else handle(format("%s [%q]=%s,",depth,k,v)) end else if tk == "number" then -- or find(k,"^%d+$") then if hexify then handle(format("%s [0x%04X]=%q,",depth,k,v)) else handle(format("%s [%s]=%q,",depth,k,v)) end elseif tk == "boolean" then handle(format("%s [%s]=%q,",depth,tostring(k),v)) elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then handle(format("%s %s=%q,",depth,k,v)) else handle(format("%s [%q]=%q,",depth,k,v)) end end elseif t == "table" then if not next(v) then if tk == "number" then -- or find(k,"^%d+$") then if hexify then handle(format("%s [0x%04X]={},",depth,k)) else handle(format("%s [%s]={},",depth,k)) end elseif tk == "boolean" then handle(format("%s [%s]={},",depth,tostring(k))) elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then handle(format("%s %s={},",depth,k)) else handle(format("%s [%q]={},",depth,k)) end elseif inline then local st = simple_table(v) if st then if tk == "number" then -- or find(k,"^%d+$") then if hexify then handle(format("%s [0x%04X]={ %s },",depth,k,concat(st,", "))) else handle(format("%s [%s]={ %s },",depth,k,concat(st,", "))) end elseif tk == "boolean" then -- or find(k,"^%d+$") then handle(format("%s [%s]={ %s },",depth,tostring(k),concat(st,", "))) elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then handle(format("%s %s={ %s },",depth,k,concat(st,", "))) else handle(format("%s [%q]={ %s },",depth,k,concat(st,", "))) end else do_serialize(v,k,depth,level+1) end else do_serialize(v,k,depth,level+1) end elseif t == "boolean" then if tk == "number" then -- or find(k,"^%d+$") then if hexify then handle(format("%s [0x%04X]=%s,",depth,k,tostring(v))) else handle(format("%s [%s]=%s,",depth,k,tostring(v))) end elseif tk == "boolean" then -- or find(k,"^%d+$") then handle(format("%s [%s]=%s,",depth,tostring(k),tostring(v))) elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then handle(format("%s %s=%s,",depth,k,tostring(v))) else handle(format("%s [%q]=%s,",depth,k,tostring(v))) end elseif t == "function" then if functions then local f = getinfo(v).what == "C" and dump(dummy) or dump(v) -- local f = getinfo(v).what == "C" and dump(function(...) return v(...) end) or dump(v) if tk == "number" then -- or find(k,"^%d+$") then if hexify then handle(format("%s [0x%04X]=loadstring(%q),",depth,k,f)) else handle(format("%s [%s]=loadstring(%q),",depth,k,f)) end elseif tk == "boolean" then handle(format("%s [%s]=loadstring(%q),",depth,tostring(k),f)) elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then handle(format("%s %s=loadstring(%q),",depth,k,f)) else handle(format("%s [%q]=loadstring(%q),",depth,k,f)) end end else if tk == "number" then -- or find(k,"^%d+$") then if hexify then handle(format("%s [0x%04X]=%q,",depth,k,tostring(v))) else handle(format("%s [%s]=%q,",depth,k,tostring(v))) end elseif tk == "boolean" then -- or find(k,"^%d+$") then handle(format("%s [%s]=%q,",depth,tostring(k),tostring(v))) elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then handle(format("%s %s=%q,",depth,k,tostring(v))) else handle(format("%s [%q]=%q,",depth,k,tostring(v))) end end --~ end end end if level > 0 then handle(format("%s},",depth)) end end -- replacing handle by a direct t[#t+1] = ... (plus test) is not much -- faster (0.03 on 1.00 for zapfino.tma) local function serialize(_handle,root,name,specification) -- handle wins local tname = type(name) if type(specification) == "table" then noquotes = specification.noquotes hexify = specification.hexify handle = _handle or specification.handle or print reduce = specification.reduce or false functions = specification.functions compact = specification.compact inline = specification.inline and compact if functions == nil then functions = true end if compact == nil then compact = true end if inline == nil then inline = compact end else noquotes = false hexify = false handle = _handle or print reduce = false compact = true inline = true functions = true end if tname == "string" then if name == "return" then handle("return {") else handle(name .. "={") end elseif tname == "number" then if hexify then handle(format("[0x%04X]={",name)) else handle("[" .. name .. "]={") end elseif tname == "boolean" then if name then handle("return {") else handle("{") end else handle("t={") end if root then -- The dummy access will initialize a table that has a delayed initialization -- using a metatable. (maybe explicitly test for metatable) if getmetatable(root) then -- todo: make this an option, maybe even per subtable local dummy = root._w_h_a_t_e_v_e_r_ root._w_h_a_t_e_v_e_r_ = nil end -- Let's forget about empty tables. if next(root) then do_serialize(root,name,"",0) end end handle("}") end --~ name: --~ --~ true : return { } --~ false : { } --~ nil : t = { } --~ string : string = { } --~ 'return' : return { } --~ number : [number] = { } function table.serialize(root,name,specification) local t, n = { }, 0 local function flush(s) n = n + 1 t[n] = s end serialize(flush,root,name,specification) return concat(t,"\n") end table.tohandle = serialize -- sometimes tables are real use (zapfino extra pro is some 85M) in which -- case a stepwise serialization is nice; actually, we could consider: -- -- for line in table.serializer(root,name,reduce,noquotes) do -- ...(line) -- end -- -- so this is on the todo list local maxtab = 2*1024 function table.tofile(filename,root,name,specification) local f = io.open(filename,'w') if f then if maxtab > 1 then local t, n = { }, 0 local function flush(s) n = n + 1 t[n] = s if n > maxtab then f:write(concat(t,"\n"),"\n") -- hm, write(sometable) should be nice t, n = { }, 0 -- we could recycle t if needed end end serialize(flush,root,name,specification) f:write(concat(t,"\n"),"\n") else local function flush(s) f:write(s,"\n") end serialize(flush,root,name,specification) end f:close() io.flush() end end local function flattened(t,f,depth) if f == nil then f = { } depth = 0xFFFF elseif tonumber(f) then -- assume then only two arguments are given depth = f f = { } elseif not depth then depth = 0xFFFF end for k, v in next, t do if type(k) ~= "number" then if depth > 0 and type(v) == "table" then flattened(v,f,depth-1) else f[k] = v end end end local n = #f for k=1,#t do local v = t[k] if depth > 0 and type(v) == "table" then flattened(v,f,depth-1) n = #f else n = n + 1 f[n] = v end end return f end table.flattened = flattened local function unnest(t,f) -- only used in mk, for old times sake if not f then -- and only relevant for token lists f = { } end for i=1,#t do local v = t[i] if type(v) == "table" then if type(v[1]) == "table" then unnest(v,f) else f[#f+1] = v end else f[#f+1] = v end end return f end function table.unnest(t) -- bad name return unnest(t) end local function are_equal(a,b,n,m) -- indexed if a and b and #a == #b then n = n or 1 m = m or #a for i=n,m do local ai, bi = a[i], b[i] if ai==bi then -- same elseif type(ai)=="table" and type(bi)=="table" then if not are_equal(ai,bi) then return false end else return false end end return true else return false end end local function identical(a,b) -- assumes same structure for ka, va in next, a do local vb = b[ka] if va == vb then -- same elseif type(va) == "table" and type(vb) == "table" then if not identical(va,vb) then return false end else return false end end return true end table.identical = identical table.are_equal = are_equal -- maybe also make a combined one function table.compact(t) if t then for k,v in next, t do if not next(v) then t[k] = nil end end end end function table.contains(t, v) if t then for i=1, #t do if t[i] == v then return i end end end return false end function table.count(t) local n = 0 for k, v in next, t do n = n + 1 end return n end function table.swapped(t,s) -- hash local n = { } if s then --~ for i=1,#s do --~ n[i] = s[i] --~ end for k, v in next, s do n[k] = v end end --~ for i=1,#t do --~ local ti = t[i] -- don't ask but t[i] can be nil --~ if ti then --~ n[ti] = i --~ end --~ end for k, v in next, t do n[v] = k end return n end function table.reversed(t) if t then local tt, tn = { }, #t if tn > 0 then local ttn = 0 for i=tn,1,-1 do ttn = ttn + 1 tt[ttn] = t[i] end end return tt end end function table.sequenced(t,sep,simple) -- hash only local s, n = { }, 0 for k, v in sortedhash(t) do if simple then if v == true then n = n + 1 s[n] = k elseif v and v~= "" then n = n + 1 s[n] = k .. "=" .. tostring(v) end else n = n + 1 s[n] = k .. "=" .. tostring(v) end end return concat(s, sep or " | ") end function table.print(t,...) if type(t) ~= "table" then print(tostring(t)) else table.tohandle(print,t,...) end end -- -- -- obsolete but we keep them for a while and might comment them later -- -- -- -- roughly: copy-loop : unpack : sub == 0.9 : 0.4 : 0.45 (so in critical apps, use unpack) function table.sub(t,i,j) return { unpack(t,i,j) } end -- slower than #t on indexed tables (#t only returns the size of the numerically indexed slice) function table.is_empty(t) return not t or not next(t) end function table.has_one_entry(t) return t and not next(t,next(t)) end -- new function table.loweredkeys(t) -- maybe utf local l = { } for k, v in next, t do l[lower(k)] = v end return l end -- new, might move (maybe duplicate) function table.unique(old) local hash = { } local new = { } local n = 0 for i=1,#old do local oi = old[i] if not hash[oi] then n = n + 1 new[n] = oi hash[oi] = true end end return new end -- function table.sorted(t,...) -- table.sort(t,...) -- return t -- still sorts in-place -- end end -- closure do -- begin closure to overcome local limits and interference if not modules then modules = { } end modules ['l-lpeg'] = { version = 1.001, comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -- a new lpeg fails on a #(1-P(":")) test and really needs a + P(-1) local lpeg = require("lpeg") -- tracing (only used when we encounter a problem in integration of lpeg in luatex) local report = texio and texio.write_nl or print --~ local lpmatch = lpeg.match --~ local lpprint = lpeg.print --~ local lpp = lpeg.P --~ local lpr = lpeg.R --~ local lps = lpeg.S --~ local lpc = lpeg.C --~ local lpb = lpeg.B --~ local lpv = lpeg.V --~ local lpcf = lpeg.Cf --~ local lpcb = lpeg.Cb --~ local lpcg = lpeg.Cg --~ local lpct = lpeg.Ct --~ local lpcs = lpeg.Cs --~ local lpcc = lpeg.Cc --~ local lpcmt = lpeg.Cmt --~ local lpcarg = lpeg.Carg --~ function lpeg.match(l,...) report("LPEG MATCH") lpprint(l) return lpmatch(l,...) end --~ function lpeg.P (l) local p = lpp (l) report("LPEG P =") lpprint(l) return p end --~ function lpeg.R (l) local p = lpr (l) report("LPEG R =") lpprint(l) return p end --~ function lpeg.S (l) local p = lps (l) report("LPEG S =") lpprint(l) return p end --~ function lpeg.C (l) local p = lpc (l) report("LPEG C =") lpprint(l) return p end --~ function lpeg.B (l) local p = lpb (l) report("LPEG B =") lpprint(l) return p end --~ function lpeg.V (l) local p = lpv (l) report("LPEG V =") lpprint(l) return p end --~ function lpeg.Cf (l) local p = lpcf (l) report("LPEG Cf =") lpprint(l) return p end --~ function lpeg.Cb (l) local p = lpcb (l) report("LPEG Cb =") lpprint(l) return p end --~ function lpeg.Cg (l) local p = lpcg (l) report("LPEG Cg =") lpprint(l) return p end --~ function lpeg.Ct (l) local p = lpct (l) report("LPEG Ct =") lpprint(l) return p end --~ function lpeg.Cs (l) local p = lpcs (l) report("LPEG Cs =") lpprint(l) return p end --~ function lpeg.Cc (l) local p = lpcc (l) report("LPEG Cc =") lpprint(l) return p end --~ function lpeg.Cmt (l) local p = lpcmt (l) report("LPEG Cmt =") lpprint(l) return p end --~ function lpeg.Carg (l) local p = lpcarg(l) report("LPEG Carg =") lpprint(l) return p end local type = type local byte, char, gmatch = string.byte, string.char, string.gmatch -- Beware, we predefine a bunch of patterns here and one reason for doing so -- is that we get consistent behaviour in some of the visualizers. lpeg.patterns = lpeg.patterns or { } -- so that we can share local patterns = lpeg.patterns local P, R, S, V, match = lpeg.P, lpeg.R, lpeg.S, lpeg.V, lpeg.match local Ct, C, Cs, Cc = lpeg.Ct, lpeg.C, lpeg.Cs, lpeg.Cc local lpegtype = lpeg.type local utfcharacters = string.utfcharacters local utfgmatch = unicode and unicode.utf8.gmatch local anything = P(1) local endofstring = P(-1) local alwaysmatched = P(true) patterns.anything = anything patterns.endofstring = endofstring patterns.beginofstring = alwaysmatched patterns.alwaysmatched = alwaysmatched local digit, sign = R('09'), S('+-') local cr, lf, crlf = P("\r"), P("\n"), P("\r\n") local newline = crlf + S("\r\n") -- cr + lf local escaped = P("\\") * anything local squote = P("'") local dquote = P('"') local space = P(" ") local utfbom_32_be = P('\000\000\254\255') local utfbom_32_le = P('\255\254\000\000') local utfbom_16_be = P('\255\254') local utfbom_16_le = P('\254\255') local utfbom_8 = P('\239\187\191') local utfbom = utfbom_32_be + utfbom_32_le + utfbom_16_be + utfbom_16_le + utfbom_8 local utftype = utfbom_32_be / "utf-32-be" + utfbom_32_le / "utf-32-le" + utfbom_16_be / "utf-16-be" + utfbom_16_le / "utf-16-le" + utfbom_8 / "utf-8" + alwaysmatched / "unknown" local utf8next = R("\128\191") patterns.utf8one = R("\000\127") patterns.utf8two = R("\194\223") * utf8next patterns.utf8three = R("\224\239") * utf8next * utf8next patterns.utf8four = R("\240\244") * utf8next * utf8next * utf8next patterns.utfbom = utfbom patterns.utftype = utftype local utf8char = patterns.utf8one + patterns.utf8two + patterns.utf8three + patterns.utf8four local validutf8char = utf8char^0 * endofstring * Cc(true) + Cc(false) patterns.utf8 = utf8char patterns.utf8char = utf8char patterns.validutf8 = validutf8char patterns.validutf8char = validutf8char patterns.digit = digit patterns.sign = sign patterns.cardinal = sign^0 * digit^1 patterns.integer = sign^0 * digit^1 patterns.float = sign^0 * digit^0 * P('.') * digit^1 patterns.cfloat = sign^0 * digit^0 * P(',') * digit^1 patterns.number = patterns.float + patterns.integer patterns.cnumber = patterns.cfloat + patterns.integer patterns.oct = P("0") * R("07")^1 patterns.octal = patterns.oct patterns.HEX = P("0x") * R("09","AF")^1 patterns.hex = P("0x") * R("09","af")^1 patterns.hexadecimal = P("0x") * R("09","AF","af")^1 patterns.lowercase = R("az") patterns.uppercase = R("AZ") patterns.letter = patterns.lowercase + patterns.uppercase patterns.space = space patterns.tab = P("\t") patterns.spaceortab = patterns.space + patterns.tab patterns.eol = S("\n\r") patterns.spacer = S(" \t\f\v") -- + char(0xc2, 0xa0) if we want utf (cf mail roberto) patterns.newline = newline patterns.emptyline = newline^1 patterns.nonspacer = 1 - patterns.spacer patterns.whitespace = patterns.eol + patterns.spacer patterns.nonwhitespace = 1 - patterns.whitespace patterns.equal = P("=") patterns.comma = P(",") patterns.commaspacer = P(",") * patterns.spacer^0 patterns.period = P(".") patterns.colon = P(":") patterns.semicolon = P(";") patterns.underscore = P("_") patterns.escaped = escaped patterns.squote = squote patterns.dquote = dquote patterns.nosquote = (escaped + (1-squote))^0 patterns.nodquote = (escaped + (1-dquote))^0 patterns.unsingle = (squote/"") * patterns.nosquote * (squote/"") patterns.undouble = (dquote/"") * patterns.nodquote * (dquote/"") patterns.unquoted = patterns.undouble + patterns.unsingle -- more often undouble patterns.unspacer = ((patterns.spacer^1)/"")^0 patterns.somecontent = (anything - newline - space)^1 -- (utf8char - newline - space)^1 patterns.beginline = #(1-newline) -- print(string.unquoted("test")) -- print(string.unquoted([["t\"est"]])) -- print(string.unquoted([["t\"est"x]])) -- print(string.unquoted("\'test\'")) -- print(string.unquoted('"test"')) -- print(string.unquoted('"test"')) function lpeg.anywhere(pattern) --slightly adapted from website return P { P(pattern) + 1 * V(1) } -- why so complex? end function lpeg.splitter(pattern, action) return (((1-P(pattern))^1)/action+1)^0 end function lpeg.tsplitter(pattern, action) return Ct((((1-P(pattern))^1)/action+1)^0) end -- probleem: separator can be lpeg and that does not hash too well, but -- it's quite okay as the key is then not garbage collected local splitters_s, splitters_m, splitters_t = { }, { }, { } local function splitat(separator,single) local splitter = (single and splitters_s[separator]) or splitters_m[separator] if not splitter then separator = P(separator) local other = C((1 - separator)^0) if single then local any = anything splitter = other * (separator * C(any^0) + "") -- ? splitters_s[separator] = splitter else splitter = other * (separator * other)^0 splitters_m[separator] = splitter end end return splitter end local function tsplitat(separator) local splitter = splitters_t[separator] if not splitter then splitter = Ct(splitat(separator)) splitters_t[separator] = splitter end return splitter end lpeg.splitat = splitat lpeg.tsplitat = tsplitat function string.splitup(str,separator) if not separator then separator = "," end return match(splitters_m[separator] or splitat(separator),str) end --~ local p = splitat("->",false) print(match(p,"oeps->what->more")) -- oeps what more --~ local p = splitat("->",true) print(match(p,"oeps->what->more")) -- oeps what->more --~ local p = splitat("->",false) print(match(p,"oeps")) -- oeps --~ local p = splitat("->",true) print(match(p,"oeps")) -- oeps local cache = { } function lpeg.split(separator,str) local c = cache[separator] if not c then c = tsplitat(separator) cache[separator] = c end return match(c,str) end function string.split(str,separator) local c = cache[separator] if not c then c = tsplitat(separator) cache[separator] = c end return match(c,str) end local spacing = patterns.spacer^0 * newline -- sort of strip local empty = spacing * Cc("") local nonempty = Cs((1-spacing)^1) * spacing^-1 local content = (empty + nonempty)^1 patterns.textline = content --~ local linesplitter = Ct(content^0) --~ --~ function string.splitlines(str) --~ return match(linesplitter,str) --~ end local linesplitter = tsplitat(newline) patterns.linesplitter = linesplitter function string.splitlines(str) return match(linesplitter,str) end local utflinesplitter = utfbom^-1 * tsplitat(newline) patterns.utflinesplitter = utflinesplitter function string.utfsplitlines(str) return match(utflinesplitter,str or "") end --~ lpeg.splitters = cache -- no longer public local cache = { } function lpeg.checkedsplit(separator,str) local c = cache[separator] if not c then separator = P(separator) local other = C((1 - separator)^1) c = Ct(separator^0 * other * (separator^1 * other)^0) cache[separator] = c end return match(c,str) end function string.checkedsplit(str,separator) local c = cache[separator] if not c then separator = P(separator) local other = C((1 - separator)^1) c = Ct(separator^0 * other * (separator^1 * other)^0) cache[separator] = c end return match(c,str) end --~ from roberto's site: local function f2(s) local c1, c2 = byte(s,1,2) return c1 * 64 + c2 - 12416 end local function f3(s) local c1, c2, c3 = byte(s,1,3) return (c1 * 64 + c2) * 64 + c3 - 925824 end local function f4(s) local c1, c2, c3, c4 = byte(s,1,4) return ((c1 * 64 + c2) * 64 + c3) * 64 + c4 - 63447168 end local utf8byte = patterns.utf8one/byte + patterns.utf8two/f2 + patterns.utf8three/f3 + patterns.utf8four/f4 patterns.utf8byte = utf8byte --~ local str = " a b c d " --~ local s = lpeg.stripper(lpeg.R("az")) print("["..lpeg.match(s,str).."]") --~ local s = lpeg.keeper(lpeg.R("az")) print("["..lpeg.match(s,str).."]") --~ local s = lpeg.stripper("ab") print("["..lpeg.match(s,str).."]") --~ local s = lpeg.keeper("ab") print("["..lpeg.match(s,str).."]") local cache = { } function lpeg.stripper(str) if type(str) == "string" then local s = cache[str] if not s then s = Cs(((S(str)^1)/"" + 1)^0) cache[str] = s end return s else return Cs(((str^1)/"" + 1)^0) end end local cache = { } function lpeg.keeper(str) if type(str) == "string" then local s = cache[str] if not s then s = Cs((((1-S(str))^1)/"" + 1)^0) cache[str] = s end return s else return Cs((((1-str)^1)/"" + 1)^0) end end function lpeg.frontstripper(str) -- or pattern (yet undocumented) return (P(str) + P(true)) * Cs(P(1)^0) end function lpeg.endstripper(str) -- or pattern (yet undocumented) return Cs((1 - P(str) * P(-1))^0) end -- Just for fun I looked at the used bytecode and -- p = (p and p + pp) or pp gets one more (testset). function lpeg.replacer(one,two) if type(one) == "table" then local no = #one if no > 0 then local p for i=1,no do local o = one[i] local pp = P(o[1]) / o[2] if p then p = p + pp else p = pp end end return Cs((p + 1)^0) end else two = two or "" return Cs((P(one)/two + 1)^0) end end local splitters_f, splitters_s = { }, { } function lpeg.firstofsplit(separator) -- always return value local splitter = splitters_f[separator] if not splitter then separator = P(separator) splitter = C((1 - separator)^0) splitters_f[separator] = splitter end return splitter end function lpeg.secondofsplit(separator) -- nil if not split local splitter = splitters_s[separator] if not splitter then separator = P(separator) splitter = (1 - separator)^0 * separator * C(anything^0) splitters_s[separator] = splitter end return splitter end function lpeg.balancer(left,right) left, right = P(left), P(right) return P { left * ((1 - left - right) + V(1))^0 * right } end --~ print(1,match(lpeg.firstofsplit(":"),"bc:de")) --~ print(2,match(lpeg.firstofsplit(":"),":de")) -- empty --~ print(3,match(lpeg.firstofsplit(":"),"bc")) --~ print(4,match(lpeg.secondofsplit(":"),"bc:de")) --~ print(5,match(lpeg.secondofsplit(":"),"bc:")) -- empty --~ print(6,match(lpeg.secondofsplit(":",""),"bc")) --~ print(7,match(lpeg.secondofsplit(":"),"bc")) --~ print(9,match(lpeg.secondofsplit(":","123"),"bc")) --~ -- slower: --~ --~ function lpeg.counter(pattern) --~ local n, pattern = 0, (lpeg.P(pattern)/function() n = n + 1 end + lpeg.anything)^0 --~ return function(str) n = 0 ; lpegmatch(pattern,str) ; return n end --~ end local nany = utf8char/"" function lpeg.counter(pattern) pattern = Cs((P(pattern)/" " + nany)^0) return function(str) return #match(pattern,str) end end if utfgmatch then function lpeg.count(str,what) -- replaces string.count if type(what) == "string" then local n = 0 for _ in utfgmatch(str,what) do n = n + 1 end return n else -- 4 times slower but still faster than / function return #match(Cs((P(what)/" " + nany)^0),str) end end else local cache = { } function lpeg.count(str,what) -- replaces string.count if type(what) == "string" then local p = cache[what] if not p then p = Cs((P(what)/" " + nany)^0) cache[p] = p end return #match(p,str) else -- 4 times slower but still faster than / function return #match(Cs((P(what)/" " + nany)^0),str) end end end local patterns_escapes = { -- also defines in l-string ["%"] = "%%", ["."] = "%.", ["+"] = "%+", ["-"] = "%-", ["*"] = "%*", ["["] = "%[", ["]"] = "%]", ["("] = "%)", [")"] = "%)", -- ["{"] = "%{", ["}"] = "%}" -- ["^"] = "%^", ["$"] = "%$", } local simple_escapes = { -- also defines in l-string ["-"] = "%-", ["."] = "%.", ["?"] = ".", ["*"] = ".*", } local p = Cs((S("-.+*%()[]") / patterns_escapes + anything)^0) local s = Cs((S("-.+*%()[]") / simple_escapes + anything)^0) function string.escapedpattern(str,simple) return match(simple and s or p,str) end -- utf extensies lpeg.UP = lpeg.P if utfcharacters then function lpeg.US(str) local p for uc in utfcharacters(str) do if p then p = p + P(uc) else p = P(uc) end end return p end elseif utfgmatch then function lpeg.US(str) local p for uc in utfgmatch(str,".") do if p then p = p + P(uc) else p = P(uc) end end return p end else function lpeg.US(str) local p local f = function(uc) if p then p = p + P(uc) else p = P(uc) end end match((utf8char/f)^0,str) return p end end local range = Cs(utf8byte) * (Cs(utf8byte) + Cc(false)) local utfchar = unicode and unicode.utf8 and unicode.utf8.char function lpeg.UR(str,more) local first, last if type(str) == "number" then first = str last = more or first else first, last = match(range,str) if not last then return P(str) end end if first == last then return P(str) elseif utfchar and last - first < 8 then -- a somewhat arbitrary criterium local p for i=first,last do if p then p = p + P(utfchar(i)) else p = P(utfchar(i)) end end return p -- nil when invalid range else local f = function(b) return b >= first and b <= last end return utf8byte / f -- nil when invalid range end end --~ lpeg.print(lpeg.R("ab","cd","gh")) --~ lpeg.print(lpeg.P("a","b","c")) --~ lpeg.print(lpeg.S("a","b","c")) --~ print(lpeg.count("äáàa",lpeg.P("á") + lpeg.P("à"))) --~ print(lpeg.count("äáàa",lpeg.UP("áà"))) --~ print(lpeg.count("äáàa",lpeg.US("àá"))) --~ print(lpeg.count("äáàa",lpeg.UR("aá"))) --~ print(lpeg.count("äáàa",lpeg.UR("àá"))) --~ print(lpeg.count("äáàa",lpeg.UR(0x0000,0xFFFF))) function lpeg.oneof(list,...) -- lpeg.oneof("elseif","else","if","then") if type(list) ~= "table" then list = { list, ... } end -- sort(list) -- longest match first local p = P(list[1]) for l=2,#list do p = p + P(list[l]) end return p end function lpeg.is_lpeg(p) return p and lpegtype(p) == "pattern" end -- For the moment here, but it might move to utilities. Beware, we need to -- have the longest keyword first, so 'aaa' comes beforte 'aa' which is why we -- loop back from the end cq. prepend. local sort, fastcopy, sortedkeys = table.sort, table.fastcopy, table.sortedkeys -- dependency! function lpeg.append(list,pp,delayed,checked) local p = pp if #list > 0 then local keys = fastcopy(list) sort(keys) for i=#keys,1,-1 do local k = keys[i] if p then p = P(k) + p else p = P(k) end end elseif delayed then -- hm, it looks like the lpeg parser resolves anyway local keys = sortedkeys(list) if p then for i=1,#keys,1 do local k = keys[i] local v = list[k] p = P(k)/list + p end else for i=1,#keys do local k = keys[i] local v = list[k] if p then p = P(k) + p else p = P(k) end end if p then p = p / list end end elseif checked then -- problem: substitution gives a capture local keys = sortedkeys(list) for i=1,#keys do local k = keys[i] local v = list[k] if p then if k == v then p = P(k) + p else p = P(k)/v + p end else if k == v then p = P(k) else p = P(k)/v end end end else local keys = sortedkeys(list) for i=1,#keys do local k = keys[i] local v = list[k] if p then p = P(k)/v + p else p = P(k)/v end end end return p end -- inspect(lpeg.append({ a = "1", aa = "1", aaa = "1" } ,nil,true)) -- inspect(lpeg.append({ ["degree celsius"] = "1", celsius = "1", degree = "1" } ,nil,true)) -- function lpeg.exact_match(words,case_insensitive) -- local pattern = concat(words) -- if case_insensitive then -- local pattern = S(upper(characters)) + S(lower(characters)) -- local list = { } -- for i=1,#words do -- list[lower(words[i])] = true -- end -- return Cmt(pattern^1, function(_,i,s) -- return list[lower(s)] and i -- end) -- else -- local pattern = S(concat(words)) -- local list = { } -- for i=1,#words do -- list[words[i]] = true -- end -- return Cmt(pattern^1, function(_,i,s) -- return list[s] and i -- end) -- end -- end -- experiment: local function make(t) local p -- for k, v in next, t do for k, v in table.sortedhash(t) do if not p then if next(v) then p = P(k) * make(v) else p = P(k) end else if next(v) then p = p + P(k) * make(v) else p = p + P(k) end end end return p end function lpeg.utfchartabletopattern(list) local tree = { } for i=1,#list do local t = tree for c in gmatch(list[i],".") do if not t[c] then t[c] = { } end t = t[c] end end return make(tree) end -- inspect ( lpeg.utfchartabletopattern { -- utfchar(0x00A0), -- nbsp -- utfchar(0x2000), -- enquad -- utfchar(0x2001), -- emquad -- utfchar(0x2002), -- enspace -- utfchar(0x2003), -- emspace -- utfchar(0x2004), -- threeperemspace -- utfchar(0x2005), -- fourperemspace -- utfchar(0x2006), -- sixperemspace -- utfchar(0x2007), -- figurespace -- utfchar(0x2008), -- punctuationspace -- utfchar(0x2009), -- breakablethinspace -- utfchar(0x200A), -- hairspace -- utfchar(0x200B), -- zerowidthspace -- utfchar(0x202F), -- narrownobreakspace -- utfchar(0x205F), -- math thinspace -- } ) end -- closure do -- begin closure to overcome local limits and interference if not modules then modules = { } end modules ['l-boolean'] = { version = 1.001, comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } local type, tonumber = type, tonumber boolean = boolean or { } local boolean = boolean function boolean.tonumber(b) if b then return 1 else return 0 end -- test and return or return end function toboolean(str,tolerant) if tolerant then local tstr = type(str) if tstr == "string" then return str == "true" or str == "yes" or str == "on" or str == "1" or str == "t" elseif tstr == "number" then return tonumber(str) ~= 0 elseif tstr == "nil" then return false else return str end elseif str == "true" then return true elseif str == "false" then return false else return str end end string.toboolean = toboolean function string.is_boolean(str,default) if type(str) == "string" then if str == "true" or str == "yes" or str == "on" or str == "t" then return true elseif str == "false" or str == "no" or str == "off" or str == "f" then return false end end return default end end -- closure do -- begin closure to overcome local limits and interference if not modules then modules = { } end modules ['l-math'] = { version = 1.001, comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } local floor, sin, cos, tan = math.floor, math.sin, math.cos, math.tan if not math.round then function math.round(x) return floor(x + 0.5) end end if not math.div then function math.div(n,m) return floor(n/m) end end if not math.mod then function math.mod(n,m) return n % m end end local pipi = 2*math.pi/360 if not math.sind then function math.sind(d) return sin(d*pipi) end function math.cosd(d) return cos(d*pipi) end function math.tand(d) return tan(d*pipi) end end if not math.odd then function math.odd (n) return n % 2 ~= 0 end function math.even(n) return n % 2 == 0 end end end -- closure do -- begin closure to overcome local limits and interference if not modules then modules = { } end modules ['l-file'] = { version = 1.001, comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -- needs a cleanup file = file or { } local file = file local insert, concat = table.insert, table.concat local find, gmatch, match, gsub, sub, char, lower = string.find, string.gmatch, string.match, string.gsub, string.sub, string.char, string.lower local lpegmatch = lpeg.match local getcurrentdir, attributes = lfs.currentdir, lfs.attributes local P, R, S, C, Cs, Cp, Cc = lpeg.P, lpeg.R, lpeg.S, lpeg.C, lpeg.Cs, lpeg.Cp, lpeg.Cc local function dirname(name,default) return match(name,"^(.+)[/\\].-$") or (default or "") end local function basename(name) return match(name,"^.+[/\\](.-)$") or name end local function nameonly(name) return (gsub(match(name,"^.+[/\\](.-)$") or name,"%..*$","")) end local function extname(name,default) return match(name,"^.+%.([^/\\]-)$") or default or "" end local function splitname(name) local n, s = match(name,"^(.+)%.([^/\\]-)$") return n or name, s or "" end file.basename = basename file.dirname = dirname file.nameonly = nameonly file.extname = extname file.suffix = extname function file.removesuffix(filename) return (gsub(filename,"%.[%a%d]+$","")) end function file.addsuffix(filename, suffix, criterium) if not suffix or suffix == "" then return filename elseif criterium == true then return filename .. "." .. suffix elseif not criterium then local n, s = splitname(filename) if not s or s == "" then return filename .. "." .. suffix else return filename end else local n, s = splitname(filename) if s and s ~= "" then local t = type(criterium) if t == "table" then -- keep if in criterium for i=1,#criterium do if s == criterium[i] then return filename end end elseif t == "string" then -- keep if criterium if s == criterium then return filename end end end return n .. "." .. suffix end end --~ print("1 " .. file.addsuffix("name","new") .. " -> name.new") --~ print("2 " .. file.addsuffix("name.old","new") .. " -> name.old") --~ print("3 " .. file.addsuffix("name.old","new",true) .. " -> name.old.new") --~ print("4 " .. file.addsuffix("name.old","new","new") .. " -> name.new") --~ print("5 " .. file.addsuffix("name.old","new","old") .. " -> name.old") --~ print("6 " .. file.addsuffix("name.old","new","foo") .. " -> name.new") --~ print("7 " .. file.addsuffix("name.old","new",{"foo","bar"}) .. " -> name.new") --~ print("8 " .. file.addsuffix("name.old","new",{"old","bar"}) .. " -> name.old") function file.replacesuffix(filename, suffix) return (gsub(filename,"%.[%a%d]+$","")) .. "." .. suffix end --~ function file.join(...) --~ local pth = concat({...},"/") --~ pth = gsub(pth,"\\","/") --~ local a, b = match(pth,"^(.*://)(.*)$") --~ if a and b then --~ return a .. gsub(b,"//+","/") --~ end --~ a, b = match(pth,"^(//)(.*)$") --~ if a and b then --~ return a .. gsub(b,"//+","/") --~ end --~ return (gsub(pth,"//+","/")) --~ end local trick_1 = char(1) local trick_2 = "^" .. trick_1 .. "/+" function file.join(...) -- rather dirty local lst = { ... } local a, b = lst[1], lst[2] if not a or a == "" then -- not a added lst[1] = trick_1 elseif b and find(a,"^/+$") and find(b,"^/") then lst[1] = "" lst[2] = gsub(b,"^/+","") end local pth = concat(lst,"/") pth = gsub(pth,"\\","/") local a, b = match(pth,"^(.*://)(.*)$") if a and b then return a .. gsub(b,"//+","/") end a, b = match(pth,"^(//)(.*)$") if a and b then return a .. gsub(b,"//+","/") end pth = gsub(pth,trick_2,"") return (gsub(pth,"//+","/")) end --~ print(file.join("//","/y")) --~ print(file.join("/","/y")) --~ print(file.join("","/y")) --~ print(file.join("/x/","/y")) --~ print(file.join("x/","/y")) --~ print(file.join("http://","/y")) --~ print(file.join("http://a","/y")) --~ print(file.join("http:///a","/y")) --~ print(file.join("//nas-1","/y")) -- We should be able to use: -- -- function file.is_writable(name) -- local a = attributes(name) or attributes(dirname(name,".")) -- return a and sub(a.permissions,2,2) == "w" -- end -- -- But after some testing Taco and I came up with: function file.is_writable(name) if lfs.isdir(name) then name = name .. "/m_t_x_t_e_s_t.tmp" local f = io.open(name,"wb") if f then f:close() os.remove(name) return true end elseif lfs.isfile(name) then local f = io.open(name,"ab") if f then f:close() return true end else local f = io.open(name,"ab") if f then f:close() os.remove(name) return true end end return false end function file.is_readable(name) local a = attributes(name) return a and sub(a.permissions,1,1) == "r" end file.isreadable = file.is_readable -- depricated file.iswritable = file.is_writable -- depricated -- todo: lpeg \\ / .. does not save much local checkedsplit = string.checkedsplit function file.splitpath(str,separator) -- string str = gsub(str,"\\","/") return checkedsplit(str,separator or io.pathseparator) end function file.joinpath(tab,separator) -- table return concat(tab,separator or io.pathseparator) -- can have trailing // end -- we can hash them weakly --~ function file.collapsepath(str) -- fails on b.c/.. --~ str = gsub(str,"\\","/") --~ if find(str,"/") then --~ str = gsub(str,"^%./",(gsub(getcurrentdir(),"\\","/")) .. "/") -- ./xx in qualified --~ str = gsub(str,"/%./","/") --~ local n, m = 1, 1 --~ while n > 0 or m > 0 do --~ str, n = gsub(str,"[^/%.]+/%.%.$","") --~ str, m = gsub(str,"[^/%.]+/%.%./","") --~ end --~ str = gsub(str,"([^/])/$","%1") --~ -- str = gsub(str,"^%./","") -- ./xx in qualified --~ str = gsub(str,"/%.$","") --~ end --~ if str == "" then str = "." end --~ return str --~ end --~ --~ The previous one fails on "a.b/c" so Taco came up with a split based --~ variant. After some skyping we got it sort of compatible with the old --~ one. After that the anchoring to currentdir was added in a better way. --~ Of course there are some optimizations too. Finally we had to deal with --~ windows drive prefixes and things like sys://. function file.collapsepath(str,anchor) if anchor and not find(str,"^/") and not find(str,"^%a:") then str = getcurrentdir() .. "/" .. str end if str == "" or str =="." then return "." elseif find(str,"^%.%.") then str = gsub(str,"\\","/") return str elseif not find(str,"%.") then str = gsub(str,"\\","/") return str end str = gsub(str,"\\","/") local starter, rest = match(str,"^(%a+:/*)(.-)$") if starter then str = rest end local oldelements = checkedsplit(str,"/") local newelements = { } local i = #oldelements while i > 0 do local element = oldelements[i] if element == '.' then -- do nothing elseif element == '..' then local n = i - 1 while n > 0 do local element = oldelements[n] if element ~= '..' and element ~= '.' then oldelements[n] = '.' break else n = n - 1 end end if n < 1 then insert(newelements,1,'..') end elseif element ~= "" then insert(newelements,1,element) end i = i - 1 end if #newelements == 0 then return starter or "." elseif starter then return starter .. concat(newelements, '/') elseif find(str,"^/") then return "/" .. concat(newelements,'/') else return concat(newelements, '/') end end --~ local function test(str) --~ print(string.format("%-20s %-15s %-15s",str,file.collapsepath(str),file.collapsepath(str,true))) --~ end --~ test("a/b.c/d") test("b.c/d") test("b.c/..") --~ test("/") test("c:/..") test("sys://..") --~ test("") test("./") test(".") test("..") test("./..") test("../..") --~ test("a") test("./a") test("/a") test("a/../..") --~ test("a/./b/..") test("a/aa/../b/bb") test("a/.././././b/..") test("a/./././b/..") --~ test("a/b/c/../..") test("./a/b/c/../..") test("a/b/c/../..") function file.robustname(str,strict) str = gsub(str,"[^%a%d%/%-%.\\]+","-") if strict then return lower(gsub(str,"^%-*(.-)%-*$","%1")) else return str end end file.readdata = io.loaddata file.savedata = io.savedata function file.copy(oldname,newname) file.savedata(newname,io.loaddata(oldname)) end -- lpeg variants, slightly faster, not always --~ local period = P(".") --~ local slashes = S("\\/") --~ local noperiod = 1-period --~ local noslashes = 1-slashes --~ local name = noperiod^1 --~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * C(noperiod^1) * -1 --~ function file.extname(name) --~ return lpegmatch(pattern,name) or "" --~ end --~ local pattern = Cs(((period * noperiod^1 * -1)/"" + 1)^1) --~ function file.removesuffix(name) --~ return lpegmatch(pattern,name) --~ end --~ local pattern = (noslashes^0 * slashes)^1 * C(noslashes^1) * -1 --~ function file.basename(name) --~ return lpegmatch(pattern,name) or name --~ end --~ local pattern = (noslashes^0 * slashes)^1 * Cp() * noslashes^1 * -1 --~ function file.dirname(name) --~ local p = lpegmatch(pattern,name) --~ if p then --~ return sub(name,1,p-2) --~ else --~ return "" --~ end --~ end --~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * Cp() * noperiod^1 * -1 --~ function file.addsuffix(name, suffix) --~ local p = lpegmatch(pattern,name) --~ if p then --~ return name --~ else --~ return name .. "." .. suffix --~ end --~ end --~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * Cp() * noperiod^1 * -1 --~ function file.replacesuffix(name,suffix) --~ local p = lpegmatch(pattern,name) --~ if p then --~ return sub(name,1,p-2) .. "." .. suffix --~ else --~ return name .. "." .. suffix --~ end --~ end --~ local pattern = (noslashes^0 * slashes)^0 * Cp() * ((noperiod^1 * period)^1 * Cp() + P(true)) * noperiod^1 * -1 --~ function file.nameonly(name) --~ local a, b = lpegmatch(pattern,name) --~ if b then --~ return sub(name,a,b-2) --~ elseif a then --~ return sub(name,a) --~ else --~ return name --~ end --~ end --~ local test = file.extname --~ local test = file.basename --~ local test = file.dirname --~ local test = file.addsuffix --~ local test = file.replacesuffix --~ local test = file.nameonly --~ print(1,test("./a/b/c/abd.def.xxx","!!!")) --~ print(2,test("./../b/c/abd.def.xxx","!!!")) --~ print(3,test("a/b/c/abd.def.xxx","!!!")) --~ print(4,test("a/b/c/def.xxx","!!!")) --~ print(5,test("a/b/c/def","!!!")) --~ print(6,test("def","!!!")) --~ print(7,test("def.xxx","!!!")) --~ local tim = os.clock() for i=1,250000 do local ext = test("abd.def.xxx","!!!") end print(os.clock()-tim) -- also rewrite previous local letter = R("az","AZ") + S("_-+") local separator = P("://") local qualified = P(".")^0 * P("/") + letter*P(":") + letter^1*separator + letter^1 * P("/") local rootbased = P("/") + letter*P(":") lpeg.patterns.qualified = qualified lpeg.patterns.rootbased = rootbased -- ./name ../name /name c: :// name/name function file.is_qualified_path(filename) return lpegmatch(qualified,filename) ~= nil end function file.is_rootbased_path(filename) return lpegmatch(rootbased,filename) ~= nil end -- actually these are schemes local slash = S("\\/") local period = P(".") local drive = C(R("az","AZ")) * P(":") local path = C(((1-slash)^0 * slash)^0) local suffix = period * C(P(1-period)^0 * P(-1)) local base = C((1-suffix)^0) drive = drive + Cc("") path = path + Cc("") base = base + Cc("") suffix = suffix + Cc("") local pattern_a = drive * path * base * suffix local pattern_b = path * base * suffix local pattern_c = C(drive * path) * C(base * suffix) function file.splitname(str,splitdrive) if splitdrive then return lpegmatch(pattern_a,str) -- returns drive, path, base, suffix else return lpegmatch(pattern_b,str) -- returns path, base, suffix end end function file.nametotable(str,splitdrive) -- returns table local path, drive, subpath, name, base, suffix = lpegmatch(pattern_c,str) if splitdrive then return { path = path, drive = drive, subpath = subpath, name = name, base = base, suffix = suffix, } else return { path = path, name = name, base = base, suffix = suffix, } end end -- function test(t) for k, v in next, t do print(v, "=>", file.splitname(v)) end end -- -- test { "c:", "c:/aa", "c:/aa/bb", "c:/aa/bb/cc", "c:/aa/bb/cc.dd", "c:/aa/bb/cc.dd.ee" } -- test { "c:", "c:aa", "c:aa/bb", "c:aa/bb/cc", "c:aa/bb/cc.dd", "c:aa/bb/cc.dd.ee" } -- test { "/aa", "/aa/bb", "/aa/bb/cc", "/aa/bb/cc.dd", "/aa/bb/cc.dd.ee" } -- test { "aa", "aa/bb", "aa/bb/cc", "aa/bb/cc.dd", "aa/bb/cc.dd.ee" } --~ -- todo: --~ --~ if os.type == "windows" then --~ local currentdir = getcurrentdir --~ function getcurrentdir() --~ return (gsub(currentdir(),"\\","/")) --~ end --~ end -- for myself: function file.strip(name,dir) local b, a = match(name,"^(.-)" .. dir .. "(.*)$") return a ~= "" and a or name end end -- closure do -- begin closure to overcome local limits and interference if not modules then modules = { } end modules ['l-io'] = { version = 1.001, comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } local io = io local byte, find, gsub, format = string.byte, string.find, string.gsub, string.format local concat = table.concat local type = type if string.find(os.getenv("PATH"),";") then io.fileseparator, io.pathseparator = "\\", ";" else io.fileseparator, io.pathseparator = "/" , ":" end function io.loaddata(filename,textmode) local f = io.open(filename,(textmode and 'r') or 'rb') if f then local data = f:read('*all') f:close() return data else return nil end end function io.savedata(filename,data,joiner) local f = io.open(filename,"wb") if f then if type(data) == "table" then f:write(concat(data,joiner or "")) elseif type(data) == "function" then data(f) else f:write(data or "") end f:close() io.flush() return true else return false end end function io.exists(filename) local f = io.open(filename) if f == nil then return false else assert(f:close()) return true end end function io.size(filename) local f = io.open(filename) if f == nil then return 0 else local s = f:seek("end") assert(f:close()) return s end end function io.noflines(f) if type(f) == "string" then local f = io.open(filename) local n = f and io.noflines(f) or 0 assert(f:close()) return n else local n = 0 for _ in f:lines() do n = n + 1 end f:seek('set',0) return n end end local nextchar = { [ 4] = function(f) return f:read(1,1,1,1) end, [ 2] = function(f) return f:read(1,1) end, [ 1] = function(f) return f:read(1) end, [-2] = function(f) local a, b = f:read(1,1) return b, a end, [-4] = function(f) local a, b, c, d = f:read(1,1,1,1) return d, c, b, a end } function io.characters(f,n) if f then return nextchar[n or 1], f end end local nextbyte = { [4] = function(f) local a, b, c, d = f:read(1,1,1,1) if d then return byte(a), byte(b), byte(c), byte(d) end end, [3] = function(f) local a, b, c = f:read(1,1,1) if b then return byte(a), byte(b), byte(c) end end, [2] = function(f) local a, b = f:read(1,1) if b then return byte(a), byte(b) end end, [1] = function (f) local a = f:read(1) if a then return byte(a) end end, [-2] = function (f) local a, b = f:read(1,1) if b then return byte(b), byte(a) end end, [-3] = function(f) local a, b, c = f:read(1,1,1) if b then return byte(c), byte(b), byte(a) end end, [-4] = function(f) local a, b, c, d = f:read(1,1,1,1) if d then return byte(d), byte(c), byte(b), byte(a) end end } function io.bytes(f,n) if f then return nextbyte[n or 1], f else return nil, nil end end function io.ask(question,default,options) while true do io.write(question) if options then io.write(format(" [%s]",concat(options,"|"))) end if default then io.write(format(" [%s]",default)) end io.write(format(" ")) io.flush() local answer = io.read() answer = gsub(answer,"^%s*(.*)%s*$","%1") if answer == "" and default then return default elseif not options then return answer else for k=1,#options do if options[k] == answer then return answer end end local pattern = "^" .. answer for k=1,#options do local v = options[k] if find(v,pattern) then return v end end end end end local function readnumber(f,n,m) if m then f:seek("set",n) n = m end if n == 1 then return byte(f:read(1)) elseif n == 2 then local a, b = byte(f:read(2),1,2) return 256 * a + b elseif n == 3 then local a, b, c = byte(f:read(3),1,3) return 256*256 * a + 256 * b + c elseif n == 4 then local a, b, c, d = byte(f:read(4),1,4) return 256*256*256 * a + 256*256 * b + 256 * c + d elseif n == 8 then local a, b = readnumber(f,4), readnumber(f,4) return 256 * a + b elseif n == 12 then local a, b, c = readnumber(f,4), readnumber(f,4), readnumber(f,4) return 256*256 * a + 256 * b + c elseif n == -2 then local b, a = byte(f:read(2),1,2) return 256*a + b elseif n == -3 then local c, b, a = byte(f:read(3),1,3) return 256*256 * a + 256 * b + c elseif n == -4 then local d, c, b, a = byte(f:read(4),1,4) return 256*256*256 * a + 256*256 * b + 256*c + d elseif n == -8 then local h, g, f, e, d, c, b, a = byte(f:read(8),1,8) return 256*256*256*256*256*256*256 * a + 256*256*256*256*256*256 * b + 256*256*256*256*256 * c + 256*256*256*256 * d + 256*256*256 * e + 256*256 * f + 256 * g + h else return 0 end end io.readnumber = readnumber function io.readstring(f,n,m) if m then f:seek("set",n) n = m end local str = gsub(f:read(n),"%z","") return str end -- if not io.i_limiter then function io.i_limiter() end end -- dummy so we can test safely if not io.o_limiter then function io.o_limiter() end end -- dummy so we can test safely end -- closure do -- begin closure to overcome local limits and interference if not modules then modules = { } end modules ['luat-basics-gen'] = { version = 1.100, comment = "companion to luatex-*.tex", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } if context then texio.write_nl("fatal error: this module is not for context") os.exit() end local dummyfunction = function() end local dummyreporter = function(c) return function(...) texio.write(c .. " : " .. string.format(...)) end end statistics = { register = dummyfunction, starttiming = dummyfunction, stoptiming = dummyfunction, elapsedtime = nil, } directives = { register = dummyfunction, enable = dummyfunction, disable = dummyfunction, } trackers = { register = dummyfunction, enable = dummyfunction, disable = dummyfunction, } experiments = { register = dummyfunction, enable = dummyfunction, disable = dummyfunction, } storage = { -- probably no longer needed register = dummyfunction, shared = { }, } logs = { new = dummyreporter, reporter = dummyreporter, messenger = dummyreporter, report = dummyfunction, } callbacks = { register = function(n,f) return callback.register(n,f) end, } utilities = { storage = { allocate = function(t) return t or { } end, mark = function(t) return t or { } end, }, } characters = characters or { data = { } } -- we need to cheat a bit here texconfig.kpse_init = true resolvers = resolvers or { } -- no fancy file helpers used local remapper = { otf = "opentype fonts", ttf = "truetype fonts", ttc = "truetype fonts", dfont = "truetype fonts", -- "truetype dictionary", cid = "cid maps", fea = "font feature files", pfa = "type1 fonts", -- this is for Khaled, in ConTeXt we don't use this! pfb = "type1 fonts", -- this is for Khaled, in ConTeXt we don't use this! } function resolvers.findfile(name,fileformat) name = string.gsub(name,"\\","\/") fileformat = fileformat and string.lower(fileformat) local found = kpse.find_file(name,(fileformat and fileformat ~= "" and (remapper[fileformat] or fileformat)) or file.extname(name,"tex")) if not found or found == "" then found = kpse.find_file(name,"other text files") end return found end function resolvers.findbinfile(name,fileformat) if not fileformat or fileformat == "" then fileformat = file.extname(name) -- string.match(name,"%.([^%.]-)$") end return resolvers.findfile(name,(fileformat and remapper[fileformat]) or fileformat) end function resolvers.resolve(s) return s end function resolvers.unresolve(s) return s end -- Caches ... I will make a real stupid version some day when I'm in the -- mood. After all, the generic code does not need the more advanced -- ConTeXt features. Cached data is not shared between ConTeXt and other -- usage as I don't want any dependency at all. Also, ConTeXt might have -- different needs and tricks added. --~ containers.usecache = true caches = { } local writable, readables = nil, { } if not caches.namespace or caches.namespace == "" or caches.namespace == "context" then caches.namespace = 'generic' end do local cachepaths = kpse.expand_path('$TEXMFCACHE') or "" if cachepaths == "" then cachepaths = kpse.expand_path('$TEXMFVAR') end if cachepaths == "" then cachepaths = kpse.expand_path('$VARTEXMF') end if cachepaths == "" then cachepaths = "." end cachepaths = string.split(cachepaths,os.type == "windows" and ";" or ":") for i=1,#cachepaths do if file.is_writable(cachepaths[i]) then writable = file.join(cachepaths[i],"luatex-cache") lfs.mkdir(writable) writable = file.join(writable,caches.namespace) lfs.mkdir(writable) break end end for i=1,#cachepaths do if file.is_readable(cachepaths[i]) then readables[#readables+1] = file.join(cachepaths[i],"luatex-cache",caches.namespace) end end if not writable then texio.write_nl("quiting: fix your writable cache path") os.exit() elseif #readables == 0 then texio.write_nl("quiting: fix your readable cache path") os.exit() elseif #readables == 1 and readables[1] == writable then texio.write(string.format("(using cache: %s)",writable)) else texio.write(string.format("(using write cache: %s)",writable)) texio.write(string.format("(using read cache: %s)",table.concat(readables, " "))) end end function caches.getwritablepath(category,subcategory) local path = file.join(writable,category) lfs.mkdir(path) path = file.join(path,subcategory) lfs.mkdir(path) return path end function caches.getreadablepaths(category,subcategory) local t = { } for i=1,#readables do t[i] = file.join(readables[i],category,subcategory) end return t end local function makefullname(path,name) if path and path ~= "" then name = "temp-" .. name -- clash prevention return file.addsuffix(file.join(path,name),"lua"), file.addsuffix(file.join(path,name),"luc") end end function caches.is_writable(path,name) local fullname = makefullname(path,name) return fullname and file.is_writable(fullname) end function caches.loaddata(paths,name) for i=1,#paths do local data = false local luaname, lucname = makefullname(paths[i],name) if lucname and lfs.isfile(lucname) then texio.write(string.format("(load: %s)",lucname)) data = loadfile(lucname) end if not data and luaname and lfs.isfile(luaname) then texio.write(string.format("(load: %s)",luaname)) data = loadfile(luaname) end return data and data() end end function caches.savedata(path,name,data) local luaname, lucname = makefullname(path,name) if luaname then texio.write(string.format("(save: %s)",luaname)) table.tofile(luaname,data,true,{ reduce = true }) if lucname and type(caches.compile) == "function" then os.remove(lucname) -- better be safe texio.write(string.format("(save: %s)",lucname)) caches.compile(data,luaname,lucname) end end end -- According to KH os.execute is not permitted in plain/latex so there is -- no reason to use the normal context way. So the method here is slightly -- different from the one we have in context. We also use different suffixes -- as we don't want any clashes (sharing cache files is not that handy as -- context moves on faster.) -- -- Beware: serialization might fail on large files (so maybe we should pcall -- this) in which case one should limit the method to luac and enable support -- for execution. caches.compilemethod = "luac" -- luac dump both function caches.compile(data,luaname,lucname) local done = false if caches.compilemethod == "luac" or caches.compilemethod == "both" then local command = "-o " .. string.quoted(lucname) .. " -s " .. string.quoted(luaname) done = os.spawn("texluac " .. command) == 0 end if not done and (caches.compilemethod == "dump" or caches.compilemethod == "both") then local d = table.serialize(data,true) if d and d ~= "" then local f = io.open(lucname,'w') if f then local s = loadstring(d) f:write(string.dump(s)) f:close() end end end end -- function table.setmetatableindex(t,f) setmetatable(t,{ __index = f }) end end -- closure do -- begin closure to overcome local limits and interference if not modules then modules = { } end modules ['data-con'] = { version = 1.100, comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } local format, lower, gsub = string.format, string.lower, string.gsub local trace_cache = false trackers.register("resolvers.cache", function(v) trace_cache = v end) local trace_containers = false trackers.register("resolvers.containers", function(v) trace_containers = v end) local trace_storage = false trackers.register("resolvers.storage", function(v) trace_storage = v end) --[[ldx--
Once we found ourselves defining similar cache constructs several times, containers were introduced. Containers are used to collect tables in memory and reuse them when possible based on (unique) hashes (to be provided by the calling function).
Caching to disk is disabled by default. Version numbers are stored in the saved table which makes it possible to change the table structures without bothering about the disk cache.
Examples of usage can be found in the font related code.
--ldx]]-- containers = containers or { } local containers = containers containers.usecache = true local report_containers = logs.reporter("resolvers","containers") local function report(container,tag,name) if trace_cache or trace_containers then report_containers("container: %s, tag: %s, name: %s",container.subcategory,tag,name or 'invalid') end end local allocated = { } local mt = { __index = function(t,k) if k == "writable" then local writable = caches.getwritablepath(t.category,t.subcategory) or { "." } t.writable = writable return writable elseif k == "readables" then local readables = caches.getreadablepaths(t.category,t.subcategory) or { "." } t.readables = readables return readables end end, __storage__ = true } function containers.define(category, subcategory, version, enabled) if category and subcategory then local c = allocated[category] if not c then c = { } allocated[category] = c end local s = c[subcategory] if not s then s = { category = category, subcategory = subcategory, storage = { }, enabled = enabled, version = version or math.pi, -- after all, this is TeX trace = false, -- writable = caches.getwritablepath and caches.getwritablepath (category,subcategory) or { "." }, -- readables = caches.getreadablepaths and caches.getreadablepaths(category,subcategory) or { "." }, } setmetatable(s,mt) c[subcategory] = s end return s end end function containers.is_usable(container, name) return container.enabled and caches and caches.is_writable(container.writable, name) end function containers.is_valid(container, name) if name and name ~= "" then local storage = container.storage[name] return storage and storage.cache_version == container.version else return false end end function containers.read(container,name) local storage = container.storage local stored = storage[name] if not stored and container.enabled and caches and containers.usecache then stored = caches.loaddata(container.readables,name) if stored and stored.cache_version == container.version then report(container,"loaded",name) else stored = nil end storage[name] = stored elseif stored then report(container,"reusing",name) end return stored end function containers.write(container, name, data) if data then data.cache_version = container.version if container.enabled and caches then local unique, shared = data.unique, data.shared data.unique, data.shared = nil, nil caches.savedata(container.writable, name, data) report(container,"saved",name) data.unique, data.shared = unique, shared end report(container,"stored",name) container.storage[name] = data end return data end function containers.content(container,name) return container.storage[name] end function containers.cleanname(name) return (gsub(lower(name),"[^%w%d]+","-")) end end -- closure do -- begin closure to overcome local limits and interference if not modules then modules = { } end modules ['luatex-fonts-nod'] = { version = 1.001, comment = "companion to luatex-fonts.lua", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } if context then texio.write_nl("fatal error: this module is not for context") os.exit() end -- Don't depend on code here as it is only needed to complement the -- font handler code. -- Attributes: if tex.attribute[0] ~= 0 then texio.write_nl("log","!") texio.write_nl("log","! Attribute 0 is reserved for ConTeXt's font feature management and has to be") texio.write_nl("log","! set to zero. Also, some attributes in the range 1-255 are used for special") texio.write_nl("log","! purposes so setting them at the TeX end might break the font handler.") texio.write_nl("log","!") tex.attribute[0] = 0 -- else no features end attributes = { } attributes.unsetvalue = -0x7FFFFFFF local numbers, last = { }, 127 function attributes.private(name) local number = numbers[name] if not number then if last < 255 then last = last + 1 end number = last numbers[name] = number end return number end -- Nodes: nodes = { } nodes.pool = { } nodes.handlers = { } local nodecodes = { } for k,v in next, node.types () do nodecodes[string.gsub(v,"_","")] = k end local whatcodes = { } for k,v in next, node.whatsits() do whatcodes[string.gsub(v,"_","")] = k end local glyphcodes = { [0] = "character", "glyph", "ligature", "ghost", "left", "right" } nodes.nodecodes = nodecodes nodes.whatcodes = whatcodes nodes.whatsitcodes = whatcodes nodes.glyphcodes = glyphcodes local free_node = node.free local remove_node = node.remove local new_node = node.new nodes.handlers.protectglyphs = node.protect_glyphs nodes.handlers.unprotectglyphs = node.unprotect_glyphs function nodes.remove(head, current, free_too) local t = current head, current = remove_node(head,current) if t then if free_too then free_node(t) t = nil else t.next, t.prev = nil, nil end end return head, current, t end function nodes.delete(head,current) return nodes.remove(head,current,true) end nodes.before = node.insert_before nodes.after = node.insert_after function nodes.pool.kern(k) local n = new_node("kern",1) n.kern = k return n end end -- closure do -- begin closure to overcome local limits and interference if not modules then modules = { } end modules ['font-ini'] = { version = 1.001, comment = "companion to font-ini.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -- basemethods -> can also be in list -- presetcontext -> defaults -- hashfeatures -> ctx version --[[ldx--Not much is happening here.
--ldx]]-- local lower = string.lower local allocate, mark = utilities.storage.allocate, utilities.storage.mark local report_defining = logs.reporter("fonts","defining") fontloader.totable = fontloader.to_table fonts = fonts or { } -- already defined in context local fonts = fonts -- some of these might move to where they are used first: fonts.hashes = { identifiers = allocate() } fonts.analyzers = { } -- not needed here fonts.readers = { } fonts.tables = { } fonts.definers = { methods = { } } fonts.specifiers = fonts.specifiers or { } -- in format ! fonts.loggers = { register = function() end } fonts.helpers = { } fonts.tracers = { } -- for the moment till we have move to moduledata end -- closure do -- begin closure to overcome local limits and interference if not modules then modules = { } end modules ['font-con'] = { version = 1.001, comment = "companion to font-ini.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -- some names of table entries will be changed (no _) local utf = unicode.utf8 local next, tostring, rawget = next, tostring, rawget local format, match, lower, gsub = string.format, string.match, string.lower, string.gsub local utfbyte = utf.byte local sort, insert, concat, sortedkeys, serialize, fastcopy = table.sort, table.insert, table.concat, table.sortedkeys, table.serialize, table.fastcopy local derivetable = table.derive local trace_defining = false trackers.register("fonts.defining", function(v) trace_defining = v end) local trace_scaling = false trackers.register("fonts.scaling" , function(v) trace_scaling = v end) local report_defining = logs.reporter("fonts","defining") -- watch out: no negative depths and negative eights permitted in regular fonts --[[ldx--Here we only implement a few helper functions.
--ldx]]-- local fonts = fonts local constructors = { } fonts.constructors = constructors local handlers = { } fonts.handlers = handlers local specifiers = fonts.specifiers local contextsetups = specifiers.contextsetups local contextnumbers = specifiers.contextnumbers local allocate = utilities.storage.allocate local setmetatableindex = table.setmetatableindex -- will be directives constructors.dontembed = allocate() constructors.autocleanup = true constructors.namemode = "fullpath" -- will be a function constructors.version = 1.01 constructors.cache = containers.define("fonts", "constructors", constructors.version, false) constructors.privateoffset = 0xF0000 -- 0x10FFFF -- Some experimental helpers (handy for tracing): -- -- todo: extra: -- -- extra_space => space.extra -- space => space.width -- space_stretch => space.stretch -- space_shrink => space.shrink -- We do keep the x-height, extra_space, space_shrink and space_stretch -- around as these are low level official names. constructors.keys = { properties = { encodingbytes = "number", embedding = "number", cidinfo = { }, format = "string", fontname = "string", fullname = "string", filename = "filename", psname = "string", name = "string", virtualized = "boolean", hasitalics = "boolean", autoitalicamount = "basepoints", nostackmath = "boolean", noglyphnames = "boolean", mode = "string", hasmath = "boolean", mathitalics = "boolean", textitalics = "boolean", finalized = "boolean", }, parameters = { mathsize = "number", scriptpercentage = "float", scriptscriptpercentage = "float", units = "cardinal", designsize = "scaledpoints", expansion = { stretch = "integerscale", -- might become float shrink = "integerscale", -- might become float step = "integerscale", -- might become float auto = "boolean", }, protrusion = { auto = "boolean", }, slantfactor = "float", extendfactor = "float", factor = "float", hfactor = "float", vfactor = "float", size = "scaledpoints", units = "scaledpoints", scaledpoints = "scaledpoints", slantperpoint = "scaledpoints", spacing = { width = "scaledpoints", stretch = "scaledpoints", shrink = "scaledpoints", extra = "scaledpoints", }, xheight = "scaledpoints", quad = "scaledpoints", ascender = "scaledpoints", descender = "scaledpoints", synonyms = { space = "spacing.width", spacestretch = "spacing.stretch", spaceshrink = "spacing.shrink", extraspace = "spacing.extra", x_height = "xheight", space_stretch = "spacing.stretch", space_shrink = "spacing.shrink", extra_space = "spacing.extra", em = "quad", ex = "xheight", slant = "slantperpoint", }, }, description = { width = "basepoints", height = "basepoints", depth = "basepoints", boundingbox = { }, }, character = { width = "scaledpoints", height = "scaledpoints", depth = "scaledpoints", italic = "scaledpoints", }, } -- This might become an interface: local designsizes = allocate() constructors.designsizes = designsizes local loadedfonts = allocate() constructors.loadedfonts = loadedfonts --[[ldx--We need to normalize the scale factor (in scaled points). This has to
do with the fact that
Beware, the boundingbox is passed as reference so we may not overwrite it in the process; numbers are of course copies. Here 65536 equals 1pt. (Due to excessive memory usage in CJK fonts, we no longer pass the boundingbox.)
--ldx]]-- -- The scaler is only used for otf and afm and virtual fonts. If -- a virtual font has italic correction make sure to set the -- hasitalics flag. Some more flags will be added in -- the future. --[[ldx--The reason why the scaler was originally split, is that for a while we experimented
with a helper function. However, in practice the
A unique hash value is generated by:
--ldx]]-- local hashmethods = { } constructors.hashmethods = hashmethods function constructors.hashfeatures(specification) -- will be overloaded local features = specification.features if features then local t, tn = { }, 0 for category, list in next, features do if next(list) then local hasher = hashmethods[category] if hasher then local hash = hasher(list) if hash then tn = tn + 1 t[tn] = category .. ":" .. hash end end end end if tn > 0 then return concat(t," & ") end end return "unknown" end hashmethods.normal = function(list) local s = { } local n = 0 for k, v in next, list do if k ~= "number" and k ~= "features" then -- I need to figure this out, features n = n + 1 s[n] = k end end if n > 0 then sort(s) for i=1,n do local k = s[i] s[i] = k .. '=' .. tostring(list[k]) end return concat(s,"+") end end --[[ldx--In principle we can share tfm tables when we are in node for a font, but then
we need to define a font switch as an id/attr switch which is no fun, so in that
case users can best use dynamic features ... so, we will not use that speedup. Okay,
when we get rid of base mode we can optimize even further by sharing, but then we
loose our testcases for
We need to check for default features. For this we provide a helper function.
--ldx]]-- function constructors.checkedfeatures(what,features) local defaults = handlers[what].features.defaults if features and next(features) then features = fastcopy(features) -- can be inherited (mt) but then no loops possible for key, value in next, defaults do if features[key] == nil then features[key] = value end end return features else return fastcopy(defaults) -- we can change features in place end end -- before scaling function constructors.initializefeatures(what,tfmdata,features,trace,report) if features and next(features) then local properties = tfmdata.properties or { } -- brrr local whathandler = handlers[what] local whatfeatures = whathandler.features local whatinitializers = whatfeatures.initializers local whatmodechecker = whatfeatures.modechecker -- properties.mode can be enforces (for instance in font-otd) local mode = properties.mode or (whatmodechecker and whatmodechecker(tfmdata,features,features.mode)) or features.mode or "base" properties.mode = mode -- also status features.mode = mode -- both properties.mode or features.mode can be changed -- local done = { } while true do local redo = false local initializers = whatfeatures.initializers[mode] if initializers then for i=1,#initializers do local step = initializers[i] local feature = step.name -- we could intercept mode here .. needs a rewrite of this whole loop then but it's cleaner that way local value = features[feature] if not value then -- disabled elseif done[feature] then -- already done else local action = step.action if trace then report("initializing feature %s to %s for mode %s for font %s",feature, tostring(value),mode or 'unknown', tfmdata.properties.fullname or 'unknown') end action(tfmdata,value,features) -- can set mode (e.g. goodies) so it can trigger a restart if mode ~= properties.mode or mode ~= features.mode then if whatmodechecker then properties.mode = whatmodechecker(tfmdata,features,properties.mode) -- force checking features.mode = properties.mode end if mode ~= properties.mode then mode = properties.mode redo = true end end done[feature] = true end if redo then break end end if not redo then break end else break end end properties.mode = mode -- to be sure return true else return false end end -- while typesetting function constructors.collectprocessors(what,tfmdata,features,trace,report) local processes, nofprocesses = { }, 0 if features and next(features) then local properties = tfmdata.properties local whathandler = handlers[what] local whatfeatures = whathandler.features local whatprocessors = whatfeatures.processors local processors = whatprocessors[properties.mode] if processors then for i=1,#processors do local step = processors[i] local feature = step.name if features[feature] then local action = step.action if trace then report("installing feature processor %s for mode %s for font %s",feature, mode or 'unknown', tfmdata.properties.fullname or 'unknown') end if action then nofprocesses = nofprocesses + 1 processes[nofprocesses] = action end end end elseif trace then report("no feature processors for mode %s for font %s", mode or 'unknown', tfmdata.properties.fullname or 'unknown') end end return processes end -- after scaling function constructors.applymanipulators(what,tfmdata,features,trace,report) if features and next(features) then local properties = tfmdata.properties local whathandler = handlers[what] local whatfeatures = whathandler.features local whatmanipulators = whatfeatures.manipulators local manipulators = whatmanipulators[properties.mode] if manipulators then for i=1,#manipulators do local step = manipulators[i] local feature = step.name local value = features[feature] if value then local action = step.action if trace then report("applying feature manipulator %s for mode %s for font %s",feature, mode or 'unknown', tfmdata.properties.fullname or 'unknown') end if action then action(tfmdata,feature,value) end end end end end end end -- closure do -- begin closure to overcome local limits and interference if not modules then modules = { } end modules ['luatex-font-enc'] = { version = 1.001, comment = "companion to luatex-*.tex", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } if context then texio.write_nl("fatal error: this module is not for context") os.exit() end local fonts = fonts fonts.encodings = { } fonts.encodings.agl = { } setmetatable(fonts.encodings.agl, { __index = function(t,k) if k == "unicodes" then texio.write("Eventually this code will disappear because map files are kind of obsolete. Some code may move to runtime or auxiliary modules.
The name to unciode related code will stay of course.
--ldx]]-- local function loadlumtable(filename) -- will move to font goodies local lumname = file.replacesuffix(file.basename(filename),"lum") local lumfile = resolvers.findfile(lumname,"map") or "" if lumfile ~= "" and lfs.isfile(lumfile) then if trace_loading or trace_mapping then report_fonts("enhance: loading %s ",lumfile) end lumunic = dofile(lumfile) return lumunic, lumfile end end local hex = R("AF","09") local hexfour = (hex*hex*hex*hex) / function(s) return tonumber(s,16) end local hexsix = (hex^1) / function(s) return tonumber(s,16) end local dec = (R("09")^1) / tonumber local period = P(".") local unicode = P("uni") * (hexfour * (period + P(-1)) * Cc(false) + Ct(hexfour^1) * Cc(true)) local ucode = P("u") * (hexsix * (period + P(-1)) * Cc(false) + Ct(hexsix ^1) * Cc(true)) local index = P("index") * dec * Cc(false) local parser = unicode + ucode + index local parsers = { } local function makenameparser(str) if not str or str == "" then return parser else local p = parsers[str] if not p then p = P(str) * period * dec * Cc(false) parsers[str] = p end return p end end --~ local parser = mappings.makenameparser("Japan1") --~ local parser = mappings.makenameparser() --~ local function test(str) --~ local b, a = lpegmatch(parser,str) --~ print((a and table.serialize(b)) or b) --~ end --~ test("a.sc") --~ test("a") --~ test("uni1234") --~ test("uni1234.xx") --~ test("uni12349876") --~ test("index1234") --~ test("Japan1.123") local function tounicode16(unicode) if unicode < 0x10000 then return format("%04X",unicode) else return format("%04X%04X",unicode/1024+0xD800,unicode%1024+0xDC00) end end local function tounicode16sequence(unicodes) local t = { } for l=1,#unicodes do local unicode = unicodes[l] if unicode < 0x10000 then t[l] = format("%04X",unicode) else t[l] = format("%04X%04X",unicode/1024+0xD800,unicode%1024+0xDC00) end end return concat(t) end local function fromunicode16(str) if #str == 4 then return tonumber(str,16) else local l, r = match(str,"(....)(....)") return (tonumber(l,16)- 0xD800)*0x400 + tonumber(r,16) - 0xDC00 end end --~ This is quite a bit faster but at the cost of some memory but if we --~ do this we will also use it elsewhere so let's not follow this route --~ now. I might use this method in the plain variant (no caching there) --~ but then I need a flag that distinguishes between code branches. --~ --~ local cache = { } --~ --~ function mappings.tounicode16(unicode) --~ local s = cache[unicode] --~ if not s then --~ if unicode < 0x10000 then --~ s = format("%04X",unicode) --~ else --~ s = format("%04X%04X",unicode/1024+0xD800,unicode%1024+0xDC00) --~ end --~ cache[unicode] = s --~ end --~ return s --~ end mappings.loadlumtable = loadlumtable mappings.makenameparser = makenameparser mappings.tounicode16 = tounicode16 mappings.tounicode16sequence = tounicode16sequence mappings.fromunicode16 = fromunicode16 local separator = S("_.") local other = C((1 - separator)^1) local ligsplitter = Ct(other * (separator * other)^0) --~ print(table.serialize(lpegmatch(ligsplitter,"this"))) --~ print(table.serialize(lpegmatch(ligsplitter,"this.that"))) --~ print(table.serialize(lpegmatch(ligsplitter,"japan1.123"))) --~ print(table.serialize(lpegmatch(ligsplitter,"such_so_more"))) --~ print(table.serialize(lpegmatch(ligsplitter,"such_so_more.that"))) function mappings.addtounicode(data,filename) local resources = data.resources local properties = data.properties local descriptions = data.descriptions local unicodes = resources.unicodes if not unicodes then return end -- we need to move this code unicodes['space'] = unicodes['space'] or 32 unicodes['hyphen'] = unicodes['hyphen'] or 45 unicodes['zwj'] = unicodes['zwj'] or 0x200D unicodes['zwnj'] = unicodes['zwnj'] or 0x200C -- the tounicode mapping is sparse and only needed for alternatives local private = fonts.constructors.privateoffset local unknown = format("%04X",utfbyte("?")) local unicodevector = fonts.encodings.agl.unicodes -- loaded runtime in context local tounicode = { } local originals = { } resources.tounicode = tounicode resources.originals = originals local lumunic, uparser, oparser local cidinfo, cidnames, cidcodes, usedmap if false then -- will become an option lumunic = loadlumtable(filename) lumunic = lumunic and lumunic.tounicode end -- cidinfo = properties.cidinfo usedmap = cidinfo and fonts.cid.getmap(cidinfo) -- if usedmap then oparser = usedmap and makenameparser(cidinfo.ordering) cidnames = usedmap.names cidcodes = usedmap.unicodes end uparser = makenameparser() local ns, nl = 0, 0 for unic, glyph in next, descriptions do local index = glyph.index local name = glyph.name if unic == -1 or unic >= private or (unic >= 0xE000 and unic <= 0xF8FF) or unic == 0xFFFE or unic == 0xFFFF then local unicode = lumunic and lumunic[name] or unicodevector[name] if unicode then originals[index] = unicode tounicode[index] = tounicode16(unicode) ns = ns + 1 end -- cidmap heuristics, beware, there is no guarantee for a match unless -- the chain resolves if (not unicode) and usedmap then local foundindex = lpegmatch(oparser,name) if foundindex then unicode = cidcodes[foundindex] -- name to number if unicode then originals[index] = unicode tounicode[index] = tounicode16(unicode) ns = ns + 1 else local reference = cidnames[foundindex] -- number to name if reference then local foundindex = lpegmatch(oparser,reference) if foundindex then unicode = cidcodes[foundindex] if unicode then originals[index] = unicode tounicode[index] = tounicode16(unicode) ns = ns + 1 end end if not unicode then local foundcodes, multiple = lpegmatch(uparser,reference) if foundcodes then originals[index] = foundcodes if multiple then tounicode[index] = tounicode16sequence(foundcodes) nl = nl + 1 unicode = true else tounicode[index] = tounicode16(foundcodes) ns = ns + 1 unicode = foundcodes end end end end end end end -- a.whatever or a_b_c.whatever or a_b_c (no numbers) if not unicode then local split = lpegmatch(ligsplitter,name) local nplit = split and #split or 0 if nplit >= 2 then local t, n = { }, 0 for l=1,nplit do local base = split[l] local u = unicodes[base] or unicodevector[base] if not u then break elseif type(u) == "table" then n = n + 1 t[n] = u[1] else n = n + 1 t[n] = u end end if n == 0 then -- done then -- nothing elseif n == 1 then originals[index] = t[1] tounicode[index] = tounicode16(t[1]) else originals[index] = t tounicode[index] = tounicode16sequence(t) end nl = nl + 1 unicode = true else -- skip: already checked and we don't want privates here end end -- last resort (we might need to catch private here as well) if not unicode then local foundcodes, multiple = lpegmatch(uparser,name) if foundcodes then if multiple then originals[index] = foundcodes tounicode[index] = tounicode16sequence(foundcodes) nl = nl + 1 unicode = true else originals[index] = foundcodes tounicode[index] = tounicode16(foundcodes) ns = ns + 1 unicode = foundcodes end end end -- if not unicode then -- originals[index] = 0xFFFD -- tounicode[index] = "FFFD" -- end end end if trace_mapping then for unic, glyph in table.sortedhash(descriptions) do local name = glyph.name local index = glyph.index local toun = tounicode[index] if toun then report_fonts("internal: 0x%05X, name: %s, unicode: U+%05X, tounicode: %s",index,name,unic,toun) else report_fonts("internal: 0x%05X, name: %s, unicode: U+%05X",index,name,unic) end end end if trace_loading and (ns > 0 or nl > 0) then report_fonts("enhance: %s tounicode entries added (%s ligatures)",nl+ns, ns) end end end -- closure do -- begin closure to overcome local limits and interference if not modules then modules = { } end modules ['luatex-fonts-syn'] = { version = 1.001, comment = "companion to luatex-*.tex", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } if context then texio.write_nl("fatal error: this module is not for context") os.exit() end -- Generic font names support. -- -- Watch out, the version number is the same as the one used in -- the mtx-fonts.lua function scripts.fonts.names as we use a -- simplified font database in the plain solution and by using -- a different number we're less dependent on context. -- -- mtxrun --script font --reload --simple -- -- The format of the file is as follows: -- -- return { -- ["version"] = 1.001, -- ["mappings"] = { -- ["somettcfontone"] = { "Some TTC Font One", "SomeFontA.ttc", 1 }, -- ["somettcfonttwo"] = { "Some TTC Font Two", "SomeFontA.ttc", 2 }, -- ["somettffont"] = { "Some TTF Font", "SomeFontB.ttf" }, -- ["someotffont"] = { "Some OTF Font", "SomeFontC.otf" }, -- }, -- } local fonts = fonts fonts.names = fonts.names or { } fonts.names.version = 1.001 -- not the same as in context fonts.names.basename = "luatex-fonts-names.lua" fonts.names.new_to_old = { } fonts.names.old_to_new = { } local data, loaded = nil, false local fileformats = { "lua", "tex", "other text files" } function fonts.names.resolve(name,sub) if not loaded then local basename = fonts.names.basename if basename and basename ~= "" then for i=1,#fileformats do local format = fileformats[i] local foundname = resolvers.findfile(basename,format) or "" if foundname ~= "" then data = dofile(foundname) texio.write("") break end end end loaded = true end if type(data) == "table" and data.version == fonts.names.version then local condensed = string.gsub(string.lower(name),"[^%a%d]","") local found = data.mappings and data.mappings[condensed] if found then local fontname, filename, subfont = found[1], found[2], found[3] if subfont then return filename, fontname else return filename, false end else return name, false -- fallback to filename end end end fonts.names.resolvespec = fonts.names.resolve -- only supported in mkiv function fonts.names.getfilename(askedname,suffix) -- only supported in mkiv return "" end end -- closure do -- begin closure to overcome local limits and interference if not modules then modules = { } end modules ['luatex-fonts-tfm'] = { version = 1.001, comment = "companion to luatex-*.tex", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } if context then texio.write_nl("fatal error: this module is not for context") os.exit() end local fonts = fonts local tfm = { } fonts.handlers.tfm = tfm fonts.formats.tfm = "type1" -- we need to have at least a value here function fonts.readers.tfm(specification) local fullname = specification.filename or "" if fullname == "" then local forced = specification.forced or "" if forced ~= "" then fullname = specification.name .. "." .. forced else fullname = specification.name end end local foundname = resolvers.findbinfile(fullname, 'tfm') or "" if foundname == "" then foundname = resolvers.findbinfile(fullname, 'ofm') or "" end if foundname ~= "" then specification.filename = foundname specification.format = "ofm" return font.read_tfm(specification.filename,specification.size) end end end -- closure do -- begin closure to overcome local limits and interference if not modules then modules = { } end modules ['font-oti'] = { version = 1.001, comment = "companion to font-ini.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } local lower = string.lower local allocate = utilities.storage.allocate local fonts = fonts local otf = { } fonts.handlers.otf = otf local otffeatures = fonts.constructors.newfeatures("otf") local registerotffeature = otffeatures.register registerotffeature { name = "features", description = "initialization of feature handler", default = true, } -- these are later hooked into node and base initializaters local otftables = otf.tables -- not always defined local function setmode(tfmdata,value) if value then tfmdata.properties.mode = lower(value) end end local function setlanguage(tfmdata,value) if value then local cleanvalue = lower(value) local languages = otftables and otftables.languages local properties = tfmdata.properties if not languages then properties.language = cleanvalue elseif languages[value] then properties.language = cleanvalue else properties.language = "dflt" end end end local function setscript(tfmdata,value) if value then local cleanvalue = lower(value) local scripts = otftables and otftables.scripts local properties = tfmdata.properties if not scripts then properties.script = cleanvalue elseif scripts[value] then properties.script = cleanvalue else properties.script = "dflt" end end end registerotffeature { name = "mode", description = "mode", initializers = { base = setmode, node = setmode, } } registerotffeature { name = "language", description = "language", initializers = { base = setlanguage, node = setlanguage, } } registerotffeature { name = "script", description = "script", initializers = { base = setscript, node = setscript, } } end -- closure do -- begin closure to overcome local limits and interference if not modules then modules = { } end modules ['font-otf'] = { version = 1.001, comment = "companion to font-ini.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -- langs -> languages enz -- anchor_classes vs kernclasses -- modification/creationtime in subfont is runtime dus zinloos -- to_table -> totable -- ascent descent -- more checking against low level calls of functions local utf = unicode.utf8 local utfbyte = utf.byte local format, gmatch, gsub, find, match, lower, strip = string.format, string.gmatch, string.gsub, string.find, string.match, string.lower, string.strip local type, next, tonumber, tostring = type, next, tonumber, tostring local abs = math.abs local getn = table.getn local lpegmatch = lpeg.match local reversed, concat, remove = table.reversed, table.concat, table.remove local ioflush = io.flush local fastcopy, tohash, derivetable = table.fastcopy, table.tohash, table.derive local allocate = utilities.storage.allocate local registertracker = trackers.register local registerdirective = directives.register local starttiming = statistics.starttiming local stoptiming = statistics.stoptiming local elapsedtime = statistics.elapsedtime local findbinfile = resolvers.findbinfile local trace_private = false registertracker("otf.private", function(v) trace_private = v end) local trace_loading = false registertracker("otf.loading", function(v) trace_loading = v end) local trace_features = false registertracker("otf.features", function(v) trace_features = v end) local trace_dynamics = false registertracker("otf.dynamics", function(v) trace_dynamics = v end) local trace_sequences = false registertracker("otf.sequences", function(v) trace_sequences = v end) local trace_markwidth = false registertracker("otf.markwidth", function(v) trace_markwidth = v end) local trace_defining = false registertracker("fonts.defining", function(v) trace_defining = v end) local report_otf = logs.reporter("fonts","otf loading") local fonts = fonts local otf = fonts.handlers.otf otf.glists = { "gsub", "gpos" } otf.version = 2.736 -- beware: also sync font-mis.lua otf.cache = containers.define("fonts", "otf", otf.version, true) local fontdata = fonts.hashes.identifiers local chardata = characters and characters.data -- not used local otffeatures = fonts.constructors.newfeatures("otf") local registerotffeature = otffeatures.register local enhancers = allocate() otf.enhancers = enhancers local patches = { } enhancers.patches = patches local definers = fonts.definers local readers = fonts.readers local constructors = fonts.constructors local forceload = false local cleanup = 0 -- mk: 0=885M 1=765M 2=735M (regular run 730M) local usemetatables = false -- .4 slower on mk but 30 M less mem so we might change the default -- will be directive local packdata = true local syncspace = true local forcenotdef = false local wildcard = "*" local default = "dflt" local fontloaderfields = fontloader.fields local mainfields = nil local glyphfields = nil -- not used yet registerdirective("fonts.otf.loader.cleanup", function(v) cleanup = tonumber(v) or (v and 1) or 0 end) registerdirective("fonts.otf.loader.force", function(v) forceload = v end) registerdirective("fonts.otf.loader.usemetatables", function(v) usemetatables = v end) registerdirective("fonts.otf.loader.pack", function(v) packdata = v end) registerdirective("fonts.otf.loader.syncspace", function(v) syncspace = v end) registerdirective("fonts.otf.loader.forcenotdef", function(v) forcenotdef = v end) local function load_featurefile(raw,featurefile) if featurefile and featurefile ~= "" then if trace_loading then report_otf("featurefile: %s", featurefile) end fontloader.apply_featurefile(raw, featurefile) end end local function showfeatureorder(rawdata,filename) local sequences = rawdata.resources.sequences if sequences and #sequences > 0 then if trace_loading then report_otf("font %s has %s sequences",filename,#sequences) report_otf(" ") end for nos=1,#sequences do local sequence = sequences[nos] local typ = sequence.type or "no-type" local name = sequence.name or "no-name" local subtables = sequence.subtables or { "no-subtables" } local features = sequence.features if trace_loading then report_otf("%3i %-15s %-20s [%s]",nos,name,typ,concat(subtables,",")) end if features then for feature, scripts in next, features do local tt = { } if type(scripts) == "table" then for script, languages in next, scripts do local ttt = { } for language, _ in next, languages do ttt[#ttt+1] = language end tt[#tt+1] = format("[%s: %s]",script,concat(ttt," ")) end if trace_loading then report_otf(" %s: %s",feature,concat(tt," ")) end else if trace_loading then report_otf(" %s: %s",feature,tostring(scripts)) end end end end end if trace_loading then report_otf("\n") end elseif trace_loading then report_otf("font %s has no sequences",filename) end end --[[ldx--We start with a lot of tables and related functions.
--ldx]]-- local valid_fields = table.tohash { -- "anchor_classes", "ascent", -- "cache_version", "cidinfo", "copyright", -- "creationtime", "descent", "design_range_bottom", "design_range_top", "design_size", "encodingchanged", "extrema_bound", "familyname", "fontname", "fontname", "fontstyle_id", "fontstyle_name", "fullname", -- "glyphs", "hasvmetrics", -- "head_optimized_for_cleartype", "horiz_base", "issans", "isserif", "italicangle", -- "kerns", -- "lookups", "macstyle", -- "modificationtime", "onlybitmaps", "origname", "os2_version", "pfminfo", -- "private", "serifcheck", "sfd_version", -- "size", "strokedfont", "strokewidth", -- "subfonts", "table_version", -- "tables", -- "ttf_tab_saved", "ttf_tables", "uni_interp", "uniqueid", "units_per_em", "upos", "use_typo_metrics", "uwidth", -- "validation_state", "version", "vert_base", "weight", "weight_width_slope_only", -- "xuid", } local ordered_enhancers = { "prepare tables", "prepare glyphs", "prepare lookups", "analyze glyphs", "analyze math", "prepare tounicode", -- maybe merge with prepare "reorganize lookups", "reorganize mark classes", "reorganize anchor classes", "reorganize glyph kerns", "reorganize glyph lookups", "reorganize glyph anchors", "merge kern classes", "reorganize features", "reorganize subtables", "check glyphs", "check metadata", "check extra features", -- after metadata "add duplicates", "check encoding", "cleanup tables", } --[[ldx--Here we go.
--ldx]]-- local actions = allocate() local before = allocate() local after = allocate() patches.before = before patches.after = after local function enhance(name,data,filename,raw) local enhancer = actions[name] if enhancer then if trace_loading then report_otf("enhance: %s (%s)",name,filename) ioflush() end enhancer(data,filename,raw) elseif trace_loading then -- report_otf("enhance: %s is undefined",name) end end function enhancers.apply(data,filename,raw) local basename = file.basename(lower(filename)) if trace_loading then report_otf("start enhancing: %s",filename) end ioflush() -- we want instant messages for e=1,#ordered_enhancers do local enhancer = ordered_enhancers[e] local b = before[enhancer] if b then for pattern, action in next, b do if find(basename,pattern) then action(data,filename,raw) end end end enhance(enhancer,data,filename,raw) local a = after[enhancer] if a then for pattern, action in next, a do if find(basename,pattern) then action(data,filename,raw) end end end ioflush() -- we want instant messages end if trace_loading then report_otf("stop enhancing") end ioflush() -- we want instant messages end -- patches.register("before","migrate metadata","cambria",function() end) function patches.register(what,where,pattern,action) local pw = patches[what] if pw then local ww = pw[where] if ww then ww[pattern] = action else pw[where] = { [pattern] = action} end end end function patches.report(fmt,...) if trace_loading then report_otf("patching: " ..fmt,...) end end function enhancers.register(what,action) -- only already registered can be overloaded actions[what] = action end function otf.load(filename,format,sub,featurefile) local name = file.basename(file.removesuffix(filename)) local attr = lfs.attributes(filename) local size = attr and attr.size or 0 local time = attr and attr.modification or 0 if featurefile then name = name .. "@" .. file.removesuffix(file.basename(featurefile)) end if sub == "" then sub = false end local hash = name if sub then hash = hash .. "-" .. sub end hash = containers.cleanname(hash) local featurefiles if featurefile then featurefiles = { } for s in gmatch(featurefile,"[^,]+") do local name = resolvers.findfile(file.addsuffix(s,'fea'),'fea') or "" if name == "" then report_otf("loading: no featurefile '%s'",s) else local attr = lfs.attributes(name) featurefiles[#featurefiles+1] = { name = name, size = attr and attr.size or 0, time = attr and attr.modification or 0, } end end if #featurefiles == 0 then featurefiles = nil end end local data = containers.read(otf.cache,hash) local reload = not data or data.size ~= size or data.time ~= time if forceload then report_otf("loading: forced reload due to hard coded flag") reload = true end if not reload then local featuredata = data.featuredata if featurefiles then if not featuredata or #featuredata ~= #featurefiles then reload = true else for i=1,#featurefiles do local fi, fd = featurefiles[i], featuredata[i] if fi.name ~= fd.name or fi.size ~= fd.size or fi.time ~= fd.time then reload = true break end end end elseif featuredata then reload = true end if reload then report_otf("loading: forced reload due to changed featurefile specification: %s",featurefile or "--") end end if reload then report_otf("loading: %s (hash: %s)",filename,hash) local fontdata, messages if sub then fontdata, messages = fontloader.open(filename,sub) else fontdata, messages = fontloader.open(filename) end if fontdata then mainfields = mainfields or (fontloaderfields and fontloaderfields(fontdata)) end if trace_loading and messages and #messages > 0 then if type(messages) == "string" then report_otf("warning: %s",messages) else for m=1,#messages do report_otf("warning: %s",tostring(messages[m])) end end else report_otf("font loaded okay") end if fontdata then if featurefiles then for i=1,#featurefiles do load_featurefile(fontdata,featurefiles[i].name) end end local unicodes = { -- names to unicodes } local splitter = lpeg.splitter(" ",unicodes) data = { size = size, time = time, format = format, featuredata = featurefiles, resources = { filename = resolvers.unresolve(filename), -- no shortcut version = otf.version, creator = "context mkiv", unicodes = unicodes, indices = { -- index to unicodes }, duplicates = { -- alternative unicodes }, variants = { -- alternative unicodes (variants) }, lookuptypes = { }, }, metadata = { -- raw metadata, not to be used }, properties = { -- normalized metadata }, descriptions = { }, goodies = { }, helpers = { tounicodelist = splitter, tounicodetable = lpeg.Ct(splitter), }, } starttiming(data) report_otf("file size: %s", size) enhancers.apply(data,filename,fontdata) if packdata then if cleanup > 0 then collectgarbage("collect") --~ lua.collectgarbage() end enhance("pack",data,filename,nil) end report_otf("saving in cache: %s",filename) data = containers.write(otf.cache, hash, data) if cleanup > 1 then collectgarbage("collect") --~ lua.collectgarbage() end stoptiming(data) if elapsedtime then -- not in generic report_otf("preprocessing and caching took %s seconds",elapsedtime(data)) end fontloader.close(fontdata) -- free memory if cleanup > 3 then collectgarbage("collect") --~ lua.collectgarbage() end data = containers.read(otf.cache, hash) -- this frees the old table and load the sparse one if cleanup > 2 then collectgarbage("collect") --~ lua.collectgarbage() end else data = nil report_otf("loading failed (file read error)") end end if data then if trace_defining then report_otf("loading from cache: %s",hash) end enhance("unpack",data,filename,nil,false) enhance("add dimensions",data,filename,nil,false) if trace_sequences then showfeatureorder(data,filename) end end return data end local mt = { __index = function(t,k) -- maybe set it if k == "height" then local ht = t.boundingbox[4] return ht < 0 and 0 or ht elseif k == "depth" then local dp = -t.boundingbox[2] return dp < 0 and 0 or dp elseif k == "width" then return 0 elseif k == "name" then -- or maybe uni* return forcenotdef and ".notdef" end end } actions["prepare tables"] = function(data,filename,raw) data.properties.hasitalics = false end actions["add dimensions"] = function(data,filename) -- todo: forget about the width if it's the defaultwidth (saves mem) -- we could also build the marks hash here (instead of storing it) if data then local descriptions = data.descriptions local resources = data.resources local defaultwidth = resources.defaultwidth or 0 local defaultheight = resources.defaultheight or 0 local defaultdepth = resources.defaultdepth or 0 if usemetatables then for _, d in next, descriptions do local wd = d.width if not wd then d.width = defaultwidth elseif trace_markwidth and wd ~= 0 and d.class == "mark" then report_otf("mark with width %s (%s) in %s",wd,d.name or "This module is a bit more split up that I'd like but since we also want to test
with plain
The specification of OpenType is kind of vague. Apart from a lack of a proper free specifications there's also the problem that Microsoft and Adobe may have their own interpretation of how and in what order to apply features. In general the Microsoft website has more detailed specifications and is a better reference. There is also some information in the FontForge help files.
Because there is so much possible, fonts might contain bugs and/or be made to work with certain rederers. These may evolve over time which may have the side effect that suddenly fonts behave differently.
After a lot of experiments (mostly by Taco, me and Idris) we're now at yet another
implementation. Of course all errors are mine and of course the code can be
improved. There are quite some optimizations going on here and processing speed
is currently acceptable. Not all functions are implemented yet, often because I
lack the fonts for testing. Many scripts are not yet supported either, but I will
look into them as soon as
Because there are different interpretations possible, I will extend the code with more (configureable) variants. I can also add hooks for users so that they can write their own extensions.
Glyphs are indexed not by unicode but in their own way. This is because there is no
relationship with unicode at all, apart from the fact that a font might cover certain
ranges of characters. One character can have multiple shapes. However, at the
The raw table as it coms from
This module is sparsely documented because it is a moving target. The table format of the reader changes and we experiment a lot with different methods for supporting features.
As with the
Incrementing the version number will force a re-cache. We jump the number by one
when there's a fix in the
We get hits on a mark, but we're not sure if the it has to be applied so we need to explicitly test for basechar, baselig and basemark entries.
--ldx]]-- function handlers.gpos_mark2base(start,kind,lookupname,markanchors,sequence) local markchar = start.char if marks[markchar] then local base = start.prev -- [glyph] [start=mark] if base and base.id == glyph_code and base.subtype<256 and base.font == currentfont then local basechar = base.char if marks[basechar] then while true do base = base.prev if base and base.id == glyph_code and base.subtype<256 and base.font == currentfont then basechar = base.char if not marks[basechar] then break end else if trace_bugs then logwarning("%s: no base for mark %s",pref(kind,lookupname),gref(markchar)) end return start, false end end end local baseanchors = descriptions[basechar] if baseanchors then baseanchors = baseanchors.anchors end if baseanchors then local baseanchors = baseanchors['basechar'] if baseanchors then local al = anchorlookups[lookupname] for anchor,ba in next, baseanchors do if al[anchor] then local ma = markanchors[anchor] if ma then local dx, dy, bound = setmark(start,base,tfmdata.parameters.factor,rlmode,ba,ma) if trace_marks then logprocess("%s, anchor %s, bound %s: anchoring mark %s to basechar %s => (%s,%s)", pref(kind,lookupname),anchor,bound,gref(markchar),gref(basechar),dx,dy) end return start, true end end end if trace_bugs then logwarning("%s, no matching anchors for mark %s and base %s",pref(kind,lookupname),gref(markchar),gref(basechar)) end end else -- if trace_bugs then -- logwarning("%s: char %s is missing in font",pref(kind,lookupname),gref(basechar)) onetimemessage(currentfont,basechar,"no base anchors",report_fonts) end elseif trace_bugs then logwarning("%s: prev node is no char",pref(kind,lookupname)) end elseif trace_bugs then logwarning("%s: mark %s is no mark",pref(kind,lookupname),gref(markchar)) end return start, false end function handlers.gpos_mark2ligature(start,kind,lookupname,markanchors,sequence) -- check chainpos variant local markchar = start.char if marks[markchar] then local base = start.prev -- [glyph] [optional marks] [start=mark] if base and base.id == glyph_code and base.subtype<256 and base.font == currentfont then local basechar = base.char if marks[basechar] then while true do base = base.prev if base and base.id == glyph_code and base.subtype<256 and base.font == currentfont then basechar = base.char if not marks[basechar] then break end else if trace_bugs then logwarning("%s: no base for mark %s",pref(kind,lookupname),gref(markchar)) end return start, false end end end local index = has_attribute(start,ligacomp) local baseanchors = descriptions[basechar] if baseanchors then baseanchors = baseanchors.anchors if baseanchors then local baseanchors = baseanchors['baselig'] if baseanchors then local al = anchorlookups[lookupname] for anchor,ba in next, baseanchors do if al[anchor] then local ma = markanchors[anchor] if ma then ba = ba[index] if ba then local dx, dy, bound = setmark(start,base,tfmdata.parameters.factor,rlmode,ba,ma) -- index if trace_marks then logprocess("%s, anchor %s, index %s, bound %s: anchoring mark %s to baselig %s at index %s => (%s,%s)", pref(kind,lookupname),anchor,index,bound,gref(markchar),gref(basechar),index,dx,dy) end return start, true end end end end if trace_bugs then logwarning("%s: no matching anchors for mark %s and baselig %s",pref(kind,lookupname),gref(markchar),gref(basechar)) end end end else -- if trace_bugs then -- logwarning("%s: char %s is missing in font",pref(kind,lookupname),gref(basechar)) onetimemessage(currentfont,basechar,"no base anchors",report_fonts) end elseif trace_bugs then logwarning("%s: prev node is no char",pref(kind,lookupname)) end elseif trace_bugs then logwarning("%s: mark %s is no mark",pref(kind,lookupname),gref(markchar)) end return start, false end function handlers.gpos_mark2mark(start,kind,lookupname,markanchors,sequence) local markchar = start.char if marks[markchar] then local base = start.prev -- [glyph] [basemark] [start=mark] -- while base and has_attribute(base,ligacomp) and has_attribute(base,ligacomp) ~= has_attribute(start,ligacomp) do -- base = base.prev -- KE: prevents mkmk for marks on different components of a ligature -- end local slc = has_attribute(start,ligacomp) if slc then -- a rather messy loop ... needs checking with husayni while base do local blc = has_attribute(base,ligacomp) if blc and blc ~= slc then base = base.prev else break end end end if base and base.id == glyph_code and base.subtype<256 and base.font == currentfont then -- subtype test can go local basechar = base.char local baseanchors = descriptions[basechar] if baseanchors then baseanchors = baseanchors.anchors if baseanchors then baseanchors = baseanchors['basemark'] if baseanchors then local al = anchorlookups[lookupname] for anchor,ba in next, baseanchors do if al[anchor] then local ma = markanchors[anchor] if ma then local dx, dy, bound = setmark(start,base,tfmdata.parameters.factor,rlmode,ba,ma) if trace_marks then logprocess("%s, anchor %s, bound %s: anchoring mark %s to basemark %s => (%s,%s)", pref(kind,lookupname),anchor,bound,gref(markchar),gref(basechar),dx,dy) end return start,true end end end if trace_bugs then logwarning("%s: no matching anchors for mark %s and basemark %s",pref(kind,lookupname),gref(markchar),gref(basechar)) end end end else -- if trace_bugs then -- logwarning("%s: char %s is missing in font",pref(kind,lookupname),gref(basechar)) onetimemessage(currentfont,basechar,"no base anchors",report_fonts) end elseif trace_bugs then logwarning("%s: prev node is no mark",pref(kind,lookupname)) end elseif trace_bugs then logwarning("%s: mark %s is no mark",pref(kind,lookupname),gref(markchar)) end return start,false end function handlers.gpos_cursive(start,kind,lookupname,exitanchors,sequence) -- to be checked local alreadydone = cursonce and has_attribute(start,cursbase) if not alreadydone then local done = false local startchar = start.char if marks[startchar] then if trace_cursive then logprocess("%s: ignoring cursive for mark %s",pref(kind,lookupname),gref(startchar)) end else local nxt = start.next while not done and nxt and nxt.id == glyph_code and nxt.subtype<256 and nxt.font == currentfont do local nextchar = nxt.char if marks[nextchar] then -- should not happen (maybe warning) nxt = nxt.next else local entryanchors = descriptions[nextchar] if entryanchors then entryanchors = entryanchors.anchors if entryanchors then entryanchors = entryanchors['centry'] if entryanchors then local al = anchorlookups[lookupname] for anchor, entry in next, entryanchors do if al[anchor] then local exit = exitanchors[anchor] if exit then local dx, dy, bound = setcursive(start,nxt,tfmdata.parameters.factor,rlmode,exit,entry,characters[startchar],characters[nextchar]) if trace_cursive then logprocess("%s: moving %s to %s cursive (%s,%s) using anchor %s and bound %s in rlmode %s",pref(kind,lookupname),gref(startchar),gref(nextchar),dx,dy,anchor,bound,rlmode) end done = true break end end end end end else -- if trace_bugs then -- logwarning("%s: char %s is missing in font",pref(kind,lookupname),gref(startchar)) onetimemessage(currentfont,startchar,"no entry anchors",report_fonts) end break end end end return start, done else if trace_cursive and trace_details then logprocess("%s, cursive %s is already done",pref(kind,lookupname),gref(start.char),alreadydone) end return start, false end end function handlers.gpos_single(start,kind,lookupname,kerns,sequence) local startchar = start.char local dx, dy, w, h = setpair(start,tfmdata.parameters.factor,rlmode,sequence.flags[4],kerns,characters[startchar]) if trace_kerns then logprocess("%s: shifting single %s by (%s,%s) and correction (%s,%s)",pref(kind,lookupname),gref(startchar),dx,dy,w,h) end return start, false end function handlers.gpos_pair(start,kind,lookupname,kerns,sequence) -- todo: kerns in disc nodes: pre, post, replace -> loop over disc too -- todo: kerns in components of ligatures local snext = start.next if not snext then return start, false else local prev, done = start, false local factor = tfmdata.parameters.factor local lookuptype = lookuptypes[lookupname] while snext and snext.id == glyph_code and snext.subtype<256 and snext.font == currentfont do local nextchar = snext.char local krn = kerns[nextchar] if not krn and marks[nextchar] then prev = snext snext = snext.next else local krn = kerns[nextchar] if not krn then -- skip elseif type(krn) == "table" then if lookuptype == "pair" then -- probably not needed local a, b = krn[2], krn[3] if a and #a > 0 then local startchar = start.char local x, y, w, h = setpair(start,factor,rlmode,sequence.flags[4],a,characters[startchar]) if trace_kerns then logprocess("%s: shifting first of pair %s and %s by (%s,%s) and correction (%s,%s)",pref(kind,lookupname),gref(startchar),gref(nextchar),x,y,w,h) end end if b and #b > 0 then local startchar = start.char local x, y, w, h = setpair(snext,factor,rlmode,sequence.flags[4],b,characters[nextchar]) if trace_kerns then logprocess("%s: shifting second of pair %s and %s by (%s,%s) and correction (%s,%s)",pref(kind,lookupname),gref(startchar),gref(nextchar),x,y,w,h) end end else -- wrong ... position has different entries report_process("%s: check this out (old kern stuff)",pref(kind,lookupname)) -- local a, b = krn[2], krn[6] -- if a and a ~= 0 then -- local k = setkern(snext,factor,rlmode,a) -- if trace_kerns then -- logprocess("%s: inserting first kern %s between %s and %s",pref(kind,lookupname),k,gref(prev.char),gref(nextchar)) -- end -- end -- if b and b ~= 0 then -- logwarning("%s: ignoring second kern xoff %s",pref(kind,lookupname),b*factor) -- end end done = true elseif krn ~= 0 then local k = setkern(snext,factor,rlmode,krn) if trace_kerns then logprocess("%s: inserting kern %s between %s and %s",pref(kind,lookupname),k,gref(prev.char),gref(nextchar)) end done = true end break end end return start, done end end --[[ldx--I will implement multiple chain replacements once I run into a font that uses it. It's not that complex to handle.
--ldx]]-- local chainmores = { } local chainprocs = { } local function logprocess(...) if trace_steps then registermessage(...) end report_subchain(...) end local logwarning = report_subchain local function logprocess(...) if trace_steps then registermessage(...) end report_chain(...) end local logwarning = report_chain -- We could share functions but that would lead to extra function calls with many -- arguments, redundant tests and confusing messages. function chainprocs.chainsub(start,stop,kind,chainname,currentcontext,lookuphash,lookuplist,chainlookupname) logwarning("%s: a direct call to chainsub cannot happen",cref(kind,chainname,chainlookupname)) return start, false end function chainmores.chainsub(start,stop,kind,chainname,currentcontext,lookuphash,lookuplist,chainlookupname,n) logprocess("%s: a direct call to chainsub cannot happen",cref(kind,chainname,chainlookupname)) return start, false end -- The reversesub is a special case, which is why we need to store the replacements -- in a bit weird way. There is no lookup and the replacement comes from the lookup -- itself. It is meant mostly for dealing with Urdu. function chainprocs.reversesub(start,stop,kind,chainname,currentcontext,lookuphash,replacements) local char = start.char local replacement = replacements[char] if replacement then if trace_singles then logprocess("%s: single reverse replacement of %s by %s",cref(kind,chainname),gref(char),gref(replacement)) end start.char = replacement return start, true else return start, false end end --[[ldx--This chain stuff is somewhat tricky since we can have a sequence of actions to be applied: single, alternate, multiple or ligature where ligature can be an invalid one in the sense that it will replace multiple by one but not neccessary one that looks like the combination (i.e. it is the counterpart of multiple then). For example, the following is valid:
Therefore we we don't really do the replacement here already unless we have the single lookup case. The efficiency of the replacements can be improved by deleting as less as needed but that would also make the code even more messy.
--ldx]]-- local function delete_till_stop(start,stop,ignoremarks) -- keeps start local n = 1 if start == stop then -- done elseif ignoremarks then repeat -- start x x m x x stop => start m local next = start.next if not marks[next.char] then delete_node(start,next) end n = n + 1 until next == stop else -- start x x x stop => start repeat local next = start.next delete_node(start,next) n = n + 1 until next == stop end return n end --[[ldx--Here we replace start by a single variant, First we delete the rest of the match.
--ldx]]-- function chainprocs.gsub_single(start,stop,kind,chainname,currentcontext,lookuphash,currentlookup,chainlookupname,chainindex) -- todo: marks ? local current = start local subtables = currentlookup.subtables if #subtables > 1 then logwarning("todo: check if we need to loop over the replacements: %s",concat(subtables," ")) end while current do if current.id == glyph_code then local currentchar = current.char local lookupname = subtables[1] -- only 1 local replacement = lookuphash[lookupname] if not replacement then if trace_bugs then logwarning("%s: no single hits",cref(kind,chainname,chainlookupname,lookupname,chainindex)) end else replacement = replacement[currentchar] if not replacement then if trace_bugs then logwarning("%s: no single for %s",cref(kind,chainname,chainlookupname,lookupname,chainindex),gref(currentchar)) end else if trace_singles then logprocess("%s: replacing single %s by %s",cref(kind,chainname,chainlookupname,lookupname,chainindex),gref(currentchar),gref(replacement)) end current.char = replacement end end return start, true elseif current == stop then break else current = current.next end end return start, false end chainmores.gsub_single = chainprocs.gsub_single --[[ldx--Here we replace start by a sequence of new glyphs. First we delete the rest of the match.
--ldx]]-- function chainprocs.gsub_multiple(start,stop,kind,chainname,currentcontext,lookuphash,currentlookup,chainlookupname) delete_till_stop(start,stop) -- we could pass ignoremarks as #3 .. local startchar = start.char local subtables = currentlookup.subtables local lookupname = subtables[1] local replacements = lookuphash[lookupname] if not replacements then if trace_bugs then logwarning("%s: no multiple hits",cref(kind,chainname,chainlookupname,lookupname)) end else replacements = replacements[startchar] if not replacements then if trace_bugs then logwarning("%s: no multiple for %s",cref(kind,chainname,chainlookupname,lookupname),gref(startchar)) end else if trace_multiples then logprocess("%s: replacing %s by multiple characters %s",cref(kind,chainname,chainlookupname,lookupname),gref(startchar),gref(replacements)) end return multiple_glyphs(start,replacements) end end return start, false end -- function chainmores.gsub_multiple(start,stop,kind,chainname,currentcontext,lookuphash,currentlookup,chainlookupname,n) -- logprocess("%s: gsub_multiple not yet supported",cref(kind,chainname,chainlookupname)) -- return start, false -- end chainmores.gsub_multiple = chainprocs.gsub_multiple --[[ldx--Here we replace start by new glyph. First we delete the rest of the match.
--ldx]]-- -- char_1 mark_1 -> char_x mark_1 (ignore marks) -- char_1 mark_1 -> char_x -- to be checked: do we always have just one glyph? -- we can also have alternates for marks -- marks come last anyway -- are there cases where we need to delete the mark function chainprocs.gsub_alternate(start,stop,kind,chainname,currentcontext,lookuphash,currentlookup,chainlookupname) local current = start local subtables = currentlookup.subtables local value = featurevalue == true and tfmdata.shared.features[kind] or featurevalue while current do if current.id == glyph_code then -- is this check needed? local currentchar = current.char local lookupname = subtables[1] local alternatives = lookuphash[lookupname] if not alternatives then if trace_bugs then logwarning("%s: no alternative hit",cref(kind,chainname,chainlookupname,lookupname)) end else alternatives = alternatives[currentchar] if alternatives then local choice = get_alternative_glyph(current,alternatives,value) if choice then if trace_alternatives then logprocess("%s: replacing %s by alternative %s (%s)",cref(kind,chainname,chainlookupname,lookupname),gref(char),gref(choice),choice) end start.char = choice else if trace_alternatives then logwarning("%s: no variant %s for %s",cref(kind,chainname,chainlookupname,lookupname),tostring(value),gref(char)) end end elseif trace_bugs then logwarning("%s: no alternative for %s",cref(kind,chainname,chainlookupname,lookupname),gref(currentchar)) end end return start, true elseif current == stop then break else current = current.next end end return start, false end -- function chainmores.gsub_alternate(start,stop,kind,chainname,currentcontext,lookuphash,currentlookup,chainlookupname,n) -- logprocess("%s: gsub_alternate not yet supported",cref(kind,chainname,chainlookupname)) -- return start, false -- end chainmores.gsub_alternate = chainprocs.gsub_alternate --[[ldx--When we replace ligatures we use a helper that handles the marks. I might change this function (move code inline and handle the marks by a separate function). We assume rather stupid ligatures (no complex disc nodes).
--ldx]]-- function chainprocs.gsub_ligature(start,stop,kind,chainname,currentcontext,lookuphash,currentlookup,chainlookupname,chainindex) local startchar = start.char local subtables = currentlookup.subtables local lookupname = subtables[1] local ligatures = lookuphash[lookupname] if not ligatures then if trace_bugs then logwarning("%s: no ligature hits",cref(kind,chainname,chainlookupname,lookupname,chainindex)) end else ligatures = ligatures[startchar] if not ligatures then if trace_bugs then logwarning("%s: no ligatures starting with %s",cref(kind,chainname,chainlookupname,lookupname,chainindex),gref(startchar)) end else local s, discfound, last, nofreplacements = start.next, false, stop, 0 while s do local id = s.id if id == disc_code then s = s.next discfound = true else local schar = s.char if marks[schar] then -- marks s = s.next else local lg = ligatures[schar] if lg then ligatures, last, nofreplacements = lg, s, nofreplacements + 1 if s == stop then break else s = s.next end else break end end end end local l2 = ligatures.ligature if l2 then if chainindex then stop = last end if trace_ligatures then if start == stop then logprocess("%s: replacing character %s by ligature %s",cref(kind,chainname,chainlookupname,lookupname,chainindex),gref(startchar),gref(l2)) else logprocess("%s: replacing character %s upto %s by ligature %s",cref(kind,chainname,chainlookupname,lookupname,chainindex),gref(startchar),gref(stop.char),gref(l2)) end end start = toligature(kind,lookupname,start,stop,l2,currentlookup.flags[1],discfound) return start, true, nofreplacements elseif trace_bugs then if start == stop then logwarning("%s: replacing character %s by ligature fails",cref(kind,chainname,chainlookupname,lookupname,chainindex),gref(startchar)) else logwarning("%s: replacing character %s upto %s by ligature fails",cref(kind,chainname,chainlookupname,lookupname,chainindex),gref(startchar),gref(stop.char)) end end end end return start, false, 0 end chainmores.gsub_ligature = chainprocs.gsub_ligature function chainprocs.gpos_mark2base(start,stop,kind,chainname,currentcontext,lookuphash,currentlookup,chainlookupname) local markchar = start.char if marks[markchar] then local subtables = currentlookup.subtables local lookupname = subtables[1] local markanchors = lookuphash[lookupname] if markanchors then markanchors = markanchors[markchar] end if markanchors then local base = start.prev -- [glyph] [start=mark] if base and base.id == glyph_code and base.subtype<256 and base.font == currentfont then local basechar = base.char if marks[basechar] then while true do base = base.prev if base and base.id == glyph_code and base.subtype<256 and base.font == currentfont then basechar = base.char if not marks[basechar] then break end else if trace_bugs then logwarning("%s: no base for mark %s",pref(kind,lookupname),gref(markchar)) end return start, false end end end local baseanchors = descriptions[basechar].anchors if baseanchors then local baseanchors = baseanchors['basechar'] if baseanchors then local al = anchorlookups[lookupname] for anchor,ba in next, baseanchors do if al[anchor] then local ma = markanchors[anchor] if ma then local dx, dy, bound = setmark(start,base,tfmdata.parameters.factor,rlmode,ba,ma) if trace_marks then logprocess("%s, anchor %s, bound %s: anchoring mark %s to basechar %s => (%s,%s)", cref(kind,chainname,chainlookupname,lookupname),anchor,bound,gref(markchar),gref(basechar),dx,dy) end return start, true end end end if trace_bugs then logwarning("%s, no matching anchors for mark %s and base %s",cref(kind,chainname,chainlookupname,lookupname),gref(markchar),gref(basechar)) end end end elseif trace_bugs then logwarning("%s: prev node is no char",cref(kind,chainname,chainlookupname,lookupname)) end elseif trace_bugs then logwarning("%s: mark %s has no anchors",cref(kind,chainname,chainlookupname,lookupname),gref(markchar)) end elseif trace_bugs then logwarning("%s: mark %s is no mark",cref(kind,chainname,chainlookupname),gref(markchar)) end return start, false end function chainprocs.gpos_mark2ligature(start,stop,kind,chainname,currentcontext,lookuphash,currentlookup,chainlookupname) local markchar = start.char if marks[markchar] then local subtables = currentlookup.subtables local lookupname = subtables[1] local markanchors = lookuphash[lookupname] if markanchors then markanchors = markanchors[markchar] end if markanchors then local base = start.prev -- [glyph] [optional marks] [start=mark] if base and base.id == glyph_code and base.subtype<256 and base.font == currentfont then local basechar = base.char if marks[basechar] then while true do base = base.prev if base and base.id == glyph_code and base.subtype<256 and base.font == currentfont then basechar = base.char if not marks[basechar] then break end else if trace_bugs then logwarning("%s: no base for mark %s",cref(kind,chainname,chainlookupname,lookupname),markchar) end return start, false end end end -- todo: like marks a ligatures hash local index = has_attribute(start,ligacomp) local baseanchors = descriptions[basechar].anchors if baseanchors then local baseanchors = baseanchors['baselig'] if baseanchors then local al = anchorlookups[lookupname] for anchor,ba in next, baseanchors do if al[anchor] then local ma = markanchors[anchor] if ma then ba = ba[index] if ba then local dx, dy, bound = setmark(start,base,tfmdata.parameters.factor,rlmode,ba,ma) -- index if trace_marks then logprocess("%s, anchor %s, bound %s: anchoring mark %s to baselig %s at index %s => (%s,%s)", cref(kind,chainname,chainlookupname,lookupname),anchor,a or bound,gref(markchar),gref(basechar),index,dx,dy) end return start, true end end end end if trace_bugs then logwarning("%s: no matching anchors for mark %s and baselig %s",cref(kind,chainname,chainlookupname,lookupname),gref(markchar),gref(basechar)) end end end elseif trace_bugs then logwarning("feature %s, lookup %s: prev node is no char",kind,lookupname) end elseif trace_bugs then logwarning("%s: mark %s has no anchors",cref(kind,chainname,chainlookupname,lookupname),gref(markchar)) end elseif trace_bugs then logwarning("%s: mark %s is no mark",cref(kind,chainname,chainlookupname),gref(markchar)) end return start, false end function chainprocs.gpos_mark2mark(start,stop,kind,chainname,currentcontext,lookuphash,currentlookup,chainlookupname) local markchar = start.char if marks[markchar] then --~ local alreadydone = markonce and has_attribute(start,markmark) --~ if not alreadydone then -- local markanchors = descriptions[markchar].anchors markanchors = markanchors and markanchors.mark local subtables = currentlookup.subtables local lookupname = subtables[1] local markanchors = lookuphash[lookupname] if markanchors then markanchors = markanchors[markchar] end if markanchors then local base = start.prev -- [glyph] [basemark] [start=mark] -- while (base and has_attribute(base,ligacomp) and has_attribute(base,ligacomp) ~= has_attribute(start,ligacomp)) do -- base = base.prev -- KE: prevents mkmk for marks on different components of a ligature -- end local slc = has_attribute(start,ligacomp) if slc then -- a rather messy loop ... needs checking with husayni while base do local blc = has_attribute(base,ligacomp) if blc and blc ~= slc then base = base.prev else break end end end if base and base.id == glyph_code and base.subtype<256 and base.font == currentfont then -- subtype test can go local basechar = base.char local baseanchors = descriptions[basechar].anchors if baseanchors then baseanchors = baseanchors['basemark'] if baseanchors then local al = anchorlookups[lookupname] for anchor,ba in next, baseanchors do if al[anchor] then local ma = markanchors[anchor] if ma then local dx, dy, bound = setmark(start,base,tfmdata.parameters.factor,rlmode,ba,ma) if trace_marks then logprocess("%s, anchor %s, bound %s: anchoring mark %s to basemark %s => (%s,%s)", cref(kind,chainname,chainlookupname,lookupname),anchor,bound,gref(markchar),gref(basechar),dx,dy) end return start, true end end end if trace_bugs then logwarning("%s: no matching anchors for mark %s and basemark %s",gref(kind,chainname,chainlookupname,lookupname),gref(markchar),gref(basechar)) end end end elseif trace_bugs then logwarning("%s: prev node is no mark",cref(kind,chainname,chainlookupname,lookupname)) end elseif trace_bugs then logwarning("%s: mark %s has no anchors",cref(kind,chainname,chainlookupname,lookupname),gref(markchar)) end --~ elseif trace_marks and trace_details then --~ logprocess("%s, mark %s is already bound (n=%s), ignoring mark2mark",pref(kind,lookupname),gref(markchar),alreadydone) --~ end elseif trace_bugs then logwarning("%s: mark %s is no mark",cref(kind,chainname,chainlookupname),gref(markchar)) end return start, false end -- ! ! ! untested ! ! ! function chainprocs.gpos_cursive(start,stop,kind,chainname,currentcontext,lookuphash,currentlookup,chainlookupname) local alreadydone = cursonce and has_attribute(start,cursbase) if not alreadydone then local startchar = start.char local subtables = currentlookup.subtables local lookupname = subtables[1] local exitanchors = lookuphash[lookupname] if exitanchors then exitanchors = exitanchors[startchar] end if exitanchors then local done = false if marks[startchar] then if trace_cursive then logprocess("%s: ignoring cursive for mark %s",pref(kind,lookupname),gref(startchar)) end else local nxt = start.next while not done and nxt and nxt.id == glyph_code and nxt.subtype<256 and nxt.font == currentfont do local nextchar = nxt.char if marks[nextchar] then -- should not happen (maybe warning) nxt = nxt.next else local entryanchors = descriptions[nextchar] if entryanchors then entryanchors = entryanchors.anchors if entryanchors then entryanchors = entryanchors['centry'] if entryanchors then local al = anchorlookups[lookupname] for anchor, entry in next, entryanchors do if al[anchor] then local exit = exitanchors[anchor] if exit then local dx, dy, bound = setcursive(start,nxt,tfmdata.parameters.factor,rlmode,exit,entry,characters[startchar],characters[nextchar]) if trace_cursive then logprocess("%s: moving %s to %s cursive (%s,%s) using anchor %s and bound %s in rlmode %s",pref(kind,lookupname),gref(startchar),gref(nextchar),dx,dy,anchor,bound,rlmode) end done = true break end end end end end else -- if trace_bugs then -- logwarning("%s: char %s is missing in font",pref(kind,lookupname),gref(startchar)) onetimemessage(currentfont,startchar,"no entry anchors",report_fonts) end break end end end return start, done else if trace_cursive and trace_details then logprocess("%s, cursive %s is already done",pref(kind,lookupname),gref(start.char),alreadydone) end return start, false end end return start, false end function chainprocs.gpos_single(start,stop,kind,chainname,currentcontext,lookuphash,currentlookup,chainlookupname,chainindex,sequence) -- untested .. needs checking for the new model local startchar = start.char local subtables = currentlookup.subtables local lookupname = subtables[1] local kerns = lookuphash[lookupname] if kerns then kerns = kerns[startchar] -- needed ? if kerns then local dx, dy, w, h = setpair(start,tfmdata.parameters.factor,rlmode,sequence.flags[4],kerns,characters[startchar]) if trace_kerns then logprocess("%s: shifting single %s by (%s,%s) and correction (%s,%s)",cref(kind,chainname,chainlookupname),gref(startchar),dx,dy,w,h) end end end return start, false end -- when machines become faster i will make a shared function function chainprocs.gpos_pair(start,stop,kind,chainname,currentcontext,lookuphash,currentlookup,chainlookupname,chainindex,sequence) -- logwarning("%s: gpos_pair not yet supported",cref(kind,chainname,chainlookupname)) local snext = start.next if snext then local startchar = start.char local subtables = currentlookup.subtables local lookupname = subtables[1] local kerns = lookuphash[lookupname] if kerns then kerns = kerns[startchar] if kerns then local lookuptype = lookuptypes[lookupname] local prev, done = start, false local factor = tfmdata.parameters.factor while snext and snext.id == glyph_code and snext.subtype<256 and snext.font == currentfont do local nextchar = snext.char local krn = kerns[nextchar] if not krn and marks[nextchar] then prev = snext snext = snext.next else if not krn then -- skip elseif type(krn) == "table" then if lookuptype == "pair" then local a, b = krn[2], krn[3] if a and #a > 0 then local startchar = start.char local x, y, w, h = setpair(start,factor,rlmode,sequence.flags[4],a,characters[startchar]) if trace_kerns then logprocess("%s: shifting first of pair %s and %s by (%s,%s) and correction (%s,%s)",cref(kind,chainname,chainlookupname),gref(startchar),gref(nextchar),x,y,w,h) end end if b and #b > 0 then local startchar = start.char local x, y, w, h = setpair(snext,factor,rlmode,sequence.flags[4],b,characters[nextchar]) if trace_kerns then logprocess("%s: shifting second of pair %s and %s by (%s,%s) and correction (%s,%s)",cref(kind,chainname,chainlookupname),gref(startchar),gref(nextchar),x,y,w,h) end end else report_process("%s: check this out (old kern stuff)",cref(kind,chainname,chainlookupname)) local a, b = krn[2], krn[6] if a and a ~= 0 then local k = setkern(snext,factor,rlmode,a) if trace_kerns then logprocess("%s: inserting first kern %s between %s and %s",cref(kind,chainname,chainlookupname),k,gref(prev.char),gref(nextchar)) end end if b and b ~= 0 then logwarning("%s: ignoring second kern xoff %s",cref(kind,chainname,chainlookupname),b*factor) end end done = true elseif krn ~= 0 then local k = setkern(snext,factor,rlmode,krn) if trace_kerns then logprocess("%s: inserting kern %s between %s and %s",cref(kind,chainname,chainlookupname),k,gref(prev.char),gref(nextchar)) end done = true end break end end return start, done end end end return start, false end -- what pointer to return, spec says stop -- to be discussed ... is bidi changer a space? -- elseif char == zwnj and sequence[n][32] then -- brrr -- somehow l or f is global -- we don't need to pass the currentcontext, saves a bit -- make a slow variant then can be activated but with more tracing local function show_skip(kind,chainname,char,ck,class) if ck[9] then logwarning("%s: skipping char %s (%s) in rule %s, lookuptype %s (%s=>%s)",cref(kind,chainname),gref(char),class,ck[1],ck[2],ck[9],ck[10]) else logwarning("%s: skipping char %s (%s) in rule %s, lookuptype %s",cref(kind,chainname),gref(char),class,ck[1],ck[2]) end end local function normal_handle_contextchain(start,kind,chainname,contexts,sequence,lookuphash) -- local rule, lookuptype, sequence, f, l, lookups = ck[1], ck[2] ,ck[3], ck[4], ck[5], ck[6] local flags = sequence.flags local done = false local skipmark = flags[1] local skipligature = flags[2] local skipbase = flags[3] local someskip = skipmark or skipligature or skipbase -- could be stored in flags for a fast test (hm, flags could be false !) local markclass = sequence.markclass -- todo, first we need a proper test local skipped = false for k=1,#contexts do local match = true local current = start local last = start local ck = contexts[k] local seq = ck[3] local s = #seq -- f..l = mid string if s == 1 then -- never happens match = current.id == glyph_code and current.subtype<256 and current.font == currentfont and seq[1][current.char] else -- maybe we need a better space check (maybe check for glue or category or combination) -- we cannot optimize for n=2 because there can be disc nodes local f, l = ck[4], ck[5] -- current match if f == 1 and f == l then -- current only -- already a hit -- match = true else -- before/current/after | before/current | current/after -- no need to test first hit (to be optimized) if f == l then -- new, else last out of sync (f is > 1) -- match = true else local n = f + 1 last = last.next while n <= l do if last then local id = last.id if id == glyph_code then if last.subtype<256 and last.font == currentfont then local char = last.char local ccd = descriptions[char] if ccd then local class = ccd.class if class == skipmark or class == skipligature or class == skipbase or (markclass and class == "mark" and not markclass[char]) then skipped = true if trace_skips then show_skip(kind,chainname,char,ck,class) end last = last.next elseif seq[n][char] then if n < l then last = last.next end n = n + 1 else match = false break end else match = false break end else match = false break end elseif id == disc_code then last = last.next else match = false break end else match = false break end end end end -- before if match and f > 1 then local prev = start.prev if prev then local n = f-1 while n >= 1 do if prev then local id = prev.id if id == glyph_code then if prev.subtype<256 and prev.font == currentfont then -- normal char local char = prev.char local ccd = descriptions[char] if ccd then local class = ccd.class if class == skipmark or class == skipligature or class == skipbase or (markclass and class == "mark" and not markclass[char]) then skipped = true if trace_skips then show_skip(kind,chainname,char,ck,class) end elseif seq[n][char] then n = n -1 else match = false break end else match = false break end else match = false break end elseif id == disc_code then -- skip 'm elseif seq[n][32] then n = n -1 else match = false break end prev = prev.prev elseif seq[n][32] then -- somehat special, as zapfino can have many preceding spaces n = n -1 else match = false break end end elseif f == 2 then match = seq[1][32] else for n=f-1,1 do if not seq[n][32] then match = false break end end end end -- after if match and s > l then local current = last and last.next if current then -- removed optimization for s-l == 1, we have to deal with marks anyway local n = l + 1 while n <= s do if current then local id = current.id if id == glyph_code then if current.subtype<256 and current.font == currentfont then -- normal char local char = current.char local ccd = descriptions[char] if ccd then local class = ccd.class if class == skipmark or class == skipligature or class == skipbase or (markclass and class == "mark" and not markclass[char]) then skipped = true if trace_skips then show_skip(kind,chainname,char,ck,class) end elseif seq[n][char] then n = n + 1 else match = false break end else match = false break end else match = false break end elseif id == disc_code then -- skip 'm elseif seq[n][32] then -- brrr n = n + 1 else match = false break end current = current.next elseif seq[n][32] then n = n + 1 else match = false break end end elseif s-l == 1 then match = seq[s][32] else for n=l+1,s do if not seq[n][32] then match = false break end end end end end if match then -- ck == currentcontext if trace_contexts then local rule, lookuptype, f, l = ck[1], ck[2], ck[4], ck[5] local char = start.char if ck[9] then logwarning("%s: rule %s matches at char %s for (%s,%s,%s) chars, lookuptype %s (%s=>%s)", cref(kind,chainname),rule,gref(char),f-1,l-f+1,s-l,lookuptype,ck[9],ck[10]) else logwarning("%s: rule %s matches at char %s for (%s,%s,%s) chars, lookuptype %s", cref(kind,chainname),rule,gref(char),f-1,l-f+1,s-l,lookuptype) end end local chainlookups = ck[6] if chainlookups then local nofchainlookups = #chainlookups -- we can speed this up if needed if nofchainlookups == 1 then local chainlookupname = chainlookups[1] local chainlookup = lookuptable[chainlookupname] if chainlookup then local cp = chainprocs[chainlookup.type] if cp then start, done = cp(start,last,kind,chainname,ck,lookuphash,chainlookup,chainlookupname,nil,sequence) else logprocess("%s: %s is not yet supported",cref(kind,chainname,chainlookupname),chainlookup.type) end else -- shouldn't happen logprocess("%s is not yet supported",cref(kind,chainname,chainlookupname)) end else local i = 1 repeat if skipped then while true do local char = start.char local ccd = descriptions[char] if ccd then local class = ccd.class if class == skipmark or class == skipligature or class == skipbase or (markclass and class == "mark" and not markclass[char]) then start = start.next else break end else break end end end local chainlookupname = chainlookups[i] local chainlookup = lookuptable[chainlookupname] -- can be false (n matches,Analyzers run per script and/or language and are needed in order to process features right.
--ldx]]-- -- todo: analyzers per script/lang, cross font, so we need an font id hash -> script -- e.g. latin -> hyphenate, arab -> 1/2/3 analyze -- its own namespace local state = attributes.private('state') function analyzers.setstate(head,font) local useunicodemarks = analyzers.useunicodemarks local tfmdata = fontdata[font] local characters = tfmdata.characters local descriptions = tfmdata.descriptions local first, last, current, n, done = nil, nil, head, 0, false -- maybe make n boolean while current do local id = current.id if id == glyph_code and current.font == font then local char = current.char local d = descriptions[char] if d then if d.class == "mark" or (useunicodemarks and categories[char] == "mn") then done = true set_attribute(current,state,5) -- mark elseif n == 0 then first, last, n = current, current, 1 set_attribute(current,state,1) -- init else last, n = current, n+1 set_attribute(current,state,2) -- medi end else -- finish if first and first == last then set_attribute(last,state,4) -- isol elseif last then set_attribute(last,state,3) -- fina end first, last, n = nil, nil, 0 end elseif id == disc_code then -- always in the middle set_attribute(current,state,2) -- midi last = current else -- finish if first and first == last then set_attribute(last,state,4) -- isol elseif last then set_attribute(last,state,3) -- fina end first, last, n = nil, nil, 0 end current = current.next end if first and first == last then set_attribute(last,state,4) -- isol elseif last then set_attribute(last,state,3) -- fina end return head, done end -- in the future we will use language/script attributes instead of the -- font related value, but then we also need dynamic features which is -- somewhat slower; and .. we need a chain of them local function analyzeinitializer(tfmdata,value) -- attr local script, language = otf.scriptandlanguage(tfmdata) -- attr local action = initializers[script] if action then if type(action) == "function" then return action(tfmdata,value) else local action = action[language] if action then return action(tfmdata,value) end end end end local function analyzeprocessor(head,font,attr) local tfmdata = fontdata[font] local script, language = otf.scriptandlanguage(tfmdata,attr) local action = methods[script] if action then if type(action) == "function" then return action(head,font,attr) else action = action[language] if action then return action(head,font,attr) end end end return head, false end registerotffeature { name = "analyze", description = "analysis of (for instance) character classes", default = true, initializers = { node = analyzeinitializer, }, processors = { position = 1, node = analyzeprocessor, } } -- latin methods.latn = analyzers.setstate -- this info eventually will go into char-def and we will have a state -- table for generic then local zwnj = 0x200C local zwj = 0x200D local isol = { [0x0600] = true, [0x0601] = true, [0x0602] = true, [0x0603] = true, [0x0608] = true, [0x060B] = true, [0x0621] = true, [0x0674] = true, [0x06DD] = true, [zwnj] = true, } local isol_fina = { [0x0622] = true, [0x0623] = true, [0x0624] = true, [0x0625] = true, [0x0627] = true, [0x0629] = true, [0x062F] = true, [0x0630] = true, [0x0631] = true, [0x0632] = true, [0x0648] = true, [0x0671] = true, [0x0672] = true, [0x0673] = true, [0x0675] = true, [0x0676] = true, [0x0677] = true, [0x0688] = true, [0x0689] = true, [0x068A] = true, [0x068B] = true, [0x068C] = true, [0x068D] = true, [0x068E] = true, [0x068F] = true, [0x0690] = true, [0x0691] = true, [0x0692] = true, [0x0693] = true, [0x0694] = true, [0x0695] = true, [0x0696] = true, [0x0697] = true, [0x0698] = true, [0x0699] = true, [0x06C0] = true, [0x06C3] = true, [0x06C4] = true, [0x06C5] = true, [0x06C6] = true, [0x06C7] = true, [0x06C8] = true, [0x06C9] = true, [0x06CA] = true, [0x06CB] = true, [0x06CD] = true, [0x06CF] = true, [0x06D2] = true, [0x06D3] = true, [0x06D5] = true, [0x06EE] = true, [0x06EF] = true, [0x0759] = true, [0x075A] = true, [0x075B] = true, [0x076B] = true, [0x076C] = true, [0x0771] = true, [0x0773] = true, [0x0774] = true, [0x0778] = true, [0x0779] = true, [0xFEF5] = true, [0xFEF7] = true, [0xFEF9] = true, [0xFEFB] = true, -- syriac [0x0710] = true, [0x0715] = true, [0x0716] = true, [0x0717] = true, [0x0718] = true, [0x0719] = true, [0x0728] = true, [0x072A] = true, [0x072C] = true, [0x071E] = true, } local isol_fina_medi_init = { [0x0626] = true, [0x0628] = true, [0x062A] = true, [0x062B] = true, [0x062C] = true, [0x062D] = true, [0x062E] = true, [0x0633] = true, [0x0634] = true, [0x0635] = true, [0x0636] = true, [0x0637] = true, [0x0638] = true, [0x0639] = true, [0x063A] = true, [0x063B] = true, [0x063C] = true, [0x063D] = true, [0x063E] = true, [0x063F] = true, [0x0640] = true, [0x0641] = true, [0x0642] = true, [0x0643] = true, [0x0644] = true, [0x0645] = true, [0x0646] = true, [0x0647] = true, [0x0649] = true, [0x064A] = true, [0x066E] = true, [0x066F] = true, [0x0678] = true, [0x0679] = true, [0x067A] = true, [0x067B] = true, [0x067C] = true, [0x067D] = true, [0x067E] = true, [0x067F] = true, [0x0680] = true, [0x0681] = true, [0x0682] = true, [0x0683] = true, [0x0684] = true, [0x0685] = true, [0x0686] = true, [0x0687] = true, [0x069A] = true, [0x069B] = true, [0x069C] = true, [0x069D] = true, [0x069E] = true, [0x069F] = true, [0x06A0] = true, [0x06A1] = true, [0x06A2] = true, [0x06A3] = true, [0x06A4] = true, [0x06A5] = true, [0x06A6] = true, [0x06A7] = true, [0x06A8] = true, [0x06A9] = true, [0x06AA] = true, [0x06AB] = true, [0x06AC] = true, [0x06AD] = true, [0x06AE] = true, [0x06AF] = true, [0x06B0] = true, [0x06B1] = true, [0x06B2] = true, [0x06B3] = true, [0x06B4] = true, [0x06B5] = true, [0x06B6] = true, [0x06B7] = true, [0x06B8] = true, [0x06B9] = true, [0x06BA] = true, [0x06BB] = true, [0x06BC] = true, [0x06BD] = true, [0x06BE] = true, [0x06BF] = true, [0x06C1] = true, [0x06C2] = true, [0x06CC] = true, [0x06CE] = true, [0x06D0] = true, [0x06D1] = true, [0x06FA] = true, [0x06FB] = true, [0x06FC] = true, [0x06FF] = true, [0x0750] = true, [0x0751] = true, [0x0752] = true, [0x0753] = true, [0x0754] = true, [0x0755] = true, [0x0756] = true, [0x0757] = true, [0x0758] = true, [0x075C] = true, [0x075D] = true, [0x075E] = true, [0x075F] = true, [0x0760] = true, [0x0761] = true, [0x0762] = true, [0x0763] = true, [0x0764] = true, [0x0765] = true, [0x0766] = true, [0x0767] = true, [0x0768] = true, [0x0769] = true, [0x076A] = true, [0x076D] = true, [0x076E] = true, [0x076F] = true, [0x0770] = true, [0x0772] = true, [0x0775] = true, [0x0776] = true, [0x0777] = true, [0x077A] = true, [0x077B] = true, [0x077C] = true, [0x077D] = true, [0x077E] = true, [0x077F] = true, [zwj] = true, -- syriac [0x0712] = true, [0x0713] = true, [0x0714] = true, [0x071A] = true, [0x071B] = true, [0x071C] = true, [0x071D] = true, [0x071F] = true, [0x0720] = true, [0x0721] = true, [0x0722] = true, [0x0723] = true, [0x0725] = true, [0x0726] = true, [0x0727] = true, [0x0729] = true, [0x072B] = true, [0x0724] = true, [0x0706] = true, [0x0707] = true, } local arab_warned = { } -- todo: gref local function warning(current,what) local char = current.char if not arab_warned[char] then log.report("analyze","arab: character %s (U+%05X) has no %s class", char, char, what) arab_warned[char] = true end end function methods.nocolor(head,font,attr) for n in traverse_id(glyph_code,head) do if not font or n.font == font then resetnodecolor(n) end end return head, true end local function finish(first,last) if last then if first == last then local fc = first.char if isol_fina_medi_init[fc] or isol_fina[fc] then set_attribute(first,state,4) -- isol if trace_analyzing then setnodecolor(first,"font:isol") end else warning(first,"isol") set_attribute(first,state,0) -- error if trace_analyzing then resetnodecolor(first) end end else local lc = last.char if isol_fina_medi_init[lc] or isol_fina[lc] then -- why isol here ? -- if laststate == 1 or laststate == 2 or laststate == 4 then set_attribute(last,state,3) -- fina if trace_analyzing then setnodecolor(last,"font:fina") end else warning(last,"fina") set_attribute(last,state,0) -- error if trace_analyzing then resetnodecolor(last) end end end first, last = nil, nil elseif first then -- first and last are either both set so we never com here local fc = first.char if isol_fina_medi_init[fc] or isol_fina[fc] then set_attribute(first,state,4) -- isol if trace_analyzing then setnodecolor(first,"font:isol") end else warning(first,"isol") set_attribute(first,state,0) -- error if trace_analyzing then resetnodecolor(first) end end first = nil end return first, last end function methods.arab(head,font,attr) -- maybe make a special version with no trace local useunicodemarks = analyzers.useunicodemarks local tfmdata = fontdata[font] local marks = tfmdata.resources.marks local first, last, current, done = nil, nil, head, false while current do if current.id == glyph_code and current.subtype<256 and current.font == font and not has_attribute(current,state) then done = true local char = current.char if marks[char] or (useunicodemarks and categories[char] == "mn") then set_attribute(current,state,5) -- mark if trace_analyzing then setnodecolor(current,"font:mark") end elseif isol[char] then -- can be zwj or zwnj too first, last = finish(first,last) set_attribute(current,state,4) -- isol if trace_analyzing then setnodecolor(current,"font:isol") end first, last = nil, nil elseif not first then if isol_fina_medi_init[char] then set_attribute(current,state,1) -- init if trace_analyzing then setnodecolor(current,"font:init") end first, last = first or current, current elseif isol_fina[char] then set_attribute(current,state,4) -- isol if trace_analyzing then setnodecolor(current,"font:isol") end first, last = nil, nil else -- no arab first, last = finish(first,last) end elseif isol_fina_medi_init[char] then first, last = first or current, current set_attribute(current,state,2) -- medi if trace_analyzing then setnodecolor(current,"font:medi") end elseif isol_fina[char] then if not has_attribute(last,state,1) then -- tricky, we need to check what last may be ! set_attribute(last,state,2) -- medi if trace_analyzing then setnodecolor(last,"font:medi") end end set_attribute(current,state,3) -- fina if trace_analyzing then setnodecolor(current,"font:fina") end first, last = nil, nil elseif char >= 0x0600 and char <= 0x06FF then if trace_analyzing then setnodecolor(current,"font:rest") end first, last = finish(first,last) else --no first, last = finish(first,last) end else first, last = finish(first,last) end current = current.next end first, last = finish(first,last) return head, done end methods.syrc = methods.arab directives.register("otf.analyze.useunicodemarks",function(v) analyzers.useunicodemarks = v end) end -- closure do -- begin closure to overcome local limits and interference if not modules then modules = { } end modules ['luatex-fonts-lua'] = { version = 1.001, comment = "companion to luatex-*.tex", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } if context then texio.write_nl("fatal error: this module is not for context") os.exit() end local fonts = fonts fonts.formats.lua = "lua" function fonts.readers.lua(specification) local fullname = specification.filename or "" if fullname == "" then local forced = specification.forced or "" if forced ~= "" then fullname = specification.name .. "." .. forced else fullname = specification.name end end local fullname = resolvers.findfile(fullname) or "" if fullname ~= "" then local loader = loadfile(fullname) loader = loader and loader() return loader and loader(specification) end end end -- closure do -- begin closure to overcome local limits and interference if not modules then modules = { } end modules ['font-def'] = { version = 1.001, comment = "companion to font-ini.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } local concat = table.concat local format, gmatch, match, find, lower, gsub = string.format, string.gmatch, string.match, string.find, string.lower, string.gsub local tostring, next = tostring, next local lpegmatch = lpeg.match local allocate = utilities.storage.allocate local trace_defining = false trackers .register("fonts.defining", function(v) trace_defining = v end) local directive_embedall = false directives.register("fonts.embedall", function(v) directive_embedall = v end) trackers.register("fonts.loading", "fonts.defining", "otf.loading", "afm.loading", "tfm.loading") trackers.register("fonts.all", "fonts.*", "otf.*", "afm.*", "tfm.*") local report_defining = logs.reporter("fonts","defining") --[[ldx--Here we deal with defining fonts. We do so by intercepting the
default loader that only handles
We hardly gain anything when we cache the final (pre scaled)
We can prefix a font specification by
The following function split the font specification into components and prepares a table that will move along as we proceed.
--ldx]]-- -- beware, we discard additional specs -- -- method:name method:name(sub) method:name(sub)*spec method:name*spec -- name name(sub) name(sub)*spec name*spec -- name@spec*oeps local splitter, splitspecifiers = nil, "" local P, C, S, Cc = lpeg.P, lpeg.C, lpeg.S, lpeg.Cc local left = P("(") local right = P(")") local colon = P(":") local space = P(" ") definers.defaultlookup = "file" local prefixpattern = P(false) local function addspecifier(symbol) splitspecifiers = splitspecifiers .. symbol local method = S(splitspecifiers) local lookup = C(prefixpattern) * colon local sub = left * C(P(1-left-right-method)^1) * right local specification = C(method) * C(P(1)^1) local name = C((1-sub-specification)^1) splitter = P((lookup + Cc("")) * name * (sub + Cc("")) * (specification + Cc(""))) end local function addlookup(str,default) prefixpattern = prefixpattern + P(str) end definers.addlookup = addlookup addlookup("file") addlookup("name") addlookup("spec") local function getspecification(str) return lpegmatch(splitter,str) end definers.getspecification = getspecification function definers.registersplit(symbol,action,verbosename) addspecifier(symbol) variants[symbol] = action if verbosename then variants[verbosename] = action end end function definers.makespecification(specification,lookup,name,sub,method,detail,size) size = size or 655360 if trace_defining then report_defining("%s -> lookup: %s, name: %s, sub: %s, method: %s, detail: %s", specification, (lookup ~= "" and lookup) or "[file]", (name ~= "" and name) or "-", (sub ~= "" and sub) or "-", (method ~= "" and method) or "-", (detail ~= "" and detail) or "-") end if not lookup or lookup == "" then lookup = definers.defaultlookup end local t = { lookup = lookup, -- forced type specification = specification, -- full specification size = size, -- size in scaled points or -1000*n name = name, -- font or filename sub = sub, -- subfont (eg in ttc) method = method, -- specification method detail = detail, -- specification resolved = "", -- resolved font name forced = "", -- forced loader features = { }, -- preprocessed features } return t end function definers.analyze(specification, size) -- can be optimized with locals local lookup, name, sub, method, detail = getspecification(specification or "") return definers.makespecification(specification, lookup, name, sub, method, detail, size) end --[[ldx--We can resolve the filename using the next function:
--ldx]]-- definers.resolvers = definers.resolvers or { } local resolvers = definers.resolvers -- todo: reporter function resolvers.file(specification) local suffix = file.suffix(specification.name) if fonts.formats[suffix] then specification.forced = suffix specification.name = file.removesuffix(specification.name) end end function resolvers.name(specification) local resolve = fonts.names.resolve if resolve then local resolved, sub = resolve(specification.name,specification.sub,specification) -- we pass specification for overloaded versions if resolved then specification.resolved = resolved specification.sub = sub local suffix = file.suffix(resolved) if fonts.formats[suffix] then specification.forced = suffix specification.name = file.removesuffix(resolved) else specification.name = resolved end end else resolvers.file(specification) end end function resolvers.spec(specification) local resolvespec = fonts.names.resolvespec if resolvespec then local resolved, sub = resolvespec(specification.name,specification.sub,specification) -- we pass specification for overloaded versions if resolved then specification.resolved = resolved specification.sub = sub specification.forced = file.extname(resolved) specification.name = file.removesuffix(resolved) end else resolvers.name(specification) end end function definers.resolve(specification) if not specification.resolved or specification.resolved == "" then -- resolved itself not per se in mapping hash local r = resolvers[specification.lookup] if r then r(specification) end end if specification.forced == "" then specification.forced = nil else specification.forced = specification.forced end specification.hash = lower(specification.name .. ' @ ' .. constructors.hashfeatures(specification)) if specification.sub and specification.sub ~= "" then specification.hash = specification.sub .. ' @ ' .. specification.hash end return specification end --[[ldx--The main read function either uses a forced reader (as determined by a lookup) or tries to resolve the name using the list of readers.
We need to cache when possible. We do cache raw tfm data (from
Watch out, here we do load a font, but we don't prepare the specification yet.
--ldx]]-- -- very experimental: function definers.applypostprocessors(tfmdata) local postprocessors = tfmdata.postprocessors if postprocessors then for i=1,#postprocessors do local extrahash = postprocessors[i](tfmdata) -- after scaling etc if type(extrahash) == "string" and extrahash ~= "" then -- e.g. a reencoding needs this extrahash = gsub(lower(extrahash),"[^a-z]","-") tfmdata.properties.fullname = format("%s-%s",tfmdata.properties.fullname,extrahash) end end end return tfmdata end -- function definers.applypostprocessors(tfmdata) -- return tfmdata -- end local function checkembedding(tfmdata) local properties = tfmdata.properties local embedding if directive_embedall then embedding = "full" elseif properties and properties.filename and constructors.dontembed[properties.filename] then embedding = "no" else embedding = "subset" end if properties then properties.embedding = embedding else tfmdata.properties = { embedding = embedding } end tfmdata.embedding = embedding end function definers.loadfont(specification) local hash = constructors.hashinstance(specification) local tfmdata = loadedfonts[hash] -- hashes by size ! if not tfmdata then local forced = specification.forced or "" if forced ~= "" then local reader = readers[lower(forced)] tfmdata = reader and reader(specification) if not tfmdata then report_defining("forced type %s of %s not found",forced,specification.name) end else local sequence = readers.sequence -- can be overloaded so only a shortcut here for s=1,#sequence do local reader = sequence[s] if readers[reader] then -- we skip not loaded readers if trace_defining then report_defining("trying (reader sequence driven) type %s for %s with file %s",reader,specification.name,specification.filename or "unknown") end tfmdata = readers[reader](specification) if tfmdata then break else specification.filename = nil end end end end if tfmdata then tfmdata = definers.applypostprocessors(tfmdata) checkembedding(tfmdata) -- todo: general postprocessor loadedfonts[hash] = tfmdata designsizes[specification.hash] = tfmdata.parameters.designsize end end if not tfmdata then report_defining("font with asked name '%s' is not found using lookup '%s'",specification.name,specification.lookup) end return tfmdata end --[[ldx--For virtual fonts we need a slightly different approach:
--ldx]]-- function constructors.readanddefine(name,size) -- no id -- maybe a dummy first local specification = definers.analyze(name,size) local method = specification.method if method and variants[method] then specification = variants[method](specification) end specification = definers.resolve(specification) local hash = constructors.hashinstance(specification) local id = definers.registered(hash) if not id then local tfmdata = definers.loadfont(specification) if tfmdata then tfmdata.properties.hash = hash id = font.define(tfmdata) definers.register(tfmdata,id) else id = 0 -- signal end end return fontdata[id], id end --[[ldx--So far the specifiers. Now comes the real definer. Here we cache based on id's. Here we also intercept the virtual font handler. Since it evolved stepwise I may rewrite this bit (combine code).
In the previously defined reader (the one resulting in aWe overload the