summaryrefslogtreecommitdiff
path: root/Master/texmf-dist/tex/luatex/luaotfload/fontloader-l-lpeg.lua
diff options
context:
space:
mode:
authorKarl Berry <karl@freefriends.org>2018-09-26 20:50:34 +0000
committerKarl Berry <karl@freefriends.org>2018-09-26 20:50:34 +0000
commit78334b0d4607b242273f9d0cde94f6a6fba51832 (patch)
tree5b604ee30bb611eeeae95ed4218b4e7fd6eb00b8 /Master/texmf-dist/tex/luatex/luaotfload/fontloader-l-lpeg.lua
parent3a0140d05a7e481750ec23a30ed5cf3cdef61eb7 (diff)
luaotfload (26sep18)
git-svn-id: svn://tug.org/texlive/trunk@48771 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Master/texmf-dist/tex/luatex/luaotfload/fontloader-l-lpeg.lua')
-rw-r--r--Master/texmf-dist/tex/luatex/luaotfload/fontloader-l-lpeg.lua300
1 files changed, 192 insertions, 108 deletions
diff --git a/Master/texmf-dist/tex/luatex/luaotfload/fontloader-l-lpeg.lua b/Master/texmf-dist/tex/luatex/luaotfload/fontloader-l-lpeg.lua
index 959ca553eb8..750d5e698d7 100644
--- a/Master/texmf-dist/tex/luatex/luaotfload/fontloader-l-lpeg.lua
+++ b/Master/texmf-dist/tex/luatex/luaotfload/fontloader-l-lpeg.lua
@@ -6,6 +6,10 @@ if not modules then modules = { } end modules ['l-lpeg'] = {
license = "see context related readme files"
}
+-- we can get too many captures (e.g. on largexml files) which makes me wonder
+-- if P(foo)/"" can't be simplfied to N(foo) i.e. some direct instruction to the
+-- lpeg virtual machine to ignore it
+
-- lpeg 12 vs lpeg 10: slower compilation, similar parsing speed (i need to check
-- if i can use new features like capture / 2 and .B (at first sight the xml
-- parser is some 5% slower)
@@ -17,7 +21,13 @@ if not modules then modules = { } end modules ['l-lpeg'] = {
-- move utf -> l-unicode
-- move string -> l-string or keep it here
-lpeg = require("lpeg")
+-- lpeg.B : backward without consumption
+-- lpeg.F = getmetatable(lpeg.P(1)).__len : forward without consumption
+
+
+lpeg = require("lpeg") -- does lpeg register itself global?
+
+local lpeg = lpeg
-- The latest lpeg doesn't have print any more, and even the new ones are not
-- available by default (only when debug mode is enabled), which is a pitty as
@@ -103,11 +113,14 @@ patterns.alwaysmatched = alwaysmatched
local sign = S('+-')
local zero = P('0')
local digit = R('09')
+local digits = digit^1
local octdigit = R("07")
+local octdigits = octdigit^1
local lowercase = R("az")
local uppercase = R("AZ")
local underscore = P("_")
local hexdigit = digit + lowercase + uppercase
+local hexdigits = hexdigit^1
local cr, lf, crlf = P("\r"), P("\n"), P("\r\n")
----- newline = crlf + S("\r\n") -- cr + lf
local newline = P("\r") * (P("\n") + P(true)) + P("\n") -- P("\r")^-1 * P("\n")^-1
@@ -187,18 +200,20 @@ local fullstripper = whitespace^0 * C((whitespace^0 * nonwhitespace^1)^0)
----- collapser = Cs(spacer^0/"" * ((spacer^1 * endofstring / "") + (spacer^1/" ") + P(1))^0)
local collapser = Cs(spacer^0/"" * nonspacer^0 * ((spacer^0/" " * nonspacer^1)^0))
+local nospacer = Cs((whitespace^1/"" + nonwhitespace^1)^0)
-local b_collapser = Cs( whitespace^0 /"" * (nonwhitespace^1 + whitespace^1/" ")^0)
-local e_collapser = Cs((whitespace^1 * P(-1)/"" + nonwhitespace^1 + whitespace^1/" ")^0)
-local m_collapser = Cs( (nonwhitespace^1 + whitespace^1/" ")^0)
+local b_collapser = Cs( whitespace^0 /"" * (nonwhitespace^1 + whitespace^1/" ")^0)
+local e_collapser = Cs((whitespace^1 * endofstring/"" + nonwhitespace^1 + whitespace^1/" ")^0)
+local m_collapser = Cs( (nonwhitespace^1 + whitespace^1/" ")^0)
-local b_stripper = Cs( spacer^0 /"" * (nonspacer^1 + spacer^1/" ")^0)
-local e_stripper = Cs((spacer^1 * P(-1)/"" + nonspacer^1 + spacer^1/" ")^0)
-local m_stripper = Cs( (nonspacer^1 + spacer^1/" ")^0)
+local b_stripper = Cs( spacer^0 /"" * (nonspacer^1 + spacer^1/" ")^0)
+local e_stripper = Cs((spacer^1 * endofstring/"" + nonspacer^1 + spacer^1/" ")^0)
+local m_stripper = Cs( (nonspacer^1 + spacer^1/" ")^0)
patterns.stripper = stripper
patterns.fullstripper = fullstripper
patterns.collapser = collapser
+patterns.nospacer = nospacer
patterns.b_collapser = b_collapser
patterns.m_collapser = m_collapser
@@ -238,43 +253,50 @@ patterns.doublequoted = dquote * patterns.nodquote * dquote
patterns.quoted = patterns.doublequoted + patterns.singlequoted
patterns.digit = digit
+patterns.digits = digits
patterns.octdigit = octdigit
+patterns.octdigits = octdigits
patterns.hexdigit = hexdigit
+patterns.hexdigits = hexdigits
patterns.sign = sign
-patterns.cardinal = digit^1
-patterns.integer = sign^-1 * digit^1
-patterns.unsigned = digit^0 * period * digit^1
+patterns.cardinal = digits
+patterns.integer = sign^-1 * digits
+patterns.unsigned = digit^0 * period * digits
patterns.float = sign^-1 * patterns.unsigned
-patterns.cunsigned = digit^0 * comma * digit^1
-patterns.cpunsigned = digit^0 * (period + comma) * digit^1
+patterns.cunsigned = digit^0 * comma * digits
+patterns.cpunsigned = digit^0 * (period + comma) * digits
patterns.cfloat = sign^-1 * patterns.cunsigned
patterns.cpfloat = sign^-1 * patterns.cpunsigned
patterns.number = patterns.float + patterns.integer
patterns.cnumber = patterns.cfloat + patterns.integer
patterns.cpnumber = patterns.cpfloat + patterns.integer
-patterns.oct = zero * octdigit^1
+patterns.oct = zero * octdigits -- hm is this ok
patterns.octal = patterns.oct
patterns.HEX = zero * P("X") * (digit+uppercase)^1
patterns.hex = zero * P("x") * (digit+lowercase)^1
-patterns.hexadecimal = zero * S("xX") * hexdigit^1
+patterns.hexadecimal = zero * S("xX") * hexdigits
patterns.hexafloat = sign^-1
* zero * S("xX")
- * (hexdigit^0 * period * hexdigit^1 + hexdigit^1 * period * hexdigit^0 + hexdigit^1)
- * (S("pP") * sign^-1 * hexdigit^1)^-1
+ * (hexdigit^0 * period * hexdigits + hexdigits * period * hexdigit^0 + hexdigits)
+ * (S("pP") * sign^-1 * hexdigits)^-1
patterns.decafloat = sign^-1
- * (digit^0 * period * digit^1 + digit^1 * period * digit^0 + digit^1)
- * S("eE") * sign^-1 * digit^1
+ * (digit^0 * period * digits + digits * period * digit^0 + digits)
+ * S("eE") * sign^-1 * digits
patterns.propername = (uppercase + lowercase + underscore) * (uppercase + lowercase + underscore + digit)^0 * endofstring
patterns.somecontent = (anything - newline - space)^1 -- (utf8char - newline - space)^1
patterns.beginline = #(1-newline)
-patterns.longtostring = Cs(whitespace^0/"" * ((patterns.quoted + nonwhitespace^1 + whitespace^1/"" * (P(-1) + Cc(" ")))^0))
+patterns.longtostring = Cs(whitespace^0/"" * ((patterns.quoted + nonwhitespace^1 + whitespace^1/"" * (endofstring + Cc(" ")))^0))
-local function anywhere(pattern) --slightly adapted from website
- return P { P(pattern) + 1 * V(1) }
+-- local function anywhere(pattern) -- slightly adapted from website
+-- return P { P(pattern) + 1 * V(1) }
+-- end
+
+function anywhere(pattern) -- faster
+ return (1-P(pattern))^0 * P(pattern)
end
lpeg.anywhere = anywhere
@@ -595,19 +617,27 @@ end
-- print(7,lpegmatch(lpeg.secondofsplit(":"),"bc"))
-- print(9,lpegmatch(lpeg.secondofsplit(":","123"),"bc"))
--- -- slower:
+-- this was slower but lpeg has been sped up in the meantime, so we no longer
+-- use this (still seems somewhat faster on long strings)
+--
+-- local nany = utf8char/""
--
-- function lpeg.counter(pattern)
--- local n, pattern = 0, (lpeg.P(pattern)/function() n = n + 1 end + lpeg.anything)^0
--- return function(str) n = 0 ; lpegmatch(pattern,str) ; return n end
+-- pattern = Cs((P(pattern)/" " + nany)^0)
+-- return function(str)
+-- return #lpegmatch(pattern,str)
+-- end
-- end
-local nany = utf8char/""
-
-function lpeg.counter(pattern)
- pattern = Cs((P(pattern)/" " + nany)^0)
- return function(str)
- return #lpegmatch(pattern,str)
+function lpeg.counter(pattern,action)
+ local n = 0
+ local pattern = (P(pattern) / function() n = n + 1 end + anything)^0
+ ----- pattern = (P(pattern) * (P(true) / function() n = n + 1 end) + anything)^0
+ ----- pattern = (P(pattern) * P(function() n = n + 1 end) + anything)^0
+ if action then
+ return function(str) n = 0 ; lpegmatch(pattern,str) ; action(n) end
+ else
+ return function(str) n = 0 ; lpegmatch(pattern,str) ; return n end
end
end
@@ -839,28 +869,42 @@ end
local p_false = P(false)
local p_true = P(true)
-local function make(t)
- local function making(t)
- local p = p_false
- local keys = sortedkeys(t)
- for i=1,#keys do
- local k = keys[i]
- if k ~= "" then
- local v = t[k]
- if v == true then
- p = p + P(k) * p_true
- elseif v == false then
- -- can't happen
- else
- p = p + P(k) * making(v)
- end
- end
- end
- if t[""] then
- p = p + p_true
- end
- return p
- end
+-- local function collapse(t,x)
+-- if type(t) ~= "table" then
+-- return t, x
+-- else
+-- local n = next(t)
+-- if n == nil then
+-- return t, x
+-- elseif next(t,n) == nil then
+-- -- one entry
+-- local k = n
+-- local v = t[k]
+-- if type(v) == "table" then
+-- return collapse(v,x..k)
+-- else
+-- return v, x .. k
+-- end
+-- else
+-- local tt = { }
+-- for k, v in next, t do
+-- local vv, kk = collapse(v,k)
+-- tt[kk] = vv
+-- end
+-- return tt, x
+-- end
+-- end
+-- end
+
+local lower = utf and utf.lower or string.lower
+local upper = utf and utf.upper or string.upper
+
+function lpeg.setutfcasers(l,u)
+ lower = l or lower
+ upper = u or upper
+end
+
+local function make1(t,rest)
local p = p_false
local keys = sortedkeys(t)
for i=1,#keys do
@@ -872,41 +916,39 @@ local function make(t)
elseif v == false then
-- can't happen
else
- p = p + P(k) * making(v)
+ p = p + P(k) * make1(v,v[""])
end
end
end
+ if rest then
+ p = p + p_true
+ end
return p
end
-local function collapse(t,x)
- if type(t) ~= "table" then
- return t, x
- else
- local n = next(t)
- if n == nil then
- return t, x
- elseif next(t,n) == nil then
- -- one entry
- local k = n
+local function make2(t,rest) -- only ascii
+ local p = p_false
+ local keys = sortedkeys(t)
+ for i=1,#keys do
+ local k = keys[i]
+ if k ~= "" then
local v = t[k]
- if type(v) == "table" then
- return collapse(v,x..k)
+ if v == true then
+ p = p + (P(lower(k))+P(upper(k))) * p_true
+ elseif v == false then
+ -- can't happen
else
- return v, x .. k
+ p = p + (P(lower(k))+P(upper(k))) * make2(v,v[""])
end
- else
- local tt = { }
- for k, v in next, t do
- local vv, kk = collapse(v,k)
- tt[kk] = vv
- end
- return tt, x
end
end
+ if rest then
+ p = p + p_true
+ end
+ return p
end
-function lpeg.utfchartabletopattern(list) -- goes to util-lpg
+local function utfchartabletopattern(list,insensitive) -- goes to util-lpg
local tree = { }
local n = #list
if n == 0 then
@@ -979,32 +1021,42 @@ function lpeg.utfchartabletopattern(list) -- goes to util-lpg
end
end
end
--- collapse(tree,"") -- needs testing, maybe optional, slightly faster because P("x")*P("X") seems slower than P"(xX") (why)
--- inspect(tree)
- return make(tree)
+ -- collapse(tree,"") -- needs testing, maybe optional, slightly faster because P("x")*P("X") seems slower than P"(xX") (why)
+ -- inspect(tree)
+ return (insensitive and make2 or make1)(tree)
+end
+
+lpeg.utfchartabletopattern = utfchartabletopattern
+
+function lpeg.utfreplacer(list,insensitive)
+ local pattern = Cs((utfchartabletopattern(list,insensitive)/list + utf8character)^0)
+ return function(str)
+ return lpegmatch(pattern,str) or str
+ end
end
+
-- local t = { "start", "stoep", "staart", "paard" }
-- local p = lpeg.Cs((lpeg.utfchartabletopattern(t)/string.upper + 1)^1)
-- local t = { "a", "abc", "ac", "abe", "abxyz", "xy", "bef","aa" }
-- local p = lpeg.Cs((lpeg.utfchartabletopattern(t)/string.upper + 1)^1)
--- inspect(lpegmatch(p,"a"))
--- inspect(lpegmatch(p,"aa"))
--- inspect(lpegmatch(p,"aaaa"))
--- inspect(lpegmatch(p,"ac"))
--- inspect(lpegmatch(p,"bc"))
--- inspect(lpegmatch(p,"zzbczz"))
--- inspect(lpegmatch(p,"zzabezz"))
--- inspect(lpegmatch(p,"ab"))
--- inspect(lpegmatch(p,"abc"))
--- inspect(lpegmatch(p,"abe"))
--- inspect(lpegmatch(p,"xa"))
--- inspect(lpegmatch(p,"bx"))
--- inspect(lpegmatch(p,"bax"))
--- inspect(lpegmatch(p,"abxyz"))
--- inspect(lpegmatch(p,"foobarbefcrap"))
+-- inspect(lpegmatch(p,"a")=="A")
+-- inspect(lpegmatch(p,"aa")=="AA")
+-- inspect(lpegmatch(p,"aaaa")=="AAAA")
+-- inspect(lpegmatch(p,"ac")=="AC")
+-- inspect(lpegmatch(p,"bc")=="bc")
+-- inspect(lpegmatch(p,"zzbczz")=="zzbczz")
+-- inspect(lpegmatch(p,"zzabezz")=="zzABEzz")
+-- inspect(lpegmatch(p,"ab")=="Ab")
+-- inspect(lpegmatch(p,"abc")=="ABC")
+-- inspect(lpegmatch(p,"abe")=="ABE")
+-- inspect(lpegmatch(p,"xa")=="xA")
+-- inspect(lpegmatch(p,"bx")=="bx")
+-- inspect(lpegmatch(p,"bax")=="bAx")
+-- inspect(lpegmatch(p,"abxyz")=="ABXYZ")
+-- inspect(lpegmatch(p,"foobarbefcrap")=="foobArBEFcrAp")
-- local t = { ["^"] = 1, ["^^"] = 2, ["^^^"] = 3, ["^^^^"] = 4 }
-- local p = lpeg.Cs((lpeg.utfchartabletopattern(t)/t + 1)^1)
@@ -1081,25 +1133,38 @@ end
-- moved here (before util-str)
------ digit = R("09")
------ period = P(".")
------ zero = P("0")
-local trailingzeros = zero^0 * -digit -- suggested by Roberto R
-local case_1 = period * trailingzeros / ""
-local case_2 = period * (digit - trailingzeros)^1 * (trailingzeros / "")
-local number = digit^1 * (case_1 + case_2)
-local stripper = Cs((number + 1)^0)
+do
+
+ local trailingzeros = zero^0 * -digit -- suggested by Roberto
+ local stripper = Cs((
+ digits * (
+ period * trailingzeros / ""
+ + period * (digit - trailingzeros)^1 * (trailingzeros / "")
+ ) + 1
+ )^0)
+
+ lpeg.patterns.stripzeros = stripper -- multiple in string
-lpeg.patterns.stripzeros = stripper
+ local nonzero = digit - zero
+ local trailingzeros = zero^1 * endofstring
+ local stripper = Cs( (1-period)^0 * (
+ period * trailingzeros/""
+ + period * (nonzero^1 + (trailingzeros/"") + zero^1)^0
+ + endofstring
+ ))
--- local sample = "bla 11.00 bla 11 bla 0.1100 bla 1.00100 bla 0.00 bla 0.001 bla 1.1100 bla 0.100100100 bla 0.00100100100"
--- collectgarbage("collect")
--- str = string.rep(sample,10000)
--- local ts = os.clock()
--- lpegmatch(stripper,str)
--- print(#str, os.clock()-ts, lpegmatch(stripper,sample))
+ lpeg.patterns.stripzero = stripper -- slightly more efficient but expects a float !
--- for practical reasone we keep this here:
+ -- local sample = "bla 11.00 bla 11 bla 0.1100 bla 1.00100 bla 0.00 bla 0.001 bla 1.1100 bla 0.100100100 bla 0.00100100100"
+ -- collectgarbage("collect")
+ -- str = string.rep(sample,10000)
+ -- local ts = os.clock()
+ -- lpegmatch(stripper,str)
+ -- print(#str, os.clock()-ts, lpegmatch(stripper,sample))
+
+end
+
+-- for practical reasons we keep this here:
local byte_to_HEX = { }
local byte_to_hex = { }
@@ -1171,3 +1236,22 @@ end
-- local h = "ADFE0345"
-- local b = lpegmatch(patterns.hextobytes,h)
-- print(h,b,string.tohex(b),string.toHEX(b))
+
+local patterns = { } -- can be made weak
+
+local function containsws(what)
+ local p = patterns[what]
+ if not p then
+ local p1 = P(what) * (whitespace + endofstring) * Cc(true)
+ local p2 = whitespace * P(p1)
+ p = P(p1) + P(1-p2)^0 * p2 + Cc(false)
+ patterns[what] = p
+ end
+ return p
+end
+
+lpeg.containsws = containsws
+
+function string.containsws(str,what)
+ return lpegmatch(patterns[what] or containsws(what),str)
+end