summaryrefslogtreecommitdiff
path: root/Master/texmf-dist/tex/context/base/l-lpeg.lua
diff options
context:
space:
mode:
authorMojca Miklavec <mojca.miklavec@gmail.com>2012-05-14 17:38:55 +0000
committerMojca Miklavec <mojca.miklavec@gmail.com>2012-05-14 17:38:55 +0000
commit15995e10bfc68edf79970c4ea4fbb6678566c46e (patch)
tree2de7ca2a83f2d37ef043ad7429a5cb945bb79ddb /Master/texmf-dist/tex/context/base/l-lpeg.lua
parentc9a39f716f1e5ec820ed3aab2c9aef25c5a9d730 (diff)
ConTeXt 2012.05.14 16:00
git-svn-id: svn://tug.org/texlive/trunk@26371 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Master/texmf-dist/tex/context/base/l-lpeg.lua')
-rw-r--r--Master/texmf-dist/tex/context/base/l-lpeg.lua267
1 files changed, 241 insertions, 26 deletions
diff --git a/Master/texmf-dist/tex/context/base/l-lpeg.lua b/Master/texmf-dist/tex/context/base/l-lpeg.lua
index ce0cc67efe7..25d48ee75d7 100644
--- a/Master/texmf-dist/tex/context/base/l-lpeg.lua
+++ b/Master/texmf-dist/tex/context/base/l-lpeg.lua
@@ -6,9 +6,51 @@ if not modules then modules = { } end modules ['l-lpeg'] = {
license = "see context related readme files"
}
+
+-- a new lpeg fails on a #(1-P(":")) test and really needs a + P(-1)
+
local lpeg = require("lpeg")
+-- tracing (only used when we encounter a problem in integration of lpeg in luatex)
+
+local report = texio and texio.write_nl or print
+
+--~ local lpmatch = lpeg.match
+--~ local lpprint = lpeg.print
+--~ local lpp = lpeg.P
+--~ local lpr = lpeg.R
+--~ local lps = lpeg.S
+--~ local lpc = lpeg.C
+--~ local lpb = lpeg.B
+--~ local lpv = lpeg.V
+--~ local lpcf = lpeg.Cf
+--~ local lpcb = lpeg.Cb
+--~ local lpcg = lpeg.Cg
+--~ local lpct = lpeg.Ct
+--~ local lpcs = lpeg.Cs
+--~ local lpcc = lpeg.Cc
+--~ local lpcmt = lpeg.Cmt
+--~ local lpcarg = lpeg.Carg
+
+--~ function lpeg.match(l,...) report("LPEG MATCH") lpprint(l) return lpmatch(l,...) end
+
+--~ function lpeg.P (l) local p = lpp (l) report("LPEG P =") lpprint(l) return p end
+--~ function lpeg.R (l) local p = lpr (l) report("LPEG R =") lpprint(l) return p end
+--~ function lpeg.S (l) local p = lps (l) report("LPEG S =") lpprint(l) return p end
+--~ function lpeg.C (l) local p = lpc (l) report("LPEG C =") lpprint(l) return p end
+--~ function lpeg.B (l) local p = lpb (l) report("LPEG B =") lpprint(l) return p end
+--~ function lpeg.V (l) local p = lpv (l) report("LPEG V =") lpprint(l) return p end
+--~ function lpeg.Cf (l) local p = lpcf (l) report("LPEG Cf =") lpprint(l) return p end
+--~ function lpeg.Cb (l) local p = lpcb (l) report("LPEG Cb =") lpprint(l) return p end
+--~ function lpeg.Cg (l) local p = lpcg (l) report("LPEG Cg =") lpprint(l) return p end
+--~ function lpeg.Ct (l) local p = lpct (l) report("LPEG Ct =") lpprint(l) return p end
+--~ function lpeg.Cs (l) local p = lpcs (l) report("LPEG Cs =") lpprint(l) return p end
+--~ function lpeg.Cc (l) local p = lpcc (l) report("LPEG Cc =") lpprint(l) return p end
+--~ function lpeg.Cmt (l) local p = lpcmt (l) report("LPEG Cmt =") lpprint(l) return p end
+--~ function lpeg.Carg (l) local p = lpcarg(l) report("LPEG Carg =") lpprint(l) return p end
+
local type = type
+local byte, char, gmatch = string.byte, string.char, string.gmatch
-- Beware, we predefine a bunch of patterns here and one reason for doing so
-- is that we get consistent behaviour in some of the visualizers.
@@ -34,7 +76,7 @@ patterns.alwaysmatched = alwaysmatched
local digit, sign = R('09'), S('+-')
local cr, lf, crlf = P("\r"), P("\n"), P("\r\n")
-local newline = crlf + cr + lf
+local newline = crlf + S("\r\n") -- cr + lf
local escaped = P("\\") * anything
local squote = P("'")
local dquote = P('"')
@@ -89,7 +131,7 @@ patterns.space = space
patterns.tab = P("\t")
patterns.spaceortab = patterns.space + patterns.tab
patterns.eol = S("\n\r")
-patterns.spacer = S(" \t\f\v") -- + string.char(0xc2, 0xa0) if we want utf (cf mail roberto)
+patterns.spacer = S(" \t\f\v") -- + char(0xc2, 0xa0) if we want utf (cf mail roberto)
patterns.newline = newline
patterns.emptyline = newline^1
patterns.nonspacer = 1 - patterns.spacer
@@ -115,16 +157,12 @@ patterns.unspacer = ((patterns.spacer^1)/"")^0
patterns.somecontent = (anything - newline - space)^1 -- (utf8char - newline - space)^1
patterns.beginline = #(1-newline)
-local unquoted = Cs(patterns.unquoted * endofstring) -- not C
-
-function string.unquoted(str)
- return match(unquoted,str) or str
-end
-
---~ print(string.unquoted("test"))
---~ print(string.unquoted([["t\"est"]]))
---~ print(string.unquoted([["t\"est"x]]))
---~ print(string.unquoted("\'test\'"))
+-- print(string.unquoted("test"))
+-- print(string.unquoted([["t\"est"]]))
+-- print(string.unquoted([["t\"est"x]]))
+-- print(string.unquoted("\'test\'"))
+-- print(string.unquoted('"test"'))
+-- print(string.unquoted('"test"'))
function lpeg.anywhere(pattern) --slightly adapted from website
return P { P(pattern) + 1 * V(1) } -- why so complex?
@@ -134,7 +172,14 @@ function lpeg.splitter(pattern, action)
return (((1-P(pattern))^1)/action+1)^0
end
-local splitters_s, splitters_m = { }, { }
+function lpeg.tsplitter(pattern, action)
+ return Ct((((1-P(pattern))^1)/action+1)^0)
+end
+
+-- probleem: separator can be lpeg and that does not hash too well, but
+-- it's quite okay as the key is then not garbage collected
+
+local splitters_s, splitters_m, splitters_t = { }, { }, { }
local function splitat(separator,single)
local splitter = (single and splitters_s[separator]) or splitters_m[separator]
@@ -153,7 +198,24 @@ local function splitat(separator,single)
return splitter
end
-lpeg.splitat = splitat
+local function tsplitat(separator)
+ local splitter = splitters_t[separator]
+ if not splitter then
+ splitter = Ct(splitat(separator))
+ splitters_t[separator] = splitter
+ end
+ return splitter
+end
+
+lpeg.splitat = splitat
+lpeg.tsplitat = tsplitat
+
+function string.splitup(str,separator)
+ if not separator then
+ separator = ","
+ end
+ return match(splitters_m[separator] or splitat(separator),str)
+end
--~ local p = splitat("->",false) print(match(p,"oeps->what->more")) -- oeps what more
--~ local p = splitat("->",true) print(match(p,"oeps->what->more")) -- oeps what->more
@@ -165,7 +227,7 @@ local cache = { }
function lpeg.split(separator,str)
local c = cache[separator]
if not c then
- c = Ct(splitat(separator))
+ c = tsplitat(separator)
cache[separator] = c
end
return match(c,str)
@@ -174,7 +236,7 @@ end
function string.split(str,separator)
local c = cache[separator]
if not c then
- c = Ct(splitat(separator))
+ c = tsplitat(separator)
cache[separator] = c
end
return match(c,str)
@@ -193,7 +255,7 @@ patterns.textline = content
--~ return match(linesplitter,str)
--~ end
-local linesplitter = Ct(splitat(newline))
+local linesplitter = tsplitat(newline)
patterns.linesplitter = linesplitter
@@ -201,12 +263,12 @@ function string.splitlines(str)
return match(linesplitter,str)
end
-local utflinesplitter = utfbom^-1 * Ct(splitat(newline))
+local utflinesplitter = utfbom^-1 * tsplitat(newline)
patterns.utflinesplitter = utflinesplitter
function string.utfsplitlines(str)
- return match(utflinesplitter,str)
+ return match(utflinesplitter,str or "")
end
--~ lpeg.splitters = cache -- no longer public
@@ -237,13 +299,11 @@ end
--~ from roberto's site:
-local f1 = string.byte
+local function f2(s) local c1, c2 = byte(s,1,2) return c1 * 64 + c2 - 12416 end
+local function f3(s) local c1, c2, c3 = byte(s,1,3) return (c1 * 64 + c2) * 64 + c3 - 925824 end
+local function f4(s) local c1, c2, c3, c4 = byte(s,1,4) return ((c1 * 64 + c2) * 64 + c3) * 64 + c4 - 63447168 end
-local function f2(s) local c1, c2 = f1(s,1,2) return c1 * 64 + c2 - 12416 end
-local function f3(s) local c1, c2, c3 = f1(s,1,3) return (c1 * 64 + c2) * 64 + c3 - 925824 end
-local function f4(s) local c1, c2, c3, c4 = f1(s,1,4) return ((c1 * 64 + c2) * 64 + c3) * 64 + c4 - 63447168 end
-
-local utf8byte = patterns.utf8one/f1 + patterns.utf8two/f2 + patterns.utf8three/f3 + patterns.utf8four/f4
+local utf8byte = patterns.utf8one/byte + patterns.utf8two/f2 + patterns.utf8three/f3 + patterns.utf8four/f4
patterns.utf8byte = utf8byte
@@ -538,4 +598,159 @@ function lpeg.is_lpeg(p)
return p and lpegtype(p) == "pattern"
end
---~ Cf(Ct("") * (Cg(C(...) * "=" * Cs(...)))^0, rawset)
+-- For the moment here, but it might move to utilities. Beware, we need to
+-- have the longest keyword first, so 'aaa' comes beforte 'aa' which is why we
+-- loop back from the end cq. prepend.
+
+local sort, fastcopy, sortedkeys = table.sort, table.fastcopy, table.sortedkeys -- dependency!
+
+function lpeg.append(list,pp,delayed,checked)
+ local p = pp
+ if #list > 0 then
+ local keys = fastcopy(list)
+ sort(keys)
+ for i=#keys,1,-1 do
+ local k = keys[i]
+ if p then
+ p = P(k) + p
+ else
+ p = P(k)
+ end
+ end
+ elseif delayed then -- hm, it looks like the lpeg parser resolves anyway
+ local keys = sortedkeys(list)
+ if p then
+ for i=1,#keys,1 do
+ local k = keys[i]
+ local v = list[k]
+ p = P(k)/list + p
+ end
+ else
+ for i=1,#keys do
+ local k = keys[i]
+ local v = list[k]
+ if p then
+ p = P(k) + p
+ else
+ p = P(k)
+ end
+ end
+ if p then
+ p = p / list
+ end
+ end
+ elseif checked then
+ -- problem: substitution gives a capture
+ local keys = sortedkeys(list)
+ for i=1,#keys do
+ local k = keys[i]
+ local v = list[k]
+ if p then
+ if k == v then
+ p = P(k) + p
+ else
+ p = P(k)/v + p
+ end
+ else
+ if k == v then
+ p = P(k)
+ else
+ p = P(k)/v
+ end
+ end
+ end
+ else
+ local keys = sortedkeys(list)
+ for i=1,#keys do
+ local k = keys[i]
+ local v = list[k]
+ if p then
+ p = P(k)/v + p
+ else
+ p = P(k)/v
+ end
+ end
+ end
+ return p
+end
+
+-- inspect(lpeg.append({ a = "1", aa = "1", aaa = "1" } ,nil,true))
+-- inspect(lpeg.append({ ["degree celsius"] = "1", celsius = "1", degree = "1" } ,nil,true))
+
+-- function lpeg.exact_match(words,case_insensitive)
+-- local pattern = concat(words)
+-- if case_insensitive then
+-- local pattern = S(upper(characters)) + S(lower(characters))
+-- local list = { }
+-- for i=1,#words do
+-- list[lower(words[i])] = true
+-- end
+-- return Cmt(pattern^1, function(_,i,s)
+-- return list[lower(s)] and i
+-- end)
+-- else
+-- local pattern = S(concat(words))
+-- local list = { }
+-- for i=1,#words do
+-- list[words[i]] = true
+-- end
+-- return Cmt(pattern^1, function(_,i,s)
+-- return list[s] and i
+-- end)
+-- end
+-- end
+
+-- experiment:
+
+local function make(t)
+ local p
+-- for k, v in next, t do
+ for k, v in table.sortedhash(t) do
+ if not p then
+ if next(v) then
+ p = P(k) * make(v)
+ else
+ p = P(k)
+ end
+ else
+ if next(v) then
+ p = p + P(k) * make(v)
+ else
+ p = p + P(k)
+ end
+ end
+ end
+ return p
+end
+
+function lpeg.utfchartabletopattern(list)
+ local tree = { }
+ for i=1,#list do
+ local t = tree
+ for c in gmatch(list[i],".") do
+ if not t[c] then
+ t[c] = { }
+ end
+ t = t[c]
+ end
+ end
+ return make(tree)
+end
+
+-- inspect ( lpeg.utfchartabletopattern {
+-- utfchar(0x00A0), -- nbsp
+-- utfchar(0x2000), -- enquad
+-- utfchar(0x2001), -- emquad
+-- utfchar(0x2002), -- enspace
+-- utfchar(0x2003), -- emspace
+-- utfchar(0x2004), -- threeperemspace
+-- utfchar(0x2005), -- fourperemspace
+-- utfchar(0x2006), -- sixperemspace
+-- utfchar(0x2007), -- figurespace
+-- utfchar(0x2008), -- punctuationspace
+-- utfchar(0x2009), -- breakablethinspace
+-- utfchar(0x200A), -- hairspace
+-- utfchar(0x200B), -- zerowidthspace
+-- utfchar(0x202F), -- narrownobreakspace
+-- utfchar(0x205F), -- math thinspace
+-- } )