diff options
Diffstat (limited to 'Master/texmf-dist/tex/context/base/l-lpeg.lua')
| -rw-r--r-- | Master/texmf-dist/tex/context/base/l-lpeg.lua | 267 |
1 files changed, 241 insertions, 26 deletions
diff --git a/Master/texmf-dist/tex/context/base/l-lpeg.lua b/Master/texmf-dist/tex/context/base/l-lpeg.lua index ce0cc67efe7..25d48ee75d7 100644 --- a/Master/texmf-dist/tex/context/base/l-lpeg.lua +++ b/Master/texmf-dist/tex/context/base/l-lpeg.lua @@ -6,9 +6,51 @@ if not modules then modules = { } end modules ['l-lpeg'] = { license = "see context related readme files" } + +-- a new lpeg fails on a #(1-P(":")) test and really needs a + P(-1) + local lpeg = require("lpeg") +-- tracing (only used when we encounter a problem in integration of lpeg in luatex) + +local report = texio and texio.write_nl or print + +--~ local lpmatch = lpeg.match +--~ local lpprint = lpeg.print +--~ local lpp = lpeg.P +--~ local lpr = lpeg.R +--~ local lps = lpeg.S +--~ local lpc = lpeg.C +--~ local lpb = lpeg.B +--~ local lpv = lpeg.V +--~ local lpcf = lpeg.Cf +--~ local lpcb = lpeg.Cb +--~ local lpcg = lpeg.Cg +--~ local lpct = lpeg.Ct +--~ local lpcs = lpeg.Cs +--~ local lpcc = lpeg.Cc +--~ local lpcmt = lpeg.Cmt +--~ local lpcarg = lpeg.Carg + +--~ function lpeg.match(l,...) report("LPEG MATCH") lpprint(l) return lpmatch(l,...) end + +--~ function lpeg.P (l) local p = lpp (l) report("LPEG P =") lpprint(l) return p end +--~ function lpeg.R (l) local p = lpr (l) report("LPEG R =") lpprint(l) return p end +--~ function lpeg.S (l) local p = lps (l) report("LPEG S =") lpprint(l) return p end +--~ function lpeg.C (l) local p = lpc (l) report("LPEG C =") lpprint(l) return p end +--~ function lpeg.B (l) local p = lpb (l) report("LPEG B =") lpprint(l) return p end +--~ function lpeg.V (l) local p = lpv (l) report("LPEG V =") lpprint(l) return p end +--~ function lpeg.Cf (l) local p = lpcf (l) report("LPEG Cf =") lpprint(l) return p end +--~ function lpeg.Cb (l) local p = lpcb (l) report("LPEG Cb =") lpprint(l) return p end +--~ function lpeg.Cg (l) local p = lpcg (l) report("LPEG Cg =") lpprint(l) return p end +--~ function lpeg.Ct (l) local p = lpct (l) report("LPEG Ct =") lpprint(l) return p end +--~ function lpeg.Cs (l) local p = lpcs (l) report("LPEG Cs =") lpprint(l) return p end +--~ function lpeg.Cc (l) local p = lpcc (l) report("LPEG Cc =") lpprint(l) return p end +--~ function lpeg.Cmt (l) local p = lpcmt (l) report("LPEG Cmt =") lpprint(l) return p end +--~ function lpeg.Carg (l) local p = lpcarg(l) report("LPEG Carg =") lpprint(l) return p end + local type = type +local byte, char, gmatch = string.byte, string.char, string.gmatch -- Beware, we predefine a bunch of patterns here and one reason for doing so -- is that we get consistent behaviour in some of the visualizers. @@ -34,7 +76,7 @@ patterns.alwaysmatched = alwaysmatched local digit, sign = R('09'), S('+-') local cr, lf, crlf = P("\r"), P("\n"), P("\r\n") -local newline = crlf + cr + lf +local newline = crlf + S("\r\n") -- cr + lf local escaped = P("\\") * anything local squote = P("'") local dquote = P('"') @@ -89,7 +131,7 @@ patterns.space = space patterns.tab = P("\t") patterns.spaceortab = patterns.space + patterns.tab patterns.eol = S("\n\r") -patterns.spacer = S(" \t\f\v") -- + string.char(0xc2, 0xa0) if we want utf (cf mail roberto) +patterns.spacer = S(" \t\f\v") -- + char(0xc2, 0xa0) if we want utf (cf mail roberto) patterns.newline = newline patterns.emptyline = newline^1 patterns.nonspacer = 1 - patterns.spacer @@ -115,16 +157,12 @@ patterns.unspacer = ((patterns.spacer^1)/"")^0 patterns.somecontent = (anything - newline - space)^1 -- (utf8char - newline - space)^1 patterns.beginline = #(1-newline) -local unquoted = Cs(patterns.unquoted * endofstring) -- not C - -function string.unquoted(str) - return match(unquoted,str) or str -end - ---~ print(string.unquoted("test")) ---~ print(string.unquoted([["t\"est"]])) ---~ print(string.unquoted([["t\"est"x]])) ---~ print(string.unquoted("\'test\'")) +-- print(string.unquoted("test")) +-- print(string.unquoted([["t\"est"]])) +-- print(string.unquoted([["t\"est"x]])) +-- print(string.unquoted("\'test\'")) +-- print(string.unquoted('"test"')) +-- print(string.unquoted('"test"')) function lpeg.anywhere(pattern) --slightly adapted from website return P { P(pattern) + 1 * V(1) } -- why so complex? @@ -134,7 +172,14 @@ function lpeg.splitter(pattern, action) return (((1-P(pattern))^1)/action+1)^0 end -local splitters_s, splitters_m = { }, { } +function lpeg.tsplitter(pattern, action) + return Ct((((1-P(pattern))^1)/action+1)^0) +end + +-- probleem: separator can be lpeg and that does not hash too well, but +-- it's quite okay as the key is then not garbage collected + +local splitters_s, splitters_m, splitters_t = { }, { }, { } local function splitat(separator,single) local splitter = (single and splitters_s[separator]) or splitters_m[separator] @@ -153,7 +198,24 @@ local function splitat(separator,single) return splitter end -lpeg.splitat = splitat +local function tsplitat(separator) + local splitter = splitters_t[separator] + if not splitter then + splitter = Ct(splitat(separator)) + splitters_t[separator] = splitter + end + return splitter +end + +lpeg.splitat = splitat +lpeg.tsplitat = tsplitat + +function string.splitup(str,separator) + if not separator then + separator = "," + end + return match(splitters_m[separator] or splitat(separator),str) +end --~ local p = splitat("->",false) print(match(p,"oeps->what->more")) -- oeps what more --~ local p = splitat("->",true) print(match(p,"oeps->what->more")) -- oeps what->more @@ -165,7 +227,7 @@ local cache = { } function lpeg.split(separator,str) local c = cache[separator] if not c then - c = Ct(splitat(separator)) + c = tsplitat(separator) cache[separator] = c end return match(c,str) @@ -174,7 +236,7 @@ end function string.split(str,separator) local c = cache[separator] if not c then - c = Ct(splitat(separator)) + c = tsplitat(separator) cache[separator] = c end return match(c,str) @@ -193,7 +255,7 @@ patterns.textline = content --~ return match(linesplitter,str) --~ end -local linesplitter = Ct(splitat(newline)) +local linesplitter = tsplitat(newline) patterns.linesplitter = linesplitter @@ -201,12 +263,12 @@ function string.splitlines(str) return match(linesplitter,str) end -local utflinesplitter = utfbom^-1 * Ct(splitat(newline)) +local utflinesplitter = utfbom^-1 * tsplitat(newline) patterns.utflinesplitter = utflinesplitter function string.utfsplitlines(str) - return match(utflinesplitter,str) + return match(utflinesplitter,str or "") end --~ lpeg.splitters = cache -- no longer public @@ -237,13 +299,11 @@ end --~ from roberto's site: -local f1 = string.byte +local function f2(s) local c1, c2 = byte(s,1,2) return c1 * 64 + c2 - 12416 end +local function f3(s) local c1, c2, c3 = byte(s,1,3) return (c1 * 64 + c2) * 64 + c3 - 925824 end +local function f4(s) local c1, c2, c3, c4 = byte(s,1,4) return ((c1 * 64 + c2) * 64 + c3) * 64 + c4 - 63447168 end -local function f2(s) local c1, c2 = f1(s,1,2) return c1 * 64 + c2 - 12416 end -local function f3(s) local c1, c2, c3 = f1(s,1,3) return (c1 * 64 + c2) * 64 + c3 - 925824 end -local function f4(s) local c1, c2, c3, c4 = f1(s,1,4) return ((c1 * 64 + c2) * 64 + c3) * 64 + c4 - 63447168 end - -local utf8byte = patterns.utf8one/f1 + patterns.utf8two/f2 + patterns.utf8three/f3 + patterns.utf8four/f4 +local utf8byte = patterns.utf8one/byte + patterns.utf8two/f2 + patterns.utf8three/f3 + patterns.utf8four/f4 patterns.utf8byte = utf8byte @@ -538,4 +598,159 @@ function lpeg.is_lpeg(p) return p and lpegtype(p) == "pattern" end ---~ Cf(Ct("") * (Cg(C(...) * "=" * Cs(...)))^0, rawset) +-- For the moment here, but it might move to utilities. Beware, we need to +-- have the longest keyword first, so 'aaa' comes beforte 'aa' which is why we +-- loop back from the end cq. prepend. + +local sort, fastcopy, sortedkeys = table.sort, table.fastcopy, table.sortedkeys -- dependency! + +function lpeg.append(list,pp,delayed,checked) + local p = pp + if #list > 0 then + local keys = fastcopy(list) + sort(keys) + for i=#keys,1,-1 do + local k = keys[i] + if p then + p = P(k) + p + else + p = P(k) + end + end + elseif delayed then -- hm, it looks like the lpeg parser resolves anyway + local keys = sortedkeys(list) + if p then + for i=1,#keys,1 do + local k = keys[i] + local v = list[k] + p = P(k)/list + p + end + else + for i=1,#keys do + local k = keys[i] + local v = list[k] + if p then + p = P(k) + p + else + p = P(k) + end + end + if p then + p = p / list + end + end + elseif checked then + -- problem: substitution gives a capture + local keys = sortedkeys(list) + for i=1,#keys do + local k = keys[i] + local v = list[k] + if p then + if k == v then + p = P(k) + p + else + p = P(k)/v + p + end + else + if k == v then + p = P(k) + else + p = P(k)/v + end + end + end + else + local keys = sortedkeys(list) + for i=1,#keys do + local k = keys[i] + local v = list[k] + if p then + p = P(k)/v + p + else + p = P(k)/v + end + end + end + return p +end + +-- inspect(lpeg.append({ a = "1", aa = "1", aaa = "1" } ,nil,true)) +-- inspect(lpeg.append({ ["degree celsius"] = "1", celsius = "1", degree = "1" } ,nil,true)) + +-- function lpeg.exact_match(words,case_insensitive) +-- local pattern = concat(words) +-- if case_insensitive then +-- local pattern = S(upper(characters)) + S(lower(characters)) +-- local list = { } +-- for i=1,#words do +-- list[lower(words[i])] = true +-- end +-- return Cmt(pattern^1, function(_,i,s) +-- return list[lower(s)] and i +-- end) +-- else +-- local pattern = S(concat(words)) +-- local list = { } +-- for i=1,#words do +-- list[words[i]] = true +-- end +-- return Cmt(pattern^1, function(_,i,s) +-- return list[s] and i +-- end) +-- end +-- end + +-- experiment: + +local function make(t) + local p +-- for k, v in next, t do + for k, v in table.sortedhash(t) do + if not p then + if next(v) then + p = P(k) * make(v) + else + p = P(k) + end + else + if next(v) then + p = p + P(k) * make(v) + else + p = p + P(k) + end + end + end + return p +end + +function lpeg.utfchartabletopattern(list) + local tree = { } + for i=1,#list do + local t = tree + for c in gmatch(list[i],".") do + if not t[c] then + t[c] = { } + end + t = t[c] + end + end + return make(tree) +end + +-- inspect ( lpeg.utfchartabletopattern { +-- utfchar(0x00A0), -- nbsp +-- utfchar(0x2000), -- enquad +-- utfchar(0x2001), -- emquad +-- utfchar(0x2002), -- enspace +-- utfchar(0x2003), -- emspace +-- utfchar(0x2004), -- threeperemspace +-- utfchar(0x2005), -- fourperemspace +-- utfchar(0x2006), -- sixperemspace +-- utfchar(0x2007), -- figurespace +-- utfchar(0x2008), -- punctuationspace +-- utfchar(0x2009), -- breakablethinspace +-- utfchar(0x200A), -- hairspace +-- utfchar(0x200B), -- zerowidthspace +-- utfchar(0x202F), -- narrownobreakspace +-- utfchar(0x205F), -- math thinspace +-- } ) |