diff options
author | Karl Berry <karl@freefriends.org> | 2013-04-08 00:43:40 +0000 |
---|---|---|
committer | Karl Berry <karl@freefriends.org> | 2013-04-08 00:43:40 +0000 |
commit | 824f7b0903de8ad7f6ee3d7656005e4c59155e06 (patch) | |
tree | abfc8673ef9916f3ab7074e811207384c301492b /Master/texmf-dist/tex/context/base/char-utf.lua | |
parent | 689aefb0727676ed3cddf331337b4be226495e72 (diff) |
context import for TL13, from www.pragma-ade.com/context/beta/cont-tmf.zip
git-svn-id: svn://tug.org/texlive/trunk@29731 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Master/texmf-dist/tex/context/base/char-utf.lua')
-rw-r--r-- | Master/texmf-dist/tex/context/base/char-utf.lua | 260 |
1 files changed, 149 insertions, 111 deletions
diff --git a/Master/texmf-dist/tex/context/base/char-utf.lua b/Master/texmf-dist/tex/context/base/char-utf.lua index a411c2d82a6..d0e40e664a6 100644 --- a/Master/texmf-dist/tex/context/base/char-utf.lua +++ b/Master/texmf-dist/tex/context/base/char-utf.lua @@ -19,10 +19,10 @@ in special kinds of output (for instance <l n='pdf'/>).</p> over a string.</p> --ldx]]-- -local utfchar, utfbyte, utfgsub = utf.char, utf.byte, utf.gsub local concat, gmatch, gsub, find = table.concat, string.gmatch, string.gsub, string.find -local utfcharacters, utfvalues = string.utfcharacters, string.utfvalues +local utfchar, utfbyte, utfcharacters, utfvalues = utf.char, utf.byte, utf.characters, utf.values local allocate = utilities.storage.allocate +local lpegmatch, lpegpatterns = lpeg.match, lpeg.patterns local charfromnumber = characters.fromnumber @@ -76,6 +76,7 @@ local decomposed = allocate { ["ﬖ"] = "վն", ["ﬗ"] = "մխ", } + characters.decomposed = decomposed local function initialize() -- maybe only 'mn' @@ -193,6 +194,20 @@ local private = { utffilters.private = private +local tohigh = lpeg.replacer(low) -- frozen, only for basic tex +local tolow = lpeg.replacer(high) -- frozen, only for basic tex + +lpegpatterns.utftohigh = tohigh +lpegpatterns.utftolow = tolow + +function utffilters.harden(str) + return lpegmatch(tohigh,str) +end + +function utffilters.soften(str) + return lpegmatch(tolow,str) +end + local function set(ch) local cb if type(ch) == "number" then @@ -201,17 +216,24 @@ local function set(ch) cb = utfbyte(ch) end if cb < 256 then + escapes[ch] = "\\" .. ch low[ch] = utfchar(0x0F0000 + cb) + if ch == "%" then + ch = "%%" -- nasty, but we need this as in replacements (also in lpeg) % is interpreted + end high[utfchar(0x0F0000 + cb)] = ch - escapes[ch] = "\\" .. ch end end private.set = set -function private.escape (str) return gsub(str,"(.)", escapes) end -function private.replace(str) return utfgsub(str,"(.)", low ) end -function private.revert (str) return utfgsub(str,"(.)", high ) end +-- function private.escape (str) return gsub(str,"(.)", escapes) end +-- function private.replace(str) return utfgsub(str,"(.)", low ) end +-- function private.revert (str) return utfgsub(str,"(.)", high ) end + +private.escape = utf.remapper(escapes) +private.replace = utf.remapper(low) +private.revert = utf.remapper(high) for ch in gmatch(special,".") do set(ch) end @@ -236,111 +258,119 @@ not collecting tokens is not only faster but also saves garbage collecting. -- I might use the combined loop at some point for the filter -- some day. ---~ function utffilters.collapse(str) -- not really tested (we could preallocate a table) ---~ if str and str ~= "" then ---~ local nstr = #str ---~ if nstr > 1 then ---~ if initialize then -- saves a call ---~ initialize() ---~ end ---~ local tokens, t, first, done, n = { }, 0, false, false, 0 ---~ for second in utfcharacters(str) do ---~ local dec = decomposed[second] ---~ if dec then ---~ if not done then ---~ if n > 0 then ---~ for s in utfcharacters(str) do ---~ if n == 1 then ---~ break ---~ else ---~ t = t + 1 ---~ tokens[t] = s ---~ n = n - 1 ---~ end ---~ end ---~ end ---~ done = true ---~ elseif first then ---~ t = t + 1 ---~ tokens[t] = first ---~ end ---~ t = t + 1 ---~ tokens[t] = dec ---~ first = false ---~ elseif done then ---~ local crs = high[second] ---~ if crs then ---~ if first then ---~ t = t + 1 ---~ tokens[t] = first ---~ end ---~ first = crs ---~ else ---~ local cgf = graphemes[first] ---~ if cgf and cgf[second] then ---~ first = cgf[second] ---~ elseif first then ---~ t = t + 1 ---~ tokens[t] = first ---~ first = second ---~ else ---~ first = second ---~ end ---~ end ---~ else ---~ local crs = high[second] ---~ if crs then ---~ for s in utfcharacters(str) do ---~ if n == 1 then ---~ break ---~ else ---~ t = t + 1 ---~ tokens[t] = s ---~ n = n - 1 ---~ end ---~ end ---~ if first then ---~ t = t + 1 ---~ tokens[t] = first ---~ end ---~ first = crs ---~ done = true ---~ else ---~ local cgf = graphemes[first] ---~ if cgf and cgf[second] then ---~ for s in utfcharacters(str) do ---~ if n == 1 then ---~ break ---~ else ---~ t = t + 1 ---~ tokens[t] = s ---~ n = n - 1 ---~ end ---~ end ---~ first = cgf[second] ---~ done = true ---~ else ---~ first = second ---~ n = n + 1 ---~ end ---~ end ---~ end ---~ end ---~ if done then ---~ if first then ---~ t = t + 1 ---~ tokens[t] = first ---~ end ---~ return concat(tokens) -- seldom called ---~ end ---~ elseif nstr > 0 then ---~ return high[str] or str ---~ end ---~ end ---~ return str ---~ end - -function utffilters.collapse(str) -- not really tested (we could preallocate a table) +-- function utffilters.collapse(str) -- not really tested (we could preallocate a table) +-- if str and str ~= "" then +-- local nstr = #str +-- if nstr > 1 then +-- if initialize then -- saves a call +-- initialize() +-- end +-- local tokens, t, first, done, n = { }, 0, false, false, 0 +-- for second in utfcharacters(str) do +-- local dec = decomposed[second] +-- if dec then +-- if not done then +-- if n > 0 then +-- for s in utfcharacters(str) do +-- if n == 1 then +-- break +-- else +-- t = t + 1 +-- tokens[t] = s +-- n = n - 1 +-- end +-- end +-- end +-- done = true +-- elseif first then +-- t = t + 1 +-- tokens[t] = first +-- end +-- t = t + 1 +-- tokens[t] = dec +-- first = false +-- elseif done then +-- local crs = high[second] +-- if crs then +-- if first then +-- t = t + 1 +-- tokens[t] = first +-- end +-- first = crs +-- else +-- local cgf = graphemes[first] +-- if cgf and cgf[second] then +-- first = cgf[second] +-- elseif first then +-- t = t + 1 +-- tokens[t] = first +-- first = second +-- else +-- first = second +-- end +-- end +-- else +-- local crs = high[second] +-- if crs then +-- for s in utfcharacters(str) do +-- if n == 1 then +-- break +-- else +-- t = t + 1 +-- tokens[t] = s +-- n = n - 1 +-- end +-- end +-- if first then +-- t = t + 1 +-- tokens[t] = first +-- end +-- first = crs +-- done = true +-- else +-- local cgf = graphemes[first] +-- if cgf and cgf[second] then +-- for s in utfcharacters(str) do +-- if n == 1 then +-- break +-- else +-- t = t + 1 +-- tokens[t] = s +-- n = n - 1 +-- end +-- end +-- first = cgf[second] +-- done = true +-- else +-- first = second +-- n = n + 1 +-- end +-- end +-- end +-- end +-- if done then +-- if first then +-- t = t + 1 +-- tokens[t] = first +-- end +-- return concat(tokens) -- seldom called +-- end +-- elseif nstr > 0 then +-- return high[str] or str +-- end +-- end +-- return str +-- end + +local skippable = table.tohash { "mkiv", "mkvi" } +local filesuffix = file.suffix + +-- we could reuse tokens but it's seldom populated anyway + +function utffilters.collapse(str,filename) -- not really tested (we could preallocate a table) + if skippable[filesuffix(filename)] then + return str + end if str and str ~= "" then local nstr = #str if nstr > 1 then @@ -481,6 +511,14 @@ if sequencers then sequencers.enableaction(textfileactions,"characters.filters.utf.decompose") end + directives.register("filters.utf.collapse", function(v) + sequencers[v and "enableaction" or "disableaction"](textfileactions,"characters.filters.utf.collapse") + end) + + directives.register("filters.utf.decompose", function(v) + sequencers[v and "enableaction" or "disableaction"](textfileactions,"characters.filters.utf.decompose") + end) + end --[[ldx-- |