summaryrefslogtreecommitdiff
path: root/Master/texmf-dist/tex/context/base/char-utf.lua
diff options
context:
space:
mode:
Diffstat (limited to 'Master/texmf-dist/tex/context/base/char-utf.lua')
-rw-r--r--Master/texmf-dist/tex/context/base/char-utf.lua260
1 files changed, 149 insertions, 111 deletions
diff --git a/Master/texmf-dist/tex/context/base/char-utf.lua b/Master/texmf-dist/tex/context/base/char-utf.lua
index a411c2d82a6..d0e40e664a6 100644
--- a/Master/texmf-dist/tex/context/base/char-utf.lua
+++ b/Master/texmf-dist/tex/context/base/char-utf.lua
@@ -19,10 +19,10 @@ in special kinds of output (for instance <l n='pdf'/>).</p>
over a string.</p>
--ldx]]--
-local utfchar, utfbyte, utfgsub = utf.char, utf.byte, utf.gsub
local concat, gmatch, gsub, find = table.concat, string.gmatch, string.gsub, string.find
-local utfcharacters, utfvalues = string.utfcharacters, string.utfvalues
+local utfchar, utfbyte, utfcharacters, utfvalues = utf.char, utf.byte, utf.characters, utf.values
local allocate = utilities.storage.allocate
+local lpegmatch, lpegpatterns = lpeg.match, lpeg.patterns
local charfromnumber = characters.fromnumber
@@ -76,6 +76,7 @@ local decomposed = allocate {
["ﬖ"] = "վն",
["ﬗ"] = "մխ",
}
+
characters.decomposed = decomposed
local function initialize() -- maybe only 'mn'
@@ -193,6 +194,20 @@ local private = {
utffilters.private = private
+local tohigh = lpeg.replacer(low) -- frozen, only for basic tex
+local tolow = lpeg.replacer(high) -- frozen, only for basic tex
+
+lpegpatterns.utftohigh = tohigh
+lpegpatterns.utftolow = tolow
+
+function utffilters.harden(str)
+ return lpegmatch(tohigh,str)
+end
+
+function utffilters.soften(str)
+ return lpegmatch(tolow,str)
+end
+
local function set(ch)
local cb
if type(ch) == "number" then
@@ -201,17 +216,24 @@ local function set(ch)
cb = utfbyte(ch)
end
if cb < 256 then
+ escapes[ch] = "\\" .. ch
low[ch] = utfchar(0x0F0000 + cb)
+ if ch == "%" then
+ ch = "%%" -- nasty, but we need this as in replacements (also in lpeg) % is interpreted
+ end
high[utfchar(0x0F0000 + cb)] = ch
- escapes[ch] = "\\" .. ch
end
end
private.set = set
-function private.escape (str) return gsub(str,"(.)", escapes) end
-function private.replace(str) return utfgsub(str,"(.)", low ) end
-function private.revert (str) return utfgsub(str,"(.)", high ) end
+-- function private.escape (str) return gsub(str,"(.)", escapes) end
+-- function private.replace(str) return utfgsub(str,"(.)", low ) end
+-- function private.revert (str) return utfgsub(str,"(.)", high ) end
+
+private.escape = utf.remapper(escapes)
+private.replace = utf.remapper(low)
+private.revert = utf.remapper(high)
for ch in gmatch(special,".") do set(ch) end
@@ -236,111 +258,119 @@ not collecting tokens is not only faster but also saves garbage collecting.
-- I might use the combined loop at some point for the filter
-- some day.
---~ function utffilters.collapse(str) -- not really tested (we could preallocate a table)
---~ if str and str ~= "" then
---~ local nstr = #str
---~ if nstr > 1 then
---~ if initialize then -- saves a call
---~ initialize()
---~ end
---~ local tokens, t, first, done, n = { }, 0, false, false, 0
---~ for second in utfcharacters(str) do
---~ local dec = decomposed[second]
---~ if dec then
---~ if not done then
---~ if n > 0 then
---~ for s in utfcharacters(str) do
---~ if n == 1 then
---~ break
---~ else
---~ t = t + 1
---~ tokens[t] = s
---~ n = n - 1
---~ end
---~ end
---~ end
---~ done = true
---~ elseif first then
---~ t = t + 1
---~ tokens[t] = first
---~ end
---~ t = t + 1
---~ tokens[t] = dec
---~ first = false
---~ elseif done then
---~ local crs = high[second]
---~ if crs then
---~ if first then
---~ t = t + 1
---~ tokens[t] = first
---~ end
---~ first = crs
---~ else
---~ local cgf = graphemes[first]
---~ if cgf and cgf[second] then
---~ first = cgf[second]
---~ elseif first then
---~ t = t + 1
---~ tokens[t] = first
---~ first = second
---~ else
---~ first = second
---~ end
---~ end
---~ else
---~ local crs = high[second]
---~ if crs then
---~ for s in utfcharacters(str) do
---~ if n == 1 then
---~ break
---~ else
---~ t = t + 1
---~ tokens[t] = s
---~ n = n - 1
---~ end
---~ end
---~ if first then
---~ t = t + 1
---~ tokens[t] = first
---~ end
---~ first = crs
---~ done = true
---~ else
---~ local cgf = graphemes[first]
---~ if cgf and cgf[second] then
---~ for s in utfcharacters(str) do
---~ if n == 1 then
---~ break
---~ else
---~ t = t + 1
---~ tokens[t] = s
---~ n = n - 1
---~ end
---~ end
---~ first = cgf[second]
---~ done = true
---~ else
---~ first = second
---~ n = n + 1
---~ end
---~ end
---~ end
---~ end
---~ if done then
---~ if first then
---~ t = t + 1
---~ tokens[t] = first
---~ end
---~ return concat(tokens) -- seldom called
---~ end
---~ elseif nstr > 0 then
---~ return high[str] or str
---~ end
---~ end
---~ return str
---~ end
-
-function utffilters.collapse(str) -- not really tested (we could preallocate a table)
+-- function utffilters.collapse(str) -- not really tested (we could preallocate a table)
+-- if str and str ~= "" then
+-- local nstr = #str
+-- if nstr > 1 then
+-- if initialize then -- saves a call
+-- initialize()
+-- end
+-- local tokens, t, first, done, n = { }, 0, false, false, 0
+-- for second in utfcharacters(str) do
+-- local dec = decomposed[second]
+-- if dec then
+-- if not done then
+-- if n > 0 then
+-- for s in utfcharacters(str) do
+-- if n == 1 then
+-- break
+-- else
+-- t = t + 1
+-- tokens[t] = s
+-- n = n - 1
+-- end
+-- end
+-- end
+-- done = true
+-- elseif first then
+-- t = t + 1
+-- tokens[t] = first
+-- end
+-- t = t + 1
+-- tokens[t] = dec
+-- first = false
+-- elseif done then
+-- local crs = high[second]
+-- if crs then
+-- if first then
+-- t = t + 1
+-- tokens[t] = first
+-- end
+-- first = crs
+-- else
+-- local cgf = graphemes[first]
+-- if cgf and cgf[second] then
+-- first = cgf[second]
+-- elseif first then
+-- t = t + 1
+-- tokens[t] = first
+-- first = second
+-- else
+-- first = second
+-- end
+-- end
+-- else
+-- local crs = high[second]
+-- if crs then
+-- for s in utfcharacters(str) do
+-- if n == 1 then
+-- break
+-- else
+-- t = t + 1
+-- tokens[t] = s
+-- n = n - 1
+-- end
+-- end
+-- if first then
+-- t = t + 1
+-- tokens[t] = first
+-- end
+-- first = crs
+-- done = true
+-- else
+-- local cgf = graphemes[first]
+-- if cgf and cgf[second] then
+-- for s in utfcharacters(str) do
+-- if n == 1 then
+-- break
+-- else
+-- t = t + 1
+-- tokens[t] = s
+-- n = n - 1
+-- end
+-- end
+-- first = cgf[second]
+-- done = true
+-- else
+-- first = second
+-- n = n + 1
+-- end
+-- end
+-- end
+-- end
+-- if done then
+-- if first then
+-- t = t + 1
+-- tokens[t] = first
+-- end
+-- return concat(tokens) -- seldom called
+-- end
+-- elseif nstr > 0 then
+-- return high[str] or str
+-- end
+-- end
+-- return str
+-- end
+
+local skippable = table.tohash { "mkiv", "mkvi" }
+local filesuffix = file.suffix
+
+-- we could reuse tokens but it's seldom populated anyway
+
+function utffilters.collapse(str,filename) -- not really tested (we could preallocate a table)
+ if skippable[filesuffix(filename)] then
+ return str
+ end
if str and str ~= "" then
local nstr = #str
if nstr > 1 then
@@ -481,6 +511,14 @@ if sequencers then
sequencers.enableaction(textfileactions,"characters.filters.utf.decompose")
end
+ directives.register("filters.utf.collapse", function(v)
+ sequencers[v and "enableaction" or "disableaction"](textfileactions,"characters.filters.utf.collapse")
+ end)
+
+ directives.register("filters.utf.decompose", function(v)
+ sequencers[v and "enableaction" or "disableaction"](textfileactions,"characters.filters.utf.decompose")
+ end)
+
end
--[[ldx--