summaryrefslogtreecommitdiff
path: root/Master/texmf-dist/tex/context/base/char-utf.lua
diff options
context:
space:
mode:
Diffstat (limited to 'Master/texmf-dist/tex/context/base/char-utf.lua')
-rw-r--r--Master/texmf-dist/tex/context/base/char-utf.lua436
1 files changed, 324 insertions, 112 deletions
diff --git a/Master/texmf-dist/tex/context/base/char-utf.lua b/Master/texmf-dist/tex/context/base/char-utf.lua
index 6dd85fdc87d..30124a6a2a6 100644
--- a/Master/texmf-dist/tex/context/base/char-utf.lua
+++ b/Master/texmf-dist/tex/context/base/char-utf.lua
@@ -19,57 +19,118 @@ in special kinds of output (for instance <l n='pdf'/>).</p>
over a string.</p>
--ldx]]--
-local utf = unicode.utf8
-local concat, gmatch = table.concat, string.gmatch
+local utfchar, utfbyte, utfgsub = utf.char, utf.byte, utf.gsub
+local concat, gmatch, gsub, find = table.concat, string.gmatch, string.gsub, string.find
local utfcharacters, utfvalues = string.utfcharacters, string.utfvalues
+local allocate = utilities.storage.allocate
-local ctxcatcodes = tex.ctxcatcodes
+local charfromnumber = characters.fromnumber
-characters = characters or { }
-characters.graphemes = characters.graphemes or { }
-characters.filters = characters.filters or { }
-characters.filters.utf = characters.filters.utf or { }
+-- todo: trackers
-characters.filters.utf.initialized = false
-characters.filters.utf.collapsing = true
-characters.filters.utf.expanding = true
+characters = characters or { }
+local characters = characters
-local graphemes = characters.graphemes
-local utffilters = characters.filters.utf
-local utfchar, utfbyte, utfgsub = utf.char, utf.byte, utf.gsub
+characters.graphemes = allocate()
+local graphemes = characters.graphemes
+
+characters.decomposed = allocate()
+local decomposed = characters.decomposed
+
+characters.mathpairs = allocate()
+local mathpairs = characters.mathpairs
+
+characters.filters = allocate()
+local filters = characters.filters
+
+filters.utf = filters.utf or { }
+local utffilters = characters.filters.utf
--[[ldx--
<p>It only makes sense to collapse at runtime, since we don't expect
source code to depend on collapsing.</p>
--ldx]]--
-function utffilters.initialize()
- if utffilters.collapsing and not utffilters.initialized then
- for k,v in next, characters.data do
- -- using vs and first testing for length is faster (.02->.01 s)
- local vs = v.specials
- if vs and #vs == 3 and vs[1] == 'char' then
- local first, second = utfchar(vs[2]), utfchar(vs[3])
- local cgf = graphemes[first]
- if not cgf then
- cgf = { }
- graphemes[first] = cgf
+-- for the moment, will be entries in char-def.lua
+
+local decomposed = allocate {
+ ["IJ"] = "IJ",
+ ["ij"] = "ij",
+ ["և"] = "եւ",
+ ["ff"] = "ff",
+ ["fi"] = "fi",
+ ["fl"] = "fl",
+ ["ffi"] = "ffi",
+ ["ffl"] = "ffl",
+ ["ſt"] = "ſt",
+ ["st"] = "st",
+ ["ﬓ"] = "մն",
+ ["ﬔ"] = "մե",
+ ["ﬕ"] = "մի",
+ ["ﬖ"] = "վն",
+ ["ﬗ"] = "մխ",
+}
+characters.decomposed = decomposed
+
+local function initialize()
+ for unicode, v in next, characters.data do
+ -- using vs and first testing for length is faster (.02->.01 s)
+ local vs = v.specials
+ if vs and #vs == 3 and vs[1] == 'char' then
+ local one, two = vs[2], vs[3]
+ local first, second, combined = utfchar(one), utfchar(two), utfchar(unicode)
+ local cgf = graphemes[first]
+ if not cgf then
+ cgf = { }
+ graphemes[first] = cgf
+ end
+ cgf[second] = combined
+ if v.mathclass or v.mathspec then
+ local mps = mathpairs[two]
+ if not mps then
+ mps = { }
+ mathpairs[two] = mps
+ end
+ mps[one] = unicode -- here unicode
+ local mps = mathpairs[second]
+ if not mps then
+ mps = { }
+ mathpairs[second] = mps
end
- cgf[second] = utfchar(k)
+ mps[first] = combined
end
+ -- else
+ -- local description = v.description
+ -- if find(description,"LIGATURE") then
+ -- if vs then
+ -- local t = { }
+ -- for i=2,#vs do
+ -- t[#t+1] = utfchar(vs[i])
+ -- end
+ -- decomposed[utfchar(unicode)] = concat(t)
+ -- else
+ -- local vs = v.shcode
+ -- if vs then
+ -- local t = { }
+ -- for i=1,#vs do
+ -- t[i] = utfchar(vs[i])
+ -- end
+ -- decomposed[utfchar(unicode)] = concat(t)
+ -- end
+ -- end
+ -- end
end
- utffilters.initialized = true
end
+ initialize = false
end
--- utffilters.add_grapheme(utfchar(318),'l','\string~')
--- utffilters.add_grapheme('c','a','b')
+-- utffilters.addgrapheme(utfchar(318),'l','\string~')
+-- utffilters.addgrapheme('c','a','b')
-function utffilters.add_grapheme(result,first,second)
- local r, f, s = tonumber(result), tonumber(first), tonumber(second)
- if r then result = utfchar(r) end
- if f then first = utfchar(f) end
- if s then second = utfchar(s) end
+function utffilters.addgrapheme(result,first,second) -- can be U+ 0x string or utf or number
+ local result = charfromnumber(result)
+ local first = charfromnumber(first)
+ local second = charfromnumber(second)
if not graphemes[first] then
graphemes[first] = { [second] = result }
else
@@ -77,31 +138,6 @@ function utffilters.add_grapheme(result,first,second)
end
end
-function utffilters.collapse(str) -- old one
- if utffilters.collapsing and str and #str > 1 then
- if not utffilters.initialized then -- saves a call
- utffilters.initialize()
- end
- local tokens, first, done = { }, false, false
- for second in utfcharacters(str) do
- local cgf = graphemes[first]
- if cgf and cgf[second] then
- first, done = cgf[second], true
- elseif first then
- tokens[#tokens+1] = first
- first = second
- else
- first = second
- end
- end
- if done then
- tokens[#tokens+1] = first
- return concat(tokens)
- end
- end
- return str
-end
-
--[[ldx--
<p>In order to deal with 8-bit output, we need to find a way to
go from <l n='utf'/> to 8-bit. This is handled in the
@@ -120,18 +156,20 @@ to their right glyph there.</p>
0x100000.</p>
--ldx]]--
-utffilters.private = {
- high = { },
- low = { },
- escapes = { },
+local low = allocate({ })
+local high = allocate({ })
+local escapes = allocate({ })
+local special = "~#$%^&_{}\\|"
+
+local private = {
+ low = low,
+ high = high,
+ escapes = escapes,
}
-local low = utffilters.private.low
-local high = utffilters.private.high
-local escapes = utffilters.private.escapes
-local special = "~#$%^&_{}\\|"
+utffilters.private = private
-function utffilters.private.set(ch)
+local function set(ch)
local cb
if type(ch) == "number" then
cb, ch = ch, utfchar(ch)
@@ -145,11 +183,11 @@ function utffilters.private.set(ch)
end
end
-function utffilters.private.replace(str) return utfgsub(str,"(.)", low ) end
-function utffilters.private.revert(str) return utfgsub(str,"(.)", high ) end
-function utffilters.private.escape(str) return utfgsub(str,"(.)", escapes) end
+private.set = set
-local set = utffilters.private.set
+function private.escape (str) return gsub(str,"(.)", escapes) end
+function private.replace(str) return utfgsub(str,"(.)", low ) end
+function private.revert (str) return utfgsub(str,"(.)", high ) end
for ch in gmatch(special,".") do set(ch) end
@@ -162,12 +200,7 @@ first snippet uses the relocated dollars.</p>
<typing>
[󰀤x󰀤] [$x$]
</typing>
---ldx]]--
-local cr = utffilters.private.high -- kan via een lpeg
-local cf = utffilters
-
---[[ldx--
<p>The next variant has lazy token collecting, on a 140 page mk.tex this saves
about .25 seconds, which is understandable because we have no graphmes and
not collecting tokens is not only faster but also saves garbage collecting.
@@ -175,20 +208,129 @@ not collecting tokens is not only faster but also saves garbage collecting.
--ldx]]--
-- lpeg variant is not faster
+--
+-- I might use the combined loop at some point for the filter
+-- some day.
+
+--~ function utffilters.collapse(str) -- not really tested (we could preallocate a table)
+--~ if str and str ~= "" then
+--~ local nstr = #str
+--~ if nstr > 1 then
+--~ if initialize then -- saves a call
+--~ initialize()
+--~ end
+--~ local tokens, t, first, done, n = { }, 0, false, false, 0
+--~ for second in utfcharacters(str) do
+--~ local dec = decomposed[second]
+--~ if dec then
+--~ if not done then
+--~ if n > 0 then
+--~ for s in utfcharacters(str) do
+--~ if n == 1 then
+--~ break
+--~ else
+--~ t = t + 1
+--~ tokens[t] = s
+--~ n = n - 1
+--~ end
+--~ end
+--~ end
+--~ done = true
+--~ elseif first then
+--~ t = t + 1
+--~ tokens[t] = first
+--~ end
+--~ t = t + 1
+--~ tokens[t] = dec
+--~ first = false
+--~ elseif done then
+--~ local crs = high[second]
+--~ if crs then
+--~ if first then
+--~ t = t + 1
+--~ tokens[t] = first
+--~ end
+--~ first = crs
+--~ else
+--~ local cgf = graphemes[first]
+--~ if cgf and cgf[second] then
+--~ first = cgf[second]
+--~ elseif first then
+--~ t = t + 1
+--~ tokens[t] = first
+--~ first = second
+--~ else
+--~ first = second
+--~ end
+--~ end
+--~ else
+--~ local crs = high[second]
+--~ if crs then
+--~ for s in utfcharacters(str) do
+--~ if n == 1 then
+--~ break
+--~ else
+--~ t = t + 1
+--~ tokens[t] = s
+--~ n = n - 1
+--~ end
+--~ end
+--~ if first then
+--~ t = t + 1
+--~ tokens[t] = first
+--~ end
+--~ first = crs
+--~ done = true
+--~ else
+--~ local cgf = graphemes[first]
+--~ if cgf and cgf[second] then
+--~ for s in utfcharacters(str) do
+--~ if n == 1 then
+--~ break
+--~ else
+--~ t = t + 1
+--~ tokens[t] = s
+--~ n = n - 1
+--~ end
+--~ end
+--~ first = cgf[second]
+--~ done = true
+--~ else
+--~ first = second
+--~ n = n + 1
+--~ end
+--~ end
+--~ end
+--~ end
+--~ if done then
+--~ if first then
+--~ t = t + 1
+--~ tokens[t] = first
+--~ end
+--~ return concat(tokens) -- seldom called
+--~ end
+--~ elseif nstr > 0 then
+--~ return high[str] or str
+--~ end
+--~ end
+--~ return str
+--~ end
function utffilters.collapse(str) -- not really tested (we could preallocate a table)
- if cf.collapsing and str then
- if #str > 1 then
- if not cf.initialized then -- saves a call
- cf.initialize()
+ if str and str ~= "" then
+ local nstr = #str
+ if nstr > 1 then
+ if initialize then -- saves a call
+ initialize()
end
- local tokens, first, done, n = { }, false, false, 0
+ local tokens, t, first, done, n = { }, 0, false, false, 0
for second in utfcharacters(str) do
if done then
- local crs = cr[second]
+ local crs = high[second]
if crs then
if first then
- tokens[#tokens+1] = first
+ t = t + 1
+ tokens[t] = first
end
first = crs
else
@@ -196,26 +338,31 @@ function utffilters.collapse(str) -- not really tested (we could preallocate a t
if cgf and cgf[second] then
first = cgf[second]
elseif first then
- tokens[#tokens+1] = first
+ t = t + 1
+ tokens[t] = first
first = second
else
first = second
end
end
else
- local crs = cr[second]
+ local crs = high[second]
if crs then
for s in utfcharacters(str) do
if n == 1 then
break
else
- tokens[#tokens+1], n = s, n - 1
+ t = t + 1
+ tokens[t] = s
+ n = n - 1
end
end
if first then
- tokens[#tokens+1] = first
+ t = t + 1
+ tokens[t] = first
end
- first, done = crs, true
+ first = crs
+ done = true
else
local cgf = graphemes[first]
if cgf and cgf[second] then
@@ -223,51 +370,116 @@ function utffilters.collapse(str) -- not really tested (we could preallocate a t
if n == 1 then
break
else
- tokens[#tokens+1], n = s, n -1
+ t = t + 1
+ tokens[t] = s
+ n = n - 1
end
end
- first, done = cgf[second], true
+ first = cgf[second]
+ done = true
else
- first, n = second, n + 1
+ first = second
+ n = n + 1
+ end
+ end
+ end
+ end
+ if done then
+ if first then
+ t = t + 1
+ tokens[t] = first
+ end
+ return concat(tokens) -- seldom called
+ end
+ elseif nstr > 0 then
+ return high[str] or str
+ end
+ end
+ return str
+end
+
+function utffilters.decompose(str)
+ if str and str ~= "" then
+ local nstr = #str
+ if nstr > 1 then
+ -- if initialize then -- saves a call
+ -- initialize()
+ -- end
+ local tokens, t, done, n = { }, 0, false, 0
+ for s in utfcharacters(str) do
+ local dec = decomposed[s]
+ if dec then
+ if not done then
+ if n > 0 then
+ for s in utfcharacters(str) do
+ if n == 1 then
+ break
+ else
+ t = t + 1
+ tokens[t] = s
+ n = n - 1
+ end
+ end
end
+ done = true
end
+ t = t + 1
+ tokens[t] = dec
+ elseif done then
+ t = t + 1
+ tokens[t] = s
+ else
+ n = n + 1
end
end
if done then
- tokens[#tokens+1] = first
return concat(tokens) -- seldom called
end
- elseif #str > 0 then
- return cr[str] or str
end
end
return str
end
+local textfileactions = resolvers.openers.helpers.textfileactions
+
+utilities.sequencers.appendaction (textfileactions,"system","characters.filters.utf.collapse")
+utilities.sequencers.disableaction(textfileactions,"characters.filters.utf.collapse")
+
+utilities.sequencers.appendaction (textfileactions,"system","characters.filters.utf.decompose")
+utilities.sequencers.disableaction(textfileactions,"characters.filters.utf.decompose")
+
+function characters.filters.utf.enable()
+ utilities.sequencers.enableaction(textfileactions,"characters.filters.utf.collapse")
+ utilities.sequencers.enableaction(textfileactions,"characters.filters.utf.decompose")
+end
+
--[[ldx--
<p>Next we implement some commands that are used in the user interface.</p>
--ldx]]--
-commands = commands or { }
-
-function commands.uchar(first,second)
- tex.sprint(ctxcatcodes,utfchar(first*256+second))
-end
+-- commands = commands or { }
+--
+-- function commands.uchar(first,second)
+-- context(utfchar(first*256+second))
+-- end
--[[ldx--
<p>A few helpers (used to be <t>luat-uni<t/>).</p>
--ldx]]--
-function utf.split(str)
- local t = { }
- for snippet in utfcharacters(str) do
- t[#t+1] = snippet
- end
- return t
-end
-
-function utf.each(str,fnc)
- for snippet in utfcharacters(str) do
- fnc(snippet)
- end
-end
+-- obsolete:
+--
+-- function utf.split(str)
+-- local t, n = { }, 0
+-- for snippet in utfcharacters(str) do
+-- n = n + 1
+-- t[n+1] = snippet
+-- end
+-- return t
+-- end
+--
+-- function utf.each(str,fnc)
+-- for snippet in utfcharacters(str) do
+-- fnc(snippet)
+-- end
+-- end