commit context 2011.05.18

git-svn-id: svn://tug.org/texlive/trunk@22719 c570f23f-e606-0410-a88d-b1316a301751

1 files changed, 324 insertions, 112 deletions

diff --git a/Master/texmf-dist/tex/context/base/char-utf.lua b/Master/texmf-dist/tex/context/base/char-utf.lua
index 6dd85fdc87d..30124a6a2a6 100644
--- a/Master/texmf-dist/tex/context/base/char-utf.lua
+++ b/Master/texmf-dist/tex/context/base/char-utf.lua
@@ -19,57 +19,118 @@ in special kinds of output (for instance <l n='pdf'/>).</p>
 over a string.</p>
 --ldx]]--
 
-local utf = unicode.utf8
-local concat, gmatch = table.concat, string.gmatch
+local utfchar, utfbyte, utfgsub = utf.char, utf.byte, utf.gsub
+local concat, gmatch, gsub, find = table.concat, string.gmatch, string.gsub, string.find
 local utfcharacters, utfvalues = string.utfcharacters, string.utfvalues
+local allocate = utilities.storage.allocate
 
-local ctxcatcodes = tex.ctxcatcodes
+local charfromnumber = characters.fromnumber
 
-characters              = characters              or { }
-characters.graphemes    = characters.graphemes    or { }
-characters.filters      = characters.filters      or { }
-characters.filters.utf  = characters.filters.utf  or { }
+-- todo: trackers
 
-characters.filters.utf.initialized = false
-characters.filters.utf.collapsing  = true
-characters.filters.utf.expanding   = true
+characters            = characters or { }
+local characters      = characters
 
-local graphemes  = characters.graphemes
-local utffilters = characters.filters.utf
-local utfchar, utfbyte, utfgsub = utf.char, utf.byte, utf.gsub
+characters.graphemes  = allocate()
+local graphemes       = characters.graphemes
+
+characters.decomposed = allocate()
+local decomposed      = characters.decomposed
+
+characters.mathpairs  = allocate()
+local mathpairs       = characters.mathpairs
+
+characters.filters    = allocate()
+local filters         = characters.filters
+
+filters.utf           = filters.utf  or { }
+local utffilters      = characters.filters.utf
 
 --[[ldx--
 <p>It only makes sense to collapse at runtime, since we don't expect
 source code to depend on collapsing.</p>
 --ldx]]--
 
-function utffilters.initialize()
-    if utffilters.collapsing and not utffilters.initialized then
-        for k,v in next, characters.data do
-            -- using vs and first testing for length is faster (.02->.01 s)
-            local vs = v.specials
-            if vs and #vs == 3 and vs[1] == 'char' then
-                local first, second = utfchar(vs[2]), utfchar(vs[3])
-                local cgf = graphemes[first]
-                if not cgf then
-                    cgf = { }
-                    graphemes[first] = cgf
+-- for the moment, will be entries in char-def.lua
+
+local decomposed = allocate {
+    ["IJ"] = "IJ",
+    ["ij"] = "ij",
+    ["և"] = "եւ",
+    ["ff"] = "ff",
+    ["fi"] = "fi",
+    ["fl"] = "fl",
+    ["ffi"] = "ffi",
+    ["ffl"] = "ffl",
+    ["ſt"] = "ſt",
+    ["st"] = "st",
+    ["ﬓ"] = "մն",
+    ["ﬔ"] = "մե",
+    ["ﬕ"] = "մի",
+    ["ﬖ"] = "վն",
+    ["ﬗ"] = "մխ",
+}
+characters.decomposed = decomposed
+
+local function initialize()
+    for unicode, v in next, characters.data do
+        -- using vs and first testing for length is faster (.02->.01 s)
+        local vs = v.specials
+        if vs and #vs == 3 and vs[1] == 'char' then
+            local one, two = vs[2], vs[3]
+            local first, second, combined = utfchar(one), utfchar(two), utfchar(unicode)
+            local cgf = graphemes[first]
+            if not cgf then
+                cgf = { }
+                graphemes[first] = cgf
+            end
+            cgf[second] = combined
+            if v.mathclass or v.mathspec then
+                local mps = mathpairs[two]
+                if not mps then
+                    mps = { }
+                    mathpairs[two] = mps
+                end
+                mps[one] = unicode -- here unicode
+                local mps = mathpairs[second]
+                if not mps then
+                    mps = { }
+                    mathpairs[second] = mps
                 end
-                cgf[second] = utfchar(k)
+                mps[first] = combined
             end
+     -- else
+     --     local description = v.description
+     --     if find(description,"LIGATURE") then
+     --         if vs then
+     --             local t = { }
+     --             for i=2,#vs do
+     --                 t[#t+1] = utfchar(vs[i])
+     --             end
+     --             decomposed[utfchar(unicode)] = concat(t)
+     --         else
+     --             local vs = v.shcode
+     --             if vs then
+     --                 local t = { }
+     --                 for i=1,#vs do
+     --                     t[i] = utfchar(vs[i])
+     --                 end
+     --                 decomposed[utfchar(unicode)] = concat(t)
+     --             end
+     --         end
+     --     end
         end
-        utffilters.initialized = true
     end
+    initialize = false
 end
 
--- utffilters.add_grapheme(utfchar(318),'l','\string~')
--- utffilters.add_grapheme('c','a','b')
+-- utffilters.addgrapheme(utfchar(318),'l','\string~')
+-- utffilters.addgrapheme('c','a','b')
 
-function utffilters.add_grapheme(result,first,second)
-    local r, f, s = tonumber(result), tonumber(first), tonumber(second)
-    if r then result = utfchar(r) end
-    if f then first  = utfchar(f) end
-    if s then second = utfchar(s) end
+function utffilters.addgrapheme(result,first,second) -- can be U+ 0x string or utf or number
+    local result = charfromnumber(result)
+    local first  = charfromnumber(first)
+    local second = charfromnumber(second)
     if not graphemes[first] then
         graphemes[first] = { [second] = result }
     else
@@ -77,31 +138,6 @@ function utffilters.add_grapheme(result,first,second)
     end
 end
 
-function utffilters.collapse(str) -- old one
-    if utffilters.collapsing and str and #str > 1 then
-        if not utffilters.initialized then -- saves a call
-            utffilters.initialize()
-        end
-        local tokens, first, done = { }, false, false
-        for second in utfcharacters(str) do
-            local cgf = graphemes[first]
-            if cgf and cgf[second] then
-                first, done = cgf[second], true
-            elseif first then
-                tokens[#tokens+1] = first
-                first = second
-            else
-                first = second
-            end
-        end
-        if done then
-            tokens[#tokens+1] = first
-            return concat(tokens)
-        end
-    end
-    return str
-end
-
 --[[ldx--
 <p>In order to deal with 8-bit output, we need to find a way to
 go from <l n='utf'/> to 8-bit. This is handled in the
@@ -120,18 +156,20 @@ to their right glyph there.</p>
 0x100000.</p>
 --ldx]]--
 
-utffilters.private = {
-    high    = { },
-    low     = { },
-    escapes = { },
+local low     = allocate({ })
+local high    = allocate({ })
+local escapes = allocate({ })
+local special = "~#$%^&_{}\\|"
+
+local private = {
+    low     = low,
+    high    = high,
+    escapes = escapes,
 }
 
-local low     = utffilters.private.low
-local high    = utffilters.private.high
-local escapes = utffilters.private.escapes
-local special = "~#$%^&_{}\\|"
+utffilters.private = private
 
-function utffilters.private.set(ch)
+local function set(ch)
     local cb
     if type(ch) == "number" then
         cb, ch = ch, utfchar(ch)
@@ -145,11 +183,11 @@ function utffilters.private.set(ch)
     end
 end
 
-function utffilters.private.replace(str) return utfgsub(str,"(.)", low    ) end
-function utffilters.private.revert(str)  return utfgsub(str,"(.)", high   ) end
-function utffilters.private.escape(str)  return utfgsub(str,"(.)", escapes) end
+private.set = set
 
-local set = utffilters.private.set
+function private.escape (str) return    gsub(str,"(.)", escapes) end
+function private.replace(str) return utfgsub(str,"(.)", low    ) end
+function private.revert (str) return utfgsub(str,"(.)", high   ) end
 
 for ch in gmatch(special,".") do set(ch) end
 
@@ -162,12 +200,7 @@ first snippet uses the relocated dollars.</p>
 <typing>
 [󰀤x󰀤] [$x$]
 </typing>
---ldx]]--
 
-local cr = utffilters.private.high -- kan via een lpeg
-local cf = utffilters
-
---[[ldx--
 <p>The next variant has lazy token collecting, on a 140 page mk.tex this saves
 about .25 seconds, which is understandable because we have no graphmes and
 not collecting tokens is not only faster but also saves garbage collecting.
@@ -175,20 +208,129 @@ not collecting tokens is not only faster but also saves garbage collecting.
 --ldx]]--
 
 -- lpeg variant is not faster
+--
+-- I might use the combined loop at some point for the filter
+-- some day.
+
+--~ function utffilters.collapse(str) -- not really tested (we could preallocate a table)
+--~     if str and str ~= "" then
+--~         local nstr = #str
+--~         if nstr > 1 then
+--~             if initialize then -- saves a call
+--~                 initialize()
+--~             end
+--~             local tokens, t, first, done, n = { }, 0, false, false, 0
+--~             for second in utfcharacters(str) do
+--~                 local dec = decomposed[second]
+--~                 if dec then
+--~                     if not done then
+--~                         if n > 0 then
+--~                             for s in utfcharacters(str) do
+--~                                 if n == 1 then
+--~                                     break
+--~                                 else
+--~                                     t = t + 1
+--~                                     tokens[t] = s
+--~                                     n = n - 1
+--~                                 end
+--~                             end
+--~                         end
+--~                         done = true
+--~                     elseif first then
+--~                         t = t + 1
+--~                         tokens[t] = first
+--~                     end
+--~                     t = t + 1
+--~                     tokens[t] = dec
+--~                     first = false
+--~                 elseif done then
+--~                     local crs = high[second]
+--~                     if crs then
+--~                         if first then
+--~                             t = t + 1
+--~                             tokens[t] = first
+--~                         end
+--~                         first = crs
+--~                     else
+--~                         local cgf = graphemes[first]
+--~                         if cgf and cgf[second] then
+--~                             first = cgf[second]
+--~                         elseif first then
+--~                             t = t + 1
+--~                             tokens[t] = first
+--~                             first = second
+--~                         else
+--~                             first = second
+--~                         end
+--~                     end
+--~                 else
+--~                     local crs = high[second]
+--~                     if crs then
+--~                         for s in utfcharacters(str) do
+--~                             if n == 1 then
+--~                                 break
+--~                             else
+--~                                 t = t + 1
+--~                                 tokens[t] = s
+--~                                 n = n - 1
+--~                             end
+--~                         end
+--~                         if first then
+--~                             t = t + 1
+--~                             tokens[t] = first
+--~                         end
+--~                         first = crs
+--~                         done = true
+--~                     else
+--~                         local cgf = graphemes[first]
+--~                         if cgf and cgf[second] then
+--~                             for s in utfcharacters(str) do
+--~                                 if n == 1 then
+--~                                     break
+--~                                 else
+--~                                     t = t + 1
+--~                                     tokens[t] = s
+--~                                     n = n - 1
+--~                                 end
+--~                             end
+--~                             first = cgf[second]
+--~                             done = true
+--~                         else
+--~                             first = second
+--~                             n = n + 1
+--~                         end
+--~                     end
+--~                 end
+--~             end
+--~             if done then
+--~                 if first then
+--~                     t = t + 1
+--~                     tokens[t] = first
+--~                 end
+--~                 return concat(tokens) -- seldom called
+--~             end
+--~         elseif nstr > 0 then
+--~             return high[str] or str
+--~         end
+--~     end
+--~     return str
+--~ end
 
 function utffilters.collapse(str) -- not really tested (we could preallocate a table)
-    if cf.collapsing and str then
-        if #str > 1 then
-            if not cf.initialized then -- saves a call
-                cf.initialize()
+    if str and str ~= "" then
+        local nstr = #str
+        if nstr > 1 then
+            if initialize then -- saves a call
+                initialize()
             end
-            local tokens, first, done, n = { }, false, false, 0
+            local tokens, t, first, done, n = { }, 0, false, false, 0
             for second in utfcharacters(str) do
                 if done then
-                    local crs = cr[second]
+                    local crs = high[second]
                     if crs then
                         if first then
-                            tokens[#tokens+1] = first
+                            t = t + 1
+                            tokens[t] = first
                         end
                         first = crs
                     else
@@ -196,26 +338,31 @@ function utffilters.collapse(str) -- not really tested (we could preallocate a t
                         if cgf and cgf[second] then
                             first = cgf[second]
                         elseif first then
-                            tokens[#tokens+1] = first
+                            t = t + 1
+                            tokens[t] = first
                             first = second
                         else
                             first = second
                         end
                     end
                 else
-                    local crs = cr[second]
+                    local crs = high[second]
                     if crs then
                         for s in utfcharacters(str) do
                             if n == 1 then
                                 break
                             else
-                                tokens[#tokens+1], n = s, n - 1
+                                t = t + 1
+                                tokens[t] = s
+                                n = n - 1
                             end
                         end
                         if first then
-                            tokens[#tokens+1] = first
+                            t = t + 1
+                            tokens[t] = first
                         end
-                        first, done = crs, true
+                        first = crs
+                        done = true
                     else
                         local cgf = graphemes[first]
                         if cgf and cgf[second] then
@@ -223,51 +370,116 @@ function utffilters.collapse(str) -- not really tested (we could preallocate a t
                                 if n == 1 then
                                     break
                                 else
-                                    tokens[#tokens+1], n = s, n -1
+                                    t = t + 1
+                                    tokens[t] = s
+                                    n = n - 1
                                 end
                             end
-                            first, done = cgf[second], true
+                            first = cgf[second]
+                            done = true
                         else
-                            first, n = second, n + 1
+                            first = second
+                            n = n + 1
+                        end
+                    end
+                end
+            end
+            if done then
+                if first then
+                    t = t + 1
+                    tokens[t] = first
+                end
+                return concat(tokens) -- seldom called
+            end
+        elseif nstr > 0 then
+            return high[str] or str
+        end
+    end
+    return str
+end
+
+function utffilters.decompose(str)
+    if str and str ~= "" then
+        local nstr = #str
+        if nstr > 1 then
+         -- if initialize then -- saves a call
+         --     initialize()
+         -- end
+            local tokens, t, done, n = { }, 0, false, 0
+            for s in utfcharacters(str) do
+                local dec = decomposed[s]
+                if dec then
+                    if not done then
+                        if n > 0 then
+                            for s in utfcharacters(str) do
+                                if n == 1 then
+                                    break
+                                else
+                                    t = t + 1
+                                    tokens[t] = s
+                                    n = n - 1
+                                end
+                            end
                         end
+                        done = true
                     end
+                    t = t + 1
+                    tokens[t] = dec
+                elseif done then
+                    t = t + 1
+                    tokens[t] = s
+                else
+                    n = n + 1
                 end
             end
             if done then
-                tokens[#tokens+1] = first
                 return concat(tokens) -- seldom called
             end
-        elseif #str > 0 then
-            return cr[str] or str
         end
     end
     return str
 end
 
+local textfileactions = resolvers.openers.helpers.textfileactions
+
+utilities.sequencers.appendaction (textfileactions,"system","characters.filters.utf.collapse")
+utilities.sequencers.disableaction(textfileactions,"characters.filters.utf.collapse")
+
+utilities.sequencers.appendaction (textfileactions,"system","characters.filters.utf.decompose")
+utilities.sequencers.disableaction(textfileactions,"characters.filters.utf.decompose")
+
+function characters.filters.utf.enable()
+    utilities.sequencers.enableaction(textfileactions,"characters.filters.utf.collapse")
+    utilities.sequencers.enableaction(textfileactions,"characters.filters.utf.decompose")
+end
+
 --[[ldx--
 <p>Next we implement some commands that are used in the user interface.</p>
 --ldx]]--
 
-commands = commands or { }
-
-function commands.uchar(first,second)
-    tex.sprint(ctxcatcodes,utfchar(first*256+second))
-end
+-- commands = commands or { }
+--
+-- function commands.uchar(first,second)
+--     context(utfchar(first*256+second))
+-- end
 
 --[[ldx--
 <p>A few helpers (used to be <t>luat-uni<t/>).</p>
 --ldx]]--
 
-function utf.split(str)
-    local t = { }
-    for snippet in utfcharacters(str) do
-        t[#t+1] = snippet
-    end
-    return t
-end
-
-function utf.each(str,fnc)
-    for snippet in utfcharacters(str) do
-        fnc(snippet)
-    end
-end
+-- obsolete:
+--
+-- function utf.split(str)
+--     local t, n = { }, 0
+--     for snippet in utfcharacters(str) do
+--         n = n + 1
+--         t[n+1] = snippet
+--     end
+--     return t
+-- end
+--
+-- function utf.each(str,fnc)
+--     for snippet in utfcharacters(str) do
+--         fnc(snippet)
+--     end
+-- end