diff options
Diffstat (limited to 'Master/texmf-dist/tex/context/base/sort-ini.lua')
-rw-r--r-- | Master/texmf-dist/tex/context/base/sort-ini.lua | 815 |
1 files changed, 625 insertions, 190 deletions
diff --git a/Master/texmf-dist/tex/context/base/sort-ini.lua b/Master/texmf-dist/tex/context/base/sort-ini.lua index b745c9aa5db..7a91910ca7d 100644 --- a/Master/texmf-dist/tex/context/base/sort-ini.lua +++ b/Master/texmf-dist/tex/context/base/sort-ini.lua @@ -9,88 +9,444 @@ if not modules then modules = { } end modules ['sort-ini'] = { -- It took a while to get there, but with Fleetwood Mac's "Don't Stop" -- playing in the background we sort of got it done. --- todo: cleanup splits (in other modules) +--[[<p>The code here evolved from the rather old mkii approach. There +we concatinate the key and (raw) entry into a new string. Numbers and +special characters get some treatment so that they sort ok. In +addition some normalization (lowercasing, accent stripping) takes +place and again data is appended ror prepended. Eventually these +strings are sorted using a regular string sorter. The relative order +of character is dealt with by weighting them. It took a while to +figure this all out but eventually it worked ok for most languages, +given that the right datatables were provided.</p> + +<p>Here we do follow a similar approach but this time we don't append +the manipulated keys and entries but create tables for each of them +with entries being tables themselves having different properties. In +these tables characters are represented by numbers and sorting takes +place using these numbers. Strings are simplified using lowercasing +as well as shape codes. Numbers are filtered and after getting an offset +they end up at the right end of the spectrum (more clever parser will +be added some day). There are definitely more solutions to the problem +and it is a nice puzzle to solve.</p> + +<p>In the future more methods can be added, as there is practically no +limit to what goes into the tables. For that we will provide hooks.</p> + +<p>Todo: decomposition with specific order of accents, this is +relatively easy to do.</p> + +<p>Todo: investigate what standards and conventions there are and see +how they map onto this mechanism. I've learned that users can come up +with any demand so nothing here is frozen.</p> + +<p>In the future index entries will become more clever, i.e. they will +have language etc properties that then can be used.</p> +]]-- + local utf = unicode.utf8 -local gsub, rep, sort, concat = string.gsub, string.rep, table.sort, table.concat +local gsub, rep, sub, sort, concat = string.gsub, string.rep, string.sub, table.sort, table.concat local utfbyte, utfchar = utf.byte, utf.char -local utfcharacters, utfvalues, strcharacters = string.utfcharacters, string.utfvalues, string.characters -local chardata = characters.data -local next, type, tonumber = next, type, tonumber - -local trace_tests = false trackers.register("sorters.tests", function(v) trace_tests = v end) - -sorters = { } -sorters.comparers = { } -sorters.splitters = { } -sorters.entries = { } -sorters.mappings = { } -sorters.replacements = { } - -sorters.ignored_offset = 0x10000 -sorters.replacement_offset = 0x10000 -sorters.digits_offset = 0x20000 -sorters.digits_maximum = 0xFFFFF - -local ignored_offset = sorters.ignored_offset -local digits_offset = sorters.digits_offset -local digits_maximum = sorters.digits_maximum - -local mappings = sorters.mappings -local entries = sorters.entries -local replacements = sorters.replacements - -local language, defaultlanguage, dummy = 'en', 'en', { } - -local currentreplacements, currentmappings, currententries - -function sorters.setlanguage(lang) - language = lang or language or defaultlanguage - currentreplacements = replacements[language] or replacements[defaultlanguage] or dummy - currentmappings = mappings [language] or mappings [defaultlanguage] or dummy - currententries = entries [language] or entries [defaultlanguage] or dummy - return currentreplacements, currentmappings, currententries -end +local utfcharacters, utfvalues = string.utfcharacters, string.utfvalues +local next, type, tonumber, rawget, rawset = next, type, tonumber, rawget, rawset -sorters.setlanguage() +local allocate = utilities.storage.allocate +local setmetatableindex = table.setmetatableindex --- maybe inline code if it's too slow +local trace_tests = false trackers.register("sorters.tests", function(v) trace_tests = v end) +local trace_methods = false trackers.register("sorters.methods", function(v) trace_methods = v end) -local function basicsort(sort_a,sort_b) - if not sort_a or not sort_b then - return 0 - elseif #sort_a > #sort_b then - if #sort_b == 0 then - return 1 - else - for i=1,#sort_b do - local ai, bi = sort_a[i], sort_b[i] - if ai > bi then - return 1 - elseif ai < bi then - return -1 +local report_sorters = logs.reporter("languages","sorters") + +local comparers = { } +local splitters = { } +local definitions = allocate() +local tracers = allocate() +local ignoredoffset = 0x10000 -- frozen +local replacementoffset = 0x10000 -- frozen +local digitsoffset = 0x20000 -- frozen +local digitsmaximum = 0xFFFFF -- frozen + +local lccodes = characters.lccodes +local lcchars = characters.lcchars +local shchars = characters.shchars +local fscodes = characters.fscodes +local fschars = characters.fschars + +local decomposed = characters.decomposed + +local variables = interfaces.variables + +local v_numbers = variables.numbers +local v_default = variables.default +local v_before = variables.before +local v_after = variables.after +local v_first = variables.first +local v_last = variables.last + +local validmethods = table.tohash { + -- "ch", -- raw character + "mm", -- minus mapping + "zm", -- zero mapping + "pm", -- plus mapping + "mc", -- lower case - 1 + "zc", -- lower case + "pc", -- lower case + 1 + "uc", -- unicode +} + +local predefinedmethods = { + [v_before] = "mm,mc,uc", + [v_after] = "pm,mc,uc", + [v_first] = "pc,mm,uc", + [v_last] = "mc,mm,uc", +} + +sorters = { + comparers = comparers, + splitters = splitters, + definitions = definitions, + tracers = tracers, + constants = { + ignoredoffset = ignoredoffset, + replacementoffset = replacementoffset, + digitsoffset = digitsoffset, + digitsmaximum = digitsmaximum, + defaultlanguage = v_default, + defaultmethod = v_before, + defaultdigits = v_numbers, + } +} + +local sorters = sorters +local constants = sorters.constants + +local data, language, method, digits +local replacements, m_mappings, z_mappings, p_mappings, entries, orders, lower, upper, method, sequence +local thefirstofsplit + +local mte = { + __index = function(t,k) + if k ~= "" and utfbyte(k) < digitsoffset then + local el + if k then + local l = lower[k] or lcchars[k] + el = rawget(t,l) + end + if not el then + local l = shchars[k] + if l and l ~= k then + if #l > 1 then + l = sub(l,1,1) -- todo + end + el = rawget(t,l) + if not el then + l = lower[k] or lcchars[l] + if l then + el = rawget(t,l) + end + end end + el = el or k end - return 1 + -- rawset(t,k,el) also make a copy? + return el end - elseif #sort_a < #sort_b then - if #sort_a == 0 then - return -1 - else - for i=1,#sort_a do - local ai, bi = sort_a[i], sort_b[i] - if ai > bi then - return 1 - elseif ai < bi then - return -1 + end +} + +local noorder = false + +local function preparetables(data) + local orders, lower, m_mappings, z_mappings, p_mappings = data.orders, data.lower, { }, { }, { } + for i=1,#orders do + local oi = orders[i] + local n = { 2 * i } + m_mappings[oi], z_mappings[oi], p_mappings[oi] = n, n, n + end + local mtm = { + __index = function(t,k) + local n, nn + if k then + if trace_tests then + report_sorters("simplifing character 0x%04X %s",utfbyte(k),k) + end + local l = lower[k] or lcchars[k] + if l then + if trace_tests then + report_sorters(" 1 lower: %s",l) + end + local ml = rawget(t,l) + if ml then + n = { } + nn = 0 + for i=1,#ml do + nn = nn + 1 + n[nn] = ml[i] + (t.__delta or 0) + end + if trace_tests then + report_sorters(" 2 order: %s",concat(n," ")) + end + end + end + if not n then + local s = shchars[k] -- maybe all components? + if s and s ~= k then + if trace_tests then + report_sorters(" 3 shape: %s",s) + end + n = { } + nn = 0 + for l in utfcharacters(s) do + local ml = rawget(t,l) + if ml then + if trace_tests then + report_sorters(" 4 keep: %s",l) + end + if ml then + for i=1,#ml do + nn = nn + 1 + n[nn] = ml[i] + end + end + else + l = lower[l] or lcchars[l] + if l then + if trace_tests then + report_sorters(" 5 lower: %s",l) + end + local ml = rawget(t,l) + if ml then + for i=1,#ml do + nn = nn + 1 + n[nn] = ml[i] + (t.__delta or 0) + end + end + end + end + end + else -- we probably never enter this branch + -- fschars returns a single char +--~ s = fschars[k] +--~ if s and s ~= k then +--~ if trace_tests then +--~ report_sorters(" 6 split: %s",s) +--~ end +--~ local ml = rawget(t,s) +--~ if ml then +--~ n = { } +--~ nn = 0 +--~ for i=1,#ml do +--~ nn = nn + 1 +--~ n[nn] = ml[i] +--~ end +--~ end +--~ end + local b = utfbyte(k) + n = decomposed[b] or { b } + if trace_tests then + report_sorters(" 6 split: %s",utf.string(b)) + end + end + if n then + if trace_tests then + report_sorters(" 7 order: %s",concat(n," ")) + end + else + n = noorder + if trace_tests then + report_sorters(" 8 order: 0") + end + end + end + else + n = noorder + if trace_tests then + report_sorters(" 9 order: 0") end end - return -1 + rawset(t,k,n) + return n end - elseif #sort_a == 0 then - return 0 - else - for i=1,#sort_a do + } + data.m_mappings = m_mappings + data.z_mappings = z_mappings + data.p_mappings = p_mappings + m_mappings.__delta = -1 + z_mappings.__delta = 0 + p_mappings.__delta = 1 + setmetatable(data.entries,mte) + setmetatable(data.m_mappings,mtm) + setmetatable(data.z_mappings,mtm) + setmetatable(data.p_mappings,mtm) + thefirstofsplit = data.firstofsplit +end + +local function update() -- prepare parent chains, needed when new languages are added + for language, data in next, definitions do + local parent = data.parent or "default" + if language ~= "default" then + setmetatableindex(data,definitions[parent] or definitions.default) + end + data.language = language + data.parent = parent + data.m_mappings = { } -- free temp data + data.z_mappings = { } -- free temp data + data.p_mappings = { } -- free temp data + end +end + +local function setlanguage(l,m,d,u) + language = (l ~= "" and l) or constants.defaultlanguage + data = definitions[language or constants.defaultlanguage] or definitions[constants.defaultlanguage] + method = (m ~= "" and m) or data.method or constants.defaultmethod + digits = (d ~= "" and d) or data.digits or constants.defaultdigits + if trace_tests then + report_sorters("setting language '%s', method '%s', digits '%s'",language,method,digits) + end + replacements = data.replacements + entries = data.entries + orders = data.orders + lower = data.lower + upper = data.upper + preparetables(data) + m_mappings = data.m_mappings + z_mappings = data.z_mappings + p_mappings = data.p_mappings + -- + method = predefinedmethods[variables[method]] or method + data.method = method + -- + data.digits = digite + -- + local seq = utilities.parsers.settings_to_array(method or "") -- check the list + sequence = { } + local nofsequence = 0 + for i=1,#seq do + local s = seq[i] + if validmethods[s] then + nofsequence = nofsequence + 1 + sequence[nofsequence] = s + else + report_sorters("invalid sorter method '%s' in '%s'",s,method) + end + end + data.sequence = sequence + if trace_tests then + report_sorters("using sort sequence: %s",concat(sequence," ")) + end + -- + return data +end + +function sorters.update() + update() + setlanguage(language,method,numberorder) -- resync current language and method +end + +function sorters.setlanguage(language,method,numberorder) + update() + setlanguage(language,method,numberorder) -- new language and method +end + +-- tricky: { 0, 0, 0 } vs { 0, 0, 0, 0 } => longer wins and mm, pm, zm can have them + +--~ local function basicsort(sort_a,sort_b) -- todo: local # +--~ if not sort_a or not sort_b then +--~ return 0 +--~ elseif #sort_a > #sort_b then +--~ if #sort_b == 0 then +--~ return 1 +--~ else +--~ for i=1,#sort_b do +--~ local ai, bi = sort_a[i], sort_b[i] +--~ if ai > bi then +--~ return 1 +--~ elseif ai < bi then +--~ return -1 +--~ end +--~ end +--~ return 1 +--~ end +--~ elseif #sort_a < #sort_b then +--~ if #sort_a == 0 then +--~ return -1 +--~ else +--~ for i=1,#sort_a do +--~ local ai, bi = sort_a[i], sort_b[i] +--~ if ai > bi then +--~ return 1 +--~ elseif ai < bi then +--~ return -1 +--~ end +--~ end +--~ return -1 +--~ end +--~ elseif #sort_a == 0 then +--~ return 0 +--~ else +--~ for i=1,#sort_a do +--~ local ai, bi = sort_a[i], sort_b[i] +--~ if ai > bi then +--~ return 1 +--~ elseif ai < bi then +--~ return -1 +--~ end +--~ end +--~ return 0 +--~ end +--~ end + +--~ function comparers.basic(a,b) -- trace ea and eb +--~ local ea, eb = a.split, b.split +--~ local na, nb = #ea, #eb +--~ if na == 0 and nb == 0 then +--~ -- simple variant (single word) +--~ local result = 0 +--~ for j=1,#sequence do +--~ local m = sequence[j] +--~ result = basicsort(ea[m],eb[m]) +--~ if result ~= 0 then +--~ return result +--~ end +--~ end +--~ return result +--~ else +--~ -- complex variant, used in register (multiple words) +--~ local result = 0 +--~ for i=1,nb < na and nb or na do +--~ local eai, ebi = ea[i], eb[i] +--~ for j=1,#sequence do +--~ local m = sequence[j] +--~ result = basicsort(eai[m],ebi[m]) +--~ if result ~= 0 then +--~ return result +--~ end +--~ end +--~ if result ~= 0 then +--~ return result +--~ end +--~ end +--~ if result ~= 0 then +--~ return result +--~ elseif na > nb then +--~ return 1 +--~ elseif nb > na then +--~ return -1 +--~ else +--~ return 0 +--~ end +--~ end +--~ end + +local function basicsort(sort_a,sort_b) + if sort_a and sort_b then + local na = #sort_a + local nb = #sort_b + if na > nb then + na = nb + end + for i=1,na do local ai, bi = sort_a[i], sort_b[i] if ai > bi then return 1 @@ -98,28 +454,56 @@ local function basicsort(sort_a,sort_b) return -1 end end - return 0 end + return 0 end -function sorters.comparers.basic(a,b) +function comparers.basic(a,b) -- trace ea and eb local ea, eb = a.split, b.split local na, nb = #ea, #eb if na == 0 and nb == 0 then -- simple variant (single word) - local result = basicsort(ea.e,eb.e) - return result == 0 and result or basicsort(ea.m,eb.m) + local result = 0 + for j=1,#sequence do + local m = sequence[j] + result = basicsort(ea[m],eb[m]) + if result ~= 0 then + return result + end + end + if result == 0 then + local la, lb = #ea.uc, #eb.uc + if la > lb then + return 1 + elseif lb > la then + return -1 + else + return 0 + end + else + return result + end else -- complex variant, used in register (multiple words) local result = 0 for i=1,nb < na and nb or na do local eai, ebi = ea[i], eb[i] - result = basicsort(eai.e,ebi.e) - if result == 0 then - result = basicsort(eai.m,ebi.m) -- only needed it there are m's + for j=1,#sequence do + local m = sequence[j] + result = basicsort(eai[m],ebi[m]) + if result ~= 0 then + return result + end end - if result ~= 0 then - break + if result == 0 then + local la, lb = #eai.uc, #ebi.uc + if la > lb then + return 1 + elseif lb > la then + return -1 + end + else + return result end end if result ~= 0 then @@ -135,22 +519,22 @@ function sorters.comparers.basic(a,b) end local function numify(s) - return rep(" ",10-#s) .. s -- or format with padd -end - -local function numify(s) - s = digits_offset + tonumber(s) - if s > digits_maximum then - s = digits_maximum + s = digitsoffset + tonumber(s) -- alternatively we can create range + if s > digitsmaximum then + s = digitsmaximum end return utfchar(s) end function sorters.strip(str) -- todo: only letters and such utf.gsub("([^%w%d])","") if str then - str = gsub(str,"\\%S*","") + str = gsub(str,"\\[\"\'~^`]*","") -- \"e + str = gsub(str,"\\%S*","") -- the rest + str = gsub(str,"%s","\001") -- can be option str = gsub(str,"[%s%[%](){}%$\"\']*","") - str = gsub(str,"(%d+)",numify) -- sort numbers properly + if digits == v_numbers then + str = gsub(str,"(%d+)",numify) -- sort numbers properly + end return str else return "" @@ -161,98 +545,191 @@ local function firstofsplit(entry) -- numbers are left padded by spaces local split = entry.split if #split > 0 then - split = split[1].s + split = split[1].ch + else + split = split.ch + end + local first = split and split[1] or "" + if thefirstofsplit then + return thefirstofsplit(first,data,entry) -- normally the first one is needed else - split = split.s + return first, entries[first] or "\000" -- tag end - local entry = split and split[1] or "" - return entry, currententries[entry] or "\000" end sorters.firstofsplit = firstofsplit --- beware, numbers get spaces in front +-- for the moment we use an inefficient bunch of tables but once +-- we know what combinations make sense we can optimize this -function sorters.splitters.utf(str) - if #currentreplacements > 0 then - for k=1,#currentreplacements do - local v = currentreplacements[k] +function splitters.utf(str) -- we could append m and u but this is cleaner, s is for tracing + if #replacements > 0 then + -- todo make an lpeg for this + for k=1,#replacements do + local v = replacements[k] str = gsub(str,v[1],v[2]) end end - local s, e, m, n = { }, { }, { }, 0 - for sc in utfcharacters(str) do -- maybe an lpeg - local ec, mc = currententries[sc], currentmappings[sc] or utfbyte(sc) - n = n + 1 - s[n] = sc - e[n] = currentmappings[ec] or mc - m[n] = mc - end - return { s = s, e = e, m = m } + local m_case, z_case, p_case, m_mapping, z_mapping, p_mapping, char, byte, n = { }, { }, { }, { }, { }, { }, { }, { }, 0 + local nm, nz, np = 0, 0, 0 + for sc in utfcharacters(str) do + local b = utfbyte(sc) + if b >= digitsoffset then + if n == 0 then + -- we need to force number to the top + z_case[1] = 0 + m_case[1] = 0 + p_case[1] = 0 + char[1] = sc + byte[1] = 0 + m_mapping[1] = 0 + z_mapping[1] = 0 + p_mapping[1] = 0 + n = 2 + else + n = n + 1 + end + z_case[n] = b + m_case[n] = b + p_case[n] = b + char[n] = sc + byte[n] = b + nm = nm + 1 + nz = nz + 1 + np = np + 1 + m_mapping[nm] = b + z_mapping[nz] = b + p_mapping[np] = b + else + n = n + 1 + local l = lower[sc] + l = l and utfbyte(l) or lccodes[b] + if type(l) == "table" then + l = l[1] -- there are currently no tables in lccodes but it can be some, day + end + z_case[n] = l + if l ~= b then + m_case[n] = l - 1 + p_case[n] = l + 1 + else + m_case[n] = l + p_case[n] = l + end + char[n], byte[n] = sc, b +local fs = fscodes[b] or b + local msc = m_mappings[sc] + if msc ~= noorder then +if not msc then + msc = m_mappings[fs] end - --- we can use one array instead (sort of like in mkii) --- but for the moment we do it this way as it is more --- handy for tracing - --- function sorters.splitters.utf(str) --- if #currentreplacements > 0 then --- for k=1,#currentreplacements do --- local v = currentreplacements[k] --- str = gsub(str,v[1],v[2]) --- end --- end --- local s, e, m, n = { }, { }, { }, 0 --- for sc in utfcharacters(str) do -- maybe an lpeg --- local ec, mc = currententries[sc], currentmappings[sc] or utfbyte(sc) --- n = n + 1 --- ec = currentmappings[ec] or mc --- s[n] = sc --- e[n] = ec --- if ec ~= mc then --- n = n + 1 --- e[n] = mc --- end --- end --- return { s = s, e = e } --- end - -function table.remap(t) - local tt = { } - for k,v in next, t do - tt[v] = k + for i=1,#msc do + nm = nm + 1 + m_mapping[nm] = msc[i] + end + end + local zsc = z_mappings[sc] + if zsc ~= noorder then +if not zsc then + zsc = z_mappings[fs] +end + for i=1,#zsc do + nz = nz + 1 + z_mapping[nz] = zsc[i] + end + end + local psc = p_mappings[sc] + if psc ~= noorder then +if not psc then + psc = p_mappings[fs] +end + for i=1,#psc do + np = np + 1 + p_mapping[np] = psc[i] + end + end + end end - return tt + + -- only those needed that are part of a sequence + +--~ local b = byte[1] +--~ if b then +--~ -- we set them to the first split code (korean) +--~ local fs = fscodes[b] or b +--~ if #m_mapping == 0 then +--~ m_mapping = { m_mappings[fs][1] } +--~ end +--~ if #z_mapping == 0 then +--~ z_mapping = { z_mappings[fs][1] } +--~ end +--~ if #p_mapping == 0 then +--~ p_mapping = { p_mappings[fs][1] } +--~ end +--~ end + + local t = { + ch = char, + uc = byte, + mc = m_case, + zc = z_case, + pc = p_case, + mm = m_mapping, + zm = z_mapping, + pm = p_mapping, + } + + return t end -local function pack(entry) - local t = { } +local function packch(entry) local split = entry.split - if #split > 0 then + if #split > 0 then -- useless test + local t = { } for i=1,#split do - local tt, li = { }, split[i].s + local tt, li = { }, split[i].ch for j=1,#li do local lij = li[j] - tt[j] = utfbyte(lij) > ignored_offset and "[]" or lij + tt[j] = utfbyte(lij) > ignoredoffset and "[]" or lij end t[i] = concat(tt) end return concat(t," + ") else - local t, li = { }, split.s + local t, li = { }, split.ch for j=1,#li do local lij = li[j] - t[j] = utfbyte(lij) > ignored_offset and "[]" or lij + t[j] = utfbyte(lij) > ignoredoffset and "[]" or lij end return concat(t) end end +local function packuc(entry) + local split = entry.split + if #split > 0 then -- useless test + local t = { } + for i=1,#split do + t[i] = concat(split[i].uc, " ") + end + return concat(t," + ") + else + return concat(split.uc," ") + end +end + function sorters.sort(entries,cmp) + if trace_tests or trace_methods then + local nofentries = #entries + report_sorters("entries: %s, language: %s, method: %s, digits: %s",nofentries,language,method,tostring(digits)) + for i=1,nofentries do + report_sorters("entry %s",table.serialize(entries[i].split,i,true,true,true)) + end + end if trace_tests then sort(entries,function(a,b) local r = cmp(a,b) - logs.report("sorter","%s %s %s",pack(a),(not r and "?") or (r<0 and "<") or (r>0 and ">") or "=",pack(b)) + local e = (not r and "?") or (r<0 and "<") or (r>0 and ">") or "=" + report_sorters("%s %s %s | %s %s %s",packch(a),e,packch(b),packuc(a),e,packuc(b)) return r == -1 end) local s @@ -263,9 +740,9 @@ function sorters.sort(entries,cmp) first = " " else s = first - logs.report("sorter",">> %s 0x%05X (%s 0x%05X)",first,utfbyte(first),letter,utfbyte(letter)) + report_sorters(">> %s 0x%05X (%s 0x%05X)",first,utfbyte(first),letter,utfbyte(letter)) end - logs.report("sorter"," %s",pack(entry)) + report_sorters(" %s | %s",packch(entry),packuc(entry)) end else sort(entries,function(a,b) @@ -273,45 +750,3 @@ function sorters.sort(entries,cmp) end) end end - --- some day we can have a characters.upper and characters.lower - -function sorters.add_uppercase_replacements(what) - local rep, new = replacements[what], { } - for i=1,#rep do - local r = rep[i] - local u = chardata[utfbyte(r[1])].uccode - if u then - new[utfchar(u)] = r[2] - end - end - for k, v in next, new do - rep[k] = v - end -end - -function sorters.add_uppercase_entries(what) - local ent, new = entries[what], { } - for k, v in next, ent do - local u = chardata[utfbyte(k)].uccode - if u then - new[utfchar(u)] = v - end - end - for k, v in next, new do - ent[k] = v - end -end - -function sorters.add_uppercase_mappings(what,offset) - local map, new, offset = mappings[what], { }, offset or 0 - for k, v in next, map do - local u = chardata[utfbyte(k)].uccode - if u then - new[utfchar(u)] = v + offset - end - end - for k, v in next, new do - map[k] = v - end -end |