summaryrefslogtreecommitdiff
path: root/Master/texmf-dist/tex/context/base/sort-ini.lua
diff options
context:
space:
mode:
Diffstat (limited to 'Master/texmf-dist/tex/context/base/sort-ini.lua')
-rw-r--r--Master/texmf-dist/tex/context/base/sort-ini.lua320
1 files changed, 244 insertions, 76 deletions
diff --git a/Master/texmf-dist/tex/context/base/sort-ini.lua b/Master/texmf-dist/tex/context/base/sort-ini.lua
index 5f045938f1a..b745c9aa5db 100644
--- a/Master/texmf-dist/tex/context/base/sort-ini.lua
+++ b/Master/texmf-dist/tex/context/base/sort-ini.lua
@@ -1,21 +1,24 @@
if not modules then modules = { } end modules ['sort-ini'] = {
version = 1.001,
- comment = "companion to sort-ini.tex",
+ comment = "companion to sort-ini.mkiv",
author = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
copyright = "PRAGMA ADE / ConTeXt Development Team",
license = "see context related readme files"
}
--- todo:
---
--- out of range
--- uppercase
--- texutil compatible
--- always expand to utf
+-- It took a while to get there, but with Fleetwood Mac's "Don't Stop"
+-- playing in the background we sort of got it done.
+
+-- todo: cleanup splits (in other modules)
local utf = unicode.utf8
-local gsub, rep = string.gsub, string.rep
+local gsub, rep, sort, concat = string.gsub, string.rep, table.sort, table.concat
+local utfbyte, utfchar = utf.byte, utf.char
local utfcharacters, utfvalues, strcharacters = string.utfcharacters, string.utfvalues, string.characters
+local chardata = characters.data
+local next, type, tonumber = next, type, tonumber
+
+local trace_tests = false trackers.register("sorters.tests", function(v) trace_tests = v end)
sorters = { }
sorters.comparers = { }
@@ -23,30 +26,49 @@ sorters.splitters = { }
sorters.entries = { }
sorters.mappings = { }
sorters.replacements = { }
-sorters.language = 'en'
-function sorters.comparers.basic(sort_a,sort_b)
- -- sm assignment is slow, will become sorters.initialize
- local sm = sorters.mappings[sorters.language or sorters.defaultlanguage] or sorters.mappings.en
- if #sort_a > #sort_b then
+sorters.ignored_offset = 0x10000
+sorters.replacement_offset = 0x10000
+sorters.digits_offset = 0x20000
+sorters.digits_maximum = 0xFFFFF
+
+local ignored_offset = sorters.ignored_offset
+local digits_offset = sorters.digits_offset
+local digits_maximum = sorters.digits_maximum
+
+local mappings = sorters.mappings
+local entries = sorters.entries
+local replacements = sorters.replacements
+
+local language, defaultlanguage, dummy = 'en', 'en', { }
+
+local currentreplacements, currentmappings, currententries
+
+function sorters.setlanguage(lang)
+ language = lang or language or defaultlanguage
+ currentreplacements = replacements[language] or replacements[defaultlanguage] or dummy
+ currentmappings = mappings [language] or mappings [defaultlanguage] or dummy
+ currententries = entries [language] or entries [defaultlanguage] or dummy
+ return currentreplacements, currentmappings, currententries
+end
+
+sorters.setlanguage()
+
+-- maybe inline code if it's too slow
+
+local function basicsort(sort_a,sort_b)
+ if not sort_a or not sort_b then
+ return 0
+ elseif #sort_a > #sort_b then
if #sort_b == 0 then
return 1
else
for i=1,#sort_b do
local ai, bi = sort_a[i], sort_b[i]
- local am, bm = sm[ai], sm[bi]
- if am and bm then
- if am > bm then
- return 1
- elseif am < bm then
- return -1
- end
- else
- if ai > bi then
- return 1
- elseif ai < bi then
- return -1
- end
+ if ai > bi then
+ return 1
+ elseif ai < bi then
+ return -1
end
end
return 1
@@ -57,19 +79,10 @@ function sorters.comparers.basic(sort_a,sort_b)
else
for i=1,#sort_a do
local ai, bi = sort_a[i], sort_b[i]
- local am, bm = sm[ai], sm[bi]
- if am and bm then
- if am > bm then
- return 1
- elseif am < bm then
- return -1
- end
- else
- if ai > bi then
- return 1
- elseif ai < bi then
- return -1
- end
+ if ai > bi then
+ return 1
+ elseif ai < bi then
+ return -1
end
end
return -1
@@ -79,71 +92,226 @@ function sorters.comparers.basic(sort_a,sort_b)
else
for i=1,#sort_a do
local ai, bi = sort_a[i], sort_b[i]
- local am, bm = sm[ai], sm[bi]
- if am and bm then
- if am > bm then
- return 1
- elseif am < bm then
- return -1
- end
- else
- if ai > bi then
- return 1
- elseif ai < bi then
- return -1
- end
+ if ai > bi then
+ return 1
+ elseif ai < bi then
+ return -1
end
end
return 0
end
end
-local function padd(s) return rep(" ",10-#s) .. s end -- or format with padd
+function sorters.comparers.basic(a,b)
+ local ea, eb = a.split, b.split
+ local na, nb = #ea, #eb
+ if na == 0 and nb == 0 then
+ -- simple variant (single word)
+ local result = basicsort(ea.e,eb.e)
+ return result == 0 and result or basicsort(ea.m,eb.m)
+ else
+ -- complex variant, used in register (multiple words)
+ local result = 0
+ for i=1,nb < na and nb or na do
+ local eai, ebi = ea[i], eb[i]
+ result = basicsort(eai.e,ebi.e)
+ if result == 0 then
+ result = basicsort(eai.m,ebi.m) -- only needed it there are m's
+ end
+ if result ~= 0 then
+ break
+ end
+ end
+ if result ~= 0 then
+ return result
+ elseif na > nb then
+ return 1
+ elseif nb > na then
+ return -1
+ else
+ return 0
+ end
+ end
+end
+
+local function numify(s)
+ return rep(" ",10-#s) .. s -- or format with padd
+end
+
+local function numify(s)
+ s = digits_offset + tonumber(s)
+ if s > digits_maximum then
+ s = digits_maximum
+ end
+ return utfchar(s)
+end
function sorters.strip(str) -- todo: only letters and such utf.gsub("([^%w%d])","")
if str then
str = gsub(str,"\\%S*","")
str = gsub(str,"[%s%[%](){}%$\"\']*","")
- str = gsub(str,"(%d+)",padd) -- sort numbers properly
+ str = gsub(str,"(%d+)",numify) -- sort numbers properly
return str
else
return ""
end
end
-function sorters.firstofsplit(split)
+local function firstofsplit(entry)
-- numbers are left padded by spaces
- local se = sorters.entries[sorters.language or sorters.defaultlanguage] -- slow, will become sorters.initialize
- local vs = split[1]
- local entry = (vs and vs[1]) or ""
- return entry, (se and se[entry]) or "\000"
+ local split = entry.split
+ if #split > 0 then
+ split = split[1].s
+ else
+ split = split.s
+ end
+ local entry = split and split[1] or ""
+ return entry, currententries[entry] or "\000"
end
-sorters.defaultlanguage = 'en'
+sorters.firstofsplit = firstofsplit
-- beware, numbers get spaces in front
-function sorters.splitters.utf(str) -- brrr, todo: language
- local r = sorters.replacements[sorters.language] or sorters.replacements[sorters.defaultlanguage] or { }
---~ local m = sorters.mappings [sorters.language] or sorters.mappings [sorters.defaultlanguage] or { }
- local u = characters.uncompose
- local b = utf.byte
- local t = { }
- for _,v in next, r do
- str = gsub(str,v[1],v[2])
+function sorters.splitters.utf(str)
+ if #currentreplacements > 0 then
+ for k=1,#currentreplacements do
+ local v = currentreplacements[k]
+ str = gsub(str,v[1],v[2])
+ end
end
- for c in utfcharacters(str) do -- maybe an lpeg
- if #c == 1 then
- t[#t+1] = c
- else
- for cc in strcharacters(c) do
- t[#t+1] = cc
+ local s, e, m, n = { }, { }, { }, 0
+ for sc in utfcharacters(str) do -- maybe an lpeg
+ local ec, mc = currententries[sc], currentmappings[sc] or utfbyte(sc)
+ n = n + 1
+ s[n] = sc
+ e[n] = currentmappings[ec] or mc
+ m[n] = mc
+ end
+ return { s = s, e = e, m = m }
+end
+
+-- we can use one array instead (sort of like in mkii)
+-- but for the moment we do it this way as it is more
+-- handy for tracing
+
+-- function sorters.splitters.utf(str)
+-- if #currentreplacements > 0 then
+-- for k=1,#currentreplacements do
+-- local v = currentreplacements[k]
+-- str = gsub(str,v[1],v[2])
+-- end
+-- end
+-- local s, e, m, n = { }, { }, { }, 0
+-- for sc in utfcharacters(str) do -- maybe an lpeg
+-- local ec, mc = currententries[sc], currentmappings[sc] or utfbyte(sc)
+-- n = n + 1
+-- ec = currentmappings[ec] or mc
+-- s[n] = sc
+-- e[n] = ec
+-- if ec ~= mc then
+-- n = n + 1
+-- e[n] = mc
+-- end
+-- end
+-- return { s = s, e = e }
+-- end
+
+function table.remap(t)
+ local tt = { }
+ for k,v in next, t do
+ tt[v] = k
+ end
+ return tt
+end
+
+local function pack(entry)
+ local t = { }
+ local split = entry.split
+ if #split > 0 then
+ for i=1,#split do
+ local tt, li = { }, split[i].s
+ for j=1,#li do
+ local lij = li[j]
+ tt[j] = utfbyte(lij) > ignored_offset and "[]" or lij
end
+ t[i] = concat(tt)
end
+ return concat(t," + ")
+ else
+ local t, li = { }, split.s
+ for j=1,#li do
+ local lij = li[j]
+ t[j] = utfbyte(lij) > ignored_offset and "[]" or lij
+ end
+ return concat(t)
end
- return t
end
function sorters.sort(entries,cmp)
- table.sort(entries,function(a,b) return cmp(a,b) == -1 end)
+ if trace_tests then
+ sort(entries,function(a,b)
+ local r = cmp(a,b)
+ logs.report("sorter","%s %s %s",pack(a),(not r and "?") or (r<0 and "<") or (r>0 and ">") or "=",pack(b))
+ return r == -1
+ end)
+ local s
+ for i=1,#entries do
+ local entry = entries[i]
+ local letter, first = firstofsplit(entry)
+ if first == s then
+ first = " "
+ else
+ s = first
+ logs.report("sorter",">> %s 0x%05X (%s 0x%05X)",first,utfbyte(first),letter,utfbyte(letter))
+ end
+ logs.report("sorter"," %s",pack(entry))
+ end
+ else
+ sort(entries,function(a,b)
+ return cmp(a,b) == -1
+ end)
+ end
+end
+
+-- some day we can have a characters.upper and characters.lower
+
+function sorters.add_uppercase_replacements(what)
+ local rep, new = replacements[what], { }
+ for i=1,#rep do
+ local r = rep[i]
+ local u = chardata[utfbyte(r[1])].uccode
+ if u then
+ new[utfchar(u)] = r[2]
+ end
+ end
+ for k, v in next, new do
+ rep[k] = v
+ end
+end
+
+function sorters.add_uppercase_entries(what)
+ local ent, new = entries[what], { }
+ for k, v in next, ent do
+ local u = chardata[utfbyte(k)].uccode
+ if u then
+ new[utfchar(u)] = v
+ end
+ end
+ for k, v in next, new do
+ ent[k] = v
+ end
+end
+
+function sorters.add_uppercase_mappings(what,offset)
+ local map, new, offset = mappings[what], { }, offset or 0
+ for k, v in next, map do
+ local u = chardata[utfbyte(k)].uccode
+ if u then
+ new[utfchar(u)] = v + offset
+ end
+ end
+ for k, v in next, new do
+ map[k] = v
+ end
end