From 97f5151099967e6c823d640a90d87bca92c44035 Mon Sep 17 00:00:00 2001 From: Norbert Preining Date: Tue, 29 Oct 2019 03:00:39 +0000 Subject: CTAN sync 201910290300 --- indexing/xindex/lualatex/xindex-DIN2.lua | 16 +-- indexing/xindex/lualatex/xindex-HAdW-eKO.lua | 16 ++- indexing/xindex/lualatex/xindex-base.lua | 22 ++++- indexing/xindex/lualatex/xindex-cfg-common.lua | 13 ++- indexing/xindex/lualatex/xindex-cfg.lua | 12 +-- indexing/xindex/lualatex/xindex-dtk.lua | 2 +- indexing/xindex/lualatex/xindex-lapp.lua | 2 +- indexing/xindex/lualatex/xindex-lib.lua | 132 +++++++++++++++---------- indexing/xindex/lualatex/xindex-pretty.lua | 2 +- indexing/xindex/lualatex/xindex-yannis.lua | 2 +- 10 files changed, 140 insertions(+), 79 deletions(-) (limited to 'indexing/xindex/lualatex') diff --git a/indexing/xindex/lualatex/xindex-DIN2.lua b/indexing/xindex/lualatex/xindex-DIN2.lua index 4a055efd9a..48231b1d83 100644 --- a/indexing/xindex/lualatex/xindex-DIN2.lua +++ b/indexing/xindex/lualatex/xindex-DIN2.lua @@ -7,7 +7,7 @@ ----------------------------------------------------------------------- if not modules then modules = { } end modules ['xindex-cfg'] = { - version = 0.16, + version = 0.18, comment = "configuration to xindex.lua", author = "Herbert Voss", copyright = "Herbert Voss", @@ -39,6 +39,8 @@ idxnewletter = "\\textbf" -- Only valid if -n is not set Each character's position in this array-like table determines its 'priority'. Several characters in the same slot have the same 'priority'. ]] + + alphabet_lower = { -- for sorting { ' ' }, -- only for internal tests { 'a', 'á', 'à', 'å', 'æ', }, @@ -46,24 +48,24 @@ alphabet_lower = { -- for sorting { 'b' }, { 'c', 'ç' }, { 'd' }, - { 'e', 'é', 'è', 'ë' }, + { 'e', 'é', 'è', 'ë', 'ê' }, { 'f' }, { 'g' }, { 'h' }, - { 'i', 'í', 'ì', 'ï' }, + { 'i', 'í', 'ì', 'î', 'ï' }, { 'j' }, { 'k' }, { 'l' }, { 'm' }, { 'n', 'ñ' }, - { 'o', 'ó', 'ò', 'ø', 'œ'}, + { 'o', 'ó', 'ò', 'ô', 'ø', 'œ', 'ø'}, { 'oe', 'ö' }, { 'p' }, { 'q' }, { 'r' }, { 's', 'š', 'ß' }, { 't' }, - { 'u', 'ú', 'ù' }, + { 'u', 'ú', 'ù', 'û'}, { 'ue', 'ü' }, { 'v' }, { 'w' }, @@ -78,7 +80,7 @@ alphabet_upper = { -- for sorting { 'B' }, { 'C', 'Ç' }, { 'D' }, - { 'E', 'È', 'È', 'Ë' }, + { 'E', 'È', 'É', 'Ë' }, { 'F' }, { 'G' }, { 'H' }, @@ -88,7 +90,7 @@ alphabet_upper = { -- for sorting { 'L' }, { 'M' }, { 'N', 'Ñ' }, - { 'O', 'Ó', 'Ò', 'Ø','Œ' }, + { 'O', 'Ó', 'Ò', 'Ø','Œ', 'Ø' }, { 'OE', 'Ö' }, { 'P' }, { 'Q' }, diff --git a/indexing/xindex/lualatex/xindex-HAdW-eKO.lua b/indexing/xindex/lualatex/xindex-HAdW-eKO.lua index 2b5fd10dfa..7b2dba332c 100644 --- a/indexing/xindex/lualatex/xindex-HAdW-eKO.lua +++ b/indexing/xindex/lualatex/xindex-HAdW-eKO.lua @@ -9,7 +9,7 @@ -- configuration for index files of the Heidelberger Akademie der Wissenschaften if not modules then modules = { } end modules ['xindex-HAdW-eKO'] = { - version = 0.16, + version = 0.18, comment = "configuration to xindex.lua", author = "Herbert Voss", copyright = "Herbert Voss", @@ -31,6 +31,13 @@ idxnewletter = "\\textbf" -- Only valid if -n is not set case_sensitive = false -- speeds up running +folium = { + de = {"f.", "ff."}, + en = {"f.", "ff."}, + fr = {"\\,sq","\\,sqq"}, +} + + --[[ Each character's position in this array-like table determines its 'priority'. Several characters in the same slot have the same 'priority'. @@ -55,7 +62,7 @@ alphabet_lower = { -- for sorting { 'p' }, { 'q' }, { 'r' }, - { 's' }, + { 's', 'ß' }, { 't' }, { 'u', 'ú', 'ù', 'ü' }, { 'v' }, @@ -94,6 +101,8 @@ alphabet_upper = { -- for sorting { 'Z' } } +--alphabet_lower_map = CreateCharListMap(alphabet_lower)-- jetzt in xindex.lua +--alphabet_upper_map = CreateCharListMap(alphabet_upper) --\indexentry{Ackers, Carolus}{VII/1-715} --\indexentry{Bremen!Adalbert I. von, Erzbischof}{VII/2/1-948} @@ -215,7 +224,7 @@ end function replaceRomanB(r) local i = romanToNumber(r) if i then return "!//"..string.format("%05d",i).."//." - else return r + else return ("!"..r..".") -- to catch something Neuneburg!a.d. Weichsel end end @@ -315,4 +324,3 @@ function specialGetPageList(v,hyperpage) end end - diff --git a/indexing/xindex/lualatex/xindex-base.lua b/indexing/xindex/lualatex/xindex-base.lua index d736f67b7d..f415505460 100644 --- a/indexing/xindex/lualatex/xindex-base.lua +++ b/indexing/xindex/lualatex/xindex-base.lua @@ -6,7 +6,7 @@ -- LICENSE: LPPL1.3 ----------------------------------------------------------------------- -local info = { version = 0.16 } +local info = { version = 0.18 } -- check config if pageNoPrefixDel ~= "" then @@ -58,10 +58,25 @@ printList(Index,2) writeLog(1,"\n------------------ end list after getEntryAndPage() ------------------------\n",2) writeLog(2,"Sorting entries: for LARGE idx files it can take some minutes ... \n",0) -if SORTprehook then Index = SORTprehook(Index) end +if SORTprehook then + writeLog(2, "PreHook begin\n", 0) + Index = SORTprehook(Index) + writeLog(2, "PreHook end\n", 0) +end + +writeLog(2, "GenerateSortKeys begin\n", 0) +GenerateSortKeys(Index) +writeLog(2, "GenerateSortKeys end\n", 0) + k = 0 +writeLog(2, "Sort begin\n", 0) table.sort(Index,UTFCompare) -if SORTposthook then Index = SORTposthook(Index) end +writeLog(2, "Sort end\n", 0) +if SORTposthook then + writeLog(2, "PostHook begin\n", 0) + Index = SORTposthook(Index) + writeLog(2, "PostHook end\n", 0) +end writeLog(1,"\n------------------ Start list after table.sort(Index,UTFCompare)------------\n",2) printList(Index,2) @@ -85,7 +100,6 @@ writeLog(1,"------------------ End list after checkParenthesis(Index) ---------- writeLog(2,"done\n",0) - writeLog(2,"Replace @-operator ...\n",0) writeLog(1,"------------------ Start list before compressEntryList(Index)------------\n",1) writeLog(1,require 'xindex-pretty'.dump(Index),3) -- only for internal dump diff --git a/indexing/xindex/lualatex/xindex-cfg-common.lua b/indexing/xindex/lualatex/xindex-cfg-common.lua index 7b84b348a3..f44967f63c 100644 --- a/indexing/xindex/lualatex/xindex-cfg-common.lua +++ b/indexing/xindex/lualatex/xindex-cfg-common.lua @@ -7,7 +7,7 @@ ----------------------------------------------------------------------- if not modules then modules = { } end modules ['xindex-cfg-common'] = { - version = 0.16, + version = 0.18, comment = "configuration to xindex.lua", author = "Herbert Voss", copyright = "Herbert Voss", @@ -115,3 +115,14 @@ alphabet_uppercase = { { 'ю', 'Ю' }, { 'я', 'Я' } } + +local function create_map(char_list) + local map = {} + for i, pair in ipairs(char_list) do + map[pair[1]] = pair[2] + end + return map +end + +alphabet_uppercase_map = create_map(alphabet_uppercase) + diff --git a/indexing/xindex/lualatex/xindex-cfg.lua b/indexing/xindex/lualatex/xindex-cfg.lua index 7c055f1c3b..89de044ca1 100644 --- a/indexing/xindex/lualatex/xindex-cfg.lua +++ b/indexing/xindex/lualatex/xindex-cfg.lua @@ -7,7 +7,7 @@ ----------------------------------------------------------------------- if not modules then modules = { } end modules ['xindex-cfg'] = { - version = 0.16, + version = 0.18, comment = "main configuration to xindex.lua", author = "Herbert Voss", copyright = "Herbert Voss", @@ -31,21 +31,21 @@ idxnewletter = "\\textbf" -- Only valid if -n is not set ]] alphabet_lower = { -- for sorting { ' ' }, -- only for internal tests - { 'a', 'á', 'à', 'ä', 'å', 'æ', }, + { 'a', 'á', 'à', 'ä', 'â', 'å', 'æ', }, { 'b' }, { 'c', 'ç' }, { 'd' }, - { 'e', 'é', 'è', 'ë' }, + { 'e', 'é', 'è', 'ë', 'ê' }, { 'f' }, { 'g' }, { 'h' }, - { 'i', 'í', 'ì', 'ï' }, + { 'i', 'í', 'ì', 'î', 'ï' }, { 'j' }, { 'k' }, { 'l' }, { 'm' }, { 'n', 'ñ' }, - { 'o', 'ó', 'ò', 'ö', 'ø', 'œ', 'ø'}, + { 'o', 'ó', 'ò', 'ö', 'ô', 'ø', 'œ', 'ø'}, { 'p' }, { 'q' }, { 'r' }, @@ -64,7 +64,7 @@ alphabet_upper = { -- for sorting { 'B' }, { 'C', 'Ç' }, { 'D' }, - { 'E', 'È', 'È', 'Ë' }, + { 'E', 'È', 'É', 'Ë' }, { 'F' }, { 'G' }, { 'H' }, diff --git a/indexing/xindex/lualatex/xindex-dtk.lua b/indexing/xindex/lualatex/xindex-dtk.lua index 746b4287d3..6c918cc28a 100644 --- a/indexing/xindex/lualatex/xindex-dtk.lua +++ b/indexing/xindex/lualatex/xindex-dtk.lua @@ -7,7 +7,7 @@ ----------------------------------------------------------------------- if not modules then modules = { } end modules ['xindex-cfg'] = { - version = 0.16, + version = 0.18, comment = "DTK configuration to xindex.lua", author = "Herbert Voss", copyright = "Herbert Voss", diff --git a/indexing/xindex/lualatex/xindex-lapp.lua b/indexing/xindex/lualatex/xindex-lapp.lua index ff4cb80a67..02dea0358b 100644 --- a/indexing/xindex/lualatex/xindex-lapp.lua +++ b/indexing/xindex/lualatex/xindex-lapp.lua @@ -28,7 +28,7 @@ --~ n 2 -------------------------------- if not modules then modules = { } end modules ['xindex-lapp'] = { - version = 0.16, + version = 0.18, comment = "read parameter for xindex.lua", author = "Steve Donovan", copyright = "Steve Donovan", diff --git a/indexing/xindex/lualatex/xindex-lib.lua b/indexing/xindex/lualatex/xindex-lib.lua index 8720d3db1b..59193f1d28 100644 --- a/indexing/xindex/lualatex/xindex-lib.lua +++ b/indexing/xindex/lualatex/xindex-lib.lua @@ -7,7 +7,7 @@ ----------------------------------------------------------------------- if not modules then modules = { } end modules ['xindex-lib'] = { - version = 0.16, + version = 0.18, comment = "main library to xindex.lua", author = "Herbert Voss", copyright = "Herbert Voss", @@ -19,16 +19,24 @@ local escapechar2 = "\"\"" local Letters = "ÄÖÜäöüabcdefghijklmnopqrstuvwxyzßABCDEFGHIJKLMNOPQRSTUVWXYZ" local Digits = "0123456789" --- Looks up the character `character´ in the alphabet and returns its 'normalization' for sorting -local function get_normalized_char( CharList, character ) +function CreateCharListMap(CharList) + local map = {} for i, alphabet_entry in ipairs(CharList) do - for _, alphabet_char in ipairs(alphabet_entry) do - if character == alphabet_char then - return alphabet_entry[1] + local first_char + for j, alphabet_char in ipairs(alphabet_entry) do + if j == 1 then + first_char = alphabet_char + else + map[alphabet_char] = first_char end end end - return character + return map +end + +-- Looks up the character `character´ in the alphabet and returns its 'normalization' for sorting +local function get_normalized_char( CharListMap, character ) + return CharListMap[character] or character end function Lower(strOrig) -- return UTF string.lower @@ -47,57 +55,60 @@ end function NormalizedLower(strOrig) -- return normalized UTF string.lower (ä -> a) local str = "" for i=1, utf.len(strOrig) do - local c = get_normalized_char(alphabet_lower,utf.sub(strOrig,i,i)) + local c = get_normalized_char(alphabet_lower_map,utf.sub(strOrig,i,i)) str = str..string.lower(c) end return str end function NormalizedUpper(strOrig) -- return normalized UTF string.upper (ä -> A) - local str = "" + local str = {} local c for i=1, utf.len(strOrig) do - c = get_normalized_char(alphabet_lower,utf.sub(strOrig,i,i)) --- str = str..string.upper(c) - str = str..c + c = get_normalized_char(alphabet_lower_map,utf.sub(strOrig,i,i)) + str[#str + 1] = c end - strOrig = str - str="" + strOrig = table.concat(str) + str = {} for i=1, utf.len(strOrig) do - c = get_normalized_char(alphabet_upper,utf.sub(strOrig,i,i)) + c = get_normalized_char(alphabet_upper_map,utf.sub(strOrig,i,i)) c = NormalizedUppercase(c) - str = str..c + str[#str + 1] = c end - return str + return table.concat(str) end function NormalizedUppercase(chr) -- return normalized UTF uppercase letter - for i, alphabet_entry in ipairs(alphabet_uppercase) do - if chr == alphabet_entry[1] then - return alphabet_entry[2] - end - end - return chr + return alphabet_uppercase_map[chr] or chr end function NormalizedChars(strOrig) -- return normalized UTF string (ä -> a) with first as upper case - local c = string.upper(get_normalized_char(alphabet_lower,utf.sub(strOrig,1,1))) - local str = c - for i=2, utf.len(strOrig) do - c = get_normalized_char(alphabet_lower,utf.sub(strOrig,i,i)) - str = str..c - end - local strOrigTrue = strOrig - strOrig = str - c = string.upper(get_normalized_char(alphabet_upper,utf.sub(strOrig,1,1))) - str = c - for i=2, utf.len(strOrig) do - c = get_normalized_char(alphabet_upper,utf.sub(strOrig,i,i)) - str = str..c - end + local c = get_normalized_char(alphabet_lower_map,utf.sub(strOrig,1,1)) + c = get_normalized_char(alphabet_upper_map, c) + c = string.upper(c) + local str = { c } + for ch in string.utfcharacters(utf.sub(strOrig, 2)) do + c = get_normalized_char(alphabet_lower_map, ch) + c = get_normalized_char(alphabet_upper_map, c) + str[#str + 1] = c + end + return table.concat(str) +end + +-- local strOrigTrue = strOrig +-- strOrig = table.concat(str) +-- c = string.upper(get_normalized_char(alphabet_upper,utf.sub(strOrig,1,1))) +-- str = { c } +-- for i=2, utf.len(strOrig) do +-- c = get_normalized_char(alphabet_upper,utf.sub(strOrig,i,i)) +-- str[#str + 1] = c +-- end +-- for ch in string.utfcharacters(strOrig, 2) do +-- str[#str + 1] = get_normalized_char(alphabet_upper, ch) +-- end -- print(strOrigTrue.." -> "..str) - return str -end +-- return table.concat(str) +--end function checkVert(str) -- get the | part local vert = string.find(str,"|",1,true) @@ -524,34 +535,42 @@ end -- str = "\indexentry {foo}{bar}" -- first, second = str:match('{([^}]*)}%s*{([^}]*)}') -> foo, bar -function UTFCompare(a,b) --- a, b are something like \indexentry{foo}{bar} --- writeLog(1,"UTFCompare: "..a["Entry"]..", "..a["pages"][1]["number"].." - "..b["Entry"]..", "..b["pages"][1]["number"].."\n",2) --- k = k + 1 --- if (k % 50) == 0 then writeLog(1,".",1) end - local A,B,Apage,Bpage +function GenerateSortKey(a) + local A, Apage if numericPage then if tonumber(a["pages"][1]["number"]) then Apage = string.format("%09d",a["pages"][1]["number"]) else Apage = string.format("%09d",romanToNumber(a["pages"][1]["number"])) end - if tonumber(b["pages"][1]["number"]) then - Bpage = string.format("%09d",b["pages"][1]["number"]) - else - Bpage = string.format("%09d",romanToNumber(b["pages"][1]["number"])) - end else Apage = string.format("%09s",a["pages"][1]["number"]) - Bpage = string.format("%09s",b["pages"][1]["number"]) end if no_caseSensitive then A = NormalizedUpper(getSortString(a["Entry"].." "..Apage..a["pages"][1]["special"]):gsub('!','')) -- replace! by empty - B = NormalizedUpper(getSortString(b["Entry"].." "..Bpage..b["pages"][1]["special"]):gsub('!','')) + a["SortKey"] = A else A = NormalizedChars(getSortString(a["Entry"].." "..Apage..a["pages"][1]["special"]):gsub('!','')) -- replace! by empty - B = NormalizedChars(getSortString(b["Entry"].." "..Bpage..b["pages"][1]["special"]):gsub('!','')) + a["SortKey"] = A end + -- return A +end + +function GenerateSortKeys(Index) + for _, entry in ipairs(Index) do + GenerateSortKey(entry) + end +end + +function UTFCompare(a,b) +-- a, b are something like \indexentry{foo}{bar} +-- writeLog(1,"UTFCompare: "..a["Entry"]..", "..a["pages"][1]["number"].." - "..b["Entry"]..", "..b["pages"][1]["number"].."\n",2) +-- k = k + 1 +-- if (k % 50) == 0 then writeLog(1,".",1) end + + local A = a["SortKey"] + local B = b["SortKey"] + writeLog(1,"UTFCompare: A--B "..A.."--"..B.."\n",2) -- print(A,B) --[[ @@ -907,7 +926,13 @@ local match = function(expr) return Ct(idx):match(expr) end +local SortStringCache = {} function getSortString(Entry) + local sortString = SortStringCache[Entry] + if sortString ~= nil then + return sortString + end + local t = match(Entry) -- require 'xindex-pretty'.dump(t) -- only for internal dump local data = t[1] @@ -920,6 +945,7 @@ function getSortString(Entry) data = data..t[i] end end + SortStringCache[Entry] = data return data end diff --git a/indexing/xindex/lualatex/xindex-pretty.lua b/indexing/xindex/lualatex/xindex-pretty.lua index 67e5508c5a..1165b6a624 100644 --- a/indexing/xindex/lualatex/xindex-pretty.lua +++ b/indexing/xindex/lualatex/xindex-pretty.lua @@ -5,7 +5,7 @@ -- if not modules then modules = { } end modules ['xindex-pretty'] = { - version = 0.16, + version = 0.18, comment = "dump a Lua table for debugging", author = "Steve Donovan", copyright = "Steve Donovan", diff --git a/indexing/xindex/lualatex/xindex-yannis.lua b/indexing/xindex/lualatex/xindex-yannis.lua index 8d54d82e3e..99b8642293 100644 --- a/indexing/xindex/lualatex/xindex-yannis.lua +++ b/indexing/xindex/lualatex/xindex-yannis.lua @@ -7,7 +7,7 @@ ----------------------------------------------------------------------- if not modules then modules = { } end modules ['xindex-yannis'] = { - version = 0.16, + version = 0.18, comment = "main configuration to xindex.lua", author = "Herbert Voss", copyright = "Herbert Voss", -- cgit v1.2.3