xindex (28oct19)

git-svn-id: svn://tug.org/texlive/trunk@52551 c570f23f-e606-0410-a88d-b1316a301751

10 files changed, 140 insertions, 79 deletions

diff --git a/Master/texmf-dist/tex/lualatex/xindex/xindex-DIN2.lua b/Master/texmf-dist/tex/lualatex/xindex/xindex-DIN2.lua
index 4a055efd9a2..48231b1d834 100644
--- a/Master/texmf-dist/tex/lualatex/xindex/xindex-DIN2.lua
+++ b/Master/texmf-dist/tex/lualatex/xindex/xindex-DIN2.lua
@@ -7,7 +7,7 @@
 -----------------------------------------------------------------------
 
 if not modules then modules = { } end modules ['xindex-cfg'] = {
-      version = 0.16,
+      version = 0.18,
       comment = "configuration to xindex.lua",
        author = "Herbert Voss",
     copyright = "Herbert Voss",
@@ -39,6 +39,8 @@ idxnewletter      = "\\textbf"  -- Only valid if -n is not set
     Each character's position in this array-like table determines its 'priority'.
     Several characters in the same slot have the same 'priority'.
 ]]
+
+
 alphabet_lower = { --   for sorting
     { ' ' },  -- only for internal tests
     { 'a', 'á', 'à', 'å', 'æ', },
@@ -46,24 +48,24 @@ alphabet_lower = { --   for sorting
     { 'b' },
     { 'c', 'ç' },
     { 'd' },
-    { 'e', 'é', 'è', 'ë' },
+    { 'e', 'é', 'è', 'ë', 'ê' },
     { 'f' },
     { 'g' },
     { 'h' },
-    { 'i', 'í', 'ì', 'ï' },
+    { 'i', 'í', 'ì', 'î', 'ï' },
     { 'j' },
     { 'k' },
     { 'l' },
     { 'm' },
     { 'n', 'ñ' },
-    { 'o', 'ó', 'ò', 'ø', 'œ'},
+    { 'o', 'ó', 'ò', 'ô', 'ø', 'œ', 'ø'},
     { 'oe', 'ö' },
     { 'p' },
     { 'q' },
     { 'r' },
     { 's', 'š', 'ß' },
     { 't' },
-    { 'u', 'ú', 'ù' },
+    { 'u', 'ú', 'ù', 'û'},
     { 'ue', 'ü' },
     { 'v' },
     { 'w' },
@@ -78,7 +80,7 @@ alphabet_upper = { -- for sorting
     { 'B' },
     { 'C', 'Ç' },
     { 'D' },
-    { 'E', 'È', 'È', 'Ë' },
+    { 'E', 'È', 'É', 'Ë' },
     { 'F' },
     { 'G' },
     { 'H' },
@@ -88,7 +90,7 @@ alphabet_upper = { -- for sorting
     { 'L' },
     { 'M' },
     { 'N', 'Ñ' },
-    { 'O', 'Ó', 'Ò', 'Ø','Œ' },
+    { 'O', 'Ó', 'Ò', 'Ø','Œ', 'Ø' },
     { 'OE', 'Ö' },
     { 'P' },
     { 'Q' },
diff --git a/Master/texmf-dist/tex/lualatex/xindex/xindex-HAdW-eKO.lua b/Master/texmf-dist/tex/lualatex/xindex/xindex-HAdW-eKO.lua
index 2b5fd10dfa8..7b2dba332cf 100644
--- a/Master/texmf-dist/tex/lualatex/xindex/xindex-HAdW-eKO.lua
+++ b/Master/texmf-dist/tex/lualatex/xindex/xindex-HAdW-eKO.lua
@@ -9,7 +9,7 @@
 -- configuration for index files of the Heidelberger Akademie der Wissenschaften
 
 if not modules then modules = { } end modules ['xindex-HAdW-eKO'] = {
-      version = 0.16,
+      version = 0.18,
       comment = "configuration to xindex.lua",
        author = "Herbert Voss",
     copyright = "Herbert Voss",
@@ -31,6 +31,13 @@ idxnewletter      = "\\textbf"  -- Only valid if -n is not set
 
 case_sensitive    = false  -- speeds up running
 
+folium = { 
+  de = {"f.", "ff."},
+  en = {"f.", "ff."},
+  fr = {"\\,sq","\\,sqq"},
+}
+
+
 --[[
     Each character's position in this array-like table determines its 'priority'.
     Several characters in the same slot have the same 'priority'.
@@ -55,7 +62,7 @@ alphabet_lower = { --   for sorting
     { 'p' },
     { 'q' },
     { 'r' },
-    { 's' },
+    { 's', 'ß' },
     { 't' },
     { 'u', 'ú', 'ù', 'ü' },
     { 'v' },
@@ -94,6 +101,8 @@ alphabet_upper = { -- for sorting
     { 'Z' }
 }
 
+--alphabet_lower_map = CreateCharListMap(alphabet_lower)-- jetzt in xindex.lua
+--alphabet_upper_map = CreateCharListMap(alphabet_upper)
 
 --\indexentry{Ackers, Carolus}{VII/1-715}
 --\indexentry{Bremen!Adalbert I. von, Erzbischof}{VII/2/1-948}
@@ -215,7 +224,7 @@ end
 function replaceRomanB(r)
   local i = romanToNumber(r)
   if i then return "!//"..string.format("%05d",i).."//."
-       else return r
+       else return ("!"..r..".")   -- to catch something Neuneburg!a.d. Weichsel
   end
 end
 
@@ -315,4 +324,3 @@ function specialGetPageList(v,hyperpage)
   end
 end
 
-
diff --git a/Master/texmf-dist/tex/lualatex/xindex/xindex-base.lua b/Master/texmf-dist/tex/lualatex/xindex/xindex-base.lua
index d736f67b7dd..f4155054605 100644
--- a/Master/texmf-dist/tex/lualatex/xindex/xindex-base.lua
+++ b/Master/texmf-dist/tex/lualatex/xindex/xindex-base.lua
@@ -6,7 +6,7 @@
 --      LICENSE:  LPPL1.3
 -----------------------------------------------------------------------
 
-local info = { version = 0.16 } 
+local info = { version = 0.18 } 
 
 -- check config
 if pageNoPrefixDel ~= "" then 
@@ -58,10 +58,25 @@ printList(Index,2)
 writeLog(1,"\n------------------ end list after getEntryAndPage() ------------------------\n",2)
 writeLog(2,"Sorting entries: for LARGE idx files it can take some minutes ... \n",0)
 
-if SORTprehook then Index = SORTprehook(Index) end
+if SORTprehook then
+  writeLog(2, "PreHook begin\n", 0)
+  Index = SORTprehook(Index)
+  writeLog(2, "PreHook end\n", 0)
+end
+
+writeLog(2, "GenerateSortKeys begin\n", 0)
+GenerateSortKeys(Index)
+writeLog(2, "GenerateSortKeys end\n", 0)
+
 k = 0
+writeLog(2, "Sort begin\n", 0)
 table.sort(Index,UTFCompare)
-if SORTposthook then Index = SORTposthook(Index) end
+writeLog(2, "Sort end\n", 0)
+if SORTposthook then
+  writeLog(2, "PostHook begin\n", 0)
+  Index = SORTposthook(Index)
+  writeLog(2, "PostHook end\n", 0)
+end
 
 writeLog(1,"\n------------------ Start list after table.sort(Index,UTFCompare)------------\n",2)
 printList(Index,2)
@@ -85,7 +100,6 @@ writeLog(1,"------------------ End list after checkParenthesis(Index) ----------
 writeLog(2,"done\n",0)
 
 
-
 writeLog(2,"Replace @-operator ...\n",0)
 writeLog(1,"------------------ Start list before compressEntryList(Index)------------\n",1)
 writeLog(1,require 'xindex-pretty'.dump(Index),3)   -- only for internal dump
diff --git a/Master/texmf-dist/tex/lualatex/xindex/xindex-cfg-common.lua b/Master/texmf-dist/tex/lualatex/xindex/xindex-cfg-common.lua
index 7b84b348a37..f44967f63c1 100644
--- a/Master/texmf-dist/tex/lualatex/xindex/xindex-cfg-common.lua
+++ b/Master/texmf-dist/tex/lualatex/xindex/xindex-cfg-common.lua
@@ -7,7 +7,7 @@
 -----------------------------------------------------------------------
 
 if not modules then modules = { } end modules ['xindex-cfg-common'] = {
-      version = 0.16,
+      version = 0.18,
       comment = "configuration to xindex.lua",
        author = "Herbert Voss",
     copyright = "Herbert Voss",
@@ -115,3 +115,14 @@ alphabet_uppercase = {
     { 'ю', 'Ю' },
     { 'я', 'Я' }
 }
+
+local function create_map(char_list)
+  local map = {}
+  for i, pair in ipairs(char_list) do
+    map[pair[1]] = pair[2]
+  end
+  return map
+end
+
+alphabet_uppercase_map = create_map(alphabet_uppercase)
+
diff --git a/Master/texmf-dist/tex/lualatex/xindex/xindex-cfg.lua b/Master/texmf-dist/tex/lualatex/xindex/xindex-cfg.lua
index 7c055f1c3b9..89de044ca1e 100644
--- a/Master/texmf-dist/tex/lualatex/xindex/xindex-cfg.lua
+++ b/Master/texmf-dist/tex/lualatex/xindex/xindex-cfg.lua
@@ -7,7 +7,7 @@
 -----------------------------------------------------------------------
 
 if not modules then modules = { } end modules ['xindex-cfg'] = {
-      version = 0.16,
+      version = 0.18,
       comment = "main configuration to xindex.lua",
        author = "Herbert Voss",
     copyright = "Herbert Voss",
@@ -31,21 +31,21 @@ idxnewletter      = "\\textbf"  -- Only valid if -n is not set
 ]]
 alphabet_lower = { --   for sorting
     { ' ' },  -- only for internal tests
-    { 'a', 'á', 'à', 'ä', 'å', 'æ', },
+    { 'a', 'á', 'à', 'ä', 'â', 'å', 'æ', },
     { 'b' },
     { 'c', 'ç' },
     { 'd' },
-    { 'e', 'é', 'è', 'ë' },
+    { 'e', 'é', 'è', 'ë', 'ê' },
     { 'f' },
     { 'g' },
     { 'h' },
-    { 'i', 'í', 'ì', 'ï' },
+    { 'i', 'í', 'ì', 'î', 'ï' },
     { 'j' },
     { 'k' },
     { 'l' },
     { 'm' },
     { 'n', 'ñ' },
-    { 'o', 'ó', 'ò', 'ö', 'ø', 'œ', 'ø'},
+    { 'o', 'ó', 'ò', 'ö', 'ô', 'ø', 'œ', 'ø'},
     { 'p' },
     { 'q' },
     { 'r' },
@@ -64,7 +64,7 @@ alphabet_upper = { -- for sorting
     { 'B' },
     { 'C', 'Ç' },
     { 'D' },
-    { 'E', 'È', 'È', 'Ë' },
+    { 'E', 'È', 'É', 'Ë' },
     { 'F' },
     { 'G' },
     { 'H' },
diff --git a/Master/texmf-dist/tex/lualatex/xindex/xindex-dtk.lua b/Master/texmf-dist/tex/lualatex/xindex/xindex-dtk.lua
index 746b4287d30..6c918cc28a4 100644
--- a/Master/texmf-dist/tex/lualatex/xindex/xindex-dtk.lua
+++ b/Master/texmf-dist/tex/lualatex/xindex/xindex-dtk.lua
@@ -7,7 +7,7 @@
 -----------------------------------------------------------------------
 
 if not modules then modules = { } end modules ['xindex-cfg'] = {
-      version = 0.16,
+      version = 0.18,
       comment = "DTK configuration to xindex.lua",
        author = "Herbert Voss",
     copyright = "Herbert Voss",
diff --git a/Master/texmf-dist/tex/lualatex/xindex/xindex-lapp.lua b/Master/texmf-dist/tex/lualatex/xindex/xindex-lapp.lua
index ff4cb80a672..02dea0358bc 100644
--- a/Master/texmf-dist/tex/lualatex/xindex/xindex-lapp.lua
+++ b/Master/texmf-dist/tex/lualatex/xindex/xindex-lapp.lua
@@ -28,7 +28,7 @@
 --~ n       2
 --------------------------------
 if not modules then modules = { } end modules ['xindex-lapp'] = {
-      version = 0.16,
+      version = 0.18,
       comment = "read parameter for xindex.lua",
        author = "Steve Donovan",
     copyright = "Steve Donovan",
diff --git a/Master/texmf-dist/tex/lualatex/xindex/xindex-lib.lua b/Master/texmf-dist/tex/lualatex/xindex/xindex-lib.lua
index 8720d3db1b2..59193f1d28a 100644
--- a/Master/texmf-dist/tex/lualatex/xindex/xindex-lib.lua
+++ b/Master/texmf-dist/tex/lualatex/xindex/xindex-lib.lua
@@ -7,7 +7,7 @@
 -----------------------------------------------------------------------
 
 if not modules then modules = { } end modules ['xindex-lib'] = {
-      version = 0.16,
+      version = 0.18,
       comment = "main library  to xindex.lua",
        author = "Herbert Voss",
     copyright = "Herbert Voss",
@@ -19,16 +19,24 @@ local escapechar2 = "\"\""
 local Letters = "ÄÖÜäöüabcdefghijklmnopqrstuvwxyzßABCDEFGHIJKLMNOPQRSTUVWXYZ"
 local Digits = "0123456789"
 
--- Looks up the character `character´ in the alphabet and returns its 'normalization' for sorting
-local function get_normalized_char( CharList, character )
+function CreateCharListMap(CharList)
+  local map = {}
   for i, alphabet_entry in ipairs(CharList) do
-    for _, alphabet_char in ipairs(alphabet_entry) do
-      if character == alphabet_char then
-        return alphabet_entry[1]
+    local first_char
+    for j, alphabet_char in ipairs(alphabet_entry) do
+      if j == 1 then
+        first_char = alphabet_char
+      else
+        map[alphabet_char] = first_char
       end
     end
   end
-  return character
+  return map
+end
+
+-- Looks up the character `character´ in the alphabet and returns its 'normalization' for sorting
+local function get_normalized_char( CharListMap, character )
+  return CharListMap[character] or character
 end
 
 function Lower(strOrig)   -- return UTF string.lower
@@ -47,57 +55,60 @@ end
 function NormalizedLower(strOrig)  -- return normalized UTF string.lower (ä -> a)
   local str = ""
   for i=1, utf.len(strOrig) do
-    local c = get_normalized_char(alphabet_lower,utf.sub(strOrig,i,i))
+    local c = get_normalized_char(alphabet_lower_map,utf.sub(strOrig,i,i))
     str = str..string.lower(c)
   end
   return str
 end
 
 function NormalizedUpper(strOrig)  -- return normalized UTF string.upper (ä -> A)
-  local str = ""
+  local str = {}
   local c
   for i=1, utf.len(strOrig) do
-    c = get_normalized_char(alphabet_lower,utf.sub(strOrig,i,i))
---    str = str..string.upper(c)
-    str = str..c 
+    c = get_normalized_char(alphabet_lower_map,utf.sub(strOrig,i,i))
+    str[#str + 1] = c
   end
-  strOrig = str
-  str=""
+  strOrig = table.concat(str)
+  str = {}
   for i=1, utf.len(strOrig) do
-    c = get_normalized_char(alphabet_upper,utf.sub(strOrig,i,i))
+    c = get_normalized_char(alphabet_upper_map,utf.sub(strOrig,i,i))
     c = NormalizedUppercase(c)
-    str = str..c
+    str[#str + 1] = c
   end
-  return str
+  return table.concat(str)
 end
 
 function NormalizedUppercase(chr)  -- return normalized UTF uppercase letter
-  for i, alphabet_entry in ipairs(alphabet_uppercase) do
-    if chr == alphabet_entry[1] then
-      return alphabet_entry[2]
-    end
-  end
-  return chr
+  return alphabet_uppercase_map[chr] or chr
 end
 
 function NormalizedChars(strOrig)  -- return normalized UTF string (ä -> a) with first as upper case
-  local c = string.upper(get_normalized_char(alphabet_lower,utf.sub(strOrig,1,1)))
-  local str = c
-  for i=2, utf.len(strOrig) do
-    c = get_normalized_char(alphabet_lower,utf.sub(strOrig,i,i))
-    str = str..c
-  end
-  local strOrigTrue = strOrig
-  strOrig = str
-  c = string.upper(get_normalized_char(alphabet_upper,utf.sub(strOrig,1,1)))
-  str = c
-  for i=2, utf.len(strOrig) do
-    c = get_normalized_char(alphabet_upper,utf.sub(strOrig,i,i))
-    str = str..c
-  end
+  local c = get_normalized_char(alphabet_lower_map,utf.sub(strOrig,1,1))
+  c = get_normalized_char(alphabet_upper_map, c)
+  c = string.upper(c)
+  local str = { c }
+  for ch in string.utfcharacters(utf.sub(strOrig, 2)) do
+    c = get_normalized_char(alphabet_lower_map, ch)
+    c = get_normalized_char(alphabet_upper_map, c)
+    str[#str + 1] = c
+  end
+  return table.concat(str)
+end
+  
+--  local strOrigTrue = strOrig
+--  strOrig = table.concat(str)
+--  c = string.upper(get_normalized_char(alphabet_upper,utf.sub(strOrig,1,1)))
+--  str = { c }
+--  for i=2, utf.len(strOrig) do
+--    c = get_normalized_char(alphabet_upper,utf.sub(strOrig,i,i))
+--    str[#str + 1] = c
+--  end
+--  for ch in string.utfcharacters(strOrig, 2) do
+--    str[#str + 1] = get_normalized_char(alphabet_upper, ch)
+--  end
 --  print(strOrigTrue.." -> "..str)
-  return str
-end
+--  return table.concat(str)
+--end
 
 function checkVert(str) -- get the | part
   local vert = string.find(str,"|",1,true)
@@ -524,34 +535,42 @@ end
 -- str = "\indexentry {foo}{bar}"
 -- first, second = str:match('{([^}]*)}%s*{([^}]*)}')  -> foo, bar
 
-function UTFCompare(a,b)  
--- a, b are something like \indexentry{foo}{bar}
---  writeLog(1,"UTFCompare:  "..a["Entry"]..", "..a["pages"][1]["number"].." - "..b["Entry"]..", "..b["pages"][1]["number"].."\n",2)
---  k = k + 1
---  if (k % 50) == 0 then writeLog(1,".",1) end
-  local A,B,Apage,Bpage
+function GenerateSortKey(a)
+  local A, Apage
   if numericPage then
     if tonumber(a["pages"][1]["number"]) then
       Apage = string.format("%09d",a["pages"][1]["number"])
     else
       Apage = string.format("%09d",romanToNumber(a["pages"][1]["number"]))
     end
-    if tonumber(b["pages"][1]["number"]) then
-      Bpage = string.format("%09d",b["pages"][1]["number"])
-    else
-      Bpage = string.format("%09d",romanToNumber(b["pages"][1]["number"]))
-    end
   else
     Apage = string.format("%09s",a["pages"][1]["number"])
-    Bpage = string.format("%09s",b["pages"][1]["number"])
   end
   if no_caseSensitive then
     A = NormalizedUpper(getSortString(a["Entry"].." "..Apage..a["pages"][1]["special"]):gsub('!','')) -- replace! by empty
-    B = NormalizedUpper(getSortString(b["Entry"].." "..Bpage..b["pages"][1]["special"]):gsub('!',''))
+    a["SortKey"] = A
   else  
     A = NormalizedChars(getSortString(a["Entry"].." "..Apage..a["pages"][1]["special"]):gsub('!','')) -- replace! by empty
-    B = NormalizedChars(getSortString(b["Entry"].." "..Bpage..b["pages"][1]["special"]):gsub('!',''))
+    a["SortKey"] = A
   end
+  -- return A
+end
+
+function GenerateSortKeys(Index)
+  for _, entry in ipairs(Index) do
+    GenerateSortKey(entry)
+  end
+end
+
+function UTFCompare(a,b)  
+-- a, b are something like \indexentry{foo}{bar}
+--  writeLog(1,"UTFCompare:  "..a["Entry"]..", "..a["pages"][1]["number"].." - "..b["Entry"]..", "..b["pages"][1]["number"].."\n",2)
+--  k = k + 1
+--  if (k % 50) == 0 then writeLog(1,".",1) end
+
+  local A = a["SortKey"]
+  local B = b["SortKey"]
+  
   writeLog(1,"UTFCompare: A--B "..A.."--"..B.."\n",2)
 -- print(A,B)
 --[[ 
@@ -907,7 +926,13 @@ local match = function(expr)
   return Ct(idx):match(expr)
 end
 
+local SortStringCache = {}
 function getSortString(Entry)
+  local sortString = SortStringCache[Entry]
+  if sortString ~= nil then
+    return sortString
+  end
+
   local t = match(Entry)
 --  require 'xindex-pretty'.dump(t)   -- only for internal dump
   local data = t[1]
@@ -920,6 +945,7 @@ function getSortString(Entry)
       data = data..t[i]
     end
   end
+  SortStringCache[Entry] = data
   return data
 end
 
diff --git a/Master/texmf-dist/tex/lualatex/xindex/xindex-pretty.lua b/Master/texmf-dist/tex/lualatex/xindex/xindex-pretty.lua
index 67e5508c5a9..1165b6a6248 100644
--- a/Master/texmf-dist/tex/lualatex/xindex/xindex-pretty.lua
+++ b/Master/texmf-dist/tex/lualatex/xindex/xindex-pretty.lua
@@ -5,7 +5,7 @@
 --
 
 if not modules then modules = { } end modules ['xindex-pretty'] = {
-      version = 0.16,
+      version = 0.18,
       comment = "dump a Lua table for debugging",
        author = "Steve Donovan",
     copyright = "Steve Donovan",
diff --git a/Master/texmf-dist/tex/lualatex/xindex/xindex-yannis.lua b/Master/texmf-dist/tex/lualatex/xindex/xindex-yannis.lua
index 8d54d82e3eb..99b8642293e 100644
--- a/Master/texmf-dist/tex/lualatex/xindex/xindex-yannis.lua
+++ b/Master/texmf-dist/tex/lualatex/xindex/xindex-yannis.lua
@@ -7,7 +7,7 @@
 -----------------------------------------------------------------------
 
 if not modules then modules = { } end modules ['xindex-yannis'] = {
-      version = 0.16,
+      version = 0.18,
       comment = "main configuration to xindex.lua",
        author = "Herbert Voss",
     copyright = "Herbert Voss",