upmendex: v0.59, support output order of symbols and numbers separately

git-svn-id: svn://tug.org/texlive/trunk@60418 c570f23f-e606-0410-a88d-b1316a301751

15 files changed, 160 insertions, 42 deletions

diff --git a/Build/source/texk/upmendex/ChangeLog b/Build/source/texk/upmendex/ChangeLog
index 9310b5e4cf5..6b5a8432ec1 100644
--- a/Build/source/texk/upmendex/ChangeLog
+++ b/Build/source/texk/upmendex/ChangeLog
@@ -1,3 +1,18 @@
+2021-09-04  TANAKA Takuji  <ttk@t-lab.opal.ne.jp>
+
+	* version 0.59  Beta version.
+	* configure.ac: Bump version.
+	* fwrite.c, styfile.c, {,ex}var.h:
+	Support output order of symbols and numbers separately
+	when "symbol_flag" is 2 in style file.
+	* sort.c:
+	Accept combination of "icu_locale" and "icu_rules" in style file.
+	If simultaneously specified, collation rules specified by "icu_rules" are
+	added on collation rules specified by "icu_locale".
+	* tests/upmendex.test, tests/rule.{idx,ind}, tests/locale2.ist,
+	tests/{locale1,attri1,locale2-rule}.ind:
+	Add a new test and update conventional tests.
+
 2021-08-29  TANAKA Takuji  <ttk@t-lab.opal.ne.jp>
 
 	* styfile.c:
diff --git a/Build/source/texk/upmendex/configure b/Build/source/texk/upmendex/configure
index f1ffea6fbb8..7f3a619e5a9 100755
--- a/Build/source/texk/upmendex/configure
+++ b/Build/source/texk/upmendex/configure
@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.71 for upmendex (TeX Live) 0.58.
+# Generated by GNU Autoconf 2.71 for upmendex (TeX Live) 0.59.
 #
 #
 # Copyright (C) 1992-1996, 1998-2017, 2020-2021 Free Software Foundation,
@@ -626,8 +626,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='upmendex (TeX Live)'
 PACKAGE_TARNAME='upmendex--tex-live-'
-PACKAGE_VERSION='0.58'
-PACKAGE_STRING='upmendex (TeX Live) 0.58'
+PACKAGE_VERSION='0.59'
+PACKAGE_STRING='upmendex (TeX Live) 0.59'
 PACKAGE_BUGREPORT=''
 PACKAGE_URL=''
 
@@ -1390,7 +1390,7 @@ if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-\`configure' configures upmendex (TeX Live) 0.58 to adapt to many kinds of systems.
+\`configure' configures upmendex (TeX Live) 0.59 to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
@@ -1462,7 +1462,7 @@ fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of upmendex (TeX Live) 0.58:";;
+     short | recursive ) echo "Configuration of upmendex (TeX Live) 0.59:";;
    esac
   cat <<\_ACEOF
 
@@ -1587,7 +1587,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
   cat <<\_ACEOF
-upmendex (TeX Live) configure 0.58
+upmendex (TeX Live) configure 0.59
 generated by GNU Autoconf 2.71
 
 Copyright (C) 2021 Free Software Foundation, Inc.
@@ -2268,7 +2268,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by upmendex (TeX Live) $as_me 0.58, which was
+It was created by upmendex (TeX Live) $as_me 0.59, which was
 generated by GNU Autoconf 2.71.  Invocation command line was
 
   $ $0$ac_configure_args_raw
@@ -8806,7 +8806,7 @@ fi
 
 # Define the identity of the package.
  PACKAGE='upmendex--tex-live-'
- VERSION='0.58'
+ VERSION='0.59'
 
 
 # Some tools Automake needs.
@@ -18942,7 +18942,7 @@ Usage: $0 [OPTIONS]
 Report bugs to <bug-libtool@gnu.org>."
 
 lt_cl_version="\
-upmendex (TeX Live) config.lt 0.58
+upmendex (TeX Live) config.lt 0.59
 configured by $0, generated by GNU Autoconf 2.71.
 
 Copyright (C) 2011 Free Software Foundation, Inc.
@@ -21114,7 +21114,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by upmendex (TeX Live) $as_me 0.58, which was
+This file was extended by upmendex (TeX Live) $as_me 0.59, which was
 generated by GNU Autoconf 2.71.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
@@ -21182,7 +21182,7 @@ ac_cs_config_escaped=`printf "%s\n" "$ac_cs_config" | sed "s/^ //; s/'/'\\\\\\\\
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config='$ac_cs_config_escaped'
 ac_cs_version="\\
-upmendex (TeX Live) config.status 0.58
+upmendex (TeX Live) config.status 0.59
 configured by $0, generated by GNU Autoconf 2.71,
   with options \\"\$ac_cs_config\\"
 
diff --git a/Build/source/texk/upmendex/configure.ac b/Build/source/texk/upmendex/configure.ac
index 7af5a44b8c9..3bc2cc06f30 100644
--- a/Build/source/texk/upmendex/configure.ac
+++ b/Build/source/texk/upmendex/configure.ac
@@ -8,8 +8,8 @@ dnl   This file is free software; the copyright holder
 dnl   gives unlimited permission to copy and/or distribute it,
 dnl   with or without modifications, as long as this notice is preserved.
 dnl
-AC_INIT([upmendex (TeX Live)], [0.58])
-AC_PREREQ([2.63])
+AC_INIT([upmendex (TeX Live)],[0.59])
+AC_PREREQ([2.71])
 AC_CONFIG_SRCDIR([main.c])
 AC_CONFIG_AUX_DIR([../../build-aux])
 AC_CONFIG_MACRO_DIR([../../m4])
diff --git a/Build/source/texk/upmendex/exvar.h b/Build/source/texk/upmendex/exvar.h
index 7b061c6655a..dd650721e30 100644
--- a/Build/source/texk/upmendex/exvar.h
+++ b/Build/source/texk/upmendex/exvar.h
@@ -26,7 +26,7 @@ extern int line_max;
 extern char indent_space[];
 extern int indent_length;
 extern int priority;
-extern char symbol[];
+extern char symhead[],numhead[];
 extern char symhead_positive[],symhead_negative[];
 extern char numhead_positive[],numhead_negative[];
 extern int symbol_flag;
diff --git a/Build/source/texk/upmendex/fwrite.c b/Build/source/texk/upmendex/fwrite.c
index a5efc14f5d4..911bbef4add 100644
--- a/Build/source/texk/upmendex/fwrite.c
+++ b/Build/source/texk/upmendex/fwrite.c
@@ -175,6 +175,23 @@ void indwrite(char *filename, struct index *ind, int pagenum)
 	perr=U_ZERO_ERROR;
 	unormalizer_NFD=unorm2_getInstance(NULL, "nfc", UNORM2_DECOMPOSE, &perr);
 
+	if (strlen(symhead)==0) {
+		if (lethead_flag>0) {
+			strcpy(symhead, symhead_positive);
+		}
+		else if (lethead_flag<0) {
+			strcpy(symhead, symhead_negative);
+		}
+	}
+	{
+		if (lethead_flag>0) {
+			strcpy(numhead, numhead_positive);
+		}
+		else if (lethead_flag<0) {
+			strcpy(numhead, numhead_negative);
+		}
+	}
+
 	for (i=line_length=0;i<lines;i++) {
 		index_normalize(ind[i].dic[0], initial, &chset);
 		if (i==0) {
@@ -280,16 +297,11 @@ void indwrite(char *filename, struct index *ind, int pagenum)
 				}
 			}
 			else {
-				if (lethead_flag!=0 && symbol_flag) {
-					if (strlen(symbol)) {
-						fprintf(fp,"%s%s%s",lethead_prefix,symbol,lethead_suffix);
-					}
-					else if (lethead_flag>0) {
-						fprintf(fp,"%s%s%s",lethead_prefix,symhead_positive,lethead_suffix);
-					}
-					else if (lethead_flag<0) {
-						fprintf(fp,"%s%s%s",lethead_prefix,symhead_negative,lethead_suffix);
-					}
+				if (lethead_flag!=0 && symbol_flag==2 && chset==CH_NUMERIC) {
+					fprintf(fp,"%s%s%s",lethead_prefix,numhead,lethead_suffix);
+				}
+				if (lethead_flag!=0 && (symbol_flag==1 || (symbol_flag==2 && chset!=CH_NUMERIC))) {
+					fprintf(fp,"%s%s%s",lethead_prefix,symhead,lethead_suffix);
 				}
 				widechar_to_multibyte(obuff,BUFFERLEN,ind[i].idx[0]);
 				SPRINTF(lbuff,"%s%s",item_0,obuff);
@@ -398,18 +410,15 @@ void indwrite(char *filename, struct index *ind, int pagenum)
 				}
 			}
 			else {
-				if (CH_LATIN<=chset_prev&&chset_prev<=CH_THAI){
-					fputs(group_skip,fp);
-					if (lethead_flag!=0 && symbol_flag) {
-						if (strlen(symbol)) {
-							fprintf(fp,"%s%s%s",lethead_prefix,symbol,lethead_suffix);
-						}
-						else if (lethead_flag>0) {
-							fprintf(fp,"%s%s%s",lethead_prefix,symhead_positive,lethead_suffix);
-						}
-						else if (lethead_flag<0) {
-							fprintf(fp,"%s%s%s",lethead_prefix,symhead_negative,lethead_suffix);
-						}
+				if (chset_prev!=chset) {
+					if ((CH_LATIN<=chset_prev&&chset_prev<=CH_THAI) || symbol_flag==2)
+						fputs(group_skip,fp);
+					if (lethead_flag!=0 && symbol_flag==2 && chset==CH_NUMERIC) {
+						fprintf(fp,"%s%s%s",lethead_prefix,numhead,lethead_suffix);
+					}
+					if (lethead_flag!=0 && (symbol_flag==1 && (CH_LATIN<=chset_prev&&chset_prev<=CH_THAI) ||
+								symbol_flag==2 && chset!=CH_NUMERIC) ) {
+						fprintf(fp,"%s%s%s",lethead_prefix,symhead,lethead_suffix);
 					}
 				}
 			}
diff --git a/Build/source/texk/upmendex/sort.c b/Build/source/texk/upmendex/sort.c
index f1914fc8e13..7d4590886a2 100644
--- a/Build/source/texk/upmendex/sort.c
+++ b/Build/source/texk/upmendex/sort.c
@@ -4,6 +4,22 @@
 #include "exkana.h"
 #include "exvar.h"
 
+#define RULEBUFSIZE  29210+STYBUFSIZE
+/*
+	length of collation rule in ICU 68.2
+
+	icu_locale          length
+	ja                    6410
+	ja@collation=unihan     61
+	ko                   12577
+	ko@collation=unihan     51
+	ko@collation=search    782
+	zh  (pinin)          26909
+	zh@collation=unihan     82
+	zh@collation=stroke  29208
+	zh@collation=zhuyin  28880
+*/
+
 int sym,nmbr,ltn,kana,hngl,hnz,cyr,grk,dvng,thai;
 
 static int wcomp(const void *p, const void *q);
@@ -17,7 +33,9 @@ void wsort(struct index *ind, int num)
 {
 	int i,order;
 	UErrorCode status;
-	UChar rules[STYBUFSIZE] = {'\0'};
+	UParseError parse_error;
+	UChar rules[RULEBUFSIZE] = {'\0'};
+	int32_t len;
 
 	for (order=1,i=0;;i++) {
 		switch (character_order[i]) {
@@ -86,8 +104,23 @@ BREAK:
 
 	status = U_ZERO_ERROR;
 	if (strlen(icu_rules)>0) {
+		if (strcmp(icu_locale,"root")!=0) {
+			icu_collator = ucol_open(icu_locale, &status);
+			if (U_FAILURE(status)) {
+				verb_printf(efp, "\n[ICU] Collator creation failed.: %s\n", u_errorName(status));
+				exit(254);
+			}
+			len = ucol_getRulesEx(icu_collator, UCOL_TAILORING_ONLY, rules, RULEBUFSIZE);
+			if (u_strlen(rules)<len) {
+				verb_printf(efp, "\n[ICU] Failed to extract collation rules by locale (%s). Need buffer size %d.\n",
+					icu_locale, len);
+				exit(254);
+			}
+			ucol_close(icu_collator);
+		}
 		unescape((unsigned char *)icu_rules, rules);
-		icu_collator = ucol_openRules(rules, -1, UCOL_OFF, UCOL_TERTIARY, NULL, &status);
+		status = U_ZERO_ERROR;
+		icu_collator = ucol_openRules(rules, -1, UCOL_OFF, UCOL_TERTIARY, &parse_error, &status);
 	} else
 		icu_collator = ucol_open(icu_locale, &status);
 	if (U_FAILURE(status)) {
@@ -339,7 +372,7 @@ static int unescape(const unsigned char *src, UChar *dest)
 			strncpy(tmp,(char *)&src[j],i-j+1);
 			tmp[i-j+1]='\0';
 			k=u_strlen(dest);
-			ret=u_unescape(tmp, &dest[k], STYBUFSIZE-k);
+			ret=u_unescape(tmp, &dest[k], RULEBUFSIZE-k);
 			if (ret==0) {
 				verb_printf(efp, "\n[ICU] Escape sequence in input seems malformed.\n");
 				exit(254);
@@ -351,7 +384,7 @@ static int unescape(const unsigned char *src, UChar *dest)
 			tmp[i-j+1]='\0';
 			k=u_strlen(dest);
 			status=U_ZERO_ERROR;
-			u_strFromUTF8(&dest[k], STYBUFSIZE-k, NULL, tmp, -1, &status);
+			u_strFromUTF8(&dest[k], RULEBUFSIZE-k, NULL, tmp, -1, &status);
 			if (U_FAILURE(status)) {
 				verb_printf(efp, "\n[ICU] Input string seems malformed.: %s\n", u_errorName(status));
 				exit(254);
diff --git a/Build/source/texk/upmendex/styfile.c b/Build/source/texk/upmendex/styfile.c
index cecf29d7255..af50bf845bb 100644
--- a/Build/source/texk/upmendex/styfile.c
+++ b/Build/source/texk/upmendex/styfile.c
@@ -117,7 +117,7 @@ void styread(const char *filename)
 			indent_length=atoi(&buff[cc]);
 			continue;
 		}
-		if (getparam(buff,"symbol",symbol)) continue;
+		if (getparam(buff,"symbol",symhead)) continue;
 		cc=scompare(buff,"symbol_flag");
 		if (cc!= -1) {
 			symbol_flag=atoi(&buff[cc]);
diff --git a/Build/source/texk/upmendex/tests/attr1.ind b/Build/source/texk/upmendex/tests/attr1.ind
index 47e44895159..6db7b19221b 100644
--- a/Build/source/texk/upmendex/tests/attr1.ind
+++ b/Build/source/texk/upmendex/tests/attr1.ind
@@ -25,6 +25,9 @@
   \item c\leaders\hbox{$\cdot$}\hfill 405
   \item Ç\leaders\hbox{$\cdot$}\hfill 406
   \item ç\leaders\hbox{$\cdot$}\hfill 407
+  \item Ch\leaders\hbox{$\cdot$}\hfill 420
+  \item cha\leaders\hbox{$\cdot$}\hfill 421
+  \item cz\leaders\hbox{$\cdot$}\hfill 422
 
   \indexspace
 
diff --git a/Build/source/texk/upmendex/tests/locale1.ind b/Build/source/texk/upmendex/tests/locale1.ind
index 56c8c2c6138..76bf98f25a5 100644
--- a/Build/source/texk/upmendex/tests/locale1.ind
+++ b/Build/source/texk/upmendex/tests/locale1.ind
@@ -23,6 +23,9 @@
 \centerline{\bfseries ◆◆◆ C ◆◆◆}\par\nobreak
   \item c\leaders\hbox{$\cdot$}\hfill 405
   \item C\leaders\hbox{$\cdot$}\hfill 404
+  \item Ch\leaders\hbox{$\cdot$}\hfill 420
+  \item cha\leaders\hbox{$\cdot$}\hfill 421
+  \item cz\leaders\hbox{$\cdot$}\hfill 422
 
   \indexspace
 
diff --git a/Build/source/texk/upmendex/tests/locale2-rule.ind b/Build/source/texk/upmendex/tests/locale2-rule.ind
new file mode 100644
index 00000000000..4054762ab75
--- /dev/null
+++ b/Build/source/texk/upmendex/tests/locale2-rule.ind
@@ -0,0 +1,44 @@
+\begin{theindex}
+
+\centerline{\bfseries ◆◆◆ Symbols ◆◆◆}\par\nobreak
+  \item #\leaders\hbox{$\cdot$}\hfill 2000
+  \item $\leaders\hbox{$\cdot$}\hfill 2001
+  \item 10\leaders\hbox{$\cdot$}\hfill 1000
+  \item 2\leaders\hbox{$\cdot$}\hfill 1001
+
+  \indexspace
+
+\centerline{\bfseries ◆◆◆ A ◆◆◆}\par\nobreak
+  \item a\leaders\hbox{$\cdot$}\hfill 401
+  \item A\leaders\hbox{$\cdot$}\hfill 400
+
+  \indexspace
+
+\centerline{\bfseries ◆◆◆ C ◆◆◆}\par\nobreak
+  \item C\leaders\hbox{$\cdot$}\hfill 404
+  \item Ç\leaders\hbox{$\cdot$}\hfill 406
+  \item c\leaders\hbox{$\cdot$}\hfill 405
+  \item ç\leaders\hbox{$\cdot$}\hfill 407
+  \item cz\leaders\hbox{$\cdot$}\hfill 422
+
+  \indexspace
+
+\centerline{\bfseries ◆◆◆ U ◆◆◆}\par\nobreak
+  \item u\leaders\hbox{$\cdot$}\hfill 409
+  \item U\leaders\hbox{$\cdot$}\hfill 408
+  \item ü\leaders\hbox{$\cdot$}\hfill 411
+  \item Ü\leaders\hbox{$\cdot$}\hfill 410
+
+  \indexspace
+
+\centerline{\bfseries ◆◆◆ B ◆◆◆}\par\nobreak
+  \item b\leaders\hbox{$\cdot$}\hfill 403
+  \item B\leaders\hbox{$\cdot$}\hfill 402
+
+  \indexspace
+
+\centerline{\bfseries ◆◆◆ CH ◆◆◆}\par\nobreak
+  \item Ch\leaders\hbox{$\cdot$}\hfill 420
+  \item cha\leaders\hbox{$\cdot$}\hfill 421
+
+\end{theindex}
diff --git a/Build/source/texk/upmendex/tests/locale2.ist b/Build/source/texk/upmendex/tests/locale2.ist
new file mode 100644
index 00000000000..89b590a21bf
--- /dev/null
+++ b/Build/source/texk/upmendex/tests/locale2.ist
@@ -0,0 +1 @@
+icu_locale "es@collation=traditional"
diff --git a/Build/source/texk/upmendex/tests/rule.idx b/Build/source/texk/upmendex/tests/rule.idx
index 155ba8e28a7..783cce9fd41 100644
--- a/Build/source/texk/upmendex/tests/rule.idx
+++ b/Build/source/texk/upmendex/tests/rule.idx
@@ -7,6 +7,9 @@
 \indexentry{c}{405}
 \indexentry{Ç}{406}
 \indexentry{ç}{407}
+\indexentry{Ch}{420}
+\indexentry{cha}{421}
+\indexentry{cz}{422}
 \indexentry{U}{408}
 \indexentry{u}{409}
 \indexentry{Ü}{410}
diff --git a/Build/source/texk/upmendex/tests/rule.ind b/Build/source/texk/upmendex/tests/rule.ind
index 6dd13b205f9..3ac1335e052 100644
--- a/Build/source/texk/upmendex/tests/rule.ind
+++ b/Build/source/texk/upmendex/tests/rule.ind
@@ -19,6 +19,9 @@
   \item Ç\leaders\hbox{$\cdot$}\hfill 406
   \item c\leaders\hbox{$\cdot$}\hfill 405
   \item ç\leaders\hbox{$\cdot$}\hfill 407
+  \item Ch\leaders\hbox{$\cdot$}\hfill 420
+  \item cha\leaders\hbox{$\cdot$}\hfill 421
+  \item cz\leaders\hbox{$\cdot$}\hfill 422
 
   \indexspace
 
diff --git a/Build/source/texk/upmendex/tests/upmendex.test b/Build/source/texk/upmendex/tests/upmendex.test
index 48787d5ae7c..a291c1dfc74 100755
--- a/Build/source/texk/upmendex/tests/upmendex.test
+++ b/Build/source/texk/upmendex/tests/upmendex.test
@@ -84,3 +84,7 @@ else
   exit 1
 fi
 
+# test for icu_locale + icu_rules
+./upmendex -s $srcdir/tests/uni.ist -s $srcdir/tests/locale2.ist -s $srcdir/tests/rule1.ist $srcdir/tests/rule.idx \
+	-o locale2-rule1.ind1 -t locale2-rule1.ilg \
+	&& diff $srcdir/tests/locale2-rule.ind locale2-rule1.ind1 || exit 1
diff --git a/Build/source/texk/upmendex/var.h b/Build/source/texk/upmendex/var.h
index 02e27149109..a7a66bfe0fa 100644
--- a/Build/source/texk/upmendex/var.h
+++ b/Build/source/texk/upmendex/var.h
@@ -27,7 +27,7 @@ int line_max=72;
 char indent_space[STYBUFSIZE]={"\t\t"};
 int indent_length=16;
 int priority=0;
-char symbol[STYBUFSIZE]={""};
+char symhead[STYBUFSIZE]={""},numhead[STYBUFSIZE]={""};
 char symhead_positive[STYBUFSIZE]={"Symbols"},symhead_negative[STYBUFSIZE]={"symbols"};
 char numhead_positive[STYBUFSIZE]={"Numbers"},numhead_negative[STYBUFSIZE]={"numbers"};
 int symbol_flag=1;