upmendex 0.60: enhance procedure of classifying characters

git-svn-id: svn://tug.org/texlive/trunk@60856 c570f23f-e606-0410-a88d-b1316a301751

13 files changed, 198 insertions, 262 deletions

diff --git a/Build/source/texk/README b/Build/source/texk/README
index 497b871809d..4af905a2e4a 100644
--- a/Build/source/texk/README
+++ b/Build/source/texk/README
@@ -108,7 +108,7 @@ ttf2pk2 - maintained here, by us
 
 ttfdump - maintained here, by us, since Taiwan upstream apparently gone.
 
-upmendex 0.59 - by Takuji Tanaka
+upmendex 0.60 - by Takuji Tanaka
   http://www.ctan.org/pkg/upmendex
   https://github.com/t-tk/upmendex-package
 
diff --git a/Build/source/texk/upmendex/ChangeLog b/Build/source/texk/upmendex/ChangeLog
index 65c5ba4519a..b5aea031038 100644
--- a/Build/source/texk/upmendex/ChangeLog
+++ b/Build/source/texk/upmendex/ChangeLog
@@ -1,3 +1,14 @@
+2021-10-24  TANAKA Takuji  <ttk@t-lab.opal.ne.jp>
+
+	* version 0.60  Beta version.
+	* configure.ac: Bump version.
+	* Makefile.am, convert.c, fwrite.c, sort.c,
+	{,ex}kana.h, ktable.h, mendex.h:
+	Enhance procedure of classifying characters:
+	Newly classifies by character type of
+	General Category in Unicode Script Property.
+	https://github.com/t-tk/upmendex-package/issues/8
+
 2021-09-19  TANAKA Takuji  <ttk@t-lab.opal.ne.jp>
 
 	* fread.c:
@@ -170,7 +181,7 @@
 
 2020-05-01  TANAKA Takuji  <ttk@t-lab.opal.ne.jp>
 
-	* main.c, version.h, Makefile.{am.in}: Add bug report addresses
+	* main.c, version.h, Makefile.am: Add bug report addresses
 	on command line help.
 	* convert.c: Tune buffer sizes to avoid tool warnings.
 
diff --git a/Build/source/texk/upmendex/Makefile.am b/Build/source/texk/upmendex/Makefile.am
index 94bea187b33..44b3871623c 100644
--- a/Build/source/texk/upmendex/Makefile.am
+++ b/Build/source/texk/upmendex/Makefile.am
@@ -19,7 +19,6 @@ upmendex_SOURCES = \
 	kana.h \
 	kp.c \
 	kp.h \
-	ktable.h \
 	main.c \
 	mendex.h \
 	version.h \
diff --git a/Build/source/texk/upmendex/Makefile.in b/Build/source/texk/upmendex/Makefile.in
index 03a3ebbb93d..4ecaadee97b 100644
--- a/Build/source/texk/upmendex/Makefile.in
+++ b/Build/source/texk/upmendex/Makefile.in
@@ -593,7 +593,6 @@ upmendex_SOURCES = \
 	kana.h \
 	kp.c \
 	kp.h \
-	ktable.h \
 	main.c \
 	mendex.h \
 	version.h \
diff --git a/Build/source/texk/upmendex/configure b/Build/source/texk/upmendex/configure
index 7f3a619e5a9..2060efd92cd 100755
--- a/Build/source/texk/upmendex/configure
+++ b/Build/source/texk/upmendex/configure
@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.71 for upmendex (TeX Live) 0.59.
+# Generated by GNU Autoconf 2.71 for upmendex (TeX Live) 0.60.
 #
 #
 # Copyright (C) 1992-1996, 1998-2017, 2020-2021 Free Software Foundation,
@@ -626,8 +626,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='upmendex (TeX Live)'
 PACKAGE_TARNAME='upmendex--tex-live-'
-PACKAGE_VERSION='0.59'
-PACKAGE_STRING='upmendex (TeX Live) 0.59'
+PACKAGE_VERSION='0.60'
+PACKAGE_STRING='upmendex (TeX Live) 0.60'
 PACKAGE_BUGREPORT=''
 PACKAGE_URL=''
 
@@ -1390,7 +1390,7 @@ if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-\`configure' configures upmendex (TeX Live) 0.59 to adapt to many kinds of systems.
+\`configure' configures upmendex (TeX Live) 0.60 to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
@@ -1462,7 +1462,7 @@ fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of upmendex (TeX Live) 0.59:";;
+     short | recursive ) echo "Configuration of upmendex (TeX Live) 0.60:";;
    esac
   cat <<\_ACEOF
 
@@ -1587,7 +1587,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
   cat <<\_ACEOF
-upmendex (TeX Live) configure 0.59
+upmendex (TeX Live) configure 0.60
 generated by GNU Autoconf 2.71
 
 Copyright (C) 2021 Free Software Foundation, Inc.
@@ -2268,7 +2268,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by upmendex (TeX Live) $as_me 0.59, which was
+It was created by upmendex (TeX Live) $as_me 0.60, which was
 generated by GNU Autoconf 2.71.  Invocation command line was
 
   $ $0$ac_configure_args_raw
@@ -8806,7 +8806,7 @@ fi
 
 # Define the identity of the package.
  PACKAGE='upmendex--tex-live-'
- VERSION='0.59'
+ VERSION='0.60'
 
 
 # Some tools Automake needs.
@@ -18942,7 +18942,7 @@ Usage: $0 [OPTIONS]
 Report bugs to <bug-libtool@gnu.org>."
 
 lt_cl_version="\
-upmendex (TeX Live) config.lt 0.59
+upmendex (TeX Live) config.lt 0.60
 configured by $0, generated by GNU Autoconf 2.71.
 
 Copyright (C) 2011 Free Software Foundation, Inc.
@@ -21114,7 +21114,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by upmendex (TeX Live) $as_me 0.59, which was
+This file was extended by upmendex (TeX Live) $as_me 0.60, which was
 generated by GNU Autoconf 2.71.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
@@ -21182,7 +21182,7 @@ ac_cs_config_escaped=`printf "%s\n" "$ac_cs_config" | sed "s/^ //; s/'/'\\\\\\\\
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config='$ac_cs_config_escaped'
 ac_cs_version="\\
-upmendex (TeX Live) config.status 0.59
+upmendex (TeX Live) config.status 0.60
 configured by $0, generated by GNU Autoconf 2.71,
   with options \\"\$ac_cs_config\\"
 
diff --git a/Build/source/texk/upmendex/configure.ac b/Build/source/texk/upmendex/configure.ac
index 3bc2cc06f30..87e9f10deb7 100644
--- a/Build/source/texk/upmendex/configure.ac
+++ b/Build/source/texk/upmendex/configure.ac
@@ -8,7 +8,7 @@ dnl   This file is free software; the copyright holder
 dnl   gives unlimited permission to copy and/or distribute it,
 dnl   with or without modifications, as long as this notice is preserved.
 dnl
-AC_INIT([upmendex (TeX Live)],[0.59])
+AC_INIT([upmendex (TeX Live)],[0.60])
 AC_PREREQ([2.71])
 AC_CONFIG_SRCDIR([main.c])
 AC_CONFIG_AUX_DIR([../../build-aux])
diff --git a/Build/source/texk/upmendex/convert.c b/Build/source/texk/upmendex/convert.c
index 74d32d73527..7b051fd4fc0 100644
--- a/Build/source/texk/upmendex/convert.c
+++ b/Build/source/texk/upmendex/convert.c
@@ -6,7 +6,6 @@
 #include "qsort.h"
 
 #include "exkana.h"
-#include "ktable.h"
 #include "exvar.h"
 
 #include "kp.h"
@@ -25,8 +24,6 @@ static int dlines=0,elines=0;
 
 static int dicvalread(const char *filename, struct dictionary *dicval, int line);
 
-#define is_alpha_numeric(a)  (((a)>=ALPHATOP && (a)<=ALPHAEND) || (a)==YENSIGN)
-
 UChar * u_xstrdup (const UChar *string)
 {
 	return u_strcpy(xmalloc((u_strlen(string)+1)*sizeof(UChar)), string);
@@ -210,76 +207,66 @@ int convert(UChar *buff1, UChar *buff2)
 				i++;
 			}
 
-			else if (buff1[i]<0x80) {
+			else if (buff1[i]<0x20 && buff1[i]!='\t') { /* ignore control characters */
+				i++;
+			}
+
+			else if (buff1[i]<0x7F) {
 				buff2[j]=buff1[i];
 				i++;
 				j++;
 			}
 
+			else if (buff1[i]<0xA0) { /* ignore control characters */
+				i++;
+			}
+
 			else if (is_latin(buff3)||is_cyrillic(buff3)||is_greek(buff3)
 				 ||is_jpn_kana(buff3)||is_kor_hngl(buff3)||is_zhuyin(buff3)
-				 ||is_numeric(buff3)||is_devanagari(buff3)||is_thai(buff3)
-					||is_comb_diacritical_mark(buff3)) {
+				 ||is_numeric(buff3)==1||is_type_symbol(buff3)==1
+				 ||is_devanagari(buff3)||is_thai(buff3)
+					||is_type_mark_or_punct(buff3)) {
 				buff2[j]=buff3[0];
 				if (wclen==2) buff2[j+1]=buff3[1];
 				i+=wclen;
 				j+=wclen;
 			}
 
-			else if (buff1[i]>=0x80) {
-				if (is_alpha_numeric(buff3[0])) {
-/*   alpha-numeric,symbols   */
-					for (k=0;k<u_strlen(symboltable);k++) {
-						if (buff3[0]==symboltable[k]) {
-							buff2[j]=k+0x20;
-							if ((buff2[j]>='a')&&(buff2[j]<='z')) buff2[j]-=32;
-							i++;
-							j+=wclen;
-							break;
-						}
-					}
-					if (k==u_strlen(symboltable)) {
-						i++;
-						buff2[j++]=buff3[0];
+			else {
+				for (k=0;k<dlines;k++) {
+/*   dictionary table   */
+					if (u_strncmp(dictable[k].dic[0],&buff1[i],u_strlen(dictable[k].dic[0]))==0) {
+						u_strncpy(&buff2[j],dictable[k].dic[1],u_strlen(dictable[k].dic[1]));
+						i+=u_strlen(dictable[k].dic[0]);
+						j+=u_strlen(dictable[k].dic[1]);
+						break;
 					}
 				}
-
-				else {
-					for (k=0;k<dlines;k++) {
-/*   dictionary table   */
-						if (u_strncmp(dictable[k].dic[0],&buff1[i],u_strlen(dictable[k].dic[0]))==0) {
-							u_strncpy(&buff2[j],dictable[k].dic[1],u_strlen(dictable[k].dic[1]));
+				if ((k==dlines)&&(elines!=0)) {
+/*   environment dictionary table   */
+					for (k=0;k<elines;k++) {
+						if (u_strncmp(envdic[k].dic[0],&buff1[i],u_strlen(envdic[k].dic[0]))==0) {
+							u_strncpy(&buff2[j],envdic[k].dic[1],u_strlen(envdic[k].dic[1]));
 							i+=u_strlen(dictable[k].dic[0]);
-							j+=u_strlen(dictable[k].dic[1]);
+							j+=u_strlen(envdic[k].dic[1]);
 							break;
 						}
 					}
-					if ((k==dlines)&&(elines!=0)) {
-/*   environment dictionary table   */
-						for (k=0;k<elines;k++) {
-							if (u_strncmp(envdic[k].dic[0],&buff1[i],u_strlen(envdic[k].dic[0]))==0) {
-								u_strncpy(&buff2[j],envdic[k].dic[1],u_strlen(envdic[k].dic[1]));
-								i+=u_strlen(dictable[k].dic[0]);
-								j+=u_strlen(envdic[k].dic[1]);
-								break;
-							}
-						}
-					}
-					if (((k==dlines)&&(elines==0))||((k==elines)&&(elines!=0))) {
-						if (is_hanzi(buff3) || force==1) {
+				}
+				if (((k==dlines)&&(elines==0))||((k==elines)&&(elines!=0))) {
+					if (is_hanzi(buff3) || is_numeric(buff3) || is_type_symbol(buff3) || force==1) {
 /*   forced convert   */
-							buff2[j]=buff3[0];
-							if (wclen==2) buff2[j+1]=buff3[1];
-							i+=wclen;
-							j+=wclen;
-						}
-						else {
-							widechar_to_multibyte(errbuff,BUFFERLEN2,&buff1[i]);
-							snprintf(errbuff2,BUFFERLEN3,"\nError: %s is no entry in dictionary file ",errbuff);
-							fputs(errbuff2,efp);
-							if (efp!=stderr) fputs(errbuff2,stderr);
-							return -1;
-						}
+						buff2[j]=buff3[0];
+						if (wclen==2) buff2[j+1]=buff3[1];
+						i+=wclen;
+						j+=wclen;
+					}
+					else {
+						widechar_to_multibyte(errbuff,BUFFERLEN2,&buff1[i]);
+						snprintf(errbuff2,BUFFERLEN3,"\nError: %s is no entry in dictionary file ",errbuff);
+						fputs(errbuff2,efp);
+						if (efp!=stderr) fputs(errbuff2,stderr);
+						return -1;
 					}
 				}
 			}
diff --git a/Build/source/texk/upmendex/exkana.h b/Build/source/texk/upmendex/exkana.h
index aeb5a18d827..9e4ba1d1972 100644
--- a/Build/source/texk/upmendex/exkana.h
+++ b/Build/source/texk/upmendex/exkana.h
@@ -4,7 +4,6 @@ extern UChar AIUEO[];
 extern UChar *aiueo;
 extern UChar kanatable[];
 extern UChar extkanatable[];
-extern UChar circkanatable[];
 
 #define SPACE    0x3000    /* 全角スペース */
 #define ALPHATOP 0xff01    /* ！ */
@@ -18,8 +17,19 @@ extern UChar circkanatable[];
 #define EXKANAEND  0x31FF  /* ㇿ */
 #define CRKANATOP  0x32D0  /* ㋐ */
 #define CRKANAEND  0x32FE  /* ㋾ */
+#define HANKANATOP 0xFF71  /* ｱ */
+#define HANKANAEND 0xFF9D  /* ﾝ */
+#define HANKANAWO  0xFF66  /* ｦ */
+#define HANKANATU  0xFF6F  /* ｯ */
+#define SQKANATOP  0x3300  /* ㌀ */
+#define SQKANAEND  0x3357  /* ㍗ */
+#define CRLATNTOP  0x24B6  /* Ⓐ */
+#define CRLATNEND  0x24E9  /* ⓩ */
 
 #define is_katakana(a)  ((a)>=KATATOP && (a)<=KATAEND)
 #define is_hiragana(a)  ((a)>=HIRATOP && (a)<=HIRAEND)
 #define is_extkana(a)   ((a)>=EXKANATOP && (a)<=EXKANAEND)
 #define is_circkana(a)  ((a)>=CRKANATOP && (a)<=CRKANAEND)
+#define is_hankana(a)   ((a)>=HANKANATOP && (a)<=HANKANAEND || (a)>=HANKANAWO && (a)<=HANKANATU)
+#define is_sqkana(a)    ((a)>=SQKANATOP && (a)<=SQKANAEND)
+#define is_circlatin(a) ((a)>=CRLATNTOP && (a)<=CRLATNEND)
diff --git a/Build/source/texk/upmendex/fwrite.c b/Build/source/texk/upmendex/fwrite.c
index a5b6aa60362..4ec8ab5a779 100644
--- a/Build/source/texk/upmendex/fwrite.c
+++ b/Build/source/texk/upmendex/fwrite.c
@@ -18,7 +18,7 @@ static void crcheck(char *lbuff, FILE *fp);
 static void index_normalize(UChar *istr, UChar *ini, int *chset);
 static int initial_cmp_char(UChar *ini, UChar ch);
 static int init_hanzi_header(void);
-static const UNormalizer2* unormalizer_NFD;
+static const UNormalizer2 *unormalizer_NFD, *unormalizer_NFKD;
 static int turkish_i;
 
 #define M_NONE      0
@@ -189,7 +189,8 @@ void indwrite(char *filename, struct index *ind, int pagenum)
 		fprintf(fp,"%s%d%s",setpage_prefix,pagenum,setpage_suffix);
 	}
 	perr=U_ZERO_ERROR;
-	unormalizer_NFD=unorm2_getInstance(NULL, "nfc", UNORM2_DECOMPOSE, &perr);
+	unormalizer_NFD =unorm2_getInstance(NULL, "nfc", UNORM2_DECOMPOSE, &perr);
+	unormalizer_NFKD=unorm2_getInstance(NULL, "nfkc", UNORM2_DECOMPOSE, &perr);
 
 	if (strlen(symhead)==0) {
 		if (lethead_flag>0) {
@@ -427,12 +428,12 @@ void indwrite(char *filename, struct index *ind, int pagenum)
 			}
 			else {
 				if (chset_prev!=chset) {
-					if ((CH_LATIN<=chset_prev&&chset_prev<=CH_THAI) || symbol_flag==2)
+					if (is_any_script(chset_prev) || symbol_flag==2)
 						fputs(group_skip,fp);
 					if (lethead_flag!=0 && symbol_flag==2 && chset==CH_NUMERIC) {
 						fprintf(fp,"%s%s%s",lethead_prefix,numhead,lethead_suffix);
 					}
-					if (lethead_flag!=0 && (symbol_flag==1 && (CH_LATIN<=chset_prev&&chset_prev<=CH_THAI) ||
+					if (lethead_flag!=0 && (symbol_flag==1 && is_any_script(chset_prev) ||
 								symbol_flag==2 && chset!=CH_NUMERIC) ) {
 						fprintf(fp,"%s%s%s",lethead_prefix,symhead,lethead_suffix);
 					}
@@ -763,6 +764,13 @@ static void index_normalize(UChar *istr, UChar *ini, int *chset)
 	*chset=charset(istr);
 	ini[1]=L'\0';
 
+	if (is_circkana(ch) || is_hankana(ch) || is_sqkana(ch) || is_circlatin(ch)) {  /* ㋐㋑㋒.. ｱｲｳ.. ㌀㌁㌂.. */
+		src[0]=ch;  src[1]=0x00;
+		perr=U_ZERO_ERROR;
+		unorm2_normalize(unormalizer_NFKD, src, 1, dest, 8, &perr);
+		if (U_SUCCESS(perr))
+			ch=dest[0];
+	}
 	if (is_hiragana(ch)) {
 		ch+=KATATOP-HIRATOP; /* hiragana -> katakana */
 	}
@@ -774,15 +782,15 @@ static void index_normalize(UChar *istr, UChar *ini, int *chset)
 		ini[0]=extkanatable[ch-EXKANATOP];
 		return;
 	}
-	if (is_circkana(ch)) {     /* ㋐㋑㋒㋓㋔ .. ㋻㋼㋽㋾ */
-		ini[0]=circkanatable[ch-CRKANATOP];
-		return;
-	}
-	else if (ch==0x309F) { ini[0]=0x3088; return; }  /* HIRAGANA YORI -> よ */
+	if      (ch==0x309F) { ini[0]=0x3088; return; }  /* HIRAGANA YORI -> よ */
 	else if (ch==0x30FF) { ini[0]=0x3053; return; }  /* KATAKANA KOTO -> こ */
 	else if (is_jpn_kana(istr)==2) {
 		c32=U16_GET_SUPPLEMENTARY(istr[0],istr[1]);
 		switch (c32) {
+			case 0x1F200:                  /* 🈀 */
+				ini[0]=0x307B; break;  /* ほ */
+			case 0x1B000:                  /* 𛀀 */
+				ini[0]=0x3048; break;  /* え */
 			case 0x1B150: case 0x1B164:
 				ini[0]=0x3090; break;  /* ゐ */
 			case 0x1B151: case 0x1B165:
@@ -889,6 +897,10 @@ static void index_normalize(UChar *istr, UChar *ini, int *chset)
 		ini[0]=ch;
 		return;
 	}
+	if (ch>=0xFF21&&ch<=0xFF3A || ch>=0xFF41&&ch<=0xFF5A) {
+		/* Fullwidth latin letter */
+		ch-=0xFF21-0x0041;
+	}
 	if (ch==0x049||ch==0x069||ch==0x130||ch==0x131||ch==0x0CE||ch==0x0EE) {
 		/* check dotted/dotless İ,I,i,ı and Î,î for Turkish */
 		strX[0] = 0x131;  strX[1] = 0x5A;  strX[2] = 0x00;  /* ıZ */
@@ -1018,6 +1030,8 @@ static void index_normalize(UChar *istr, UChar *ini, int *chset)
 			}
 		}
 	}
+	if (ch==0x0AA) ch=L'A';
+	if (ch==0x0BA) ch=L'O';
 	ini[0]=u_toupper(ch);
 	return;
 }
diff --git a/Build/source/texk/upmendex/kana.h b/Build/source/texk/upmendex/kana.h
index 5c006cb7f3e..41307f2cb4c 100644
--- a/Build/source/texk/upmendex/kana.h
+++ b/Build/source/texk/upmendex/kana.h
@@ -192,56 +192,6 @@ UChar extkanatable[]={
 	0x308d, /* ろ */
 	0
 };
-UChar circkanatable[]={
-	0x3042, /* あ */
-	0x3044, /* い */
-	0x3046, /* う */
-	0x3048, /* え */
-	0x304a, /* お */
-	0x304b, /* か */
-	0x304d, /* き */
-	0x304f, /* く */
-	0x3051, /* け */
-	0x3053, /* こ */
-	0x3055, /* さ */
-	0x3057, /* し */
-	0x3059, /* す */
-	0x305b, /* せ */
-	0x305d, /* そ */
-	0x305f, /* た */
-	0x3061, /* ち */
-	0x3064, /* つ */
-	0x3066, /* て */
-	0x3068, /* と */
-	0x306a, /* な */
-	0x306b, /* に */
-	0x306c, /* ぬ */
-	0x306d, /* ね */
-	0x306e, /* の */
-	0x306f, /* は */
-	0x3072, /* ひ */
-	0x3075, /* ふ */
-	0x3078, /* へ */
-	0x307b, /* ほ */
-	0x307e, /* ま */
-	0x307f, /* み */
-	0x3080, /* む */
-	0x3081, /* め */
-	0x3082, /* も */
-	0x3084, /* や */
-	0x3086, /* ゆ */
-	0x3088, /* よ */
-	0x3089, /* ら */
-	0x308a, /* り */
-	0x308b, /* る */
-	0x308c, /* れ */
-	0x308d, /* ろ */
-	0x308f, /* わ */
-	0x3090, /* ゐ */
-	0x3091, /* ゑ */
-	0x3092, /* を */
-	0
-};
 
 
 UChar GANADA[]={
diff --git a/Build/source/texk/upmendex/ktable.h b/Build/source/texk/upmendex/ktable.h
deleted file mode 100644
index 9a3a00ef49b..00000000000
--- a/Build/source/texk/upmendex/ktable.h
+++ /dev/null
@@ -1,98 +0,0 @@
-UChar symboltable[]={
-	0x3000, /* スペース */
-	0xff01, /* ！ */
-	0xff02, /* ” */
-	0xff03, /* ＃ */
-	0xff04, /* ＄ */
-	0xff05, /* ％ */
-	0xff06, /* ＆ */
-	0xff07, /* ’ */
-	0xff08, /* （ */
-	0xff09, /* ） */
-	0xff0a, /* ＊ */
-	0xff0b, /* ＋ */
-	0xff0c, /* ， */
-	0xff0d, /* − */
-	0xff0e, /* ． */
-	0xff0f, /* ／ */
-	0xff10, /* ０ */
-	0xff11, /* １ */
-	0xff12, /* ２ */
-	0xff13, /* ３ */
-	0xff14, /* ４ */
-	0xff15, /* ５ */
-	0xff16, /* ６ */
-	0xff17, /* ７ */
-	0xff18, /* ８ */
-	0xff19, /* ９ */
-	0xff1a, /* ： */
-	0xff1b, /* ； */
-	0xff1c, /* ＜ */
-	0xff1d, /* ＝ */
-	0xff1e, /* ＞ */
-	0xff1f, /* ？ */
-	0xff20, /* ＠ */
-	0xff21, /* Ａ */
-	0xff22, /* Ｂ */
-	0xff23, /* Ｃ */
-	0xff24, /* Ｄ */
-	0xff25, /* Ｅ */
-	0xff26, /* Ｆ */
-	0xff27, /* Ｇ */
-	0xff28, /* Ｈ */
-	0xff29, /* Ｉ */
-	0xff2a, /* Ｊ */
-	0xff2b, /* Ｋ */
-	0xff2c, /* Ｌ */
-	0xff2d, /* Ｍ */
-	0xff2e, /* Ｎ */
-	0xff2f, /* Ｏ */
-	0xff30, /* Ｐ */
-	0xff31, /* Ｑ */
-	0xff32, /* Ｒ */
-	0xff33, /* Ｓ */
-	0xff34, /* Ｔ */
-	0xff35, /* Ｕ */
-	0xff36, /* Ｖ */
-	0xff37, /* Ｗ */
-	0xff38, /* Ｘ */
-	0xff39, /* Ｙ */
-	0xff3a, /* Ｚ */
-	0xff3b, /* ［ */
-	0xffe5, /* ￥ */
-	0xff3d, /* ］ */
-	0xff3e, /* ＾ */
-	0xff3f, /* ＿ */
-	0xff40, /* ‘ */
-	0xff41, /* ａ */
-	0xff42, /* ｂ */
-	0xff43, /* ｃ */
-	0xff44, /* ｄ */
-	0xff45, /* ｅ */
-	0xff46, /* ｆ */
-	0xff47, /* ｇ */
-	0xff48, /* ｈ */
-	0xff49, /* ｉ */
-	0xff4a, /* ｊ */
-	0xff4b, /* ｋ */
-	0xff4c, /* ｌ */
-	0xff4d, /* ｍ */
-	0xff4e, /* ｎ */
-	0xff4f, /* ｏ */
-	0xff50, /* ｐ */
-	0xff51, /* ｑ */
-	0xff52, /* ｒ */
-	0xff53, /* ｓ */
-	0xff54, /* ｔ */
-	0xff55, /* ｕ */
-	0xff56, /* ｖ */
-	0xff57, /* ｗ */
-	0xff58, /* ｘ */
-	0xff59, /* ｙ */
-	0xff5a, /* ｚ */
-	0xff5b, /* ｛ */
-	0xff5c, /* ｜ */
-	0xff5d, /* ｝ */
-	0xff5f, /* 〜 */
-	0
-};
diff --git a/Build/source/texk/upmendex/mendex.h b/Build/source/texk/upmendex/mendex.h
index d14e5c66753..9eff1cc85d2 100644
--- a/Build/source/texk/upmendex/mendex.h
+++ b/Build/source/texk/upmendex/mendex.h
@@ -49,7 +49,6 @@ int lastpage(const char *filename);
 /* sort.c */
 void wsort(struct index *ind, int num);
 void pagesort(struct index *ind, int num);
-int is_alphanumeric(UChar *c);
 int is_latin(UChar *c);
 int is_numeric(UChar *c);
 int is_jpn_kana(UChar *c);
@@ -60,7 +59,8 @@ int is_cyrillic(UChar *c);
 int is_greek(UChar *c);
 int is_devanagari(UChar *c);
 int is_thai(UChar *c);
-int is_comb_diacritical_mark(UChar *c);
+int is_type_mark_or_punct(UChar *c);
+int is_type_symbol(UChar *c);
 int chkcontinue(struct page *p, int num);
 int ss_comp(UChar *s1, UChar *s2);
 
@@ -75,6 +75,7 @@ int ss_comp(UChar *s1, UChar *s2);
 #define CH_THAI         8
 #define CH_SYMBOL   0x100
 #define CH_NUMERIC  0x101
+#define  is_any_script(a)  ((CH_LATIN<=(a) && (a)<=CH_THAI))
 
 /* sort.c */
 int charset(UChar *c);
diff --git a/Build/source/texk/upmendex/sort.c b/Build/source/texk/upmendex/sort.c
index 26584e420aa..4d9c40441fa 100644
--- a/Build/source/texk/upmendex/sort.c
+++ b/Build/source/texk/upmendex/sort.c
@@ -290,13 +290,15 @@ static int pcomp(const void *p, const void *q)
 
 static int ordering(UChar *c)
 {
-	if (*c<0x80) {
-		if (is_latin(c)) return ltn;
-		else if (is_numeric(c)) return nmbr;
-		else return sym;
+	if      (*c<0x20)                return sym;  /* control */
+	else if (*c<0x7F) {
+		if      (is_latin(c))    return ltn;
+		else if (is_numeric(c))  return nmbr;
+		else                     return sym;
 	}
+	else if (*c<0xA0)                return sym;  /* control */
 	else {
-		if (is_latin(c)) return ltn;
+		if      (is_latin(c))    return ltn;
 		else if (is_jpn_kana(c)) return kana;
 		else if (is_kor_hngl(c)) return hngl;
 		else if (is_hanzi(c))    return hnz;
@@ -305,20 +307,21 @@ static int ordering(UChar *c)
 		else if (is_numeric(c))  return nmbr;
 		else if (is_devanagari(c)) return dvng;
 		else if (is_thai(c))     return thai;
-		else return sym;
+		else                     return sym;
 	}
 }
 
 int charset(UChar *c)
 {
-	if (*c==0x00) return CH_UNKNOWN;
-	else if (*c<0x80) {
-		if (is_latin(c)) return CH_LATIN;
-		else if (is_numeric(c)) return CH_NUMERIC;
-		else return CH_SYMBOL;
+	if      (*c<0x20)                return CH_UNKNOWN;  /* control */
+	else if (*c<0x7F) {
+		if      (is_latin(c))    return CH_LATIN;
+		else if (is_numeric(c))  return CH_NUMERIC;
+		else                     return CH_SYMBOL;
 	}
+	else if (*c<0xA0)                return CH_UNKNOWN;  /* control */
 	else {
-		if (is_latin(c)) return CH_LATIN;
+		if      (is_latin(c))    return CH_LATIN;
 		else if (is_jpn_kana(c)) return CH_KANA;
 		else if (is_kor_hngl(c)) return CH_HANGUL;
 		else if (is_hanzi(c))    return CH_HANZI;
@@ -327,7 +330,7 @@ int charset(UChar *c)
 		else if (is_numeric(c))  return CH_NUMERIC;
 		else if (is_devanagari(c)) return CH_DEVANAGARI;
 		else if (is_thai(c))     return CH_THAI;
-		else return CH_SYMBOL;
+		else                     return CH_SYMBOL;
 	}
 }
 
@@ -347,11 +350,18 @@ static int get_charset_juncture(UChar *str)
 		else l = k-1;
 		chset_l=charset(&str[l]);
 		chset_k=charset(&str[k]);
+		if (chset0==CH_UNKNOWN && is_any_script(chset_l)) {
+			chset0=chset_l;
+		}
+		if (chset0!=CH_UNKNOWN && is_any_script(chset_k)) {
+#if 0
 		if (chset0==CH_UNKNOWN && chset_l!=CH_SYMBOL && chset_l!=CH_NUMERIC) {
 			chset0=chset_l;
 		}
 		if (chset_k!=CH_SYMBOL && chset_k!=CH_NUMERIC) {
+#endif
 			if (chset0!=chset_k) {
+			  fprintf(stderr,"sort.c: ###DBG1000 %d %d %d len:%d %05x %05x\n", chset0, chset_l, chset_k, k, str[l], str[k]);
 				len=k;
 				return len;
 			}
@@ -396,17 +406,11 @@ static int unescape(const unsigned char *src, UChar *dest)
 	return -1;
 }
 
-int is_alphanumeric(UChar *c)
-{
-	if (((*c>=L'A')&&(*c<=L'Z'))||((*c>=L'a')&&(*c<=L'z'))||((*c>=L'0')&&(*c<=L'9')))
-		return 1;
-	else return 0;
-}
-
 int is_latin(UChar *c)
 {
 	if (((*c>=L'A')&&(*c<=L'Z'))||((*c>=L'a')&&(*c<=L'z'))) return 1;
-	else if ((*c>=0x00C0)&&(*c<=0x00D6)) return 1; /* Latin-1 Supplement */
+	else if ((*c==0x00AA)||(*c==0x00BA)) return 1; /* Latin-1 Supplement */
+	else if ((*c>=0x00C0)&&(*c<=0x00D6)) return 1;
 	else if ((*c>=0x00D8)&&(*c<=0x00F6)) return 1;
 	else if ((*c>=0x00F8)&&(*c<=0x00FF)) return 1;
 	else if ((*c>=0x0100)&&(*c<=0x024F)) return 1; /* Latin Extended-A,B */
@@ -416,28 +420,57 @@ int is_latin(UChar *c)
 	else if ((*c>=0xAB30)&&(*c<=0xAB6F)) return 1; /* Latin Extended-E */
 	else if ((*c>=0x1E00)&&(*c<=0x1EFF)) return 1; /* Latin Extended Additional */
 	else if ((*c>=0xFB00)&&(*c<=0xFB06)) return 1; /* Latin ligatures */
+	else if ((*c>=0xFF21)&&(*c<=0xFF3A)) return 1; /* Fullwidth Latin Capital Letter */
+	else if ((*c>=0xFF41)&&(*c<=0xFF5A)) return 1; /* Fullwidth Latin Small Letter */
+		/* Property of followings is "Common, So (other symbol)", but seem to be treated as Latin by ICU collator */
+	else if ((*c>=0x24B6)&&(*c<=0x24CF)) return 1; /* CIRCLED LATIN CAPITAL LETTER */
+	else if ((*c>=0x24D0)&&(*c<=0x24E9)) return 1; /* CIRCLED LATIN SMALL LETTER */
 	else return 0;
 }
 
 int is_numeric(UChar *c)
 {
+	UChar32 c32;
+
 	if ((*c>=L'0')&&(*c<=L'9')) return 1;
-	else if ((*c>=0x0966)&&(*c<=0x096F)) return 1; /* Devanagari Digit */
-	else if ((*c>=0x0E50)&&(*c<=0x0E59)) return 1; /* Thai Digit */
-	else return 0;
+	else if ((*c>=0xFF10)&&(*c<=0xFF19)) return 1; /* Fullwidth Digit */
+		/* followings do not seem to be treated as numbers by ICU collator though charType is U_OTHER_NUMBER */
+	else if ((*c>=0x3192)&&(*c<=0x3195)) return 0; /* IDEOGRAPHIC ANNOTATION ONE MARK..IDEOGRAPHIC ANNOTATION FOUR MARK */
+	else if ((*c>=0x3220)&&(*c<=0x3229)) return 0; /* PARENTHESIZED IDEOGRAPH ONE..PARENTHESIZED IDEOGRAPH TEN */
+	else if ((*c>=0x3280)&&(*c<=0x3289)) return 0; /* CIRCLED IDEOGRAPH ONE..CIRCLED IDEOGRAPH TEN */
+	else if ((*c>=0xA830)&&(*c<=0xA835)) return 0; /* NORTH INDIC FRACTION ONE QUARTER..NORTH INDIC FRACTION THREE SIXTEENTHS */
+
+	if (is_surrogate_pair(c))
+		c32=U16_GET_SUPPLEMENTARY(*c,*(c+1));
+	else c32=*c;
+
+	switch (u_charType(c32)) {
+	case U_DECIMAL_DIGIT_NUMBER:
+		return 1;
+	case U_OTHER_NUMBER:
+		return 2;
+	default:
+		return 0;
+	}
 }
 
 int is_jpn_kana(UChar *c)
 {
 	UChar32 c32;
 
-	if      ((*c>=0x3040)&&(*c<=0x30FF)) return 1; /* Hiragana, Katakana */
+	if       (*c==0x30A0)                return 0; /* KATAKANA-HIRAGANA DOUBLE HYPHEN */
+	else if  (*c==0x30FB)                return 0; /* KATAKANA MIDDLE DOT */
+	else if ((*c>=0x3040)&&(*c<=0x30FF)) return 1; /* Hiragana, Katakana */
 	else if ((*c>=0x31F0)&&(*c<=0x31FF)) return 1; /* Katakana Phonetic Extensions */
 	else if ((*c>=0x32D0)&&(*c<=0x32FE)) return 1; /* Circled Katakana */
+	else if ((*c>=0xFF66)&&(*c<=0xFF9F)) return 1; /* Halfwidth Katakana */
+	else if ((*c>=0x3300)&&(*c<=0x3357)) return 1; /* Squared Katakana words */
 
 	if (is_surrogate_pair(c)) {
 		c32=U16_GET_SUPPLEMENTARY(*c,*(c+1));
 		if ((c32>=0x1B130) && (c32<=0x1B16F)) return 2; /* Small Kana Extensions */
+		else if ((c32==0x1B000))              return 2; /* KATAKANA LETTER ARCHAIC E */
+		else if ((c32==0x1F200))              return 2; /* SQUARE HIRAGANA HOKA */
 	}
 	return 0;
 		/* ICU 65 does not seem to support
@@ -503,7 +536,8 @@ int is_greek(UChar *c)
 
 int is_devanagari(UChar *c)
 {
-	if      ((*c>=0x0966)&&(*c<=0x096F)) return 0; /* Devanagari Digit */
+	if      ((*c>=0x0964)&&(*c<=0x0965)) return 0; /* Generic punctuation for scripts of India */
+	else if ((*c>=0x0966)&&(*c<=0x096F)) return 0; /* Devanagari Digit */
 	else if ((*c>=0x0900)&&(*c<=0x097F)) return 1; /* Devanagari */
 	else if ((*c>=0xA8E0)&&(*c<=0xA8FF)) return 1; /* Devanagari Extended */
 	else return 0;
@@ -517,14 +551,43 @@ int is_thai(UChar *c)
 	else return 0;
 }
 
-int is_comb_diacritical_mark(UChar *c)
+int is_type_mark_or_punct(UChar *c)
 {
-	if      ((*c>=0x02B0)&&(*c<=0x02FF)) return 1; /* Spacing Modifier Letters */
-	else if ((*c>=0x0300)&&(*c<=0x036F)) return 1; /* Combining Diacritical Marks */
-	else if ((*c>=0x1DC0)&&(*c<=0x1DFF)) return 1; /* Combining Diacritical Marks Supplement */
-	else if ((*c>=0x1AB0)&&(*c<=0x1AFF)) return 1; /* Combining Diacritical Marks Extended */
-	else if ((*c>=0x3099)&&(*c<=0x309A)) return 1; /* Combining Kana Voiced Sound Marks */
-	else return 0;
+	UChar32 c32;
+
+	if (is_surrogate_pair(c))
+		c32=U16_GET_SUPPLEMENTARY(*c,*(c+1));
+	else c32=*c;
+
+	switch (u_charType(c32)) {
+	case U_MODIFIER_LETTER:
+	case U_DASH_PUNCTUATION: case U_START_PUNCTUATION: case U_END_PUNCTUATION:
+	case U_CONNECTOR_PUNCTUATION: case U_OTHER_PUNCTUATION:
+	case U_INITIAL_PUNCTUATION: case U_FINAL_PUNCTUATION:
+	case U_NON_SPACING_MARK: case U_ENCLOSING_MARK: case U_COMBINING_SPACING_MARK:
+		return 1;
+	default:
+		return 0;
+	}
+}
+
+int is_type_symbol(UChar *c)
+{
+	UChar32 c32;
+
+	if (is_surrogate_pair(c))
+		c32=U16_GET_SUPPLEMENTARY(*c,*(c+1));
+	else c32=*c;
+
+	switch (u_charType(c32)) {
+	case U_MODIFIER_SYMBOL:
+		return 1;
+	case U_MATH_SYMBOL: case U_CURRENCY_SYMBOL:
+	case U_OTHER_SYMBOL:
+		return 2;
+	default:
+		return 0;
+	}
 }
 
 int chkcontinue(struct page *p, int num)