summaryrefslogtreecommitdiff
path: root/indexing/upmendex/source
diff options
context:
space:
mode:
authorNorbert Preining <norbert@preining.info>2021-11-22 03:03:38 +0000
committerNorbert Preining <norbert@preining.info>2021-11-22 03:03:38 +0000
commit3b45580e298046d6c1cb0431da31bea04bc6613a (patch)
treeee2b508b3e9ce370c5134f941650c4316ce9ec8c /indexing/upmendex/source
parent25fba613796cd9c1b2042e28152d451d8cddef4a (diff)
CTAN sync 202111220303
Diffstat (limited to 'indexing/upmendex/source')
-rw-r--r--indexing/upmendex/source/ChangeLog107
-rw-r--r--indexing/upmendex/source/Makefile.am3
-rw-r--r--indexing/upmendex/source/Makefile.in3
-rw-r--r--indexing/upmendex/source/aclocal.m410
-rwxr-xr-xindexing/upmendex/source/configure22
-rw-r--r--indexing/upmendex/source/configure.ac6
-rw-r--r--indexing/upmendex/source/convert.c93
-rw-r--r--indexing/upmendex/source/exkana.h12
-rw-r--r--indexing/upmendex/source/exvar.h5
-rw-r--r--indexing/upmendex/source/fread.c81
-rw-r--r--indexing/upmendex/source/fwrite.c199
-rw-r--r--indexing/upmendex/source/kana.h50
-rw-r--r--indexing/upmendex/source/ktable.h98
-rw-r--r--indexing/upmendex/source/mendex.h13
-rw-r--r--indexing/upmendex/source/sort.c356
-rw-r--r--indexing/upmendex/source/styfile.c142
-rw-r--r--indexing/upmendex/source/tests/attr1.ind40
-rw-r--r--indexing/upmendex/source/tests/attr1.ist2
-rw-r--r--indexing/upmendex/source/tests/locale1.ind48
-rw-r--r--indexing/upmendex/source/tests/locale1.ist1
-rw-r--r--indexing/upmendex/source/tests/locale2-rule.ind44
-rw-r--r--indexing/upmendex/source/tests/locale2.ist1
-rw-r--r--indexing/upmendex/source/tests/pprec3.ist11
-rw-r--r--indexing/upmendex/source/tests/pprec4.ist11
-rw-r--r--indexing/upmendex/source/tests/pprecB-0.ind5
-rw-r--r--indexing/upmendex/source/tests/pprecB-3.ind5
-rw-r--r--indexing/upmendex/source/tests/pprecB-4.ind5
-rw-r--r--indexing/upmendex/source/tests/pprecB.idx9
-rw-r--r--indexing/upmendex/source/tests/range.idx126
-rw-r--r--indexing/upmendex/source/tests/range1.ind25
-rw-r--r--indexing/upmendex/source/tests/range1.ist3
-rw-r--r--indexing/upmendex/source/tests/range2.ind26
-rw-r--r--indexing/upmendex/source/tests/range2.ist4
-rw-r--r--indexing/upmendex/source/tests/range3.ind25
-rw-r--r--indexing/upmendex/source/tests/range3.ist4
-rw-r--r--indexing/upmendex/source/tests/rangetwo.idx2
-rw-r--r--indexing/upmendex/source/tests/rangetwo.ind5
-rw-r--r--indexing/upmendex/source/tests/rangetwo.ist2
-rw-r--r--indexing/upmendex/source/tests/rule.idx20
-rw-r--r--indexing/upmendex/source/tests/rule.ind40
-rw-r--r--indexing/upmendex/source/tests/rule1.ist3
-rw-r--r--indexing/upmendex/source/tests/rule2.ist3
-rw-r--r--indexing/upmendex/source/tests/rule3.ist4
-rw-r--r--indexing/upmendex/source/tests/rule4.ist4
-rwxr-xr-xindexing/upmendex/source/tests/upmendex.test74
-rw-r--r--indexing/upmendex/source/var.h7
46 files changed, 1298 insertions, 461 deletions
diff --git a/indexing/upmendex/source/ChangeLog b/indexing/upmendex/source/ChangeLog
index 497e0e6e26..b2b3cd2f1e 100644
--- a/indexing/upmendex/source/ChangeLog
+++ b/indexing/upmendex/source/ChangeLog
@@ -1,3 +1,108 @@
+2021-11-20 TANAKA Takuji <ttk@t-lab.opal.ne.jp>
+
+ * version 1.00 Stable version.
+ * configure.ac: Bump version.
+ * {,ex}var.h, fwrite.c, styfile.c:
+ Add options "script_preamble" and "script_postamble"
+ in style file.
+ * sort.c: Add Latin Extended-F and -G.
+ Add U_FORMAT_CHAR to is_type_mark_or_punct().
+ * mendex.h, var.h, convert.c, fwrite.c, sort.c, styfile.c:
+ Supports Arabic, Hebrew scripts (experimental).
+ * upmendex.ja.txt:
+ Update document.
+
+2021-10-24 TANAKA Takuji <ttk@t-lab.opal.ne.jp>
+
+ * version 0.60 Beta version.
+ * configure.ac: Bump version.
+ * Makefile.am, convert.c, fwrite.c, sort.c,
+ {,ex}kana.h, ktable.h, mendex.h:
+ Enhance procedure of classifying characters:
+ Newly classifies by character type of
+ General Category in Unicode Script Property.
+ https://github.com/t-tk/upmendex-package/issues/8
+
+2021-09-19 TANAKA Takuji <ttk@t-lab.opal.ne.jp>
+
+ * fread.c:
+ Modify heuristic detection of page number types.
+ * tests/upmendex.test, tests/range.idx, tests/range[123].ind,
+ tests/pprecB.idx, tests/pprec[34].ist, tests/pprecB-[034].ind:
+ Update tests.
+
+2021-09-18 TANAKA Takuji <ttk@t-lab.opal.ne.jp>
+
+ * fread.c, fwrite.c, sort.c, mendex.h, {,ex}var.h:
+ Extend levels of page numbers from 3 to 10
+ for compatibility with makeindex.
+ Fix a bug of page ranges with multi-level page numbers.
+ * fread.c:
+ Strict check for "page_precedence" and "page_compositor" in style file.
+ * fwrite.c:
+ Do not break lines before "delim_n".
+ * tests/upmendex.test, tests/range.idx, tests/range[123].{ist,ind},
+ removed tests/rangetwo.*:
+ Update tests.
+
+2021-09-05 TANAKA Takuji <ttk@t-lab.opal.ne.jp>
+
+ * fwrite.c:
+ Support I-Y ordering in Lithuanian.
+
+2021-09-04 TANAKA Takuji <ttk@t-lab.opal.ne.jp>
+
+ * version 0.59 Beta version.
+ * configure.ac: Bump version.
+ * fwrite.c, styfile.c, {,ex}var.h:
+ Support output order of symbols and numbers separately
+ when "symbol_flag" is 2 in style file.
+ * sort.c:
+ Accept combination of "icu_locale" and "icu_rules" in style file.
+ If simultaneously specified, collation rules specified by "icu_rules" are
+ added on collation rules specified by "icu_locale".
+ * tests/upmendex.test, tests/rule.{idx,ind}, tests/locale2.ist,
+ tests/{locale1,attri1,locale2-rule}.ind:
+ Add a new test and update conventional tests.
+
+2021-08-29 TANAKA Takuji <ttk@t-lab.opal.ne.jp>
+
+ * styfile.c:
+ Strict check for "icu_attributes" in style file.
+ * sort.c:
+ Strict check for "character_order" in style file.
+ * fwrite.c:
+ Fix a bug when "symbol_flag" is 0.
+ * tests/upmendex.test, tests/attr1.{ist,ind},
+ tests/locale1.ind, tests/rule.{idx,ind}:
+ Add a new test and update conventional tests.
+
+2021-08-28 TANAKA Takuji <ttk@t-lab.opal.ne.jp>
+
+ * styfile.c:
+ Add a alias "headings_flag" of the keyword "heading_flag"
+ in style file for compatibility with makeindex.
+ Warn if specifier in style file is unknown.
+
+2021-08-26 TANAKA Takuji <ttk@t-lab.opal.ne.jp>
+
+ * sort.c:
+ Initialize a variable.
+ * tests/upmendex.test, tests/locale1.{ist,ind}:
+ Add a new test for "icu_locale".
+
+2021-08-25 TANAKA Takuji <ttk@t-lab.opal.ne.jp>
+
+ * sort.c:
+ Initialize a string. Check malformed input for "icu_rules".
+ * tests/upmendex.test, tests/rule[34].ist:
+ Add new tests.
+
+2021-08-23 TANAKA Takuji <ttk@t-lab.opal.ne.jp>
+
+ * tests/upmendex.test, tests/rule[12].ist, tests/rule.{idx,ind}:
+ Add new tests for the keyword "icu_rules" in style file.
+
2021-08-08 TANAKA Takuji <ttk@t-lab.opal.ne.jp>
* version 0.58 Beta version.
@@ -90,7 +195,7 @@
2020-05-01 TANAKA Takuji <ttk@t-lab.opal.ne.jp>
- * main.c, version.h, Makefile.{am.in}: Add bug report addresses
+ * main.c, version.h, Makefile.am: Add bug report addresses
on command line help.
* convert.c: Tune buffer sizes to avoid tool warnings.
diff --git a/indexing/upmendex/source/Makefile.am b/indexing/upmendex/source/Makefile.am
index 94bea187b3..451f0df1eb 100644
--- a/indexing/upmendex/source/Makefile.am
+++ b/indexing/upmendex/source/Makefile.am
@@ -1,4 +1,4 @@
-## $Id$
+## $Id: Makefile.am 60856 2021-10-24 05:44:35Z takuji $
## Makefile.am for the TeX Live subdirectory texk/upmendex/
##
## Copyright 2015-2019 Karl Berry <tex-live@tug.org>
@@ -19,7 +19,6 @@ upmendex_SOURCES = \
kana.h \
kp.c \
kp.h \
- ktable.h \
main.c \
mendex.h \
version.h \
diff --git a/indexing/upmendex/source/Makefile.in b/indexing/upmendex/source/Makefile.in
index 11cff850bc..4ecaadee97 100644
--- a/indexing/upmendex/source/Makefile.in
+++ b/indexing/upmendex/source/Makefile.in
@@ -1,4 +1,4 @@
-# Makefile.in generated by automake 1.16.4 from Makefile.am.
+# Makefile.in generated by automake 1.16.5 from Makefile.am.
# @configure_input@
# Copyright (C) 1994-2021 Free Software Foundation, Inc.
@@ -593,7 +593,6 @@ upmendex_SOURCES = \
kana.h \
kp.c \
kp.h \
- ktable.h \
main.c \
mendex.h \
version.h \
diff --git a/indexing/upmendex/source/aclocal.m4 b/indexing/upmendex/source/aclocal.m4
index 284ca6d457..8d45a4e2bc 100644
--- a/indexing/upmendex/source/aclocal.m4
+++ b/indexing/upmendex/source/aclocal.m4
@@ -1,4 +1,4 @@
-# generated automatically by aclocal 1.16.4 -*- Autoconf -*-
+# generated automatically by aclocal 1.16.5 -*- Autoconf -*-
# Copyright (C) 1996-2021 Free Software Foundation, Inc.
@@ -35,7 +35,7 @@ AC_DEFUN([AM_AUTOMAKE_VERSION],
[am__api_version='1.16'
dnl Some users find AM_AUTOMAKE_VERSION and mistake it for a way to
dnl require some minimum version. Point them to the right macro.
-m4_if([$1], [1.16.4], [],
+m4_if([$1], [1.16.5], [],
[AC_FATAL([Do not call $0, use AM_INIT_AUTOMAKE([$1]).])])dnl
])
@@ -51,7 +51,7 @@ m4_define([_AM_AUTOCONF_VERSION], [])
# Call AM_AUTOMAKE_VERSION and AM_AUTOMAKE_VERSION so they can be traced.
# This function is AC_REQUIREd by AM_INIT_AUTOMAKE.
AC_DEFUN([AM_SET_CURRENT_AUTOMAKE_VERSION],
-[AM_AUTOMAKE_VERSION([1.16.4])dnl
+[AM_AUTOMAKE_VERSION([1.16.5])dnl
m4_ifndef([AC_AUTOCONF_VERSION],
[m4_copy([m4_PACKAGE_VERSION], [AC_AUTOCONF_VERSION])])dnl
_AM_AUTOCONF_VERSION(m4_defn([AC_AUTOCONF_VERSION]))])
@@ -428,6 +428,10 @@ m4_defn([AC_PROG_CC])
# release and drop the old call support.
AC_DEFUN([AM_INIT_AUTOMAKE],
[AC_PREREQ([2.65])dnl
+m4_ifdef([_$0_ALREADY_INIT],
+ [m4_fatal([$0 expanded multiple times
+]m4_defn([_$0_ALREADY_INIT]))],
+ [m4_define([_$0_ALREADY_INIT], m4_expansion_stack)])dnl
dnl Autoconf wants to disallow AM_ names. We explicitly allow
dnl the ones we care about.
m4_pattern_allow([^AM_[A-Z]+FLAGS$])dnl
diff --git a/indexing/upmendex/source/configure b/indexing/upmendex/source/configure
index f1ffea6fbb..e5aa682d08 100755
--- a/indexing/upmendex/source/configure
+++ b/indexing/upmendex/source/configure
@@ -1,6 +1,6 @@
#! /bin/sh
# Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.71 for upmendex (TeX Live) 0.58.
+# Generated by GNU Autoconf 2.71 for upmendex (TeX Live) 1.00.
#
#
# Copyright (C) 1992-1996, 1998-2017, 2020-2021 Free Software Foundation,
@@ -626,8 +626,8 @@ MAKEFLAGS=
# Identity of this package.
PACKAGE_NAME='upmendex (TeX Live)'
PACKAGE_TARNAME='upmendex--tex-live-'
-PACKAGE_VERSION='0.58'
-PACKAGE_STRING='upmendex (TeX Live) 0.58'
+PACKAGE_VERSION='1.00'
+PACKAGE_STRING='upmendex (TeX Live) 1.00'
PACKAGE_BUGREPORT=''
PACKAGE_URL=''
@@ -1390,7 +1390,7 @@ if test "$ac_init_help" = "long"; then
# Omit some internal or obsolete options to make the list less imposing.
# This message is too long to be a string in the A/UX 3.1 sh.
cat <<_ACEOF
-\`configure' configures upmendex (TeX Live) 0.58 to adapt to many kinds of systems.
+\`configure' configures upmendex (TeX Live) 1.00 to adapt to many kinds of systems.
Usage: $0 [OPTION]... [VAR=VALUE]...
@@ -1462,7 +1462,7 @@ fi
if test -n "$ac_init_help"; then
case $ac_init_help in
- short | recursive ) echo "Configuration of upmendex (TeX Live) 0.58:";;
+ short | recursive ) echo "Configuration of upmendex (TeX Live) 1.00:";;
esac
cat <<\_ACEOF
@@ -1587,7 +1587,7 @@ fi
test -n "$ac_init_help" && exit $ac_status
if $ac_init_version; then
cat <<\_ACEOF
-upmendex (TeX Live) configure 0.58
+upmendex (TeX Live) configure 1.00
generated by GNU Autoconf 2.71
Copyright (C) 2021 Free Software Foundation, Inc.
@@ -2268,7 +2268,7 @@ cat >config.log <<_ACEOF
This file contains any messages produced by compilers while
running configure, to aid debugging if configure makes a mistake.
-It was created by upmendex (TeX Live) $as_me 0.58, which was
+It was created by upmendex (TeX Live) $as_me 1.00, which was
generated by GNU Autoconf 2.71. Invocation command line was
$ $0$ac_configure_args_raw
@@ -8806,7 +8806,7 @@ fi
# Define the identity of the package.
PACKAGE='upmendex--tex-live-'
- VERSION='0.58'
+ VERSION='1.00'
# Some tools Automake needs.
@@ -18942,7 +18942,7 @@ Usage: $0 [OPTIONS]
Report bugs to <bug-libtool@gnu.org>."
lt_cl_version="\
-upmendex (TeX Live) config.lt 0.58
+upmendex (TeX Live) config.lt 1.00
configured by $0, generated by GNU Autoconf 2.71.
Copyright (C) 2011 Free Software Foundation, Inc.
@@ -21114,7 +21114,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
# report actual input values of CONFIG_FILES etc. instead of their
# values after options handling.
ac_log="
-This file was extended by upmendex (TeX Live) $as_me 0.58, which was
+This file was extended by upmendex (TeX Live) $as_me 1.00, which was
generated by GNU Autoconf 2.71. Invocation command line was
CONFIG_FILES = $CONFIG_FILES
@@ -21182,7 +21182,7 @@ ac_cs_config_escaped=`printf "%s\n" "$ac_cs_config" | sed "s/^ //; s/'/'\\\\\\\\
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
ac_cs_config='$ac_cs_config_escaped'
ac_cs_version="\\
-upmendex (TeX Live) config.status 0.58
+upmendex (TeX Live) config.status 1.00
configured by $0, generated by GNU Autoconf 2.71,
with options \\"\$ac_cs_config\\"
diff --git a/indexing/upmendex/source/configure.ac b/indexing/upmendex/source/configure.ac
index 7af5a44b8c..b21873a1ed 100644
--- a/indexing/upmendex/source/configure.ac
+++ b/indexing/upmendex/source/configure.ac
@@ -1,4 +1,4 @@
-dnl $Id$
+dnl $Id: configure.ac 61096 2021-11-20 09:29:33Z takuji $
dnl Process this file with autoconf to produce a configure script.
dnl
dnl Copyright 2015-2021 Karl Berry <tex-live@tug.org>
@@ -8,8 +8,8 @@ dnl This file is free software; the copyright holder
dnl gives unlimited permission to copy and/or distribute it,
dnl with or without modifications, as long as this notice is preserved.
dnl
-AC_INIT([upmendex (TeX Live)], [0.58])
-AC_PREREQ([2.63])
+AC_INIT([upmendex (TeX Live)],[1.00])
+AC_PREREQ([2.71])
AC_CONFIG_SRCDIR([main.c])
AC_CONFIG_AUX_DIR([../../build-aux])
AC_CONFIG_MACRO_DIR([../../m4])
diff --git a/indexing/upmendex/source/convert.c b/indexing/upmendex/source/convert.c
index 74d32d7352..a7cae61038 100644
--- a/indexing/upmendex/source/convert.c
+++ b/indexing/upmendex/source/convert.c
@@ -6,7 +6,6 @@
#include "qsort.h"
#include "exkana.h"
-#include "ktable.h"
#include "exvar.h"
#include "kp.h"
@@ -25,8 +24,6 @@ static int dlines=0,elines=0;
static int dicvalread(const char *filename, struct dictionary *dicval, int line);
-#define is_alpha_numeric(a) (((a)>=ALPHATOP && (a)<=ALPHAEND) || (a)==YENSIGN)
-
UChar * u_xstrdup (const UChar *string)
{
return u_strcpy(xmalloc((u_strlen(string)+1)*sizeof(UChar)), string);
@@ -210,76 +207,66 @@ int convert(UChar *buff1, UChar *buff2)
i++;
}
- else if (buff1[i]<0x80) {
+ else if (buff1[i]<0x20 && buff1[i]!='\t') { /* ignore control characters */
+ i++;
+ }
+
+ else if (buff1[i]<0x7F) {
buff2[j]=buff1[i];
i++;
j++;
}
+ else if (buff1[i]<0xA0) { /* ignore control characters */
+ i++;
+ }
+
else if (is_latin(buff3)||is_cyrillic(buff3)||is_greek(buff3)
||is_jpn_kana(buff3)||is_kor_hngl(buff3)||is_zhuyin(buff3)
- ||is_numeric(buff3)||is_devanagari(buff3)||is_thai(buff3)
- ||is_comb_diacritical_mark(buff3)) {
+ ||is_numeric(buff3)==1||is_type_symbol(buff3)==1
+ ||is_devanagari(buff3)||is_thai(buff3)||is_arabic(buff3)||is_hebrew(buff3)
+ ||is_type_mark_or_punct(buff3)) {
buff2[j]=buff3[0];
if (wclen==2) buff2[j+1]=buff3[1];
i+=wclen;
j+=wclen;
}
- else if (buff1[i]>=0x80) {
- if (is_alpha_numeric(buff3[0])) {
-/* alpha-numeric,symbols */
- for (k=0;k<u_strlen(symboltable);k++) {
- if (buff3[0]==symboltable[k]) {
- buff2[j]=k+0x20;
- if ((buff2[j]>='a')&&(buff2[j]<='z')) buff2[j]-=32;
- i++;
- j+=wclen;
- break;
- }
- }
- if (k==u_strlen(symboltable)) {
- i++;
- buff2[j++]=buff3[0];
+ else {
+ for (k=0;k<dlines;k++) {
+/* dictionary table */
+ if (u_strncmp(dictable[k].dic[0],&buff1[i],u_strlen(dictable[k].dic[0]))==0) {
+ u_strncpy(&buff2[j],dictable[k].dic[1],u_strlen(dictable[k].dic[1]));
+ i+=u_strlen(dictable[k].dic[0]);
+ j+=u_strlen(dictable[k].dic[1]);
+ break;
}
}
-
- else {
- for (k=0;k<dlines;k++) {
-/* dictionary table */
- if (u_strncmp(dictable[k].dic[0],&buff1[i],u_strlen(dictable[k].dic[0]))==0) {
- u_strncpy(&buff2[j],dictable[k].dic[1],u_strlen(dictable[k].dic[1]));
+ if ((k==dlines)&&(elines!=0)) {
+/* environment dictionary table */
+ for (k=0;k<elines;k++) {
+ if (u_strncmp(envdic[k].dic[0],&buff1[i],u_strlen(envdic[k].dic[0]))==0) {
+ u_strncpy(&buff2[j],envdic[k].dic[1],u_strlen(envdic[k].dic[1]));
i+=u_strlen(dictable[k].dic[0]);
- j+=u_strlen(dictable[k].dic[1]);
+ j+=u_strlen(envdic[k].dic[1]);
break;
}
}
- if ((k==dlines)&&(elines!=0)) {
-/* environment dictionary table */
- for (k=0;k<elines;k++) {
- if (u_strncmp(envdic[k].dic[0],&buff1[i],u_strlen(envdic[k].dic[0]))==0) {
- u_strncpy(&buff2[j],envdic[k].dic[1],u_strlen(envdic[k].dic[1]));
- i+=u_strlen(dictable[k].dic[0]);
- j+=u_strlen(envdic[k].dic[1]);
- break;
- }
- }
- }
- if (((k==dlines)&&(elines==0))||((k==elines)&&(elines!=0))) {
- if (is_hanzi(buff3) || force==1) {
+ }
+ if (((k==dlines)&&(elines==0))||((k==elines)&&(elines!=0))) {
+ if (is_hanzi(buff3) || is_numeric(buff3) || is_type_symbol(buff3) || force==1) {
/* forced convert */
- buff2[j]=buff3[0];
- if (wclen==2) buff2[j+1]=buff3[1];
- i+=wclen;
- j+=wclen;
- }
- else {
- widechar_to_multibyte(errbuff,BUFFERLEN2,&buff1[i]);
- snprintf(errbuff2,BUFFERLEN3,"\nError: %s is no entry in dictionary file ",errbuff);
- fputs(errbuff2,efp);
- if (efp!=stderr) fputs(errbuff2,stderr);
- return -1;
- }
+ buff2[j]=buff3[0];
+ if (wclen==2) buff2[j+1]=buff3[1];
+ i+=wclen;
+ j+=wclen;
+ }
+ else {
+ widechar_to_multibyte(errbuff,BUFFERLEN2,&buff1[i]);
+ snprintf(errbuff2,BUFFERLEN3,"\nError: %s is no entry in dictionary file ",errbuff);
+ fputs(errbuff2,efp);
+ if (efp!=stderr) fputs(errbuff2,stderr);
+ return -1;
}
}
}
diff --git a/indexing/upmendex/source/exkana.h b/indexing/upmendex/source/exkana.h
index aeb5a18d82..9e4ba1d197 100644
--- a/indexing/upmendex/source/exkana.h
+++ b/indexing/upmendex/source/exkana.h
@@ -4,7 +4,6 @@ extern UChar AIUEO[];
extern UChar *aiueo;
extern UChar kanatable[];
extern UChar extkanatable[];
-extern UChar circkanatable[];
#define SPACE 0x3000 /* 全角スペース */
#define ALPHATOP 0xff01 /* ! */
@@ -18,8 +17,19 @@ extern UChar circkanatable[];
#define EXKANAEND 0x31FF /* ㇿ */
#define CRKANATOP 0x32D0 /* ㋐ */
#define CRKANAEND 0x32FE /* ㋾ */
+#define HANKANATOP 0xFF71 /* ア */
+#define HANKANAEND 0xFF9D /* ン */
+#define HANKANAWO 0xFF66 /* ヲ */
+#define HANKANATU 0xFF6F /* ッ */
+#define SQKANATOP 0x3300 /* ㌀ */
+#define SQKANAEND 0x3357 /* ㍗ */
+#define CRLATNTOP 0x24B6 /* Ⓐ */
+#define CRLATNEND 0x24E9 /* ⓩ */
#define is_katakana(a) ((a)>=KATATOP && (a)<=KATAEND)
#define is_hiragana(a) ((a)>=HIRATOP && (a)<=HIRAEND)
#define is_extkana(a) ((a)>=EXKANATOP && (a)<=EXKANAEND)
#define is_circkana(a) ((a)>=CRKANATOP && (a)<=CRKANAEND)
+#define is_hankana(a) ((a)>=HANKANATOP && (a)<=HANKANAEND || (a)>=HANKANAWO && (a)<=HANKANATU)
+#define is_sqkana(a) ((a)>=SQKANATOP && (a)<=SQKANAEND)
+#define is_circlatin(a) ((a)>=CRLATNTOP && (a)<=CRLATNEND)
diff --git a/indexing/upmendex/source/exvar.h b/indexing/upmendex/source/exvar.h
index 7b061c6655..790f327e27 100644
--- a/indexing/upmendex/source/exvar.h
+++ b/indexing/upmendex/source/exvar.h
@@ -2,7 +2,7 @@ extern FILE *efp;
extern int lines,idxcount,acc,reject;
extern int prange,fsti,lorder,bcomp,force,fpage,gflg,verb,debug;
-extern int warn,scount,pattr[3];
+extern int warn,scount,pattr[];
extern struct index *ind;
@@ -26,7 +26,7 @@ extern int line_max;
extern char indent_space[];
extern int indent_length;
extern int priority;
-extern char symbol[];
+extern char symhead[],numhead[];
extern char symhead_positive[],symhead_negative[];
extern char numhead_positive[],numhead_negative[];
extern int symbol_flag;
@@ -35,6 +35,7 @@ extern UChar atama[],hangul_head[],hanzi_head[],kana_head[];
extern UChar devanagari_head[],thai_head[];
extern char page_compositor[],page_precedence[];
extern char character_order[];
+extern char script_preamble[][STYBUFSIZE],script_postamble[][STYBUFSIZE];
extern char icu_locale[],icu_rules[];
extern int icu_attributes[];
diff --git a/indexing/upmendex/source/fread.c b/indexing/upmendex/source/fread.c
index a336022e11..66d3e0b144 100644
--- a/indexing/upmendex/source/fread.c
+++ b/indexing/upmendex/source/fread.c
@@ -344,7 +344,7 @@ LOOP:
copy_multibyte_char(buff, table, &j, &k);
}
- table[k]='\0';
+ table[k]='\0';
for (k=0;k<=ind[l].num;k++) {
if (strcmp(ind[l].p[k].page,table)==0) {
@@ -356,9 +356,9 @@ LOOP:
ind[l].num++;
if (!((ind[l].num)%16)) ind[l].p=(struct page *)xrealloc(ind[l].p,sizeof(struct page)*((int)((ind[l].num)/16)+1)*16);
- ind[l].p[ind[l].num].page=xstrdup(table);
+ ind[l].p[ind[l].num].page=xstrdup(table);
- ind[l].p[ind[l].num].enc=xstrdup(estr);
+ ind[l].p[ind[l].num].enc=xstrdup(estr);
chkpageattr(&ind[l].p[ind[l].num]);
}
}
@@ -388,15 +388,15 @@ LOOP:
nest++;
if (buff[j]==arg_close) {
if (nest==0) {
- table[k]='\0';
- ind[i].p[0].page=xstrdup(table);
+ table[k]='\0';
+ ind[i].p[0].page=xstrdup(table);
break;
}
else nest--;
}
copy_multibyte_char(buff, table, &j, &k);
}
- ind[l].p[0].enc=xstrdup(estr);
+ ind[l].p[0].enc=xstrdup(estr);
chkpageattr(&ind[i].p[0]);
}
}
@@ -493,25 +493,54 @@ int widechar_to_multibyte(char *mbstr, int32_t size, UChar *wcstr)
static void chkpageattr(struct page *p)
{
- int i,j,cc=0;
+ int i,j,cc=0,cnt,pplen,pclen;
+ char buff[16],*pcpos,*page0;
+ pplen=strlen(page_precedence);
+ pclen=strlen(page_compositor);
for (i=0;i<strlen(p->page);i++) {
- if (strncmp(page_compositor,&p->page[i],strlen(page_compositor))==0) {
+ page0=&p->page[i];
+ if (strncmp(page_compositor,page0,pclen)==0) {
p->attr[cc]=pattr[cc];
cc++;
- i+=strlen(page_compositor)-1;
+ i+=pclen-1;
+ if (cc>=PAGE_COMPOSIT_DEPTH) {
+ if (pclen>0)
+ verb_printf(efp, "\nToo many fields of page number \"%s\".\n", p->page);
+ else
+ verb_printf(efp, "\nIllegular page_comositor specification.\n");
+ exit(253);
+ }
}
else {
-ATTRLOOP:
- if (!((p->page[i]>='0' && p->page[i]<='9') || (p->page[i]>='A' && p->page[i]<='Z') || (p->page[i]>='a' && p->page[i]<='z'))) {
+ cnt=0;
+ if (!((*page0>='0' && *page0<='9') || (*page0>='A' && *page0<='Z') || (*page0>='a' && *page0<='z'))) {
p->attr[cc]= -1;
if (cc<2) p->attr[++cc]= -1;
return;
}
+ pcpos=strstr(page0,page_compositor);
+ j=pcpos ? pcpos-page0 : strlen(page0);
+ if (j>15) {
+ verb_printf(efp, "\nToo long page number string \"%s\".\n", page0);
+ exit(253);
+ }
+ strncpy(buff,page0,j);
+ buff[j]='\0';
+ATTRLOOP:
+ cnt++;
+ if (cnt>pplen) {
+ verb_printf(efp, "\nFailed to find page type for page \"%s\" in page_precedence specification (%s).\n",
+ page0, page_precedence);
+ exit(253);
+ }
+
switch(page_precedence[pattr[cc]]) {
case 'r':
- if (strchr("ivxlcdm",p->page[i])==NULL) {
- if (pattr[cc]<strlen(page_precedence)-1)
+ if (strchr("ivxlcdm",*page0)==NULL ||
+ (strchr("lcdm",*page0) && strchr(page_precedence,'a') && strlen(buff)==1 && pcpos)) {
+ /* heuristic detection as alphabet since L=50, C=100, D=100, M=1000 are quite large */
+ if (pattr[cc]<pplen-1)
pattr[cc]++;
else pattr[cc]=0;
for (j=cc+1;j<3;j++) pattr[j]=0;
@@ -519,8 +548,10 @@ ATTRLOOP:
}
break;
case 'R':
- if (strchr("IVXLCDM",p->page[i])==NULL) {
- if (pattr[cc]<strlen(page_precedence)-1)
+ if (strchr("IVXLCDM",*page0)==NULL ||
+ (strchr("LCDM",*page0) && strchr(page_precedence,'A') && strlen(buff)==1 && pcpos)) {
+ /* heuristic detection as alphabet since L=50, C=100, D=100, M=1000 are quite large */
+ if (pattr[cc]<pplen-1)
pattr[cc]++;
else pattr[cc]=0;
for (j=cc+1;j<3;j++) pattr[j]=0;
@@ -528,8 +559,8 @@ ATTRLOOP:
}
break;
case 'n':
- if (p->page[i]<'0' || p->page[i]>'9') {
- if (pattr[cc]<strlen(page_precedence)-1)
+ if (*page0<'0' || *page0>'9') {
+ if (pattr[cc]<pplen-1)
pattr[cc]++;
else pattr[cc]=0;
for (j=cc+1;j<3;j++) pattr[j]=0;
@@ -537,8 +568,10 @@ ATTRLOOP:
}
break;
case 'a':
- if (p->page[i]<'a' || p->page[i]>'z') {
- if (pattr[cc]<strlen(page_precedence)-1)
+ if (*page0<'a' || *page0>'z' || strlen(buff)>1 ||
+ (strchr("ivx",*page0) && strchr(page_precedence,'r') && !pcpos)) {
+ /* heuristic detection as roman number since I=1, V=5, X=10 are quite small */
+ if (pattr[cc]<pplen-1)
pattr[cc]++;
else pattr[cc]=0;
for (j=cc+1;j<3;j++) pattr[j]=0;
@@ -546,8 +579,10 @@ ATTRLOOP:
}
break;
case 'A':
- if (p->page[i]<'A' || p->page[i]>'Z') {
- if (pattr[cc]<strlen(page_precedence)-1)
+ if (*page0<'A' || *page0>'Z' || strlen(buff)>1 ||
+ (strchr("IVX",*page0) && strchr(page_precedence,'R') && !pcpos)) {
+ /* heuristic detection as roman number since I=1, V=5, X=10 are quite small */
+ if (pattr[cc]<pplen-1)
pattr[cc]++;
else pattr[cc]=0;
for (j=cc+1;j<3;j++) pattr[j]=0;
@@ -555,7 +590,9 @@ ATTRLOOP:
}
break;
default:
- break;
+ verb_printf(efp, "\nUnknown page type '%c' in page_precedence specification (%s).\n",
+ page_precedence[pattr[cc]], page_precedence);
+ exit(253);
}
}
}
diff --git a/indexing/upmendex/source/fwrite.c b/indexing/upmendex/source/fwrite.c
index d1685ede1c..9ce382589e 100644
--- a/indexing/upmendex/source/fwrite.c
+++ b/indexing/upmendex/source/fwrite.c
@@ -13,12 +13,12 @@ int line_length=0;
static void printpage(struct index *ind, FILE *fp, int num, char *lbuff);
static int range_check(struct index ind, int count, char *lbuff);
-static void linecheck(char *lbuff, char *tmpbuff);
+static void linecheck(char *lbuff, char *tmpbuff, int force);
static void crcheck(char *lbuff, FILE *fp);
static void index_normalize(UChar *istr, UChar *ini, int *chset);
static int initial_cmp_char(UChar *ini, UChar ch);
static int init_hanzi_header(void);
-static const UNormalizer2* unormalizer_NFD;
+static const UNormalizer2 *unormalizer_NFD, *unormalizer_NFKD;
static int turkish_i;
#define M_NONE 0
@@ -93,7 +93,7 @@ static void fprint_uchar(FILE *fp, const UChar *a, const int mode, const int len
} else
olen=wclen;
if (olen>INITIALLENGTH) {
- warn_printf(efp, "\nWarning, Too long (%d) header.\n", olen);
+ warn_printf(efp, "\nWarning: Too long (%d) header.\n", olen);
wclen=INITIALLENGTH;
} else
wclen=olen;
@@ -128,7 +128,7 @@ void warn_printf(FILE *fp, const char *format, ...)
vsnprintf(print_buff, sizeof print_buff, format, argptr);
va_end(argptr);
- warn++;
+ warn++;
fputs(print_buff, stderr);
if (fp!=stderr) fputs(print_buff, fp);
}
@@ -146,11 +146,27 @@ void verb_printf(FILE *fp, const char *format, ...)
if (fp!=stderr) fputs(print_buff, fp);
}
+static int pnumconv2(struct page *p)
+{
+ int j,k,cc,pclen;
+
+ pclen=strlen(page_compositor);
+ for (j=k=cc=0;j<strlen(p->page);j++) {
+ if (strncmp(p->page+j,page_compositor,pclen)==0) {
+ j+=pclen;
+ k=j;
+ cc++;
+ continue;
+ }
+ }
+ return pnumconv(p->page+k,p->attr[cc]);
+}
+
/* write ind file */
void indwrite(char *filename, struct index *ind, int pagenum)
{
- int i,j,hpoint=0,tpoint=0,ipoint=0,jpoint=0;
+ int i,j,hpoint=0,tpoint=0,ipoint=0,jpoint=0,block_open=0;
char lbuff[BUFFERLEN],obuff[BUFFERLEN];
UChar datama[256],initial[INITIALLENGTH],initial_prev[INITIALLENGTH];
int chset,chset_prev;
@@ -173,12 +189,34 @@ void indwrite(char *filename, struct index *ind, int pagenum)
fprintf(fp,"%s%d%s",setpage_prefix,pagenum,setpage_suffix);
}
perr=U_ZERO_ERROR;
- unormalizer_NFD=unorm2_getInstance(NULL, "nfc", UNORM2_DECOMPOSE, &perr);
+ unormalizer_NFD =unorm2_getInstance(NULL, "nfc", UNORM2_DECOMPOSE, &perr);
+ unormalizer_NFKD=unorm2_getInstance(NULL, "nfkc", UNORM2_DECOMPOSE, &perr);
+
+ if (strlen(symhead)==0) {
+ if (lethead_flag>0) {
+ strcpy(symhead, symhead_positive);
+ }
+ else if (lethead_flag<0) {
+ strcpy(symhead, symhead_negative);
+ }
+ }
+ {
+ if (lethead_flag>0) {
+ strcpy(numhead, numhead_positive);
+ }
+ else if (lethead_flag<0) {
+ strcpy(numhead, numhead_negative);
+ }
+ }
for (i=line_length=0;i<lines;i++) {
index_normalize(ind[i].dic[0], initial, &chset);
if (i==0) {
- if ((CH_LATIN<=chset&&chset<=CH_GREEK) || chset==CH_HANZI) {
+ if (is_any_script(chset) && strlen(script_preamble[chset])) {
+ fputs(script_preamble[chset],fp);
+ block_open=chset;
+ }
+ if ((CH_LATIN<=chset&&chset<=CH_GREEK) || chset==CH_HANZI || (CH_ARABIC<=chset&&chset<=CH_HEBREW)) {
if (lethead_flag!=0) {
fputs(lethead_prefix,fp);
fprint_uchar(fp,initial,lethead_flag,-1);
@@ -280,16 +318,11 @@ void indwrite(char *filename, struct index *ind, int pagenum)
}
}
else {
- if (lethead_flag!=0) {
- if (symbol_flag && strlen(symbol)) {
- fprintf(fp,"%s%s%s",lethead_prefix,symbol,lethead_suffix);
- }
- else if (lethead_flag>0) {
- fprintf(fp,"%s%s%s",lethead_prefix,symhead_positive,lethead_suffix);
- }
- else if (lethead_flag<0) {
- fprintf(fp,"%s%s%s",lethead_prefix,symhead_negative,lethead_suffix);
- }
+ if (lethead_flag!=0 && symbol_flag==2 && chset==CH_NUMERIC) {
+ fprintf(fp,"%s%s%s",lethead_prefix,numhead,lethead_suffix);
+ }
+ if (lethead_flag!=0 && (symbol_flag==1 || (symbol_flag==2 && chset!=CH_NUMERIC))) {
+ fprintf(fp,"%s%s%s",lethead_prefix,symhead,lethead_suffix);
}
widechar_to_multibyte(obuff,BUFFERLEN,ind[i].idx[0]);
SPRINTF(lbuff,"%s%s",item_0,obuff);
@@ -323,7 +356,19 @@ void indwrite(char *filename, struct index *ind, int pagenum)
}
else {
index_normalize(ind[i-1].dic[0], initial_prev, &chset_prev);
- if ((CH_LATIN<=chset&&chset<=CH_GREEK) || chset==CH_HANZI) {
+ if (chset!=chset_prev && is_any_script(chset_prev) && block_open) {
+ if (strlen(script_postamble[chset_prev])) {
+ fputs(script_postamble[chset_prev],fp);
+ }
+ block_open=0;
+ }
+ if (chset!=chset_prev && is_any_script(chset)) {
+ if (strlen(script_preamble[chset])) {
+ fputs(script_preamble[chset],fp);
+ block_open=chset;
+ }
+ }
+ if ((CH_LATIN<=chset&&chset<=CH_GREEK) || chset==CH_HANZI || (CH_ARABIC<=chset&&chset<=CH_HEBREW)) {
if (chset!=chset_prev || ss_comp(initial,initial_prev)) {
fputs(group_skip,fp);
if (lethead_flag!=0) {
@@ -398,18 +443,15 @@ void indwrite(char *filename, struct index *ind, int pagenum)
}
}
else {
- if (CH_LATIN<=chset_prev&&chset_prev<=CH_THAI){
- fputs(group_skip,fp);
- if (lethead_flag!=0 && symbol_flag) {
- if (strlen(symbol)) {
- fprintf(fp,"%s%s%s",lethead_prefix,symbol,lethead_suffix);
- }
- else if (lethead_flag>0) {
- fprintf(fp,"%s%s%s",lethead_prefix,symhead_positive,lethead_suffix);
- }
- else if (lethead_flag<0) {
- fprintf(fp,"%s%s%s",lethead_prefix,symhead_negative,lethead_suffix);
- }
+ if (chset_prev!=chset) {
+ if (is_any_script(chset_prev) || symbol_flag==2)
+ fputs(group_skip,fp);
+ if (lethead_flag!=0 && symbol_flag==2 && chset==CH_NUMERIC) {
+ fprintf(fp,"%s%s%s",lethead_prefix,numhead,lethead_suffix);
+ }
+ if (lethead_flag!=0 && (symbol_flag==1 && is_any_script(chset_prev) ||
+ symbol_flag==2 && chset!=CH_NUMERIC) ) {
+ fprintf(fp,"%s%s%s",lethead_prefix,symhead,lethead_suffix);
}
}
}
@@ -479,6 +521,9 @@ void indwrite(char *filename, struct index *ind, int pagenum)
printpage(ind,fp,i,lbuff);
}
}
+ if (is_any_script(chset) && strlen(script_postamble[chset]) && block_open) {
+ fputs(script_postamble[chset],fp);
+ }
fputs(postamble,fp);
if (fp!=stdout) fclose(fp);
@@ -498,7 +543,9 @@ static void printpage(struct index *ind, FILE *fp, int num, char *lbuff)
for(j=0;j<ind[num].num;j++) {
cc=range_check(ind[num],j,lbuff);
if (cc>j) {
- if (pnumconv(ind[num].p[j].page,ind[num].p[j].attr[0])==pnumconv(ind[num].p[cc].page,ind[num].p[cc].attr[0])) {
+ int epage = pnumconv2(&ind[num].p[cc]);
+ int bpage = pnumconv2(&ind[num].p[j]);
+ if (epage==bpage) {
j=cc-1;
continue;
}
@@ -509,20 +556,18 @@ static void printpage(struct index *ind, FILE *fp, int num, char *lbuff)
if (strlen(ind[num].p[j].enc)>0) {
SPRINTF(buff,"%s%s%s",encap_prefix,ind[num].p[j].enc,encap_infix);
}
- if (strlen(suffix_3p)>0 && (pnumconv(ind[num].p[cc].page,ind[num].p[cc].attr[0])-pnumconv(ind[num].p[j].page,ind[num].p[j].attr[0]))==2) {
- SAPPENDF(buff,"%s",ind[num].p[j].page);
+ /* print beginning of range */
+ SAPPENDF(buff,"%s",ind[num].p[j].page);
+ if (strlen(suffix_3p)>0 && epage-bpage==2) {
SAPPENDF(buff,"%s",suffix_3p);
}
- else if (strlen(suffix_mp)>0 && (pnumconv(ind[num].p[cc].page,ind[num].p[cc].attr[0])-pnumconv(ind[num].p[j].page,ind[num].p[j].attr[0]))>=2) {
- SAPPENDF(buff,"%s",ind[num].p[j].page);
+ else if (strlen(suffix_mp)>0 && epage-bpage>=2) {
SAPPENDF(buff,"%s",suffix_mp);
}
- else if (strlen(suffix_2p)>0 && (pnumconv(ind[num].p[cc].page,ind[num].p[cc].attr[0])-pnumconv(ind[num].p[j].page,ind[num].p[j].attr[0]))==1) {
- SAPPENDF(buff,"%s",ind[num].p[j].page);
+ else if (strlen(suffix_2p)>0 && epage-bpage==1) {
SAPPENDF(buff,"%s",suffix_2p);
}
else {
- SAPPENDF(buff,"%s",ind[num].p[j].page);
SAPPENDF(buff,"%s",delim_r);
SAPPENDF(buff,"%s",ind[num].p[cc].page);
}
@@ -531,14 +576,14 @@ static void printpage(struct index *ind, FILE *fp, int num, char *lbuff)
if (strlen(ind[num].p[j].enc)>0) {
SAPPENDF(tmpbuff,"%s",encap_suffix);
}
- linecheck(lbuff,tmpbuff);
+ linecheck(lbuff,tmpbuff, FALSE);
j=cc;
if (j==ind[num].num) {
goto PRINT;
}
else {
SAPPENDF(tmpbuff,"%s",delim_n);
- linecheck(lbuff,tmpbuff);
+ linecheck(lbuff,tmpbuff, TRUE);
}
}
else if (strlen(ind[num].p[j].enc)>0) {
@@ -560,19 +605,19 @@ static void printpage(struct index *ind, FILE *fp, int num, char *lbuff)
SAPPENDF(tmpbuff,"%s",ind[num].p[j].page);
SAPPENDF(tmpbuff,"%s",encap_suffix);
SAPPENDF(tmpbuff,"%s",delim_n);
- linecheck(lbuff,tmpbuff);
+ linecheck(lbuff,tmpbuff, FALSE);
}
else {
SAPPENDF(tmpbuff,"%s",ind[num].p[j].page);
SAPPENDF(tmpbuff,"%s",delim_n);
- linecheck(lbuff,tmpbuff);
+ linecheck(lbuff,tmpbuff, FALSE);
}
}
else {
/* no encap */
SAPPENDF(tmpbuff,"%s",ind[num].p[j].page);
SAPPENDF(tmpbuff,"%s",delim_n);
- linecheck(lbuff,tmpbuff);
+ linecheck(lbuff,tmpbuff, FALSE);
}
}
@@ -606,7 +651,7 @@ static void printpage(struct index *ind, FILE *fp, int num, char *lbuff)
else {
SAPPENDF(tmpbuff,"%s",ind[num].p[j].page);
}
- linecheck(lbuff,tmpbuff);
+ linecheck(lbuff,tmpbuff, FALSE);
PRINT:
fputs(lbuff,fp);
@@ -652,7 +697,7 @@ static int range_check(struct index ind, int count, char *lbuff)
if (strlen(ind.p[j].enc)>0) {
SPRINTF(tmpbuff,"%s%s%s%s%s%s",encap_prefix,ind.p[j].enc,encap_infix
,ind.p[j].page,encap_suffix,delim_n);
- linecheck(lbuff,tmpbuff);
+ linecheck(lbuff,tmpbuff, FALSE);
}
}
}
@@ -682,8 +727,8 @@ static int range_check(struct index ind, int count, char *lbuff)
break;
}
}
- cc1=pnumconv(ind.p[i-1].page,ind.p[i-1].attr[0]);
- cc2=pnumconv(ind.p[count].page,ind.p[count].attr[0]);
+ cc1=pnumconv2(&ind.p[i-1]);
+ cc2=pnumconv2(&ind.p[count]);
if (cc1>=cc2+2 || (cc1>=cc2+1 && strlen(suffix_2p)) || force) {
return i-1;
}
@@ -691,9 +736,9 @@ static int range_check(struct index ind, int count, char *lbuff)
}
/* check line length */
-static void linecheck(char *lbuff, char *tmpbuff)
+static void linecheck(char *lbuff, char *tmpbuff, int force)
{
- if (line_length+strlen(tmpbuff)>line_max) {
+ if (line_length+strlen(tmpbuff)>line_max && !force) {
SAPPENDF(lbuff,"\n");
SAPPENDF(lbuff,"%s",indent_space);
SAPPENDF(lbuff,"%s",tmpbuff);
@@ -731,12 +776,20 @@ static void index_normalize(UChar *istr, UChar *ini, int *chset)
UChar32 c32;
UErrorCode perr;
UCollationResult order;
- static int hanzi_mode=0;
+ UCollationStrength strgth;
+ static int hanzi_mode=0, i_y_mode=0;
ch=istr[0];
*chset=charset(istr);
ini[1]=L'\0';
+ if (is_circkana(ch) || is_hankana(ch) || is_sqkana(ch) || is_circlatin(ch)) { /* ㋐㋑㋒.. アイウ.. ㌀㌁㌂.. */
+ src[0]=ch; src[1]=0x00;
+ perr=U_ZERO_ERROR;
+ unorm2_normalize(unormalizer_NFKD, src, 1, dest, 8, &perr);
+ if (U_SUCCESS(perr))
+ ch=dest[0];
+ }
if (is_hiragana(ch)) {
ch+=KATATOP-HIRATOP; /* hiragana -> katakana */
}
@@ -748,15 +801,15 @@ static void index_normalize(UChar *istr, UChar *ini, int *chset)
ini[0]=extkanatable[ch-EXKANATOP];
return;
}
- if (is_circkana(ch)) { /* ㋐㋑㋒㋓㋔ .. ㋻㋼㋽㋾ */
- ini[0]=circkanatable[ch-CRKANATOP];
- return;
- }
- else if (ch==0x309F) { ini[0]=0x3088; return; } /* HIRAGANA YORI -> よ */
+ if (ch==0x309F) { ini[0]=0x3088; return; } /* HIRAGANA YORI -> よ */
else if (ch==0x30FF) { ini[0]=0x3053; return; } /* KATAKANA KOTO -> こ */
else if (is_jpn_kana(istr)==2) {
c32=U16_GET_SUPPLEMENTARY(istr[0],istr[1]);
switch (c32) {
+ case 0x1F200: /* 🈀 */
+ ini[0]=0x307B; break; /* ほ */
+ case 0x1B000: /* 𛀀 */
+ ini[0]=0x3048; break; /* え */
case 0x1B150: case 0x1B164:
ini[0]=0x3090; break; /* ゐ */
case 0x1B151: case 0x1B165:
@@ -852,17 +905,29 @@ static void index_normalize(UChar *istr, UChar *ini, int *chset)
u_strcpy(ini,hz_index[lo-1].idx);
return;
}
- else if (is_devanagari(&ch)||is_thai(&ch)) {
- if (ch==0x929||0x931||0x934||(0x958<=ch&&ch<=0x95F)) {
+ else if (is_devanagari(&ch)||is_thai(&ch)||is_arabic(&ch)||is_hebrew(&ch)) {
+ if (ch==0x929||ch==0x931||ch==0x934||(0x958<=ch&&ch<=0x95F) /* Devanagary */
+ ||(0x622<=ch&&ch<=0x626)||ch==0x6C0||ch==0x6C2||ch==0x6D3 /* Arabic */
+ ||(0xFB50<=ch&&ch<=0xFDFF) /* Arabic Presentation Forms-A */
+ ||(0xFE70<=ch&&ch<=0xFEFF) /* Arabic Presentation Forms-B */
+ ||(0xFB1D<=ch&&ch<=0xFB4F) /* Hebrew presentation forms */
+ ) {
src[0]=ch; src[1]=0x00;
perr=U_ZERO_ERROR;
unorm2_normalize(unormalizer_NFD, src, 1, dest, 8, &perr);
if (U_SUCCESS(perr))
ch=dest[0]; /* without modifier */
}
+ else if (ch==0x5DA||ch==0x5DD||ch==0x05DF||ch==0x5E3||ch==0x05E5) { /* Hebrew letter final */
+ ch++;
+ }
ini[0]=ch;
return;
}
+ if (ch>=0xFF21&&ch<=0xFF3A || ch>=0xFF41&&ch<=0xFF5A) {
+ /* Fullwidth latin letter */
+ ch-=0xFF21-0x0041;
+ }
if (ch==0x049||ch==0x069||ch==0x130||ch==0x131||ch==0x0CE||ch==0x0EE) {
/* check dotted/dotless İ,I,i,ı and Î,î for Turkish */
strX[0] = 0x131; strX[1] = 0x5A; strX[2] = 0x00; /* ıZ */
@@ -874,6 +939,22 @@ static void index_normalize(UChar *istr, UChar *ini, int *chset)
return;
}
}
+ if (ch==0x059||ch==0x079) {
+ /* check Y versus I for Lithuanian */
+ if (i_y_mode==0) {
+ strgth = ucol_getStrength(icu_collator);
+ ucol_setStrength(icu_collator, UCOL_PRIMARY);
+ strX[0] = 0x059; strX[1] = 0x00; /* Y */
+ strZ[0] = 0x049; strZ[1] = 0x00; /* I */
+ order = ucol_strcoll(icu_collator, strZ, -1, strX, -1);
+ if (order==UCOL_EQUAL) i_y_mode=2; else i_y_mode=1;
+ ucol_setStrength(icu_collator, strgth);
+ }
+ if (i_y_mode==2) {
+ ini[0] = 0x049; /* I */
+ return;
+ }
+ }
if (ch==0x0C6||ch==0x0E6||ch==0x152||ch==0x153||ch==0x132||ch==0x133
||ch==0x0DF||ch==0x1E9E||ch==0x13F||ch==0x140||ch==0x490||ch==0x491) {
strX[0] = u_toupper(ch); strX[1] = 0x00; /* ex. "Æ" "Œ" */
@@ -976,6 +1057,8 @@ static void index_normalize(UChar *istr, UChar *ini, int *chset)
}
}
}
+ if (ch==0x0AA) ch=L'A';
+ if (ch==0x0BA) ch=L'O';
ini[0]=u_toupper(ch);
return;
}
@@ -1033,7 +1116,7 @@ static int init_hanzi_header(void)
len=u_strlen(pch0);
}
if (len>=INITIALLENGTH) {
- warn_printf(efp, "\nWarning, Too long (%d) hanzi header.\n", len);
+ warn_printf(efp, "\nWarning: Too long (%d) hanzi header.\n", len);
len=INITIALLENGTH-1;
}
u_strncpy(hz_index[k].idx,pch0,len);
diff --git a/indexing/upmendex/source/kana.h b/indexing/upmendex/source/kana.h
index 5c006cb7f3..41307f2cb4 100644
--- a/indexing/upmendex/source/kana.h
+++ b/indexing/upmendex/source/kana.h
@@ -192,56 +192,6 @@ UChar extkanatable[]={
0x308d, /* ろ */
0
};
-UChar circkanatable[]={
- 0x3042, /* あ */
- 0x3044, /* い */
- 0x3046, /* う */
- 0x3048, /* え */
- 0x304a, /* お */
- 0x304b, /* か */
- 0x304d, /* き */
- 0x304f, /* く */
- 0x3051, /* け */
- 0x3053, /* こ */
- 0x3055, /* さ */
- 0x3057, /* し */
- 0x3059, /* す */
- 0x305b, /* せ */
- 0x305d, /* そ */
- 0x305f, /* た */
- 0x3061, /* ち */
- 0x3064, /* つ */
- 0x3066, /* て */
- 0x3068, /* と */
- 0x306a, /* な */
- 0x306b, /* に */
- 0x306c, /* ぬ */
- 0x306d, /* ね */
- 0x306e, /* の */
- 0x306f, /* は */
- 0x3072, /* ひ */
- 0x3075, /* ふ */
- 0x3078, /* へ */
- 0x307b, /* ほ */
- 0x307e, /* ま */
- 0x307f, /* み */
- 0x3080, /* む */
- 0x3081, /* め */
- 0x3082, /* も */
- 0x3084, /* や */
- 0x3086, /* ゆ */
- 0x3088, /* よ */
- 0x3089, /* ら */
- 0x308a, /* り */
- 0x308b, /* る */
- 0x308c, /* れ */
- 0x308d, /* ろ */
- 0x308f, /* わ */
- 0x3090, /* ゐ */
- 0x3091, /* ゑ */
- 0x3092, /* を */
- 0
-};
UChar GANADA[]={
diff --git a/indexing/upmendex/source/ktable.h b/indexing/upmendex/source/ktable.h
deleted file mode 100644
index 9a3a00ef49..0000000000
--- a/indexing/upmendex/source/ktable.h
+++ /dev/null
@@ -1,98 +0,0 @@
-UChar symboltable[]={
- 0x3000, /* スペース */
- 0xff01, /* ! */
- 0xff02, /* ” */
- 0xff03, /* # */
- 0xff04, /* $ */
- 0xff05, /* % */
- 0xff06, /* & */
- 0xff07, /* ’ */
- 0xff08, /* ( */
- 0xff09, /* ) */
- 0xff0a, /* * */
- 0xff0b, /* + */
- 0xff0c, /* , */
- 0xff0d, /* − */
- 0xff0e, /* . */
- 0xff0f, /* / */
- 0xff10, /* 0 */
- 0xff11, /* 1 */
- 0xff12, /* 2 */
- 0xff13, /* 3 */
- 0xff14, /* 4 */
- 0xff15, /* 5 */
- 0xff16, /* 6 */
- 0xff17, /* 7 */
- 0xff18, /* 8 */
- 0xff19, /* 9 */
- 0xff1a, /* : */
- 0xff1b, /* ; */
- 0xff1c, /* < */
- 0xff1d, /* = */
- 0xff1e, /* > */
- 0xff1f, /* ? */
- 0xff20, /* @ */
- 0xff21, /* A */
- 0xff22, /* B */
- 0xff23, /* C */
- 0xff24, /* D */
- 0xff25, /* E */
- 0xff26, /* F */
- 0xff27, /* G */
- 0xff28, /* H */
- 0xff29, /* I */
- 0xff2a, /* J */
- 0xff2b, /* K */
- 0xff2c, /* L */
- 0xff2d, /* M */
- 0xff2e, /* N */
- 0xff2f, /* O */
- 0xff30, /* P */
- 0xff31, /* Q */
- 0xff32, /* R */
- 0xff33, /* S */
- 0xff34, /* T */
- 0xff35, /* U */
- 0xff36, /* V */
- 0xff37, /* W */
- 0xff38, /* X */
- 0xff39, /* Y */
- 0xff3a, /* Z */
- 0xff3b, /* [ */
- 0xffe5, /* ¥ */
- 0xff3d, /* ] */
- 0xff3e, /* ^ */
- 0xff3f, /* _ */
- 0xff40, /* ‘ */
- 0xff41, /* a */
- 0xff42, /* b */
- 0xff43, /* c */
- 0xff44, /* d */
- 0xff45, /* e */
- 0xff46, /* f */
- 0xff47, /* g */
- 0xff48, /* h */
- 0xff49, /* i */
- 0xff4a, /* j */
- 0xff4b, /* k */
- 0xff4c, /* l */
- 0xff4d, /* m */
- 0xff4e, /* n */
- 0xff4f, /* o */
- 0xff50, /* p */
- 0xff51, /* q */
- 0xff52, /* r */
- 0xff53, /* s */
- 0xff54, /* t */
- 0xff55, /* u */
- 0xff56, /* v */
- 0xff57, /* w */
- 0xff58, /* x */
- 0xff59, /* y */
- 0xff5a, /* z */
- 0xff5b, /* { */
- 0xff5c, /* | */
- 0xff5d, /* } */
- 0xff5f, /* 〜 */
- 0
-};
diff --git a/indexing/upmendex/source/mendex.h b/indexing/upmendex/source/mendex.h
index c6703761fc..091862d61e 100644
--- a/indexing/upmendex/source/mendex.h
+++ b/indexing/upmendex/source/mendex.h
@@ -11,10 +11,12 @@
#include <unicode/uchar.h>
#include <unicode/ucol.h>
+#define PAGE_COMPOSIT_DEPTH 10
+
struct page {
char *page;
char *enc;
- int attr[3];
+ int attr[PAGE_COMPOSIT_DEPTH];
};
struct index {
@@ -47,7 +49,6 @@ int lastpage(const char *filename);
/* sort.c */
void wsort(struct index *ind, int num);
void pagesort(struct index *ind, int num);
-int is_alphanumeric(UChar *c);
int is_latin(UChar *c);
int is_numeric(UChar *c);
int is_jpn_kana(UChar *c);
@@ -58,7 +59,10 @@ int is_cyrillic(UChar *c);
int is_greek(UChar *c);
int is_devanagari(UChar *c);
int is_thai(UChar *c);
-int is_comb_diacritical_mark(UChar *c);
+int is_arabic(UChar *c);
+int is_hebrew(UChar *c);
+int is_type_mark_or_punct(UChar *c);
+int is_type_symbol(UChar *c);
int chkcontinue(struct page *p, int num);
int ss_comp(UChar *s1, UChar *s2);
@@ -71,8 +75,11 @@ int ss_comp(UChar *s1, UChar *s2);
#define CH_HANZI 6
#define CH_DEVANAGARI 7
#define CH_THAI 8
+#define CH_ARABIC 9
+#define CH_HEBREW 10
#define CH_SYMBOL 0x100
#define CH_NUMERIC 0x101
+#define is_any_script(a) ((CH_LATIN<=(a) && (a)<=CH_HEBREW))
/* sort.c */
int charset(UChar *c);
diff --git a/indexing/upmendex/source/sort.c b/indexing/upmendex/source/sort.c
index e3a47c12b7..bcecd765cf 100644
--- a/indexing/upmendex/source/sort.c
+++ b/indexing/upmendex/source/sort.c
@@ -4,7 +4,23 @@
#include "exkana.h"
#include "exvar.h"
-int sym,nmbr,ltn,kana,hngl,hnz,cyr,grk,dvng,thai;
+#define RULEBUFSIZE 29210+STYBUFSIZE
+/*
+ length of collation rule in ICU 68.2
+
+ icu_locale length
+ ja 6410
+ ja@collation=unihan 61
+ ko 12577
+ ko@collation=unihan 51
+ ko@collation=search 782
+ zh (pinin) 26909
+ zh@collation=unihan 82
+ zh@collation=stroke 29208
+ zh@collation=zhuyin 28880
+*/
+
+int sym,nmbr,ltn,kana,hngl,hnz,cyr,grk,dvng,thai,arab,hbrw;
static int wcomp(const void *p, const void *q);
static int pcomp(const void *p, const void *q);
@@ -17,7 +33,9 @@ void wsort(struct index *ind, int num)
{
int i,order;
UErrorCode status;
- UChar rules[STYBUFSIZE];
+ UParseError parse_error;
+ UChar rules[RULEBUFSIZE] = {'\0'};
+ int32_t len;
for (order=1,i=0;;i++) {
switch (character_order[i]) {
@@ -65,7 +83,16 @@ void wsort(struct index *ind, int num)
thai=order++;
break;
+ case 'a':
+ arab=order++;
+ break;
+
+ case 'h':
+ hbrw=order++;
+ break;
+
default:
+ verb_printf(efp,"\nWarning: Illegal input for character_order (%c).",character_order[i]);
break;
}
}
@@ -82,11 +109,28 @@ BREAK:
if (grk==0) grk=order++;
if (dvng==0) dvng=order++;
if (thai==0) thai=order++;
+ if (arab==0) arab=order++;
+ if (hbrw==0) hbrw=order++;
status = U_ZERO_ERROR;
if (strlen(icu_rules)>0) {
+ if (strcmp(icu_locale,"root")!=0) {
+ icu_collator = ucol_open(icu_locale, &status);
+ if (U_FAILURE(status)) {
+ verb_printf(efp, "\n[ICU] Collator creation failed.: %s\n", u_errorName(status));
+ exit(254);
+ }
+ len = ucol_getRulesEx(icu_collator, UCOL_TAILORING_ONLY, rules, RULEBUFSIZE);
+ if (u_strlen(rules)<len) {
+ verb_printf(efp, "\n[ICU] Failed to extract collation rules by locale (%s). Need buffer size %d.\n",
+ icu_locale, len);
+ exit(254);
+ }
+ ucol_close(icu_collator);
+ }
unescape((unsigned char *)icu_rules, rules);
- icu_collator = ucol_openRules(rules, -1, UCOL_OFF, UCOL_TERTIARY, NULL, &status);
+ status = U_ZERO_ERROR;
+ icu_collator = ucol_openRules(rules, -1, UCOL_OFF, UCOL_TERTIARY, &parse_error, &status);
} else
icu_collator = ucol_open(icu_locale, &status);
if (U_FAILURE(status)) {
@@ -94,11 +138,11 @@ BREAK:
exit(254);
}
if (status == U_USING_DEFAULT_WARNING) {
- warn_printf(efp, "\nWarning, [ICU] U_USING_DEFAULT_WARNING for locale %s\n",
+ warn_printf(efp, "\nWarning: [ICU] U_USING_DEFAULT_WARNING for locale %s\n",
icu_locale);
}
if (status == U_USING_FALLBACK_WARNING) {
- warn_printf(efp, "\nWarning, [ICU] U_USING_FALLBACK_WARNING for locale %s\n",
+ warn_printf(efp, "\nWarning: [ICU] U_USING_FALLBACK_WARNING for locale %s\n",
icu_locale);
}
for (i=0;i<UCOL_ATTRIBUTE_COUNT;i++) {
@@ -107,7 +151,7 @@ BREAK:
ucol_setAttribute(icu_collator, i, icu_attributes[i], &status);
}
if (U_FAILURE(status)) {
- warn_printf(efp, "\nWarning, [ICU] Failed to set attribute (%d): %s\n",
+ warn_printf(efp, "\nWarning: [ICU] Failed to set attribute (%d): %s\n",
i, u_errorName(status));
}
}
@@ -214,12 +258,12 @@ void pagesort(struct index *ind, int num)
static int pcomp(const void *p, const void *q)
{
int i,j,cc=0,num1,num2;
- char buff[16];
+ char buff[16],*p0,*p1;
const struct page *page1 = p, *page2 = q;
scount++;
- for (i=0;i<3;i++) {
+ for (i=0;i<PAGE_COMPOSIT_DEPTH;i++) {
if ((page1->attr[i]<0)&&(page2->attr[i]<0)) return 0;
else if ((page1->attr[i]<0)&&(page2->attr[i]>=0)) return -1;
else if ((page2->attr[i]<0)&&(page1->attr[i]>=0)) return 1;
@@ -227,18 +271,18 @@ static int pcomp(const void *p, const void *q)
if (page1->attr[i]>page2->attr[i]) return 1;
if (page1->attr[i]<page2->attr[i]) return -1;
- for (j=cc;j<strlen(page1->page);j++) {
- if (strncmp(&page1->page[j],page_compositor,strlen(page_compositor))==0) break;
- }
- strncpy(buff,&page1->page[cc],j-cc);
- buff[j-cc]='\0';
+ p0=&page1->page[cc];
+ p1=strstr(p0, page_compositor);
+ j=p1 ? p1-p0 : strlen(p0);
+ strncpy(buff,p0,j);
+ buff[j]='\0';
num1=pnumconv(buff,page1->attr[i]);
- for (j=cc;j<strlen(page2->page);j++) {
- if (strncmp(&page2->page[j],page_compositor,strlen(page_compositor))==0) break;
- }
- strncpy(buff,&page2->page[cc],j-cc);
- buff[j-cc]='\0';
+ p0=&page2->page[cc];
+ p1=strstr(p0, page_compositor);
+ j=p1 ? p1-p0 : strlen(p0);
+ strncpy(buff,p0,j);
+ buff[j]='\0';
num2=pnumconv(buff,page2->attr[i]);
if (num1>num2) return 1;
@@ -247,7 +291,8 @@ static int pcomp(const void *p, const void *q)
if (page1->enc[0]=='(' || page2->enc[0]==')') return -1;
if (page1->enc[0]==')' || page2->enc[0]=='(') return 1;
- cc=j+strlen(page_compositor);
+ if (p1) cc+=j+strlen(page_compositor);
+ else return 0;
}
return 0;
@@ -255,13 +300,15 @@ static int pcomp(const void *p, const void *q)
static int ordering(UChar *c)
{
- if (*c<0x80) {
- if (is_latin(c)) return ltn;
- else if (is_numeric(c)) return nmbr;
- else return sym;
+ if (*c<0x20) return sym; /* control */
+ else if (*c<0x7F) {
+ if (is_latin(c)) return ltn;
+ else if (is_numeric(c)) return nmbr;
+ else return sym;
}
+ else if (*c<0xA0) return sym; /* control */
else {
- if (is_latin(c)) return ltn;
+ if (is_latin(c)) return ltn;
else if (is_jpn_kana(c)) return kana;
else if (is_kor_hngl(c)) return hngl;
else if (is_hanzi(c)) return hnz;
@@ -270,20 +317,23 @@ static int ordering(UChar *c)
else if (is_numeric(c)) return nmbr;
else if (is_devanagari(c)) return dvng;
else if (is_thai(c)) return thai;
- else return sym;
+ else if (is_arabic(c)) return arab;
+ else if (is_hebrew(c)) return hbrw;
+ else return sym;
}
}
int charset(UChar *c)
{
- if (*c==0x00) return CH_UNKNOWN;
- else if (*c<0x80) {
- if (is_latin(c)) return CH_LATIN;
- else if (is_numeric(c)) return CH_NUMERIC;
- else return CH_SYMBOL;
+ if (*c<0x20) return CH_UNKNOWN; /* control */
+ else if (*c<0x7F) {
+ if (is_latin(c)) return CH_LATIN;
+ else if (is_numeric(c)) return CH_NUMERIC;
+ else return CH_SYMBOL;
}
+ else if (*c<0xA0) return CH_UNKNOWN; /* control */
else {
- if (is_latin(c)) return CH_LATIN;
+ if (is_latin(c)) return CH_LATIN;
else if (is_jpn_kana(c)) return CH_KANA;
else if (is_kor_hngl(c)) return CH_HANGUL;
else if (is_hanzi(c)) return CH_HANZI;
@@ -292,7 +342,9 @@ int charset(UChar *c)
else if (is_numeric(c)) return CH_NUMERIC;
else if (is_devanagari(c)) return CH_DEVANAGARI;
else if (is_thai(c)) return CH_THAI;
- else return CH_SYMBOL;
+ else if (is_arabic(c)) return CH_ARABIC;
+ else if (is_hebrew(c)) return CH_HEBREW;
+ else return CH_SYMBOL;
}
}
@@ -312,10 +364,10 @@ static int get_charset_juncture(UChar *str)
else l = k-1;
chset_l=charset(&str[l]);
chset_k=charset(&str[k]);
- if (chset0==CH_UNKNOWN && chset_l!=CH_SYMBOL && chset_l!=CH_NUMERIC) {
+ if (chset0==CH_UNKNOWN && is_any_script(chset_l)) {
chset0=chset_l;
}
- if (chset_k!=CH_SYMBOL && chset_k!=CH_NUMERIC) {
+ if (chset0!=CH_UNKNOWN && is_any_script(chset_k)) {
if (chset0!=chset_k) {
len=k;
return len;
@@ -324,10 +376,11 @@ static int get_charset_juncture(UChar *str)
}
}
-static int unescape(const unsigned char *src, UChar *dist)
+static int unescape(const unsigned char *src, UChar *dest)
{
- int i,j,k;
+ int i,j,k,ret;
char tmp[STYBUFSIZE];
+ UErrorCode status;
for (i=j=0;i<STYBUFSIZE;i++) {
if (src[i]=='\0') {
@@ -336,32 +389,37 @@ static int unescape(const unsigned char *src, UChar *dist)
else if (src[i]< 0x80 && (src[i+1]>=0x80 || src[i+1]=='\0')) {
strncpy(tmp,(char *)&src[j],i-j+1);
tmp[i-j+1]='\0';
- k=u_strlen(dist);
- u_unescape(tmp, &dist[k], STYBUFSIZE-k);
+ k=u_strlen(dest);
+ ret=u_unescape(tmp, &dest[k], RULEBUFSIZE-k);
+ if (ret==0) {
+ verb_printf(efp, "\n[ICU] Escape sequence in input seems malformed.\n");
+ exit(254);
+ }
j=i+1;
}
else if (src[i]>=0x80 && (src[i+1]< 0x80 || src[i+1]=='\0')) {
strncpy(tmp,(char *)&src[j],i-j+1);
tmp[i-j+1]='\0';
- k=u_strlen(dist);
- multibyte_to_widechar(&dist[k], STYBUFSIZE-k, tmp);
+ k=u_strlen(dest);
+ status=U_ZERO_ERROR;
+ u_strFromUTF8(&dest[k], RULEBUFSIZE-k, NULL, tmp, -1, &status);
+ if (U_FAILURE(status)) {
+ verb_printf(efp, "\n[ICU] Input string seems malformed.: %s\n", u_errorName(status));
+ exit(254);
+ }
j=i+1;
}
}
return -1;
}
-int is_alphanumeric(UChar *c)
-{
- if (((*c>=L'A')&&(*c<=L'Z'))||((*c>=L'a')&&(*c<=L'z'))||((*c>=L'0')&&(*c<=L'9')))
- return 1;
- else return 0;
-}
-
int is_latin(UChar *c)
{
+ UChar32 c32;
+
if (((*c>=L'A')&&(*c<=L'Z'))||((*c>=L'a')&&(*c<=L'z'))) return 1;
- else if ((*c>=0x00C0)&&(*c<=0x00D6)) return 1; /* Latin-1 Supplement */
+ else if ((*c==0x00AA)||(*c==0x00BA)) return 1; /* Latin-1 Supplement */
+ else if ((*c>=0x00C0)&&(*c<=0x00D6)) return 1;
else if ((*c>=0x00D8)&&(*c<=0x00F6)) return 1;
else if ((*c>=0x00F8)&&(*c<=0x00FF)) return 1;
else if ((*c>=0x0100)&&(*c<=0x024F)) return 1; /* Latin Extended-A,B */
@@ -371,28 +429,63 @@ int is_latin(UChar *c)
else if ((*c>=0xAB30)&&(*c<=0xAB6F)) return 1; /* Latin Extended-E */
else if ((*c>=0x1E00)&&(*c<=0x1EFF)) return 1; /* Latin Extended Additional */
else if ((*c>=0xFB00)&&(*c<=0xFB06)) return 1; /* Latin ligatures */
- else return 0;
+ else if ((*c>=0xFF21)&&(*c<=0xFF3A)) return 1; /* Fullwidth Latin Capital Letter */
+ else if ((*c>=0xFF41)&&(*c<=0xFF5A)) return 1; /* Fullwidth Latin Small Letter */
+ /* Property of followings is "Common, So (other symbol)", but seem to be treated as Latin by ICU collator */
+ else if ((*c>=0x24B6) /* CIRCLED LATIN CAPITAL LETTER */
+ &&(*c<=0x24E9)) return 1; /* CIRCLED LATIN SMALL LETTER */
+
+ if (is_surrogate_pair(c)) {
+ c32=U16_GET_SUPPLEMENTARY(*c,*(c+1));
+ if ((c32>=0x10780) && (c32<=0x107BF)) return 2; /* Latin Extended-F */
+ else if ((c32>=0x1DF00) && (c32<=0x1DFFF)) return 2; /* Latin Extended-G */
+ }
+ return 0;
}
int is_numeric(UChar *c)
{
+ UChar32 c32;
+
if ((*c>=L'0')&&(*c<=L'9')) return 1;
- else if ((*c>=0x0966)&&(*c<=0x096F)) return 1; /* Devanagari Digit */
- else if ((*c>=0x0E50)&&(*c<=0x0E59)) return 1; /* Thai Digit */
- else return 0;
+ else if ((*c>=0xFF10)&&(*c<=0xFF19)) return 1; /* Fullwidth Digit */
+ /* followings do not seem to be treated as numbers by ICU collator though charType is U_OTHER_NUMBER */
+ else if ((*c>=0x3192)&&(*c<=0x3195)) return 0; /* IDEOGRAPHIC ANNOTATION ONE MARK..IDEOGRAPHIC ANNOTATION FOUR MARK */
+ else if ((*c>=0x3220)&&(*c<=0x3229)) return 0; /* PARENTHESIZED IDEOGRAPH ONE..PARENTHESIZED IDEOGRAPH TEN */
+ else if ((*c>=0x3280)&&(*c<=0x3289)) return 0; /* CIRCLED IDEOGRAPH ONE..CIRCLED IDEOGRAPH TEN */
+ else if ((*c>=0xA830)&&(*c<=0xA835)) return 0; /* NORTH INDIC FRACTION ONE QUARTER..NORTH INDIC FRACTION THREE SIXTEENTHS */
+
+ if (is_surrogate_pair(c))
+ c32=U16_GET_SUPPLEMENTARY(*c,*(c+1));
+ else c32=*c;
+
+ switch (u_charType(c32)) {
+ case U_DECIMAL_DIGIT_NUMBER:
+ return 1;
+ case U_OTHER_NUMBER:
+ return 2;
+ default:
+ return 0;
+ }
}
int is_jpn_kana(UChar *c)
{
UChar32 c32;
- if ((*c>=0x3040)&&(*c<=0x30FF)) return 1; /* Hiragana, Katakana */
+ if (*c==0x30A0) return 0; /* KATAKANA-HIRAGANA DOUBLE HYPHEN */
+ else if (*c==0x30FB) return 0; /* KATAKANA MIDDLE DOT */
+ else if ((*c>=0x3040)&&(*c<=0x30FF)) return 1; /* Hiragana, Katakana */
else if ((*c>=0x31F0)&&(*c<=0x31FF)) return 1; /* Katakana Phonetic Extensions */
else if ((*c>=0x32D0)&&(*c<=0x32FE)) return 1; /* Circled Katakana */
+ else if ((*c>=0xFF66)&&(*c<=0xFF9F)) return 1; /* Halfwidth Katakana */
+ else if ((*c>=0x3300)&&(*c<=0x3357)) return 1; /* Squared Katakana words */
if (is_surrogate_pair(c)) {
c32=U16_GET_SUPPLEMENTARY(*c,*(c+1));
if ((c32>=0x1B130) && (c32<=0x1B16F)) return 2; /* Small Kana Extensions */
+ else if ((c32==0x1B000)) return 2; /* KATAKANA LETTER ARCHAIC E */
+ else if ((c32==0x1F200)) return 2; /* SQUARE HIRAGANA HOKA */
}
return 0;
/* ICU 65 does not seem to support
@@ -416,11 +509,11 @@ int is_hanzi(UChar *c)
{
UChar32 c32;
- if ((*c>=0x2E80)&&(*c<=0x2EFF)) return 1; /* CJK Radicals Supplement */
- else if ((*c>=0x2F00)&&(*c<=0x2FDF)) return 1; /* Kangxi Radicals */
+ if ((*c>=0x2E80) /* CJK Radicals Supplement */
+ &&(*c<=0x2FDF)) return 1; /* Kangxi Radicals */
else if ((*c>=0x31C0)&&(*c<=0x31EF)) return 1; /* CJK Strokes */
- else if ((*c>=0x3300)&&(*c<=0x33FF)) return 1; /* CJK Compatibility */
- else if ((*c>=0x3400)&&(*c<=0x4DBF)) return 1; /* CJK Unified Ideographs Extension A */
+ else if ((*c>=0x3300) /* CJK Compatibility */
+ &&(*c<=0x4DBF)) return 1; /* CJK Unified Ideographs Extension A */
else if ((*c>=0x4E00)&&(*c<=0x9FFF)) return 1; /* CJK Unified Ideographs */
else if ((*c>=0xF900)&&(*c<=0xFAFF)) return 1; /* CJK Compatibility Ideographs */
@@ -442,7 +535,9 @@ int is_zhuyin(UChar *c)
int is_cyrillic(UChar *c)
{
- if ((*c>=0x0400)&&(*c<=0x052F)) return 1; /* Cyrillic, Cyrillic Supplement */
+ if ((*c==0x0482)) return 0; /* Cyrillic Thousands Sign */
+ else if ((*c>=0x0400) /* Cyrillic */
+ &&(*c<=0x052F)) return 1; /* Cyrillic Supplement */
else if ((*c>=0x1C80)&&(*c<=0x1C8F)) return 1; /* Cyrillic Extended-C */
else if ((*c>=0x2DE0)&&(*c<=0x2DFF)) return 1; /* Cyrillic Extended-A */
else if ((*c>=0xA640)&&(*c<=0xA69F)) return 1; /* Cyrillic Extended-B */
@@ -451,14 +546,16 @@ int is_cyrillic(UChar *c)
int is_greek(UChar *c)
{
- if ((*c>=0x0370)&&(*c<=0x03FF)) return 1; /* Greek */
+ if ((*c==0x03F6)) return 0; /* Greek Reversed Lunate Epsilon Symbol */
+ else if ((*c>=0x0370)&&(*c<=0x03FF)) return 1; /* Greek */
else if ((*c>=0x1F00)&&(*c<=0x1FFF)) return 1; /* Greek Extended */
else return 0;
}
int is_devanagari(UChar *c)
{
- if ((*c>=0x0966)&&(*c<=0x096F)) return 0; /* Devanagari Digit */
+ if ((*c>=0x0964) /* Generic punctuation for scripts of India */
+ &&(*c<=0x096F)) return 0; /* Devanagari Digit */
else if ((*c>=0x0900)&&(*c<=0x097F)) return 1; /* Devanagari */
else if ((*c>=0xA8E0)&&(*c<=0xA8FF)) return 1; /* Devanagari Extended */
else return 0;
@@ -472,47 +569,132 @@ int is_thai(UChar *c)
else return 0;
}
-int is_comb_diacritical_mark(UChar *c)
+int is_arabic(UChar *c)
+{
+ if ((*c>=0x0600) /* ARABIC NUMBER SIGN..ARABIC SIGN SAMVAT */
+ /* ARABIC NUMBER MARK ABOVE */
+ &&(*c<=0x0608)) return 0; /* ARABIC-INDIC CUBE ROOT..ARABIC RAY */
+ else if ((*c==0x060B)) return 0; /* AFGHANI SIGN */
+ else if ((*c==0x060C)) return 0; /* ARABIC COMMA */
+ else if ((*c>=0x060E)&&(*c<=0x060F)) return 0; /* ARABIC POETIC VERSE SIGN..ARABIC SIGN MISRA */
+ else if ((*c>=0x0660)&&(*c<=0x0669)) return 0; /* ARABIC-INDIC DIGIT ZERO..ARABIC-INDIC DIGIT NINE */
+ else if ((*c==0x061B)) return 0; /* ARABIC SEMICOLON */
+ else if ((*c==0x061C)) return 0; /* ARABIC LETTER MARK */
+ else if ((*c==0x061F)) return 0; /* ARABIC QUESTION MARK */
+ else if ((*c==0x0640)) return 0; /* ARABIC TATWEEL */
+ else if ((*c==0x06DD)) return 0; /* ARABIC END OF AYAH */
+ else if ((*c==0x06DE)) return 0; /* ARABIC START OF RUB EL HIZB */
+ else if ((*c==0x06E9)) return 0; /* ARABIC PLACE OF SAJDAH */
+ else if ((*c>=0x06F0)&&(*c<=0x06F9)) return 0; /* EXTENDED ARABIC-INDIC DIGIT ZERO..EXTENDED ARABIC-INDIC DIGIT NINE */
+ else if ((*c>=0x06FD)&&(*c<=0x06FE)) return 0; /* ARABIC SIGN SINDHI AMPERSAND..ARABIC SIGN SINDHI POSTPOSITION MEN */
+ else if ((*c==0x08E2)) return 0; /* ARABIC DISPUTED END OF AYAH */
+ else if ((*c>=0x0890)&&(*c<=0x0891)) return 0; /* ARABIC POUND MARK ABOVE..ARABIC PIASTRE MARK ABOVE */
+ else if ((*c>=0xFD40)&&(*c<=0xFD4F)) return 0; /* ARABIC LIGATURE RAHIMAHU ALLAAH..ARABIC LIGATURE RAHIMAHUM ALLAAH */
+ else if ((*c==0xFDCF)) return 0; /* ARABIC LIGATURE SALAAMUHU ALAYNAA */
+ else if ((*c==0xFDFC)) return 0; /* RIAL SIGH */
+ else if ((*c>=0xFDFD)&&(*c<=0xFDFF)) return 0; /* ARABIC LIGATURE BISMILLAH AR-RAHMAN AR-RAHEEM..ARABIC LIGATURE AZZA WA JALL */
+
+ else if ((*c>=0x0600)&&(*c<=0x06FF)) return 1; /* Arabic */
+ else if ((*c>=0x0750)&&(*c<=0x077F)) return 1; /* Arabic Supplement */
+ else if ((*c>=0x0870) /* Arabic Extended-B */
+ &&(*c<=0x08FF)) return 1; /* Arabic Extended-A */
+ else if ((*c>=0xFB50)&&(*c<=0xFDFF)) return 1; /* Arabic Presentation Forms-A */
+ else if ((*c>=0xFE70)&&(*c<=0xFEFF)) return 1; /* Arabic Presentation Forms-B */
+ else return 0;
+}
+
+int is_hebrew(UChar *c)
{
- if ((*c>=0x02B0)&&(*c<=0x02FF)) return 1; /* Spacing Modifier Letters */
- else if ((*c>=0x0300)&&(*c<=0x036F)) return 1; /* Combining Diacritical Marks */
- else if ((*c>=0x1DC0)&&(*c<=0x1DFF)) return 1; /* Combining Diacritical Marks Supplement */
- else if ((*c>=0x1AB0)&&(*c<=0x1AFF)) return 1; /* Combining Diacritical Marks Extended */
- else if ((*c>=0x3099)&&(*c<=0x309A)) return 1; /* Combining Kana Voiced Sound Marks */
+ if ((*c==0xFB29)) return 0; /* Hebrew Letter Alternative Plus Sign */
+ else if ((*c>=0x0590)&&(*c<=0x05FF)) return 1; /* Hebrew */
+ else if ((*c>=0xFB1D)&&(*c<=0xFB4F)) return 1; /* Hebrew presentation forms */
else return 0;
}
+int is_type_mark_or_punct(UChar *c)
+{
+ UChar32 c32;
+
+ if (is_surrogate_pair(c))
+ c32=U16_GET_SUPPLEMENTARY(*c,*(c+1));
+ else c32=*c;
+
+ switch (u_charType(c32)) {
+ case U_MODIFIER_LETTER:
+ case U_DASH_PUNCTUATION: case U_START_PUNCTUATION: case U_END_PUNCTUATION:
+ case U_CONNECTOR_PUNCTUATION: case U_OTHER_PUNCTUATION:
+ case U_INITIAL_PUNCTUATION: case U_FINAL_PUNCTUATION:
+ case U_NON_SPACING_MARK: case U_ENCLOSING_MARK: case U_COMBINING_SPACING_MARK:
+ case U_FORMAT_CHAR:
+ return 1;
+ default:
+ return 0;
+ }
+}
+
+int is_type_symbol(UChar *c)
+{
+ UChar32 c32;
+
+ if (is_surrogate_pair(c))
+ c32=U16_GET_SUPPLEMENTARY(*c,*(c+1));
+ else c32=*c;
+
+ switch (u_charType(c32)) {
+ case U_MODIFIER_SYMBOL:
+ return 1;
+ case U_MATH_SYMBOL: case U_CURRENCY_SYMBOL:
+ case U_OTHER_SYMBOL:
+ return 2;
+ default:
+ return 0;
+ }
+}
+
int chkcontinue(struct page *p, int num)
{
- int i,j,cc=0,num1,num2;
- char buff[16];
+ int i,j,cc=0,num1,num2,k1,k2;
+ char buff1[16],buff2[16],*p0,*p1;
- for (i=0;i<3;i++) {
+ for (i=0;i<PAGE_COMPOSIT_DEPTH;i++) {
if ((p[num].attr[i]<0)&&(p[num+1].attr[i]<0)) return 1;
else if (p[num].attr[i]!=p[num+1].attr[i]) return 0;
- for (j=cc;j<strlen(p[num].page);j++) {
- if (strncmp(&p[num].page[j],page_compositor,strlen(page_compositor))==0) break;
+ p0=&p[num].page[cc];
+ p1=strstr(p0, page_compositor);
+ if (p1) {
+ j=p1-p0;
+ k1=j;
+ } else {
+ j=strlen(p0);
+ k1=0;
}
- strncpy(buff,&p[num].page[cc],j);
- buff[j]='\0';
- num1=pnumconv(buff,p[num].attr[i]);
-
- for (j=cc;j<strlen(p[num+1].page);j++) {
- if (strncmp(&p[num+1].page[j],page_compositor,strlen(page_compositor))==0) break;
+ strncpy(buff1,p0,j);
+ buff1[j]='\0';
+ num1=pnumconv(buff1,p[num].attr[i]);
+
+ p0=&p[num+1].page[cc];
+ p1=strstr(p0, page_compositor);
+ if (p1) {
+ j=p1-p0;
+ k2=j;
+ } else {
+ j=strlen(p0);
+ k2=0;
}
- strncpy(buff,&p[num+1].page[cc],j);
- buff[j]='\0';
- num2=pnumconv(buff,p[num+1].attr[i]);
-
- if (num1==num2 || num1+1==num2) {
- if (i==2) return 1;
- if ((p[num].attr[i+1]<0)&&(p[num+1].attr[i+1]<0)) return 1;
- else return 0;
+ strncpy(buff2,p0,j);
+ buff2[j]='\0';
+ num2=pnumconv(buff2,p[num+1].attr[i]);
+
+ if (k1>0 || k2>0) {
+ if (k1!=k2) return 0;
+ if (strcmp(buff1,buff2)) return 0;
+ cc+=k1+strlen(page_compositor);
+ continue;
}
- else if (num1!=num2) return 0;
- cc=j+strlen(page_compositor);
+ if (num1==num2 || num1+1==num2) return 1;
+ else return 0;
}
return 1;
diff --git a/indexing/upmendex/source/styfile.c b/indexing/upmendex/source/styfile.c
index 619e4a5625..f029679049 100644
--- a/indexing/upmendex/source/styfile.c
+++ b/indexing/upmendex/source/styfile.c
@@ -76,13 +76,9 @@ void styread(const char *filename)
if (getparam(buff,"symhead_negative",symhead_negative)) continue;
if (getparam(buff,"numhead_positive",numhead_positive)) continue;
if (getparam(buff,"numhead_negative",numhead_negative)) continue;
- cc=scompare(buff,"lethead_flag");
- if (cc!= -1) {
- lethead_flag=atoi(&buff[cc]);
- continue;
- }
- cc=scompare(buff,"heading_flag");
- if (cc!= -1) {
+ if ( (cc=scompare(buff,"lethead_flag")) != -1 ||
+ (cc=scompare(buff,"heading_flag")) != -1 ||
+ (cc=scompare(buff,"headings_flag")) != -1 ) {
lethead_flag=atoi(&buff[cc]);
continue;
}
@@ -121,7 +117,7 @@ void styread(const char *filename)
indent_length=atoi(&buff[cc]);
continue;
}
- if (getparam(buff,"symbol",symbol)) continue;
+ if (getparam(buff,"symbol",symhead)) continue;
cc=scompare(buff,"symbol_flag");
if (cc!= -1) {
symbol_flag=atoi(&buff[cc]);
@@ -164,6 +160,50 @@ void styread(const char *filename)
continue;
}
if (getparam(buff,"icu_attributes", icu_attr_str )) continue;
+
+ cc=scompare(buff,"script_preamble");
+ if (cc!= -1) {
+ strcpy(tmp,buff+strlen("script_preamble"));
+ if (getparam(tmp,"latin", script_preamble[CH_LATIN] )) continue;
+ if (getparam(tmp,"cyrillic", script_preamble[CH_CYRILLIC] )) continue;
+ if (getparam(tmp,"greek", script_preamble[CH_GREEK] )) continue;
+ if (getparam(tmp,"kana", script_preamble[CH_KANA] )) continue;
+ if (getparam(tmp,"hangul", script_preamble[CH_HANGUL] )) continue;
+ if (getparam(tmp,"hanzi", script_preamble[CH_HANZI] )) continue;
+ if (getparam(tmp,"devanagari",script_preamble[CH_DEVANAGARI] )) continue;
+ if (getparam(tmp,"thai", script_preamble[CH_THAI] )) continue;
+ if (getparam(tmp,"arabic", script_preamble[CH_ARABIC] )) continue;
+ if (getparam(tmp,"hebrew", script_preamble[CH_HEBREW] )) continue;
+ if (strlen(tmp)>0) {
+ verb_printf(efp,"\nWarning: Unknown script for specifier \"script_preamble\" (%s).", tmp);
+ }
+ continue;
+ }
+ cc=scompare(buff,"script_postamble");
+ if (cc!= -1) {
+ strcpy(tmp,buff+strlen("script_postamble"));
+ if (getparam(tmp,"latin", script_postamble[CH_LATIN] )) continue;
+ if (getparam(tmp,"cyrillic", script_postamble[CH_CYRILLIC] )) continue;
+ if (getparam(tmp,"greek", script_postamble[CH_GREEK] )) continue;
+ if (getparam(tmp,"kana", script_postamble[CH_KANA] )) continue;
+ if (getparam(tmp,"hangul", script_postamble[CH_HANGUL] )) continue;
+ if (getparam(tmp,"hanzi", script_postamble[CH_HANZI] )) continue;
+ if (getparam(tmp,"devanagari",script_postamble[CH_DEVANAGARI] )) continue;
+ if (getparam(tmp,"thai", script_postamble[CH_THAI] )) continue;
+ if (getparam(tmp,"arabic", script_postamble[CH_ARABIC] )) continue;
+ if (getparam(tmp,"hebrew", script_postamble[CH_HEBREW] )) continue;
+ if (strlen(tmp)>0) {
+ verb_printf(efp,"\nWarning: Unknown script for specifier \"script_postamble\" (%s).", tmp);
+ }
+ continue;
+ }
+
+ cc=strcspn(buff," \t\r\n");
+ if (cc>0) buff[cc]='\0';
+ if (buff[0]=='%' || buff[0]=='\n') continue;
+ if (strlen(buff)>0) {
+ verb_printf(efp,"\nWarning: Unknown specifier (%s).", buff);
+ }
}
fclose(fp);
@@ -316,54 +356,62 @@ static int sstrncmp(const char *s1, const char *s2, size_t len)
void set_icu_attributes(void)
{
int i,attr;
- char *pos, *tmp;
+ char *pos, *head, *tmp;
for (i=0;i<UCOL_ATTRIBUTE_COUNT;i++) icu_attributes[i]=UCOL_DEFAULT;
tmp=icu_attr_str;
- if ((pos=strstr(tmp,"alternate:"))>0) {
+ head=strtok(tmp, " \t\r\n");
+ while (head!=NULL) {
+ pos=head;
+ if (strncmp(pos,"alternate:",10)==0) {
pos+=10; attr=UCOL_ALTERNATE_HANDLING;
- if (strstr(pos,"shifted")) icu_attributes[attr]=UCOL_SHIFTED;
- else if (strstr(pos,"non-ignorable")) icu_attributes[attr]=UCOL_NON_IGNORABLE;
- else verb_printf(efp,"\nWarning: Illegal input for icu_attributes (alternate).");
- }
- if ((pos=strstr(tmp,"strength:"))>0) {
+ if (strcmp(pos,"shifted")==0) icu_attributes[attr]=UCOL_SHIFTED;
+ else if (strcmp(pos,"non-ignorable")==0) icu_attributes[attr]=UCOL_NON_IGNORABLE;
+ else goto illegal;
+ } else
+ if (strncmp(pos,"strength:",9)==0) {
pos+=9; attr=UCOL_STRENGTH;
- if (strstr(pos,"primary")) icu_attributes[attr]=UCOL_PRIMARY;
- else if (strstr(pos,"secondary")) icu_attributes[attr]=UCOL_SECONDARY;
- else if (strstr(pos,"tertiary")) icu_attributes[attr]=UCOL_TERTIARY;
- else if (strstr(pos,"quaternary")) icu_attributes[attr]=UCOL_QUATERNARY;
- else if (strstr(pos,"identical")) icu_attributes[attr]=UCOL_IDENTICAL;
- else verb_printf(efp,"\nWarning: Illegal input for icu_attributes (strength).");
- }
- if ((pos=strstr(tmp,"french-collation:"))>0) {
+ if (strcmp(pos,"primary")==0) icu_attributes[attr]=UCOL_PRIMARY;
+ else if (strcmp(pos,"secondary")==0) icu_attributes[attr]=UCOL_SECONDARY;
+ else if (strcmp(pos,"tertiary")==0) icu_attributes[attr]=UCOL_TERTIARY;
+ else if (strcmp(pos,"quaternary")==0) icu_attributes[attr]=UCOL_QUATERNARY;
+ else if (strcmp(pos,"identical")==0) icu_attributes[attr]=UCOL_IDENTICAL;
+ else goto illegal;
+ } else
+ if (strncmp(pos,"french-collation:",17)==0) {
pos+=17; attr=UCOL_FRENCH_COLLATION;
- if (strstr(pos,"on")) icu_attributes[attr]=UCOL_ON;
- else if (strstr(pos,"off")) icu_attributes[attr]=UCOL_OFF;
- else verb_printf(efp,"\nWarning: Illegal input for icu_attributes (french-collation).");
- }
- if ((pos=strstr(tmp,"case-first:"))>0) {
+ if (strcmp(pos,"on")==0) icu_attributes[attr]=UCOL_ON;
+ else if (strcmp(pos,"off")==0) icu_attributes[attr]=UCOL_OFF;
+ else goto illegal;
+ } else
+ if (strncmp(pos,"case-first:",11)==0) {
pos+=11; attr=UCOL_CASE_FIRST;
- if (strstr(pos,"off")) icu_attributes[attr]=UCOL_OFF;
- else if (strstr(pos,"upper-first")) icu_attributes[attr]=UCOL_UPPER_FIRST;
- else if (strstr(pos,"lower-first")) icu_attributes[attr]=UCOL_LOWER_FIRST;
- else verb_printf(efp,"\nWarning: Illegal input for icu_attributes (case-first).");
- }
- if ((pos=strstr(tmp,"case-level:"))>0) {
+ if (strcmp(pos,"off")==0) icu_attributes[attr]=UCOL_OFF;
+ else if (strcmp(pos,"upper-first")==0) icu_attributes[attr]=UCOL_UPPER_FIRST;
+ else if (strcmp(pos,"lower-first")==0) icu_attributes[attr]=UCOL_LOWER_FIRST;
+ else goto illegal;
+ } else
+ if (strncmp(pos,"case-level:",11)==0) {
pos+=11; attr=UCOL_CASE_LEVEL;
- if (strstr(pos,"on")) icu_attributes[attr]=UCOL_ON;
- else if (strstr(pos,"off")) icu_attributes[attr]=UCOL_OFF;
- else verb_printf(efp,"\nWarning: Illegal input for icu_attributes (case-level).");
- }
- if ((pos=strstr(tmp,"normalization-mode:"))>0) {
+ if (strcmp(pos,"on")==0) icu_attributes[attr]=UCOL_ON;
+ else if (strcmp(pos,"off")==0) icu_attributes[attr]=UCOL_OFF;
+ else goto illegal;
+ } else
+ if (strncmp(pos,"normalization-mode:",19)==0) {
pos+=19; attr=UCOL_NORMALIZATION_MODE;
- if (strstr(pos,"on")) icu_attributes[attr]=UCOL_ON;
- else if (strstr(pos,"off")) icu_attributes[attr]=UCOL_OFF;
- else verb_printf(efp,"\nWarning: Illegal input for icu_attributes (normalization-mode).");
- }
- if ((pos=strstr(tmp,"numeric-ordering:"))>0) {
+ if (strcmp(pos,"on")==0) icu_attributes[attr]=UCOL_ON;
+ else if (strcmp(pos,"off")==0) icu_attributes[attr]=UCOL_OFF;
+ else goto illegal;
+ } else
+ if (strncmp(pos,"numeric-ordering:",17)==0) {
pos+=17; attr=UCOL_NUMERIC_COLLATION;
- if (strstr(pos,"on")) icu_attributes[attr]=UCOL_ON;
- else if (strstr(pos,"off")) icu_attributes[attr]=UCOL_OFF;
- else verb_printf(efp,"\nWarning: Illegal input for icu_attributes (numeric-ordering).");
+ if (strcmp(pos,"on")==0) icu_attributes[attr]=UCOL_ON;
+ else if (strcmp(pos,"off")==0) icu_attributes[attr]=UCOL_OFF;
+ else goto illegal;
+ } else {
+ illegal:
+ verb_printf(efp,"\nWarning: Illegal input for icu_attributes (%s).",head);
+ }
+ head=strtok(NULL, " \t\r\n");
}
}
diff --git a/indexing/upmendex/source/tests/attr1.ind b/indexing/upmendex/source/tests/attr1.ind
new file mode 100644
index 0000000000..6db7b19221
--- /dev/null
+++ b/indexing/upmendex/source/tests/attr1.ind
@@ -0,0 +1,40 @@
+\begin{theindex}
+
+\centerline{\bfseries ◆◆◆ Symbols ◆◆◆}\par\nobreak
+ \item #\leaders\hbox{$\cdot$}\hfill 2000
+ \item $\leaders\hbox{$\cdot$}\hfill 2001
+ \item 2\leaders\hbox{$\cdot$}\hfill 1001
+ \item 10\leaders\hbox{$\cdot$}\hfill 1000
+
+ \indexspace
+
+\centerline{\bfseries ◆◆◆ A ◆◆◆}\par\nobreak
+ \item A\leaders\hbox{$\cdot$}\hfill 400
+ \item a\leaders\hbox{$\cdot$}\hfill 401
+
+ \indexspace
+
+\centerline{\bfseries ◆◆◆ B ◆◆◆}\par\nobreak
+ \item B\leaders\hbox{$\cdot$}\hfill 402
+ \item b\leaders\hbox{$\cdot$}\hfill 403
+
+ \indexspace
+
+\centerline{\bfseries ◆◆◆ C ◆◆◆}\par\nobreak
+ \item C\leaders\hbox{$\cdot$}\hfill 404
+ \item c\leaders\hbox{$\cdot$}\hfill 405
+ \item Ç\leaders\hbox{$\cdot$}\hfill 406
+ \item ç\leaders\hbox{$\cdot$}\hfill 407
+ \item Ch\leaders\hbox{$\cdot$}\hfill 420
+ \item cha\leaders\hbox{$\cdot$}\hfill 421
+ \item cz\leaders\hbox{$\cdot$}\hfill 422
+
+ \indexspace
+
+\centerline{\bfseries ◆◆◆ U ◆◆◆}\par\nobreak
+ \item U\leaders\hbox{$\cdot$}\hfill 408
+ \item u\leaders\hbox{$\cdot$}\hfill 409
+ \item Ü\leaders\hbox{$\cdot$}\hfill 410
+ \item ü\leaders\hbox{$\cdot$}\hfill 411
+
+\end{theindex}
diff --git a/indexing/upmendex/source/tests/attr1.ist b/indexing/upmendex/source/tests/attr1.ist
new file mode 100644
index 0000000000..301aff0a6a
--- /dev/null
+++ b/indexing/upmendex/source/tests/attr1.ist
@@ -0,0 +1,2 @@
+icu_attributes "case-first:upper-first
+numeric-ordering:on "
diff --git a/indexing/upmendex/source/tests/locale1.ind b/indexing/upmendex/source/tests/locale1.ind
new file mode 100644
index 0000000000..76bf98f25a
--- /dev/null
+++ b/indexing/upmendex/source/tests/locale1.ind
@@ -0,0 +1,48 @@
+\begin{theindex}
+
+\centerline{\bfseries ◆◆◆ Symbols ◆◆◆}\par\nobreak
+ \item #\leaders\hbox{$\cdot$}\hfill 2000
+ \item $\leaders\hbox{$\cdot$}\hfill 2001
+ \item 10\leaders\hbox{$\cdot$}\hfill 1000
+ \item 2\leaders\hbox{$\cdot$}\hfill 1001
+
+ \indexspace
+
+\centerline{\bfseries ◆◆◆ A ◆◆◆}\par\nobreak
+ \item a\leaders\hbox{$\cdot$}\hfill 401
+ \item A\leaders\hbox{$\cdot$}\hfill 400
+
+ \indexspace
+
+\centerline{\bfseries ◆◆◆ B ◆◆◆}\par\nobreak
+ \item b\leaders\hbox{$\cdot$}\hfill 403
+ \item B\leaders\hbox{$\cdot$}\hfill 402
+
+ \indexspace
+
+\centerline{\bfseries ◆◆◆ C ◆◆◆}\par\nobreak
+ \item c\leaders\hbox{$\cdot$}\hfill 405
+ \item C\leaders\hbox{$\cdot$}\hfill 404
+ \item Ch\leaders\hbox{$\cdot$}\hfill 420
+ \item cha\leaders\hbox{$\cdot$}\hfill 421
+ \item cz\leaders\hbox{$\cdot$}\hfill 422
+
+ \indexspace
+
+\centerline{\bfseries ◆◆◆ Ç ◆◆◆}\par\nobreak
+ \item ç\leaders\hbox{$\cdot$}\hfill 407
+ \item Ç\leaders\hbox{$\cdot$}\hfill 406
+
+ \indexspace
+
+\centerline{\bfseries ◆◆◆ U ◆◆◆}\par\nobreak
+ \item u\leaders\hbox{$\cdot$}\hfill 409
+ \item U\leaders\hbox{$\cdot$}\hfill 408
+
+ \indexspace
+
+\centerline{\bfseries ◆◆◆ Ü ◆◆◆}\par\nobreak
+ \item ü\leaders\hbox{$\cdot$}\hfill 411
+ \item Ü\leaders\hbox{$\cdot$}\hfill 410
+
+\end{theindex}
diff --git a/indexing/upmendex/source/tests/locale1.ist b/indexing/upmendex/source/tests/locale1.ist
new file mode 100644
index 0000000000..cf9e67da6e
--- /dev/null
+++ b/indexing/upmendex/source/tests/locale1.ist
@@ -0,0 +1 @@
+icu_locale "az"
diff --git a/indexing/upmendex/source/tests/locale2-rule.ind b/indexing/upmendex/source/tests/locale2-rule.ind
new file mode 100644
index 0000000000..4054762ab7
--- /dev/null
+++ b/indexing/upmendex/source/tests/locale2-rule.ind
@@ -0,0 +1,44 @@
+\begin{theindex}
+
+\centerline{\bfseries ◆◆◆ Symbols ◆◆◆}\par\nobreak
+ \item #\leaders\hbox{$\cdot$}\hfill 2000
+ \item $\leaders\hbox{$\cdot$}\hfill 2001
+ \item 10\leaders\hbox{$\cdot$}\hfill 1000
+ \item 2\leaders\hbox{$\cdot$}\hfill 1001
+
+ \indexspace
+
+\centerline{\bfseries ◆◆◆ A ◆◆◆}\par\nobreak
+ \item a\leaders\hbox{$\cdot$}\hfill 401
+ \item A\leaders\hbox{$\cdot$}\hfill 400
+
+ \indexspace
+
+\centerline{\bfseries ◆◆◆ C ◆◆◆}\par\nobreak
+ \item C\leaders\hbox{$\cdot$}\hfill 404
+ \item Ç\leaders\hbox{$\cdot$}\hfill 406
+ \item c\leaders\hbox{$\cdot$}\hfill 405
+ \item ç\leaders\hbox{$\cdot$}\hfill 407
+ \item cz\leaders\hbox{$\cdot$}\hfill 422
+
+ \indexspace
+
+\centerline{\bfseries ◆◆◆ U ◆◆◆}\par\nobreak
+ \item u\leaders\hbox{$\cdot$}\hfill 409
+ \item U\leaders\hbox{$\cdot$}\hfill 408
+ \item ü\leaders\hbox{$\cdot$}\hfill 411
+ \item Ü\leaders\hbox{$\cdot$}\hfill 410
+
+ \indexspace
+
+\centerline{\bfseries ◆◆◆ B ◆◆◆}\par\nobreak
+ \item b\leaders\hbox{$\cdot$}\hfill 403
+ \item B\leaders\hbox{$\cdot$}\hfill 402
+
+ \indexspace
+
+\centerline{\bfseries ◆◆◆ CH ◆◆◆}\par\nobreak
+ \item Ch\leaders\hbox{$\cdot$}\hfill 420
+ \item cha\leaders\hbox{$\cdot$}\hfill 421
+
+\end{theindex}
diff --git a/indexing/upmendex/source/tests/locale2.ist b/indexing/upmendex/source/tests/locale2.ist
new file mode 100644
index 0000000000..89b590a21b
--- /dev/null
+++ b/indexing/upmendex/source/tests/locale2.ist
@@ -0,0 +1 @@
+icu_locale "es@collation=traditional"
diff --git a/indexing/upmendex/source/tests/pprec3.ist b/indexing/upmendex/source/tests/pprec3.ist
new file mode 100644
index 0000000000..b8f33af1dd
--- /dev/null
+++ b/indexing/upmendex/source/tests/pprec3.ist
@@ -0,0 +1,11 @@
+% -*- coding: utf-8 -*-
+
+page_precedence "naA"
+
+delim_0 "[[delim0]]"
+delim_1 "[[delim1]]"
+delim_2 "[[delim2]]"
+delim_r "[[delimr]]"
+suffix_2p "[[sfx 2p]]"
+suffix_3p "[[sfx 3p]]"
+
diff --git a/indexing/upmendex/source/tests/pprec4.ist b/indexing/upmendex/source/tests/pprec4.ist
new file mode 100644
index 0000000000..768fec7642
--- /dev/null
+++ b/indexing/upmendex/source/tests/pprec4.ist
@@ -0,0 +1,11 @@
+% -*- coding: utf-8 -*-
+
+page_precedence "Ana"
+
+delim_0 "[[delim0]]"
+delim_1 "[[delim1]]"
+delim_2 "[[delim2]]"
+delim_r "[[delimr]]"
+suffix_2p "[[sfx 2p]]"
+suffix_3p "[[sfx 3p]]"
+
diff --git a/indexing/upmendex/source/tests/pprecB-0.ind b/indexing/upmendex/source/tests/pprecB-0.ind
new file mode 100644
index 0000000000..1e23d9fb4a
--- /dev/null
+++ b/indexing/upmendex/source/tests/pprecB-0.ind
@@ -0,0 +1,5 @@
+\begin{theindex}
+
+ \item entryA[[delim0]]1[[sfx 3p]], a[[sfx 3p]], A[[sfx 3p]]
+
+\end{theindex}
diff --git a/indexing/upmendex/source/tests/pprecB-3.ind b/indexing/upmendex/source/tests/pprecB-3.ind
new file mode 100644
index 0000000000..1e23d9fb4a
--- /dev/null
+++ b/indexing/upmendex/source/tests/pprecB-3.ind
@@ -0,0 +1,5 @@
+\begin{theindex}
+
+ \item entryA[[delim0]]1[[sfx 3p]], a[[sfx 3p]], A[[sfx 3p]]
+
+\end{theindex}
diff --git a/indexing/upmendex/source/tests/pprecB-4.ind b/indexing/upmendex/source/tests/pprecB-4.ind
new file mode 100644
index 0000000000..3943ad4854
--- /dev/null
+++ b/indexing/upmendex/source/tests/pprecB-4.ind
@@ -0,0 +1,5 @@
+\begin{theindex}
+
+ \item entryA[[delim0]]A[[sfx 3p]], 1[[sfx 3p]], a[[sfx 3p]]
+
+\end{theindex}
diff --git a/indexing/upmendex/source/tests/pprecB.idx b/indexing/upmendex/source/tests/pprecB.idx
new file mode 100644
index 0000000000..a1d8fc31ae
--- /dev/null
+++ b/indexing/upmendex/source/tests/pprecB.idx
@@ -0,0 +1,9 @@
+\indexentry{entryA}{a}
+\indexentry{entryA}{b}
+\indexentry{entryA}{c}
+\indexentry{entryA}{1}
+\indexentry{entryA}{2}
+\indexentry{entryA}{3}
+\indexentry{entryA}{A}
+\indexentry{entryA}{B}
+\indexentry{entryA}{C}
diff --git a/indexing/upmendex/source/tests/range.idx b/indexing/upmendex/source/tests/range.idx
new file mode 100644
index 0000000000..3eccb113aa
--- /dev/null
+++ b/indexing/upmendex/source/tests/range.idx
@@ -0,0 +1,126 @@
+\indexentry{entryA}{1}
+\indexentry{entryA}{2}
+\indexentry{entryA}{iii}
+\indexentry{entryA}{iv}
+\indexentry{entryA}{IX}
+\indexentry{entryA}{X}
+
+\indexentry{entryB}{1}
+\indexentry{entryB}{2}
+\indexentry{entryB}{3}
+\indexentry{entryB}{iii}
+\indexentry{entryB}{iv}
+\indexentry{entryB}{v}
+\indexentry{entryB}{IX}
+\indexentry{entryB}{X}
+\indexentry{entryB}{XI}
+
+\indexentry{entryC}{1}
+\indexentry{entryC}{2}
+\indexentry{entryC}{3}
+\indexentry{entryC}{4}
+\indexentry{entryC|bold}{iii}
+\indexentry{entryC|bold}{iv}
+\indexentry{entryC|bold}{v}
+\indexentry{entryC|bold}{vi}
+\indexentry{entryC}{IX}
+\indexentry{entryC|bold}{X}
+\indexentry{entryC}{XI}
+\indexentry{entryC}{XII}
+
+\indexentry{entryD}{c-1}
+\indexentry{entryD}{c-2}
+\indexentry{entryD}{i-iii}
+\indexentry{entryD}{i-iv}
+\indexentry{entryD}{d-IX}
+\indexentry{entryD}{d-X}
+
+\indexentry{entryE}{c-1}
+\indexentry{entryE}{c-2}
+\indexentry{entryE}{c-3}
+\indexentry{entryE}{i-iii}
+\indexentry{entryE}{i-iv}
+\indexentry{entryE}{i-v}
+\indexentry{entryE}{d-IX}
+\indexentry{entryE}{d-X}
+\indexentry{entryE}{d-XI}
+
+\indexentry{entryF}{C-I}
+\indexentry{entryF}{C-II}
+\indexentry{entryF}{C-III}
+\indexentry{entryF}{C-IV}
+\indexentry{entryF}{V-LIII}
+\indexentry{entryF}{V-LIV}
+\indexentry{entryF}{V-LV}
+\indexentry{entryF}{V-LVI}
+\indexentry{entryF}{D-DI}
+\indexentry{entryF}{D-DII}
+\indexentry{entryF}{D-DIII}
+\indexentry{entryF}{D-DIV}
+
+\indexentry{entryP}{A-a-1-VII-iii}
+\indexentry{entryP}{A-a-2-VII-iii}
+\indexentry{entryP}{A-a-3-VII-iii}
+\indexentry{entryP}{A-a-1-VIII-iii}
+\indexentry{entryP}{A-a-1-IX-iii}
+\indexentry{entryP}{A-a-1-VII-iv}
+\indexentry{entryP}{A-a-1-VII-v}
+
+\indexentry{entryQ}{1-2-3-4-5-6-7-8-9-10}
+\indexentry{entryQ}{1-2-3-4-5-6-7-8-9-11}
+\indexentry{entryQ}{1-2-3-4-5-6-7-8-9-14}
+\indexentry{entryQ}{1-2-3-4-5-6-7-8-9-15}
+\indexentry{entryQ}{1-2-3-4-5-6-7-8-9-16}
+\indexentry{entryQ}{1-2-3-4-5-6-7-8-9-22}
+\indexentry{entryQ}{1-2-3-4-5-6-7-8-9-23}
+\indexentry{entryQ}{1-2-3-4-5-6-7-8-9-24}
+\indexentry{entryQ}{1-2-3-4-5-6-7-8-9-25}
+\indexentry{entryQ}{1-2-3-4-5-6-7-8-9-i}
+\indexentry{entryQ}{1-2-3-4-5-6-7-8-9-ii}
+\indexentry{entryQ}{1-2-3-4-5-6-7-8-9-iii}
+\indexentry{entryQ}{1-2-3-4-5-6-7-8-9-iv}
+\indexentry{entryQ}{1-2-3-4-5-6-7-8-9-v}
+
+\indexentry{entryA1|(}{1}
+\indexentry{entryA1|)}{2}
+\indexentry{entryA1|(}{iii}
+\indexentry{entryA1|)}{iv}
+\indexentry{entryA1|(}{IX}
+\indexentry{entryA1|)}{X}
+
+\indexentry{entryB1|(}{1}
+\indexentry{entryB1|)}{3}
+\indexentry{entryB1|(}{iii}
+\indexentry{entryB1|)}{v}
+\indexentry{entryB1|(}{IX}
+\indexentry{entryB1|)}{XI}
+
+\indexentry{entryC1|(}{1}
+\indexentry{entryC1|)}{4}
+\indexentry{entryC1|(bold}{iii}
+\indexentry{entryC1|)}{vi}
+\indexentry{entryC1|(}{IX}
+\indexentry{entryC1|bold}{X}
+\indexentry{entryC1|)}{XII}
+
+\indexentry{entryD1|(}{c-1}
+\indexentry{entryD1|)}{c-2}
+\indexentry{entryD1|(}{i-iii}
+\indexentry{entryD1|)}{i-iv}
+\indexentry{entryD1|(}{d-IX}
+\indexentry{entryD1|)}{d-X}
+
+\indexentry{entryE1|(}{c-1}
+\indexentry{entryE1|)}{c-3}
+\indexentry{entryE1|(}{i-iii}
+\indexentry{entryE1|)}{i-v}
+\indexentry{entryE1|(}{d-IX}
+\indexentry{entryE1|)}{d-XI}
+
+\indexentry{entryF1|(}{C-I}
+\indexentry{entryF1|)}{C-IV}
+\indexentry{entryF1|(}{V-LIII}
+\indexentry{entryF1|)}{V-LVI}
+\indexentry{entryF1|(}{D-DI}
+\indexentry{entryF1|bold}{D-DII}
+\indexentry{entryF1|)}{D-DIV}
diff --git a/indexing/upmendex/source/tests/range1.ind b/indexing/upmendex/source/tests/range1.ind
new file mode 100644
index 0000000000..c5f047a55e
--- /dev/null
+++ b/indexing/upmendex/source/tests/range1.ind
@@ -0,0 +1,25 @@
+\begin{theindex}
+
+ \item entryA, 1[[sfx 2p]], iii[[sfx 2p]], IX[[sfx 2p]]
+ \item entryA1, 1[[sfx 2p]], iii[[sfx 2p]], IX[[sfx 2p]]
+ \item entryB, 1[[sfx mp]], iii[[sfx mp]], IX[[sfx mp]]
+ \item entryB1, 1[[sfx mp]], iii[[sfx mp]], IX[[sfx mp]]
+ \item entryC, 1[[sfx mp]], \bold{iii[[sfx mp]]}, IX, \bold{X},
+ XI[[sfx 2p]]
+ \item entryC1, 1[[sfx mp]], \bold{iii[[sfx mp]]}, \bold{X},
+ IX[[sfx mp]]
+ \item entryD, c-1[[sfx 2p]], d-IX[[sfx 2p]], i-iii[[sfx 2p]]
+ \item entryD1, c-1[[sfx 2p]], d-IX[[sfx 2p]], i-iii[[sfx 2p]]
+ \item entryE, c-1[[sfx mp]], d-IX[[sfx mp]], i-iii[[sfx mp]]
+ \item entryE1, c-1[[sfx mp]], d-IX[[sfx mp]], i-iii[[sfx mp]]
+ \item entryF, C-I[[sfx mp]], D-DI[[sfx mp]], V-LIII[[sfx mp]]
+ \item entryF1, C-I[[sfx mp]], \bold{D-DII}, D-DI[[sfx mp]],
+ V-LIII[[sfx mp]]
+ \item entryP, A-a-1-VII-iii[[sfx mp]], A-a-1-VIII-iii, A-a-1-IX-iii,
+ A-a-2-VII-iii, A-a-3-VII-iii
+ \item entryQ, 1-2-3-4-5-6-7-8-9-10[[sfx 2p]],
+ 1-2-3-4-5-6-7-8-9-14[[sfx mp]],
+ 1-2-3-4-5-6-7-8-9-22[[sfx mp]],
+ 1-2-3-4-5-6-7-8-9-i[[sfx mp]]
+
+\end{theindex}
diff --git a/indexing/upmendex/source/tests/range1.ist b/indexing/upmendex/source/tests/range1.ist
new file mode 100644
index 0000000000..116c2d8713
--- /dev/null
+++ b/indexing/upmendex/source/tests/range1.ist
@@ -0,0 +1,3 @@
+page_precedence "nrRaA"
+suffix_2p "[[sfx 2p]]"
+suffix_mp "[[sfx mp]]"
diff --git a/indexing/upmendex/source/tests/range2.ind b/indexing/upmendex/source/tests/range2.ind
new file mode 100644
index 0000000000..789eb3f75c
--- /dev/null
+++ b/indexing/upmendex/source/tests/range2.ind
@@ -0,0 +1,26 @@
+\begin{theindex}
+
+ \item entryA, 1[[sfx 2p]], iii[[sfx 2p]], IX[[sfx 2p]]
+ \item entryA1, 1[[sfx 2p]], iii[[sfx 2p]], IX[[sfx 2p]]
+ \item entryB, 1[[sfx 3p]], iii[[sfx 3p]], IX[[sfx 3p]]
+ \item entryB1, 1[[sfx 3p]], iii[[sfx 3p]], IX[[sfx 3p]]
+ \item entryC, 1[[delim r]]4, \bold{iii[[delim r]]vi}, IX, \bold{X},
+ XI[[sfx 2p]]
+ \item entryC1, 1[[delim r]]4, \bold{iii[[delim r]]vi}, \bold{X},
+ IX[[delim r]]XII
+ \item entryD, c-1[[sfx 2p]], d-IX[[sfx 2p]], i-iii[[sfx 2p]]
+ \item entryD1, c-1[[sfx 2p]], d-IX[[sfx 2p]], i-iii[[sfx 2p]]
+ \item entryE, c-1[[sfx 3p]], d-IX[[sfx 3p]], i-iii[[sfx 3p]]
+ \item entryE1, c-1[[sfx 3p]], d-IX[[sfx 3p]], i-iii[[sfx 3p]]
+ \item entryF, C-I[[delim r]]C-IV, D-DI[[delim r]]D-DIV,
+ V-LIII[[delim r]]V-LVI
+ \item entryF1, C-I[[delim r]]C-IV, \bold{D-DII}, D-DI[[delim r]]D-DIV,
+ V-LIII[[delim r]]V-LVI
+ \item entryP, A-a-1-VII-iii[[sfx 3p]], A-a-1-VIII-iii, A-a-1-IX-iii,
+ A-a-2-VII-iii, A-a-3-VII-iii
+ \item entryQ, 1-2-3-4-5-6-7-8-9-10[[sfx 2p]],
+ 1-2-3-4-5-6-7-8-9-14[[sfx 3p]],
+ 1-2-3-4-5-6-7-8-9-22[[delim r]]1-2-3-4-5-6-7-8-9-25,
+ 1-2-3-4-5-6-7-8-9-i[[delim r]]1-2-3-4-5-6-7-8-9-v
+
+\end{theindex}
diff --git a/indexing/upmendex/source/tests/range2.ist b/indexing/upmendex/source/tests/range2.ist
new file mode 100644
index 0000000000..552e5fb674
--- /dev/null
+++ b/indexing/upmendex/source/tests/range2.ist
@@ -0,0 +1,4 @@
+page_precedence "nrRaA"
+delim_r "[[delim r]]"
+suffix_2p "[[sfx 2p]]"
+suffix_3p "[[sfx 3p]]"
diff --git a/indexing/upmendex/source/tests/range3.ind b/indexing/upmendex/source/tests/range3.ind
new file mode 100644
index 0000000000..519730a138
--- /dev/null
+++ b/indexing/upmendex/source/tests/range3.ind
@@ -0,0 +1,25 @@
+\begin{theindex}
+
+ \item entryA, 1[[sfx 2p]], iii[[sfx 2p]], IX[[sfx 2p]]
+ \item entryA1, 1[[sfx 2p]], iii[[sfx 2p]], IX[[sfx 2p]]
+ \item entryB, 1[[sfx 3p]], iii[[sfx 3p]], IX[[sfx 3p]]
+ \item entryB1, 1[[sfx 3p]], iii[[sfx 3p]], IX[[sfx 3p]]
+ \item entryC, 1[[sfx mp]], \bold{iii[[sfx mp]]}, IX, \bold{X},
+ XI[[sfx 2p]]
+ \item entryC1, 1[[sfx mp]], \bold{iii[[sfx mp]]}, \bold{X},
+ IX[[sfx mp]]
+ \item entryD, c-1[[sfx 2p]], d-IX[[sfx 2p]], i-iii[[sfx 2p]]
+ \item entryD1, c-1[[sfx 2p]], d-IX[[sfx 2p]], i-iii[[sfx 2p]]
+ \item entryE, c-1[[sfx 3p]], d-IX[[sfx 3p]], i-iii[[sfx 3p]]
+ \item entryE1, c-1[[sfx 3p]], d-IX[[sfx 3p]], i-iii[[sfx 3p]]
+ \item entryF, C-I[[sfx mp]], D-DI[[sfx mp]], V-LIII[[sfx mp]]
+ \item entryF1, C-I[[sfx mp]], \bold{D-DII}, D-DI[[sfx mp]],
+ V-LIII[[sfx mp]]
+ \item entryP, A-a-1-VII-iii[[sfx 3p]], A-a-1-VIII-iii, A-a-1-IX-iii,
+ A-a-2-VII-iii, A-a-3-VII-iii
+ \item entryQ, 1-2-3-4-5-6-7-8-9-10[[sfx 2p]],
+ 1-2-3-4-5-6-7-8-9-14[[sfx 3p]],
+ 1-2-3-4-5-6-7-8-9-22[[sfx mp]],
+ 1-2-3-4-5-6-7-8-9-i[[sfx mp]]
+
+\end{theindex}
diff --git a/indexing/upmendex/source/tests/range3.ist b/indexing/upmendex/source/tests/range3.ist
new file mode 100644
index 0000000000..b751471b43
--- /dev/null
+++ b/indexing/upmendex/source/tests/range3.ist
@@ -0,0 +1,4 @@
+page_precedence "nrRaA"
+suffix_2p "[[sfx 2p]]"
+suffix_3p "[[sfx 3p]]"
+suffix_mp "[[sfx mp]]"
diff --git a/indexing/upmendex/source/tests/rangetwo.idx b/indexing/upmendex/source/tests/rangetwo.idx
deleted file mode 100644
index d941f7e831..0000000000
--- a/indexing/upmendex/source/tests/rangetwo.idx
+++ /dev/null
@@ -1,2 +0,0 @@
-\indexentry{entryA}{1}
-\indexentry{entryA}{2}
diff --git a/indexing/upmendex/source/tests/rangetwo.ind b/indexing/upmendex/source/tests/rangetwo.ind
deleted file mode 100644
index 1119d2fa8e..0000000000
--- a/indexing/upmendex/source/tests/rangetwo.ind
+++ /dev/null
@@ -1,5 +0,0 @@
-\begin{theindex}
-
- \item entryA, 1[[sfx2p]]
-
-\end{theindex}
diff --git a/indexing/upmendex/source/tests/rangetwo.ist b/indexing/upmendex/source/tests/rangetwo.ist
deleted file mode 100644
index 3e986c8935..0000000000
--- a/indexing/upmendex/source/tests/rangetwo.ist
+++ /dev/null
@@ -1,2 +0,0 @@
-suffix_2p "[[sfx2p]]"
-
diff --git a/indexing/upmendex/source/tests/rule.idx b/indexing/upmendex/source/tests/rule.idx
new file mode 100644
index 0000000000..783cce9fd4
--- /dev/null
+++ b/indexing/upmendex/source/tests/rule.idx
@@ -0,0 +1,20 @@
+% -*- coding: utf-8 -*-
+\indexentry{A}{400}
+\indexentry{a}{401}
+\indexentry{B}{402}
+\indexentry{b}{403}
+\indexentry{C}{404}
+\indexentry{c}{405}
+\indexentry{Ç}{406}
+\indexentry{ç}{407}
+\indexentry{Ch}{420}
+\indexentry{cha}{421}
+\indexentry{cz}{422}
+\indexentry{U}{408}
+\indexentry{u}{409}
+\indexentry{Ü}{410}
+\indexentry{ü}{411}
+\indexentry{10}{1000}
+\indexentry{2}{1001}
+\indexentry{#}{2000}
+\indexentry{$}{2001}
diff --git a/indexing/upmendex/source/tests/rule.ind b/indexing/upmendex/source/tests/rule.ind
new file mode 100644
index 0000000000..3ac1335e05
--- /dev/null
+++ b/indexing/upmendex/source/tests/rule.ind
@@ -0,0 +1,40 @@
+\begin{theindex}
+
+\centerline{\bfseries ◆◆◆ Symbols ◆◆◆}\par\nobreak
+ \item #\leaders\hbox{$\cdot$}\hfill 2000
+ \item $\leaders\hbox{$\cdot$}\hfill 2001
+ \item 10\leaders\hbox{$\cdot$}\hfill 1000
+ \item 2\leaders\hbox{$\cdot$}\hfill 1001
+
+ \indexspace
+
+\centerline{\bfseries ◆◆◆ A ◆◆◆}\par\nobreak
+ \item a\leaders\hbox{$\cdot$}\hfill 401
+ \item A\leaders\hbox{$\cdot$}\hfill 400
+
+ \indexspace
+
+\centerline{\bfseries ◆◆◆ C ◆◆◆}\par\nobreak
+ \item C\leaders\hbox{$\cdot$}\hfill 404
+ \item Ç\leaders\hbox{$\cdot$}\hfill 406
+ \item c\leaders\hbox{$\cdot$}\hfill 405
+ \item ç\leaders\hbox{$\cdot$}\hfill 407
+ \item Ch\leaders\hbox{$\cdot$}\hfill 420
+ \item cha\leaders\hbox{$\cdot$}\hfill 421
+ \item cz\leaders\hbox{$\cdot$}\hfill 422
+
+ \indexspace
+
+\centerline{\bfseries ◆◆◆ U ◆◆◆}\par\nobreak
+ \item u\leaders\hbox{$\cdot$}\hfill 409
+ \item U\leaders\hbox{$\cdot$}\hfill 408
+ \item ü\leaders\hbox{$\cdot$}\hfill 411
+ \item Ü\leaders\hbox{$\cdot$}\hfill 410
+
+ \indexspace
+
+\centerline{\bfseries ◆◆◆ B ◆◆◆}\par\nobreak
+ \item b\leaders\hbox{$\cdot$}\hfill 403
+ \item B\leaders\hbox{$\cdot$}\hfill 402
+
+\end{theindex}
diff --git a/indexing/upmendex/source/tests/rule1.ist b/indexing/upmendex/source/tests/rule1.ist
new file mode 100644
index 0000000000..c986e39e96
--- /dev/null
+++ b/indexing/upmendex/source/tests/rule1.ist
@@ -0,0 +1,3 @@
+% -*- coding: utf-8 -*-
+
+icu_rules "&a<<<A<C<<Ç<<c<<ç<u<<U<<ü<<Ü<b<<B"
diff --git a/indexing/upmendex/source/tests/rule2.ist b/indexing/upmendex/source/tests/rule2.ist
new file mode 100644
index 0000000000..a61af0a546
--- /dev/null
+++ b/indexing/upmendex/source/tests/rule2.ist
@@ -0,0 +1,3 @@
+% -*- coding: utf-8 -*-
+
+icu_rules "&a<<<A<C<<\U000000c7<<c<<\u00E7<u<<U<<\x{fc}<<\xDC<b<<B"
diff --git a/indexing/upmendex/source/tests/rule3.ist b/indexing/upmendex/source/tests/rule3.ist
new file mode 100644
index 0000000000..7ab835a801
--- /dev/null
+++ b/indexing/upmendex/source/tests/rule3.ist
@@ -0,0 +1,4 @@
+% -*- coding: latin-1 -*-
+% To test malformed input
+
+icu_rules "&a<<<A<C<<<<c<<<u<<U<<<<<b<<B"
diff --git a/indexing/upmendex/source/tests/rule4.ist b/indexing/upmendex/source/tests/rule4.ist
new file mode 100644
index 0000000000..0c30dceba2
--- /dev/null
+++ b/indexing/upmendex/source/tests/rule4.ist
@@ -0,0 +1,4 @@
+% -*- coding: utf-8 -*-
+% To test malformed input
+
+icu_rules "&a<<<A<C<<\U000000c7<<c<<\uE7<u<<U<<\x{fc}<<\xDC<b<<B"
diff --git a/indexing/upmendex/source/tests/upmendex.test b/indexing/upmendex/source/tests/upmendex.test
index b64c045af9..84dfbd17be 100755
--- a/indexing/upmendex/source/tests/upmendex.test
+++ b/indexing/upmendex/source/tests/upmendex.test
@@ -1,5 +1,5 @@
#! /bin/sh -vx
-# $Id$
+# $Id: upmendex.test 60549 2021-09-19 08:57:18Z takuji $
# Copyright 2017 Karl Berry <tex-live@tug.org>
# Copyright 2013 Peter Breitenlohner <tex-live@tug.org>
# You may freely use, modify and/or distribute this file.
@@ -7,7 +7,7 @@
TEXMFCNF=$srcdir/../kpathsea
export TEXMFCNF
-rm -f foo.* uni.* pprec*.* rangetwo.*
+rm -f foo.* uni.* range*.* pprec*.* locale*.* attr*.* rule*.*
./upmendex $srcdir/tests/foo.idx -o foo.ind1 -t foo.ilg1 \
&& diff $srcdir/tests/foo.ind foo.ind1 || exit 1
@@ -28,10 +28,18 @@ cat $srcdir/tests/uni.idx | \
$srcdir/tests/uni.idx -o uni.ind3 -t uni.ilg3 \
&& diff $srcdir/tests/uni.ind uni.ind3 || exit 1
-# test for two-element range suffix_2p
-./upmendex -s $srcdir/tests/rangetwo.ist $srcdir/tests/rangetwo.idx \
- -o rangetwo.ind1 -t rangetwo.ilg \
- && diff $srcdir/tests/rangetwo.ind rangetwo.ind1 || exit 1
+# test for range suffix_2p, suffix_3p, suffix_mp
+./upmendex -s $srcdir/tests/range1.ist $srcdir/tests/range.idx \
+ -o range1.ind1 -t range1.ilg \
+ && diff $srcdir/tests/range1.ind range1.ind1 || exit 1
+
+./upmendex -s $srcdir/tests/range2.ist $srcdir/tests/range.idx \
+ -o range2.ind1 -t range2.ilg \
+ && diff $srcdir/tests/range2.ind range2.ind1 || exit 1
+
+./upmendex -s $srcdir/tests/range3.ist $srcdir/tests/range.idx \
+ -o range3.ind1 -t range3.ilg \
+ && diff $srcdir/tests/range3.ind range3.ind1 || exit 1
# test for page_precedence and suffix_3p
./upmendex -s $srcdir/tests/pprec0.ist $srcdir/tests/pprecA.idx \
@@ -46,3 +54,57 @@ cat $srcdir/tests/uni.idx | \
-o pprecA-2.ind1 -t pprecA-2.ilg \
&& diff $srcdir/tests/pprecA-2.ind pprecA-2.ind1 || exit 1
+./upmendex -s $srcdir/tests/pprec0.ist $srcdir/tests/pprecB.idx \
+ -o pprecB-0.ind1 -t pprecB-0.ilg \
+ && diff $srcdir/tests/pprecB-0.ind pprecB-0.ind1 || exit 1
+
+./upmendex -s $srcdir/tests/pprec3.ist $srcdir/tests/pprecB.idx \
+ -o pprecB-3.ind1 -t pprecB-3.ilg \
+ && diff $srcdir/tests/pprecB-3.ind pprecB-3.ind1 || exit 1
+
+./upmendex -s $srcdir/tests/pprec4.ist $srcdir/tests/pprecB.idx \
+ -o pprecB-4.ind1 -t pprecB-4.ilg \
+ && diff $srcdir/tests/pprecB-4.ind pprecB-4.ind1 || exit 1
+
+# test for icu_locale
+./upmendex -s $srcdir/tests/uni.ist -s $srcdir/tests/locale1.ist $srcdir/tests/rule.idx \
+ -o locale1.ind1 -t locale1.ilg \
+ && diff $srcdir/tests/locale1.ind locale1.ind1 || exit 1
+
+# test for icu_attributes
+./upmendex -s $srcdir/tests/uni.ist -s $srcdir/tests/attr1.ist $srcdir/tests/rule.idx \
+ -o attr1.ind1 -t attr1.ilg \
+ && diff $srcdir/tests/attr1.ind attr1.ind1 || exit 1
+
+# test for icu_rules
+./upmendex -s $srcdir/tests/uni.ist -s $srcdir/tests/rule1.ist $srcdir/tests/rule.idx \
+ -o rule1.ind1 -t rule1.ilg \
+ && diff $srcdir/tests/rule.ind rule1.ind1 || exit 1
+
+./upmendex -s $srcdir/tests/uni.ist -s $srcdir/tests/rule2.ist $srcdir/tests/rule.idx \
+ -o rule2.ind1 -t rule2.ilg \
+ && diff $srcdir/tests/rule.ind rule2.ind1 || exit 1
+
+# malformed input
+./upmendex -s $srcdir/tests/uni.ist -s $srcdir/tests/rule3.ist $srcdir/tests/rule.idx \
+ -o rule3.ind1 -t rule3.ilg
+if [ $? -eq 254 ]; then
+ echo "Error check: OK"
+else
+ echo "Error check: NG"
+ exit 1
+fi
+
+./upmendex -s $srcdir/tests/uni.ist -s $srcdir/tests/rule4.ist $srcdir/tests/rule.idx \
+ -o rule4.ind1 -t rule4.ilg
+if [ $? -eq 254 ]; then
+ echo "Error check: OK"
+else
+ echo "Error check: NG"
+ exit 1
+fi
+
+# test for icu_locale + icu_rules
+./upmendex -s $srcdir/tests/uni.ist -s $srcdir/tests/locale2.ist -s $srcdir/tests/rule1.ist $srcdir/tests/rule.idx \
+ -o locale2-rule1.ind1 -t locale2-rule1.ilg \
+ && diff $srcdir/tests/locale2-rule.ind locale2-rule1.ind1 || exit 1
diff --git a/indexing/upmendex/source/var.h b/indexing/upmendex/source/var.h
index 02e2714910..c9275fcc91 100644
--- a/indexing/upmendex/source/var.h
+++ b/indexing/upmendex/source/var.h
@@ -4,7 +4,7 @@ FILE *efp;
int lines,idxcount,acc=0,reject=0;
int prange=1,fsti=0,lorder=0,bcomp=0,force=0,fpage=0,gflg=0,verb=1,debug=0;
-int warn=0,scount=0,pattr[3]={0,0,0};
+int warn=0,scount=0,pattr[PAGE_COMPOSIT_DEPTH]={0};
/* static char roman[]={"ivxlcdm"},Roman[]={"IVXLCDM"}; */
struct index *ind;
@@ -27,7 +27,7 @@ int line_max=72;
char indent_space[STYBUFSIZE]={"\t\t"};
int indent_length=16;
int priority=0;
-char symbol[STYBUFSIZE]={""};
+char symhead[STYBUFSIZE]={""},numhead[STYBUFSIZE]={""};
char symhead_positive[STYBUFSIZE]={"Symbols"},symhead_negative[STYBUFSIZE]={"symbols"};
char numhead_positive[STYBUFSIZE]={"Numbers"},numhead_negative[STYBUFSIZE]={"numbers"};
int symbol_flag=1;
@@ -35,7 +35,8 @@ int letter_head=1;
UChar atama[STYBUFSIZE],hangul_head[STYBUFSIZE],hanzi_head[STYBUFSIZE]={L'\0'},kana_head[STYBUFSIZE]={L'\0'};
UChar devanagari_head[STYBUFSIZE],thai_head[STYBUFSIZE];
char page_compositor[STYBUFSIZE]={"-"},page_precedence[STYBUFSIZE]={"rnaRA"};
-char character_order[STYBUFSIZE]={"SNLGCJKHDT"};
+char character_order[STYBUFSIZE]={"SNLGCJKHDTah"};
+char script_preamble[11][STYBUFSIZE],script_postamble[11][STYBUFSIZE];
char icu_locale[STYBUFSIZE]={"root"},icu_rules[STYBUFSIZE]={""};
int icu_attributes[UCOL_ATTRIBUTE_COUNT];