diff options
Diffstat (limited to 'indexing/upmendex/source')
32 files changed, 792 insertions, 222 deletions
diff --git a/indexing/upmendex/source/ChangeLog b/indexing/upmendex/source/ChangeLog index b2b3cd2f1e..ac5a331ffb 100644 --- a/indexing/upmendex/source/ChangeLog +++ b/indexing/upmendex/source/ChangeLog @@ -1,3 +1,93 @@ +2023-02-26 TANAKA Takuji <ttk@t-lab.opal.ne.jp> + + * hanzi.h: Update references. + +2022-11-20 TANAKA Takuji <ttk@t-lab.opal.ne.jp> + + * version 1.07 Stable version. + * configure.ac: Bump version. + * sort.c, fwrite.c, {,ex}hanzi.h: + Improve detection of hanzi header to support + Chinese polyphone (多音字). + +2022-11-16 TANAKA Takuji <ttk@t-lab.opal.ne.jp> + + * version 1.06 Stable version. + * configure.ac: Bump version. + * sort.c: Add parameter option "@" to character_order, + which denotes character order is defined by icu_rules. + * fwrite.c: Better treatment for Marathi KSSA, JNYA. + * upmendex.ja.txt: Update document. + +2022-10-08 TANAKA Takuji <ttk@t-lab.opal.ne.jp> + + * version 1.05 Stable version. + * configure.ac: Bump version. + * main.c, sort.c, convert.c, fwrite.c, {,ex}kana.h, mendex.h: + Support U+1B001 Hiragana Letter Archaic Ye. + * fwrite.c: + Strict check for dotted/dotless I/i in Turkish. + Strict check for Thai reordering. + +2022-09-17 TANAKA Takuji <ttk@t-lab.opal.ne.jp> + + * fwrite.c: + Fix a bug on Thai words started with vowel followed by consonant. + +2022-09-11 TANAKA Takuji <ttk@t-lab.opal.ne.jp> + + * version 1.03 Stable version. + * configure.ac: Bump version. + * main.c, sort.c, convert.c, fwrite.c, kana.h: + Support U+1B11F..1B122 Hiragana/Katakana Letter Archaic Wu, Yi, Ye. + +2022-09-04 TANAKA Takuji <ttk@t-lab.opal.ne.jp> + + * version 1.02 Stable version. + * configure.ac: Bump version. + * fwrite.c: + Support U+1B132, U+1B155 Hiragana/Katakana Letter Small Ko. + They have not been supported yet but will be supported in ICU collator. + Support U+0626 Arabic Letter Yeh with Hamza Above for Uyghur. + Support U+0149 Latin Small Letter N Preceded By Apostrophe for Afrikaans. + Support digraphs dd, ff, ng, ph, rh for Welsh. + * sort.c: Add Latin Extended-F and -G. + Add CJK Unified Ideograph Extension H, Cyrillic Extended-D, + Devanagari Extended-A and Arabic Extended-C in Unicode 15.0. + +2022-09-02 TANAKA Takuji <ttk@t-lab.opal.ne.jp> + + * main.c: + Moved calls of KP_entry_filetype() after checking options. + This change prevents unwanted looking for texmf.cnf. + https://github.com/texjporg/tex-jp-build/pull/144 + +2022-05-22 TANAKA Takuji <ttk@t-lab.opal.ne.jp> + + * version 1.01 Stable version. + * configure.ac: Bump version. + * fread.c: + Modify heuristic detection of page number types + for ambiguous ones between roman number and alphabet. + * tests/upmendex.test, tests/pprec[567].ist, + tests/romalp[ABCD].idx, tests/romalpA-[56].ind, + tests/romalp[BD]-[567].ind, tests/romalpC-5.ind, Makefile.am: + Update tests. + +2022-03-21 Karl Berry <karl@tug.org> + + * TL'22 release. + +2022-03-07 Hironobu Yamashita <h.y.acetaminophen@gmail.com> + + * fwrite.c: output to file instead of accumulating too much. + https://github.com/texjporg/tex-jp-build/issues/137 + +2022-01-18 Karl Berry <karl@freefriends.org> + + * COPYRIGHT, + * main.c: 2022. + 2021-11-20 TANAKA Takuji <ttk@t-lab.opal.ne.jp> * version 1.00 Stable version. diff --git a/indexing/upmendex/source/Makefile.am b/indexing/upmendex/source/Makefile.am index 451f0df1eb..3dd956d530 100644 --- a/indexing/upmendex/source/Makefile.am +++ b/indexing/upmendex/source/Makefile.am @@ -1,4 +1,4 @@ -## $Id: Makefile.am 60856 2021-10-24 05:44:35Z takuji $ +## $Id$ ## Makefile.am for the TeX Live subdirectory texk/upmendex/ ## ## Copyright 2015-2019 Karl Berry <tex-live@tug.org> @@ -57,7 +57,20 @@ EXTRA_DIST += $(TESTS) ## tests/upmendex.test EXTRA_DIST += tests/foo.idx tests/foo.ind \ - tests/uni.idx tests/uni.dict tests/uni.ist tests/uni.ind \ - tests/pprec0.ist tests/pprec1.ist tests/pprec2.ist tests/pprecA.idx \ - tests/pprecA-0.ind tests/pprecA-1.ind tests/pprecA-2.ind -DISTCLEANFILES = foo.* uni.* pprec*.* + tests/uni.idx tests/uni.dict tests/uni.ist \ + tests/uni1.ist tests/uni2.ist tests/uni.ind \ + tests/range.idx tests/range2.ist tests/range2.ist tests/range3.ist \ + tests/range1.ind tests/range2.ind tests/range3.ind \ + tests/pprec0.ist tests/pprec1.ist tests/pprec2.ist tests/pprec3.ist \ + tests/pprec4.ist tests/pprec5.ist tests/pprec6.ist tests/pprec7.ist \ + tests/pprecA.idx tests/pprecA-0.ind tests/pprecA-1.ind tests/pprecA-2.ind \ + tests/pprecB.idx tests/pprecB-0.ind tests/pprecB-3.ind tests/pprecB-4.ind \ + tests/romalpA.idx tests/romalpA-5.ind tests/romalpA-6.ind \ + tests/romalpB.idx tests/romalpB-5.ind tests/romalpB-6.ind tests/romalpB-7.ind \ + tests/romalpC.idx tests/romalpC-5.ind \ + tests/romalpD.idx tests/romalpD-5.ind tests/romalpD-6.ind tests/romalpD-7.ind +DISTCLEANFILES = foo.* uni.* range*.* pprec*.* romalp*.* +EXTRA_DIST += tests/locale1.ist tests/locale2.ist tests/rule.idx \ + tests/attr1.ist tests/attr1.ind tests/locale1.ind tests/locale2-rule.ind \ + tests/rule1.ist tests/rule2.ist tests/rule3.ist tests/rule4.ist rule.ind +DISTCLEANFILES += locale*.* attr*.* rule*.* diff --git a/indexing/upmendex/source/Makefile.in b/indexing/upmendex/source/Makefile.in index 4ecaadee97..2b18866de8 100644 --- a/indexing/upmendex/source/Makefile.in +++ b/indexing/upmendex/source/Makefile.in @@ -609,11 +609,26 @@ nodist_EXTRA_upmendex_SOURCES = dummy.cxx LDADD = $(KPATHSEA_LIBS) $(ICU_LIBS) $(PTHREAD_LIB) $(ICU_LIBS_EXTRA) EXTRA_DIST = COPYRIGHT indexsty upmendex.ja.txt $(TESTS) tests/foo.idx \ tests/foo.ind tests/uni.idx tests/uni.dict tests/uni.ist \ - tests/uni.ind tests/pprec0.ist tests/pprec1.ist \ - tests/pprec2.ist tests/pprecA.idx tests/pprecA-0.ind \ - tests/pprecA-1.ind tests/pprecA-2.ind + tests/uni1.ist tests/uni2.ist tests/uni.ind tests/range.idx \ + tests/range2.ist tests/range2.ist tests/range3.ist \ + tests/range1.ind tests/range2.ind tests/range3.ind \ + tests/pprec0.ist tests/pprec1.ist tests/pprec2.ist \ + tests/pprec3.ist tests/pprec4.ist tests/pprec5.ist \ + tests/pprec6.ist tests/pprec7.ist tests/pprecA.idx \ + tests/pprecA-0.ind tests/pprecA-1.ind tests/pprecA-2.ind \ + tests/pprecB.idx tests/pprecB-0.ind tests/pprecB-3.ind \ + tests/pprecB-4.ind tests/romalpA.idx tests/romalpA-5.ind \ + tests/romalpA-6.ind tests/romalpB.idx tests/romalpB-5.ind \ + tests/romalpB-6.ind tests/romalpB-7.ind tests/romalpC.idx \ + tests/romalpC-5.ind tests/romalpD.idx tests/romalpD-5.ind \ + tests/romalpD-6.ind tests/romalpD-7.ind tests/locale1.ist \ + tests/locale2.ist tests/rule.idx tests/attr1.ist \ + tests/attr1.ind tests/locale1.ind tests/locale2-rule.ind \ + tests/rule1.ist tests/rule2.ist tests/rule3.ist \ + tests/rule4.ist rule.ind TESTS = tests/upmendex.test -DISTCLEANFILES = foo.* uni.* pprec*.* +DISTCLEANFILES = foo.* uni.* range*.* pprec*.* romalp*.* locale*.* \ + attr*.* rule*.* all: c-auto.h $(MAKE) $(AM_MAKEFLAGS) all-am diff --git a/indexing/upmendex/source/configure b/indexing/upmendex/source/configure index e5aa682d08..834f962202 100755 --- a/indexing/upmendex/source/configure +++ b/indexing/upmendex/source/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.71 for upmendex (TeX Live) 1.00. +# Generated by GNU Autoconf 2.71 for upmendex (TeX Live) 1.07. # # # Copyright (C) 1992-1996, 1998-2017, 2020-2021 Free Software Foundation, @@ -626,8 +626,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='upmendex (TeX Live)' PACKAGE_TARNAME='upmendex--tex-live-' -PACKAGE_VERSION='1.00' -PACKAGE_STRING='upmendex (TeX Live) 1.00' +PACKAGE_VERSION='1.07' +PACKAGE_STRING='upmendex (TeX Live) 1.07' PACKAGE_BUGREPORT='' PACKAGE_URL='' @@ -1390,7 +1390,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures upmendex (TeX Live) 1.00 to adapt to many kinds of systems. +\`configure' configures upmendex (TeX Live) 1.07 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1462,7 +1462,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of upmendex (TeX Live) 1.00:";; + short | recursive ) echo "Configuration of upmendex (TeX Live) 1.07:";; esac cat <<\_ACEOF @@ -1587,7 +1587,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -upmendex (TeX Live) configure 1.00 +upmendex (TeX Live) configure 1.07 generated by GNU Autoconf 2.71 Copyright (C) 2021 Free Software Foundation, Inc. @@ -2268,7 +2268,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by upmendex (TeX Live) $as_me 1.00, which was +It was created by upmendex (TeX Live) $as_me 1.07, which was generated by GNU Autoconf 2.71. Invocation command line was $ $0$ac_configure_args_raw @@ -8806,7 +8806,7 @@ fi # Define the identity of the package. PACKAGE='upmendex--tex-live-' - VERSION='1.00' + VERSION='1.07' # Some tools Automake needs. @@ -14509,7 +14509,7 @@ ac_config_headers="$ac_config_headers c-auto.h:c-auto.in" -printf "%s\n" "#define TL_VERSION \"TeX Live 2022/dev\"" >>confdefs.h +printf "%s\n" "#define TL_VERSION \"TeX Live 2023\"" >>confdefs.h { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking if C99 variadic macros are supported" >&5 @@ -18942,7 +18942,7 @@ Usage: $0 [OPTIONS] Report bugs to <bug-libtool@gnu.org>." lt_cl_version="\ -upmendex (TeX Live) config.lt 1.00 +upmendex (TeX Live) config.lt 1.07 configured by $0, generated by GNU Autoconf 2.71. Copyright (C) 2011 Free Software Foundation, Inc. @@ -21114,7 +21114,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by upmendex (TeX Live) $as_me 1.00, which was +This file was extended by upmendex (TeX Live) $as_me 1.07, which was generated by GNU Autoconf 2.71. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -21182,7 +21182,7 @@ ac_cs_config_escaped=`printf "%s\n" "$ac_cs_config" | sed "s/^ //; s/'/'\\\\\\\\ cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config='$ac_cs_config_escaped' ac_cs_version="\\ -upmendex (TeX Live) config.status 1.00 +upmendex (TeX Live) config.status 1.07 configured by $0, generated by GNU Autoconf 2.71, with options \\"\$ac_cs_config\\" diff --git a/indexing/upmendex/source/configure.ac b/indexing/upmendex/source/configure.ac index b21873a1ed..9b5c1511b6 100644 --- a/indexing/upmendex/source/configure.ac +++ b/indexing/upmendex/source/configure.ac @@ -1,4 +1,4 @@ -dnl $Id: configure.ac 61096 2021-11-20 09:29:33Z takuji $ +dnl $Id$ dnl Process this file with autoconf to produce a configure script. dnl dnl Copyright 2015-2021 Karl Berry <tex-live@tug.org> @@ -8,7 +8,7 @@ dnl This file is free software; the copyright holder dnl gives unlimited permission to copy and/or distribute it, dnl with or without modifications, as long as this notice is preserved. dnl -AC_INIT([upmendex (TeX Live)],[1.00]) +AC_INIT([upmendex (TeX Live)],[1.07]) AC_PREREQ([2.71]) AC_CONFIG_SRCDIR([main.c]) AC_CONFIG_AUX_DIR([../../build-aux]) diff --git a/indexing/upmendex/source/convert.c b/indexing/upmendex/source/convert.c index a7cae61038..2760059a06 100644 --- a/indexing/upmendex/source/convert.c +++ b/indexing/upmendex/source/convert.c @@ -43,7 +43,17 @@ void initkanatable(void) aiueo=u_xstrdup(AIUEO); for (i=0;;i++) { if (aiueo[i]==0) break; - aiueo[i]+=KATATOP-HIRATOP; /* hiragana -> katakana */ + if (aiueo[i]==0xd82c) { + i++; + if (aiueo[i]==0xdd1f) aiueo[i]+=3; /* U+1B11F -> U+1B122 */ + if (aiueo[i]==0xdd21) { + UChar ch[] = {0xd82c, 0xdc01, 0x0}; + if (is_jpn_kana(ch)) + AIUEO[i]=0xdc01; /* U+1B121 -> U+1B001 */ + } + } + else + aiueo[i]+=KATATOP-HIRATOP; /* hiragana -> katakana */ } u_strcpy(atama,akasatana); diff --git a/indexing/upmendex/source/exhanzi.h b/indexing/upmendex/source/exhanzi.h index 3f8f6ded84..c61ee2bfff 100644 --- a/indexing/upmendex/source/exhanzi.h +++ b/indexing/upmendex/source/exhanzi.h @@ -8,6 +8,7 @@ extern struct hanzi_index hz_index[]; extern int hz_index_len; +extern int hanzi_mode; extern struct hanzi_index HZ_RADICAL[]; extern struct hanzi_index HZ_STROKE[]; diff --git a/indexing/upmendex/source/exkana.h b/indexing/upmendex/source/exkana.h index 9e4ba1d197..799c429540 100644 --- a/indexing/upmendex/source/exkana.h +++ b/indexing/upmendex/source/exkana.h @@ -4,6 +4,7 @@ extern UChar AIUEO[]; extern UChar *aiueo; extern UChar kanatable[]; extern UChar extkanatable[]; +extern int kana_ye_mode; #define SPACE 0x3000 /* 全角スペース */ #define ALPHATOP 0xff01 /* ! */ diff --git a/indexing/upmendex/source/fread.c b/indexing/upmendex/source/fread.c index 66d3e0b144..ab3dd4c077 100644 --- a/indexing/upmendex/source/fread.c +++ b/indexing/upmendex/source/fread.c @@ -268,7 +268,7 @@ LOOP: nest++; else if (buff[j]==arg_close) { if (nest==0) { - table[k]='\0'; + table[k]='\0'; ind[0].p[0].page=xstrdup(table); break; } @@ -495,6 +495,7 @@ static void chkpageattr(struct page *p) { int i,j,cc=0,cnt,pplen,pclen; char buff[16],*pcpos,*page0; + static char pattr_prev[PAGE_COMPOSIT_DEPTH] = {0}; pplen=strlen(page_precedence); pclen=strlen(page_compositor); @@ -538,23 +539,25 @@ ATTRLOOP: switch(page_precedence[pattr[cc]]) { case 'r': if (strchr("ivxlcdm",*page0)==NULL || - (strchr("lcdm",*page0) && strchr(page_precedence,'a') && strlen(buff)==1 && pcpos)) { + (strchr("lcdm",*page0) && strchr(page_precedence,'a') && strlen(buff)==1 && + pattr_prev[cc]!='r')) { /* heuristic detection as alphabet since L=50, C=100, D=100, M=1000 are quite large */ if (pattr[cc]<pplen-1) pattr[cc]++; else pattr[cc]=0; - for (j=cc+1;j<3;j++) pattr[j]=0; + for (j=cc+1;j<PAGE_COMPOSIT_DEPTH;j++) pattr[j]=0; goto ATTRLOOP; } break; case 'R': if (strchr("IVXLCDM",*page0)==NULL || - (strchr("LCDM",*page0) && strchr(page_precedence,'A') && strlen(buff)==1 && pcpos)) { + (strchr("LCDM",*page0) && strchr(page_precedence,'A') && strlen(buff)==1 && + pattr_prev[cc]!='R')) { /* heuristic detection as alphabet since L=50, C=100, D=100, M=1000 are quite large */ if (pattr[cc]<pplen-1) pattr[cc]++; else pattr[cc]=0; - for (j=cc+1;j<3;j++) pattr[j]=0; + for (j=cc+1;j<PAGE_COMPOSIT_DEPTH;j++) pattr[j]=0; goto ATTRLOOP; } break; @@ -563,29 +566,31 @@ ATTRLOOP: if (pattr[cc]<pplen-1) pattr[cc]++; else pattr[cc]=0; - for (j=cc+1;j<3;j++) pattr[j]=0; + for (j=cc+1;j<PAGE_COMPOSIT_DEPTH;j++) pattr[j]=0; goto ATTRLOOP; } break; case 'a': if (*page0<'a' || *page0>'z' || strlen(buff)>1 || - (strchr("ivx",*page0) && strchr(page_precedence,'r') && !pcpos)) { + (strchr("ivx",*page0) && strchr(page_precedence,'r') && + pattr_prev[cc]!='a')) { /* heuristic detection as roman number since I=1, V=5, X=10 are quite small */ if (pattr[cc]<pplen-1) pattr[cc]++; else pattr[cc]=0; - for (j=cc+1;j<3;j++) pattr[j]=0; + for (j=cc+1;j<PAGE_COMPOSIT_DEPTH;j++) pattr[j]=0; goto ATTRLOOP; } break; case 'A': if (*page0<'A' || *page0>'Z' || strlen(buff)>1 || - (strchr("IVX",*page0) && strchr(page_precedence,'R') && !pcpos)) { + (strchr("IVX",*page0) && strchr(page_precedence,'R') && + pattr_prev[cc]!='A')) { /* heuristic detection as roman number since I=1, V=5, X=10 are quite small */ if (pattr[cc]<pplen-1) pattr[cc]++; else pattr[cc]=0; - for (j=cc+1;j<3;j++) pattr[j]=0; + for (j=cc+1;j<PAGE_COMPOSIT_DEPTH;j++) pattr[j]=0; goto ATTRLOOP; } break; @@ -594,6 +599,7 @@ ATTRLOOP: page_precedence[pattr[cc]], page_precedence); exit(253); } + pattr_prev[cc] = page_precedence[pattr[cc]]; } } p->attr[cc]=pattr[cc]; diff --git a/indexing/upmendex/source/fwrite.c b/indexing/upmendex/source/fwrite.c index 9ce382589e..f33230df8e 100644 --- a/indexing/upmendex/source/fwrite.c +++ b/indexing/upmendex/source/fwrite.c @@ -12,11 +12,11 @@ int line_length=0; static void printpage(struct index *ind, FILE *fp, int num, char *lbuff); -static int range_check(struct index ind, int count, char *lbuff); -static void linecheck(char *lbuff, char *tmpbuff, int force); +static int range_check(struct index ind, int count, char *lbuff, FILE *fp); +static void linecheck(char *lbuff, char *tmpbuff, FILE *fp, int force); static void crcheck(char *lbuff, FILE *fp); static void index_normalize(UChar *istr, UChar *ini, int *chset); -static int initial_cmp_char(UChar *ini, UChar ch); +static int initial_cmp_char(UChar *ini, UChar *ch); static int init_hanzi_header(void); static const UNormalizer2 *unormalizer_NFD, *unormalizer_NFKD; static int turkish_i; @@ -86,7 +86,7 @@ static void fprint_uchar(FILE *fp, const UChar *a, const int mode, const int len olen=u_strToUpper(istr,INITIALLENGTH,istr,wclen,"",&perr); } else if (mode==M_TO_LOWER) { perr=U_ZERO_ERROR; - olen=u_strToLower(istr,INITIALLENGTH,istr,wclen, istr[0]==0x130&&turkish_i?"tr":"", &perr); + olen=u_strToLower(istr,INITIALLENGTH,istr,wclen, istr[0]==0x130&&turkish_i==2?"tr":"", &perr); } else if (mode==M_TO_TITLE) { perr=U_ZERO_ERROR; olen=u_strToTitle(istr,INITIALLENGTH,istr,wclen,NULL,"",&perr); @@ -166,9 +166,9 @@ static int pnumconv2(struct page *p) /* write ind file */ void indwrite(char *filename, struct index *ind, int pagenum) { - int i,j,hpoint=0,tpoint=0,ipoint=0,jpoint=0,block_open=0; + int i,j,k,hpoint=0,tpoint=0,ipoint=0,jpoint=0,block_open=0; char lbuff[BUFFERLEN],obuff[BUFFERLEN]; - UChar datama[256],initial[INITIALLENGTH],initial_prev[INITIALLENGTH]; + UChar initial[INITIALLENGTH],initial_prev[INITIALLENGTH]; int chset,chset_prev; FILE *fp=NULL; UErrorCode perr; @@ -182,7 +182,6 @@ void indwrite(char *filename, struct index *ind, int pagenum) #endif } - convert(atama,datama); fputs(preamble,fp); if (fpage>0) { @@ -228,31 +227,35 @@ void indwrite(char *filename, struct index *ind, int pagenum) else if (chset==CH_KANA) { if (lethead_flag!=0) { fputs(lethead_prefix,fp); - for (j=hpoint;j<(u_strlen(datama));j++) { - if (initial_cmp_char(initial,datama[j])) { - fprint_uchar(fp,&atama[j-1],M_NONE,1); + for (j=hpoint;j<(u_strlen(kana_head));) { + if (initial_cmp_char(initial,&kana_head[j])) { + k=j; U16_BACK_1(kana_head, 0, k); + fprint_uchar(fp,&kana_head[k],M_NONE,1); hpoint=j; break; } + U16_FWD_1(kana_head, j, -1); } - if (j==(u_strlen(datama))) { - fprint_uchar(fp,&atama[j-1],M_NONE,1); + if (j==(u_strlen(kana_head))) { + k=j; U16_BACK_1(kana_head, 0, k); + fprint_uchar(fp,&kana_head[k],M_NONE,1); } fputs(lethead_suffix,fp); } widechar_to_multibyte(obuff,BUFFERLEN,ind[i].idx[0]); SPRINTF(lbuff,"%s%s",item_0,obuff); - for (hpoint=0;hpoint<(u_strlen(datama));hpoint++) { - if (initial_cmp_char(initial,datama[hpoint])) { + for (hpoint=0;hpoint<(u_strlen(kana_head));) { + if (initial_cmp_char(initial,&kana_head[hpoint])) { break; } + U16_FWD_1(kana_head, hpoint, -1); } } else if (chset==CH_HANGUL) { if (lethead_flag!=0) { fputs(lethead_prefix,fp); for (j=tpoint;j<(u_strlen(hangul_head));j++) { - if (initial_cmp_char(initial,hangul_head[j])) { + if (initial_cmp_char(initial,&hangul_head[j])) { fprint_uchar(fp,&hangul_head[j-1],M_NONE,1); tpoint=j; break; @@ -266,7 +269,7 @@ void indwrite(char *filename, struct index *ind, int pagenum) widechar_to_multibyte(obuff,BUFFERLEN,ind[i].idx[0]); SPRINTF(lbuff,"%s%s",item_0,obuff); for (tpoint=0;tpoint<(u_strlen(hangul_head));tpoint++) { - if (initial_cmp_char(initial,hangul_head[tpoint])) { + if (initial_cmp_char(initial,&hangul_head[tpoint])) { break; } } @@ -274,31 +277,35 @@ void indwrite(char *filename, struct index *ind, int pagenum) else if (chset==CH_DEVANAGARI) { if (lethead_flag!=0) { fputs(lethead_prefix,fp); - for (j=jpoint;j<(u_strlen(devanagari_head));j++) { - if (initial_cmp_char(initial,devanagari_head[j])) { - fprint_uchar(fp,&devanagari_head[j-1],M_NONE,1); + for (j=jpoint;j<(u_strlen(devanagari_head));) { + if (initial_cmp_char(initial,&devanagari_head[j])) { + k=j; U16_BACK_1(devanagari_head, 0, k); + fprint_uchar(fp,&devanagari_head[k],M_NONE,1); jpoint=j; break; } + U16_FWD_1(devanagari_head, j, -1); } if (j==(u_strlen(devanagari_head))) { - fprint_uchar(fp,&devanagari_head[j-1],M_NONE,1); + k=j; U16_BACK_1(devanagari_head, 0, k); + fprint_uchar(fp,&devanagari_head[k],M_NONE,1); } fputs(lethead_suffix,fp); } widechar_to_multibyte(obuff,BUFFERLEN,ind[i].idx[0]); SPRINTF(lbuff,"%s%s",item_0,obuff); - for (jpoint=0;jpoint<(u_strlen(devanagari_head));jpoint++) { - if (initial_cmp_char(initial,devanagari_head[jpoint])) { + for (jpoint=0;jpoint<(u_strlen(devanagari_head));) { + if (initial_cmp_char(initial,&devanagari_head[jpoint])) { break; } + U16_FWD_1(devanagari_head, jpoint, -1); } } else if (chset==CH_THAI) { if (lethead_flag!=0) { fputs(lethead_prefix,fp); for (j=ipoint;j<(u_strlen(thai_head));j++) { - if (initial_cmp_char(initial,thai_head[j])) { + if (initial_cmp_char(initial,&thai_head[j])) { fprint_uchar(fp,&thai_head[j-1],M_NONE,1); ipoint=j; break; @@ -312,7 +319,7 @@ void indwrite(char *filename, struct index *ind, int pagenum) widechar_to_multibyte(obuff,BUFFERLEN,ind[i].idx[0]); SPRINTF(lbuff,"%s%s",item_0,obuff); for (ipoint=0;ipoint<(u_strlen(thai_head));ipoint++) { - if (initial_cmp_char(initial,thai_head[ipoint])) { + if (initial_cmp_char(initial,&thai_head[ipoint])) { break; } } @@ -379,24 +386,26 @@ void indwrite(char *filename, struct index *ind, int pagenum) } } else if (chset==CH_KANA) { - for (j=hpoint;j<(u_strlen(datama));j++) { - if (initial_cmp_char(initial,datama[j])) { + for (j=hpoint;j<(u_strlen(kana_head));) { + if (initial_cmp_char(initial,&kana_head[j])) { break; } + U16_FWD_1(kana_head, j, -1); } if ((j!=hpoint)||(j==0)) { hpoint=j; fputs(group_skip,fp); if (lethead_flag!=0) { + k=j; U16_BACK_1(kana_head, 0, k); fputs(lethead_prefix,fp); - fprint_uchar(fp,&atama[j-1],M_NONE,1); + fprint_uchar(fp,&kana_head[k],M_NONE,1); fputs(lethead_suffix,fp); } } } else if (chset==CH_HANGUL) { for (j=tpoint;j<(u_strlen(hangul_head));j++) { - if (initial_cmp_char(initial,hangul_head[j])) { + if (initial_cmp_char(initial,&hangul_head[j])) { break; } } @@ -411,24 +420,26 @@ void indwrite(char *filename, struct index *ind, int pagenum) } } else if (chset==CH_DEVANAGARI) { - for (j=jpoint;j<(u_strlen(devanagari_head));j++) { - if (initial_cmp_char(initial,devanagari_head[j])) { + for (j=jpoint;j<(u_strlen(devanagari_head));) { + if (initial_cmp_char(initial,&devanagari_head[j])) { break; } + U16_FWD_1(devanagari_head, j, -1); } if ((j!=jpoint)||(j==0)) { jpoint=j; fputs(group_skip,fp); if (lethead_flag!=0) { + k=j; U16_BACK_1(devanagari_head, 0, k); fputs(lethead_prefix,fp); - fprint_uchar(fp,&devanagari_head[j-1],M_NONE,1); + fprint_uchar(fp,&devanagari_head[k],M_NONE,1); fputs(lethead_suffix,fp); } } } else if (chset==CH_THAI) { for (j=ipoint;j<(u_strlen(thai_head));j++) { - if (initial_cmp_char(initial,thai_head[j])) { + if (initial_cmp_char(initial,&thai_head[j])) { break; } } @@ -541,7 +552,7 @@ static void printpage(struct index *ind, FILE *fp, int num, char *lbuff) line_length=strlen(lbuff); for(j=0;j<ind[num].num;j++) { - cc=range_check(ind[num],j,lbuff); + cc=range_check(ind[num],j,lbuff,fp); if (cc>j) { int epage = pnumconv2(&ind[num].p[cc]); int bpage = pnumconv2(&ind[num].p[j]); @@ -576,14 +587,14 @@ static void printpage(struct index *ind, FILE *fp, int num, char *lbuff) if (strlen(ind[num].p[j].enc)>0) { SAPPENDF(tmpbuff,"%s",encap_suffix); } - linecheck(lbuff,tmpbuff, FALSE); + linecheck(lbuff,tmpbuff,fp, FALSE); j=cc; if (j==ind[num].num) { goto PRINT; } else { SAPPENDF(tmpbuff,"%s",delim_n); - linecheck(lbuff,tmpbuff, TRUE); + linecheck(lbuff,tmpbuff,fp, TRUE); } } else if (strlen(ind[num].p[j].enc)>0) { @@ -605,19 +616,19 @@ static void printpage(struct index *ind, FILE *fp, int num, char *lbuff) SAPPENDF(tmpbuff,"%s",ind[num].p[j].page); SAPPENDF(tmpbuff,"%s",encap_suffix); SAPPENDF(tmpbuff,"%s",delim_n); - linecheck(lbuff,tmpbuff, FALSE); + linecheck(lbuff,tmpbuff,fp, FALSE); } else { SAPPENDF(tmpbuff,"%s",ind[num].p[j].page); SAPPENDF(tmpbuff,"%s",delim_n); - linecheck(lbuff,tmpbuff, FALSE); + linecheck(lbuff,tmpbuff,fp, FALSE); } } else { /* no encap */ SAPPENDF(tmpbuff,"%s",ind[num].p[j].page); SAPPENDF(tmpbuff,"%s",delim_n); - linecheck(lbuff,tmpbuff, FALSE); + linecheck(lbuff,tmpbuff,fp, FALSE); } } @@ -651,7 +662,7 @@ static void printpage(struct index *ind, FILE *fp, int num, char *lbuff) else { SAPPENDF(tmpbuff,"%s",ind[num].p[j].page); } - linecheck(lbuff,tmpbuff, FALSE); + linecheck(lbuff,tmpbuff,fp, FALSE); PRINT: fputs(lbuff,fp); @@ -659,7 +670,7 @@ PRINT: lbuff[0]='\0'; } -static int range_check(struct index ind, int count, char *lbuff) +static int range_check(struct index ind, int count, char *lbuff, FILE *fp) { int i,j,k,cc1,cc2,start,force=0; char tmpbuff[BUFFERLEN],errbuff[BUFFERLEN],obuff[BUFFERLEN]; @@ -697,7 +708,7 @@ static int range_check(struct index ind, int count, char *lbuff) if (strlen(ind.p[j].enc)>0) { SPRINTF(tmpbuff,"%s%s%s%s%s%s",encap_prefix,ind.p[j].enc,encap_infix ,ind.p[j].page,encap_suffix,delim_n); - linecheck(lbuff,tmpbuff, FALSE); + linecheck(lbuff,tmpbuff,fp, FALSE); } } } @@ -736,10 +747,12 @@ static int range_check(struct index ind, int count, char *lbuff) } /* check line length */ -static void linecheck(char *lbuff, char *tmpbuff, int force) +static void linecheck(char *lbuff, char *tmpbuff, FILE *fp, int force) { if (line_length+strlen(tmpbuff)>line_max && !force) { SAPPENDF(lbuff,"\n"); + fputs(lbuff,fp); + lbuff[0]='\0'; SAPPENDF(lbuff,"%s",indent_space); SAPPENDF(lbuff,"%s",tmpbuff); line_length=indent_length+strlen(tmpbuff); @@ -771,13 +784,13 @@ static void crcheck(char *lbuff, FILE *fp) static void index_normalize(UChar *istr, UChar *ini, int *chset) { - int k, len, hi, lo, mi; + int k, hi, lo, mi; UChar ch,src[2],dest[8],strX[4],strY[4],strZ[4]; UChar32 c32; UErrorCode perr; UCollationResult order; UCollationStrength strgth; - static int hanzi_mode=0, i_y_mode=0; + static int i_y_mode=0; ch=istr[0]; *chset=charset(istr); @@ -810,6 +823,15 @@ static void index_normalize(UChar *istr, UChar *ini, int *chset) ini[0]=0x307B; break; /* ほ */ case 0x1B000: /* 𛀀 */ ini[0]=0x3048; break; /* え */ + case 0x1B11F: case 0x1B122: /* Archaic WU 𛄟 𛄢 */ + ini[0]=0xD82C; ini[1]=0xDD1F; ini[2]=L'\0'; + break; + case 0x1B120: /* Archaic YI 𛄠 */ + ini[0]=0xD82C; ini[1]=0xDD20; ini[2]=L'\0'; break; + case 0x1B121: case 0x1B001: /* Archaic YE 𛄡 𛀁 */ + ini[0]=0xD82C; ini[1]=0xDD21; ini[2]=L'\0'; break; + case 0x1B132: case 0x1B155: + ini[0]=0x3053; break; /* こ */ case 0x1B150: case 0x1B164: ini[0]=0x3090; break; /* ゐ */ case 0x1B151: case 0x1B165: @@ -885,27 +907,58 @@ static void index_normalize(UChar *istr, UChar *ini, int *chset) ini[0]=ch; return; } - else if ((len=is_hanzi(istr))>0) { + else if (is_hanzi(istr)) { if (hanzi_mode==0) hanzi_mode=init_hanzi_header(); if (hanzi_mode==HANZI_UNKNOWN) { u_strcpy(ini, hz_index[0].idx); return; } - strX[0] =istr[0]; - if (len==2){ strX[1] =istr[1]; } - strX[len]=L'\0'; lo=0; hi=hz_index_len; while (lo<hi) { mi = (lo+hi)/2; u_strcpy(strZ,hz_index[mi].threshold); - order = ucol_strcoll(icu_collator, strZ, -1, strX, -1); + order = ucol_strcoll(icu_collator, strZ, -1, istr, -1); if (order!=UCOL_GREATER) lo=mi+1; else hi=mi; } u_strcpy(ini,hz_index[lo-1].idx); return; } - else if (is_devanagari(&ch)||is_thai(&ch)||is_arabic(&ch)||is_hebrew(&ch)) { + else if (is_thai(&ch)) { + if ((istr[0]>=0x0E40 && istr[0]<=0x0E44) && (istr[1]>=0x0E01 && istr[1]<=0x0E2E)) { + /* Thai reordering :: Vowel followed by Consonant */ + /* https://unicode-org.github.io/icu/userguide/collation/concepts.html#thailao-reordering */ + ini[0]=istr[1]; + } else { + ini[0]=istr[0]; + } + return; + } + else if (is_devanagari(istr)==2) { + ini[0]=istr[0]; ini[1]=istr[1]; ini[2]=L'\0'; + return; + } + else if (is_devanagari(&ch)||is_arabic(&ch)||is_hebrew(&ch)) { + if (ch==0x626) { /* Arabic Letter Yeh with Hamza Above for Uyghur */ + strY[0]=0x626; strY[1]=L'\0'; /* Yeh with Hamza Above */ + strZ[0]=0x628; strZ[1]=L'\0'; /* Beh */ + order = ucol_strcoll(icu_collator, strZ, -1, strY, -1); + if (order==UCOL_LESS) { + ini[0]=strY[0]; ini[1]=strY[1]; + return; + } + } + /* Devanagary, Marathi */ + if ( (istr[0]==0x915 && istr[1]==0x94D && istr[2]== 0x937) /* KSSA क्ष */ + || (istr[0]==0x91C && istr[1]==0x94D && istr[2]== 0x91E)) { /* JNYA ज्ञ */ + strY[0]=istr[0]; strY[1]=istr[1]; strY[2]=istr[2]; strY[3]=L'\0'; + strZ[0]=0x939; strZ[1]=L'\0'; /* HA ह */ + order = ucol_strcoll(icu_collator, strZ, -1, strY, -1); + if (order==UCOL_LESS) { + u_strcpy(ini,strY); + return; + } + } if (ch==0x929||ch==0x931||ch==0x934||(0x958<=ch&&ch<=0x95F) /* Devanagary */ ||(0x622<=ch&&ch<=0x626)||ch==0x6C0||ch==0x6C2||ch==0x6D3 /* Arabic */ ||(0xFB50<=ch&&ch<=0xFDFF) /* Arabic Presentation Forms-A */ @@ -930,12 +983,17 @@ static void index_normalize(UChar *istr, UChar *ini, int *chset) } if (ch==0x049||ch==0x069||ch==0x130||ch==0x131||ch==0x0CE||ch==0x0EE) { /* check dotted/dotless İ,I,i,ı and Î,î for Turkish */ - strX[0] = 0x131; strX[1] = 0x5A; strX[2] = 0x00; /* ıZ */ - strZ[0] = 0x069; strZ[1] = 0x00; /* i */ - order = ucol_strcoll(icu_collator, strZ, -1, strX, -1); - if (order==UCOL_GREATER) { + if (turkish_i==0) { + strgth = ucol_getStrength(icu_collator); + ucol_setStrength(icu_collator, UCOL_SECONDARY); + strX[0] = 0x131; strX[1] = 0x069; strX[2] = 0x00; /* ıi */ + strZ[0] = 0x049; strZ[1] = 0x130; strZ[2] = 0x00; /* Iİ */ + order = ucol_strcoll(icu_collator, strZ, -1, strX, -1); + turkish_i = (order==UCOL_EQUAL) ? 2 : 1; + ucol_setStrength(icu_collator, strgth); + } + if (turkish_i==2) { ini[0] = (ch==0x049||ch==0x131) ? 0x131 : 0x130; /* ı or İ */ - turkish_i=1; return; } } @@ -947,7 +1005,7 @@ static void index_normalize(UChar *istr, UChar *ini, int *chset) strX[0] = 0x059; strX[1] = 0x00; /* Y */ strZ[0] = 0x049; strZ[1] = 0x00; /* I */ order = ucol_strcoll(icu_collator, strZ, -1, strX, -1); - if (order==UCOL_EQUAL) i_y_mode=2; else i_y_mode=1; + i_y_mode = (order==UCOL_EQUAL) ? 2 : 1; ucol_setStrength(icu_collator, strgth); } if (i_y_mode==2) { @@ -956,7 +1014,7 @@ static void index_normalize(UChar *istr, UChar *ini, int *chset) } } if (ch==0x0C6||ch==0x0E6||ch==0x152||ch==0x153||ch==0x132||ch==0x133 - ||ch==0x0DF||ch==0x1E9E||ch==0x13F||ch==0x140||ch==0x490||ch==0x491) { + ||ch==0x0DF||ch==0x1E9E||ch==0x13F||ch==0x140||ch==0x149||ch==0x490||ch==0x491) { strX[0] = u_toupper(ch); strX[1] = 0x00; /* ex. "Æ" "Œ" */ switch (ch) { case 0x0C6: case 0x0E6: /* Æ æ */ @@ -972,6 +1030,8 @@ static void index_normalize(UChar *istr, UChar *ini, int *chset) strZ[0] = 0x49; break; /* I */ case 0x13F: case 0x140: /* Ŀ ŀ */ strZ[0] = 0x4C; break; /* L */ + case 0x149: /* ʼn */ + strZ[0] = 0x4E; break; /* N */ case 0x490: case 0x491: /* Ґ ґ */ strZ[0] = 0x413; break; /* Г */ } @@ -1039,9 +1099,22 @@ static void index_normalize(UChar *istr, UChar *ini, int *chset) } } } + /* NG for Welsh */ + if (strX[0]==0x4E && strX[1]==0x47) { /* NG */ + strY[0]=0x4E; strY[1]=L'\0'; /* N */ + strZ[0]=0x4E; strZ[1]=0x47; strZ[2]=0x5A; strZ[3]=L'\0'; /* NGZ */ + order = ucol_strcoll(icu_collator, strZ, -1, strY, -1); + if (order==UCOL_LESS) { + ini[0]=strX[0]; ini[1]=strX[1]; /* NG */ + ini[2]=L'\0'; + return; + } + } /* other digraphs */ - if(((strX[0]==0x43 || strX[0]==0x44 || strX[0]==0x53 || strX[0]==0x54 || strX[0]==0x58 || strX[0]==0x5A) - && strX[1]==0x48) || /* CH DH SH TH XH ZH */ + if(((strX[0]==0x43 || strX[0]==0x44 || strX[0]==0x50 || strX[0]==0x52 || strX[0]==0x53 || strX[0]==0x54 || + strX[0]==0x58 || strX[0]==0x5A) && strX[1]==0x48) || /* CH DH PH RH SH TH XH ZH */ + (strX[0]==0x44 && strX[1]==0x44) || /* DD */ + (strX[0]==0x46 && strX[1]==0x46) || /* FF */ (strX[0]==0x4C && strX[1]==0x4C) || /* LL */ ((strX[0]==0x47 || strX[0]==0x4C || strX[0]==0x4E) && strX[1]==0x4A) || /* GJ LJ NJ */ (strX[0]==0x52 && strX[1]==0x52) || /* RR */ @@ -1063,12 +1136,14 @@ static void index_normalize(UChar *istr, UChar *ini, int *chset) return; } -static int initial_cmp_char(UChar *ini, UChar ch) +static int initial_cmp_char(UChar *ini, UChar *ch) { - UChar initial_tmp[INITIALLENGTH],istr[2]; - int chset; - istr[0]=ch; - istr[1]=L'\0'; + UChar initial_tmp[INITIALLENGTH],istr[3]; + int chset, l; + l = is_surrogate_pair(ch) ? 2 : 1; + istr[0]=ch[0]; + if (l==2) istr[1]=ch[1]; + istr[l]=L'\0'; index_normalize(istr, initial_tmp, &chset); return (ss_comp(ini, initial_tmp)<0); diff --git a/indexing/upmendex/source/hanzi.h b/indexing/upmendex/source/hanzi.h index e16cb79313..f1ec18fee5 100644 --- a/indexing/upmendex/source/hanzi.h +++ b/indexing/upmendex/source/hanzi.h @@ -2,6 +2,7 @@ struct hanzi_index hz_index[HZIDXSIZE]; int hz_index_len; +int hanzi_mode=0; /* U+FDD0 (Noncharacter) is used. @@ -226,8 +227,9 @@ struct hanzi_index HZ_RADICAL[]={ /* Reference: - cldr - Revision 12061: /trunk/common/collation - http://unicode.org/repos/cldr/trunk/common/collation/zh.xml + CLDR 42 + https://cldr.unicode.org/index/downloads/cldr-42 + https://github.com/unicode-org/cldr/blob/release-42/common/collation/zh.xml <collation type='stroke' alt='short'> U+FDD0 (Noncharacter) is used. */ @@ -277,8 +279,9 @@ struct hanzi_index HZ_STROKE[]={ /* Reference: - cldr - Revision 12061: /trunk/common/collation - http://unicode.org/repos/cldr/trunk/common/collation/zh.xml + CLDR 42 + https://cldr.unicode.org/index/downloads/cldr-42 + https://github.com/unicode-org/cldr/blob/release-42/common/collation/zh.xml <collation type='pinyin'> U+FDD0 (Noncharacter) is used. */ @@ -311,8 +314,9 @@ struct hanzi_index HZ_PINYIN[]={ /* Reference: - cldr - Revision 12061: /trunk/common/collation - http://unicode.org/repos/cldr/trunk/common/collation/zh.xml + CLDR 42 + https://cldr.unicode.org/index/downloads/cldr-42 + https://github.com/unicode-org/cldr/blob/release-42/common/collation/zh.xml <collation type='zhuyin'> U+FDD0 (Noncharacter) is used. */ diff --git a/indexing/upmendex/source/kana.h b/indexing/upmendex/source/kana.h index 41307f2cb4..2f8aa6ccd6 100644 --- a/indexing/upmendex/source/kana.h +++ b/indexing/upmendex/source/kana.h @@ -65,7 +65,9 @@ UChar AIUEO[]={ 0x3081, /* め */ 0x3082, /* も */ 0x3084, /* や */ + 0xd82c, 0xdd20, /* U+1B120 Katakana Letter Archaic YI 𛄠 */ 0x3086, /* ゆ */ + 0xd82c, 0xdd21, /* U+1B121 Katakana Letter Archaic YE 𛄡 */ 0x3088, /* よ */ 0x3089, /* ら */ 0x308a, /* り */ @@ -74,6 +76,7 @@ UChar AIUEO[]={ 0x308d, /* ろ */ 0x308f, /* わ */ 0x3090, /* ゐ */ + 0xd82c, 0xdd1f, /* U+1B11F Hiragana Letter Archaic WU 𛄟 */ 0x3091, /* ゑ */ 0x3092, /* を */ 0x3093, /* ん */ @@ -192,6 +195,7 @@ UChar extkanatable[]={ 0x308d, /* ろ */ 0 }; +int kana_ye_mode=0; UChar GANADA[]={ diff --git a/indexing/upmendex/source/main.c b/indexing/upmendex/source/main.c index 3bce9d1f3d..ed8b95d836 100644 --- a/indexing/upmendex/source/main.c +++ b/indexing/upmendex/source/main.c @@ -45,15 +45,6 @@ int main(int argc, char **argv) } #endif - kp_ist.var_name = "INDEXSTYLE"; - kp_ist.path = DEFAULT_INDEXSTYLES; /* default path. */ - kp_ist.suffix = "ist"; - KP_entry_filetype(&kp_ist); - kp_dict.var_name = "INDEXDICTIONARY"; - kp_dict.path = DEFAULT_INDEXDICTS; /* default path */ - kp_dict.suffix = "dict"; - KP_entry_filetype(&kp_dict); - /* check options */ for (i=1,j=k=0;i<argc && j<256;i++) { @@ -160,7 +151,7 @@ int main(int argc, char **argv) default: fprintf(stderr,"upmendex - index processor, %s (%s).\n",VERSION, TL_VERSION); - fprintf(stderr," Copyright 2009 ASCII MEDIA WORKS, 2015-2021 TANAKA Takuji\n"); + fprintf(stderr," Copyright 2009 ASCII MEDIA WORKS, 2015-2023 TANAKA Takuji\n"); fprintf(stderr," using ICU version %s\n",icu_version); fprintf(stderr,"usage:\n"); fprintf(stderr,"%% upmendex [-ilqrcgf] [-s sty] [-d dic] [-o ind] [-t log] [-p no] [--] [idx0 idx1 ...]\n"); @@ -191,6 +182,15 @@ int main(int argc, char **argv) } idxcount=j+fsti; + kp_ist.var_name = "INDEXSTYLE"; + kp_ist.path = DEFAULT_INDEXSTYLES; /* default path. */ + kp_ist.suffix = "ist"; + KP_entry_filetype(&kp_ist); + kp_dict.var_name = "INDEXDICTIONARY"; + kp_dict.path = DEFAULT_INDEXDICTS; /* default path */ + kp_dict.suffix = "dict"; + KP_entry_filetype(&kp_dict); + /* check option errors */ if (idxcount==0) idxcount=fsti=1; @@ -241,10 +241,6 @@ int main(int argc, char **argv) VERSION, icu_version, TL_VERSION); } -/* init kanatable */ - - initkanatable(); - /* init hangul,devanagari,thai *_head table */ u_strcpy(hangul_head,GANADA); u_strcpy(devanagari_head,DVNG_HEAD); @@ -255,6 +251,11 @@ int main(int argc, char **argv) } set_icu_attributes(); + init_icu_collator(); + +/* init kanatable */ + + initkanatable(); /* read dictionary */ @@ -283,7 +284,7 @@ int main(int argc, char **argv) default: break; } - if (u_strlen(kana_head)>0) u_strcpy(atama,kana_head); + if (u_strlen(kana_head)==0) u_strcpy(kana_head,atama); /* read idx file */ diff --git a/indexing/upmendex/source/mendex.h b/indexing/upmendex/source/mendex.h index 091862d61e..d4bad85e8c 100644 --- a/indexing/upmendex/source/mendex.h +++ b/indexing/upmendex/source/mendex.h @@ -47,6 +47,7 @@ int dicread(const char *filename); int lastpage(const char *filename); /* sort.c */ +void init_icu_collator(); void wsort(struct index *ind, int num); void pagesort(struct index *ind, int num); int is_latin(UChar *c); diff --git a/indexing/upmendex/source/sort.c b/indexing/upmendex/source/sort.c index bcecd765cf..6ba8818156 100644 --- a/indexing/upmendex/source/sort.c +++ b/indexing/upmendex/source/sort.c @@ -3,6 +3,7 @@ #include "exkana.h" #include "exvar.h" +#include "exhanzi.h" #define RULEBUFSIZE 29210+STYBUFSIZE /* @@ -28,15 +29,65 @@ static int ordering(UChar *c); static int get_charset_juncture(UChar *str); static int unescape(const unsigned char *src, UChar *dist); -/* sort index */ -void wsort(struct index *ind, int num) +/* init ICU collator */ +void init_icu_collator() { - int i,order; UErrorCode status; UParseError parse_error; UChar rules[RULEBUFSIZE] = {'\0'}; + int i; int32_t len; + status = U_ZERO_ERROR; + if (strlen(icu_rules)>0) { + if (strcmp(icu_locale,"root")!=0) { + icu_collator = ucol_open(icu_locale, &status); + if (U_FAILURE(status)) { + verb_printf(efp, "\n[ICU] Collator creation failed.: %s\n", u_errorName(status)); + exit(254); + } + len = ucol_getRulesEx(icu_collator, UCOL_TAILORING_ONLY, rules, RULEBUFSIZE); + if (u_strlen(rules)<len) { + verb_printf(efp, "\n[ICU] Failed to extract collation rules by locale (%s). Need buffer size %d.\n", + icu_locale, len); + exit(254); + } + ucol_close(icu_collator); + } + unescape((unsigned char *)icu_rules, rules); + status = U_ZERO_ERROR; + icu_collator = ucol_openRules(rules, -1, UCOL_OFF, UCOL_TERTIARY, &parse_error, &status); + } else + icu_collator = ucol_open(icu_locale, &status); + if (U_FAILURE(status)) { + verb_printf(efp, "\n[ICU] Collator creation failed.: %s\n", u_errorName(status)); + exit(254); + } + if (status == U_USING_DEFAULT_WARNING) { + warn_printf(efp, "\nWarning: [ICU] U_USING_DEFAULT_WARNING for locale %s\n", + icu_locale); + } + if (status == U_USING_FALLBACK_WARNING) { + warn_printf(efp, "\nWarning: [ICU] U_USING_FALLBACK_WARNING for locale %s\n", + icu_locale); + } + for (i=0;i<UCOL_ATTRIBUTE_COUNT;i++) { + if (icu_attributes[i]!=UCOL_DEFAULT) { + status = U_ZERO_ERROR; + ucol_setAttribute(icu_collator, i, icu_attributes[i], &status); + } + if (U_FAILURE(status)) { + warn_printf(efp, "\nWarning: [ICU] Failed to set attribute (%d): %s\n", + i, u_errorName(status)); + } + } +} + +/* sort index */ +void wsort(struct index *ind, int num) +{ + int i,order; + for (order=1,i=0;;i++) { switch (character_order[i]) { case '\0': @@ -91,6 +142,10 @@ void wsort(struct index *ind, int num) hbrw=order++; break; + case '@': + sym=nmbr=ltn=kana=hngl=hnz=cyr=grk=dvng=thai=arab=hbrw=order++; + break; + default: verb_printf(efp,"\nWarning: Illegal input for character_order (%c).",character_order[i]); break; @@ -112,49 +167,6 @@ BREAK: if (arab==0) arab=order++; if (hbrw==0) hbrw=order++; - status = U_ZERO_ERROR; - if (strlen(icu_rules)>0) { - if (strcmp(icu_locale,"root")!=0) { - icu_collator = ucol_open(icu_locale, &status); - if (U_FAILURE(status)) { - verb_printf(efp, "\n[ICU] Collator creation failed.: %s\n", u_errorName(status)); - exit(254); - } - len = ucol_getRulesEx(icu_collator, UCOL_TAILORING_ONLY, rules, RULEBUFSIZE); - if (u_strlen(rules)<len) { - verb_printf(efp, "\n[ICU] Failed to extract collation rules by locale (%s). Need buffer size %d.\n", - icu_locale, len); - exit(254); - } - ucol_close(icu_collator); - } - unescape((unsigned char *)icu_rules, rules); - status = U_ZERO_ERROR; - icu_collator = ucol_openRules(rules, -1, UCOL_OFF, UCOL_TERTIARY, &parse_error, &status); - } else - icu_collator = ucol_open(icu_locale, &status); - if (U_FAILURE(status)) { - verb_printf(efp, "\n[ICU] Collator creation failed.: %s\n", u_errorName(status)); - exit(254); - } - if (status == U_USING_DEFAULT_WARNING) { - warn_printf(efp, "\nWarning: [ICU] U_USING_DEFAULT_WARNING for locale %s\n", - icu_locale); - } - if (status == U_USING_FALLBACK_WARNING) { - warn_printf(efp, "\nWarning: [ICU] U_USING_FALLBACK_WARNING for locale %s\n", - icu_locale); - } - for (i=0;i<UCOL_ATTRIBUTE_COUNT;i++) { - if (icu_attributes[i]!=UCOL_DEFAULT) { - status = U_ZERO_ERROR; - ucol_setAttribute(icu_collator, i, icu_attributes[i], &status); - } - if (U_FAILURE(status)) { - warn_printf(efp, "\nWarning: [ICU] Failed to set attribute (%d): %s\n", - i, u_errorName(status)); - } - } qsort(ind,num,sizeof(struct index),wcomp); } @@ -415,8 +427,6 @@ static int unescape(const unsigned char *src, UChar *dest) int is_latin(UChar *c) { - UChar32 c32; - if (((*c>=L'A')&&(*c<=L'Z'))||((*c>=L'a')&&(*c<=L'z'))) return 1; else if ((*c==0x00AA)||(*c==0x00BA)) return 1; /* Latin-1 Supplement */ else if ((*c>=0x00C0)&&(*c<=0x00D6)) return 1; @@ -436,6 +446,7 @@ int is_latin(UChar *c) &&(*c<=0x24E9)) return 1; /* CIRCLED LATIN SMALL LETTER */ if (is_surrogate_pair(c)) { + UChar32 c32; c32=U16_GET_SUPPLEMENTARY(*c,*(c+1)); if ((c32>=0x10780) && (c32<=0x107BF)) return 2; /* Latin Extended-F */ else if ((c32>=0x1DF00) && (c32<=0x1DFFF)) return 2; /* Latin Extended-G */ @@ -471,8 +482,6 @@ int is_numeric(UChar *c) int is_jpn_kana(UChar *c) { - UChar32 c32; - if (*c==0x30A0) return 0; /* KATAKANA-HIRAGANA DOUBLE HYPHEN */ else if (*c==0x30FB) return 0; /* KATAKANA MIDDLE DOT */ else if ((*c>=0x3040)&&(*c<=0x30FF)) return 1; /* Hiragana, Katakana */ @@ -482,14 +491,34 @@ int is_jpn_kana(UChar *c) else if ((*c>=0x3300)&&(*c<=0x3357)) return 1; /* Squared Katakana words */ if (is_surrogate_pair(c)) { + UChar32 c32; c32=U16_GET_SUPPLEMENTARY(*c,*(c+1)); if ((c32>=0x1B130) && (c32<=0x1B16F)) return 2; /* Small Kana Extensions */ else if ((c32==0x1B000)) return 2; /* KATAKANA LETTER ARCHAIC E */ + else if ((c32>=0x1B11F) /* HIRAGANA LETTER ARCHAIC WU */ + && (c32<=0x1B122)) return 2; /* KATAKANA LETTER ARCHAIC WU */ else if ((c32==0x1F200)) return 2; /* SQUARE HIRAGANA HOKA */ + else if (c32==0x1B001) { + /* check whether U+1B001 is HIRAGANA LETTER ARCHAIC YE or not. + It may be HENTAIGANA LETTER E-1 */ + if (kana_ye_mode==0) { + UCollationResult order; + UCollationStrength strgth; + UChar strX[4],strZ[4]; + strgth = ucol_getStrength(icu_collator); + ucol_setStrength(icu_collator, UCOL_PRIMARY); + strX[0] = 0xD82C; strX[1] = 0xDC01; strX[2] = L'\0'; /* U+1B001 */ + strZ[0] = 0xD82C; strZ[1] = 0xDD21; strZ[2] = L'\0'; /* U+1B121 */ + order = ucol_strcoll(icu_collator, strZ, -1, strX, -1); + kana_ye_mode = (order==UCOL_EQUAL) ? 2 : 1; + ucol_setStrength(icu_collator, strgth); + } + if (kana_ye_mode==2) return 2; + } } return 0; - /* ICU 65 does not seem to support - "Kana Supplement" and "Kana Extended-A" yet. (2020/02/16) */ + /* ICU 71.1 does not seem to support + most of "Kana Supplement" and "Kana Extended-A" yet. (2022/09/11) */ } int is_kor_hngl(UChar *c) @@ -507,8 +536,6 @@ int is_kor_hngl(UChar *c) int is_hanzi(UChar *c) { - UChar32 c32; - if ((*c>=0x2E80) /* CJK Radicals Supplement */ &&(*c<=0x2FDF)) return 1; /* Kangxi Radicals */ else if ((*c>=0x31C0)&&(*c<=0x31EF)) return 1; /* CJK Strokes */ @@ -518,10 +545,17 @@ int is_hanzi(UChar *c) else if ((*c>=0xF900)&&(*c<=0xFAFF)) return 1; /* CJK Compatibility Ideographs */ if (is_surrogate_pair(c)) { + UChar32 c32; c32=U16_GET_SUPPLEMENTARY(*c,*(c+1)); if ((c32>=0x20000) && /* CJK Unified Ideographs Extension B,C,D,E,F */ /* CJK Compatibility Ideographs Supplement */ - (c32<=0x3134F)) return 2; /* CJK Unified Ideographs Extension G */ + (c32<=0x323AF)) return 2; /* CJK Unified Ideographs Extension G,H */ + } + if (*c==0xFDD0) { /* Noncharacter */ + if (hanzi_mode==HANZI_PINYIN && + *(c+1)>=L'A' && *(c+1)<=L'Z' ) return -1; /* Pinyin Index */ + if (hanzi_mode==HANZI_ZHUYIN && + *(c+1)>=0x3105 && *(c+1)<=0x3129) return -1; /* Zhuyin Index */ } return 0; } @@ -541,7 +575,13 @@ int is_cyrillic(UChar *c) else if ((*c>=0x1C80)&&(*c<=0x1C8F)) return 1; /* Cyrillic Extended-C */ else if ((*c>=0x2DE0)&&(*c<=0x2DFF)) return 1; /* Cyrillic Extended-A */ else if ((*c>=0xA640)&&(*c<=0xA69F)) return 1; /* Cyrillic Extended-B */ - else return 0; + + if (is_surrogate_pair(c)) { + UChar32 c32; + c32=U16_GET_SUPPLEMENTARY(*c,*(c+1)); + if ((c32>=0x1E030) && (c32<=0x1E08F)) return 2; /* Cyrillic Extended-D */ + } + return 0; } int is_greek(UChar *c) @@ -558,7 +598,13 @@ int is_devanagari(UChar *c) &&(*c<=0x096F)) return 0; /* Devanagari Digit */ else if ((*c>=0x0900)&&(*c<=0x097F)) return 1; /* Devanagari */ else if ((*c>=0xA8E0)&&(*c<=0xA8FF)) return 1; /* Devanagari Extended */ - else return 0; + + if (is_surrogate_pair(c)) { + UChar32 c32; + c32=U16_GET_SUPPLEMENTARY(*c,*(c+1)); + if ((c32>=0x11B00) && (c32<=0x11B5F)) return 2; /* Devanagari Extended-A */ + } + return 0; } int is_thai(UChar *c) @@ -600,7 +646,13 @@ int is_arabic(UChar *c) &&(*c<=0x08FF)) return 1; /* Arabic Extended-A */ else if ((*c>=0xFB50)&&(*c<=0xFDFF)) return 1; /* Arabic Presentation Forms-A */ else if ((*c>=0xFE70)&&(*c<=0xFEFF)) return 1; /* Arabic Presentation Forms-B */ - else return 0; + + if (is_surrogate_pair(c)) { + UChar32 c32; + c32=U16_GET_SUPPLEMENTARY(*c,*(c+1)); + if ((c32>=0x10EC0) && (c32<=0x10EFF)) return 2; /* Arabic Extended-C */ + } + return 0; } int is_hebrew(UChar *c) diff --git a/indexing/upmendex/source/tests/pprec5.ist b/indexing/upmendex/source/tests/pprec5.ist new file mode 100644 index 0000000000..400d70c786 --- /dev/null +++ b/indexing/upmendex/source/tests/pprec5.ist @@ -0,0 +1,3 @@ +% page_precedence "rnaRA" + +line_max 299 diff --git a/indexing/upmendex/source/tests/pprec6.ist b/indexing/upmendex/source/tests/pprec6.ist new file mode 100644 index 0000000000..5463ac25a4 --- /dev/null +++ b/indexing/upmendex/source/tests/pprec6.ist @@ -0,0 +1,3 @@ +page_precedence "rnR" + +line_max 299 diff --git a/indexing/upmendex/source/tests/pprec7.ist b/indexing/upmendex/source/tests/pprec7.ist new file mode 100644 index 0000000000..f600c5b230 --- /dev/null +++ b/indexing/upmendex/source/tests/pprec7.ist @@ -0,0 +1,3 @@ +page_precedence "naA" + +line_max 299 diff --git a/indexing/upmendex/source/tests/romalpA-5.ind b/indexing/upmendex/source/tests/romalpA-5.ind new file mode 100644 index 0000000000..fd0b0ec449 --- /dev/null +++ b/indexing/upmendex/source/tests/romalpA-5.ind @@ -0,0 +1,5 @@ +\begin{theindex} + + \item foo, I-13, I-15--I-17, II-10, II-20, II-30 + +\end{theindex} diff --git a/indexing/upmendex/source/tests/romalpA-6.ind b/indexing/upmendex/source/tests/romalpA-6.ind new file mode 100644 index 0000000000..fd0b0ec449 --- /dev/null +++ b/indexing/upmendex/source/tests/romalpA-6.ind @@ -0,0 +1,5 @@ +\begin{theindex} + + \item foo, I-13, I-15--I-17, II-10, II-20, II-30 + +\end{theindex} diff --git a/indexing/upmendex/source/tests/romalpA.idx b/indexing/upmendex/source/tests/romalpA.idx new file mode 100644 index 0000000000..03b0646678 --- /dev/null +++ b/indexing/upmendex/source/tests/romalpA.idx @@ -0,0 +1,7 @@ +\indexentry{foo}{I-13} +\indexentry{foo}{I-15} +\indexentry{foo}{I-16} +\indexentry{foo}{I-17} +\indexentry{foo}{II-10} +\indexentry{foo}{II-20} +\indexentry{foo}{II-30} diff --git a/indexing/upmendex/source/tests/romalpB-5.ind b/indexing/upmendex/source/tests/romalpB-5.ind new file mode 100644 index 0000000000..de3d9903bb --- /dev/null +++ b/indexing/upmendex/source/tests/romalpB-5.ind @@ -0,0 +1,14 @@ +\begin{theindex} + + \item bar, I-i-10-C-c, I-i-20-C-c, I-v-10-C-c, I-v-20-C-c, I-x-10-C-c, I-x-20-C-c, I-l-10-C-c, I-l-20-C-c, I-c-10-C-c, I-c-20-C-c, I-d-10-C-c, I-d-20-C-c, I-m-10-C-c, I-m-20-C-c + \item baz, I-i-10-C-c, I-i-10-D-c, I-i-10-I-c, I-i-10-L-c, I-i-10-M-c, I-i-10-V-c, I-i-10-X-c, I-i-20-C-c, I-i-20-D-c, I-i-20-I-c, I-i-20-L-c, I-i-20-M-c, I-i-20-V-c, I-i-20-X-c + + \indexspace + + \item foo, I-i-10-C-c, I-i-20-C-c, V-i-10-C-c, V-i-20-C-c, X-i-10-C-c, X-i-20-C-c, L-i-10-C-c, L-i-20-C-c, C-i-10-C-c, C-i-20-C-c, D-i-10-C-c, D-i-20-C-c, M-i-10-C-c, M-i-20-C-c + + \indexspace + + \item qux, I-i-10-C-c, I-i-10-C-d, I-i-10-C-i, I-i-10-C-l, I-i-10-C-m, I-i-10-C-v, I-i-10-C-x, I-i-20-C-c, I-i-20-C-d, I-i-20-C-i, I-i-20-C-l, I-i-20-C-m, I-i-20-C-v, I-i-20-C-x + +\end{theindex} diff --git a/indexing/upmendex/source/tests/romalpB-6.ind b/indexing/upmendex/source/tests/romalpB-6.ind new file mode 100644 index 0000000000..b5d7d040b5 --- /dev/null +++ b/indexing/upmendex/source/tests/romalpB-6.ind @@ -0,0 +1,14 @@ +\begin{theindex} + + \item bar, I-i-10-C-c, I-i-20-C-c, I-v-10-C-c, I-v-20-C-c, I-x-10-C-c, I-x-20-C-c, I-l-10-C-c, I-l-20-C-c, I-c-10-C-c, I-c-20-C-c, I-d-10-C-c, I-d-20-C-c, I-m-10-C-c, I-m-20-C-c + \item baz, I-i-10-I-c, I-i-10-V-c, I-i-10-X-c, I-i-10-L-c, I-i-10-C-c, I-i-10-D-c, I-i-10-M-c, I-i-20-I-c, I-i-20-V-c, I-i-20-X-c, I-i-20-L-c, I-i-20-C-c, I-i-20-D-c, I-i-20-M-c + + \indexspace + + \item foo, I-i-10-C-c, I-i-20-C-c, V-i-10-C-c, V-i-20-C-c, X-i-10-C-c, X-i-20-C-c, L-i-10-C-c, L-i-20-C-c, C-i-10-C-c, C-i-20-C-c, D-i-10-C-c, D-i-20-C-c, M-i-10-C-c, M-i-20-C-c + + \indexspace + + \item qux, I-i-10-C-i, I-i-10-C-v, I-i-10-C-x, I-i-10-C-l, I-i-10-C-c, I-i-10-C-d, I-i-10-C-m, I-i-20-C-i, I-i-20-C-v, I-i-20-C-x, I-i-20-C-l, I-i-20-C-c, I-i-20-C-d, I-i-20-C-m + +\end{theindex} diff --git a/indexing/upmendex/source/tests/romalpB-7.ind b/indexing/upmendex/source/tests/romalpB-7.ind new file mode 100644 index 0000000000..f65c311dfe --- /dev/null +++ b/indexing/upmendex/source/tests/romalpB-7.ind @@ -0,0 +1,14 @@ +\begin{theindex} + + \item bar, I-c-10-C-c, I-c-20-C-c, I-d-10-C-c, I-d-20-C-c, I-i-10-C-c, I-i-20-C-c, I-l-10-C-c, I-l-20-C-c, I-m-10-C-c, I-m-20-C-c, I-v-10-C-c, I-v-20-C-c, I-x-10-C-c, I-x-20-C-c + \item baz, I-i-10-C-c, I-i-10-D-c, I-i-10-I-c, I-i-10-L-c, I-i-10-M-c, I-i-10-V-c, I-i-10-X-c, I-i-20-C-c, I-i-20-D-c, I-i-20-I-c, I-i-20-L-c, I-i-20-M-c, I-i-20-V-c, I-i-20-X-c + + \indexspace + + \item foo, C-i-10-C-c, C-i-20-C-c, D-i-10-C-c, D-i-20-C-c, I-i-10-C-c, I-i-20-C-c, L-i-10-C-c, L-i-20-C-c, M-i-10-C-c, M-i-20-C-c, V-i-10-C-c, V-i-20-C-c, X-i-10-C-c, X-i-20-C-c + + \indexspace + + \item qux, I-i-10-C-c, I-i-10-C-d, I-i-10-C-i, I-i-10-C-l, I-i-10-C-m, I-i-10-C-v, I-i-10-C-x, I-i-20-C-c, I-i-20-C-d, I-i-20-C-i, I-i-20-C-l, I-i-20-C-m, I-i-20-C-v, I-i-20-C-x + +\end{theindex} diff --git a/indexing/upmendex/source/tests/romalpB.idx b/indexing/upmendex/source/tests/romalpB.idx new file mode 100644 index 0000000000..a56460db3b --- /dev/null +++ b/indexing/upmendex/source/tests/romalpB.idx @@ -0,0 +1,60 @@ +\indexentry{foo}{I-i-10-C-c} +\indexentry{foo}{I-i-20-C-c} +\indexentry{foo}{V-i-10-C-c} +\indexentry{foo}{V-i-20-C-c} +\indexentry{foo}{X-i-10-C-c} +\indexentry{foo}{X-i-20-C-c} +\indexentry{foo}{L-i-10-C-c} +\indexentry{foo}{L-i-20-C-c} +\indexentry{foo}{C-i-10-C-c} +\indexentry{foo}{C-i-20-C-c} +\indexentry{foo}{D-i-10-C-c} +\indexentry{foo}{D-i-20-C-c} +\indexentry{foo}{M-i-10-C-c} +\indexentry{foo}{M-i-20-C-c} + +\indexentry{bar}{I-i-10-C-c} +\indexentry{bar}{I-i-20-C-c} +\indexentry{bar}{I-v-10-C-c} +\indexentry{bar}{I-v-20-C-c} +\indexentry{bar}{I-x-10-C-c} +\indexentry{bar}{I-x-20-C-c} +\indexentry{bar}{I-l-10-C-c} +\indexentry{bar}{I-l-20-C-c} +\indexentry{bar}{I-c-10-C-c} +\indexentry{bar}{I-c-20-C-c} +\indexentry{bar}{I-d-10-C-c} +\indexentry{bar}{I-d-20-C-c} +\indexentry{bar}{I-m-10-C-c} +\indexentry{bar}{I-m-20-C-c} + +\indexentry{baz}{I-i-10-C-c} +\indexentry{baz}{I-i-20-C-c} +\indexentry{baz}{I-i-10-D-c} +\indexentry{baz}{I-i-20-D-c} +\indexentry{baz}{I-i-10-I-c} +\indexentry{baz}{I-i-20-I-c} +\indexentry{baz}{I-i-10-L-c} +\indexentry{baz}{I-i-20-L-c} +\indexentry{baz}{I-i-10-M-c} +\indexentry{baz}{I-i-20-M-c} +\indexentry{baz}{I-i-10-V-c} +\indexentry{baz}{I-i-20-V-c} +\indexentry{baz}{I-i-10-X-c} +\indexentry{baz}{I-i-20-X-c} + +\indexentry{qux}{I-i-10-C-c} +\indexentry{qux}{I-i-20-C-c} +\indexentry{qux}{I-i-10-C-d} +\indexentry{qux}{I-i-20-C-d} +\indexentry{qux}{I-i-10-C-i} +\indexentry{qux}{I-i-20-C-i} +\indexentry{qux}{I-i-10-C-l} +\indexentry{qux}{I-i-20-C-l} +\indexentry{qux}{I-i-10-C-m} +\indexentry{qux}{I-i-20-C-m} +\indexentry{qux}{I-i-10-C-v} +\indexentry{qux}{I-i-20-C-v} +\indexentry{qux}{I-i-10-C-x} +\indexentry{qux}{I-i-20-C-x} + diff --git a/indexing/upmendex/source/tests/romalpC-5.ind b/indexing/upmendex/source/tests/romalpC-5.ind new file mode 100644 index 0000000000..2edc90d1ef --- /dev/null +++ b/indexing/upmendex/source/tests/romalpC-5.ind @@ -0,0 +1,14 @@ +\begin{theindex} + + \item bar, I-i-10-C-c, I-i-20-C-c, I-ii-10-C-c, I-ii-20-C-c, I-iii-10-C-c, I-iii-20-C-c, I-v-10-C-c, I-v-20-C-c, I-x-10-C-c, I-x-20-C-c, I-l-10-C-c, I-l-20-C-c, I-c-10-C-c, I-c-20-C-c, I-d-10-C-c, I-d-20-C-c, I-m-10-C-c, I-m-20-C-c + \item baz, I-i-10-A-c, I-i-10-B-c, I-i-10-C-c, I-i-10-D-c, I-i-10-I-c, I-i-10-L-c, I-i-10-M-c, I-i-10-V-c, I-i-10-X-c, I-i-20-A-c, I-i-20-B-c, I-i-20-C-c, I-i-20-D-c, I-i-20-I-c, I-i-20-L-c, I-i-20-M-c, I-i-20-V-c, I-i-20-X-c + + \indexspace + + \item foo, I-i-10-C-c, I-i-20-C-c, II-i-10-C-c, II-i-20-C-c, III-i-10-C-c, III-i-20-C-c, V-i-10-C-c, V-i-20-C-c, X-i-10-C-c, X-i-20-C-c, L-i-10-C-c, L-i-20-C-c, C-i-10-C-c, C-i-20-C-c, D-i-10-C-c, D-i-20-C-c, M-i-10-C-c, M-i-20-C-c + + \indexspace + + \item qux, I-i-10-C-a--I-i-10-C-d, I-i-10-C-i, I-i-10-C-l, I-i-10-C-m, I-i-10-C-v, I-i-10-C-x, I-i-20-C-a--I-i-20-C-d, I-i-20-C-i, I-i-20-C-l, I-i-20-C-m, I-i-20-C-v, I-i-20-C-x + +\end{theindex} diff --git a/indexing/upmendex/source/tests/romalpC.idx b/indexing/upmendex/source/tests/romalpC.idx new file mode 100644 index 0000000000..208686338a --- /dev/null +++ b/indexing/upmendex/source/tests/romalpC.idx @@ -0,0 +1,76 @@ +\indexentry{foo}{I-i-10-C-c} +\indexentry{foo}{I-i-20-C-c} +\indexentry{foo}{II-i-10-C-c} +\indexentry{foo}{II-i-20-C-c} +\indexentry{foo}{III-i-10-C-c} +\indexentry{foo}{III-i-20-C-c} +\indexentry{foo}{V-i-10-C-c} +\indexentry{foo}{V-i-20-C-c} +\indexentry{foo}{X-i-10-C-c} +\indexentry{foo}{X-i-20-C-c} +\indexentry{foo}{L-i-10-C-c} +\indexentry{foo}{L-i-20-C-c} +\indexentry{foo}{C-i-10-C-c} +\indexentry{foo}{C-i-20-C-c} +\indexentry{foo}{D-i-10-C-c} +\indexentry{foo}{D-i-20-C-c} +\indexentry{foo}{M-i-10-C-c} +\indexentry{foo}{M-i-20-C-c} + +\indexentry{bar}{I-i-10-C-c} +\indexentry{bar}{I-i-20-C-c} +\indexentry{bar}{I-ii-10-C-c} +\indexentry{bar}{I-ii-20-C-c} +\indexentry{bar}{I-iii-10-C-c} +\indexentry{bar}{I-iii-20-C-c} +\indexentry{bar}{I-v-10-C-c} +\indexentry{bar}{I-v-20-C-c} +\indexentry{bar}{I-x-10-C-c} +\indexentry{bar}{I-x-20-C-c} +\indexentry{bar}{I-l-10-C-c} +\indexentry{bar}{I-l-20-C-c} +\indexentry{bar}{I-c-10-C-c} +\indexentry{bar}{I-c-20-C-c} +\indexentry{bar}{I-d-10-C-c} +\indexentry{bar}{I-d-20-C-c} +\indexentry{bar}{I-m-10-C-c} +\indexentry{bar}{I-m-20-C-c} + +\indexentry{baz}{I-i-10-A-c} +\indexentry{baz}{I-i-20-A-c} +\indexentry{baz}{I-i-10-B-c} +\indexentry{baz}{I-i-20-B-c} +\indexentry{baz}{I-i-10-C-c} +\indexentry{baz}{I-i-20-C-c} +\indexentry{baz}{I-i-10-D-c} +\indexentry{baz}{I-i-20-D-c} +\indexentry{baz}{I-i-10-I-c} +\indexentry{baz}{I-i-20-I-c} +\indexentry{baz}{I-i-10-L-c} +\indexentry{baz}{I-i-20-L-c} +\indexentry{baz}{I-i-10-M-c} +\indexentry{baz}{I-i-20-M-c} +\indexentry{baz}{I-i-10-V-c} +\indexentry{baz}{I-i-20-V-c} +\indexentry{baz}{I-i-10-X-c} +\indexentry{baz}{I-i-20-X-c} + +\indexentry{qux}{I-i-10-C-a} +\indexentry{qux}{I-i-20-C-a} +\indexentry{qux}{I-i-10-C-b} +\indexentry{qux}{I-i-20-C-b} +\indexentry{qux}{I-i-10-C-c} +\indexentry{qux}{I-i-20-C-c} +\indexentry{qux}{I-i-10-C-d} +\indexentry{qux}{I-i-20-C-d} +\indexentry{qux}{I-i-10-C-i} +\indexentry{qux}{I-i-20-C-i} +\indexentry{qux}{I-i-10-C-l} +\indexentry{qux}{I-i-20-C-l} +\indexentry{qux}{I-i-10-C-m} +\indexentry{qux}{I-i-20-C-m} +\indexentry{qux}{I-i-10-C-v} +\indexentry{qux}{I-i-20-C-v} +\indexentry{qux}{I-i-10-C-x} +\indexentry{qux}{I-i-20-C-x} + diff --git a/indexing/upmendex/source/tests/romalpD-5.ind b/indexing/upmendex/source/tests/romalpD-5.ind new file mode 100644 index 0000000000..f85a08325d --- /dev/null +++ b/indexing/upmendex/source/tests/romalpD-5.ind @@ -0,0 +1,9 @@ +\begin{theindex} + + \item bar, 1--3, c, d, i, l, m, v, x, I, V, X, L, C, D, M + + \indexspace + + \item foo, i, v, x, l, c, d, m, 1--3, C, D, I, L, M, V, X + +\end{theindex} diff --git a/indexing/upmendex/source/tests/romalpD-6.ind b/indexing/upmendex/source/tests/romalpD-6.ind new file mode 100644 index 0000000000..7514b21947 --- /dev/null +++ b/indexing/upmendex/source/tests/romalpD-6.ind @@ -0,0 +1,9 @@ +\begin{theindex} + + \item bar, i, v, x, l, c, d, m, 1--3, I, V, X, L, C, D, M + + \indexspace + + \item foo, i, v, x, l, c, d, m, 1--3, I, V, X, L, C, D, M + +\end{theindex} diff --git a/indexing/upmendex/source/tests/romalpD-7.ind b/indexing/upmendex/source/tests/romalpD-7.ind new file mode 100644 index 0000000000..779a39c4ff --- /dev/null +++ b/indexing/upmendex/source/tests/romalpD-7.ind @@ -0,0 +1,9 @@ +\begin{theindex} + + \item bar, 1--3, c, d, i, l, m, v, x, C, D, I, L, M, V, X + + \indexspace + + \item foo, 1--3, c, d, i, l, m, v, x, C, D, I, L, M, V, X + +\end{theindex} diff --git a/indexing/upmendex/source/tests/romalpD.idx b/indexing/upmendex/source/tests/romalpD.idx new file mode 100644 index 0000000000..3247aefbf7 --- /dev/null +++ b/indexing/upmendex/source/tests/romalpD.idx @@ -0,0 +1,35 @@ +\indexentry{foo}{1} +\indexentry{foo}{2} +\indexentry{foo}{3} +\indexentry{foo}{C} +\indexentry{foo}{D} +\indexentry{foo}{I} +\indexentry{foo}{L} +\indexentry{foo}{M} +\indexentry{foo}{V} +\indexentry{foo}{X} +\indexentry{foo}{i} +\indexentry{foo}{v} +\indexentry{foo}{x} +\indexentry{foo}{l} +\indexentry{foo}{c} +\indexentry{foo}{d} +\indexentry{foo}{m} + +\indexentry{bar}{1} +\indexentry{bar}{2} +\indexentry{bar}{3} +\indexentry{bar}{I} +\indexentry{bar}{V} +\indexentry{bar}{X} +\indexentry{bar}{L} +\indexentry{bar}{C} +\indexentry{bar}{D} +\indexentry{bar}{M} +\indexentry{bar}{c} +\indexentry{bar}{d} +\indexentry{bar}{i} +\indexentry{bar}{l} +\indexentry{bar}{m} +\indexentry{bar}{v} +\indexentry{bar}{x} diff --git a/indexing/upmendex/source/tests/upmendex.test b/indexing/upmendex/source/tests/upmendex.test index 84dfbd17be..0eef034cad 100755 --- a/indexing/upmendex/source/tests/upmendex.test +++ b/indexing/upmendex/source/tests/upmendex.test @@ -1,89 +1,103 @@ #! /bin/sh -vx -# $Id: upmendex.test 60549 2021-09-19 08:57:18Z takuji $ -# Copyright 2017 Karl Berry <tex-live@tug.org> +# $Id$ +# Copyright 2017-2022 Karl Berry <tex-live@tug.org> # Copyright 2013 Peter Breitenlohner <tex-live@tug.org> # You may freely use, modify and/or distribute this file. TEXMFCNF=$srcdir/../kpathsea export TEXMFCNF -rm -f foo.* uni.* range*.* pprec*.* locale*.* attr*.* rule*.* +rm -f foo.* uni.* range*.* pprec*.* romalp*.* locale*.* attr*.* rule*.* +rc=0 ./upmendex $srcdir/tests/foo.idx -o foo.ind1 -t foo.ilg1 \ - && diff $srcdir/tests/foo.ind foo.ind1 || exit 1 + && diff $srcdir/tests/foo.ind foo.ind1 || rc=1 cat $srcdir/tests/foo.idx | ./upmendex >foo.ind2 2>foo.ilg2 \ - && diff $srcdir/tests/foo.ind foo.ind2 || exit 1 + && diff $srcdir/tests/foo.ind foo.ind2 || rc=2 ./upmendex -d $srcdir/tests/uni.dict -s $srcdir/tests/uni.ist \ $srcdir/tests/uni.idx -o uni.ind1 -t uni.ilg1 \ - && diff $srcdir/tests/uni.ind uni.ind1 || exit 1 + && diff $srcdir/tests/uni.ind uni.ind1 || rc=3 cat $srcdir/tests/uni.idx | \ ./upmendex -d $srcdir/tests/uni.dict -s $srcdir/tests/uni.ist \ >uni.ind2 2>uni.ilg2 \ - && diff $srcdir/tests/uni.ind uni.ind2 || exit 1 + && diff $srcdir/tests/uni.ind uni.ind2 || rc=4 ./upmendex -d $srcdir/tests/uni.dict -s $srcdir/tests/uni1.ist -s $srcdir/tests/uni2.ist \ $srcdir/tests/uni.idx -o uni.ind3 -t uni.ilg3 \ - && diff $srcdir/tests/uni.ind uni.ind3 || exit 1 + && diff $srcdir/tests/uni.ind uni.ind3 || rc=5 -# test for range suffix_2p, suffix_3p, suffix_mp -./upmendex -s $srcdir/tests/range1.ist $srcdir/tests/range.idx \ - -o range1.ind1 -t range1.ilg \ - && diff $srcdir/tests/range1.ind range1.ind1 || exit 1 -./upmendex -s $srcdir/tests/range2.ist $srcdir/tests/range.idx \ - -o range2.ind1 -t range2.ilg \ - && diff $srcdir/tests/range2.ind range2.ind1 || exit 1 +# test for range suffix_2p, suffix_3p, suffix_mp +for num in 1 2 3; do +./upmendex -s $srcdir/tests/range$num.ist $srcdir/tests/range.idx \ + -o range$num.ind1 -t range$num.ilg \ + && diff $srcdir/tests/range$num.ind range$num.ind1 || rc=6 +done -./upmendex -s $srcdir/tests/range3.ist $srcdir/tests/range.idx \ - -o range3.ind1 -t range3.ilg \ - && diff $srcdir/tests/range3.ind range3.ind1 || exit 1 # test for page_precedence and suffix_3p -./upmendex -s $srcdir/tests/pprec0.ist $srcdir/tests/pprecA.idx \ - -o pprecA-0.ind1 -t pprecA-0.ilg \ - && diff $srcdir/tests/pprecA-0.ind pprecA-0.ind1 || exit 1 - -./upmendex -s $srcdir/tests/pprec1.ist $srcdir/tests/pprecA.idx \ - -o pprecA-1.ind1 -t pprecA-1.ilg \ - && diff $srcdir/tests/pprecA-1.ind pprecA-1.ind1 || exit 1 - -./upmendex -s $srcdir/tests/pprec2.ist $srcdir/tests/pprecA.idx \ - -o pprecA-2.ind1 -t pprecA-2.ilg \ - && diff $srcdir/tests/pprecA-2.ind pprecA-2.ind1 || exit 1 +sfx=A +for num in 0 1 2; do +./upmendex -s $srcdir/tests/pprec$num.ist $srcdir/tests/pprec$sfx.idx \ + -o pprec$sfx-$num.ind1 -t pprec$sfx-$num.ilg \ + && diff $srcdir/tests/pprec$sfx-$num.ind pprec$sfx-$num.ind1 || rc=7 +done +sfx=B +for num in 0 3 4; do +./upmendex -s $srcdir/tests/pprec$num.ist $srcdir/tests/pprec$sfx.idx \ + -o pprec$sfx-$num.ind1 -t pprec$sfx-$num.ilg \ + && diff $srcdir/tests/pprec$sfx-$num.ind pprec$sfx-$num.ind1 || rc=8 +done + + +# test for page_precedence, heuristic detection if a letter is Roman or Alpha +sfx=A +for num in 5 6; do +./upmendex -s $srcdir/tests/pprec$num.ist $srcdir/tests/romalp$sfx.idx \ + -o romalp$sfx-$num.ind1 -t romalp$sfx-$num.ilg \ + && diff $srcdir/tests/romalp$sfx-$num.ind romalp$sfx-$num.ind1 || rc=9 +done +sfx=B +for num in 5 6 7; do +./upmendex -s $srcdir/tests/pprec$num.ist $srcdir/tests/romalp$sfx.idx \ + -o romalp$sfx-$num.ind1 -t romalp$sfx-$num.ilg \ + && diff $srcdir/tests/romalp$sfx-$num.ind romalp$sfx-$num.ind1 || rc=10 +done +sfx=C +for num in 5; do +./upmendex -s $srcdir/tests/pprec$num.ist $srcdir/tests/romalp$sfx.idx \ + -o romalp$sfx-$num.ind1 -t romalp$sfx-$num.ilg \ + && diff $srcdir/tests/romalp$sfx-$num.ind romalp$sfx-$num.ind1 || rc=11 +done +sfx=D +for num in 5 6 7; do +./upmendex -s $srcdir/tests/pprec$num.ist $srcdir/tests/romalp$sfx.idx \ + -o romalp$sfx-$num.ind1 -t romalp$sfx-$num.ilg \ + && diff $srcdir/tests/romalp$sfx-$num.ind romalp$sfx-$num.ind1 || rc=12 +done -./upmendex -s $srcdir/tests/pprec0.ist $srcdir/tests/pprecB.idx \ - -o pprecB-0.ind1 -t pprecB-0.ilg \ - && diff $srcdir/tests/pprecB-0.ind pprecB-0.ind1 || exit 1 - -./upmendex -s $srcdir/tests/pprec3.ist $srcdir/tests/pprecB.idx \ - -o pprecB-3.ind1 -t pprecB-3.ilg \ - && diff $srcdir/tests/pprecB-3.ind pprecB-3.ind1 || exit 1 - -./upmendex -s $srcdir/tests/pprec4.ist $srcdir/tests/pprecB.idx \ - -o pprecB-4.ind1 -t pprecB-4.ilg \ - && diff $srcdir/tests/pprecB-4.ind pprecB-4.ind1 || exit 1 # test for icu_locale ./upmendex -s $srcdir/tests/uni.ist -s $srcdir/tests/locale1.ist $srcdir/tests/rule.idx \ -o locale1.ind1 -t locale1.ilg \ - && diff $srcdir/tests/locale1.ind locale1.ind1 || exit 1 + && diff $srcdir/tests/locale1.ind locale1.ind1 || rc=51 # test for icu_attributes ./upmendex -s $srcdir/tests/uni.ist -s $srcdir/tests/attr1.ist $srcdir/tests/rule.idx \ -o attr1.ind1 -t attr1.ilg \ - && diff $srcdir/tests/attr1.ind attr1.ind1 || exit 1 + && diff $srcdir/tests/attr1.ind attr1.ind1 || rc=52 # test for icu_rules ./upmendex -s $srcdir/tests/uni.ist -s $srcdir/tests/rule1.ist $srcdir/tests/rule.idx \ -o rule1.ind1 -t rule1.ilg \ - && diff $srcdir/tests/rule.ind rule1.ind1 || exit 1 + && diff $srcdir/tests/rule.ind rule1.ind1 || rc=53 ./upmendex -s $srcdir/tests/uni.ist -s $srcdir/tests/rule2.ist $srcdir/tests/rule.idx \ -o rule2.ind1 -t rule2.ilg \ - && diff $srcdir/tests/rule.ind rule2.ind1 || exit 1 + && diff $srcdir/tests/rule.ind rule2.ind1 || rc=54 # malformed input ./upmendex -s $srcdir/tests/uni.ist -s $srcdir/tests/rule3.ist $srcdir/tests/rule.idx \ @@ -107,4 +121,7 @@ fi # test for icu_locale + icu_rules ./upmendex -s $srcdir/tests/uni.ist -s $srcdir/tests/locale2.ist -s $srcdir/tests/rule1.ist $srcdir/tests/rule.idx \ -o locale2-rule1.ind1 -t locale2-rule1.ilg \ - && diff $srcdir/tests/locale2-rule.ind locale2-rule1.ind1 || exit 1 + && diff $srcdir/tests/locale2-rule.ind locale2-rule1.ind1 || rc=55 + + +exit $rc |