diff options
author | Norbert Preining <norbert@preining.info> | 2021-11-22 03:03:38 +0000 |
---|---|---|
committer | Norbert Preining <norbert@preining.info> | 2021-11-22 03:03:38 +0000 |
commit | 3b45580e298046d6c1cb0431da31bea04bc6613a (patch) | |
tree | ee2b508b3e9ce370c5134f941650c4316ce9ec8c /indexing/upmendex/source | |
parent | 25fba613796cd9c1b2042e28152d451d8cddef4a (diff) |
CTAN sync 202111220303
Diffstat (limited to 'indexing/upmendex/source')
46 files changed, 1298 insertions, 461 deletions
diff --git a/indexing/upmendex/source/ChangeLog b/indexing/upmendex/source/ChangeLog index 497e0e6e26..b2b3cd2f1e 100644 --- a/indexing/upmendex/source/ChangeLog +++ b/indexing/upmendex/source/ChangeLog @@ -1,3 +1,108 @@ +2021-11-20 TANAKA Takuji <ttk@t-lab.opal.ne.jp> + + * version 1.00 Stable version. + * configure.ac: Bump version. + * {,ex}var.h, fwrite.c, styfile.c: + Add options "script_preamble" and "script_postamble" + in style file. + * sort.c: Add Latin Extended-F and -G. + Add U_FORMAT_CHAR to is_type_mark_or_punct(). + * mendex.h, var.h, convert.c, fwrite.c, sort.c, styfile.c: + Supports Arabic, Hebrew scripts (experimental). + * upmendex.ja.txt: + Update document. + +2021-10-24 TANAKA Takuji <ttk@t-lab.opal.ne.jp> + + * version 0.60 Beta version. + * configure.ac: Bump version. + * Makefile.am, convert.c, fwrite.c, sort.c, + {,ex}kana.h, ktable.h, mendex.h: + Enhance procedure of classifying characters: + Newly classifies by character type of + General Category in Unicode Script Property. + https://github.com/t-tk/upmendex-package/issues/8 + +2021-09-19 TANAKA Takuji <ttk@t-lab.opal.ne.jp> + + * fread.c: + Modify heuristic detection of page number types. + * tests/upmendex.test, tests/range.idx, tests/range[123].ind, + tests/pprecB.idx, tests/pprec[34].ist, tests/pprecB-[034].ind: + Update tests. + +2021-09-18 TANAKA Takuji <ttk@t-lab.opal.ne.jp> + + * fread.c, fwrite.c, sort.c, mendex.h, {,ex}var.h: + Extend levels of page numbers from 3 to 10 + for compatibility with makeindex. + Fix a bug of page ranges with multi-level page numbers. + * fread.c: + Strict check for "page_precedence" and "page_compositor" in style file. + * fwrite.c: + Do not break lines before "delim_n". + * tests/upmendex.test, tests/range.idx, tests/range[123].{ist,ind}, + removed tests/rangetwo.*: + Update tests. + +2021-09-05 TANAKA Takuji <ttk@t-lab.opal.ne.jp> + + * fwrite.c: + Support I-Y ordering in Lithuanian. + +2021-09-04 TANAKA Takuji <ttk@t-lab.opal.ne.jp> + + * version 0.59 Beta version. + * configure.ac: Bump version. + * fwrite.c, styfile.c, {,ex}var.h: + Support output order of symbols and numbers separately + when "symbol_flag" is 2 in style file. + * sort.c: + Accept combination of "icu_locale" and "icu_rules" in style file. + If simultaneously specified, collation rules specified by "icu_rules" are + added on collation rules specified by "icu_locale". + * tests/upmendex.test, tests/rule.{idx,ind}, tests/locale2.ist, + tests/{locale1,attri1,locale2-rule}.ind: + Add a new test and update conventional tests. + +2021-08-29 TANAKA Takuji <ttk@t-lab.opal.ne.jp> + + * styfile.c: + Strict check for "icu_attributes" in style file. + * sort.c: + Strict check for "character_order" in style file. + * fwrite.c: + Fix a bug when "symbol_flag" is 0. + * tests/upmendex.test, tests/attr1.{ist,ind}, + tests/locale1.ind, tests/rule.{idx,ind}: + Add a new test and update conventional tests. + +2021-08-28 TANAKA Takuji <ttk@t-lab.opal.ne.jp> + + * styfile.c: + Add a alias "headings_flag" of the keyword "heading_flag" + in style file for compatibility with makeindex. + Warn if specifier in style file is unknown. + +2021-08-26 TANAKA Takuji <ttk@t-lab.opal.ne.jp> + + * sort.c: + Initialize a variable. + * tests/upmendex.test, tests/locale1.{ist,ind}: + Add a new test for "icu_locale". + +2021-08-25 TANAKA Takuji <ttk@t-lab.opal.ne.jp> + + * sort.c: + Initialize a string. Check malformed input for "icu_rules". + * tests/upmendex.test, tests/rule[34].ist: + Add new tests. + +2021-08-23 TANAKA Takuji <ttk@t-lab.opal.ne.jp> + + * tests/upmendex.test, tests/rule[12].ist, tests/rule.{idx,ind}: + Add new tests for the keyword "icu_rules" in style file. + 2021-08-08 TANAKA Takuji <ttk@t-lab.opal.ne.jp> * version 0.58 Beta version. @@ -90,7 +195,7 @@ 2020-05-01 TANAKA Takuji <ttk@t-lab.opal.ne.jp> - * main.c, version.h, Makefile.{am.in}: Add bug report addresses + * main.c, version.h, Makefile.am: Add bug report addresses on command line help. * convert.c: Tune buffer sizes to avoid tool warnings. diff --git a/indexing/upmendex/source/Makefile.am b/indexing/upmendex/source/Makefile.am index 94bea187b3..451f0df1eb 100644 --- a/indexing/upmendex/source/Makefile.am +++ b/indexing/upmendex/source/Makefile.am @@ -1,4 +1,4 @@ -## $Id$ +## $Id: Makefile.am 60856 2021-10-24 05:44:35Z takuji $ ## Makefile.am for the TeX Live subdirectory texk/upmendex/ ## ## Copyright 2015-2019 Karl Berry <tex-live@tug.org> @@ -19,7 +19,6 @@ upmendex_SOURCES = \ kana.h \ kp.c \ kp.h \ - ktable.h \ main.c \ mendex.h \ version.h \ diff --git a/indexing/upmendex/source/Makefile.in b/indexing/upmendex/source/Makefile.in index 11cff850bc..4ecaadee97 100644 --- a/indexing/upmendex/source/Makefile.in +++ b/indexing/upmendex/source/Makefile.in @@ -1,4 +1,4 @@ -# Makefile.in generated by automake 1.16.4 from Makefile.am. +# Makefile.in generated by automake 1.16.5 from Makefile.am. # @configure_input@ # Copyright (C) 1994-2021 Free Software Foundation, Inc. @@ -593,7 +593,6 @@ upmendex_SOURCES = \ kana.h \ kp.c \ kp.h \ - ktable.h \ main.c \ mendex.h \ version.h \ diff --git a/indexing/upmendex/source/aclocal.m4 b/indexing/upmendex/source/aclocal.m4 index 284ca6d457..8d45a4e2bc 100644 --- a/indexing/upmendex/source/aclocal.m4 +++ b/indexing/upmendex/source/aclocal.m4 @@ -1,4 +1,4 @@ -# generated automatically by aclocal 1.16.4 -*- Autoconf -*- +# generated automatically by aclocal 1.16.5 -*- Autoconf -*- # Copyright (C) 1996-2021 Free Software Foundation, Inc. @@ -35,7 +35,7 @@ AC_DEFUN([AM_AUTOMAKE_VERSION], [am__api_version='1.16' dnl Some users find AM_AUTOMAKE_VERSION and mistake it for a way to dnl require some minimum version. Point them to the right macro. -m4_if([$1], [1.16.4], [], +m4_if([$1], [1.16.5], [], [AC_FATAL([Do not call $0, use AM_INIT_AUTOMAKE([$1]).])])dnl ]) @@ -51,7 +51,7 @@ m4_define([_AM_AUTOCONF_VERSION], []) # Call AM_AUTOMAKE_VERSION and AM_AUTOMAKE_VERSION so they can be traced. # This function is AC_REQUIREd by AM_INIT_AUTOMAKE. AC_DEFUN([AM_SET_CURRENT_AUTOMAKE_VERSION], -[AM_AUTOMAKE_VERSION([1.16.4])dnl +[AM_AUTOMAKE_VERSION([1.16.5])dnl m4_ifndef([AC_AUTOCONF_VERSION], [m4_copy([m4_PACKAGE_VERSION], [AC_AUTOCONF_VERSION])])dnl _AM_AUTOCONF_VERSION(m4_defn([AC_AUTOCONF_VERSION]))]) @@ -428,6 +428,10 @@ m4_defn([AC_PROG_CC]) # release and drop the old call support. AC_DEFUN([AM_INIT_AUTOMAKE], [AC_PREREQ([2.65])dnl +m4_ifdef([_$0_ALREADY_INIT], + [m4_fatal([$0 expanded multiple times +]m4_defn([_$0_ALREADY_INIT]))], + [m4_define([_$0_ALREADY_INIT], m4_expansion_stack)])dnl dnl Autoconf wants to disallow AM_ names. We explicitly allow dnl the ones we care about. m4_pattern_allow([^AM_[A-Z]+FLAGS$])dnl diff --git a/indexing/upmendex/source/configure b/indexing/upmendex/source/configure index f1ffea6fbb..e5aa682d08 100755 --- a/indexing/upmendex/source/configure +++ b/indexing/upmendex/source/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.71 for upmendex (TeX Live) 0.58. +# Generated by GNU Autoconf 2.71 for upmendex (TeX Live) 1.00. # # # Copyright (C) 1992-1996, 1998-2017, 2020-2021 Free Software Foundation, @@ -626,8 +626,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='upmendex (TeX Live)' PACKAGE_TARNAME='upmendex--tex-live-' -PACKAGE_VERSION='0.58' -PACKAGE_STRING='upmendex (TeX Live) 0.58' +PACKAGE_VERSION='1.00' +PACKAGE_STRING='upmendex (TeX Live) 1.00' PACKAGE_BUGREPORT='' PACKAGE_URL='' @@ -1390,7 +1390,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures upmendex (TeX Live) 0.58 to adapt to many kinds of systems. +\`configure' configures upmendex (TeX Live) 1.00 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1462,7 +1462,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of upmendex (TeX Live) 0.58:";; + short | recursive ) echo "Configuration of upmendex (TeX Live) 1.00:";; esac cat <<\_ACEOF @@ -1587,7 +1587,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -upmendex (TeX Live) configure 0.58 +upmendex (TeX Live) configure 1.00 generated by GNU Autoconf 2.71 Copyright (C) 2021 Free Software Foundation, Inc. @@ -2268,7 +2268,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by upmendex (TeX Live) $as_me 0.58, which was +It was created by upmendex (TeX Live) $as_me 1.00, which was generated by GNU Autoconf 2.71. Invocation command line was $ $0$ac_configure_args_raw @@ -8806,7 +8806,7 @@ fi # Define the identity of the package. PACKAGE='upmendex--tex-live-' - VERSION='0.58' + VERSION='1.00' # Some tools Automake needs. @@ -18942,7 +18942,7 @@ Usage: $0 [OPTIONS] Report bugs to <bug-libtool@gnu.org>." lt_cl_version="\ -upmendex (TeX Live) config.lt 0.58 +upmendex (TeX Live) config.lt 1.00 configured by $0, generated by GNU Autoconf 2.71. Copyright (C) 2011 Free Software Foundation, Inc. @@ -21114,7 +21114,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by upmendex (TeX Live) $as_me 0.58, which was +This file was extended by upmendex (TeX Live) $as_me 1.00, which was generated by GNU Autoconf 2.71. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -21182,7 +21182,7 @@ ac_cs_config_escaped=`printf "%s\n" "$ac_cs_config" | sed "s/^ //; s/'/'\\\\\\\\ cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config='$ac_cs_config_escaped' ac_cs_version="\\ -upmendex (TeX Live) config.status 0.58 +upmendex (TeX Live) config.status 1.00 configured by $0, generated by GNU Autoconf 2.71, with options \\"\$ac_cs_config\\" diff --git a/indexing/upmendex/source/configure.ac b/indexing/upmendex/source/configure.ac index 7af5a44b8c..b21873a1ed 100644 --- a/indexing/upmendex/source/configure.ac +++ b/indexing/upmendex/source/configure.ac @@ -1,4 +1,4 @@ -dnl $Id$ +dnl $Id: configure.ac 61096 2021-11-20 09:29:33Z takuji $ dnl Process this file with autoconf to produce a configure script. dnl dnl Copyright 2015-2021 Karl Berry <tex-live@tug.org> @@ -8,8 +8,8 @@ dnl This file is free software; the copyright holder dnl gives unlimited permission to copy and/or distribute it, dnl with or without modifications, as long as this notice is preserved. dnl -AC_INIT([upmendex (TeX Live)], [0.58]) -AC_PREREQ([2.63]) +AC_INIT([upmendex (TeX Live)],[1.00]) +AC_PREREQ([2.71]) AC_CONFIG_SRCDIR([main.c]) AC_CONFIG_AUX_DIR([../../build-aux]) AC_CONFIG_MACRO_DIR([../../m4]) diff --git a/indexing/upmendex/source/convert.c b/indexing/upmendex/source/convert.c index 74d32d7352..a7cae61038 100644 --- a/indexing/upmendex/source/convert.c +++ b/indexing/upmendex/source/convert.c @@ -6,7 +6,6 @@ #include "qsort.h" #include "exkana.h" -#include "ktable.h" #include "exvar.h" #include "kp.h" @@ -25,8 +24,6 @@ static int dlines=0,elines=0; static int dicvalread(const char *filename, struct dictionary *dicval, int line); -#define is_alpha_numeric(a) (((a)>=ALPHATOP && (a)<=ALPHAEND) || (a)==YENSIGN) - UChar * u_xstrdup (const UChar *string) { return u_strcpy(xmalloc((u_strlen(string)+1)*sizeof(UChar)), string); @@ -210,76 +207,66 @@ int convert(UChar *buff1, UChar *buff2) i++; } - else if (buff1[i]<0x80) { + else if (buff1[i]<0x20 && buff1[i]!='\t') { /* ignore control characters */ + i++; + } + + else if (buff1[i]<0x7F) { buff2[j]=buff1[i]; i++; j++; } + else if (buff1[i]<0xA0) { /* ignore control characters */ + i++; + } + else if (is_latin(buff3)||is_cyrillic(buff3)||is_greek(buff3) ||is_jpn_kana(buff3)||is_kor_hngl(buff3)||is_zhuyin(buff3) - ||is_numeric(buff3)||is_devanagari(buff3)||is_thai(buff3) - ||is_comb_diacritical_mark(buff3)) { + ||is_numeric(buff3)==1||is_type_symbol(buff3)==1 + ||is_devanagari(buff3)||is_thai(buff3)||is_arabic(buff3)||is_hebrew(buff3) + ||is_type_mark_or_punct(buff3)) { buff2[j]=buff3[0]; if (wclen==2) buff2[j+1]=buff3[1]; i+=wclen; j+=wclen; } - else if (buff1[i]>=0x80) { - if (is_alpha_numeric(buff3[0])) { -/* alpha-numeric,symbols */ - for (k=0;k<u_strlen(symboltable);k++) { - if (buff3[0]==symboltable[k]) { - buff2[j]=k+0x20; - if ((buff2[j]>='a')&&(buff2[j]<='z')) buff2[j]-=32; - i++; - j+=wclen; - break; - } - } - if (k==u_strlen(symboltable)) { - i++; - buff2[j++]=buff3[0]; + else { + for (k=0;k<dlines;k++) { +/* dictionary table */ + if (u_strncmp(dictable[k].dic[0],&buff1[i],u_strlen(dictable[k].dic[0]))==0) { + u_strncpy(&buff2[j],dictable[k].dic[1],u_strlen(dictable[k].dic[1])); + i+=u_strlen(dictable[k].dic[0]); + j+=u_strlen(dictable[k].dic[1]); + break; } } - - else { - for (k=0;k<dlines;k++) { -/* dictionary table */ - if (u_strncmp(dictable[k].dic[0],&buff1[i],u_strlen(dictable[k].dic[0]))==0) { - u_strncpy(&buff2[j],dictable[k].dic[1],u_strlen(dictable[k].dic[1])); + if ((k==dlines)&&(elines!=0)) { +/* environment dictionary table */ + for (k=0;k<elines;k++) { + if (u_strncmp(envdic[k].dic[0],&buff1[i],u_strlen(envdic[k].dic[0]))==0) { + u_strncpy(&buff2[j],envdic[k].dic[1],u_strlen(envdic[k].dic[1])); i+=u_strlen(dictable[k].dic[0]); - j+=u_strlen(dictable[k].dic[1]); + j+=u_strlen(envdic[k].dic[1]); break; } } - if ((k==dlines)&&(elines!=0)) { -/* environment dictionary table */ - for (k=0;k<elines;k++) { - if (u_strncmp(envdic[k].dic[0],&buff1[i],u_strlen(envdic[k].dic[0]))==0) { - u_strncpy(&buff2[j],envdic[k].dic[1],u_strlen(envdic[k].dic[1])); - i+=u_strlen(dictable[k].dic[0]); - j+=u_strlen(envdic[k].dic[1]); - break; - } - } - } - if (((k==dlines)&&(elines==0))||((k==elines)&&(elines!=0))) { - if (is_hanzi(buff3) || force==1) { + } + if (((k==dlines)&&(elines==0))||((k==elines)&&(elines!=0))) { + if (is_hanzi(buff3) || is_numeric(buff3) || is_type_symbol(buff3) || force==1) { /* forced convert */ - buff2[j]=buff3[0]; - if (wclen==2) buff2[j+1]=buff3[1]; - i+=wclen; - j+=wclen; - } - else { - widechar_to_multibyte(errbuff,BUFFERLEN2,&buff1[i]); - snprintf(errbuff2,BUFFERLEN3,"\nError: %s is no entry in dictionary file ",errbuff); - fputs(errbuff2,efp); - if (efp!=stderr) fputs(errbuff2,stderr); - return -1; - } + buff2[j]=buff3[0]; + if (wclen==2) buff2[j+1]=buff3[1]; + i+=wclen; + j+=wclen; + } + else { + widechar_to_multibyte(errbuff,BUFFERLEN2,&buff1[i]); + snprintf(errbuff2,BUFFERLEN3,"\nError: %s is no entry in dictionary file ",errbuff); + fputs(errbuff2,efp); + if (efp!=stderr) fputs(errbuff2,stderr); + return -1; } } } diff --git a/indexing/upmendex/source/exkana.h b/indexing/upmendex/source/exkana.h index aeb5a18d82..9e4ba1d197 100644 --- a/indexing/upmendex/source/exkana.h +++ b/indexing/upmendex/source/exkana.h @@ -4,7 +4,6 @@ extern UChar AIUEO[]; extern UChar *aiueo; extern UChar kanatable[]; extern UChar extkanatable[]; -extern UChar circkanatable[]; #define SPACE 0x3000 /* 全角スペース */ #define ALPHATOP 0xff01 /* ! */ @@ -18,8 +17,19 @@ extern UChar circkanatable[]; #define EXKANAEND 0x31FF /* ㇿ */ #define CRKANATOP 0x32D0 /* ㋐ */ #define CRKANAEND 0x32FE /* ㋾ */ +#define HANKANATOP 0xFF71 /* ア */ +#define HANKANAEND 0xFF9D /* ン */ +#define HANKANAWO 0xFF66 /* ヲ */ +#define HANKANATU 0xFF6F /* ッ */ +#define SQKANATOP 0x3300 /* ㌀ */ +#define SQKANAEND 0x3357 /* ㍗ */ +#define CRLATNTOP 0x24B6 /* Ⓐ */ +#define CRLATNEND 0x24E9 /* ⓩ */ #define is_katakana(a) ((a)>=KATATOP && (a)<=KATAEND) #define is_hiragana(a) ((a)>=HIRATOP && (a)<=HIRAEND) #define is_extkana(a) ((a)>=EXKANATOP && (a)<=EXKANAEND) #define is_circkana(a) ((a)>=CRKANATOP && (a)<=CRKANAEND) +#define is_hankana(a) ((a)>=HANKANATOP && (a)<=HANKANAEND || (a)>=HANKANAWO && (a)<=HANKANATU) +#define is_sqkana(a) ((a)>=SQKANATOP && (a)<=SQKANAEND) +#define is_circlatin(a) ((a)>=CRLATNTOP && (a)<=CRLATNEND) diff --git a/indexing/upmendex/source/exvar.h b/indexing/upmendex/source/exvar.h index 7b061c6655..790f327e27 100644 --- a/indexing/upmendex/source/exvar.h +++ b/indexing/upmendex/source/exvar.h @@ -2,7 +2,7 @@ extern FILE *efp; extern int lines,idxcount,acc,reject; extern int prange,fsti,lorder,bcomp,force,fpage,gflg,verb,debug; -extern int warn,scount,pattr[3]; +extern int warn,scount,pattr[]; extern struct index *ind; @@ -26,7 +26,7 @@ extern int line_max; extern char indent_space[]; extern int indent_length; extern int priority; -extern char symbol[]; +extern char symhead[],numhead[]; extern char symhead_positive[],symhead_negative[]; extern char numhead_positive[],numhead_negative[]; extern int symbol_flag; @@ -35,6 +35,7 @@ extern UChar atama[],hangul_head[],hanzi_head[],kana_head[]; extern UChar devanagari_head[],thai_head[]; extern char page_compositor[],page_precedence[]; extern char character_order[]; +extern char script_preamble[][STYBUFSIZE],script_postamble[][STYBUFSIZE]; extern char icu_locale[],icu_rules[]; extern int icu_attributes[]; diff --git a/indexing/upmendex/source/fread.c b/indexing/upmendex/source/fread.c index a336022e11..66d3e0b144 100644 --- a/indexing/upmendex/source/fread.c +++ b/indexing/upmendex/source/fread.c @@ -344,7 +344,7 @@ LOOP: copy_multibyte_char(buff, table, &j, &k); } - table[k]='\0'; + table[k]='\0'; for (k=0;k<=ind[l].num;k++) { if (strcmp(ind[l].p[k].page,table)==0) { @@ -356,9 +356,9 @@ LOOP: ind[l].num++; if (!((ind[l].num)%16)) ind[l].p=(struct page *)xrealloc(ind[l].p,sizeof(struct page)*((int)((ind[l].num)/16)+1)*16); - ind[l].p[ind[l].num].page=xstrdup(table); + ind[l].p[ind[l].num].page=xstrdup(table); - ind[l].p[ind[l].num].enc=xstrdup(estr); + ind[l].p[ind[l].num].enc=xstrdup(estr); chkpageattr(&ind[l].p[ind[l].num]); } } @@ -388,15 +388,15 @@ LOOP: nest++; if (buff[j]==arg_close) { if (nest==0) { - table[k]='\0'; - ind[i].p[0].page=xstrdup(table); + table[k]='\0'; + ind[i].p[0].page=xstrdup(table); break; } else nest--; } copy_multibyte_char(buff, table, &j, &k); } - ind[l].p[0].enc=xstrdup(estr); + ind[l].p[0].enc=xstrdup(estr); chkpageattr(&ind[i].p[0]); } } @@ -493,25 +493,54 @@ int widechar_to_multibyte(char *mbstr, int32_t size, UChar *wcstr) static void chkpageattr(struct page *p) { - int i,j,cc=0; + int i,j,cc=0,cnt,pplen,pclen; + char buff[16],*pcpos,*page0; + pplen=strlen(page_precedence); + pclen=strlen(page_compositor); for (i=0;i<strlen(p->page);i++) { - if (strncmp(page_compositor,&p->page[i],strlen(page_compositor))==0) { + page0=&p->page[i]; + if (strncmp(page_compositor,page0,pclen)==0) { p->attr[cc]=pattr[cc]; cc++; - i+=strlen(page_compositor)-1; + i+=pclen-1; + if (cc>=PAGE_COMPOSIT_DEPTH) { + if (pclen>0) + verb_printf(efp, "\nToo many fields of page number \"%s\".\n", p->page); + else + verb_printf(efp, "\nIllegular page_comositor specification.\n"); + exit(253); + } } else { -ATTRLOOP: - if (!((p->page[i]>='0' && p->page[i]<='9') || (p->page[i]>='A' && p->page[i]<='Z') || (p->page[i]>='a' && p->page[i]<='z'))) { + cnt=0; + if (!((*page0>='0' && *page0<='9') || (*page0>='A' && *page0<='Z') || (*page0>='a' && *page0<='z'))) { p->attr[cc]= -1; if (cc<2) p->attr[++cc]= -1; return; } + pcpos=strstr(page0,page_compositor); + j=pcpos ? pcpos-page0 : strlen(page0); + if (j>15) { + verb_printf(efp, "\nToo long page number string \"%s\".\n", page0); + exit(253); + } + strncpy(buff,page0,j); + buff[j]='\0'; +ATTRLOOP: + cnt++; + if (cnt>pplen) { + verb_printf(efp, "\nFailed to find page type for page \"%s\" in page_precedence specification (%s).\n", + page0, page_precedence); + exit(253); + } + switch(page_precedence[pattr[cc]]) { case 'r': - if (strchr("ivxlcdm",p->page[i])==NULL) { - if (pattr[cc]<strlen(page_precedence)-1) + if (strchr("ivxlcdm",*page0)==NULL || + (strchr("lcdm",*page0) && strchr(page_precedence,'a') && strlen(buff)==1 && pcpos)) { + /* heuristic detection as alphabet since L=50, C=100, D=100, M=1000 are quite large */ + if (pattr[cc]<pplen-1) pattr[cc]++; else pattr[cc]=0; for (j=cc+1;j<3;j++) pattr[j]=0; @@ -519,8 +548,10 @@ ATTRLOOP: } break; case 'R': - if (strchr("IVXLCDM",p->page[i])==NULL) { - if (pattr[cc]<strlen(page_precedence)-1) + if (strchr("IVXLCDM",*page0)==NULL || + (strchr("LCDM",*page0) && strchr(page_precedence,'A') && strlen(buff)==1 && pcpos)) { + /* heuristic detection as alphabet since L=50, C=100, D=100, M=1000 are quite large */ + if (pattr[cc]<pplen-1) pattr[cc]++; else pattr[cc]=0; for (j=cc+1;j<3;j++) pattr[j]=0; @@ -528,8 +559,8 @@ ATTRLOOP: } break; case 'n': - if (p->page[i]<'0' || p->page[i]>'9') { - if (pattr[cc]<strlen(page_precedence)-1) + if (*page0<'0' || *page0>'9') { + if (pattr[cc]<pplen-1) pattr[cc]++; else pattr[cc]=0; for (j=cc+1;j<3;j++) pattr[j]=0; @@ -537,8 +568,10 @@ ATTRLOOP: } break; case 'a': - if (p->page[i]<'a' || p->page[i]>'z') { - if (pattr[cc]<strlen(page_precedence)-1) + if (*page0<'a' || *page0>'z' || strlen(buff)>1 || + (strchr("ivx",*page0) && strchr(page_precedence,'r') && !pcpos)) { + /* heuristic detection as roman number since I=1, V=5, X=10 are quite small */ + if (pattr[cc]<pplen-1) pattr[cc]++; else pattr[cc]=0; for (j=cc+1;j<3;j++) pattr[j]=0; @@ -546,8 +579,10 @@ ATTRLOOP: } break; case 'A': - if (p->page[i]<'A' || p->page[i]>'Z') { - if (pattr[cc]<strlen(page_precedence)-1) + if (*page0<'A' || *page0>'Z' || strlen(buff)>1 || + (strchr("IVX",*page0) && strchr(page_precedence,'R') && !pcpos)) { + /* heuristic detection as roman number since I=1, V=5, X=10 are quite small */ + if (pattr[cc]<pplen-1) pattr[cc]++; else pattr[cc]=0; for (j=cc+1;j<3;j++) pattr[j]=0; @@ -555,7 +590,9 @@ ATTRLOOP: } break; default: - break; + verb_printf(efp, "\nUnknown page type '%c' in page_precedence specification (%s).\n", + page_precedence[pattr[cc]], page_precedence); + exit(253); } } } diff --git a/indexing/upmendex/source/fwrite.c b/indexing/upmendex/source/fwrite.c index d1685ede1c..9ce382589e 100644 --- a/indexing/upmendex/source/fwrite.c +++ b/indexing/upmendex/source/fwrite.c @@ -13,12 +13,12 @@ int line_length=0; static void printpage(struct index *ind, FILE *fp, int num, char *lbuff); static int range_check(struct index ind, int count, char *lbuff); -static void linecheck(char *lbuff, char *tmpbuff); +static void linecheck(char *lbuff, char *tmpbuff, int force); static void crcheck(char *lbuff, FILE *fp); static void index_normalize(UChar *istr, UChar *ini, int *chset); static int initial_cmp_char(UChar *ini, UChar ch); static int init_hanzi_header(void); -static const UNormalizer2* unormalizer_NFD; +static const UNormalizer2 *unormalizer_NFD, *unormalizer_NFKD; static int turkish_i; #define M_NONE 0 @@ -93,7 +93,7 @@ static void fprint_uchar(FILE *fp, const UChar *a, const int mode, const int len } else olen=wclen; if (olen>INITIALLENGTH) { - warn_printf(efp, "\nWarning, Too long (%d) header.\n", olen); + warn_printf(efp, "\nWarning: Too long (%d) header.\n", olen); wclen=INITIALLENGTH; } else wclen=olen; @@ -128,7 +128,7 @@ void warn_printf(FILE *fp, const char *format, ...) vsnprintf(print_buff, sizeof print_buff, format, argptr); va_end(argptr); - warn++; + warn++; fputs(print_buff, stderr); if (fp!=stderr) fputs(print_buff, fp); } @@ -146,11 +146,27 @@ void verb_printf(FILE *fp, const char *format, ...) if (fp!=stderr) fputs(print_buff, fp); } +static int pnumconv2(struct page *p) +{ + int j,k,cc,pclen; + + pclen=strlen(page_compositor); + for (j=k=cc=0;j<strlen(p->page);j++) { + if (strncmp(p->page+j,page_compositor,pclen)==0) { + j+=pclen; + k=j; + cc++; + continue; + } + } + return pnumconv(p->page+k,p->attr[cc]); +} + /* write ind file */ void indwrite(char *filename, struct index *ind, int pagenum) { - int i,j,hpoint=0,tpoint=0,ipoint=0,jpoint=0; + int i,j,hpoint=0,tpoint=0,ipoint=0,jpoint=0,block_open=0; char lbuff[BUFFERLEN],obuff[BUFFERLEN]; UChar datama[256],initial[INITIALLENGTH],initial_prev[INITIALLENGTH]; int chset,chset_prev; @@ -173,12 +189,34 @@ void indwrite(char *filename, struct index *ind, int pagenum) fprintf(fp,"%s%d%s",setpage_prefix,pagenum,setpage_suffix); } perr=U_ZERO_ERROR; - unormalizer_NFD=unorm2_getInstance(NULL, "nfc", UNORM2_DECOMPOSE, &perr); + unormalizer_NFD =unorm2_getInstance(NULL, "nfc", UNORM2_DECOMPOSE, &perr); + unormalizer_NFKD=unorm2_getInstance(NULL, "nfkc", UNORM2_DECOMPOSE, &perr); + + if (strlen(symhead)==0) { + if (lethead_flag>0) { + strcpy(symhead, symhead_positive); + } + else if (lethead_flag<0) { + strcpy(symhead, symhead_negative); + } + } + { + if (lethead_flag>0) { + strcpy(numhead, numhead_positive); + } + else if (lethead_flag<0) { + strcpy(numhead, numhead_negative); + } + } for (i=line_length=0;i<lines;i++) { index_normalize(ind[i].dic[0], initial, &chset); if (i==0) { - if ((CH_LATIN<=chset&&chset<=CH_GREEK) || chset==CH_HANZI) { + if (is_any_script(chset) && strlen(script_preamble[chset])) { + fputs(script_preamble[chset],fp); + block_open=chset; + } + if ((CH_LATIN<=chset&&chset<=CH_GREEK) || chset==CH_HANZI || (CH_ARABIC<=chset&&chset<=CH_HEBREW)) { if (lethead_flag!=0) { fputs(lethead_prefix,fp); fprint_uchar(fp,initial,lethead_flag,-1); @@ -280,16 +318,11 @@ void indwrite(char *filename, struct index *ind, int pagenum) } } else { - if (lethead_flag!=0) { - if (symbol_flag && strlen(symbol)) { - fprintf(fp,"%s%s%s",lethead_prefix,symbol,lethead_suffix); - } - else if (lethead_flag>0) { - fprintf(fp,"%s%s%s",lethead_prefix,symhead_positive,lethead_suffix); - } - else if (lethead_flag<0) { - fprintf(fp,"%s%s%s",lethead_prefix,symhead_negative,lethead_suffix); - } + if (lethead_flag!=0 && symbol_flag==2 && chset==CH_NUMERIC) { + fprintf(fp,"%s%s%s",lethead_prefix,numhead,lethead_suffix); + } + if (lethead_flag!=0 && (symbol_flag==1 || (symbol_flag==2 && chset!=CH_NUMERIC))) { + fprintf(fp,"%s%s%s",lethead_prefix,symhead,lethead_suffix); } widechar_to_multibyte(obuff,BUFFERLEN,ind[i].idx[0]); SPRINTF(lbuff,"%s%s",item_0,obuff); @@ -323,7 +356,19 @@ void indwrite(char *filename, struct index *ind, int pagenum) } else { index_normalize(ind[i-1].dic[0], initial_prev, &chset_prev); - if ((CH_LATIN<=chset&&chset<=CH_GREEK) || chset==CH_HANZI) { + if (chset!=chset_prev && is_any_script(chset_prev) && block_open) { + if (strlen(script_postamble[chset_prev])) { + fputs(script_postamble[chset_prev],fp); + } + block_open=0; + } + if (chset!=chset_prev && is_any_script(chset)) { + if (strlen(script_preamble[chset])) { + fputs(script_preamble[chset],fp); + block_open=chset; + } + } + if ((CH_LATIN<=chset&&chset<=CH_GREEK) || chset==CH_HANZI || (CH_ARABIC<=chset&&chset<=CH_HEBREW)) { if (chset!=chset_prev || ss_comp(initial,initial_prev)) { fputs(group_skip,fp); if (lethead_flag!=0) { @@ -398,18 +443,15 @@ void indwrite(char *filename, struct index *ind, int pagenum) } } else { - if (CH_LATIN<=chset_prev&&chset_prev<=CH_THAI){ - fputs(group_skip,fp); - if (lethead_flag!=0 && symbol_flag) { - if (strlen(symbol)) { - fprintf(fp,"%s%s%s",lethead_prefix,symbol,lethead_suffix); - } - else if (lethead_flag>0) { - fprintf(fp,"%s%s%s",lethead_prefix,symhead_positive,lethead_suffix); - } - else if (lethead_flag<0) { - fprintf(fp,"%s%s%s",lethead_prefix,symhead_negative,lethead_suffix); - } + if (chset_prev!=chset) { + if (is_any_script(chset_prev) || symbol_flag==2) + fputs(group_skip,fp); + if (lethead_flag!=0 && symbol_flag==2 && chset==CH_NUMERIC) { + fprintf(fp,"%s%s%s",lethead_prefix,numhead,lethead_suffix); + } + if (lethead_flag!=0 && (symbol_flag==1 && is_any_script(chset_prev) || + symbol_flag==2 && chset!=CH_NUMERIC) ) { + fprintf(fp,"%s%s%s",lethead_prefix,symhead,lethead_suffix); } } } @@ -479,6 +521,9 @@ void indwrite(char *filename, struct index *ind, int pagenum) printpage(ind,fp,i,lbuff); } } + if (is_any_script(chset) && strlen(script_postamble[chset]) && block_open) { + fputs(script_postamble[chset],fp); + } fputs(postamble,fp); if (fp!=stdout) fclose(fp); @@ -498,7 +543,9 @@ static void printpage(struct index *ind, FILE *fp, int num, char *lbuff) for(j=0;j<ind[num].num;j++) { cc=range_check(ind[num],j,lbuff); if (cc>j) { - if (pnumconv(ind[num].p[j].page,ind[num].p[j].attr[0])==pnumconv(ind[num].p[cc].page,ind[num].p[cc].attr[0])) { + int epage = pnumconv2(&ind[num].p[cc]); + int bpage = pnumconv2(&ind[num].p[j]); + if (epage==bpage) { j=cc-1; continue; } @@ -509,20 +556,18 @@ static void printpage(struct index *ind, FILE *fp, int num, char *lbuff) if (strlen(ind[num].p[j].enc)>0) { SPRINTF(buff,"%s%s%s",encap_prefix,ind[num].p[j].enc,encap_infix); } - if (strlen(suffix_3p)>0 && (pnumconv(ind[num].p[cc].page,ind[num].p[cc].attr[0])-pnumconv(ind[num].p[j].page,ind[num].p[j].attr[0]))==2) { - SAPPENDF(buff,"%s",ind[num].p[j].page); + /* print beginning of range */ + SAPPENDF(buff,"%s",ind[num].p[j].page); + if (strlen(suffix_3p)>0 && epage-bpage==2) { SAPPENDF(buff,"%s",suffix_3p); } - else if (strlen(suffix_mp)>0 && (pnumconv(ind[num].p[cc].page,ind[num].p[cc].attr[0])-pnumconv(ind[num].p[j].page,ind[num].p[j].attr[0]))>=2) { - SAPPENDF(buff,"%s",ind[num].p[j].page); + else if (strlen(suffix_mp)>0 && epage-bpage>=2) { SAPPENDF(buff,"%s",suffix_mp); } - else if (strlen(suffix_2p)>0 && (pnumconv(ind[num].p[cc].page,ind[num].p[cc].attr[0])-pnumconv(ind[num].p[j].page,ind[num].p[j].attr[0]))==1) { - SAPPENDF(buff,"%s",ind[num].p[j].page); + else if (strlen(suffix_2p)>0 && epage-bpage==1) { SAPPENDF(buff,"%s",suffix_2p); } else { - SAPPENDF(buff,"%s",ind[num].p[j].page); SAPPENDF(buff,"%s",delim_r); SAPPENDF(buff,"%s",ind[num].p[cc].page); } @@ -531,14 +576,14 @@ static void printpage(struct index *ind, FILE *fp, int num, char *lbuff) if (strlen(ind[num].p[j].enc)>0) { SAPPENDF(tmpbuff,"%s",encap_suffix); } - linecheck(lbuff,tmpbuff); + linecheck(lbuff,tmpbuff, FALSE); j=cc; if (j==ind[num].num) { goto PRINT; } else { SAPPENDF(tmpbuff,"%s",delim_n); - linecheck(lbuff,tmpbuff); + linecheck(lbuff,tmpbuff, TRUE); } } else if (strlen(ind[num].p[j].enc)>0) { @@ -560,19 +605,19 @@ static void printpage(struct index *ind, FILE *fp, int num, char *lbuff) SAPPENDF(tmpbuff,"%s",ind[num].p[j].page); SAPPENDF(tmpbuff,"%s",encap_suffix); SAPPENDF(tmpbuff,"%s",delim_n); - linecheck(lbuff,tmpbuff); + linecheck(lbuff,tmpbuff, FALSE); } else { SAPPENDF(tmpbuff,"%s",ind[num].p[j].page); SAPPENDF(tmpbuff,"%s",delim_n); - linecheck(lbuff,tmpbuff); + linecheck(lbuff,tmpbuff, FALSE); } } else { /* no encap */ SAPPENDF(tmpbuff,"%s",ind[num].p[j].page); SAPPENDF(tmpbuff,"%s",delim_n); - linecheck(lbuff,tmpbuff); + linecheck(lbuff,tmpbuff, FALSE); } } @@ -606,7 +651,7 @@ static void printpage(struct index *ind, FILE *fp, int num, char *lbuff) else { SAPPENDF(tmpbuff,"%s",ind[num].p[j].page); } - linecheck(lbuff,tmpbuff); + linecheck(lbuff,tmpbuff, FALSE); PRINT: fputs(lbuff,fp); @@ -652,7 +697,7 @@ static int range_check(struct index ind, int count, char *lbuff) if (strlen(ind.p[j].enc)>0) { SPRINTF(tmpbuff,"%s%s%s%s%s%s",encap_prefix,ind.p[j].enc,encap_infix ,ind.p[j].page,encap_suffix,delim_n); - linecheck(lbuff,tmpbuff); + linecheck(lbuff,tmpbuff, FALSE); } } } @@ -682,8 +727,8 @@ static int range_check(struct index ind, int count, char *lbuff) break; } } - cc1=pnumconv(ind.p[i-1].page,ind.p[i-1].attr[0]); - cc2=pnumconv(ind.p[count].page,ind.p[count].attr[0]); + cc1=pnumconv2(&ind.p[i-1]); + cc2=pnumconv2(&ind.p[count]); if (cc1>=cc2+2 || (cc1>=cc2+1 && strlen(suffix_2p)) || force) { return i-1; } @@ -691,9 +736,9 @@ static int range_check(struct index ind, int count, char *lbuff) } /* check line length */ -static void linecheck(char *lbuff, char *tmpbuff) +static void linecheck(char *lbuff, char *tmpbuff, int force) { - if (line_length+strlen(tmpbuff)>line_max) { + if (line_length+strlen(tmpbuff)>line_max && !force) { SAPPENDF(lbuff,"\n"); SAPPENDF(lbuff,"%s",indent_space); SAPPENDF(lbuff,"%s",tmpbuff); @@ -731,12 +776,20 @@ static void index_normalize(UChar *istr, UChar *ini, int *chset) UChar32 c32; UErrorCode perr; UCollationResult order; - static int hanzi_mode=0; + UCollationStrength strgth; + static int hanzi_mode=0, i_y_mode=0; ch=istr[0]; *chset=charset(istr); ini[1]=L'\0'; + if (is_circkana(ch) || is_hankana(ch) || is_sqkana(ch) || is_circlatin(ch)) { /* ㋐㋑㋒.. アイウ.. ㌀㌁㌂.. */ + src[0]=ch; src[1]=0x00; + perr=U_ZERO_ERROR; + unorm2_normalize(unormalizer_NFKD, src, 1, dest, 8, &perr); + if (U_SUCCESS(perr)) + ch=dest[0]; + } if (is_hiragana(ch)) { ch+=KATATOP-HIRATOP; /* hiragana -> katakana */ } @@ -748,15 +801,15 @@ static void index_normalize(UChar *istr, UChar *ini, int *chset) ini[0]=extkanatable[ch-EXKANATOP]; return; } - if (is_circkana(ch)) { /* ㋐㋑㋒㋓㋔ .. ㋻㋼㋽㋾ */ - ini[0]=circkanatable[ch-CRKANATOP]; - return; - } - else if (ch==0x309F) { ini[0]=0x3088; return; } /* HIRAGANA YORI -> よ */ + if (ch==0x309F) { ini[0]=0x3088; return; } /* HIRAGANA YORI -> よ */ else if (ch==0x30FF) { ini[0]=0x3053; return; } /* KATAKANA KOTO -> こ */ else if (is_jpn_kana(istr)==2) { c32=U16_GET_SUPPLEMENTARY(istr[0],istr[1]); switch (c32) { + case 0x1F200: /* 🈀 */ + ini[0]=0x307B; break; /* ほ */ + case 0x1B000: /* 𛀀 */ + ini[0]=0x3048; break; /* え */ case 0x1B150: case 0x1B164: ini[0]=0x3090; break; /* ゐ */ case 0x1B151: case 0x1B165: @@ -852,17 +905,29 @@ static void index_normalize(UChar *istr, UChar *ini, int *chset) u_strcpy(ini,hz_index[lo-1].idx); return; } - else if (is_devanagari(&ch)||is_thai(&ch)) { - if (ch==0x929||0x931||0x934||(0x958<=ch&&ch<=0x95F)) { + else if (is_devanagari(&ch)||is_thai(&ch)||is_arabic(&ch)||is_hebrew(&ch)) { + if (ch==0x929||ch==0x931||ch==0x934||(0x958<=ch&&ch<=0x95F) /* Devanagary */ + ||(0x622<=ch&&ch<=0x626)||ch==0x6C0||ch==0x6C2||ch==0x6D3 /* Arabic */ + ||(0xFB50<=ch&&ch<=0xFDFF) /* Arabic Presentation Forms-A */ + ||(0xFE70<=ch&&ch<=0xFEFF) /* Arabic Presentation Forms-B */ + ||(0xFB1D<=ch&&ch<=0xFB4F) /* Hebrew presentation forms */ + ) { src[0]=ch; src[1]=0x00; perr=U_ZERO_ERROR; unorm2_normalize(unormalizer_NFD, src, 1, dest, 8, &perr); if (U_SUCCESS(perr)) ch=dest[0]; /* without modifier */ } + else if (ch==0x5DA||ch==0x5DD||ch==0x05DF||ch==0x5E3||ch==0x05E5) { /* Hebrew letter final */ + ch++; + } ini[0]=ch; return; } + if (ch>=0xFF21&&ch<=0xFF3A || ch>=0xFF41&&ch<=0xFF5A) { + /* Fullwidth latin letter */ + ch-=0xFF21-0x0041; + } if (ch==0x049||ch==0x069||ch==0x130||ch==0x131||ch==0x0CE||ch==0x0EE) { /* check dotted/dotless İ,I,i,ı and Î,î for Turkish */ strX[0] = 0x131; strX[1] = 0x5A; strX[2] = 0x00; /* ıZ */ @@ -874,6 +939,22 @@ static void index_normalize(UChar *istr, UChar *ini, int *chset) return; } } + if (ch==0x059||ch==0x079) { + /* check Y versus I for Lithuanian */ + if (i_y_mode==0) { + strgth = ucol_getStrength(icu_collator); + ucol_setStrength(icu_collator, UCOL_PRIMARY); + strX[0] = 0x059; strX[1] = 0x00; /* Y */ + strZ[0] = 0x049; strZ[1] = 0x00; /* I */ + order = ucol_strcoll(icu_collator, strZ, -1, strX, -1); + if (order==UCOL_EQUAL) i_y_mode=2; else i_y_mode=1; + ucol_setStrength(icu_collator, strgth); + } + if (i_y_mode==2) { + ini[0] = 0x049; /* I */ + return; + } + } if (ch==0x0C6||ch==0x0E6||ch==0x152||ch==0x153||ch==0x132||ch==0x133 ||ch==0x0DF||ch==0x1E9E||ch==0x13F||ch==0x140||ch==0x490||ch==0x491) { strX[0] = u_toupper(ch); strX[1] = 0x00; /* ex. "Æ" "Œ" */ @@ -976,6 +1057,8 @@ static void index_normalize(UChar *istr, UChar *ini, int *chset) } } } + if (ch==0x0AA) ch=L'A'; + if (ch==0x0BA) ch=L'O'; ini[0]=u_toupper(ch); return; } @@ -1033,7 +1116,7 @@ static int init_hanzi_header(void) len=u_strlen(pch0); } if (len>=INITIALLENGTH) { - warn_printf(efp, "\nWarning, Too long (%d) hanzi header.\n", len); + warn_printf(efp, "\nWarning: Too long (%d) hanzi header.\n", len); len=INITIALLENGTH-1; } u_strncpy(hz_index[k].idx,pch0,len); diff --git a/indexing/upmendex/source/kana.h b/indexing/upmendex/source/kana.h index 5c006cb7f3..41307f2cb4 100644 --- a/indexing/upmendex/source/kana.h +++ b/indexing/upmendex/source/kana.h @@ -192,56 +192,6 @@ UChar extkanatable[]={ 0x308d, /* ろ */ 0 }; -UChar circkanatable[]={ - 0x3042, /* あ */ - 0x3044, /* い */ - 0x3046, /* う */ - 0x3048, /* え */ - 0x304a, /* お */ - 0x304b, /* か */ - 0x304d, /* き */ - 0x304f, /* く */ - 0x3051, /* け */ - 0x3053, /* こ */ - 0x3055, /* さ */ - 0x3057, /* し */ - 0x3059, /* す */ - 0x305b, /* せ */ - 0x305d, /* そ */ - 0x305f, /* た */ - 0x3061, /* ち */ - 0x3064, /* つ */ - 0x3066, /* て */ - 0x3068, /* と */ - 0x306a, /* な */ - 0x306b, /* に */ - 0x306c, /* ぬ */ - 0x306d, /* ね */ - 0x306e, /* の */ - 0x306f, /* は */ - 0x3072, /* ひ */ - 0x3075, /* ふ */ - 0x3078, /* へ */ - 0x307b, /* ほ */ - 0x307e, /* ま */ - 0x307f, /* み */ - 0x3080, /* む */ - 0x3081, /* め */ - 0x3082, /* も */ - 0x3084, /* や */ - 0x3086, /* ゆ */ - 0x3088, /* よ */ - 0x3089, /* ら */ - 0x308a, /* り */ - 0x308b, /* る */ - 0x308c, /* れ */ - 0x308d, /* ろ */ - 0x308f, /* わ */ - 0x3090, /* ゐ */ - 0x3091, /* ゑ */ - 0x3092, /* を */ - 0 -}; UChar GANADA[]={ diff --git a/indexing/upmendex/source/ktable.h b/indexing/upmendex/source/ktable.h deleted file mode 100644 index 9a3a00ef49..0000000000 --- a/indexing/upmendex/source/ktable.h +++ /dev/null @@ -1,98 +0,0 @@ -UChar symboltable[]={ - 0x3000, /* スペース */ - 0xff01, /* ! */ - 0xff02, /* ” */ - 0xff03, /* # */ - 0xff04, /* $ */ - 0xff05, /* % */ - 0xff06, /* & */ - 0xff07, /* ’ */ - 0xff08, /* ( */ - 0xff09, /* ) */ - 0xff0a, /* * */ - 0xff0b, /* + */ - 0xff0c, /* , */ - 0xff0d, /* − */ - 0xff0e, /* . */ - 0xff0f, /* / */ - 0xff10, /* 0 */ - 0xff11, /* 1 */ - 0xff12, /* 2 */ - 0xff13, /* 3 */ - 0xff14, /* 4 */ - 0xff15, /* 5 */ - 0xff16, /* 6 */ - 0xff17, /* 7 */ - 0xff18, /* 8 */ - 0xff19, /* 9 */ - 0xff1a, /* : */ - 0xff1b, /* ; */ - 0xff1c, /* < */ - 0xff1d, /* = */ - 0xff1e, /* > */ - 0xff1f, /* ? */ - 0xff20, /* @ */ - 0xff21, /* A */ - 0xff22, /* B */ - 0xff23, /* C */ - 0xff24, /* D */ - 0xff25, /* E */ - 0xff26, /* F */ - 0xff27, /* G */ - 0xff28, /* H */ - 0xff29, /* I */ - 0xff2a, /* J */ - 0xff2b, /* K */ - 0xff2c, /* L */ - 0xff2d, /* M */ - 0xff2e, /* N */ - 0xff2f, /* O */ - 0xff30, /* P */ - 0xff31, /* Q */ - 0xff32, /* R */ - 0xff33, /* S */ - 0xff34, /* T */ - 0xff35, /* U */ - 0xff36, /* V */ - 0xff37, /* W */ - 0xff38, /* X */ - 0xff39, /* Y */ - 0xff3a, /* Z */ - 0xff3b, /* [ */ - 0xffe5, /* ¥ */ - 0xff3d, /* ] */ - 0xff3e, /* ^ */ - 0xff3f, /* _ */ - 0xff40, /* ‘ */ - 0xff41, /* a */ - 0xff42, /* b */ - 0xff43, /* c */ - 0xff44, /* d */ - 0xff45, /* e */ - 0xff46, /* f */ - 0xff47, /* g */ - 0xff48, /* h */ - 0xff49, /* i */ - 0xff4a, /* j */ - 0xff4b, /* k */ - 0xff4c, /* l */ - 0xff4d, /* m */ - 0xff4e, /* n */ - 0xff4f, /* o */ - 0xff50, /* p */ - 0xff51, /* q */ - 0xff52, /* r */ - 0xff53, /* s */ - 0xff54, /* t */ - 0xff55, /* u */ - 0xff56, /* v */ - 0xff57, /* w */ - 0xff58, /* x */ - 0xff59, /* y */ - 0xff5a, /* z */ - 0xff5b, /* { */ - 0xff5c, /* | */ - 0xff5d, /* } */ - 0xff5f, /* 〜 */ - 0 -}; diff --git a/indexing/upmendex/source/mendex.h b/indexing/upmendex/source/mendex.h index c6703761fc..091862d61e 100644 --- a/indexing/upmendex/source/mendex.h +++ b/indexing/upmendex/source/mendex.h @@ -11,10 +11,12 @@ #include <unicode/uchar.h> #include <unicode/ucol.h> +#define PAGE_COMPOSIT_DEPTH 10 + struct page { char *page; char *enc; - int attr[3]; + int attr[PAGE_COMPOSIT_DEPTH]; }; struct index { @@ -47,7 +49,6 @@ int lastpage(const char *filename); /* sort.c */ void wsort(struct index *ind, int num); void pagesort(struct index *ind, int num); -int is_alphanumeric(UChar *c); int is_latin(UChar *c); int is_numeric(UChar *c); int is_jpn_kana(UChar *c); @@ -58,7 +59,10 @@ int is_cyrillic(UChar *c); int is_greek(UChar *c); int is_devanagari(UChar *c); int is_thai(UChar *c); -int is_comb_diacritical_mark(UChar *c); +int is_arabic(UChar *c); +int is_hebrew(UChar *c); +int is_type_mark_or_punct(UChar *c); +int is_type_symbol(UChar *c); int chkcontinue(struct page *p, int num); int ss_comp(UChar *s1, UChar *s2); @@ -71,8 +75,11 @@ int ss_comp(UChar *s1, UChar *s2); #define CH_HANZI 6 #define CH_DEVANAGARI 7 #define CH_THAI 8 +#define CH_ARABIC 9 +#define CH_HEBREW 10 #define CH_SYMBOL 0x100 #define CH_NUMERIC 0x101 +#define is_any_script(a) ((CH_LATIN<=(a) && (a)<=CH_HEBREW)) /* sort.c */ int charset(UChar *c); diff --git a/indexing/upmendex/source/sort.c b/indexing/upmendex/source/sort.c index e3a47c12b7..bcecd765cf 100644 --- a/indexing/upmendex/source/sort.c +++ b/indexing/upmendex/source/sort.c @@ -4,7 +4,23 @@ #include "exkana.h" #include "exvar.h" -int sym,nmbr,ltn,kana,hngl,hnz,cyr,grk,dvng,thai; +#define RULEBUFSIZE 29210+STYBUFSIZE +/* + length of collation rule in ICU 68.2 + + icu_locale length + ja 6410 + ja@collation=unihan 61 + ko 12577 + ko@collation=unihan 51 + ko@collation=search 782 + zh (pinin) 26909 + zh@collation=unihan 82 + zh@collation=stroke 29208 + zh@collation=zhuyin 28880 +*/ + +int sym,nmbr,ltn,kana,hngl,hnz,cyr,grk,dvng,thai,arab,hbrw; static int wcomp(const void *p, const void *q); static int pcomp(const void *p, const void *q); @@ -17,7 +33,9 @@ void wsort(struct index *ind, int num) { int i,order; UErrorCode status; - UChar rules[STYBUFSIZE]; + UParseError parse_error; + UChar rules[RULEBUFSIZE] = {'\0'}; + int32_t len; for (order=1,i=0;;i++) { switch (character_order[i]) { @@ -65,7 +83,16 @@ void wsort(struct index *ind, int num) thai=order++; break; + case 'a': + arab=order++; + break; + + case 'h': + hbrw=order++; + break; + default: + verb_printf(efp,"\nWarning: Illegal input for character_order (%c).",character_order[i]); break; } } @@ -82,11 +109,28 @@ BREAK: if (grk==0) grk=order++; if (dvng==0) dvng=order++; if (thai==0) thai=order++; + if (arab==0) arab=order++; + if (hbrw==0) hbrw=order++; status = U_ZERO_ERROR; if (strlen(icu_rules)>0) { + if (strcmp(icu_locale,"root")!=0) { + icu_collator = ucol_open(icu_locale, &status); + if (U_FAILURE(status)) { + verb_printf(efp, "\n[ICU] Collator creation failed.: %s\n", u_errorName(status)); + exit(254); + } + len = ucol_getRulesEx(icu_collator, UCOL_TAILORING_ONLY, rules, RULEBUFSIZE); + if (u_strlen(rules)<len) { + verb_printf(efp, "\n[ICU] Failed to extract collation rules by locale (%s). Need buffer size %d.\n", + icu_locale, len); + exit(254); + } + ucol_close(icu_collator); + } unescape((unsigned char *)icu_rules, rules); - icu_collator = ucol_openRules(rules, -1, UCOL_OFF, UCOL_TERTIARY, NULL, &status); + status = U_ZERO_ERROR; + icu_collator = ucol_openRules(rules, -1, UCOL_OFF, UCOL_TERTIARY, &parse_error, &status); } else icu_collator = ucol_open(icu_locale, &status); if (U_FAILURE(status)) { @@ -94,11 +138,11 @@ BREAK: exit(254); } if (status == U_USING_DEFAULT_WARNING) { - warn_printf(efp, "\nWarning, [ICU] U_USING_DEFAULT_WARNING for locale %s\n", + warn_printf(efp, "\nWarning: [ICU] U_USING_DEFAULT_WARNING for locale %s\n", icu_locale); } if (status == U_USING_FALLBACK_WARNING) { - warn_printf(efp, "\nWarning, [ICU] U_USING_FALLBACK_WARNING for locale %s\n", + warn_printf(efp, "\nWarning: [ICU] U_USING_FALLBACK_WARNING for locale %s\n", icu_locale); } for (i=0;i<UCOL_ATTRIBUTE_COUNT;i++) { @@ -107,7 +151,7 @@ BREAK: ucol_setAttribute(icu_collator, i, icu_attributes[i], &status); } if (U_FAILURE(status)) { - warn_printf(efp, "\nWarning, [ICU] Failed to set attribute (%d): %s\n", + warn_printf(efp, "\nWarning: [ICU] Failed to set attribute (%d): %s\n", i, u_errorName(status)); } } @@ -214,12 +258,12 @@ void pagesort(struct index *ind, int num) static int pcomp(const void *p, const void *q) { int i,j,cc=0,num1,num2; - char buff[16]; + char buff[16],*p0,*p1; const struct page *page1 = p, *page2 = q; scount++; - for (i=0;i<3;i++) { + for (i=0;i<PAGE_COMPOSIT_DEPTH;i++) { if ((page1->attr[i]<0)&&(page2->attr[i]<0)) return 0; else if ((page1->attr[i]<0)&&(page2->attr[i]>=0)) return -1; else if ((page2->attr[i]<0)&&(page1->attr[i]>=0)) return 1; @@ -227,18 +271,18 @@ static int pcomp(const void *p, const void *q) if (page1->attr[i]>page2->attr[i]) return 1; if (page1->attr[i]<page2->attr[i]) return -1; - for (j=cc;j<strlen(page1->page);j++) { - if (strncmp(&page1->page[j],page_compositor,strlen(page_compositor))==0) break; - } - strncpy(buff,&page1->page[cc],j-cc); - buff[j-cc]='\0'; + p0=&page1->page[cc]; + p1=strstr(p0, page_compositor); + j=p1 ? p1-p0 : strlen(p0); + strncpy(buff,p0,j); + buff[j]='\0'; num1=pnumconv(buff,page1->attr[i]); - for (j=cc;j<strlen(page2->page);j++) { - if (strncmp(&page2->page[j],page_compositor,strlen(page_compositor))==0) break; - } - strncpy(buff,&page2->page[cc],j-cc); - buff[j-cc]='\0'; + p0=&page2->page[cc]; + p1=strstr(p0, page_compositor); + j=p1 ? p1-p0 : strlen(p0); + strncpy(buff,p0,j); + buff[j]='\0'; num2=pnumconv(buff,page2->attr[i]); if (num1>num2) return 1; @@ -247,7 +291,8 @@ static int pcomp(const void *p, const void *q) if (page1->enc[0]=='(' || page2->enc[0]==')') return -1; if (page1->enc[0]==')' || page2->enc[0]=='(') return 1; - cc=j+strlen(page_compositor); + if (p1) cc+=j+strlen(page_compositor); + else return 0; } return 0; @@ -255,13 +300,15 @@ static int pcomp(const void *p, const void *q) static int ordering(UChar *c) { - if (*c<0x80) { - if (is_latin(c)) return ltn; - else if (is_numeric(c)) return nmbr; - else return sym; + if (*c<0x20) return sym; /* control */ + else if (*c<0x7F) { + if (is_latin(c)) return ltn; + else if (is_numeric(c)) return nmbr; + else return sym; } + else if (*c<0xA0) return sym; /* control */ else { - if (is_latin(c)) return ltn; + if (is_latin(c)) return ltn; else if (is_jpn_kana(c)) return kana; else if (is_kor_hngl(c)) return hngl; else if (is_hanzi(c)) return hnz; @@ -270,20 +317,23 @@ static int ordering(UChar *c) else if (is_numeric(c)) return nmbr; else if (is_devanagari(c)) return dvng; else if (is_thai(c)) return thai; - else return sym; + else if (is_arabic(c)) return arab; + else if (is_hebrew(c)) return hbrw; + else return sym; } } int charset(UChar *c) { - if (*c==0x00) return CH_UNKNOWN; - else if (*c<0x80) { - if (is_latin(c)) return CH_LATIN; - else if (is_numeric(c)) return CH_NUMERIC; - else return CH_SYMBOL; + if (*c<0x20) return CH_UNKNOWN; /* control */ + else if (*c<0x7F) { + if (is_latin(c)) return CH_LATIN; + else if (is_numeric(c)) return CH_NUMERIC; + else return CH_SYMBOL; } + else if (*c<0xA0) return CH_UNKNOWN; /* control */ else { - if (is_latin(c)) return CH_LATIN; + if (is_latin(c)) return CH_LATIN; else if (is_jpn_kana(c)) return CH_KANA; else if (is_kor_hngl(c)) return CH_HANGUL; else if (is_hanzi(c)) return CH_HANZI; @@ -292,7 +342,9 @@ int charset(UChar *c) else if (is_numeric(c)) return CH_NUMERIC; else if (is_devanagari(c)) return CH_DEVANAGARI; else if (is_thai(c)) return CH_THAI; - else return CH_SYMBOL; + else if (is_arabic(c)) return CH_ARABIC; + else if (is_hebrew(c)) return CH_HEBREW; + else return CH_SYMBOL; } } @@ -312,10 +364,10 @@ static int get_charset_juncture(UChar *str) else l = k-1; chset_l=charset(&str[l]); chset_k=charset(&str[k]); - if (chset0==CH_UNKNOWN && chset_l!=CH_SYMBOL && chset_l!=CH_NUMERIC) { + if (chset0==CH_UNKNOWN && is_any_script(chset_l)) { chset0=chset_l; } - if (chset_k!=CH_SYMBOL && chset_k!=CH_NUMERIC) { + if (chset0!=CH_UNKNOWN && is_any_script(chset_k)) { if (chset0!=chset_k) { len=k; return len; @@ -324,10 +376,11 @@ static int get_charset_juncture(UChar *str) } } -static int unescape(const unsigned char *src, UChar *dist) +static int unescape(const unsigned char *src, UChar *dest) { - int i,j,k; + int i,j,k,ret; char tmp[STYBUFSIZE]; + UErrorCode status; for (i=j=0;i<STYBUFSIZE;i++) { if (src[i]=='\0') { @@ -336,32 +389,37 @@ static int unescape(const unsigned char *src, UChar *dist) else if (src[i]< 0x80 && (src[i+1]>=0x80 || src[i+1]=='\0')) { strncpy(tmp,(char *)&src[j],i-j+1); tmp[i-j+1]='\0'; - k=u_strlen(dist); - u_unescape(tmp, &dist[k], STYBUFSIZE-k); + k=u_strlen(dest); + ret=u_unescape(tmp, &dest[k], RULEBUFSIZE-k); + if (ret==0) { + verb_printf(efp, "\n[ICU] Escape sequence in input seems malformed.\n"); + exit(254); + } j=i+1; } else if (src[i]>=0x80 && (src[i+1]< 0x80 || src[i+1]=='\0')) { strncpy(tmp,(char *)&src[j],i-j+1); tmp[i-j+1]='\0'; - k=u_strlen(dist); - multibyte_to_widechar(&dist[k], STYBUFSIZE-k, tmp); + k=u_strlen(dest); + status=U_ZERO_ERROR; + u_strFromUTF8(&dest[k], RULEBUFSIZE-k, NULL, tmp, -1, &status); + if (U_FAILURE(status)) { + verb_printf(efp, "\n[ICU] Input string seems malformed.: %s\n", u_errorName(status)); + exit(254); + } j=i+1; } } return -1; } -int is_alphanumeric(UChar *c) -{ - if (((*c>=L'A')&&(*c<=L'Z'))||((*c>=L'a')&&(*c<=L'z'))||((*c>=L'0')&&(*c<=L'9'))) - return 1; - else return 0; -} - int is_latin(UChar *c) { + UChar32 c32; + if (((*c>=L'A')&&(*c<=L'Z'))||((*c>=L'a')&&(*c<=L'z'))) return 1; - else if ((*c>=0x00C0)&&(*c<=0x00D6)) return 1; /* Latin-1 Supplement */ + else if ((*c==0x00AA)||(*c==0x00BA)) return 1; /* Latin-1 Supplement */ + else if ((*c>=0x00C0)&&(*c<=0x00D6)) return 1; else if ((*c>=0x00D8)&&(*c<=0x00F6)) return 1; else if ((*c>=0x00F8)&&(*c<=0x00FF)) return 1; else if ((*c>=0x0100)&&(*c<=0x024F)) return 1; /* Latin Extended-A,B */ @@ -371,28 +429,63 @@ int is_latin(UChar *c) else if ((*c>=0xAB30)&&(*c<=0xAB6F)) return 1; /* Latin Extended-E */ else if ((*c>=0x1E00)&&(*c<=0x1EFF)) return 1; /* Latin Extended Additional */ else if ((*c>=0xFB00)&&(*c<=0xFB06)) return 1; /* Latin ligatures */ - else return 0; + else if ((*c>=0xFF21)&&(*c<=0xFF3A)) return 1; /* Fullwidth Latin Capital Letter */ + else if ((*c>=0xFF41)&&(*c<=0xFF5A)) return 1; /* Fullwidth Latin Small Letter */ + /* Property of followings is "Common, So (other symbol)", but seem to be treated as Latin by ICU collator */ + else if ((*c>=0x24B6) /* CIRCLED LATIN CAPITAL LETTER */ + &&(*c<=0x24E9)) return 1; /* CIRCLED LATIN SMALL LETTER */ + + if (is_surrogate_pair(c)) { + c32=U16_GET_SUPPLEMENTARY(*c,*(c+1)); + if ((c32>=0x10780) && (c32<=0x107BF)) return 2; /* Latin Extended-F */ + else if ((c32>=0x1DF00) && (c32<=0x1DFFF)) return 2; /* Latin Extended-G */ + } + return 0; } int is_numeric(UChar *c) { + UChar32 c32; + if ((*c>=L'0')&&(*c<=L'9')) return 1; - else if ((*c>=0x0966)&&(*c<=0x096F)) return 1; /* Devanagari Digit */ - else if ((*c>=0x0E50)&&(*c<=0x0E59)) return 1; /* Thai Digit */ - else return 0; + else if ((*c>=0xFF10)&&(*c<=0xFF19)) return 1; /* Fullwidth Digit */ + /* followings do not seem to be treated as numbers by ICU collator though charType is U_OTHER_NUMBER */ + else if ((*c>=0x3192)&&(*c<=0x3195)) return 0; /* IDEOGRAPHIC ANNOTATION ONE MARK..IDEOGRAPHIC ANNOTATION FOUR MARK */ + else if ((*c>=0x3220)&&(*c<=0x3229)) return 0; /* PARENTHESIZED IDEOGRAPH ONE..PARENTHESIZED IDEOGRAPH TEN */ + else if ((*c>=0x3280)&&(*c<=0x3289)) return 0; /* CIRCLED IDEOGRAPH ONE..CIRCLED IDEOGRAPH TEN */ + else if ((*c>=0xA830)&&(*c<=0xA835)) return 0; /* NORTH INDIC FRACTION ONE QUARTER..NORTH INDIC FRACTION THREE SIXTEENTHS */ + + if (is_surrogate_pair(c)) + c32=U16_GET_SUPPLEMENTARY(*c,*(c+1)); + else c32=*c; + + switch (u_charType(c32)) { + case U_DECIMAL_DIGIT_NUMBER: + return 1; + case U_OTHER_NUMBER: + return 2; + default: + return 0; + } } int is_jpn_kana(UChar *c) { UChar32 c32; - if ((*c>=0x3040)&&(*c<=0x30FF)) return 1; /* Hiragana, Katakana */ + if (*c==0x30A0) return 0; /* KATAKANA-HIRAGANA DOUBLE HYPHEN */ + else if (*c==0x30FB) return 0; /* KATAKANA MIDDLE DOT */ + else if ((*c>=0x3040)&&(*c<=0x30FF)) return 1; /* Hiragana, Katakana */ else if ((*c>=0x31F0)&&(*c<=0x31FF)) return 1; /* Katakana Phonetic Extensions */ else if ((*c>=0x32D0)&&(*c<=0x32FE)) return 1; /* Circled Katakana */ + else if ((*c>=0xFF66)&&(*c<=0xFF9F)) return 1; /* Halfwidth Katakana */ + else if ((*c>=0x3300)&&(*c<=0x3357)) return 1; /* Squared Katakana words */ if (is_surrogate_pair(c)) { c32=U16_GET_SUPPLEMENTARY(*c,*(c+1)); if ((c32>=0x1B130) && (c32<=0x1B16F)) return 2; /* Small Kana Extensions */ + else if ((c32==0x1B000)) return 2; /* KATAKANA LETTER ARCHAIC E */ + else if ((c32==0x1F200)) return 2; /* SQUARE HIRAGANA HOKA */ } return 0; /* ICU 65 does not seem to support @@ -416,11 +509,11 @@ int is_hanzi(UChar *c) { UChar32 c32; - if ((*c>=0x2E80)&&(*c<=0x2EFF)) return 1; /* CJK Radicals Supplement */ - else if ((*c>=0x2F00)&&(*c<=0x2FDF)) return 1; /* Kangxi Radicals */ + if ((*c>=0x2E80) /* CJK Radicals Supplement */ + &&(*c<=0x2FDF)) return 1; /* Kangxi Radicals */ else if ((*c>=0x31C0)&&(*c<=0x31EF)) return 1; /* CJK Strokes */ - else if ((*c>=0x3300)&&(*c<=0x33FF)) return 1; /* CJK Compatibility */ - else if ((*c>=0x3400)&&(*c<=0x4DBF)) return 1; /* CJK Unified Ideographs Extension A */ + else if ((*c>=0x3300) /* CJK Compatibility */ + &&(*c<=0x4DBF)) return 1; /* CJK Unified Ideographs Extension A */ else if ((*c>=0x4E00)&&(*c<=0x9FFF)) return 1; /* CJK Unified Ideographs */ else if ((*c>=0xF900)&&(*c<=0xFAFF)) return 1; /* CJK Compatibility Ideographs */ @@ -442,7 +535,9 @@ int is_zhuyin(UChar *c) int is_cyrillic(UChar *c) { - if ((*c>=0x0400)&&(*c<=0x052F)) return 1; /* Cyrillic, Cyrillic Supplement */ + if ((*c==0x0482)) return 0; /* Cyrillic Thousands Sign */ + else if ((*c>=0x0400) /* Cyrillic */ + &&(*c<=0x052F)) return 1; /* Cyrillic Supplement */ else if ((*c>=0x1C80)&&(*c<=0x1C8F)) return 1; /* Cyrillic Extended-C */ else if ((*c>=0x2DE0)&&(*c<=0x2DFF)) return 1; /* Cyrillic Extended-A */ else if ((*c>=0xA640)&&(*c<=0xA69F)) return 1; /* Cyrillic Extended-B */ @@ -451,14 +546,16 @@ int is_cyrillic(UChar *c) int is_greek(UChar *c) { - if ((*c>=0x0370)&&(*c<=0x03FF)) return 1; /* Greek */ + if ((*c==0x03F6)) return 0; /* Greek Reversed Lunate Epsilon Symbol */ + else if ((*c>=0x0370)&&(*c<=0x03FF)) return 1; /* Greek */ else if ((*c>=0x1F00)&&(*c<=0x1FFF)) return 1; /* Greek Extended */ else return 0; } int is_devanagari(UChar *c) { - if ((*c>=0x0966)&&(*c<=0x096F)) return 0; /* Devanagari Digit */ + if ((*c>=0x0964) /* Generic punctuation for scripts of India */ + &&(*c<=0x096F)) return 0; /* Devanagari Digit */ else if ((*c>=0x0900)&&(*c<=0x097F)) return 1; /* Devanagari */ else if ((*c>=0xA8E0)&&(*c<=0xA8FF)) return 1; /* Devanagari Extended */ else return 0; @@ -472,47 +569,132 @@ int is_thai(UChar *c) else return 0; } -int is_comb_diacritical_mark(UChar *c) +int is_arabic(UChar *c) +{ + if ((*c>=0x0600) /* ARABIC NUMBER SIGN..ARABIC SIGN SAMVAT */ + /* ARABIC NUMBER MARK ABOVE */ + &&(*c<=0x0608)) return 0; /* ARABIC-INDIC CUBE ROOT..ARABIC RAY */ + else if ((*c==0x060B)) return 0; /* AFGHANI SIGN */ + else if ((*c==0x060C)) return 0; /* ARABIC COMMA */ + else if ((*c>=0x060E)&&(*c<=0x060F)) return 0; /* ARABIC POETIC VERSE SIGN..ARABIC SIGN MISRA */ + else if ((*c>=0x0660)&&(*c<=0x0669)) return 0; /* ARABIC-INDIC DIGIT ZERO..ARABIC-INDIC DIGIT NINE */ + else if ((*c==0x061B)) return 0; /* ARABIC SEMICOLON */ + else if ((*c==0x061C)) return 0; /* ARABIC LETTER MARK */ + else if ((*c==0x061F)) return 0; /* ARABIC QUESTION MARK */ + else if ((*c==0x0640)) return 0; /* ARABIC TATWEEL */ + else if ((*c==0x06DD)) return 0; /* ARABIC END OF AYAH */ + else if ((*c==0x06DE)) return 0; /* ARABIC START OF RUB EL HIZB */ + else if ((*c==0x06E9)) return 0; /* ARABIC PLACE OF SAJDAH */ + else if ((*c>=0x06F0)&&(*c<=0x06F9)) return 0; /* EXTENDED ARABIC-INDIC DIGIT ZERO..EXTENDED ARABIC-INDIC DIGIT NINE */ + else if ((*c>=0x06FD)&&(*c<=0x06FE)) return 0; /* ARABIC SIGN SINDHI AMPERSAND..ARABIC SIGN SINDHI POSTPOSITION MEN */ + else if ((*c==0x08E2)) return 0; /* ARABIC DISPUTED END OF AYAH */ + else if ((*c>=0x0890)&&(*c<=0x0891)) return 0; /* ARABIC POUND MARK ABOVE..ARABIC PIASTRE MARK ABOVE */ + else if ((*c>=0xFD40)&&(*c<=0xFD4F)) return 0; /* ARABIC LIGATURE RAHIMAHU ALLAAH..ARABIC LIGATURE RAHIMAHUM ALLAAH */ + else if ((*c==0xFDCF)) return 0; /* ARABIC LIGATURE SALAAMUHU ALAYNAA */ + else if ((*c==0xFDFC)) return 0; /* RIAL SIGH */ + else if ((*c>=0xFDFD)&&(*c<=0xFDFF)) return 0; /* ARABIC LIGATURE BISMILLAH AR-RAHMAN AR-RAHEEM..ARABIC LIGATURE AZZA WA JALL */ + + else if ((*c>=0x0600)&&(*c<=0x06FF)) return 1; /* Arabic */ + else if ((*c>=0x0750)&&(*c<=0x077F)) return 1; /* Arabic Supplement */ + else if ((*c>=0x0870) /* Arabic Extended-B */ + &&(*c<=0x08FF)) return 1; /* Arabic Extended-A */ + else if ((*c>=0xFB50)&&(*c<=0xFDFF)) return 1; /* Arabic Presentation Forms-A */ + else if ((*c>=0xFE70)&&(*c<=0xFEFF)) return 1; /* Arabic Presentation Forms-B */ + else return 0; +} + +int is_hebrew(UChar *c) { - if ((*c>=0x02B0)&&(*c<=0x02FF)) return 1; /* Spacing Modifier Letters */ - else if ((*c>=0x0300)&&(*c<=0x036F)) return 1; /* Combining Diacritical Marks */ - else if ((*c>=0x1DC0)&&(*c<=0x1DFF)) return 1; /* Combining Diacritical Marks Supplement */ - else if ((*c>=0x1AB0)&&(*c<=0x1AFF)) return 1; /* Combining Diacritical Marks Extended */ - else if ((*c>=0x3099)&&(*c<=0x309A)) return 1; /* Combining Kana Voiced Sound Marks */ + if ((*c==0xFB29)) return 0; /* Hebrew Letter Alternative Plus Sign */ + else if ((*c>=0x0590)&&(*c<=0x05FF)) return 1; /* Hebrew */ + else if ((*c>=0xFB1D)&&(*c<=0xFB4F)) return 1; /* Hebrew presentation forms */ else return 0; } +int is_type_mark_or_punct(UChar *c) +{ + UChar32 c32; + + if (is_surrogate_pair(c)) + c32=U16_GET_SUPPLEMENTARY(*c,*(c+1)); + else c32=*c; + + switch (u_charType(c32)) { + case U_MODIFIER_LETTER: + case U_DASH_PUNCTUATION: case U_START_PUNCTUATION: case U_END_PUNCTUATION: + case U_CONNECTOR_PUNCTUATION: case U_OTHER_PUNCTUATION: + case U_INITIAL_PUNCTUATION: case U_FINAL_PUNCTUATION: + case U_NON_SPACING_MARK: case U_ENCLOSING_MARK: case U_COMBINING_SPACING_MARK: + case U_FORMAT_CHAR: + return 1; + default: + return 0; + } +} + +int is_type_symbol(UChar *c) +{ + UChar32 c32; + + if (is_surrogate_pair(c)) + c32=U16_GET_SUPPLEMENTARY(*c,*(c+1)); + else c32=*c; + + switch (u_charType(c32)) { + case U_MODIFIER_SYMBOL: + return 1; + case U_MATH_SYMBOL: case U_CURRENCY_SYMBOL: + case U_OTHER_SYMBOL: + return 2; + default: + return 0; + } +} + int chkcontinue(struct page *p, int num) { - int i,j,cc=0,num1,num2; - char buff[16]; + int i,j,cc=0,num1,num2,k1,k2; + char buff1[16],buff2[16],*p0,*p1; - for (i=0;i<3;i++) { + for (i=0;i<PAGE_COMPOSIT_DEPTH;i++) { if ((p[num].attr[i]<0)&&(p[num+1].attr[i]<0)) return 1; else if (p[num].attr[i]!=p[num+1].attr[i]) return 0; - for (j=cc;j<strlen(p[num].page);j++) { - if (strncmp(&p[num].page[j],page_compositor,strlen(page_compositor))==0) break; + p0=&p[num].page[cc]; + p1=strstr(p0, page_compositor); + if (p1) { + j=p1-p0; + k1=j; + } else { + j=strlen(p0); + k1=0; } - strncpy(buff,&p[num].page[cc],j); - buff[j]='\0'; - num1=pnumconv(buff,p[num].attr[i]); - - for (j=cc;j<strlen(p[num+1].page);j++) { - if (strncmp(&p[num+1].page[j],page_compositor,strlen(page_compositor))==0) break; + strncpy(buff1,p0,j); + buff1[j]='\0'; + num1=pnumconv(buff1,p[num].attr[i]); + + p0=&p[num+1].page[cc]; + p1=strstr(p0, page_compositor); + if (p1) { + j=p1-p0; + k2=j; + } else { + j=strlen(p0); + k2=0; } - strncpy(buff,&p[num+1].page[cc],j); - buff[j]='\0'; - num2=pnumconv(buff,p[num+1].attr[i]); - - if (num1==num2 || num1+1==num2) { - if (i==2) return 1; - if ((p[num].attr[i+1]<0)&&(p[num+1].attr[i+1]<0)) return 1; - else return 0; + strncpy(buff2,p0,j); + buff2[j]='\0'; + num2=pnumconv(buff2,p[num+1].attr[i]); + + if (k1>0 || k2>0) { + if (k1!=k2) return 0; + if (strcmp(buff1,buff2)) return 0; + cc+=k1+strlen(page_compositor); + continue; } - else if (num1!=num2) return 0; - cc=j+strlen(page_compositor); + if (num1==num2 || num1+1==num2) return 1; + else return 0; } return 1; diff --git a/indexing/upmendex/source/styfile.c b/indexing/upmendex/source/styfile.c index 619e4a5625..f029679049 100644 --- a/indexing/upmendex/source/styfile.c +++ b/indexing/upmendex/source/styfile.c @@ -76,13 +76,9 @@ void styread(const char *filename) if (getparam(buff,"symhead_negative",symhead_negative)) continue; if (getparam(buff,"numhead_positive",numhead_positive)) continue; if (getparam(buff,"numhead_negative",numhead_negative)) continue; - cc=scompare(buff,"lethead_flag"); - if (cc!= -1) { - lethead_flag=atoi(&buff[cc]); - continue; - } - cc=scompare(buff,"heading_flag"); - if (cc!= -1) { + if ( (cc=scompare(buff,"lethead_flag")) != -1 || + (cc=scompare(buff,"heading_flag")) != -1 || + (cc=scompare(buff,"headings_flag")) != -1 ) { lethead_flag=atoi(&buff[cc]); continue; } @@ -121,7 +117,7 @@ void styread(const char *filename) indent_length=atoi(&buff[cc]); continue; } - if (getparam(buff,"symbol",symbol)) continue; + if (getparam(buff,"symbol",symhead)) continue; cc=scompare(buff,"symbol_flag"); if (cc!= -1) { symbol_flag=atoi(&buff[cc]); @@ -164,6 +160,50 @@ void styread(const char *filename) continue; } if (getparam(buff,"icu_attributes", icu_attr_str )) continue; + + cc=scompare(buff,"script_preamble"); + if (cc!= -1) { + strcpy(tmp,buff+strlen("script_preamble")); + if (getparam(tmp,"latin", script_preamble[CH_LATIN] )) continue; + if (getparam(tmp,"cyrillic", script_preamble[CH_CYRILLIC] )) continue; + if (getparam(tmp,"greek", script_preamble[CH_GREEK] )) continue; + if (getparam(tmp,"kana", script_preamble[CH_KANA] )) continue; + if (getparam(tmp,"hangul", script_preamble[CH_HANGUL] )) continue; + if (getparam(tmp,"hanzi", script_preamble[CH_HANZI] )) continue; + if (getparam(tmp,"devanagari",script_preamble[CH_DEVANAGARI] )) continue; + if (getparam(tmp,"thai", script_preamble[CH_THAI] )) continue; + if (getparam(tmp,"arabic", script_preamble[CH_ARABIC] )) continue; + if (getparam(tmp,"hebrew", script_preamble[CH_HEBREW] )) continue; + if (strlen(tmp)>0) { + verb_printf(efp,"\nWarning: Unknown script for specifier \"script_preamble\" (%s).", tmp); + } + continue; + } + cc=scompare(buff,"script_postamble"); + if (cc!= -1) { + strcpy(tmp,buff+strlen("script_postamble")); + if (getparam(tmp,"latin", script_postamble[CH_LATIN] )) continue; + if (getparam(tmp,"cyrillic", script_postamble[CH_CYRILLIC] )) continue; + if (getparam(tmp,"greek", script_postamble[CH_GREEK] )) continue; + if (getparam(tmp,"kana", script_postamble[CH_KANA] )) continue; + if (getparam(tmp,"hangul", script_postamble[CH_HANGUL] )) continue; + if (getparam(tmp,"hanzi", script_postamble[CH_HANZI] )) continue; + if (getparam(tmp,"devanagari",script_postamble[CH_DEVANAGARI] )) continue; + if (getparam(tmp,"thai", script_postamble[CH_THAI] )) continue; + if (getparam(tmp,"arabic", script_postamble[CH_ARABIC] )) continue; + if (getparam(tmp,"hebrew", script_postamble[CH_HEBREW] )) continue; + if (strlen(tmp)>0) { + verb_printf(efp,"\nWarning: Unknown script for specifier \"script_postamble\" (%s).", tmp); + } + continue; + } + + cc=strcspn(buff," \t\r\n"); + if (cc>0) buff[cc]='\0'; + if (buff[0]=='%' || buff[0]=='\n') continue; + if (strlen(buff)>0) { + verb_printf(efp,"\nWarning: Unknown specifier (%s).", buff); + } } fclose(fp); @@ -316,54 +356,62 @@ static int sstrncmp(const char *s1, const char *s2, size_t len) void set_icu_attributes(void) { int i,attr; - char *pos, *tmp; + char *pos, *head, *tmp; for (i=0;i<UCOL_ATTRIBUTE_COUNT;i++) icu_attributes[i]=UCOL_DEFAULT; tmp=icu_attr_str; - if ((pos=strstr(tmp,"alternate:"))>0) { + head=strtok(tmp, " \t\r\n"); + while (head!=NULL) { + pos=head; + if (strncmp(pos,"alternate:",10)==0) { pos+=10; attr=UCOL_ALTERNATE_HANDLING; - if (strstr(pos,"shifted")) icu_attributes[attr]=UCOL_SHIFTED; - else if (strstr(pos,"non-ignorable")) icu_attributes[attr]=UCOL_NON_IGNORABLE; - else verb_printf(efp,"\nWarning: Illegal input for icu_attributes (alternate)."); - } - if ((pos=strstr(tmp,"strength:"))>0) { + if (strcmp(pos,"shifted")==0) icu_attributes[attr]=UCOL_SHIFTED; + else if (strcmp(pos,"non-ignorable")==0) icu_attributes[attr]=UCOL_NON_IGNORABLE; + else goto illegal; + } else + if (strncmp(pos,"strength:",9)==0) { pos+=9; attr=UCOL_STRENGTH; - if (strstr(pos,"primary")) icu_attributes[attr]=UCOL_PRIMARY; - else if (strstr(pos,"secondary")) icu_attributes[attr]=UCOL_SECONDARY; - else if (strstr(pos,"tertiary")) icu_attributes[attr]=UCOL_TERTIARY; - else if (strstr(pos,"quaternary")) icu_attributes[attr]=UCOL_QUATERNARY; - else if (strstr(pos,"identical")) icu_attributes[attr]=UCOL_IDENTICAL; - else verb_printf(efp,"\nWarning: Illegal input for icu_attributes (strength)."); - } - if ((pos=strstr(tmp,"french-collation:"))>0) { + if (strcmp(pos,"primary")==0) icu_attributes[attr]=UCOL_PRIMARY; + else if (strcmp(pos,"secondary")==0) icu_attributes[attr]=UCOL_SECONDARY; + else if (strcmp(pos,"tertiary")==0) icu_attributes[attr]=UCOL_TERTIARY; + else if (strcmp(pos,"quaternary")==0) icu_attributes[attr]=UCOL_QUATERNARY; + else if (strcmp(pos,"identical")==0) icu_attributes[attr]=UCOL_IDENTICAL; + else goto illegal; + } else + if (strncmp(pos,"french-collation:",17)==0) { pos+=17; attr=UCOL_FRENCH_COLLATION; - if (strstr(pos,"on")) icu_attributes[attr]=UCOL_ON; - else if (strstr(pos,"off")) icu_attributes[attr]=UCOL_OFF; - else verb_printf(efp,"\nWarning: Illegal input for icu_attributes (french-collation)."); - } - if ((pos=strstr(tmp,"case-first:"))>0) { + if (strcmp(pos,"on")==0) icu_attributes[attr]=UCOL_ON; + else if (strcmp(pos,"off")==0) icu_attributes[attr]=UCOL_OFF; + else goto illegal; + } else + if (strncmp(pos,"case-first:",11)==0) { pos+=11; attr=UCOL_CASE_FIRST; - if (strstr(pos,"off")) icu_attributes[attr]=UCOL_OFF; - else if (strstr(pos,"upper-first")) icu_attributes[attr]=UCOL_UPPER_FIRST; - else if (strstr(pos,"lower-first")) icu_attributes[attr]=UCOL_LOWER_FIRST; - else verb_printf(efp,"\nWarning: Illegal input for icu_attributes (case-first)."); - } - if ((pos=strstr(tmp,"case-level:"))>0) { + if (strcmp(pos,"off")==0) icu_attributes[attr]=UCOL_OFF; + else if (strcmp(pos,"upper-first")==0) icu_attributes[attr]=UCOL_UPPER_FIRST; + else if (strcmp(pos,"lower-first")==0) icu_attributes[attr]=UCOL_LOWER_FIRST; + else goto illegal; + } else + if (strncmp(pos,"case-level:",11)==0) { pos+=11; attr=UCOL_CASE_LEVEL; - if (strstr(pos,"on")) icu_attributes[attr]=UCOL_ON; - else if (strstr(pos,"off")) icu_attributes[attr]=UCOL_OFF; - else verb_printf(efp,"\nWarning: Illegal input for icu_attributes (case-level)."); - } - if ((pos=strstr(tmp,"normalization-mode:"))>0) { + if (strcmp(pos,"on")==0) icu_attributes[attr]=UCOL_ON; + else if (strcmp(pos,"off")==0) icu_attributes[attr]=UCOL_OFF; + else goto illegal; + } else + if (strncmp(pos,"normalization-mode:",19)==0) { pos+=19; attr=UCOL_NORMALIZATION_MODE; - if (strstr(pos,"on")) icu_attributes[attr]=UCOL_ON; - else if (strstr(pos,"off")) icu_attributes[attr]=UCOL_OFF; - else verb_printf(efp,"\nWarning: Illegal input for icu_attributes (normalization-mode)."); - } - if ((pos=strstr(tmp,"numeric-ordering:"))>0) { + if (strcmp(pos,"on")==0) icu_attributes[attr]=UCOL_ON; + else if (strcmp(pos,"off")==0) icu_attributes[attr]=UCOL_OFF; + else goto illegal; + } else + if (strncmp(pos,"numeric-ordering:",17)==0) { pos+=17; attr=UCOL_NUMERIC_COLLATION; - if (strstr(pos,"on")) icu_attributes[attr]=UCOL_ON; - else if (strstr(pos,"off")) icu_attributes[attr]=UCOL_OFF; - else verb_printf(efp,"\nWarning: Illegal input for icu_attributes (numeric-ordering)."); + if (strcmp(pos,"on")==0) icu_attributes[attr]=UCOL_ON; + else if (strcmp(pos,"off")==0) icu_attributes[attr]=UCOL_OFF; + else goto illegal; + } else { + illegal: + verb_printf(efp,"\nWarning: Illegal input for icu_attributes (%s).",head); + } + head=strtok(NULL, " \t\r\n"); } } diff --git a/indexing/upmendex/source/tests/attr1.ind b/indexing/upmendex/source/tests/attr1.ind new file mode 100644 index 0000000000..6db7b19221 --- /dev/null +++ b/indexing/upmendex/source/tests/attr1.ind @@ -0,0 +1,40 @@ +\begin{theindex} + +\centerline{\bfseries ◆◆◆ Symbols ◆◆◆}\par\nobreak + \item #\leaders\hbox{$\cdot$}\hfill 2000 + \item $\leaders\hbox{$\cdot$}\hfill 2001 + \item 2\leaders\hbox{$\cdot$}\hfill 1001 + \item 10\leaders\hbox{$\cdot$}\hfill 1000 + + \indexspace + +\centerline{\bfseries ◆◆◆ A ◆◆◆}\par\nobreak + \item A\leaders\hbox{$\cdot$}\hfill 400 + \item a\leaders\hbox{$\cdot$}\hfill 401 + + \indexspace + +\centerline{\bfseries ◆◆◆ B ◆◆◆}\par\nobreak + \item B\leaders\hbox{$\cdot$}\hfill 402 + \item b\leaders\hbox{$\cdot$}\hfill 403 + + \indexspace + +\centerline{\bfseries ◆◆◆ C ◆◆◆}\par\nobreak + \item C\leaders\hbox{$\cdot$}\hfill 404 + \item c\leaders\hbox{$\cdot$}\hfill 405 + \item Ç\leaders\hbox{$\cdot$}\hfill 406 + \item ç\leaders\hbox{$\cdot$}\hfill 407 + \item Ch\leaders\hbox{$\cdot$}\hfill 420 + \item cha\leaders\hbox{$\cdot$}\hfill 421 + \item cz\leaders\hbox{$\cdot$}\hfill 422 + + \indexspace + +\centerline{\bfseries ◆◆◆ U ◆◆◆}\par\nobreak + \item U\leaders\hbox{$\cdot$}\hfill 408 + \item u\leaders\hbox{$\cdot$}\hfill 409 + \item Ü\leaders\hbox{$\cdot$}\hfill 410 + \item ü\leaders\hbox{$\cdot$}\hfill 411 + +\end{theindex} diff --git a/indexing/upmendex/source/tests/attr1.ist b/indexing/upmendex/source/tests/attr1.ist new file mode 100644 index 0000000000..301aff0a6a --- /dev/null +++ b/indexing/upmendex/source/tests/attr1.ist @@ -0,0 +1,2 @@ +icu_attributes "case-first:upper-first +numeric-ordering:on " diff --git a/indexing/upmendex/source/tests/locale1.ind b/indexing/upmendex/source/tests/locale1.ind new file mode 100644 index 0000000000..76bf98f25a --- /dev/null +++ b/indexing/upmendex/source/tests/locale1.ind @@ -0,0 +1,48 @@ +\begin{theindex} + +\centerline{\bfseries ◆◆◆ Symbols ◆◆◆}\par\nobreak + \item #\leaders\hbox{$\cdot$}\hfill 2000 + \item $\leaders\hbox{$\cdot$}\hfill 2001 + \item 10\leaders\hbox{$\cdot$}\hfill 1000 + \item 2\leaders\hbox{$\cdot$}\hfill 1001 + + \indexspace + +\centerline{\bfseries ◆◆◆ A ◆◆◆}\par\nobreak + \item a\leaders\hbox{$\cdot$}\hfill 401 + \item A\leaders\hbox{$\cdot$}\hfill 400 + + \indexspace + +\centerline{\bfseries ◆◆◆ B ◆◆◆}\par\nobreak + \item b\leaders\hbox{$\cdot$}\hfill 403 + \item B\leaders\hbox{$\cdot$}\hfill 402 + + \indexspace + +\centerline{\bfseries ◆◆◆ C ◆◆◆}\par\nobreak + \item c\leaders\hbox{$\cdot$}\hfill 405 + \item C\leaders\hbox{$\cdot$}\hfill 404 + \item Ch\leaders\hbox{$\cdot$}\hfill 420 + \item cha\leaders\hbox{$\cdot$}\hfill 421 + \item cz\leaders\hbox{$\cdot$}\hfill 422 + + \indexspace + +\centerline{\bfseries ◆◆◆ Ç ◆◆◆}\par\nobreak + \item ç\leaders\hbox{$\cdot$}\hfill 407 + \item Ç\leaders\hbox{$\cdot$}\hfill 406 + + \indexspace + +\centerline{\bfseries ◆◆◆ U ◆◆◆}\par\nobreak + \item u\leaders\hbox{$\cdot$}\hfill 409 + \item U\leaders\hbox{$\cdot$}\hfill 408 + + \indexspace + +\centerline{\bfseries ◆◆◆ Ü ◆◆◆}\par\nobreak + \item ü\leaders\hbox{$\cdot$}\hfill 411 + \item Ü\leaders\hbox{$\cdot$}\hfill 410 + +\end{theindex} diff --git a/indexing/upmendex/source/tests/locale1.ist b/indexing/upmendex/source/tests/locale1.ist new file mode 100644 index 0000000000..cf9e67da6e --- /dev/null +++ b/indexing/upmendex/source/tests/locale1.ist @@ -0,0 +1 @@ +icu_locale "az" diff --git a/indexing/upmendex/source/tests/locale2-rule.ind b/indexing/upmendex/source/tests/locale2-rule.ind new file mode 100644 index 0000000000..4054762ab7 --- /dev/null +++ b/indexing/upmendex/source/tests/locale2-rule.ind @@ -0,0 +1,44 @@ +\begin{theindex} + +\centerline{\bfseries ◆◆◆ Symbols ◆◆◆}\par\nobreak + \item #\leaders\hbox{$\cdot$}\hfill 2000 + \item $\leaders\hbox{$\cdot$}\hfill 2001 + \item 10\leaders\hbox{$\cdot$}\hfill 1000 + \item 2\leaders\hbox{$\cdot$}\hfill 1001 + + \indexspace + +\centerline{\bfseries ◆◆◆ A ◆◆◆}\par\nobreak + \item a\leaders\hbox{$\cdot$}\hfill 401 + \item A\leaders\hbox{$\cdot$}\hfill 400 + + \indexspace + +\centerline{\bfseries ◆◆◆ C ◆◆◆}\par\nobreak + \item C\leaders\hbox{$\cdot$}\hfill 404 + \item Ç\leaders\hbox{$\cdot$}\hfill 406 + \item c\leaders\hbox{$\cdot$}\hfill 405 + \item ç\leaders\hbox{$\cdot$}\hfill 407 + \item cz\leaders\hbox{$\cdot$}\hfill 422 + + \indexspace + +\centerline{\bfseries ◆◆◆ U ◆◆◆}\par\nobreak + \item u\leaders\hbox{$\cdot$}\hfill 409 + \item U\leaders\hbox{$\cdot$}\hfill 408 + \item ü\leaders\hbox{$\cdot$}\hfill 411 + \item Ü\leaders\hbox{$\cdot$}\hfill 410 + + \indexspace + +\centerline{\bfseries ◆◆◆ B ◆◆◆}\par\nobreak + \item b\leaders\hbox{$\cdot$}\hfill 403 + \item B\leaders\hbox{$\cdot$}\hfill 402 + + \indexspace + +\centerline{\bfseries ◆◆◆ CH ◆◆◆}\par\nobreak + \item Ch\leaders\hbox{$\cdot$}\hfill 420 + \item cha\leaders\hbox{$\cdot$}\hfill 421 + +\end{theindex} diff --git a/indexing/upmendex/source/tests/locale2.ist b/indexing/upmendex/source/tests/locale2.ist new file mode 100644 index 0000000000..89b590a21b --- /dev/null +++ b/indexing/upmendex/source/tests/locale2.ist @@ -0,0 +1 @@ +icu_locale "es@collation=traditional" diff --git a/indexing/upmendex/source/tests/pprec3.ist b/indexing/upmendex/source/tests/pprec3.ist new file mode 100644 index 0000000000..b8f33af1dd --- /dev/null +++ b/indexing/upmendex/source/tests/pprec3.ist @@ -0,0 +1,11 @@ +% -*- coding: utf-8 -*- + +page_precedence "naA" + +delim_0 "[[delim0]]" +delim_1 "[[delim1]]" +delim_2 "[[delim2]]" +delim_r "[[delimr]]" +suffix_2p "[[sfx 2p]]" +suffix_3p "[[sfx 3p]]" + diff --git a/indexing/upmendex/source/tests/pprec4.ist b/indexing/upmendex/source/tests/pprec4.ist new file mode 100644 index 0000000000..768fec7642 --- /dev/null +++ b/indexing/upmendex/source/tests/pprec4.ist @@ -0,0 +1,11 @@ +% -*- coding: utf-8 -*- + +page_precedence "Ana" + +delim_0 "[[delim0]]" +delim_1 "[[delim1]]" +delim_2 "[[delim2]]" +delim_r "[[delimr]]" +suffix_2p "[[sfx 2p]]" +suffix_3p "[[sfx 3p]]" + diff --git a/indexing/upmendex/source/tests/pprecB-0.ind b/indexing/upmendex/source/tests/pprecB-0.ind new file mode 100644 index 0000000000..1e23d9fb4a --- /dev/null +++ b/indexing/upmendex/source/tests/pprecB-0.ind @@ -0,0 +1,5 @@ +\begin{theindex} + + \item entryA[[delim0]]1[[sfx 3p]], a[[sfx 3p]], A[[sfx 3p]] + +\end{theindex} diff --git a/indexing/upmendex/source/tests/pprecB-3.ind b/indexing/upmendex/source/tests/pprecB-3.ind new file mode 100644 index 0000000000..1e23d9fb4a --- /dev/null +++ b/indexing/upmendex/source/tests/pprecB-3.ind @@ -0,0 +1,5 @@ +\begin{theindex} + + \item entryA[[delim0]]1[[sfx 3p]], a[[sfx 3p]], A[[sfx 3p]] + +\end{theindex} diff --git a/indexing/upmendex/source/tests/pprecB-4.ind b/indexing/upmendex/source/tests/pprecB-4.ind new file mode 100644 index 0000000000..3943ad4854 --- /dev/null +++ b/indexing/upmendex/source/tests/pprecB-4.ind @@ -0,0 +1,5 @@ +\begin{theindex} + + \item entryA[[delim0]]A[[sfx 3p]], 1[[sfx 3p]], a[[sfx 3p]] + +\end{theindex} diff --git a/indexing/upmendex/source/tests/pprecB.idx b/indexing/upmendex/source/tests/pprecB.idx new file mode 100644 index 0000000000..a1d8fc31ae --- /dev/null +++ b/indexing/upmendex/source/tests/pprecB.idx @@ -0,0 +1,9 @@ +\indexentry{entryA}{a} +\indexentry{entryA}{b} +\indexentry{entryA}{c} +\indexentry{entryA}{1} +\indexentry{entryA}{2} +\indexentry{entryA}{3} +\indexentry{entryA}{A} +\indexentry{entryA}{B} +\indexentry{entryA}{C} diff --git a/indexing/upmendex/source/tests/range.idx b/indexing/upmendex/source/tests/range.idx new file mode 100644 index 0000000000..3eccb113aa --- /dev/null +++ b/indexing/upmendex/source/tests/range.idx @@ -0,0 +1,126 @@ +\indexentry{entryA}{1} +\indexentry{entryA}{2} +\indexentry{entryA}{iii} +\indexentry{entryA}{iv} +\indexentry{entryA}{IX} +\indexentry{entryA}{X} + +\indexentry{entryB}{1} +\indexentry{entryB}{2} +\indexentry{entryB}{3} +\indexentry{entryB}{iii} +\indexentry{entryB}{iv} +\indexentry{entryB}{v} +\indexentry{entryB}{IX} +\indexentry{entryB}{X} +\indexentry{entryB}{XI} + +\indexentry{entryC}{1} +\indexentry{entryC}{2} +\indexentry{entryC}{3} +\indexentry{entryC}{4} +\indexentry{entryC|bold}{iii} +\indexentry{entryC|bold}{iv} +\indexentry{entryC|bold}{v} +\indexentry{entryC|bold}{vi} +\indexentry{entryC}{IX} +\indexentry{entryC|bold}{X} +\indexentry{entryC}{XI} +\indexentry{entryC}{XII} + +\indexentry{entryD}{c-1} +\indexentry{entryD}{c-2} +\indexentry{entryD}{i-iii} +\indexentry{entryD}{i-iv} +\indexentry{entryD}{d-IX} +\indexentry{entryD}{d-X} + +\indexentry{entryE}{c-1} +\indexentry{entryE}{c-2} +\indexentry{entryE}{c-3} +\indexentry{entryE}{i-iii} +\indexentry{entryE}{i-iv} +\indexentry{entryE}{i-v} +\indexentry{entryE}{d-IX} +\indexentry{entryE}{d-X} +\indexentry{entryE}{d-XI} + +\indexentry{entryF}{C-I} +\indexentry{entryF}{C-II} +\indexentry{entryF}{C-III} +\indexentry{entryF}{C-IV} +\indexentry{entryF}{V-LIII} +\indexentry{entryF}{V-LIV} +\indexentry{entryF}{V-LV} +\indexentry{entryF}{V-LVI} +\indexentry{entryF}{D-DI} +\indexentry{entryF}{D-DII} +\indexentry{entryF}{D-DIII} +\indexentry{entryF}{D-DIV} + +\indexentry{entryP}{A-a-1-VII-iii} +\indexentry{entryP}{A-a-2-VII-iii} +\indexentry{entryP}{A-a-3-VII-iii} +\indexentry{entryP}{A-a-1-VIII-iii} +\indexentry{entryP}{A-a-1-IX-iii} +\indexentry{entryP}{A-a-1-VII-iv} +\indexentry{entryP}{A-a-1-VII-v} + +\indexentry{entryQ}{1-2-3-4-5-6-7-8-9-10} +\indexentry{entryQ}{1-2-3-4-5-6-7-8-9-11} +\indexentry{entryQ}{1-2-3-4-5-6-7-8-9-14} +\indexentry{entryQ}{1-2-3-4-5-6-7-8-9-15} +\indexentry{entryQ}{1-2-3-4-5-6-7-8-9-16} +\indexentry{entryQ}{1-2-3-4-5-6-7-8-9-22} +\indexentry{entryQ}{1-2-3-4-5-6-7-8-9-23} +\indexentry{entryQ}{1-2-3-4-5-6-7-8-9-24} +\indexentry{entryQ}{1-2-3-4-5-6-7-8-9-25} +\indexentry{entryQ}{1-2-3-4-5-6-7-8-9-i} +\indexentry{entryQ}{1-2-3-4-5-6-7-8-9-ii} +\indexentry{entryQ}{1-2-3-4-5-6-7-8-9-iii} +\indexentry{entryQ}{1-2-3-4-5-6-7-8-9-iv} +\indexentry{entryQ}{1-2-3-4-5-6-7-8-9-v} + +\indexentry{entryA1|(}{1} +\indexentry{entryA1|)}{2} +\indexentry{entryA1|(}{iii} +\indexentry{entryA1|)}{iv} +\indexentry{entryA1|(}{IX} +\indexentry{entryA1|)}{X} + +\indexentry{entryB1|(}{1} +\indexentry{entryB1|)}{3} +\indexentry{entryB1|(}{iii} +\indexentry{entryB1|)}{v} +\indexentry{entryB1|(}{IX} +\indexentry{entryB1|)}{XI} + +\indexentry{entryC1|(}{1} +\indexentry{entryC1|)}{4} +\indexentry{entryC1|(bold}{iii} +\indexentry{entryC1|)}{vi} +\indexentry{entryC1|(}{IX} +\indexentry{entryC1|bold}{X} +\indexentry{entryC1|)}{XII} + +\indexentry{entryD1|(}{c-1} +\indexentry{entryD1|)}{c-2} +\indexentry{entryD1|(}{i-iii} +\indexentry{entryD1|)}{i-iv} +\indexentry{entryD1|(}{d-IX} +\indexentry{entryD1|)}{d-X} + +\indexentry{entryE1|(}{c-1} +\indexentry{entryE1|)}{c-3} +\indexentry{entryE1|(}{i-iii} +\indexentry{entryE1|)}{i-v} +\indexentry{entryE1|(}{d-IX} +\indexentry{entryE1|)}{d-XI} + +\indexentry{entryF1|(}{C-I} +\indexentry{entryF1|)}{C-IV} +\indexentry{entryF1|(}{V-LIII} +\indexentry{entryF1|)}{V-LVI} +\indexentry{entryF1|(}{D-DI} +\indexentry{entryF1|bold}{D-DII} +\indexentry{entryF1|)}{D-DIV} diff --git a/indexing/upmendex/source/tests/range1.ind b/indexing/upmendex/source/tests/range1.ind new file mode 100644 index 0000000000..c5f047a55e --- /dev/null +++ b/indexing/upmendex/source/tests/range1.ind @@ -0,0 +1,25 @@ +\begin{theindex} + + \item entryA, 1[[sfx 2p]], iii[[sfx 2p]], IX[[sfx 2p]] + \item entryA1, 1[[sfx 2p]], iii[[sfx 2p]], IX[[sfx 2p]] + \item entryB, 1[[sfx mp]], iii[[sfx mp]], IX[[sfx mp]] + \item entryB1, 1[[sfx mp]], iii[[sfx mp]], IX[[sfx mp]] + \item entryC, 1[[sfx mp]], \bold{iii[[sfx mp]]}, IX, \bold{X}, + XI[[sfx 2p]] + \item entryC1, 1[[sfx mp]], \bold{iii[[sfx mp]]}, \bold{X}, + IX[[sfx mp]] + \item entryD, c-1[[sfx 2p]], d-IX[[sfx 2p]], i-iii[[sfx 2p]] + \item entryD1, c-1[[sfx 2p]], d-IX[[sfx 2p]], i-iii[[sfx 2p]] + \item entryE, c-1[[sfx mp]], d-IX[[sfx mp]], i-iii[[sfx mp]] + \item entryE1, c-1[[sfx mp]], d-IX[[sfx mp]], i-iii[[sfx mp]] + \item entryF, C-I[[sfx mp]], D-DI[[sfx mp]], V-LIII[[sfx mp]] + \item entryF1, C-I[[sfx mp]], \bold{D-DII}, D-DI[[sfx mp]], + V-LIII[[sfx mp]] + \item entryP, A-a-1-VII-iii[[sfx mp]], A-a-1-VIII-iii, A-a-1-IX-iii, + A-a-2-VII-iii, A-a-3-VII-iii + \item entryQ, 1-2-3-4-5-6-7-8-9-10[[sfx 2p]], + 1-2-3-4-5-6-7-8-9-14[[sfx mp]], + 1-2-3-4-5-6-7-8-9-22[[sfx mp]], + 1-2-3-4-5-6-7-8-9-i[[sfx mp]] + +\end{theindex} diff --git a/indexing/upmendex/source/tests/range1.ist b/indexing/upmendex/source/tests/range1.ist new file mode 100644 index 0000000000..116c2d8713 --- /dev/null +++ b/indexing/upmendex/source/tests/range1.ist @@ -0,0 +1,3 @@ +page_precedence "nrRaA" +suffix_2p "[[sfx 2p]]" +suffix_mp "[[sfx mp]]" diff --git a/indexing/upmendex/source/tests/range2.ind b/indexing/upmendex/source/tests/range2.ind new file mode 100644 index 0000000000..789eb3f75c --- /dev/null +++ b/indexing/upmendex/source/tests/range2.ind @@ -0,0 +1,26 @@ +\begin{theindex} + + \item entryA, 1[[sfx 2p]], iii[[sfx 2p]], IX[[sfx 2p]] + \item entryA1, 1[[sfx 2p]], iii[[sfx 2p]], IX[[sfx 2p]] + \item entryB, 1[[sfx 3p]], iii[[sfx 3p]], IX[[sfx 3p]] + \item entryB1, 1[[sfx 3p]], iii[[sfx 3p]], IX[[sfx 3p]] + \item entryC, 1[[delim r]]4, \bold{iii[[delim r]]vi}, IX, \bold{X}, + XI[[sfx 2p]] + \item entryC1, 1[[delim r]]4, \bold{iii[[delim r]]vi}, \bold{X}, + IX[[delim r]]XII + \item entryD, c-1[[sfx 2p]], d-IX[[sfx 2p]], i-iii[[sfx 2p]] + \item entryD1, c-1[[sfx 2p]], d-IX[[sfx 2p]], i-iii[[sfx 2p]] + \item entryE, c-1[[sfx 3p]], d-IX[[sfx 3p]], i-iii[[sfx 3p]] + \item entryE1, c-1[[sfx 3p]], d-IX[[sfx 3p]], i-iii[[sfx 3p]] + \item entryF, C-I[[delim r]]C-IV, D-DI[[delim r]]D-DIV, + V-LIII[[delim r]]V-LVI + \item entryF1, C-I[[delim r]]C-IV, \bold{D-DII}, D-DI[[delim r]]D-DIV, + V-LIII[[delim r]]V-LVI + \item entryP, A-a-1-VII-iii[[sfx 3p]], A-a-1-VIII-iii, A-a-1-IX-iii, + A-a-2-VII-iii, A-a-3-VII-iii + \item entryQ, 1-2-3-4-5-6-7-8-9-10[[sfx 2p]], + 1-2-3-4-5-6-7-8-9-14[[sfx 3p]], + 1-2-3-4-5-6-7-8-9-22[[delim r]]1-2-3-4-5-6-7-8-9-25, + 1-2-3-4-5-6-7-8-9-i[[delim r]]1-2-3-4-5-6-7-8-9-v + +\end{theindex} diff --git a/indexing/upmendex/source/tests/range2.ist b/indexing/upmendex/source/tests/range2.ist new file mode 100644 index 0000000000..552e5fb674 --- /dev/null +++ b/indexing/upmendex/source/tests/range2.ist @@ -0,0 +1,4 @@ +page_precedence "nrRaA" +delim_r "[[delim r]]" +suffix_2p "[[sfx 2p]]" +suffix_3p "[[sfx 3p]]" diff --git a/indexing/upmendex/source/tests/range3.ind b/indexing/upmendex/source/tests/range3.ind new file mode 100644 index 0000000000..519730a138 --- /dev/null +++ b/indexing/upmendex/source/tests/range3.ind @@ -0,0 +1,25 @@ +\begin{theindex} + + \item entryA, 1[[sfx 2p]], iii[[sfx 2p]], IX[[sfx 2p]] + \item entryA1, 1[[sfx 2p]], iii[[sfx 2p]], IX[[sfx 2p]] + \item entryB, 1[[sfx 3p]], iii[[sfx 3p]], IX[[sfx 3p]] + \item entryB1, 1[[sfx 3p]], iii[[sfx 3p]], IX[[sfx 3p]] + \item entryC, 1[[sfx mp]], \bold{iii[[sfx mp]]}, IX, \bold{X}, + XI[[sfx 2p]] + \item entryC1, 1[[sfx mp]], \bold{iii[[sfx mp]]}, \bold{X}, + IX[[sfx mp]] + \item entryD, c-1[[sfx 2p]], d-IX[[sfx 2p]], i-iii[[sfx 2p]] + \item entryD1, c-1[[sfx 2p]], d-IX[[sfx 2p]], i-iii[[sfx 2p]] + \item entryE, c-1[[sfx 3p]], d-IX[[sfx 3p]], i-iii[[sfx 3p]] + \item entryE1, c-1[[sfx 3p]], d-IX[[sfx 3p]], i-iii[[sfx 3p]] + \item entryF, C-I[[sfx mp]], D-DI[[sfx mp]], V-LIII[[sfx mp]] + \item entryF1, C-I[[sfx mp]], \bold{D-DII}, D-DI[[sfx mp]], + V-LIII[[sfx mp]] + \item entryP, A-a-1-VII-iii[[sfx 3p]], A-a-1-VIII-iii, A-a-1-IX-iii, + A-a-2-VII-iii, A-a-3-VII-iii + \item entryQ, 1-2-3-4-5-6-7-8-9-10[[sfx 2p]], + 1-2-3-4-5-6-7-8-9-14[[sfx 3p]], + 1-2-3-4-5-6-7-8-9-22[[sfx mp]], + 1-2-3-4-5-6-7-8-9-i[[sfx mp]] + +\end{theindex} diff --git a/indexing/upmendex/source/tests/range3.ist b/indexing/upmendex/source/tests/range3.ist new file mode 100644 index 0000000000..b751471b43 --- /dev/null +++ b/indexing/upmendex/source/tests/range3.ist @@ -0,0 +1,4 @@ +page_precedence "nrRaA" +suffix_2p "[[sfx 2p]]" +suffix_3p "[[sfx 3p]]" +suffix_mp "[[sfx mp]]" diff --git a/indexing/upmendex/source/tests/rangetwo.idx b/indexing/upmendex/source/tests/rangetwo.idx deleted file mode 100644 index d941f7e831..0000000000 --- a/indexing/upmendex/source/tests/rangetwo.idx +++ /dev/null @@ -1,2 +0,0 @@ -\indexentry{entryA}{1} -\indexentry{entryA}{2} diff --git a/indexing/upmendex/source/tests/rangetwo.ind b/indexing/upmendex/source/tests/rangetwo.ind deleted file mode 100644 index 1119d2fa8e..0000000000 --- a/indexing/upmendex/source/tests/rangetwo.ind +++ /dev/null @@ -1,5 +0,0 @@ -\begin{theindex} - - \item entryA, 1[[sfx2p]] - -\end{theindex} diff --git a/indexing/upmendex/source/tests/rangetwo.ist b/indexing/upmendex/source/tests/rangetwo.ist deleted file mode 100644 index 3e986c8935..0000000000 --- a/indexing/upmendex/source/tests/rangetwo.ist +++ /dev/null @@ -1,2 +0,0 @@ -suffix_2p "[[sfx2p]]" - diff --git a/indexing/upmendex/source/tests/rule.idx b/indexing/upmendex/source/tests/rule.idx new file mode 100644 index 0000000000..783cce9fd4 --- /dev/null +++ b/indexing/upmendex/source/tests/rule.idx @@ -0,0 +1,20 @@ +% -*- coding: utf-8 -*- +\indexentry{A}{400} +\indexentry{a}{401} +\indexentry{B}{402} +\indexentry{b}{403} +\indexentry{C}{404} +\indexentry{c}{405} +\indexentry{Ç}{406} +\indexentry{ç}{407} +\indexentry{Ch}{420} +\indexentry{cha}{421} +\indexentry{cz}{422} +\indexentry{U}{408} +\indexentry{u}{409} +\indexentry{Ü}{410} +\indexentry{ü}{411} +\indexentry{10}{1000} +\indexentry{2}{1001} +\indexentry{#}{2000} +\indexentry{$}{2001} diff --git a/indexing/upmendex/source/tests/rule.ind b/indexing/upmendex/source/tests/rule.ind new file mode 100644 index 0000000000..3ac1335e05 --- /dev/null +++ b/indexing/upmendex/source/tests/rule.ind @@ -0,0 +1,40 @@ +\begin{theindex} + +\centerline{\bfseries ◆◆◆ Symbols ◆◆◆}\par\nobreak + \item #\leaders\hbox{$\cdot$}\hfill 2000 + \item $\leaders\hbox{$\cdot$}\hfill 2001 + \item 10\leaders\hbox{$\cdot$}\hfill 1000 + \item 2\leaders\hbox{$\cdot$}\hfill 1001 + + \indexspace + +\centerline{\bfseries ◆◆◆ A ◆◆◆}\par\nobreak + \item a\leaders\hbox{$\cdot$}\hfill 401 + \item A\leaders\hbox{$\cdot$}\hfill 400 + + \indexspace + +\centerline{\bfseries ◆◆◆ C ◆◆◆}\par\nobreak + \item C\leaders\hbox{$\cdot$}\hfill 404 + \item Ç\leaders\hbox{$\cdot$}\hfill 406 + \item c\leaders\hbox{$\cdot$}\hfill 405 + \item ç\leaders\hbox{$\cdot$}\hfill 407 + \item Ch\leaders\hbox{$\cdot$}\hfill 420 + \item cha\leaders\hbox{$\cdot$}\hfill 421 + \item cz\leaders\hbox{$\cdot$}\hfill 422 + + \indexspace + +\centerline{\bfseries ◆◆◆ U ◆◆◆}\par\nobreak + \item u\leaders\hbox{$\cdot$}\hfill 409 + \item U\leaders\hbox{$\cdot$}\hfill 408 + \item ü\leaders\hbox{$\cdot$}\hfill 411 + \item Ü\leaders\hbox{$\cdot$}\hfill 410 + + \indexspace + +\centerline{\bfseries ◆◆◆ B ◆◆◆}\par\nobreak + \item b\leaders\hbox{$\cdot$}\hfill 403 + \item B\leaders\hbox{$\cdot$}\hfill 402 + +\end{theindex} diff --git a/indexing/upmendex/source/tests/rule1.ist b/indexing/upmendex/source/tests/rule1.ist new file mode 100644 index 0000000000..c986e39e96 --- /dev/null +++ b/indexing/upmendex/source/tests/rule1.ist @@ -0,0 +1,3 @@ +% -*- coding: utf-8 -*- + +icu_rules "&a<<<A<C<<Ç<<c<<ç<u<<U<<ü<<Ü<b<<B" diff --git a/indexing/upmendex/source/tests/rule2.ist b/indexing/upmendex/source/tests/rule2.ist new file mode 100644 index 0000000000..a61af0a546 --- /dev/null +++ b/indexing/upmendex/source/tests/rule2.ist @@ -0,0 +1,3 @@ +% -*- coding: utf-8 -*- + +icu_rules "&a<<<A<C<<\U000000c7<<c<<\u00E7<u<<U<<\x{fc}<<\xDC<b<<B" diff --git a/indexing/upmendex/source/tests/rule3.ist b/indexing/upmendex/source/tests/rule3.ist new file mode 100644 index 0000000000..7ab835a801 --- /dev/null +++ b/indexing/upmendex/source/tests/rule3.ist @@ -0,0 +1,4 @@ +% -*- coding: latin-1 -*- +% To test malformed input + +icu_rules "&a<<<A<C<<<<c<<<u<<U<<<<<b<<B" diff --git a/indexing/upmendex/source/tests/rule4.ist b/indexing/upmendex/source/tests/rule4.ist new file mode 100644 index 0000000000..0c30dceba2 --- /dev/null +++ b/indexing/upmendex/source/tests/rule4.ist @@ -0,0 +1,4 @@ +% -*- coding: utf-8 -*- +% To test malformed input + +icu_rules "&a<<<A<C<<\U000000c7<<c<<\uE7<u<<U<<\x{fc}<<\xDC<b<<B" diff --git a/indexing/upmendex/source/tests/upmendex.test b/indexing/upmendex/source/tests/upmendex.test index b64c045af9..84dfbd17be 100755 --- a/indexing/upmendex/source/tests/upmendex.test +++ b/indexing/upmendex/source/tests/upmendex.test @@ -1,5 +1,5 @@ #! /bin/sh -vx -# $Id$ +# $Id: upmendex.test 60549 2021-09-19 08:57:18Z takuji $ # Copyright 2017 Karl Berry <tex-live@tug.org> # Copyright 2013 Peter Breitenlohner <tex-live@tug.org> # You may freely use, modify and/or distribute this file. @@ -7,7 +7,7 @@ TEXMFCNF=$srcdir/../kpathsea export TEXMFCNF -rm -f foo.* uni.* pprec*.* rangetwo.* +rm -f foo.* uni.* range*.* pprec*.* locale*.* attr*.* rule*.* ./upmendex $srcdir/tests/foo.idx -o foo.ind1 -t foo.ilg1 \ && diff $srcdir/tests/foo.ind foo.ind1 || exit 1 @@ -28,10 +28,18 @@ cat $srcdir/tests/uni.idx | \ $srcdir/tests/uni.idx -o uni.ind3 -t uni.ilg3 \ && diff $srcdir/tests/uni.ind uni.ind3 || exit 1 -# test for two-element range suffix_2p -./upmendex -s $srcdir/tests/rangetwo.ist $srcdir/tests/rangetwo.idx \ - -o rangetwo.ind1 -t rangetwo.ilg \ - && diff $srcdir/tests/rangetwo.ind rangetwo.ind1 || exit 1 +# test for range suffix_2p, suffix_3p, suffix_mp +./upmendex -s $srcdir/tests/range1.ist $srcdir/tests/range.idx \ + -o range1.ind1 -t range1.ilg \ + && diff $srcdir/tests/range1.ind range1.ind1 || exit 1 + +./upmendex -s $srcdir/tests/range2.ist $srcdir/tests/range.idx \ + -o range2.ind1 -t range2.ilg \ + && diff $srcdir/tests/range2.ind range2.ind1 || exit 1 + +./upmendex -s $srcdir/tests/range3.ist $srcdir/tests/range.idx \ + -o range3.ind1 -t range3.ilg \ + && diff $srcdir/tests/range3.ind range3.ind1 || exit 1 # test for page_precedence and suffix_3p ./upmendex -s $srcdir/tests/pprec0.ist $srcdir/tests/pprecA.idx \ @@ -46,3 +54,57 @@ cat $srcdir/tests/uni.idx | \ -o pprecA-2.ind1 -t pprecA-2.ilg \ && diff $srcdir/tests/pprecA-2.ind pprecA-2.ind1 || exit 1 +./upmendex -s $srcdir/tests/pprec0.ist $srcdir/tests/pprecB.idx \ + -o pprecB-0.ind1 -t pprecB-0.ilg \ + && diff $srcdir/tests/pprecB-0.ind pprecB-0.ind1 || exit 1 + +./upmendex -s $srcdir/tests/pprec3.ist $srcdir/tests/pprecB.idx \ + -o pprecB-3.ind1 -t pprecB-3.ilg \ + && diff $srcdir/tests/pprecB-3.ind pprecB-3.ind1 || exit 1 + +./upmendex -s $srcdir/tests/pprec4.ist $srcdir/tests/pprecB.idx \ + -o pprecB-4.ind1 -t pprecB-4.ilg \ + && diff $srcdir/tests/pprecB-4.ind pprecB-4.ind1 || exit 1 + +# test for icu_locale +./upmendex -s $srcdir/tests/uni.ist -s $srcdir/tests/locale1.ist $srcdir/tests/rule.idx \ + -o locale1.ind1 -t locale1.ilg \ + && diff $srcdir/tests/locale1.ind locale1.ind1 || exit 1 + +# test for icu_attributes +./upmendex -s $srcdir/tests/uni.ist -s $srcdir/tests/attr1.ist $srcdir/tests/rule.idx \ + -o attr1.ind1 -t attr1.ilg \ + && diff $srcdir/tests/attr1.ind attr1.ind1 || exit 1 + +# test for icu_rules +./upmendex -s $srcdir/tests/uni.ist -s $srcdir/tests/rule1.ist $srcdir/tests/rule.idx \ + -o rule1.ind1 -t rule1.ilg \ + && diff $srcdir/tests/rule.ind rule1.ind1 || exit 1 + +./upmendex -s $srcdir/tests/uni.ist -s $srcdir/tests/rule2.ist $srcdir/tests/rule.idx \ + -o rule2.ind1 -t rule2.ilg \ + && diff $srcdir/tests/rule.ind rule2.ind1 || exit 1 + +# malformed input +./upmendex -s $srcdir/tests/uni.ist -s $srcdir/tests/rule3.ist $srcdir/tests/rule.idx \ + -o rule3.ind1 -t rule3.ilg +if [ $? -eq 254 ]; then + echo "Error check: OK" +else + echo "Error check: NG" + exit 1 +fi + +./upmendex -s $srcdir/tests/uni.ist -s $srcdir/tests/rule4.ist $srcdir/tests/rule.idx \ + -o rule4.ind1 -t rule4.ilg +if [ $? -eq 254 ]; then + echo "Error check: OK" +else + echo "Error check: NG" + exit 1 +fi + +# test for icu_locale + icu_rules +./upmendex -s $srcdir/tests/uni.ist -s $srcdir/tests/locale2.ist -s $srcdir/tests/rule1.ist $srcdir/tests/rule.idx \ + -o locale2-rule1.ind1 -t locale2-rule1.ilg \ + && diff $srcdir/tests/locale2-rule.ind locale2-rule1.ind1 || exit 1 diff --git a/indexing/upmendex/source/var.h b/indexing/upmendex/source/var.h index 02e2714910..c9275fcc91 100644 --- a/indexing/upmendex/source/var.h +++ b/indexing/upmendex/source/var.h @@ -4,7 +4,7 @@ FILE *efp; int lines,idxcount,acc=0,reject=0; int prange=1,fsti=0,lorder=0,bcomp=0,force=0,fpage=0,gflg=0,verb=1,debug=0; -int warn=0,scount=0,pattr[3]={0,0,0}; +int warn=0,scount=0,pattr[PAGE_COMPOSIT_DEPTH]={0}; /* static char roman[]={"ivxlcdm"},Roman[]={"IVXLCDM"}; */ struct index *ind; @@ -27,7 +27,7 @@ int line_max=72; char indent_space[STYBUFSIZE]={"\t\t"}; int indent_length=16; int priority=0; -char symbol[STYBUFSIZE]={""}; +char symhead[STYBUFSIZE]={""},numhead[STYBUFSIZE]={""}; char symhead_positive[STYBUFSIZE]={"Symbols"},symhead_negative[STYBUFSIZE]={"symbols"}; char numhead_positive[STYBUFSIZE]={"Numbers"},numhead_negative[STYBUFSIZE]={"numbers"}; int symbol_flag=1; @@ -35,7 +35,8 @@ int letter_head=1; UChar atama[STYBUFSIZE],hangul_head[STYBUFSIZE],hanzi_head[STYBUFSIZE]={L'\0'},kana_head[STYBUFSIZE]={L'\0'}; UChar devanagari_head[STYBUFSIZE],thai_head[STYBUFSIZE]; char page_compositor[STYBUFSIZE]={"-"},page_precedence[STYBUFSIZE]={"rnaRA"}; -char character_order[STYBUFSIZE]={"SNLGCJKHDT"}; +char character_order[STYBUFSIZE]={"SNLGCJKHDTah"}; +char script_preamble[11][STYBUFSIZE],script_postamble[11][STYBUFSIZE]; char icu_locale[STYBUFSIZE]={"root"},icu_rules[STYBUFSIZE]={""}; int icu_attributes[UCOL_ATTRIBUTE_COUNT]; |