summaryrefslogtreecommitdiff
path: root/indexing
diff options
context:
space:
mode:
authorNorbert Preining <norbert@preining.info>2021-11-22 03:03:38 +0000
committerNorbert Preining <norbert@preining.info>2021-11-22 03:03:38 +0000
commit3b45580e298046d6c1cb0431da31bea04bc6613a (patch)
treeee2b508b3e9ce370c5134f941650c4316ce9ec8c /indexing
parent25fba613796cd9c1b2042e28152d451d8cddef4a (diff)
CTAN sync 202111220303
Diffstat (limited to 'indexing')
-rw-r--r--indexing/upmendex/README.md24
-rw-r--r--indexing/upmendex/doc/samples/Makefile4
-rw-r--r--indexing/upmendex/doc/samples/alphabet/alpha_es.idx2
-rw-r--r--indexing/upmendex/doc/samples/mdxsty01.ist2
-rw-r--r--indexing/upmendex/doc/samples/mdxsty02.ist2
-rw-r--r--indexing/upmendex/doc/samples/mdxsty03.ist33
-rw-r--r--indexing/upmendex/doc/samples/multi.idx11
-rw-r--r--indexing/upmendex/doc/samples/option/mdxsty_j02.ist6
-rw-r--r--indexing/upmendex/doc/upmendex.ja.txt64
-rw-r--r--indexing/upmendex/man/upmendex.199
-rw-r--r--indexing/upmendex/man/upmendex.man1.pdfbin252502 -> 253880 bytes
-rw-r--r--indexing/upmendex/source/ChangeLog107
-rw-r--r--indexing/upmendex/source/Makefile.am3
-rw-r--r--indexing/upmendex/source/Makefile.in3
-rw-r--r--indexing/upmendex/source/aclocal.m410
-rwxr-xr-xindexing/upmendex/source/configure22
-rw-r--r--indexing/upmendex/source/configure.ac6
-rw-r--r--indexing/upmendex/source/convert.c93
-rw-r--r--indexing/upmendex/source/exkana.h12
-rw-r--r--indexing/upmendex/source/exvar.h5
-rw-r--r--indexing/upmendex/source/fread.c81
-rw-r--r--indexing/upmendex/source/fwrite.c199
-rw-r--r--indexing/upmendex/source/kana.h50
-rw-r--r--indexing/upmendex/source/ktable.h98
-rw-r--r--indexing/upmendex/source/mendex.h13
-rw-r--r--indexing/upmendex/source/sort.c356
-rw-r--r--indexing/upmendex/source/styfile.c142
-rw-r--r--indexing/upmendex/source/tests/attr1.ind40
-rw-r--r--indexing/upmendex/source/tests/attr1.ist2
-rw-r--r--indexing/upmendex/source/tests/locale1.ind48
-rw-r--r--indexing/upmendex/source/tests/locale1.ist1
-rw-r--r--indexing/upmendex/source/tests/locale2-rule.ind44
-rw-r--r--indexing/upmendex/source/tests/locale2.ist1
-rw-r--r--indexing/upmendex/source/tests/pprec3.ist11
-rw-r--r--indexing/upmendex/source/tests/pprec4.ist11
-rw-r--r--indexing/upmendex/source/tests/pprecB-0.ind5
-rw-r--r--indexing/upmendex/source/tests/pprecB-3.ind5
-rw-r--r--indexing/upmendex/source/tests/pprecB-4.ind5
-rw-r--r--indexing/upmendex/source/tests/pprecB.idx9
-rw-r--r--indexing/upmendex/source/tests/range.idx126
-rw-r--r--indexing/upmendex/source/tests/range1.ind25
-rw-r--r--indexing/upmendex/source/tests/range1.ist3
-rw-r--r--indexing/upmendex/source/tests/range2.ind26
-rw-r--r--indexing/upmendex/source/tests/range2.ist4
-rw-r--r--indexing/upmendex/source/tests/range3.ind25
-rw-r--r--indexing/upmendex/source/tests/range3.ist4
-rw-r--r--indexing/upmendex/source/tests/rangetwo.idx2
-rw-r--r--indexing/upmendex/source/tests/rangetwo.ind5
-rw-r--r--indexing/upmendex/source/tests/rangetwo.ist2
-rw-r--r--indexing/upmendex/source/tests/rule.idx20
-rw-r--r--indexing/upmendex/source/tests/rule.ind40
-rw-r--r--indexing/upmendex/source/tests/rule1.ist3
-rw-r--r--indexing/upmendex/source/tests/rule2.ist3
-rw-r--r--indexing/upmendex/source/tests/rule3.ist4
-rw-r--r--indexing/upmendex/source/tests/rule4.ist4
-rwxr-xr-xindexing/upmendex/source/tests/upmendex.test74
-rw-r--r--indexing/upmendex/source/var.h7
57 files changed, 1484 insertions, 522 deletions
diff --git a/indexing/upmendex/README.md b/indexing/upmendex/README.md
index 5aec451f1b..89d0c0e4de 100644
--- a/indexing/upmendex/README.md
+++ b/indexing/upmendex/README.md
@@ -1,7 +1,7 @@
upmendex --- Multilingual index processor
==========================================
-2021.08.08 Ver0.58
+2021.11.20 Ver1.00
TANAKA, Takuji
<ttk(at)t-lab.opal.ne.jp>
@@ -19,7 +19,7 @@ upmendex is a multilingual index processor with following features:
* Support Latin (including non-English), Greek, Cyrillic,
Korean Hangul and Chinese Han (Hanzi ideographs) scripts
as well as Japanese Kana.
- * Support Devanagari and Thai scripts (experimental).
+ * Support Devanagari, Thai, Arabic and Hebrew scripts (experimental).
* Support four kinds of sort orders (Pinyin, Radical-Stroke,
Stroke and Zhuyin) for Chinese Han scripts (Hanzi ideographs).
* Apply International Components for Unicode (ICU)[4]
@@ -36,13 +36,13 @@ upmendex is a multilingual index processor with following features:
* man/ :: manuals
### Building upmendex
-The source files are distributed at GitHub[2] and work with TeX Live svn r60191.
+The source files are distributed at GitHub[2] and work with TeX Live svn r61096.
Ref. [TeX Live and Subversion](http://www.tug.org/texlive/svn/)
Tested with ICU 68.2 .
### Status
-Beta version.
+Stable version.
No warranty.
### Copyright
@@ -51,13 +51,13 @@ It is as same as [the BSD 3-Clause License](https://opensource.org/licenses/BSD-
### Supported locales
In ICU collator, default "root" locale covers several lauguages:
-English, French, Germany, Italian, Portuguese and so on.
+English, French, German, Italian, Portuguese and so on.
Much of languages and collation options are supported by
setting into "icu_locale" in style files.
Currently following locales are available:
#### Latin script
- "az", "az@collation=search" (Azerbaijani), "ca", "ca@collation=search" (Catalan),
+ "az", "az@collation=search" (Azerbaijani), "ca@collation=search" (Catalan),
"cs", "cs@collation=search" (Czech), "da", "da@collation=search" (Danish),
"de@collation=phonebook", "de@collation=search", "de-AT@collation=phonebook" (German),
"eo" (Espelanto), "es", "es@collation=traditional", "es@collation=search" (Spanish),
@@ -70,6 +70,10 @@ Currently following locales are available:
"sv", "sv@collation=search", "sv@collation=standard" (Swedish),
"tr" (Turkish), "vi", "vi@collation=traditional" (Vietnamese)
+##### Supported by default "root" collator. Do not need to set locale
+ "ca" (Catalan), "de" (German), "en" (English), "fr" (French), "id" (Indonesian),
+ "it" (Italian), "jv" (Javanese), "ms" (Malay), "nl" (Dutch), "pt" (Portuguese)
+
#### Cyrillic script
"be" (Belarusian), "bg" (Bulgarian), "kk" (Kazakh), "ky" (Kyrgyz),
"mk" (Macedonian), "ru" (Russian), "sr" (Serbian), "uk" (Ukraine)
@@ -89,11 +93,17 @@ Currently following locales are available:
#### Thai script (experimental)
"th" (Thai)
+#### Arabic (experimental)
+ "ar", "ar@collation=compat" (Arabic), "fa" (Persian), "ps" (Pashto), "ur" (Urdu)
+
+#### Hebrew (experimental)
+ "he", "he@collation=search" (Hebrew), "yi" (Yiddish)
+
### References
1. [ASCII Nihongo TeX (Publishing TeX)](https://asciidwango.github.io/ptex/)
ASCII MEDIA WORKS (web site by DWANGO Co., Ltd.)
2. [Source/Document distribution of upmendex --- multilingual index processor @ GitHub](https://github.com/t-tk/upmendex-package)
3. [upTeX, upLaTeX ― unicode version of pTeX, pLaTeX](http://www.t-lab.opal.ne.jp/tex/uptex_en.html)
-4. [International Components for Unicode (ICU)](http://site.icu-project.org/)
+4. [International Components for Unicode (ICU)](https://icu.unicode.org/)
5. [upmendex @ CTAN](https://ctan.org/pkg/upmendex/)
diff --git a/indexing/upmendex/doc/samples/Makefile b/indexing/upmendex/doc/samples/Makefile
index 533ddaf743..ce33305212 100644
--- a/indexing/upmendex/doc/samples/Makefile
+++ b/indexing/upmendex/doc/samples/Makefile
@@ -8,8 +8,8 @@ scripts:
done
styles:
- for num in 00 01 02; do \
- $(UPMENDEX) multi.idx -s mdxsty$$num.ist -o multi_$$num.ind ; \
+ for num in 00 01 02 03; do \
+ $(UPMENDEX) multi.idx -s mdxsty$$num.ist -t multi_$$num.ilg -o multi_$$num.ind ; \
done
compare:
diff --git a/indexing/upmendex/doc/samples/alphabet/alpha_es.idx b/indexing/upmendex/doc/samples/alphabet/alpha_es.idx
index 7c3982df83..1a81aa998b 100644
--- a/indexing/upmendex/doc/samples/alphabet/alpha_es.idx
+++ b/indexing/upmendex/doc/samples/alphabet/alpha_es.idx
@@ -93,3 +93,5 @@
\indexentry{qu}{6004}
\indexentry{qua}{6005}
\indexentry{quz}{6006}
+\indexentry{ª}{501}
+\indexentry{º}{502}
diff --git a/indexing/upmendex/doc/samples/mdxsty01.ist b/indexing/upmendex/doc/samples/mdxsty01.ist
index 56e6a89f0c..24f4f5ff3b 100644
--- a/indexing/upmendex/doc/samples/mdxsty01.ist
+++ b/indexing/upmendex/doc/samples/mdxsty01.ist
@@ -8,5 +8,5 @@ lethead_flag 1
lethead_prefix "\n\\centerline{\\bfseries --- "
lethead_suffix " ---}\\par\\nobreak"
-character_order "NSTKDJHGCL"
+character_order "NSTKDJHGCLah"
diff --git a/indexing/upmendex/doc/samples/mdxsty02.ist b/indexing/upmendex/doc/samples/mdxsty02.ist
index 03353d85eb..f0536dd863 100644
--- a/indexing/upmendex/doc/samples/mdxsty02.ist
+++ b/indexing/upmendex/doc/samples/mdxsty02.ist
@@ -8,7 +8,7 @@ lethead_flag -1
lethead_prefix "\n\\centerline{\\bfseries --- "
lethead_suffix " ---}\\par\\nobreak"
-character_order "JKHGCLTDSN"
+character_order "JKHGCLTDhaSN"
hangul_head "가나다라마바사아카"
%% hangul_head "㉠㉡㉢㉣㉤㉥㉦㉧㉨㉩㉪㉫㉬㉭"
diff --git a/indexing/upmendex/doc/samples/mdxsty03.ist b/indexing/upmendex/doc/samples/mdxsty03.ist
new file mode 100644
index 0000000000..8b57069919
--- /dev/null
+++ b/indexing/upmendex/doc/samples/mdxsty03.ist
@@ -0,0 +1,33 @@
+% -*- coding: utf-8 -*-
+
+delim_0 "\\leaders\\hbox{$\\cdot$}\\hfill "
+delim_1 "\\leaders\\hbox{$\\cdot$}\\hfill "
+delim_2 "\\leaders\\hbox{$\\cdot$}\\hfill "
+
+headings_flag 1
+heading_prefix "\n\\centerline{\\bfseries --- "
+heading_suffix " ---}\\par\\nobreak"
+
+character_order "SahHJKDTLCGN"
+symbol_flag 2
+
+script_preamble latin "\n% begin latin"
+script_postamble latin "\n% end latin"
+script_preamble cyrillic "\n% begin cyrillic"
+script_postamble cyrillic "\n% end cyrillic"
+script_preamble greek "\n% begin greek"
+script_postamble greek "\n% end greek"
+script_preamble kana "\n% begin kana"
+script_postamble kana "\n% end kana"
+script_preamble hangul "\n% begin hangul"
+script_postamble hangul "\n% end hangul"
+script_preamble hanzi "\n% begin hanzi"
+script_postamble hanzi "\n% end hanzi"
+script_preamble devanagari "\n% begin devanagari"
+script_postamble devanagari "\n% end devanagari"
+script_preamble thai "\n% begin thai"
+script_postamble thai "\n% end thai"
+script_preamble arabic "\n% begin arabic"
+script_postamble arabic "\n% end arabic"
+script_preamble hebrew "\n% begin hebrew"
+script_postamble hebrew "\n% end hebrew"
diff --git a/indexing/upmendex/doc/samples/multi.idx b/indexing/upmendex/doc/samples/multi.idx
index 8241647256..5567532210 100644
--- a/indexing/upmendex/doc/samples/multi.idx
+++ b/indexing/upmendex/doc/samples/multi.idx
@@ -5,7 +5,10 @@
\indexentry{もりおうがい@森鷗外}{5}
\indexentry{세종대왕@세종대왕(世宗大王)}{6}
\indexentry{王羲之}{7}
-\indexentry{0123456789}{8}
-\indexentry{¥¥¥¥¥}{9}
-\indexentry{พ่อขุนรามคำแหงมหาราช}{10}
-\indexentry{गौतम सिद्धार्थ}{11}
+\indexentry{0120¹₂③❹➄➏🄈⑻⒐㉈🄋🄌}{8}
+\indexentry{€$$¢¢££¥¥₩₩฿₹}{9}
+\indexentry{☃☎♥⚽☺☝✈🀅🍣👻}{10}
+\indexentry{พ่อขุนรามคำแหงมหาราช}{11}
+\indexentry{गौतम सिद्धार्थ}{12}
+\indexentry{محمد‎}{13}
+\indexentry{אֱלִיעֶזֶר בֶּן־יְהוּדָה‎}{14}
diff --git a/indexing/upmendex/doc/samples/option/mdxsty_j02.ist b/indexing/upmendex/doc/samples/option/mdxsty_j02.ist
index 3b084ba9d8..b5cac2a050 100644
--- a/indexing/upmendex/doc/samples/option/mdxsty_j02.ist
+++ b/indexing/upmendex/doc/samples/option/mdxsty_j02.ist
@@ -4,9 +4,9 @@ delim_0 "\\leaders\\hbox{$\\cdot$}\\hfill "
delim_1 "\\leaders\\hbox{$\\cdot$}\\hfill "
delim_2 "\\leaders\\hbox{$\\cdot$}\\hfill "
-lethead_flag 1
-lethead_prefix "\n\\centerline{\\bfseries --- "
-lethead_suffix " ---}\\par\\nobreak"
+headings_flag 1
+heading_prefix "\n\\centerline{\\bfseries --- "
+heading_suffix " ---}\\par\\nobreak"
priority 1
letter_head 2
diff --git a/indexing/upmendex/doc/upmendex.ja.txt b/indexing/upmendex/doc/upmendex.ja.txt
index 072a5fe873..22b2ab6230 100644
--- a/indexing/upmendex/doc/upmendex.ja.txt
+++ b/indexing/upmendex/doc/upmendex.ja.txt
@@ -123,7 +123,7 @@ upmendex - 索引整形ツール
preamble <文字列>
"\\begin{theindex}\n"
- 出力ファイルの文字列。
+ 出力ファイルの先頭の文字列。
postamble <文字列>
"\n\n\\end{theindex}\n"
@@ -167,6 +167,10 @@ upmendex - 索引整形ツール
0
lethead_flagと同じ。
+ headings_flag <数値>
+ 0
+ lethead_flagと同じ。
+
kana_head <文字列>
""
仮名見出し文字。見出し文字を文字列で指定する。
@@ -283,23 +287,36 @@ upmendex - 索引整形ツール
symhead_positive <文字列>
"Symbols"
- lethead_flag または heading_flag が正数の場合に数字・記号の頭文字として
+ lethead_flag または heading_flag または headings_flag が正数の場合に記号の頭文字として
出力する文字列。
symhead_negative <文字列>
"symbols"
- lethead_flag または heading_flag が負数の場合に数字・記号の頭文字として
+ lethead_flag または heading_flag または headings_flag が負数の場合に記号の頭文字として
出力する文字列。
symbol <文字列>
""
- symbol_flag が0でない場合に、数字・記号の頭文字として出力する文字列。
+ symbol_flag が0でない場合に、記号の頭文字として出力する文字列。
文字列が定義されていれば、symhead_positive および symhead_negative より
優先される。((up)mendex専用)
+ numhead_positive <文字列>
+ "Numbers"
+ lethead_flag または heading_flag が正数かつ symbol_flag が2の場合に数字の頭文字として
+ 出力する文字列。
+
+ numhead_negative <文字列>
+ "numbers"
+ lethead_flag または heading_flag が負数かつ symbol_flag が2の場合に数字の頭文字として
+ 出力する文字列。
+
symbol_flag <数値>
1
- symbol の出力フラグ。0のとき出力しない。((up)mendex専用)
+ symbol の出力フラグ。0のとき見出しを出力しない。
+ 1のとき数字を記号の一部として扱い記号の見出しを出力する。
+ 2のとき数字と記号を別の集合に分類し数字と記号の見出しを出力する。
+ ((up)mendex専用)
letter_head <数値>
1
@@ -313,14 +330,31 @@ upmendex - 索引整形ツール
((up)mendex専用)
character_order <文字列>
- "SNLGCJKHDT"
+ "SNLGCJKHDTah"
記号、英字、日本語の優先順位。'S'は記号、'N'は数字、'L'はラテン文字、
'G'はギリシャ文字、'C'はキリル文字、'J'は日本語(かな)、'K'はハングル、
- 'H'は漢字、'D'はデーヴァナーガリー、'T'はタイ文字を表す。
- upendexでは索引項目の分類として「数字」は「記号」に含める仕様なので、
+ 'H'は漢字、'D'はデーヴァナーガリー、'T'はタイ文字、
+ 'a'はアラビア文字、'h'はヘブライ文字を表す。
+ symbol_flag=1のとき、索引項目の分類として「数字」は「記号」に含めるので、
'S'と'N'は必ず隣り合わせること(数字と数字以外の記号の順序入れ替えは可能)。
(upmendex専用)
+ script_preamble <文字列1> <文字列2>
+ ""
+ 各スクリプトごとのブロックの先頭の文字列を文字列2に指定する。
+ スクリプト名1個を以下の中から文字列1に指定しなければならない:
+ 'latin', 'cyrillic', 'greek', 'kana', 'hangul', 'hanzi',
+ 'devanagari', 'thai', 'arabic', 'hebrew'
+ (upmendex専用)
+
+ script_postamble <文字列1> <文字列2>
+ ""
+ 各スクリプトごとのブロックの末尾の文字列を文字列2に指定する。
+ スクリプト名1個を以下の中から文字列1に指定しなければならない:
+ 'latin', 'cyrillic', 'greek', 'kana', 'hangul', 'hanzi',
+ 'devanagari', 'thai', 'arabic', 'hebrew'
+ (upmendex専用)
+
icu_locale <文字列>
""
ICU collatorにおいて従うlocale。
@@ -331,20 +365,20 @@ upmendex - 索引整形ツール
""
ICU collatorにおいてlocaleによらずに照合順序を指定する場合、
照合順序のルールを示す文字列。
- ( Ref. http://userguide.icu-project.org/collation/customization
- https://unicode-org.github.io/icu/userguide/collation/customization/
+ ( Ref. https://unicode-org.github.io/icu/userguide/collation/customization/
http://www.unicode.org/reports/tr35/tr35-collation.html#Rules )
UTF-8のUnicode文字及び、以下のエスケープ文字列が使用可能である:
\Uhhhhhhhh (16進数[0-9A-Fa-f]を8桁), \uhhhh (16進数を4桁),
\xhh (16進数を2桁), \x{h...} (16進数を1〜8桁), \ooo (8進数[0-7]を3桁)。
+ icu_localeとicu_rulesを同時に指定した場合、icu_localeで指定したルールの上に
+ icu_rulesで指定したルールを追加する。
空文字列(デフォルト)のときは、localeによる照合順序のルールに従う。
(upmendex専用)
icu_attributes <文字列>
""
ICU collatorのattribute指定。
- ( Ref. http://userguide.icu-project.org/collation/customization#TOC-Default-Options
- https://unicode-org.github.io/icu/userguide/collation/customization/#default-options
+ ( Ref. https://unicode-org.github.io/icu/userguide/collation/customization/#default-options
http://www.unicode.org/reports/tr35/tr35-collation.html#Setting_Options )
以下の文字列を解釈する:
"alternate:shifted", "alternate:non-ignorable",
@@ -365,7 +399,7 @@ upmendex - 索引整形ツール
カナに揃え、拗音、撥音、濁点を除いた読みを付けなければなりませんでした(自動的に
揃えるバージョンもある)。
upmendex ではカナについてはInternational Components for Unicode (ICU)による
-ソートを行います。また漢字については辞書ファイルを設定することにより各索引語ごと
+ソートを行います。また漢字と記号については辞書ファイルを設定することにより各索引語ごと
に読みを付ける作業をかなり解消できます。
辞書ファイルは <熟語 読み> のリストで構成されます。熟語と読みの区切りはタブま
@@ -376,7 +410,7 @@ upmendex - 索引整形ツール
漢字 かんじ
読み よみ
環境 かんきょう
- α アルファ
+ $ ドル
辞書に登録する熟語は、読み方が1通りになるよう送り仮名を付けてください。
「表」、「性質」のように送り仮名によらず2通りの読み方ができる語についてはどち
@@ -445,5 +479,5 @@ upmendex - 索引整形ツール
参考
International Components for Unicode (ICU)
-http://site.icu-project.org/
+http://icu.unicode.org/
https://unicode-org.github.io/icu/
diff --git a/indexing/upmendex/man/upmendex.1 b/indexing/upmendex/man/upmendex.1
index b46ab922de..21c89844fa 100644
--- a/indexing/upmendex/man/upmendex.1
+++ b/indexing/upmendex/man/upmendex.1
@@ -53,9 +53,9 @@ and ignore leading and trailing blank(s).
By default, blanks in the index key are retained.
.TP 10
\fB-g\fR
-Make Japanese index head \fIA\fR-line (\fIA, Ka, Sa, ...\fR) of the gojuon table (Japanese
-syllabary). By default, all the characters in the gojuon table
-are used.
+Make Japanese index head \fIA\fR-line (\fIA, Ka, Sa, ...\fR; 10 characters)
+of the \fIgojuon\fR table (Japanese syllabary). By default, all 48 characters
+in the \fIgojuon\fR table are used.
.TP 10
\fB-f\fR
Force to output characters even if the scripts are not supported by \fIupmendex\fR.
@@ -88,6 +88,7 @@ Show summary of options.
.TP 10
\fB--\fR
Arguments after \fB--\fR are not taken as options.
+This is useful when the input file name starts with '-'.
.SH "STYLE FILE"
The style file informs \fIupmendex\fR about the format of the
@@ -106,7 +107,7 @@ A line begun by \'%\' is a comment.
\fBkeyword\fR <string>
"\\\\indexentry"
.RS
-Command with an argument of index entry.
+Command with an argument of index entry which is going to be processed.
.RE
.TP 30
\fBarg_open\fR <char>
@@ -220,7 +221,7 @@ Prefix of heading for newly appeared heading letter.
\fBheading_prefix\fR <string>
""
.RS
-Same as \fBlethead_prefix\fR.
+Same as \fBlethead_prefix\fR. (compatible with makeindex)
.RE
.TP 30
\fBlethead_suffix\fR <string>
@@ -232,7 +233,7 @@ Suffix of heading for newly appeared heading letter.
\fBheading_suffix\fR <string>
""
.RS
-Same as \fBlethead_suffix\fR.
+Same as \fBlethead_suffix\fR. (compatible with makeindex)
.RE
.TP 30
\fBlethead_flag\fR <number>
@@ -246,7 +247,13 @@ lowercase and titlecase.
\fBheading_flag\fR <number>
0
.RS
-Same as \fBlethead_flag\fR.
+Same as \fBlethead_flag\fR. (Note: makeindex uses a different name \fBheadings_flag\fR)
+.RE
+.TP 30
+\fBheadings_flag\fR <number>
+0
+.RS
+Same as \fBlethead_flag\fR. (compatible with makeindex)
.RE
.TP 30
\fBkana_head\fR <string>
@@ -448,31 +455,47 @@ Length of space for indent which inserted to top of folded line.
\fBsymhead_positive\fR <string>
"Symbols"
.RS
-Strings to output as heading letter for numbers and symbols
-when lethead_flag or heading_flag is positive number.
+Strings to output as heading letter for symbols
+when lethead_flag or heading_flag or headings_flag is positive number.
.RE
.TP 30
\fBsymhead_negative\fR <string>
"symbols"
.RS
-Strings to output as heading letter for numbers and symbols
-when lethead_flag or heading_flag is negative number.
+Strings to output as heading letter for symbols
+when lethead_flag or heading_flag or headings_flag is negative number.
.RE
.TP 30
\fBsymbol\fR <string>
""
.RS
-Strings to output as heading letter for numbers and symbols
+Strings to output as heading letter for symbols
when symbol_flag is non zero.
.RE
If specified, the option is prior to symhead_positive and symhead_negative.
(Extended by (up)mendex)
.RE
.TP 30
+\fBnumhead_positive\fR <string>
+"Numbers"
+.RS
+Strings to output as heading letter for numbers
+when lethead_flag or heading_flag or headings_flag is positive number and symbol_flag is 2.
+.RE
+.TP 30
+\fBnumhead_negative\fR <string>
+"numbers"
+.RS
+Strings to output as heading letter for numbers
+when lethead_flag or heading_flag or headings_flag is negative number and symbol_flag is 2.
+.RE
+.TP 30
\fBsymbol_flag\fR <number>
1
.RS
-Flag to output of symbol. If \'0\', do not output.
+Flag to output of symbol. If \'0\', do not output headings for symbols and numbers.
+If \'1\', output symbols and numbers as a group of symbols.
+If \'2\', output symbols and numbers separately.
(Extended by (up)mendex)
.RE
.TP 30
@@ -495,14 +518,34 @@ Japanese sequence and non-Japanese sequence in sorting procedure.
.RE
.TP 30
\fBcharacter_order\fR <string>
-"SNLGCJKHDT"
+"SNLGCJKHDTah"
.RS
Order of scripts and symbols.
-\'S\', \'N\', \'L\', \'G\', \'C\', \'J\', \'K\', \'H\', \'D\' and \'T\' respectively
-denotes symbol, number, Latin, Greek, Cyrillic, Japanese Kana, Korean Hangul, Hanja,
-Devanagari and Thai script.
-Please make sure that \'S\' and \'N\' are next to each other,
-since \fBupmendex\fR considers numbers as a part of symbol.
+\'S\', \'N\', \'L\', \'G\', \'C\', \'J\', \'K\', \'H\', \'D\', \'T\', \'a\' and \'h\'
+respectively denotes symbol, number, Latin, Greek, Cyrillic, Japanese Kana, Korean Hangul, Hanja,
+Devanagari, Thai, Arabic and Hebrew script.
+Please make sure that \'S\' and \'N\' are next to each other if symbol_flag=1,
+since numbers are classified as a part of symbol.
+(Extended by upmendex)
+.RE
+.TP 30
+\fBscript_preamble\fR <string 1> <string 2>
+""
+.RS
+Preamble of script block in output file, specified by string 2.
+One of script names must be specified in the string 1:
+\'latin\', \'cyrillic\', \'greek\', \'kana\', \'hangul\', \'hanzi\', \'devanagari\',
+\'thai\', \'arabic\', or \'hebrew\'.
+(Extended by upmendex)
+.RE
+.TP 30
+\fBscript_postamble\fR <string 1> <string 2>
+""
+.RS
+Postamble of script block in output file, specified by string 2.
+One of script names must be specified in the string 1:
+\'latin\', \'cyrillic\', \'greek\', \'kana\', \'hangul\', \'hanzi\', \'devanagari\',
+\'thai\', \'arabic\', or \'hebrew\'.
(Extended by upmendex)
.RE
.TP 30
@@ -522,11 +565,12 @@ Unicode characters in UTF-8 encoding and following escape sequences are accepted
\fB\\Uhhhhhhhh\fR (8-digit hexadecimal [0-9A-Fa-f]), \fB\\uhhhh\fR (4-digit hexadecimal),
\fB\\xhh\fR (2-digit hexadecimal), \fB\\x{h...}\fR (1..8-digit hexadecimal), and
\fB\\ooo\fR (3-digit octal [0-7]).
+If icu_rules and icu_locale are simultaneously specified, collation rules specified
+by icu_rules are added on collation rules specified by icu_locale.
By default, locale is used.
(Extended by upmendex)
.RE
-Ref. <http://userguide.icu-project.org/collation/customization>,
-<https://unicode-org.github.io/icu/userguide/collation/customization/>,
+Ref. <https://unicode-org.github.io/icu/userguide/collation/customization/>,
<http://www.unicode.org/reports/tr35/tr35-collation.html#Rules>
.RE
.TP 30
@@ -545,8 +589,7 @@ Followings are available:
"numeric-ordering:on", "numeric-ordering:off"
(Extended by upmendex)
.RE
-Ref. <http://userguide.icu-project.org/collation/customization>,
-<https://unicode-org.github.io/icu/userguide/collation/customization/#default-options>,
+Ref. <https://unicode-org.github.io/icu/userguide/collation/customization/#default-options>,
<http://www.unicode.org/reports/tr35/tr35-collation.html#Setting_Options>
.RE
.PP
@@ -559,8 +602,8 @@ of manually specifying a reading for every kanji word.
Japanese kanji words are usually sorted by the syllables of their readings
(\fI\'Yomi\'\fR), which can be represented by kana (Hiragana, Katakana) scripts.
\fIupmendex\fR accepts index words specified in kana expression directly on
-an input file, and also accepts conversion from index words to kana scripts
-by referring to Japanese dictionaries.
+an input file, and also accepts conversion from index words in Kanji or symbols
+to phonogram scripts by referring to Japanese dictionaries.
.RE
.LP
@@ -577,7 +620,7 @@ Examples of internal simplification of syllables are shown below.
.LP
The dictionary file consists of list with <\'index_word\' \'reading\'>.
The index word can be written in any scripts (kanji, kana, etc),
-and the reading must be in Hiragana or Katakana scripts.
+and the reading can be in any phonograms such as Hiragana or Katakana scripts.
The delimiter between the index word and its reading is one or more tab(s) or space(s).
.RE
An example of a Japanese dictionary is shown below.
@@ -590,7 +633,7 @@ An example of a Japanese dictionary is shown below.
.br
環境 かんきょう
.br
-α アルファ
+$ ドル
.RE
.LP
Here, each index word is allowed to have only one Yomi.
@@ -672,7 +715,7 @@ wrong page numbers might be output.
.BR mendex(1).
.br
International Components for Unicode (ICU):
-<http://site.icu-project.org/>,
+<http://icu.unicode.org/>,
<https://unicode-org.github.io/icu/>
.SH AUTHOR
This manual page was written by Takuji Tanaka
diff --git a/indexing/upmendex/man/upmendex.man1.pdf b/indexing/upmendex/man/upmendex.man1.pdf
index a63130ad5e..195a8190ef 100644
--- a/indexing/upmendex/man/upmendex.man1.pdf
+++ b/indexing/upmendex/man/upmendex.man1.pdf
Binary files differ
diff --git a/indexing/upmendex/source/ChangeLog b/indexing/upmendex/source/ChangeLog
index 497e0e6e26..b2b3cd2f1e 100644
--- a/indexing/upmendex/source/ChangeLog
+++ b/indexing/upmendex/source/ChangeLog
@@ -1,3 +1,108 @@
+2021-11-20 TANAKA Takuji <ttk@t-lab.opal.ne.jp>
+
+ * version 1.00 Stable version.
+ * configure.ac: Bump version.
+ * {,ex}var.h, fwrite.c, styfile.c:
+ Add options "script_preamble" and "script_postamble"
+ in style file.
+ * sort.c: Add Latin Extended-F and -G.
+ Add U_FORMAT_CHAR to is_type_mark_or_punct().
+ * mendex.h, var.h, convert.c, fwrite.c, sort.c, styfile.c:
+ Supports Arabic, Hebrew scripts (experimental).
+ * upmendex.ja.txt:
+ Update document.
+
+2021-10-24 TANAKA Takuji <ttk@t-lab.opal.ne.jp>
+
+ * version 0.60 Beta version.
+ * configure.ac: Bump version.
+ * Makefile.am, convert.c, fwrite.c, sort.c,
+ {,ex}kana.h, ktable.h, mendex.h:
+ Enhance procedure of classifying characters:
+ Newly classifies by character type of
+ General Category in Unicode Script Property.
+ https://github.com/t-tk/upmendex-package/issues/8
+
+2021-09-19 TANAKA Takuji <ttk@t-lab.opal.ne.jp>
+
+ * fread.c:
+ Modify heuristic detection of page number types.
+ * tests/upmendex.test, tests/range.idx, tests/range[123].ind,
+ tests/pprecB.idx, tests/pprec[34].ist, tests/pprecB-[034].ind:
+ Update tests.
+
+2021-09-18 TANAKA Takuji <ttk@t-lab.opal.ne.jp>
+
+ * fread.c, fwrite.c, sort.c, mendex.h, {,ex}var.h:
+ Extend levels of page numbers from 3 to 10
+ for compatibility with makeindex.
+ Fix a bug of page ranges with multi-level page numbers.
+ * fread.c:
+ Strict check for "page_precedence" and "page_compositor" in style file.
+ * fwrite.c:
+ Do not break lines before "delim_n".
+ * tests/upmendex.test, tests/range.idx, tests/range[123].{ist,ind},
+ removed tests/rangetwo.*:
+ Update tests.
+
+2021-09-05 TANAKA Takuji <ttk@t-lab.opal.ne.jp>
+
+ * fwrite.c:
+ Support I-Y ordering in Lithuanian.
+
+2021-09-04 TANAKA Takuji <ttk@t-lab.opal.ne.jp>
+
+ * version 0.59 Beta version.
+ * configure.ac: Bump version.
+ * fwrite.c, styfile.c, {,ex}var.h:
+ Support output order of symbols and numbers separately
+ when "symbol_flag" is 2 in style file.
+ * sort.c:
+ Accept combination of "icu_locale" and "icu_rules" in style file.
+ If simultaneously specified, collation rules specified by "icu_rules" are
+ added on collation rules specified by "icu_locale".
+ * tests/upmendex.test, tests/rule.{idx,ind}, tests/locale2.ist,
+ tests/{locale1,attri1,locale2-rule}.ind:
+ Add a new test and update conventional tests.
+
+2021-08-29 TANAKA Takuji <ttk@t-lab.opal.ne.jp>
+
+ * styfile.c:
+ Strict check for "icu_attributes" in style file.
+ * sort.c:
+ Strict check for "character_order" in style file.
+ * fwrite.c:
+ Fix a bug when "symbol_flag" is 0.
+ * tests/upmendex.test, tests/attr1.{ist,ind},
+ tests/locale1.ind, tests/rule.{idx,ind}:
+ Add a new test and update conventional tests.
+
+2021-08-28 TANAKA Takuji <ttk@t-lab.opal.ne.jp>
+
+ * styfile.c:
+ Add a alias "headings_flag" of the keyword "heading_flag"
+ in style file for compatibility with makeindex.
+ Warn if specifier in style file is unknown.
+
+2021-08-26 TANAKA Takuji <ttk@t-lab.opal.ne.jp>
+
+ * sort.c:
+ Initialize a variable.
+ * tests/upmendex.test, tests/locale1.{ist,ind}:
+ Add a new test for "icu_locale".
+
+2021-08-25 TANAKA Takuji <ttk@t-lab.opal.ne.jp>
+
+ * sort.c:
+ Initialize a string. Check malformed input for "icu_rules".
+ * tests/upmendex.test, tests/rule[34].ist:
+ Add new tests.
+
+2021-08-23 TANAKA Takuji <ttk@t-lab.opal.ne.jp>
+
+ * tests/upmendex.test, tests/rule[12].ist, tests/rule.{idx,ind}:
+ Add new tests for the keyword "icu_rules" in style file.
+
2021-08-08 TANAKA Takuji <ttk@t-lab.opal.ne.jp>
* version 0.58 Beta version.
@@ -90,7 +195,7 @@
2020-05-01 TANAKA Takuji <ttk@t-lab.opal.ne.jp>
- * main.c, version.h, Makefile.{am.in}: Add bug report addresses
+ * main.c, version.h, Makefile.am: Add bug report addresses
on command line help.
* convert.c: Tune buffer sizes to avoid tool warnings.
diff --git a/indexing/upmendex/source/Makefile.am b/indexing/upmendex/source/Makefile.am
index 94bea187b3..451f0df1eb 100644
--- a/indexing/upmendex/source/Makefile.am
+++ b/indexing/upmendex/source/Makefile.am
@@ -1,4 +1,4 @@
-## $Id$
+## $Id: Makefile.am 60856 2021-10-24 05:44:35Z takuji $
## Makefile.am for the TeX Live subdirectory texk/upmendex/
##
## Copyright 2015-2019 Karl Berry <tex-live@tug.org>
@@ -19,7 +19,6 @@ upmendex_SOURCES = \
kana.h \
kp.c \
kp.h \
- ktable.h \
main.c \
mendex.h \
version.h \
diff --git a/indexing/upmendex/source/Makefile.in b/indexing/upmendex/source/Makefile.in
index 11cff850bc..4ecaadee97 100644
--- a/indexing/upmendex/source/Makefile.in
+++ b/indexing/upmendex/source/Makefile.in
@@ -1,4 +1,4 @@
-# Makefile.in generated by automake 1.16.4 from Makefile.am.
+# Makefile.in generated by automake 1.16.5 from Makefile.am.
# @configure_input@
# Copyright (C) 1994-2021 Free Software Foundation, Inc.
@@ -593,7 +593,6 @@ upmendex_SOURCES = \
kana.h \
kp.c \
kp.h \
- ktable.h \
main.c \
mendex.h \
version.h \
diff --git a/indexing/upmendex/source/aclocal.m4 b/indexing/upmendex/source/aclocal.m4
index 284ca6d457..8d45a4e2bc 100644
--- a/indexing/upmendex/source/aclocal.m4
+++ b/indexing/upmendex/source/aclocal.m4
@@ -1,4 +1,4 @@
-# generated automatically by aclocal 1.16.4 -*- Autoconf -*-
+# generated automatically by aclocal 1.16.5 -*- Autoconf -*-
# Copyright (C) 1996-2021 Free Software Foundation, Inc.
@@ -35,7 +35,7 @@ AC_DEFUN([AM_AUTOMAKE_VERSION],
[am__api_version='1.16'
dnl Some users find AM_AUTOMAKE_VERSION and mistake it for a way to
dnl require some minimum version. Point them to the right macro.
-m4_if([$1], [1.16.4], [],
+m4_if([$1], [1.16.5], [],
[AC_FATAL([Do not call $0, use AM_INIT_AUTOMAKE([$1]).])])dnl
])
@@ -51,7 +51,7 @@ m4_define([_AM_AUTOCONF_VERSION], [])
# Call AM_AUTOMAKE_VERSION and AM_AUTOMAKE_VERSION so they can be traced.
# This function is AC_REQUIREd by AM_INIT_AUTOMAKE.
AC_DEFUN([AM_SET_CURRENT_AUTOMAKE_VERSION],
-[AM_AUTOMAKE_VERSION([1.16.4])dnl
+[AM_AUTOMAKE_VERSION([1.16.5])dnl
m4_ifndef([AC_AUTOCONF_VERSION],
[m4_copy([m4_PACKAGE_VERSION], [AC_AUTOCONF_VERSION])])dnl
_AM_AUTOCONF_VERSION(m4_defn([AC_AUTOCONF_VERSION]))])
@@ -428,6 +428,10 @@ m4_defn([AC_PROG_CC])
# release and drop the old call support.
AC_DEFUN([AM_INIT_AUTOMAKE],
[AC_PREREQ([2.65])dnl
+m4_ifdef([_$0_ALREADY_INIT],
+ [m4_fatal([$0 expanded multiple times
+]m4_defn([_$0_ALREADY_INIT]))],
+ [m4_define([_$0_ALREADY_INIT], m4_expansion_stack)])dnl
dnl Autoconf wants to disallow AM_ names. We explicitly allow
dnl the ones we care about.
m4_pattern_allow([^AM_[A-Z]+FLAGS$])dnl
diff --git a/indexing/upmendex/source/configure b/indexing/upmendex/source/configure
index f1ffea6fbb..e5aa682d08 100755
--- a/indexing/upmendex/source/configure
+++ b/indexing/upmendex/source/configure
@@ -1,6 +1,6 @@
#! /bin/sh
# Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.71 for upmendex (TeX Live) 0.58.
+# Generated by GNU Autoconf 2.71 for upmendex (TeX Live) 1.00.
#
#
# Copyright (C) 1992-1996, 1998-2017, 2020-2021 Free Software Foundation,
@@ -626,8 +626,8 @@ MAKEFLAGS=
# Identity of this package.
PACKAGE_NAME='upmendex (TeX Live)'
PACKAGE_TARNAME='upmendex--tex-live-'
-PACKAGE_VERSION='0.58'
-PACKAGE_STRING='upmendex (TeX Live) 0.58'
+PACKAGE_VERSION='1.00'
+PACKAGE_STRING='upmendex (TeX Live) 1.00'
PACKAGE_BUGREPORT=''
PACKAGE_URL=''
@@ -1390,7 +1390,7 @@ if test "$ac_init_help" = "long"; then
# Omit some internal or obsolete options to make the list less imposing.
# This message is too long to be a string in the A/UX 3.1 sh.
cat <<_ACEOF
-\`configure' configures upmendex (TeX Live) 0.58 to adapt to many kinds of systems.
+\`configure' configures upmendex (TeX Live) 1.00 to adapt to many kinds of systems.
Usage: $0 [OPTION]... [VAR=VALUE]...
@@ -1462,7 +1462,7 @@ fi
if test -n "$ac_init_help"; then
case $ac_init_help in
- short | recursive ) echo "Configuration of upmendex (TeX Live) 0.58:";;
+ short | recursive ) echo "Configuration of upmendex (TeX Live) 1.00:";;
esac
cat <<\_ACEOF
@@ -1587,7 +1587,7 @@ fi
test -n "$ac_init_help" && exit $ac_status
if $ac_init_version; then
cat <<\_ACEOF
-upmendex (TeX Live) configure 0.58
+upmendex (TeX Live) configure 1.00
generated by GNU Autoconf 2.71
Copyright (C) 2021 Free Software Foundation, Inc.
@@ -2268,7 +2268,7 @@ cat >config.log <<_ACEOF
This file contains any messages produced by compilers while
running configure, to aid debugging if configure makes a mistake.
-It was created by upmendex (TeX Live) $as_me 0.58, which was
+It was created by upmendex (TeX Live) $as_me 1.00, which was
generated by GNU Autoconf 2.71. Invocation command line was
$ $0$ac_configure_args_raw
@@ -8806,7 +8806,7 @@ fi
# Define the identity of the package.
PACKAGE='upmendex--tex-live-'
- VERSION='0.58'
+ VERSION='1.00'
# Some tools Automake needs.
@@ -18942,7 +18942,7 @@ Usage: $0 [OPTIONS]
Report bugs to <bug-libtool@gnu.org>."
lt_cl_version="\
-upmendex (TeX Live) config.lt 0.58
+upmendex (TeX Live) config.lt 1.00
configured by $0, generated by GNU Autoconf 2.71.
Copyright (C) 2011 Free Software Foundation, Inc.
@@ -21114,7 +21114,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
# report actual input values of CONFIG_FILES etc. instead of their
# values after options handling.
ac_log="
-This file was extended by upmendex (TeX Live) $as_me 0.58, which was
+This file was extended by upmendex (TeX Live) $as_me 1.00, which was
generated by GNU Autoconf 2.71. Invocation command line was
CONFIG_FILES = $CONFIG_FILES
@@ -21182,7 +21182,7 @@ ac_cs_config_escaped=`printf "%s\n" "$ac_cs_config" | sed "s/^ //; s/'/'\\\\\\\\
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
ac_cs_config='$ac_cs_config_escaped'
ac_cs_version="\\
-upmendex (TeX Live) config.status 0.58
+upmendex (TeX Live) config.status 1.00
configured by $0, generated by GNU Autoconf 2.71,
with options \\"\$ac_cs_config\\"
diff --git a/indexing/upmendex/source/configure.ac b/indexing/upmendex/source/configure.ac
index 7af5a44b8c..b21873a1ed 100644
--- a/indexing/upmendex/source/configure.ac
+++ b/indexing/upmendex/source/configure.ac
@@ -1,4 +1,4 @@
-dnl $Id$
+dnl $Id: configure.ac 61096 2021-11-20 09:29:33Z takuji $
dnl Process this file with autoconf to produce a configure script.
dnl
dnl Copyright 2015-2021 Karl Berry <tex-live@tug.org>
@@ -8,8 +8,8 @@ dnl This file is free software; the copyright holder
dnl gives unlimited permission to copy and/or distribute it,
dnl with or without modifications, as long as this notice is preserved.
dnl
-AC_INIT([upmendex (TeX Live)], [0.58])
-AC_PREREQ([2.63])
+AC_INIT([upmendex (TeX Live)],[1.00])
+AC_PREREQ([2.71])
AC_CONFIG_SRCDIR([main.c])
AC_CONFIG_AUX_DIR([../../build-aux])
AC_CONFIG_MACRO_DIR([../../m4])
diff --git a/indexing/upmendex/source/convert.c b/indexing/upmendex/source/convert.c
index 74d32d7352..a7cae61038 100644
--- a/indexing/upmendex/source/convert.c
+++ b/indexing/upmendex/source/convert.c
@@ -6,7 +6,6 @@
#include "qsort.h"
#include "exkana.h"
-#include "ktable.h"
#include "exvar.h"
#include "kp.h"
@@ -25,8 +24,6 @@ static int dlines=0,elines=0;
static int dicvalread(const char *filename, struct dictionary *dicval, int line);
-#define is_alpha_numeric(a) (((a)>=ALPHATOP && (a)<=ALPHAEND) || (a)==YENSIGN)
-
UChar * u_xstrdup (const UChar *string)
{
return u_strcpy(xmalloc((u_strlen(string)+1)*sizeof(UChar)), string);
@@ -210,76 +207,66 @@ int convert(UChar *buff1, UChar *buff2)
i++;
}
- else if (buff1[i]<0x80) {
+ else if (buff1[i]<0x20 && buff1[i]!='\t') { /* ignore control characters */
+ i++;
+ }
+
+ else if (buff1[i]<0x7F) {
buff2[j]=buff1[i];
i++;
j++;
}
+ else if (buff1[i]<0xA0) { /* ignore control characters */
+ i++;
+ }
+
else if (is_latin(buff3)||is_cyrillic(buff3)||is_greek(buff3)
||is_jpn_kana(buff3)||is_kor_hngl(buff3)||is_zhuyin(buff3)
- ||is_numeric(buff3)||is_devanagari(buff3)||is_thai(buff3)
- ||is_comb_diacritical_mark(buff3)) {
+ ||is_numeric(buff3)==1||is_type_symbol(buff3)==1
+ ||is_devanagari(buff3)||is_thai(buff3)||is_arabic(buff3)||is_hebrew(buff3)
+ ||is_type_mark_or_punct(buff3)) {
buff2[j]=buff3[0];
if (wclen==2) buff2[j+1]=buff3[1];
i+=wclen;
j+=wclen;
}
- else if (buff1[i]>=0x80) {
- if (is_alpha_numeric(buff3[0])) {
-/* alpha-numeric,symbols */
- for (k=0;k<u_strlen(symboltable);k++) {
- if (buff3[0]==symboltable[k]) {
- buff2[j]=k+0x20;
- if ((buff2[j]>='a')&&(buff2[j]<='z')) buff2[j]-=32;
- i++;
- j+=wclen;
- break;
- }
- }
- if (k==u_strlen(symboltable)) {
- i++;
- buff2[j++]=buff3[0];
+ else {
+ for (k=0;k<dlines;k++) {
+/* dictionary table */
+ if (u_strncmp(dictable[k].dic[0],&buff1[i],u_strlen(dictable[k].dic[0]))==0) {
+ u_strncpy(&buff2[j],dictable[k].dic[1],u_strlen(dictable[k].dic[1]));
+ i+=u_strlen(dictable[k].dic[0]);
+ j+=u_strlen(dictable[k].dic[1]);
+ break;
}
}
-
- else {
- for (k=0;k<dlines;k++) {
-/* dictionary table */
- if (u_strncmp(dictable[k].dic[0],&buff1[i],u_strlen(dictable[k].dic[0]))==0) {
- u_strncpy(&buff2[j],dictable[k].dic[1],u_strlen(dictable[k].dic[1]));
+ if ((k==dlines)&&(elines!=0)) {
+/* environment dictionary table */
+ for (k=0;k<elines;k++) {
+ if (u_strncmp(envdic[k].dic[0],&buff1[i],u_strlen(envdic[k].dic[0]))==0) {
+ u_strncpy(&buff2[j],envdic[k].dic[1],u_strlen(envdic[k].dic[1]));
i+=u_strlen(dictable[k].dic[0]);
- j+=u_strlen(dictable[k].dic[1]);
+ j+=u_strlen(envdic[k].dic[1]);
break;
}
}
- if ((k==dlines)&&(elines!=0)) {
-/* environment dictionary table */
- for (k=0;k<elines;k++) {
- if (u_strncmp(envdic[k].dic[0],&buff1[i],u_strlen(envdic[k].dic[0]))==0) {
- u_strncpy(&buff2[j],envdic[k].dic[1],u_strlen(envdic[k].dic[1]));
- i+=u_strlen(dictable[k].dic[0]);
- j+=u_strlen(envdic[k].dic[1]);
- break;
- }
- }
- }
- if (((k==dlines)&&(elines==0))||((k==elines)&&(elines!=0))) {
- if (is_hanzi(buff3) || force==1) {
+ }
+ if (((k==dlines)&&(elines==0))||((k==elines)&&(elines!=0))) {
+ if (is_hanzi(buff3) || is_numeric(buff3) || is_type_symbol(buff3) || force==1) {
/* forced convert */
- buff2[j]=buff3[0];
- if (wclen==2) buff2[j+1]=buff3[1];
- i+=wclen;
- j+=wclen;
- }
- else {
- widechar_to_multibyte(errbuff,BUFFERLEN2,&buff1[i]);
- snprintf(errbuff2,BUFFERLEN3,"\nError: %s is no entry in dictionary file ",errbuff);
- fputs(errbuff2,efp);
- if (efp!=stderr) fputs(errbuff2,stderr);
- return -1;
- }
+ buff2[j]=buff3[0];
+ if (wclen==2) buff2[j+1]=buff3[1];
+ i+=wclen;
+ j+=wclen;
+ }
+ else {
+ widechar_to_multibyte(errbuff,BUFFERLEN2,&buff1[i]);
+ snprintf(errbuff2,BUFFERLEN3,"\nError: %s is no entry in dictionary file ",errbuff);
+ fputs(errbuff2,efp);
+ if (efp!=stderr) fputs(errbuff2,stderr);
+ return -1;
}
}
}
diff --git a/indexing/upmendex/source/exkana.h b/indexing/upmendex/source/exkana.h
index aeb5a18d82..9e4ba1d197 100644
--- a/indexing/upmendex/source/exkana.h
+++ b/indexing/upmendex/source/exkana.h
@@ -4,7 +4,6 @@ extern UChar AIUEO[];
extern UChar *aiueo;
extern UChar kanatable[];
extern UChar extkanatable[];
-extern UChar circkanatable[];
#define SPACE 0x3000 /* 全角スペース */
#define ALPHATOP 0xff01 /* ! */
@@ -18,8 +17,19 @@ extern UChar circkanatable[];
#define EXKANAEND 0x31FF /* ㇿ */
#define CRKANATOP 0x32D0 /* ㋐ */
#define CRKANAEND 0x32FE /* ㋾ */
+#define HANKANATOP 0xFF71 /* ア */
+#define HANKANAEND 0xFF9D /* ン */
+#define HANKANAWO 0xFF66 /* ヲ */
+#define HANKANATU 0xFF6F /* ッ */
+#define SQKANATOP 0x3300 /* ㌀ */
+#define SQKANAEND 0x3357 /* ㍗ */
+#define CRLATNTOP 0x24B6 /* Ⓐ */
+#define CRLATNEND 0x24E9 /* ⓩ */
#define is_katakana(a) ((a)>=KATATOP && (a)<=KATAEND)
#define is_hiragana(a) ((a)>=HIRATOP && (a)<=HIRAEND)
#define is_extkana(a) ((a)>=EXKANATOP && (a)<=EXKANAEND)
#define is_circkana(a) ((a)>=CRKANATOP && (a)<=CRKANAEND)
+#define is_hankana(a) ((a)>=HANKANATOP && (a)<=HANKANAEND || (a)>=HANKANAWO && (a)<=HANKANATU)
+#define is_sqkana(a) ((a)>=SQKANATOP && (a)<=SQKANAEND)
+#define is_circlatin(a) ((a)>=CRLATNTOP && (a)<=CRLATNEND)
diff --git a/indexing/upmendex/source/exvar.h b/indexing/upmendex/source/exvar.h
index 7b061c6655..790f327e27 100644
--- a/indexing/upmendex/source/exvar.h
+++ b/indexing/upmendex/source/exvar.h
@@ -2,7 +2,7 @@ extern FILE *efp;
extern int lines,idxcount,acc,reject;
extern int prange,fsti,lorder,bcomp,force,fpage,gflg,verb,debug;
-extern int warn,scount,pattr[3];
+extern int warn,scount,pattr[];
extern struct index *ind;
@@ -26,7 +26,7 @@ extern int line_max;
extern char indent_space[];
extern int indent_length;
extern int priority;
-extern char symbol[];
+extern char symhead[],numhead[];
extern char symhead_positive[],symhead_negative[];
extern char numhead_positive[],numhead_negative[];
extern int symbol_flag;
@@ -35,6 +35,7 @@ extern UChar atama[],hangul_head[],hanzi_head[],kana_head[];
extern UChar devanagari_head[],thai_head[];
extern char page_compositor[],page_precedence[];
extern char character_order[];
+extern char script_preamble[][STYBUFSIZE],script_postamble[][STYBUFSIZE];
extern char icu_locale[],icu_rules[];
extern int icu_attributes[];
diff --git a/indexing/upmendex/source/fread.c b/indexing/upmendex/source/fread.c
index a336022e11..66d3e0b144 100644
--- a/indexing/upmendex/source/fread.c
+++ b/indexing/upmendex/source/fread.c
@@ -344,7 +344,7 @@ LOOP:
copy_multibyte_char(buff, table, &j, &k);
}
- table[k]='\0';
+ table[k]='\0';
for (k=0;k<=ind[l].num;k++) {
if (strcmp(ind[l].p[k].page,table)==0) {
@@ -356,9 +356,9 @@ LOOP:
ind[l].num++;
if (!((ind[l].num)%16)) ind[l].p=(struct page *)xrealloc(ind[l].p,sizeof(struct page)*((int)((ind[l].num)/16)+1)*16);
- ind[l].p[ind[l].num].page=xstrdup(table);
+ ind[l].p[ind[l].num].page=xstrdup(table);
- ind[l].p[ind[l].num].enc=xstrdup(estr);
+ ind[l].p[ind[l].num].enc=xstrdup(estr);
chkpageattr(&ind[l].p[ind[l].num]);
}
}
@@ -388,15 +388,15 @@ LOOP:
nest++;
if (buff[j]==arg_close) {
if (nest==0) {
- table[k]='\0';
- ind[i].p[0].page=xstrdup(table);
+ table[k]='\0';
+ ind[i].p[0].page=xstrdup(table);
break;
}
else nest--;
}
copy_multibyte_char(buff, table, &j, &k);
}
- ind[l].p[0].enc=xstrdup(estr);
+ ind[l].p[0].enc=xstrdup(estr);
chkpageattr(&ind[i].p[0]);
}
}
@@ -493,25 +493,54 @@ int widechar_to_multibyte(char *mbstr, int32_t size, UChar *wcstr)
static void chkpageattr(struct page *p)
{
- int i,j,cc=0;
+ int i,j,cc=0,cnt,pplen,pclen;
+ char buff[16],*pcpos,*page0;
+ pplen=strlen(page_precedence);
+ pclen=strlen(page_compositor);
for (i=0;i<strlen(p->page);i++) {
- if (strncmp(page_compositor,&p->page[i],strlen(page_compositor))==0) {
+ page0=&p->page[i];
+ if (strncmp(page_compositor,page0,pclen)==0) {
p->attr[cc]=pattr[cc];
cc++;
- i+=strlen(page_compositor)-1;
+ i+=pclen-1;
+ if (cc>=PAGE_COMPOSIT_DEPTH) {
+ if (pclen>0)
+ verb_printf(efp, "\nToo many fields of page number \"%s\".\n", p->page);
+ else
+ verb_printf(efp, "\nIllegular page_comositor specification.\n");
+ exit(253);
+ }
}
else {
-ATTRLOOP:
- if (!((p->page[i]>='0' && p->page[i]<='9') || (p->page[i]>='A' && p->page[i]<='Z') || (p->page[i]>='a' && p->page[i]<='z'))) {
+ cnt=0;
+ if (!((*page0>='0' && *page0<='9') || (*page0>='A' && *page0<='Z') || (*page0>='a' && *page0<='z'))) {
p->attr[cc]= -1;
if (cc<2) p->attr[++cc]= -1;
return;
}
+ pcpos=strstr(page0,page_compositor);
+ j=pcpos ? pcpos-page0 : strlen(page0);
+ if (j>15) {
+ verb_printf(efp, "\nToo long page number string \"%s\".\n", page0);
+ exit(253);
+ }
+ strncpy(buff,page0,j);
+ buff[j]='\0';
+ATTRLOOP:
+ cnt++;
+ if (cnt>pplen) {
+ verb_printf(efp, "\nFailed to find page type for page \"%s\" in page_precedence specification (%s).\n",
+ page0, page_precedence);
+ exit(253);
+ }
+
switch(page_precedence[pattr[cc]]) {
case 'r':
- if (strchr("ivxlcdm",p->page[i])==NULL) {
- if (pattr[cc]<strlen(page_precedence)-1)
+ if (strchr("ivxlcdm",*page0)==NULL ||
+ (strchr("lcdm",*page0) && strchr(page_precedence,'a') && strlen(buff)==1 && pcpos)) {
+ /* heuristic detection as alphabet since L=50, C=100, D=100, M=1000 are quite large */
+ if (pattr[cc]<pplen-1)
pattr[cc]++;
else pattr[cc]=0;
for (j=cc+1;j<3;j++) pattr[j]=0;
@@ -519,8 +548,10 @@ ATTRLOOP:
}
break;
case 'R':
- if (strchr("IVXLCDM",p->page[i])==NULL) {
- if (pattr[cc]<strlen(page_precedence)-1)
+ if (strchr("IVXLCDM",*page0)==NULL ||
+ (strchr("LCDM",*page0) && strchr(page_precedence,'A') && strlen(buff)==1 && pcpos)) {
+ /* heuristic detection as alphabet since L=50, C=100, D=100, M=1000 are quite large */
+ if (pattr[cc]<pplen-1)
pattr[cc]++;
else pattr[cc]=0;
for (j=cc+1;j<3;j++) pattr[j]=0;
@@ -528,8 +559,8 @@ ATTRLOOP:
}
break;
case 'n':
- if (p->page[i]<'0' || p->page[i]>'9') {
- if (pattr[cc]<strlen(page_precedence)-1)
+ if (*page0<'0' || *page0>'9') {
+ if (pattr[cc]<pplen-1)
pattr[cc]++;
else pattr[cc]=0;
for (j=cc+1;j<3;j++) pattr[j]=0;
@@ -537,8 +568,10 @@ ATTRLOOP:
}
break;
case 'a':
- if (p->page[i]<'a' || p->page[i]>'z') {
- if (pattr[cc]<strlen(page_precedence)-1)
+ if (*page0<'a' || *page0>'z' || strlen(buff)>1 ||
+ (strchr("ivx",*page0) && strchr(page_precedence,'r') && !pcpos)) {
+ /* heuristic detection as roman number since I=1, V=5, X=10 are quite small */
+ if (pattr[cc]<pplen-1)
pattr[cc]++;
else pattr[cc]=0;
for (j=cc+1;j<3;j++) pattr[j]=0;
@@ -546,8 +579,10 @@ ATTRLOOP:
}
break;
case 'A':
- if (p->page[i]<'A' || p->page[i]>'Z') {
- if (pattr[cc]<strlen(page_precedence)-1)
+ if (*page0<'A' || *page0>'Z' || strlen(buff)>1 ||
+ (strchr("IVX",*page0) && strchr(page_precedence,'R') && !pcpos)) {
+ /* heuristic detection as roman number since I=1, V=5, X=10 are quite small */
+ if (pattr[cc]<pplen-1)
pattr[cc]++;
else pattr[cc]=0;
for (j=cc+1;j<3;j++) pattr[j]=0;
@@ -555,7 +590,9 @@ ATTRLOOP:
}
break;
default:
- break;
+ verb_printf(efp, "\nUnknown page type '%c' in page_precedence specification (%s).\n",
+ page_precedence[pattr[cc]], page_precedence);
+ exit(253);
}
}
}
diff --git a/indexing/upmendex/source/fwrite.c b/indexing/upmendex/source/fwrite.c
index d1685ede1c..9ce382589e 100644
--- a/indexing/upmendex/source/fwrite.c
+++ b/indexing/upmendex/source/fwrite.c
@@ -13,12 +13,12 @@ int line_length=0;
static void printpage(struct index *ind, FILE *fp, int num, char *lbuff);
static int range_check(struct index ind, int count, char *lbuff);
-static void linecheck(char *lbuff, char *tmpbuff);
+static void linecheck(char *lbuff, char *tmpbuff, int force);
static void crcheck(char *lbuff, FILE *fp);
static void index_normalize(UChar *istr, UChar *ini, int *chset);
static int initial_cmp_char(UChar *ini, UChar ch);
static int init_hanzi_header(void);
-static const UNormalizer2* unormalizer_NFD;
+static const UNormalizer2 *unormalizer_NFD, *unormalizer_NFKD;
static int turkish_i;
#define M_NONE 0
@@ -93,7 +93,7 @@ static void fprint_uchar(FILE *fp, const UChar *a, const int mode, const int len
} else
olen=wclen;
if (olen>INITIALLENGTH) {
- warn_printf(efp, "\nWarning, Too long (%d) header.\n", olen);
+ warn_printf(efp, "\nWarning: Too long (%d) header.\n", olen);
wclen=INITIALLENGTH;
} else
wclen=olen;
@@ -128,7 +128,7 @@ void warn_printf(FILE *fp, const char *format, ...)
vsnprintf(print_buff, sizeof print_buff, format, argptr);
va_end(argptr);
- warn++;
+ warn++;
fputs(print_buff, stderr);
if (fp!=stderr) fputs(print_buff, fp);
}
@@ -146,11 +146,27 @@ void verb_printf(FILE *fp, const char *format, ...)
if (fp!=stderr) fputs(print_buff, fp);
}
+static int pnumconv2(struct page *p)
+{
+ int j,k,cc,pclen;
+
+ pclen=strlen(page_compositor);
+ for (j=k=cc=0;j<strlen(p->page);j++) {
+ if (strncmp(p->page+j,page_compositor,pclen)==0) {
+ j+=pclen;
+ k=j;
+ cc++;
+ continue;
+ }
+ }
+ return pnumconv(p->page+k,p->attr[cc]);
+}
+
/* write ind file */
void indwrite(char *filename, struct index *ind, int pagenum)
{
- int i,j,hpoint=0,tpoint=0,ipoint=0,jpoint=0;
+ int i,j,hpoint=0,tpoint=0,ipoint=0,jpoint=0,block_open=0;
char lbuff[BUFFERLEN],obuff[BUFFERLEN];
UChar datama[256],initial[INITIALLENGTH],initial_prev[INITIALLENGTH];
int chset,chset_prev;
@@ -173,12 +189,34 @@ void indwrite(char *filename, struct index *ind, int pagenum)
fprintf(fp,"%s%d%s",setpage_prefix,pagenum,setpage_suffix);
}
perr=U_ZERO_ERROR;
- unormalizer_NFD=unorm2_getInstance(NULL, "nfc", UNORM2_DECOMPOSE, &perr);
+ unormalizer_NFD =unorm2_getInstance(NULL, "nfc", UNORM2_DECOMPOSE, &perr);
+ unormalizer_NFKD=unorm2_getInstance(NULL, "nfkc", UNORM2_DECOMPOSE, &perr);
+
+ if (strlen(symhead)==0) {
+ if (lethead_flag>0) {
+ strcpy(symhead, symhead_positive);
+ }
+ else if (lethead_flag<0) {
+ strcpy(symhead, symhead_negative);
+ }
+ }
+ {
+ if (lethead_flag>0) {
+ strcpy(numhead, numhead_positive);
+ }
+ else if (lethead_flag<0) {
+ strcpy(numhead, numhead_negative);
+ }
+ }
for (i=line_length=0;i<lines;i++) {
index_normalize(ind[i].dic[0], initial, &chset);
if (i==0) {
- if ((CH_LATIN<=chset&&chset<=CH_GREEK) || chset==CH_HANZI) {
+ if (is_any_script(chset) && strlen(script_preamble[chset])) {
+ fputs(script_preamble[chset],fp);
+ block_open=chset;
+ }
+ if ((CH_LATIN<=chset&&chset<=CH_GREEK) || chset==CH_HANZI || (CH_ARABIC<=chset&&chset<=CH_HEBREW)) {
if (lethead_flag!=0) {
fputs(lethead_prefix,fp);
fprint_uchar(fp,initial,lethead_flag,-1);
@@ -280,16 +318,11 @@ void indwrite(char *filename, struct index *ind, int pagenum)
}
}
else {
- if (lethead_flag!=0) {
- if (symbol_flag && strlen(symbol)) {
- fprintf(fp,"%s%s%s",lethead_prefix,symbol,lethead_suffix);
- }
- else if (lethead_flag>0) {
- fprintf(fp,"%s%s%s",lethead_prefix,symhead_positive,lethead_suffix);
- }
- else if (lethead_flag<0) {
- fprintf(fp,"%s%s%s",lethead_prefix,symhead_negative,lethead_suffix);
- }
+ if (lethead_flag!=0 && symbol_flag==2 && chset==CH_NUMERIC) {
+ fprintf(fp,"%s%s%s",lethead_prefix,numhead,lethead_suffix);
+ }
+ if (lethead_flag!=0 && (symbol_flag==1 || (symbol_flag==2 && chset!=CH_NUMERIC))) {
+ fprintf(fp,"%s%s%s",lethead_prefix,symhead,lethead_suffix);
}
widechar_to_multibyte(obuff,BUFFERLEN,ind[i].idx[0]);
SPRINTF(lbuff,"%s%s",item_0,obuff);
@@ -323,7 +356,19 @@ void indwrite(char *filename, struct index *ind, int pagenum)
}
else {
index_normalize(ind[i-1].dic[0], initial_prev, &chset_prev);
- if ((CH_LATIN<=chset&&chset<=CH_GREEK) || chset==CH_HANZI) {
+ if (chset!=chset_prev && is_any_script(chset_prev) && block_open) {
+ if (strlen(script_postamble[chset_prev])) {
+ fputs(script_postamble[chset_prev],fp);
+ }
+ block_open=0;
+ }
+ if (chset!=chset_prev && is_any_script(chset)) {
+ if (strlen(script_preamble[chset])) {
+ fputs(script_preamble[chset],fp);
+ block_open=chset;
+ }
+ }
+ if ((CH_LATIN<=chset&&chset<=CH_GREEK) || chset==CH_HANZI || (CH_ARABIC<=chset&&chset<=CH_HEBREW)) {
if (chset!=chset_prev || ss_comp(initial,initial_prev)) {
fputs(group_skip,fp);
if (lethead_flag!=0) {
@@ -398,18 +443,15 @@ void indwrite(char *filename, struct index *ind, int pagenum)
}
}
else {
- if (CH_LATIN<=chset_prev&&chset_prev<=CH_THAI){
- fputs(group_skip,fp);
- if (lethead_flag!=0 && symbol_flag) {
- if (strlen(symbol)) {
- fprintf(fp,"%s%s%s",lethead_prefix,symbol,lethead_suffix);
- }
- else if (lethead_flag>0) {
- fprintf(fp,"%s%s%s",lethead_prefix,symhead_positive,lethead_suffix);
- }
- else if (lethead_flag<0) {
- fprintf(fp,"%s%s%s",lethead_prefix,symhead_negative,lethead_suffix);
- }
+ if (chset_prev!=chset) {
+ if (is_any_script(chset_prev) || symbol_flag==2)
+ fputs(group_skip,fp);
+ if (lethead_flag!=0 && symbol_flag==2 && chset==CH_NUMERIC) {
+ fprintf(fp,"%s%s%s",lethead_prefix,numhead,lethead_suffix);
+ }
+ if (lethead_flag!=0 && (symbol_flag==1 && is_any_script(chset_prev) ||
+ symbol_flag==2 && chset!=CH_NUMERIC) ) {
+ fprintf(fp,"%s%s%s",lethead_prefix,symhead,lethead_suffix);
}
}
}
@@ -479,6 +521,9 @@ void indwrite(char *filename, struct index *ind, int pagenum)
printpage(ind,fp,i,lbuff);
}
}
+ if (is_any_script(chset) && strlen(script_postamble[chset]) && block_open) {
+ fputs(script_postamble[chset],fp);
+ }
fputs(postamble,fp);
if (fp!=stdout) fclose(fp);
@@ -498,7 +543,9 @@ static void printpage(struct index *ind, FILE *fp, int num, char *lbuff)
for(j=0;j<ind[num].num;j++) {
cc=range_check(ind[num],j,lbuff);
if (cc>j) {
- if (pnumconv(ind[num].p[j].page,ind[num].p[j].attr[0])==pnumconv(ind[num].p[cc].page,ind[num].p[cc].attr[0])) {
+ int epage = pnumconv2(&ind[num].p[cc]);
+ int bpage = pnumconv2(&ind[num].p[j]);
+ if (epage==bpage) {
j=cc-1;
continue;
}
@@ -509,20 +556,18 @@ static void printpage(struct index *ind, FILE *fp, int num, char *lbuff)
if (strlen(ind[num].p[j].enc)>0) {
SPRINTF(buff,"%s%s%s",encap_prefix,ind[num].p[j].enc,encap_infix);
}
- if (strlen(suffix_3p)>0 && (pnumconv(ind[num].p[cc].page,ind[num].p[cc].attr[0])-pnumconv(ind[num].p[j].page,ind[num].p[j].attr[0]))==2) {
- SAPPENDF(buff,"%s",ind[num].p[j].page);
+ /* print beginning of range */
+ SAPPENDF(buff,"%s",ind[num].p[j].page);
+ if (strlen(suffix_3p)>0 && epage-bpage==2) {
SAPPENDF(buff,"%s",suffix_3p);
}
- else if (strlen(suffix_mp)>0 && (pnumconv(ind[num].p[cc].page,ind[num].p[cc].attr[0])-pnumconv(ind[num].p[j].page,ind[num].p[j].attr[0]))>=2) {
- SAPPENDF(buff,"%s",ind[num].p[j].page);
+ else if (strlen(suffix_mp)>0 && epage-bpage>=2) {
SAPPENDF(buff,"%s",suffix_mp);
}
- else if (strlen(suffix_2p)>0 && (pnumconv(ind[num].p[cc].page,ind[num].p[cc].attr[0])-pnumconv(ind[num].p[j].page,ind[num].p[j].attr[0]))==1) {
- SAPPENDF(buff,"%s",ind[num].p[j].page);
+ else if (strlen(suffix_2p)>0 && epage-bpage==1) {
SAPPENDF(buff,"%s",suffix_2p);
}
else {
- SAPPENDF(buff,"%s",ind[num].p[j].page);
SAPPENDF(buff,"%s",delim_r);
SAPPENDF(buff,"%s",ind[num].p[cc].page);
}
@@ -531,14 +576,14 @@ static void printpage(struct index *ind, FILE *fp, int num, char *lbuff)
if (strlen(ind[num].p[j].enc)>0) {
SAPPENDF(tmpbuff,"%s",encap_suffix);
}
- linecheck(lbuff,tmpbuff);
+ linecheck(lbuff,tmpbuff, FALSE);
j=cc;
if (j==ind[num].num) {
goto PRINT;
}
else {
SAPPENDF(tmpbuff,"%s",delim_n);
- linecheck(lbuff,tmpbuff);
+ linecheck(lbuff,tmpbuff, TRUE);
}
}
else if (strlen(ind[num].p[j].enc)>0) {
@@ -560,19 +605,19 @@ static void printpage(struct index *ind, FILE *fp, int num, char *lbuff)
SAPPENDF(tmpbuff,"%s",ind[num].p[j].page);
SAPPENDF(tmpbuff,"%s",encap_suffix);
SAPPENDF(tmpbuff,"%s",delim_n);
- linecheck(lbuff,tmpbuff);
+ linecheck(lbuff,tmpbuff, FALSE);
}
else {
SAPPENDF(tmpbuff,"%s",ind[num].p[j].page);
SAPPENDF(tmpbuff,"%s",delim_n);
- linecheck(lbuff,tmpbuff);
+ linecheck(lbuff,tmpbuff, FALSE);
}
}
else {
/* no encap */
SAPPENDF(tmpbuff,"%s",ind[num].p[j].page);
SAPPENDF(tmpbuff,"%s",delim_n);
- linecheck(lbuff,tmpbuff);
+ linecheck(lbuff,tmpbuff, FALSE);
}
}
@@ -606,7 +651,7 @@ static void printpage(struct index *ind, FILE *fp, int num, char *lbuff)
else {
SAPPENDF(tmpbuff,"%s",ind[num].p[j].page);
}
- linecheck(lbuff,tmpbuff);
+ linecheck(lbuff,tmpbuff, FALSE);
PRINT:
fputs(lbuff,fp);
@@ -652,7 +697,7 @@ static int range_check(struct index ind, int count, char *lbuff)
if (strlen(ind.p[j].enc)>0) {
SPRINTF(tmpbuff,"%s%s%s%s%s%s",encap_prefix,ind.p[j].enc,encap_infix
,ind.p[j].page,encap_suffix,delim_n);
- linecheck(lbuff,tmpbuff);
+ linecheck(lbuff,tmpbuff, FALSE);
}
}
}
@@ -682,8 +727,8 @@ static int range_check(struct index ind, int count, char *lbuff)
break;
}
}
- cc1=pnumconv(ind.p[i-1].page,ind.p[i-1].attr[0]);
- cc2=pnumconv(ind.p[count].page,ind.p[count].attr[0]);
+ cc1=pnumconv2(&ind.p[i-1]);
+ cc2=pnumconv2(&ind.p[count]);
if (cc1>=cc2+2 || (cc1>=cc2+1 && strlen(suffix_2p)) || force) {
return i-1;
}
@@ -691,9 +736,9 @@ static int range_check(struct index ind, int count, char *lbuff)
}
/* check line length */
-static void linecheck(char *lbuff, char *tmpbuff)
+static void linecheck(char *lbuff, char *tmpbuff, int force)
{
- if (line_length+strlen(tmpbuff)>line_max) {
+ if (line_length+strlen(tmpbuff)>line_max && !force) {
SAPPENDF(lbuff,"\n");
SAPPENDF(lbuff,"%s",indent_space);
SAPPENDF(lbuff,"%s",tmpbuff);
@@ -731,12 +776,20 @@ static void index_normalize(UChar *istr, UChar *ini, int *chset)
UChar32 c32;
UErrorCode perr;
UCollationResult order;
- static int hanzi_mode=0;
+ UCollationStrength strgth;
+ static int hanzi_mode=0, i_y_mode=0;
ch=istr[0];
*chset=charset(istr);
ini[1]=L'\0';
+ if (is_circkana(ch) || is_hankana(ch) || is_sqkana(ch) || is_circlatin(ch)) { /* ㋐㋑㋒.. アイウ.. ㌀㌁㌂.. */
+ src[0]=ch; src[1]=0x00;
+ perr=U_ZERO_ERROR;
+ unorm2_normalize(unormalizer_NFKD, src, 1, dest, 8, &perr);
+ if (U_SUCCESS(perr))
+ ch=dest[0];
+ }
if (is_hiragana(ch)) {
ch+=KATATOP-HIRATOP; /* hiragana -> katakana */
}
@@ -748,15 +801,15 @@ static void index_normalize(UChar *istr, UChar *ini, int *chset)
ini[0]=extkanatable[ch-EXKANATOP];
return;
}
- if (is_circkana(ch)) { /* ㋐㋑㋒㋓㋔ .. ㋻㋼㋽㋾ */
- ini[0]=circkanatable[ch-CRKANATOP];
- return;
- }
- else if (ch==0x309F) { ini[0]=0x3088; return; } /* HIRAGANA YORI -> よ */
+ if (ch==0x309F) { ini[0]=0x3088; return; } /* HIRAGANA YORI -> よ */
else if (ch==0x30FF) { ini[0]=0x3053; return; } /* KATAKANA KOTO -> こ */
else if (is_jpn_kana(istr)==2) {
c32=U16_GET_SUPPLEMENTARY(istr[0],istr[1]);
switch (c32) {
+ case 0x1F200: /* 🈀 */
+ ini[0]=0x307B; break; /* ほ */
+ case 0x1B000: /* 𛀀 */
+ ini[0]=0x3048; break; /* え */
case 0x1B150: case 0x1B164:
ini[0]=0x3090; break; /* ゐ */
case 0x1B151: case 0x1B165:
@@ -852,17 +905,29 @@ static void index_normalize(UChar *istr, UChar *ini, int *chset)
u_strcpy(ini,hz_index[lo-1].idx);
return;
}
- else if (is_devanagari(&ch)||is_thai(&ch)) {
- if (ch==0x929||0x931||0x934||(0x958<=ch&&ch<=0x95F)) {
+ else if (is_devanagari(&ch)||is_thai(&ch)||is_arabic(&ch)||is_hebrew(&ch)) {
+ if (ch==0x929||ch==0x931||ch==0x934||(0x958<=ch&&ch<=0x95F) /* Devanagary */
+ ||(0x622<=ch&&ch<=0x626)||ch==0x6C0||ch==0x6C2||ch==0x6D3 /* Arabic */
+ ||(0xFB50<=ch&&ch<=0xFDFF) /* Arabic Presentation Forms-A */
+ ||(0xFE70<=ch&&ch<=0xFEFF) /* Arabic Presentation Forms-B */
+ ||(0xFB1D<=ch&&ch<=0xFB4F) /* Hebrew presentation forms */
+ ) {
src[0]=ch; src[1]=0x00;
perr=U_ZERO_ERROR;
unorm2_normalize(unormalizer_NFD, src, 1, dest, 8, &perr);
if (U_SUCCESS(perr))
ch=dest[0]; /* without modifier */
}
+ else if (ch==0x5DA||ch==0x5DD||ch==0x05DF||ch==0x5E3||ch==0x05E5) { /* Hebrew letter final */
+ ch++;
+ }
ini[0]=ch;
return;
}
+ if (ch>=0xFF21&&ch<=0xFF3A || ch>=0xFF41&&ch<=0xFF5A) {
+ /* Fullwidth latin letter */
+ ch-=0xFF21-0x0041;
+ }
if (ch==0x049||ch==0x069||ch==0x130||ch==0x131||ch==0x0CE||ch==0x0EE) {
/* check dotted/dotless İ,I,i,ı and Î,î for Turkish */
strX[0] = 0x131; strX[1] = 0x5A; strX[2] = 0x00; /* ıZ */
@@ -874,6 +939,22 @@ static void index_normalize(UChar *istr, UChar *ini, int *chset)
return;
}
}
+ if (ch==0x059||ch==0x079) {
+ /* check Y versus I for Lithuanian */
+ if (i_y_mode==0) {
+ strgth = ucol_getStrength(icu_collator);
+ ucol_setStrength(icu_collator, UCOL_PRIMARY);
+ strX[0] = 0x059; strX[1] = 0x00; /* Y */
+ strZ[0] = 0x049; strZ[1] = 0x00; /* I */
+ order = ucol_strcoll(icu_collator, strZ, -1, strX, -1);
+ if (order==UCOL_EQUAL) i_y_mode=2; else i_y_mode=1;
+ ucol_setStrength(icu_collator, strgth);
+ }
+ if (i_y_mode==2) {
+ ini[0] = 0x049; /* I */
+ return;
+ }
+ }
if (ch==0x0C6||ch==0x0E6||ch==0x152||ch==0x153||ch==0x132||ch==0x133
||ch==0x0DF||ch==0x1E9E||ch==0x13F||ch==0x140||ch==0x490||ch==0x491) {
strX[0] = u_toupper(ch); strX[1] = 0x00; /* ex. "Æ" "Œ" */
@@ -976,6 +1057,8 @@ static void index_normalize(UChar *istr, UChar *ini, int *chset)
}
}
}
+ if (ch==0x0AA) ch=L'A';
+ if (ch==0x0BA) ch=L'O';
ini[0]=u_toupper(ch);
return;
}
@@ -1033,7 +1116,7 @@ static int init_hanzi_header(void)
len=u_strlen(pch0);
}
if (len>=INITIALLENGTH) {
- warn_printf(efp, "\nWarning, Too long (%d) hanzi header.\n", len);
+ warn_printf(efp, "\nWarning: Too long (%d) hanzi header.\n", len);
len=INITIALLENGTH-1;
}
u_strncpy(hz_index[k].idx,pch0,len);
diff --git a/indexing/upmendex/source/kana.h b/indexing/upmendex/source/kana.h
index 5c006cb7f3..41307f2cb4 100644
--- a/indexing/upmendex/source/kana.h
+++ b/indexing/upmendex/source/kana.h
@@ -192,56 +192,6 @@ UChar extkanatable[]={
0x308d, /* ろ */
0
};
-UChar circkanatable[]={
- 0x3042, /* あ */
- 0x3044, /* い */
- 0x3046, /* う */
- 0x3048, /* え */
- 0x304a, /* お */
- 0x304b, /* か */
- 0x304d, /* き */
- 0x304f, /* く */
- 0x3051, /* け */
- 0x3053, /* こ */
- 0x3055, /* さ */
- 0x3057, /* し */
- 0x3059, /* す */
- 0x305b, /* せ */
- 0x305d, /* そ */
- 0x305f, /* た */
- 0x3061, /* ち */
- 0x3064, /* つ */
- 0x3066, /* て */
- 0x3068, /* と */
- 0x306a, /* な */
- 0x306b, /* に */
- 0x306c, /* ぬ */
- 0x306d, /* ね */
- 0x306e, /* の */
- 0x306f, /* は */
- 0x3072, /* ひ */
- 0x3075, /* ふ */
- 0x3078, /* へ */
- 0x307b, /* ほ */
- 0x307e, /* ま */
- 0x307f, /* み */
- 0x3080, /* む */
- 0x3081, /* め */
- 0x3082, /* も */
- 0x3084, /* や */
- 0x3086, /* ゆ */
- 0x3088, /* よ */
- 0x3089, /* ら */
- 0x308a, /* り */
- 0x308b, /* る */
- 0x308c, /* れ */
- 0x308d, /* ろ */
- 0x308f, /* わ */
- 0x3090, /* ゐ */
- 0x3091, /* ゑ */
- 0x3092, /* を */
- 0
-};
UChar GANADA[]={
diff --git a/indexing/upmendex/source/ktable.h b/indexing/upmendex/source/ktable.h
deleted file mode 100644
index 9a3a00ef49..0000000000
--- a/indexing/upmendex/source/ktable.h
+++ /dev/null
@@ -1,98 +0,0 @@
-UChar symboltable[]={
- 0x3000, /* スペース */
- 0xff01, /* ! */
- 0xff02, /* ” */
- 0xff03, /* # */
- 0xff04, /* $ */
- 0xff05, /* % */
- 0xff06, /* & */
- 0xff07, /* ’ */
- 0xff08, /* ( */
- 0xff09, /* ) */
- 0xff0a, /* * */
- 0xff0b, /* + */
- 0xff0c, /* , */
- 0xff0d, /* − */
- 0xff0e, /* . */
- 0xff0f, /* / */
- 0xff10, /* 0 */
- 0xff11, /* 1 */
- 0xff12, /* 2 */
- 0xff13, /* 3 */
- 0xff14, /* 4 */
- 0xff15, /* 5 */
- 0xff16, /* 6 */
- 0xff17, /* 7 */
- 0xff18, /* 8 */
- 0xff19, /* 9 */
- 0xff1a, /* : */
- 0xff1b, /* ; */
- 0xff1c, /* < */
- 0xff1d, /* = */
- 0xff1e, /* > */
- 0xff1f, /* ? */
- 0xff20, /* @ */
- 0xff21, /* A */
- 0xff22, /* B */
- 0xff23, /* C */
- 0xff24, /* D */
- 0xff25, /* E */
- 0xff26, /* F */
- 0xff27, /* G */
- 0xff28, /* H */
- 0xff29, /* I */
- 0xff2a, /* J */
- 0xff2b, /* K */
- 0xff2c, /* L */
- 0xff2d, /* M */
- 0xff2e, /* N */
- 0xff2f, /* O */
- 0xff30, /* P */
- 0xff31, /* Q */
- 0xff32, /* R */
- 0xff33, /* S */
- 0xff34, /* T */
- 0xff35, /* U */
- 0xff36, /* V */
- 0xff37, /* W */
- 0xff38, /* X */
- 0xff39, /* Y */
- 0xff3a, /* Z */
- 0xff3b, /* [ */
- 0xffe5, /* ¥ */
- 0xff3d, /* ] */
- 0xff3e, /* ^ */
- 0xff3f, /* _ */
- 0xff40, /* ‘ */
- 0xff41, /* a */
- 0xff42, /* b */
- 0xff43, /* c */
- 0xff44, /* d */
- 0xff45, /* e */
- 0xff46, /* f */
- 0xff47, /* g */
- 0xff48, /* h */
- 0xff49, /* i */
- 0xff4a, /* j */
- 0xff4b, /* k */
- 0xff4c, /* l */
- 0xff4d, /* m */
- 0xff4e, /* n */
- 0xff4f, /* o */
- 0xff50, /* p */
- 0xff51, /* q */
- 0xff52, /* r */
- 0xff53, /* s */
- 0xff54, /* t */
- 0xff55, /* u */
- 0xff56, /* v */
- 0xff57, /* w */
- 0xff58, /* x */
- 0xff59, /* y */
- 0xff5a, /* z */
- 0xff5b, /* { */
- 0xff5c, /* | */
- 0xff5d, /* } */
- 0xff5f, /* 〜 */
- 0
-};
diff --git a/indexing/upmendex/source/mendex.h b/indexing/upmendex/source/mendex.h
index c6703761fc..091862d61e 100644
--- a/indexing/upmendex/source/mendex.h
+++ b/indexing/upmendex/source/mendex.h
@@ -11,10 +11,12 @@
#include <unicode/uchar.h>
#include <unicode/ucol.h>
+#define PAGE_COMPOSIT_DEPTH 10
+
struct page {
char *page;
char *enc;
- int attr[3];
+ int attr[PAGE_COMPOSIT_DEPTH];
};
struct index {
@@ -47,7 +49,6 @@ int lastpage(const char *filename);
/* sort.c */
void wsort(struct index *ind, int num);
void pagesort(struct index *ind, int num);
-int is_alphanumeric(UChar *c);
int is_latin(UChar *c);
int is_numeric(UChar *c);
int is_jpn_kana(UChar *c);
@@ -58,7 +59,10 @@ int is_cyrillic(UChar *c);
int is_greek(UChar *c);
int is_devanagari(UChar *c);
int is_thai(UChar *c);
-int is_comb_diacritical_mark(UChar *c);
+int is_arabic(UChar *c);
+int is_hebrew(UChar *c);
+int is_type_mark_or_punct(UChar *c);
+int is_type_symbol(UChar *c);
int chkcontinue(struct page *p, int num);
int ss_comp(UChar *s1, UChar *s2);
@@ -71,8 +75,11 @@ int ss_comp(UChar *s1, UChar *s2);
#define CH_HANZI 6
#define CH_DEVANAGARI 7
#define CH_THAI 8
+#define CH_ARABIC 9
+#define CH_HEBREW 10
#define CH_SYMBOL 0x100
#define CH_NUMERIC 0x101
+#define is_any_script(a) ((CH_LATIN<=(a) && (a)<=CH_HEBREW))
/* sort.c */
int charset(UChar *c);
diff --git a/indexing/upmendex/source/sort.c b/indexing/upmendex/source/sort.c
index e3a47c12b7..bcecd765cf 100644
--- a/indexing/upmendex/source/sort.c
+++ b/indexing/upmendex/source/sort.c
@@ -4,7 +4,23 @@
#include "exkana.h"
#include "exvar.h"
-int sym,nmbr,ltn,kana,hngl,hnz,cyr,grk,dvng,thai;
+#define RULEBUFSIZE 29210+STYBUFSIZE
+/*
+ length of collation rule in ICU 68.2
+
+ icu_locale length
+ ja 6410
+ ja@collation=unihan 61
+ ko 12577
+ ko@collation=unihan 51
+ ko@collation=search 782
+ zh (pinin) 26909
+ zh@collation=unihan 82
+ zh@collation=stroke 29208
+ zh@collation=zhuyin 28880
+*/
+
+int sym,nmbr,ltn,kana,hngl,hnz,cyr,grk,dvng,thai,arab,hbrw;
static int wcomp(const void *p, const void *q);
static int pcomp(const void *p, const void *q);
@@ -17,7 +33,9 @@ void wsort(struct index *ind, int num)
{
int i,order;
UErrorCode status;
- UChar rules[STYBUFSIZE];
+ UParseError parse_error;
+ UChar rules[RULEBUFSIZE] = {'\0'};
+ int32_t len;
for (order=1,i=0;;i++) {
switch (character_order[i]) {
@@ -65,7 +83,16 @@ void wsort(struct index *ind, int num)
thai=order++;
break;
+ case 'a':
+ arab=order++;
+ break;
+
+ case 'h':
+ hbrw=order++;
+ break;
+
default:
+ verb_printf(efp,"\nWarning: Illegal input for character_order (%c).",character_order[i]);
break;
}
}
@@ -82,11 +109,28 @@ BREAK:
if (grk==0) grk=order++;
if (dvng==0) dvng=order++;
if (thai==0) thai=order++;
+ if (arab==0) arab=order++;
+ if (hbrw==0) hbrw=order++;
status = U_ZERO_ERROR;
if (strlen(icu_rules)>0) {
+ if (strcmp(icu_locale,"root")!=0) {
+ icu_collator = ucol_open(icu_locale, &status);
+ if (U_FAILURE(status)) {
+ verb_printf(efp, "\n[ICU] Collator creation failed.: %s\n", u_errorName(status));
+ exit(254);
+ }
+ len = ucol_getRulesEx(icu_collator, UCOL_TAILORING_ONLY, rules, RULEBUFSIZE);
+ if (u_strlen(rules)<len) {
+ verb_printf(efp, "\n[ICU] Failed to extract collation rules by locale (%s). Need buffer size %d.\n",
+ icu_locale, len);
+ exit(254);
+ }
+ ucol_close(icu_collator);
+ }
unescape((unsigned char *)icu_rules, rules);
- icu_collator = ucol_openRules(rules, -1, UCOL_OFF, UCOL_TERTIARY, NULL, &status);
+ status = U_ZERO_ERROR;
+ icu_collator = ucol_openRules(rules, -1, UCOL_OFF, UCOL_TERTIARY, &parse_error, &status);
} else
icu_collator = ucol_open(icu_locale, &status);
if (U_FAILURE(status)) {
@@ -94,11 +138,11 @@ BREAK:
exit(254);
}
if (status == U_USING_DEFAULT_WARNING) {
- warn_printf(efp, "\nWarning, [ICU] U_USING_DEFAULT_WARNING for locale %s\n",
+ warn_printf(efp, "\nWarning: [ICU] U_USING_DEFAULT_WARNING for locale %s\n",
icu_locale);
}
if (status == U_USING_FALLBACK_WARNING) {
- warn_printf(efp, "\nWarning, [ICU] U_USING_FALLBACK_WARNING for locale %s\n",
+ warn_printf(efp, "\nWarning: [ICU] U_USING_FALLBACK_WARNING for locale %s\n",
icu_locale);
}
for (i=0;i<UCOL_ATTRIBUTE_COUNT;i++) {
@@ -107,7 +151,7 @@ BREAK:
ucol_setAttribute(icu_collator, i, icu_attributes[i], &status);
}
if (U_FAILURE(status)) {
- warn_printf(efp, "\nWarning, [ICU] Failed to set attribute (%d): %s\n",
+ warn_printf(efp, "\nWarning: [ICU] Failed to set attribute (%d): %s\n",
i, u_errorName(status));
}
}
@@ -214,12 +258,12 @@ void pagesort(struct index *ind, int num)
static int pcomp(const void *p, const void *q)
{
int i,j,cc=0,num1,num2;
- char buff[16];
+ char buff[16],*p0,*p1;
const struct page *page1 = p, *page2 = q;
scount++;
- for (i=0;i<3;i++) {
+ for (i=0;i<PAGE_COMPOSIT_DEPTH;i++) {
if ((page1->attr[i]<0)&&(page2->attr[i]<0)) return 0;
else if ((page1->attr[i]<0)&&(page2->attr[i]>=0)) return -1;
else if ((page2->attr[i]<0)&&(page1->attr[i]>=0)) return 1;
@@ -227,18 +271,18 @@ static int pcomp(const void *p, const void *q)
if (page1->attr[i]>page2->attr[i]) return 1;
if (page1->attr[i]<page2->attr[i]) return -1;
- for (j=cc;j<strlen(page1->page);j++) {
- if (strncmp(&page1->page[j],page_compositor,strlen(page_compositor))==0) break;
- }
- strncpy(buff,&page1->page[cc],j-cc);
- buff[j-cc]='\0';
+ p0=&page1->page[cc];
+ p1=strstr(p0, page_compositor);
+ j=p1 ? p1-p0 : strlen(p0);
+ strncpy(buff,p0,j);
+ buff[j]='\0';
num1=pnumconv(buff,page1->attr[i]);
- for (j=cc;j<strlen(page2->page);j++) {
- if (strncmp(&page2->page[j],page_compositor,strlen(page_compositor))==0) break;
- }
- strncpy(buff,&page2->page[cc],j-cc);
- buff[j-cc]='\0';
+ p0=&page2->page[cc];
+ p1=strstr(p0, page_compositor);
+ j=p1 ? p1-p0 : strlen(p0);
+ strncpy(buff,p0,j);
+ buff[j]='\0';
num2=pnumconv(buff,page2->attr[i]);
if (num1>num2) return 1;
@@ -247,7 +291,8 @@ static int pcomp(const void *p, const void *q)
if (page1->enc[0]=='(' || page2->enc[0]==')') return -1;
if (page1->enc[0]==')' || page2->enc[0]=='(') return 1;
- cc=j+strlen(page_compositor);
+ if (p1) cc+=j+strlen(page_compositor);
+ else return 0;
}
return 0;
@@ -255,13 +300,15 @@ static int pcomp(const void *p, const void *q)
static int ordering(UChar *c)
{
- if (*c<0x80) {
- if (is_latin(c)) return ltn;
- else if (is_numeric(c)) return nmbr;
- else return sym;
+ if (*c<0x20) return sym; /* control */
+ else if (*c<0x7F) {
+ if (is_latin(c)) return ltn;
+ else if (is_numeric(c)) return nmbr;
+ else return sym;
}
+ else if (*c<0xA0) return sym; /* control */
else {
- if (is_latin(c)) return ltn;
+ if (is_latin(c)) return ltn;
else if (is_jpn_kana(c)) return kana;
else if (is_kor_hngl(c)) return hngl;
else if (is_hanzi(c)) return hnz;
@@ -270,20 +317,23 @@ static int ordering(UChar *c)
else if (is_numeric(c)) return nmbr;
else if (is_devanagari(c)) return dvng;
else if (is_thai(c)) return thai;
- else return sym;
+ else if (is_arabic(c)) return arab;
+ else if (is_hebrew(c)) return hbrw;
+ else return sym;
}
}
int charset(UChar *c)
{
- if (*c==0x00) return CH_UNKNOWN;
- else if (*c<0x80) {
- if (is_latin(c)) return CH_LATIN;
- else if (is_numeric(c)) return CH_NUMERIC;
- else return CH_SYMBOL;
+ if (*c<0x20) return CH_UNKNOWN; /* control */
+ else if (*c<0x7F) {
+ if (is_latin(c)) return CH_LATIN;
+ else if (is_numeric(c)) return CH_NUMERIC;
+ else return CH_SYMBOL;
}
+ else if (*c<0xA0) return CH_UNKNOWN; /* control */
else {
- if (is_latin(c)) return CH_LATIN;
+ if (is_latin(c)) return CH_LATIN;
else if (is_jpn_kana(c)) return CH_KANA;
else if (is_kor_hngl(c)) return CH_HANGUL;
else if (is_hanzi(c)) return CH_HANZI;
@@ -292,7 +342,9 @@ int charset(UChar *c)
else if (is_numeric(c)) return CH_NUMERIC;
else if (is_devanagari(c)) return CH_DEVANAGARI;
else if (is_thai(c)) return CH_THAI;
- else return CH_SYMBOL;
+ else if (is_arabic(c)) return CH_ARABIC;
+ else if (is_hebrew(c)) return CH_HEBREW;
+ else return CH_SYMBOL;
}
}
@@ -312,10 +364,10 @@ static int get_charset_juncture(UChar *str)
else l = k-1;
chset_l=charset(&str[l]);
chset_k=charset(&str[k]);
- if (chset0==CH_UNKNOWN && chset_l!=CH_SYMBOL && chset_l!=CH_NUMERIC) {
+ if (chset0==CH_UNKNOWN && is_any_script(chset_l)) {
chset0=chset_l;
}
- if (chset_k!=CH_SYMBOL && chset_k!=CH_NUMERIC) {
+ if (chset0!=CH_UNKNOWN && is_any_script(chset_k)) {
if (chset0!=chset_k) {
len=k;
return len;
@@ -324,10 +376,11 @@ static int get_charset_juncture(UChar *str)
}
}
-static int unescape(const unsigned char *src, UChar *dist)
+static int unescape(const unsigned char *src, UChar *dest)
{
- int i,j,k;
+ int i,j,k,ret;
char tmp[STYBUFSIZE];
+ UErrorCode status;
for (i=j=0;i<STYBUFSIZE;i++) {
if (src[i]=='\0') {
@@ -336,32 +389,37 @@ static int unescape(const unsigned char *src, UChar *dist)
else if (src[i]< 0x80 && (src[i+1]>=0x80 || src[i+1]=='\0')) {
strncpy(tmp,(char *)&src[j],i-j+1);
tmp[i-j+1]='\0';
- k=u_strlen(dist);
- u_unescape(tmp, &dist[k], STYBUFSIZE-k);
+ k=u_strlen(dest);
+ ret=u_unescape(tmp, &dest[k], RULEBUFSIZE-k);
+ if (ret==0) {
+ verb_printf(efp, "\n[ICU] Escape sequence in input seems malformed.\n");
+ exit(254);
+ }
j=i+1;
}
else if (src[i]>=0x80 && (src[i+1]< 0x80 || src[i+1]=='\0')) {
strncpy(tmp,(char *)&src[j],i-j+1);
tmp[i-j+1]='\0';
- k=u_strlen(dist);
- multibyte_to_widechar(&dist[k], STYBUFSIZE-k, tmp);
+ k=u_strlen(dest);
+ status=U_ZERO_ERROR;
+ u_strFromUTF8(&dest[k], RULEBUFSIZE-k, NULL, tmp, -1, &status);
+ if (U_FAILURE(status)) {
+ verb_printf(efp, "\n[ICU] Input string seems malformed.: %s\n", u_errorName(status));
+ exit(254);
+ }
j=i+1;
}
}
return -1;
}
-int is_alphanumeric(UChar *c)
-{
- if (((*c>=L'A')&&(*c<=L'Z'))||((*c>=L'a')&&(*c<=L'z'))||((*c>=L'0')&&(*c<=L'9')))
- return 1;
- else return 0;
-}
-
int is_latin(UChar *c)
{
+ UChar32 c32;
+
if (((*c>=L'A')&&(*c<=L'Z'))||((*c>=L'a')&&(*c<=L'z'))) return 1;
- else if ((*c>=0x00C0)&&(*c<=0x00D6)) return 1; /* Latin-1 Supplement */
+ else if ((*c==0x00AA)||(*c==0x00BA)) return 1; /* Latin-1 Supplement */
+ else if ((*c>=0x00C0)&&(*c<=0x00D6)) return 1;
else if ((*c>=0x00D8)&&(*c<=0x00F6)) return 1;
else if ((*c>=0x00F8)&&(*c<=0x00FF)) return 1;
else if ((*c>=0x0100)&&(*c<=0x024F)) return 1; /* Latin Extended-A,B */
@@ -371,28 +429,63 @@ int is_latin(UChar *c)
else if ((*c>=0xAB30)&&(*c<=0xAB6F)) return 1; /* Latin Extended-E */
else if ((*c>=0x1E00)&&(*c<=0x1EFF)) return 1; /* Latin Extended Additional */
else if ((*c>=0xFB00)&&(*c<=0xFB06)) return 1; /* Latin ligatures */
- else return 0;
+ else if ((*c>=0xFF21)&&(*c<=0xFF3A)) return 1; /* Fullwidth Latin Capital Letter */
+ else if ((*c>=0xFF41)&&(*c<=0xFF5A)) return 1; /* Fullwidth Latin Small Letter */
+ /* Property of followings is "Common, So (other symbol)", but seem to be treated as Latin by ICU collator */
+ else if ((*c>=0x24B6) /* CIRCLED LATIN CAPITAL LETTER */
+ &&(*c<=0x24E9)) return 1; /* CIRCLED LATIN SMALL LETTER */
+
+ if (is_surrogate_pair(c)) {
+ c32=U16_GET_SUPPLEMENTARY(*c,*(c+1));
+ if ((c32>=0x10780) && (c32<=0x107BF)) return 2; /* Latin Extended-F */
+ else if ((c32>=0x1DF00) && (c32<=0x1DFFF)) return 2; /* Latin Extended-G */
+ }
+ return 0;
}
int is_numeric(UChar *c)
{
+ UChar32 c32;
+
if ((*c>=L'0')&&(*c<=L'9')) return 1;
- else if ((*c>=0x0966)&&(*c<=0x096F)) return 1; /* Devanagari Digit */
- else if ((*c>=0x0E50)&&(*c<=0x0E59)) return 1; /* Thai Digit */
- else return 0;
+ else if ((*c>=0xFF10)&&(*c<=0xFF19)) return 1; /* Fullwidth Digit */
+ /* followings do not seem to be treated as numbers by ICU collator though charType is U_OTHER_NUMBER */
+ else if ((*c>=0x3192)&&(*c<=0x3195)) return 0; /* IDEOGRAPHIC ANNOTATION ONE MARK..IDEOGRAPHIC ANNOTATION FOUR MARK */
+ else if ((*c>=0x3220)&&(*c<=0x3229)) return 0; /* PARENTHESIZED IDEOGRAPH ONE..PARENTHESIZED IDEOGRAPH TEN */
+ else if ((*c>=0x3280)&&(*c<=0x3289)) return 0; /* CIRCLED IDEOGRAPH ONE..CIRCLED IDEOGRAPH TEN */
+ else if ((*c>=0xA830)&&(*c<=0xA835)) return 0; /* NORTH INDIC FRACTION ONE QUARTER..NORTH INDIC FRACTION THREE SIXTEENTHS */
+
+ if (is_surrogate_pair(c))
+ c32=U16_GET_SUPPLEMENTARY(*c,*(c+1));
+ else c32=*c;
+
+ switch (u_charType(c32)) {
+ case U_DECIMAL_DIGIT_NUMBER:
+ return 1;
+ case U_OTHER_NUMBER:
+ return 2;
+ default:
+ return 0;
+ }
}
int is_jpn_kana(UChar *c)
{
UChar32 c32;
- if ((*c>=0x3040)&&(*c<=0x30FF)) return 1; /* Hiragana, Katakana */
+ if (*c==0x30A0) return 0; /* KATAKANA-HIRAGANA DOUBLE HYPHEN */
+ else if (*c==0x30FB) return 0; /* KATAKANA MIDDLE DOT */
+ else if ((*c>=0x3040)&&(*c<=0x30FF)) return 1; /* Hiragana, Katakana */
else if ((*c>=0x31F0)&&(*c<=0x31FF)) return 1; /* Katakana Phonetic Extensions */
else if ((*c>=0x32D0)&&(*c<=0x32FE)) return 1; /* Circled Katakana */
+ else if ((*c>=0xFF66)&&(*c<=0xFF9F)) return 1; /* Halfwidth Katakana */
+ else if ((*c>=0x3300)&&(*c<=0x3357)) return 1; /* Squared Katakana words */
if (is_surrogate_pair(c)) {
c32=U16_GET_SUPPLEMENTARY(*c,*(c+1));
if ((c32>=0x1B130) && (c32<=0x1B16F)) return 2; /* Small Kana Extensions */
+ else if ((c32==0x1B000)) return 2; /* KATAKANA LETTER ARCHAIC E */
+ else if ((c32==0x1F200)) return 2; /* SQUARE HIRAGANA HOKA */
}
return 0;
/* ICU 65 does not seem to support
@@ -416,11 +509,11 @@ int is_hanzi(UChar *c)
{
UChar32 c32;
- if ((*c>=0x2E80)&&(*c<=0x2EFF)) return 1; /* CJK Radicals Supplement */
- else if ((*c>=0x2F00)&&(*c<=0x2FDF)) return 1; /* Kangxi Radicals */
+ if ((*c>=0x2E80) /* CJK Radicals Supplement */
+ &&(*c<=0x2FDF)) return 1; /* Kangxi Radicals */
else if ((*c>=0x31C0)&&(*c<=0x31EF)) return 1; /* CJK Strokes */
- else if ((*c>=0x3300)&&(*c<=0x33FF)) return 1; /* CJK Compatibility */
- else if ((*c>=0x3400)&&(*c<=0x4DBF)) return 1; /* CJK Unified Ideographs Extension A */
+ else if ((*c>=0x3300) /* CJK Compatibility */
+ &&(*c<=0x4DBF)) return 1; /* CJK Unified Ideographs Extension A */
else if ((*c>=0x4E00)&&(*c<=0x9FFF)) return 1; /* CJK Unified Ideographs */
else if ((*c>=0xF900)&&(*c<=0xFAFF)) return 1; /* CJK Compatibility Ideographs */
@@ -442,7 +535,9 @@ int is_zhuyin(UChar *c)
int is_cyrillic(UChar *c)
{
- if ((*c>=0x0400)&&(*c<=0x052F)) return 1; /* Cyrillic, Cyrillic Supplement */
+ if ((*c==0x0482)) return 0; /* Cyrillic Thousands Sign */
+ else if ((*c>=0x0400) /* Cyrillic */
+ &&(*c<=0x052F)) return 1; /* Cyrillic Supplement */
else if ((*c>=0x1C80)&&(*c<=0x1C8F)) return 1; /* Cyrillic Extended-C */
else if ((*c>=0x2DE0)&&(*c<=0x2DFF)) return 1; /* Cyrillic Extended-A */
else if ((*c>=0xA640)&&(*c<=0xA69F)) return 1; /* Cyrillic Extended-B */
@@ -451,14 +546,16 @@ int is_cyrillic(UChar *c)
int is_greek(UChar *c)
{
- if ((*c>=0x0370)&&(*c<=0x03FF)) return 1; /* Greek */
+ if ((*c==0x03F6)) return 0; /* Greek Reversed Lunate Epsilon Symbol */
+ else if ((*c>=0x0370)&&(*c<=0x03FF)) return 1; /* Greek */
else if ((*c>=0x1F00)&&(*c<=0x1FFF)) return 1; /* Greek Extended */
else return 0;
}
int is_devanagari(UChar *c)
{
- if ((*c>=0x0966)&&(*c<=0x096F)) return 0; /* Devanagari Digit */
+ if ((*c>=0x0964) /* Generic punctuation for scripts of India */
+ &&(*c<=0x096F)) return 0; /* Devanagari Digit */
else if ((*c>=0x0900)&&(*c<=0x097F)) return 1; /* Devanagari */
else if ((*c>=0xA8E0)&&(*c<=0xA8FF)) return 1; /* Devanagari Extended */
else return 0;
@@ -472,47 +569,132 @@ int is_thai(UChar *c)
else return 0;
}
-int is_comb_diacritical_mark(UChar *c)
+int is_arabic(UChar *c)
+{
+ if ((*c>=0x0600) /* ARABIC NUMBER SIGN..ARABIC SIGN SAMVAT */
+ /* ARABIC NUMBER MARK ABOVE */
+ &&(*c<=0x0608)) return 0; /* ARABIC-INDIC CUBE ROOT..ARABIC RAY */
+ else if ((*c==0x060B)) return 0; /* AFGHANI SIGN */
+ else if ((*c==0x060C)) return 0; /* ARABIC COMMA */
+ else if ((*c>=0x060E)&&(*c<=0x060F)) return 0; /* ARABIC POETIC VERSE SIGN..ARABIC SIGN MISRA */
+ else if ((*c>=0x0660)&&(*c<=0x0669)) return 0; /* ARABIC-INDIC DIGIT ZERO..ARABIC-INDIC DIGIT NINE */
+ else if ((*c==0x061B)) return 0; /* ARABIC SEMICOLON */
+ else if ((*c==0x061C)) return 0; /* ARABIC LETTER MARK */
+ else if ((*c==0x061F)) return 0; /* ARABIC QUESTION MARK */
+ else if ((*c==0x0640)) return 0; /* ARABIC TATWEEL */
+ else if ((*c==0x06DD)) return 0; /* ARABIC END OF AYAH */
+ else if ((*c==0x06DE)) return 0; /* ARABIC START OF RUB EL HIZB */
+ else if ((*c==0x06E9)) return 0; /* ARABIC PLACE OF SAJDAH */
+ else if ((*c>=0x06F0)&&(*c<=0x06F9)) return 0; /* EXTENDED ARABIC-INDIC DIGIT ZERO..EXTENDED ARABIC-INDIC DIGIT NINE */
+ else if ((*c>=0x06FD)&&(*c<=0x06FE)) return 0; /* ARABIC SIGN SINDHI AMPERSAND..ARABIC SIGN SINDHI POSTPOSITION MEN */
+ else if ((*c==0x08E2)) return 0; /* ARABIC DISPUTED END OF AYAH */
+ else if ((*c>=0x0890)&&(*c<=0x0891)) return 0; /* ARABIC POUND MARK ABOVE..ARABIC PIASTRE MARK ABOVE */
+ else if ((*c>=0xFD40)&&(*c<=0xFD4F)) return 0; /* ARABIC LIGATURE RAHIMAHU ALLAAH..ARABIC LIGATURE RAHIMAHUM ALLAAH */
+ else if ((*c==0xFDCF)) return 0; /* ARABIC LIGATURE SALAAMUHU ALAYNAA */
+ else if ((*c==0xFDFC)) return 0; /* RIAL SIGH */
+ else if ((*c>=0xFDFD)&&(*c<=0xFDFF)) return 0; /* ARABIC LIGATURE BISMILLAH AR-RAHMAN AR-RAHEEM..ARABIC LIGATURE AZZA WA JALL */
+
+ else if ((*c>=0x0600)&&(*c<=0x06FF)) return 1; /* Arabic */
+ else if ((*c>=0x0750)&&(*c<=0x077F)) return 1; /* Arabic Supplement */
+ else if ((*c>=0x0870) /* Arabic Extended-B */
+ &&(*c<=0x08FF)) return 1; /* Arabic Extended-A */
+ else if ((*c>=0xFB50)&&(*c<=0xFDFF)) return 1; /* Arabic Presentation Forms-A */
+ else if ((*c>=0xFE70)&&(*c<=0xFEFF)) return 1; /* Arabic Presentation Forms-B */
+ else return 0;
+}
+
+int is_hebrew(UChar *c)
{
- if ((*c>=0x02B0)&&(*c<=0x02FF)) return 1; /* Spacing Modifier Letters */
- else if ((*c>=0x0300)&&(*c<=0x036F)) return 1; /* Combining Diacritical Marks */
- else if ((*c>=0x1DC0)&&(*c<=0x1DFF)) return 1; /* Combining Diacritical Marks Supplement */
- else if ((*c>=0x1AB0)&&(*c<=0x1AFF)) return 1; /* Combining Diacritical Marks Extended */
- else if ((*c>=0x3099)&&(*c<=0x309A)) return 1; /* Combining Kana Voiced Sound Marks */
+ if ((*c==0xFB29)) return 0; /* Hebrew Letter Alternative Plus Sign */
+ else if ((*c>=0x0590)&&(*c<=0x05FF)) return 1; /* Hebrew */
+ else if ((*c>=0xFB1D)&&(*c<=0xFB4F)) return 1; /* Hebrew presentation forms */
else return 0;
}
+int is_type_mark_or_punct(UChar *c)
+{
+ UChar32 c32;
+
+ if (is_surrogate_pair(c))
+ c32=U16_GET_SUPPLEMENTARY(*c,*(c+1));
+ else c32=*c;
+
+ switch (u_charType(c32)) {
+ case U_MODIFIER_LETTER:
+ case U_DASH_PUNCTUATION: case U_START_PUNCTUATION: case U_END_PUNCTUATION:
+ case U_CONNECTOR_PUNCTUATION: case U_OTHER_PUNCTUATION:
+ case U_INITIAL_PUNCTUATION: case U_FINAL_PUNCTUATION:
+ case U_NON_SPACING_MARK: case U_ENCLOSING_MARK: case U_COMBINING_SPACING_MARK:
+ case U_FORMAT_CHAR:
+ return 1;
+ default:
+ return 0;
+ }
+}
+
+int is_type_symbol(UChar *c)
+{
+ UChar32 c32;
+
+ if (is_surrogate_pair(c))
+ c32=U16_GET_SUPPLEMENTARY(*c,*(c+1));
+ else c32=*c;
+
+ switch (u_charType(c32)) {
+ case U_MODIFIER_SYMBOL:
+ return 1;
+ case U_MATH_SYMBOL: case U_CURRENCY_SYMBOL:
+ case U_OTHER_SYMBOL:
+ return 2;
+ default:
+ return 0;
+ }
+}
+
int chkcontinue(struct page *p, int num)
{
- int i,j,cc=0,num1,num2;
- char buff[16];
+ int i,j,cc=0,num1,num2,k1,k2;
+ char buff1[16],buff2[16],*p0,*p1;
- for (i=0;i<3;i++) {
+ for (i=0;i<PAGE_COMPOSIT_DEPTH;i++) {
if ((p[num].attr[i]<0)&&(p[num+1].attr[i]<0)) return 1;
else if (p[num].attr[i]!=p[num+1].attr[i]) return 0;
- for (j=cc;j<strlen(p[num].page);j++) {
- if (strncmp(&p[num].page[j],page_compositor,strlen(page_compositor))==0) break;
+ p0=&p[num].page[cc];
+ p1=strstr(p0, page_compositor);
+ if (p1) {
+ j=p1-p0;
+ k1=j;
+ } else {
+ j=strlen(p0);
+ k1=0;
}
- strncpy(buff,&p[num].page[cc],j);
- buff[j]='\0';
- num1=pnumconv(buff,p[num].attr[i]);
-
- for (j=cc;j<strlen(p[num+1].page);j++) {
- if (strncmp(&p[num+1].page[j],page_compositor,strlen(page_compositor))==0) break;
+ strncpy(buff1,p0,j);
+ buff1[j]='\0';
+ num1=pnumconv(buff1,p[num].attr[i]);
+
+ p0=&p[num+1].page[cc];
+ p1=strstr(p0, page_compositor);
+ if (p1) {
+ j=p1-p0;
+ k2=j;
+ } else {
+ j=strlen(p0);
+ k2=0;
}
- strncpy(buff,&p[num+1].page[cc],j);
- buff[j]='\0';
- num2=pnumconv(buff,p[num+1].attr[i]);
-
- if (num1==num2 || num1+1==num2) {
- if (i==2) return 1;
- if ((p[num].attr[i+1]<0)&&(p[num+1].attr[i+1]<0)) return 1;
- else return 0;
+ strncpy(buff2,p0,j);
+ buff2[j]='\0';
+ num2=pnumconv(buff2,p[num+1].attr[i]);
+
+ if (k1>0 || k2>0) {
+ if (k1!=k2) return 0;
+ if (strcmp(buff1,buff2)) return 0;
+ cc+=k1+strlen(page_compositor);
+ continue;
}
- else if (num1!=num2) return 0;
- cc=j+strlen(page_compositor);
+ if (num1==num2 || num1+1==num2) return 1;
+ else return 0;
}
return 1;
diff --git a/indexing/upmendex/source/styfile.c b/indexing/upmendex/source/styfile.c
index 619e4a5625..f029679049 100644
--- a/indexing/upmendex/source/styfile.c
+++ b/indexing/upmendex/source/styfile.c
@@ -76,13 +76,9 @@ void styread(const char *filename)
if (getparam(buff,"symhead_negative",symhead_negative)) continue;
if (getparam(buff,"numhead_positive",numhead_positive)) continue;
if (getparam(buff,"numhead_negative",numhead_negative)) continue;
- cc=scompare(buff,"lethead_flag");
- if (cc!= -1) {
- lethead_flag=atoi(&buff[cc]);
- continue;
- }
- cc=scompare(buff,"heading_flag");
- if (cc!= -1) {
+ if ( (cc=scompare(buff,"lethead_flag")) != -1 ||
+ (cc=scompare(buff,"heading_flag")) != -1 ||
+ (cc=scompare(buff,"headings_flag")) != -1 ) {
lethead_flag=atoi(&buff[cc]);
continue;
}
@@ -121,7 +117,7 @@ void styread(const char *filename)
indent_length=atoi(&buff[cc]);
continue;
}
- if (getparam(buff,"symbol",symbol)) continue;
+ if (getparam(buff,"symbol",symhead)) continue;
cc=scompare(buff,"symbol_flag");
if (cc!= -1) {
symbol_flag=atoi(&buff[cc]);
@@ -164,6 +160,50 @@ void styread(const char *filename)
continue;
}
if (getparam(buff,"icu_attributes", icu_attr_str )) continue;
+
+ cc=scompare(buff,"script_preamble");
+ if (cc!= -1) {
+ strcpy(tmp,buff+strlen("script_preamble"));
+ if (getparam(tmp,"latin", script_preamble[CH_LATIN] )) continue;
+ if (getparam(tmp,"cyrillic", script_preamble[CH_CYRILLIC] )) continue;
+ if (getparam(tmp,"greek", script_preamble[CH_GREEK] )) continue;
+ if (getparam(tmp,"kana", script_preamble[CH_KANA] )) continue;
+ if (getparam(tmp,"hangul", script_preamble[CH_HANGUL] )) continue;
+ if (getparam(tmp,"hanzi", script_preamble[CH_HANZI] )) continue;
+ if (getparam(tmp,"devanagari",script_preamble[CH_DEVANAGARI] )) continue;
+ if (getparam(tmp,"thai", script_preamble[CH_THAI] )) continue;
+ if (getparam(tmp,"arabic", script_preamble[CH_ARABIC] )) continue;
+ if (getparam(tmp,"hebrew", script_preamble[CH_HEBREW] )) continue;
+ if (strlen(tmp)>0) {
+ verb_printf(efp,"\nWarning: Unknown script for specifier \"script_preamble\" (%s).", tmp);
+ }
+ continue;
+ }
+ cc=scompare(buff,"script_postamble");
+ if (cc!= -1) {
+ strcpy(tmp,buff+strlen("script_postamble"));
+ if (getparam(tmp,"latin", script_postamble[CH_LATIN] )) continue;
+ if (getparam(tmp,"cyrillic", script_postamble[CH_CYRILLIC] )) continue;
+ if (getparam(tmp,"greek", script_postamble[CH_GREEK] )) continue;
+ if (getparam(tmp,"kana", script_postamble[CH_KANA] )) continue;
+ if (getparam(tmp,"hangul", script_postamble[CH_HANGUL] )) continue;
+ if (getparam(tmp,"hanzi", script_postamble[CH_HANZI] )) continue;
+ if (getparam(tmp,"devanagari",script_postamble[CH_DEVANAGARI] )) continue;
+ if (getparam(tmp,"thai", script_postamble[CH_THAI] )) continue;
+ if (getparam(tmp,"arabic", script_postamble[CH_ARABIC] )) continue;
+ if (getparam(tmp,"hebrew", script_postamble[CH_HEBREW] )) continue;
+ if (strlen(tmp)>0) {
+ verb_printf(efp,"\nWarning: Unknown script for specifier \"script_postamble\" (%s).", tmp);
+ }
+ continue;
+ }
+
+ cc=strcspn(buff," \t\r\n");
+ if (cc>0) buff[cc]='\0';
+ if (buff[0]=='%' || buff[0]=='\n') continue;
+ if (strlen(buff)>0) {
+ verb_printf(efp,"\nWarning: Unknown specifier (%s).", buff);
+ }
}
fclose(fp);
@@ -316,54 +356,62 @@ static int sstrncmp(const char *s1, const char *s2, size_t len)
void set_icu_attributes(void)
{
int i,attr;
- char *pos, *tmp;
+ char *pos, *head, *tmp;
for (i=0;i<UCOL_ATTRIBUTE_COUNT;i++) icu_attributes[i]=UCOL_DEFAULT;
tmp=icu_attr_str;
- if ((pos=strstr(tmp,"alternate:"))>0) {
+ head=strtok(tmp, " \t\r\n");
+ while (head!=NULL) {
+ pos=head;
+ if (strncmp(pos,"alternate:",10)==0) {
pos+=10; attr=UCOL_ALTERNATE_HANDLING;
- if (strstr(pos,"shifted")) icu_attributes[attr]=UCOL_SHIFTED;
- else if (strstr(pos,"non-ignorable")) icu_attributes[attr]=UCOL_NON_IGNORABLE;
- else verb_printf(efp,"\nWarning: Illegal input for icu_attributes (alternate).");
- }
- if ((pos=strstr(tmp,"strength:"))>0) {
+ if (strcmp(pos,"shifted")==0) icu_attributes[attr]=UCOL_SHIFTED;
+ else if (strcmp(pos,"non-ignorable")==0) icu_attributes[attr]=UCOL_NON_IGNORABLE;
+ else goto illegal;
+ } else
+ if (strncmp(pos,"strength:",9)==0) {
pos+=9; attr=UCOL_STRENGTH;
- if (strstr(pos,"primary")) icu_attributes[attr]=UCOL_PRIMARY;
- else if (strstr(pos,"secondary")) icu_attributes[attr]=UCOL_SECONDARY;
- else if (strstr(pos,"tertiary")) icu_attributes[attr]=UCOL_TERTIARY;
- else if (strstr(pos,"quaternary")) icu_attributes[attr]=UCOL_QUATERNARY;
- else if (strstr(pos,"identical")) icu_attributes[attr]=UCOL_IDENTICAL;
- else verb_printf(efp,"\nWarning: Illegal input for icu_attributes (strength).");
- }
- if ((pos=strstr(tmp,"french-collation:"))>0) {
+ if (strcmp(pos,"primary")==0) icu_attributes[attr]=UCOL_PRIMARY;
+ else if (strcmp(pos,"secondary")==0) icu_attributes[attr]=UCOL_SECONDARY;
+ else if (strcmp(pos,"tertiary")==0) icu_attributes[attr]=UCOL_TERTIARY;
+ else if (strcmp(pos,"quaternary")==0) icu_attributes[attr]=UCOL_QUATERNARY;
+ else if (strcmp(pos,"identical")==0) icu_attributes[attr]=UCOL_IDENTICAL;
+ else goto illegal;
+ } else
+ if (strncmp(pos,"french-collation:",17)==0) {
pos+=17; attr=UCOL_FRENCH_COLLATION;
- if (strstr(pos,"on")) icu_attributes[attr]=UCOL_ON;
- else if (strstr(pos,"off")) icu_attributes[attr]=UCOL_OFF;
- else verb_printf(efp,"\nWarning: Illegal input for icu_attributes (french-collation).");
- }
- if ((pos=strstr(tmp,"case-first:"))>0) {
+ if (strcmp(pos,"on")==0) icu_attributes[attr]=UCOL_ON;
+ else if (strcmp(pos,"off")==0) icu_attributes[attr]=UCOL_OFF;
+ else goto illegal;
+ } else
+ if (strncmp(pos,"case-first:",11)==0) {
pos+=11; attr=UCOL_CASE_FIRST;
- if (strstr(pos,"off")) icu_attributes[attr]=UCOL_OFF;
- else if (strstr(pos,"upper-first")) icu_attributes[attr]=UCOL_UPPER_FIRST;
- else if (strstr(pos,"lower-first")) icu_attributes[attr]=UCOL_LOWER_FIRST;
- else verb_printf(efp,"\nWarning: Illegal input for icu_attributes (case-first).");
- }
- if ((pos=strstr(tmp,"case-level:"))>0) {
+ if (strcmp(pos,"off")==0) icu_attributes[attr]=UCOL_OFF;
+ else if (strcmp(pos,"upper-first")==0) icu_attributes[attr]=UCOL_UPPER_FIRST;
+ else if (strcmp(pos,"lower-first")==0) icu_attributes[attr]=UCOL_LOWER_FIRST;
+ else goto illegal;
+ } else
+ if (strncmp(pos,"case-level:",11)==0) {
pos+=11; attr=UCOL_CASE_LEVEL;
- if (strstr(pos,"on")) icu_attributes[attr]=UCOL_ON;
- else if (strstr(pos,"off")) icu_attributes[attr]=UCOL_OFF;
- else verb_printf(efp,"\nWarning: Illegal input for icu_attributes (case-level).");
- }
- if ((pos=strstr(tmp,"normalization-mode:"))>0) {
+ if (strcmp(pos,"on")==0) icu_attributes[attr]=UCOL_ON;
+ else if (strcmp(pos,"off")==0) icu_attributes[attr]=UCOL_OFF;
+ else goto illegal;
+ } else
+ if (strncmp(pos,"normalization-mode:",19)==0) {
pos+=19; attr=UCOL_NORMALIZATION_MODE;
- if (strstr(pos,"on")) icu_attributes[attr]=UCOL_ON;
- else if (strstr(pos,"off")) icu_attributes[attr]=UCOL_OFF;
- else verb_printf(efp,"\nWarning: Illegal input for icu_attributes (normalization-mode).");
- }
- if ((pos=strstr(tmp,"numeric-ordering:"))>0) {
+ if (strcmp(pos,"on")==0) icu_attributes[attr]=UCOL_ON;
+ else if (strcmp(pos,"off")==0) icu_attributes[attr]=UCOL_OFF;
+ else goto illegal;
+ } else
+ if (strncmp(pos,"numeric-ordering:",17)==0) {
pos+=17; attr=UCOL_NUMERIC_COLLATION;
- if (strstr(pos,"on")) icu_attributes[attr]=UCOL_ON;
- else if (strstr(pos,"off")) icu_attributes[attr]=UCOL_OFF;
- else verb_printf(efp,"\nWarning: Illegal input for icu_attributes (numeric-ordering).");
+ if (strcmp(pos,"on")==0) icu_attributes[attr]=UCOL_ON;
+ else if (strcmp(pos,"off")==0) icu_attributes[attr]=UCOL_OFF;
+ else goto illegal;
+ } else {
+ illegal:
+ verb_printf(efp,"\nWarning: Illegal input for icu_attributes (%s).",head);
+ }
+ head=strtok(NULL, " \t\r\n");
}
}
diff --git a/indexing/upmendex/source/tests/attr1.ind b/indexing/upmendex/source/tests/attr1.ind
new file mode 100644
index 0000000000..6db7b19221
--- /dev/null
+++ b/indexing/upmendex/source/tests/attr1.ind
@@ -0,0 +1,40 @@
+\begin{theindex}
+
+\centerline{\bfseries ◆◆◆ Symbols ◆◆◆}\par\nobreak
+ \item #\leaders\hbox{$\cdot$}\hfill 2000
+ \item $\leaders\hbox{$\cdot$}\hfill 2001
+ \item 2\leaders\hbox{$\cdot$}\hfill 1001
+ \item 10\leaders\hbox{$\cdot$}\hfill 1000
+
+ \indexspace
+
+\centerline{\bfseries ◆◆◆ A ◆◆◆}\par\nobreak
+ \item A\leaders\hbox{$\cdot$}\hfill 400
+ \item a\leaders\hbox{$\cdot$}\hfill 401
+
+ \indexspace
+
+\centerline{\bfseries ◆◆◆ B ◆◆◆}\par\nobreak
+ \item B\leaders\hbox{$\cdot$}\hfill 402
+ \item b\leaders\hbox{$\cdot$}\hfill 403
+
+ \indexspace
+
+\centerline{\bfseries ◆◆◆ C ◆◆◆}\par\nobreak
+ \item C\leaders\hbox{$\cdot$}\hfill 404
+ \item c\leaders\hbox{$\cdot$}\hfill 405
+ \item Ç\leaders\hbox{$\cdot$}\hfill 406
+ \item ç\leaders\hbox{$\cdot$}\hfill 407
+ \item Ch\leaders\hbox{$\cdot$}\hfill 420
+ \item cha\leaders\hbox{$\cdot$}\hfill 421
+ \item cz\leaders\hbox{$\cdot$}\hfill 422
+
+ \indexspace
+
+\centerline{\bfseries ◆◆◆ U ◆◆◆}\par\nobreak
+ \item U\leaders\hbox{$\cdot$}\hfill 408
+ \item u\leaders\hbox{$\cdot$}\hfill 409
+ \item Ü\leaders\hbox{$\cdot$}\hfill 410
+ \item ü\leaders\hbox{$\cdot$}\hfill 411
+
+\end{theindex}
diff --git a/indexing/upmendex/source/tests/attr1.ist b/indexing/upmendex/source/tests/attr1.ist
new file mode 100644
index 0000000000..301aff0a6a
--- /dev/null
+++ b/indexing/upmendex/source/tests/attr1.ist
@@ -0,0 +1,2 @@
+icu_attributes "case-first:upper-first
+numeric-ordering:on "
diff --git a/indexing/upmendex/source/tests/locale1.ind b/indexing/upmendex/source/tests/locale1.ind
new file mode 100644
index 0000000000..76bf98f25a
--- /dev/null
+++ b/indexing/upmendex/source/tests/locale1.ind
@@ -0,0 +1,48 @@
+\begin{theindex}
+
+\centerline{\bfseries ◆◆◆ Symbols ◆◆◆}\par\nobreak
+ \item #\leaders\hbox{$\cdot$}\hfill 2000
+ \item $\leaders\hbox{$\cdot$}\hfill 2001
+ \item 10\leaders\hbox{$\cdot$}\hfill 1000
+ \item 2\leaders\hbox{$\cdot$}\hfill 1001
+
+ \indexspace
+
+\centerline{\bfseries ◆◆◆ A ◆◆◆}\par\nobreak
+ \item a\leaders\hbox{$\cdot$}\hfill 401
+ \item A\leaders\hbox{$\cdot$}\hfill 400
+
+ \indexspace
+
+\centerline{\bfseries ◆◆◆ B ◆◆◆}\par\nobreak
+ \item b\leaders\hbox{$\cdot$}\hfill 403
+ \item B\leaders\hbox{$\cdot$}\hfill 402
+
+ \indexspace
+
+\centerline{\bfseries ◆◆◆ C ◆◆◆}\par\nobreak
+ \item c\leaders\hbox{$\cdot$}\hfill 405
+ \item C\leaders\hbox{$\cdot$}\hfill 404
+ \item Ch\leaders\hbox{$\cdot$}\hfill 420
+ \item cha\leaders\hbox{$\cdot$}\hfill 421
+ \item cz\leaders\hbox{$\cdot$}\hfill 422
+
+ \indexspace
+
+\centerline{\bfseries ◆◆◆ Ç ◆◆◆}\par\nobreak
+ \item ç\leaders\hbox{$\cdot$}\hfill 407
+ \item Ç\leaders\hbox{$\cdot$}\hfill 406
+
+ \indexspace
+
+\centerline{\bfseries ◆◆◆ U ◆◆◆}\par\nobreak
+ \item u\leaders\hbox{$\cdot$}\hfill 409
+ \item U\leaders\hbox{$\cdot$}\hfill 408
+
+ \indexspace
+
+\centerline{\bfseries ◆◆◆ Ü ◆◆◆}\par\nobreak
+ \item ü\leaders\hbox{$\cdot$}\hfill 411
+ \item Ü\leaders\hbox{$\cdot$}\hfill 410
+
+\end{theindex}
diff --git a/indexing/upmendex/source/tests/locale1.ist b/indexing/upmendex/source/tests/locale1.ist
new file mode 100644
index 0000000000..cf9e67da6e
--- /dev/null
+++ b/indexing/upmendex/source/tests/locale1.ist
@@ -0,0 +1 @@
+icu_locale "az"
diff --git a/indexing/upmendex/source/tests/locale2-rule.ind b/indexing/upmendex/source/tests/locale2-rule.ind
new file mode 100644
index 0000000000..4054762ab7
--- /dev/null
+++ b/indexing/upmendex/source/tests/locale2-rule.ind
@@ -0,0 +1,44 @@
+\begin{theindex}
+
+\centerline{\bfseries ◆◆◆ Symbols ◆◆◆}\par\nobreak
+ \item #\leaders\hbox{$\cdot$}\hfill 2000
+ \item $\leaders\hbox{$\cdot$}\hfill 2001
+ \item 10\leaders\hbox{$\cdot$}\hfill 1000
+ \item 2\leaders\hbox{$\cdot$}\hfill 1001
+
+ \indexspace
+
+\centerline{\bfseries ◆◆◆ A ◆◆◆}\par\nobreak
+ \item a\leaders\hbox{$\cdot$}\hfill 401
+ \item A\leaders\hbox{$\cdot$}\hfill 400
+
+ \indexspace
+
+\centerline{\bfseries ◆◆◆ C ◆◆◆}\par\nobreak
+ \item C\leaders\hbox{$\cdot$}\hfill 404
+ \item Ç\leaders\hbox{$\cdot$}\hfill 406
+ \item c\leaders\hbox{$\cdot$}\hfill 405
+ \item ç\leaders\hbox{$\cdot$}\hfill 407
+ \item cz\leaders\hbox{$\cdot$}\hfill 422
+
+ \indexspace
+
+\centerline{\bfseries ◆◆◆ U ◆◆◆}\par\nobreak
+ \item u\leaders\hbox{$\cdot$}\hfill 409
+ \item U\leaders\hbox{$\cdot$}\hfill 408
+ \item ü\leaders\hbox{$\cdot$}\hfill 411
+ \item Ü\leaders\hbox{$\cdot$}\hfill 410
+
+ \indexspace
+
+\centerline{\bfseries ◆◆◆ B ◆◆◆}\par\nobreak
+ \item b\leaders\hbox{$\cdot$}\hfill 403
+ \item B\leaders\hbox{$\cdot$}\hfill 402
+
+ \indexspace
+
+\centerline{\bfseries ◆◆◆ CH ◆◆◆}\par\nobreak
+ \item Ch\leaders\hbox{$\cdot$}\hfill 420
+ \item cha\leaders\hbox{$\cdot$}\hfill 421
+
+\end{theindex}
diff --git a/indexing/upmendex/source/tests/locale2.ist b/indexing/upmendex/source/tests/locale2.ist
new file mode 100644
index 0000000000..89b590a21b
--- /dev/null
+++ b/indexing/upmendex/source/tests/locale2.ist
@@ -0,0 +1 @@
+icu_locale "es@collation=traditional"
diff --git a/indexing/upmendex/source/tests/pprec3.ist b/indexing/upmendex/source/tests/pprec3.ist
new file mode 100644
index 0000000000..b8f33af1dd
--- /dev/null
+++ b/indexing/upmendex/source/tests/pprec3.ist
@@ -0,0 +1,11 @@
+% -*- coding: utf-8 -*-
+
+page_precedence "naA"
+
+delim_0 "[[delim0]]"
+delim_1 "[[delim1]]"
+delim_2 "[[delim2]]"
+delim_r "[[delimr]]"
+suffix_2p "[[sfx 2p]]"
+suffix_3p "[[sfx 3p]]"
+
diff --git a/indexing/upmendex/source/tests/pprec4.ist b/indexing/upmendex/source/tests/pprec4.ist
new file mode 100644
index 0000000000..768fec7642
--- /dev/null
+++ b/indexing/upmendex/source/tests/pprec4.ist
@@ -0,0 +1,11 @@
+% -*- coding: utf-8 -*-
+
+page_precedence "Ana"
+
+delim_0 "[[delim0]]"
+delim_1 "[[delim1]]"
+delim_2 "[[delim2]]"
+delim_r "[[delimr]]"
+suffix_2p "[[sfx 2p]]"
+suffix_3p "[[sfx 3p]]"
+
diff --git a/indexing/upmendex/source/tests/pprecB-0.ind b/indexing/upmendex/source/tests/pprecB-0.ind
new file mode 100644
index 0000000000..1e23d9fb4a
--- /dev/null
+++ b/indexing/upmendex/source/tests/pprecB-0.ind
@@ -0,0 +1,5 @@
+\begin{theindex}
+
+ \item entryA[[delim0]]1[[sfx 3p]], a[[sfx 3p]], A[[sfx 3p]]
+
+\end{theindex}
diff --git a/indexing/upmendex/source/tests/pprecB-3.ind b/indexing/upmendex/source/tests/pprecB-3.ind
new file mode 100644
index 0000000000..1e23d9fb4a
--- /dev/null
+++ b/indexing/upmendex/source/tests/pprecB-3.ind
@@ -0,0 +1,5 @@
+\begin{theindex}
+
+ \item entryA[[delim0]]1[[sfx 3p]], a[[sfx 3p]], A[[sfx 3p]]
+
+\end{theindex}
diff --git a/indexing/upmendex/source/tests/pprecB-4.ind b/indexing/upmendex/source/tests/pprecB-4.ind
new file mode 100644
index 0000000000..3943ad4854
--- /dev/null
+++ b/indexing/upmendex/source/tests/pprecB-4.ind
@@ -0,0 +1,5 @@
+\begin{theindex}
+
+ \item entryA[[delim0]]A[[sfx 3p]], 1[[sfx 3p]], a[[sfx 3p]]
+
+\end{theindex}
diff --git a/indexing/upmendex/source/tests/pprecB.idx b/indexing/upmendex/source/tests/pprecB.idx
new file mode 100644
index 0000000000..a1d8fc31ae
--- /dev/null
+++ b/indexing/upmendex/source/tests/pprecB.idx
@@ -0,0 +1,9 @@
+\indexentry{entryA}{a}
+\indexentry{entryA}{b}
+\indexentry{entryA}{c}
+\indexentry{entryA}{1}
+\indexentry{entryA}{2}
+\indexentry{entryA}{3}
+\indexentry{entryA}{A}
+\indexentry{entryA}{B}
+\indexentry{entryA}{C}
diff --git a/indexing/upmendex/source/tests/range.idx b/indexing/upmendex/source/tests/range.idx
new file mode 100644
index 0000000000..3eccb113aa
--- /dev/null
+++ b/indexing/upmendex/source/tests/range.idx
@@ -0,0 +1,126 @@
+\indexentry{entryA}{1}
+\indexentry{entryA}{2}
+\indexentry{entryA}{iii}
+\indexentry{entryA}{iv}
+\indexentry{entryA}{IX}
+\indexentry{entryA}{X}
+
+\indexentry{entryB}{1}
+\indexentry{entryB}{2}
+\indexentry{entryB}{3}
+\indexentry{entryB}{iii}
+\indexentry{entryB}{iv}
+\indexentry{entryB}{v}
+\indexentry{entryB}{IX}
+\indexentry{entryB}{X}
+\indexentry{entryB}{XI}
+
+\indexentry{entryC}{1}
+\indexentry{entryC}{2}
+\indexentry{entryC}{3}
+\indexentry{entryC}{4}
+\indexentry{entryC|bold}{iii}
+\indexentry{entryC|bold}{iv}
+\indexentry{entryC|bold}{v}
+\indexentry{entryC|bold}{vi}
+\indexentry{entryC}{IX}
+\indexentry{entryC|bold}{X}
+\indexentry{entryC}{XI}
+\indexentry{entryC}{XII}
+
+\indexentry{entryD}{c-1}
+\indexentry{entryD}{c-2}
+\indexentry{entryD}{i-iii}
+\indexentry{entryD}{i-iv}
+\indexentry{entryD}{d-IX}
+\indexentry{entryD}{d-X}
+
+\indexentry{entryE}{c-1}
+\indexentry{entryE}{c-2}
+\indexentry{entryE}{c-3}
+\indexentry{entryE}{i-iii}
+\indexentry{entryE}{i-iv}
+\indexentry{entryE}{i-v}
+\indexentry{entryE}{d-IX}
+\indexentry{entryE}{d-X}
+\indexentry{entryE}{d-XI}
+
+\indexentry{entryF}{C-I}
+\indexentry{entryF}{C-II}
+\indexentry{entryF}{C-III}
+\indexentry{entryF}{C-IV}
+\indexentry{entryF}{V-LIII}
+\indexentry{entryF}{V-LIV}
+\indexentry{entryF}{V-LV}
+\indexentry{entryF}{V-LVI}
+\indexentry{entryF}{D-DI}
+\indexentry{entryF}{D-DII}
+\indexentry{entryF}{D-DIII}
+\indexentry{entryF}{D-DIV}
+
+\indexentry{entryP}{A-a-1-VII-iii}
+\indexentry{entryP}{A-a-2-VII-iii}
+\indexentry{entryP}{A-a-3-VII-iii}
+\indexentry{entryP}{A-a-1-VIII-iii}
+\indexentry{entryP}{A-a-1-IX-iii}
+\indexentry{entryP}{A-a-1-VII-iv}
+\indexentry{entryP}{A-a-1-VII-v}
+
+\indexentry{entryQ}{1-2-3-4-5-6-7-8-9-10}
+\indexentry{entryQ}{1-2-3-4-5-6-7-8-9-11}
+\indexentry{entryQ}{1-2-3-4-5-6-7-8-9-14}
+\indexentry{entryQ}{1-2-3-4-5-6-7-8-9-15}
+\indexentry{entryQ}{1-2-3-4-5-6-7-8-9-16}
+\indexentry{entryQ}{1-2-3-4-5-6-7-8-9-22}
+\indexentry{entryQ}{1-2-3-4-5-6-7-8-9-23}
+\indexentry{entryQ}{1-2-3-4-5-6-7-8-9-24}
+\indexentry{entryQ}{1-2-3-4-5-6-7-8-9-25}
+\indexentry{entryQ}{1-2-3-4-5-6-7-8-9-i}
+\indexentry{entryQ}{1-2-3-4-5-6-7-8-9-ii}
+\indexentry{entryQ}{1-2-3-4-5-6-7-8-9-iii}
+\indexentry{entryQ}{1-2-3-4-5-6-7-8-9-iv}
+\indexentry{entryQ}{1-2-3-4-5-6-7-8-9-v}
+
+\indexentry{entryA1|(}{1}
+\indexentry{entryA1|)}{2}
+\indexentry{entryA1|(}{iii}
+\indexentry{entryA1|)}{iv}
+\indexentry{entryA1|(}{IX}
+\indexentry{entryA1|)}{X}
+
+\indexentry{entryB1|(}{1}
+\indexentry{entryB1|)}{3}
+\indexentry{entryB1|(}{iii}
+\indexentry{entryB1|)}{v}
+\indexentry{entryB1|(}{IX}
+\indexentry{entryB1|)}{XI}
+
+\indexentry{entryC1|(}{1}
+\indexentry{entryC1|)}{4}
+\indexentry{entryC1|(bold}{iii}
+\indexentry{entryC1|)}{vi}
+\indexentry{entryC1|(}{IX}
+\indexentry{entryC1|bold}{X}
+\indexentry{entryC1|)}{XII}
+
+\indexentry{entryD1|(}{c-1}
+\indexentry{entryD1|)}{c-2}
+\indexentry{entryD1|(}{i-iii}
+\indexentry{entryD1|)}{i-iv}
+\indexentry{entryD1|(}{d-IX}
+\indexentry{entryD1|)}{d-X}
+
+\indexentry{entryE1|(}{c-1}
+\indexentry{entryE1|)}{c-3}
+\indexentry{entryE1|(}{i-iii}
+\indexentry{entryE1|)}{i-v}
+\indexentry{entryE1|(}{d-IX}
+\indexentry{entryE1|)}{d-XI}
+
+\indexentry{entryF1|(}{C-I}
+\indexentry{entryF1|)}{C-IV}
+\indexentry{entryF1|(}{V-LIII}
+\indexentry{entryF1|)}{V-LVI}
+\indexentry{entryF1|(}{D-DI}
+\indexentry{entryF1|bold}{D-DII}
+\indexentry{entryF1|)}{D-DIV}
diff --git a/indexing/upmendex/source/tests/range1.ind b/indexing/upmendex/source/tests/range1.ind
new file mode 100644
index 0000000000..c5f047a55e
--- /dev/null
+++ b/indexing/upmendex/source/tests/range1.ind
@@ -0,0 +1,25 @@
+\begin{theindex}
+
+ \item entryA, 1[[sfx 2p]], iii[[sfx 2p]], IX[[sfx 2p]]
+ \item entryA1, 1[[sfx 2p]], iii[[sfx 2p]], IX[[sfx 2p]]
+ \item entryB, 1[[sfx mp]], iii[[sfx mp]], IX[[sfx mp]]
+ \item entryB1, 1[[sfx mp]], iii[[sfx mp]], IX[[sfx mp]]
+ \item entryC, 1[[sfx mp]], \bold{iii[[sfx mp]]}, IX, \bold{X},
+ XI[[sfx 2p]]
+ \item entryC1, 1[[sfx mp]], \bold{iii[[sfx mp]]}, \bold{X},
+ IX[[sfx mp]]
+ \item entryD, c-1[[sfx 2p]], d-IX[[sfx 2p]], i-iii[[sfx 2p]]
+ \item entryD1, c-1[[sfx 2p]], d-IX[[sfx 2p]], i-iii[[sfx 2p]]
+ \item entryE, c-1[[sfx mp]], d-IX[[sfx mp]], i-iii[[sfx mp]]
+ \item entryE1, c-1[[sfx mp]], d-IX[[sfx mp]], i-iii[[sfx mp]]
+ \item entryF, C-I[[sfx mp]], D-DI[[sfx mp]], V-LIII[[sfx mp]]
+ \item entryF1, C-I[[sfx mp]], \bold{D-DII}, D-DI[[sfx mp]],
+ V-LIII[[sfx mp]]
+ \item entryP, A-a-1-VII-iii[[sfx mp]], A-a-1-VIII-iii, A-a-1-IX-iii,
+ A-a-2-VII-iii, A-a-3-VII-iii
+ \item entryQ, 1-2-3-4-5-6-7-8-9-10[[sfx 2p]],
+ 1-2-3-4-5-6-7-8-9-14[[sfx mp]],
+ 1-2-3-4-5-6-7-8-9-22[[sfx mp]],
+ 1-2-3-4-5-6-7-8-9-i[[sfx mp]]
+
+\end{theindex}
diff --git a/indexing/upmendex/source/tests/range1.ist b/indexing/upmendex/source/tests/range1.ist
new file mode 100644
index 0000000000..116c2d8713
--- /dev/null
+++ b/indexing/upmendex/source/tests/range1.ist
@@ -0,0 +1,3 @@
+page_precedence "nrRaA"
+suffix_2p "[[sfx 2p]]"
+suffix_mp "[[sfx mp]]"
diff --git a/indexing/upmendex/source/tests/range2.ind b/indexing/upmendex/source/tests/range2.ind
new file mode 100644
index 0000000000..789eb3f75c
--- /dev/null
+++ b/indexing/upmendex/source/tests/range2.ind
@@ -0,0 +1,26 @@
+\begin{theindex}
+
+ \item entryA, 1[[sfx 2p]], iii[[sfx 2p]], IX[[sfx 2p]]
+ \item entryA1, 1[[sfx 2p]], iii[[sfx 2p]], IX[[sfx 2p]]
+ \item entryB, 1[[sfx 3p]], iii[[sfx 3p]], IX[[sfx 3p]]
+ \item entryB1, 1[[sfx 3p]], iii[[sfx 3p]], IX[[sfx 3p]]
+ \item entryC, 1[[delim r]]4, \bold{iii[[delim r]]vi}, IX, \bold{X},
+ XI[[sfx 2p]]
+ \item entryC1, 1[[delim r]]4, \bold{iii[[delim r]]vi}, \bold{X},
+ IX[[delim r]]XII
+ \item entryD, c-1[[sfx 2p]], d-IX[[sfx 2p]], i-iii[[sfx 2p]]
+ \item entryD1, c-1[[sfx 2p]], d-IX[[sfx 2p]], i-iii[[sfx 2p]]
+ \item entryE, c-1[[sfx 3p]], d-IX[[sfx 3p]], i-iii[[sfx 3p]]
+ \item entryE1, c-1[[sfx 3p]], d-IX[[sfx 3p]], i-iii[[sfx 3p]]
+ \item entryF, C-I[[delim r]]C-IV, D-DI[[delim r]]D-DIV,
+ V-LIII[[delim r]]V-LVI
+ \item entryF1, C-I[[delim r]]C-IV, \bold{D-DII}, D-DI[[delim r]]D-DIV,
+ V-LIII[[delim r]]V-LVI
+ \item entryP, A-a-1-VII-iii[[sfx 3p]], A-a-1-VIII-iii, A-a-1-IX-iii,
+ A-a-2-VII-iii, A-a-3-VII-iii
+ \item entryQ, 1-2-3-4-5-6-7-8-9-10[[sfx 2p]],
+ 1-2-3-4-5-6-7-8-9-14[[sfx 3p]],
+ 1-2-3-4-5-6-7-8-9-22[[delim r]]1-2-3-4-5-6-7-8-9-25,
+ 1-2-3-4-5-6-7-8-9-i[[delim r]]1-2-3-4-5-6-7-8-9-v
+
+\end{theindex}
diff --git a/indexing/upmendex/source/tests/range2.ist b/indexing/upmendex/source/tests/range2.ist
new file mode 100644
index 0000000000..552e5fb674
--- /dev/null
+++ b/indexing/upmendex/source/tests/range2.ist
@@ -0,0 +1,4 @@
+page_precedence "nrRaA"
+delim_r "[[delim r]]"
+suffix_2p "[[sfx 2p]]"
+suffix_3p "[[sfx 3p]]"
diff --git a/indexing/upmendex/source/tests/range3.ind b/indexing/upmendex/source/tests/range3.ind
new file mode 100644
index 0000000000..519730a138
--- /dev/null
+++ b/indexing/upmendex/source/tests/range3.ind
@@ -0,0 +1,25 @@
+\begin{theindex}
+
+ \item entryA, 1[[sfx 2p]], iii[[sfx 2p]], IX[[sfx 2p]]
+ \item entryA1, 1[[sfx 2p]], iii[[sfx 2p]], IX[[sfx 2p]]
+ \item entryB, 1[[sfx 3p]], iii[[sfx 3p]], IX[[sfx 3p]]
+ \item entryB1, 1[[sfx 3p]], iii[[sfx 3p]], IX[[sfx 3p]]
+ \item entryC, 1[[sfx mp]], \bold{iii[[sfx mp]]}, IX, \bold{X},
+ XI[[sfx 2p]]
+ \item entryC1, 1[[sfx mp]], \bold{iii[[sfx mp]]}, \bold{X},
+ IX[[sfx mp]]
+ \item entryD, c-1[[sfx 2p]], d-IX[[sfx 2p]], i-iii[[sfx 2p]]
+ \item entryD1, c-1[[sfx 2p]], d-IX[[sfx 2p]], i-iii[[sfx 2p]]
+ \item entryE, c-1[[sfx 3p]], d-IX[[sfx 3p]], i-iii[[sfx 3p]]
+ \item entryE1, c-1[[sfx 3p]], d-IX[[sfx 3p]], i-iii[[sfx 3p]]
+ \item entryF, C-I[[sfx mp]], D-DI[[sfx mp]], V-LIII[[sfx mp]]
+ \item entryF1, C-I[[sfx mp]], \bold{D-DII}, D-DI[[sfx mp]],
+ V-LIII[[sfx mp]]
+ \item entryP, A-a-1-VII-iii[[sfx 3p]], A-a-1-VIII-iii, A-a-1-IX-iii,
+ A-a-2-VII-iii, A-a-3-VII-iii
+ \item entryQ, 1-2-3-4-5-6-7-8-9-10[[sfx 2p]],
+ 1-2-3-4-5-6-7-8-9-14[[sfx 3p]],
+ 1-2-3-4-5-6-7-8-9-22[[sfx mp]],
+ 1-2-3-4-5-6-7-8-9-i[[sfx mp]]
+
+\end{theindex}
diff --git a/indexing/upmendex/source/tests/range3.ist b/indexing/upmendex/source/tests/range3.ist
new file mode 100644
index 0000000000..b751471b43
--- /dev/null
+++ b/indexing/upmendex/source/tests/range3.ist
@@ -0,0 +1,4 @@
+page_precedence "nrRaA"
+suffix_2p "[[sfx 2p]]"
+suffix_3p "[[sfx 3p]]"
+suffix_mp "[[sfx mp]]"
diff --git a/indexing/upmendex/source/tests/rangetwo.idx b/indexing/upmendex/source/tests/rangetwo.idx
deleted file mode 100644
index d941f7e831..0000000000
--- a/indexing/upmendex/source/tests/rangetwo.idx
+++ /dev/null
@@ -1,2 +0,0 @@
-\indexentry{entryA}{1}
-\indexentry{entryA}{2}
diff --git a/indexing/upmendex/source/tests/rangetwo.ind b/indexing/upmendex/source/tests/rangetwo.ind
deleted file mode 100644
index 1119d2fa8e..0000000000
--- a/indexing/upmendex/source/tests/rangetwo.ind
+++ /dev/null
@@ -1,5 +0,0 @@
-\begin{theindex}
-
- \item entryA, 1[[sfx2p]]
-
-\end{theindex}
diff --git a/indexing/upmendex/source/tests/rangetwo.ist b/indexing/upmendex/source/tests/rangetwo.ist
deleted file mode 100644
index 3e986c8935..0000000000
--- a/indexing/upmendex/source/tests/rangetwo.ist
+++ /dev/null
@@ -1,2 +0,0 @@
-suffix_2p "[[sfx2p]]"
-
diff --git a/indexing/upmendex/source/tests/rule.idx b/indexing/upmendex/source/tests/rule.idx
new file mode 100644
index 0000000000..783cce9fd4
--- /dev/null
+++ b/indexing/upmendex/source/tests/rule.idx
@@ -0,0 +1,20 @@
+% -*- coding: utf-8 -*-
+\indexentry{A}{400}
+\indexentry{a}{401}
+\indexentry{B}{402}
+\indexentry{b}{403}
+\indexentry{C}{404}
+\indexentry{c}{405}
+\indexentry{Ç}{406}
+\indexentry{ç}{407}
+\indexentry{Ch}{420}
+\indexentry{cha}{421}
+\indexentry{cz}{422}
+\indexentry{U}{408}
+\indexentry{u}{409}
+\indexentry{Ü}{410}
+\indexentry{ü}{411}
+\indexentry{10}{1000}
+\indexentry{2}{1001}
+\indexentry{#}{2000}
+\indexentry{$}{2001}
diff --git a/indexing/upmendex/source/tests/rule.ind b/indexing/upmendex/source/tests/rule.ind
new file mode 100644
index 0000000000..3ac1335e05
--- /dev/null
+++ b/indexing/upmendex/source/tests/rule.ind
@@ -0,0 +1,40 @@
+\begin{theindex}
+
+\centerline{\bfseries ◆◆◆ Symbols ◆◆◆}\par\nobreak
+ \item #\leaders\hbox{$\cdot$}\hfill 2000
+ \item $\leaders\hbox{$\cdot$}\hfill 2001
+ \item 10\leaders\hbox{$\cdot$}\hfill 1000
+ \item 2\leaders\hbox{$\cdot$}\hfill 1001
+
+ \indexspace
+
+\centerline{\bfseries ◆◆◆ A ◆◆◆}\par\nobreak
+ \item a\leaders\hbox{$\cdot$}\hfill 401
+ \item A\leaders\hbox{$\cdot$}\hfill 400
+
+ \indexspace
+
+\centerline{\bfseries ◆◆◆ C ◆◆◆}\par\nobreak
+ \item C\leaders\hbox{$\cdot$}\hfill 404
+ \item Ç\leaders\hbox{$\cdot$}\hfill 406
+ \item c\leaders\hbox{$\cdot$}\hfill 405
+ \item ç\leaders\hbox{$\cdot$}\hfill 407
+ \item Ch\leaders\hbox{$\cdot$}\hfill 420
+ \item cha\leaders\hbox{$\cdot$}\hfill 421
+ \item cz\leaders\hbox{$\cdot$}\hfill 422
+
+ \indexspace
+
+\centerline{\bfseries ◆◆◆ U ◆◆◆}\par\nobreak
+ \item u\leaders\hbox{$\cdot$}\hfill 409
+ \item U\leaders\hbox{$\cdot$}\hfill 408
+ \item ü\leaders\hbox{$\cdot$}\hfill 411
+ \item Ü\leaders\hbox{$\cdot$}\hfill 410
+
+ \indexspace
+
+\centerline{\bfseries ◆◆◆ B ◆◆◆}\par\nobreak
+ \item b\leaders\hbox{$\cdot$}\hfill 403
+ \item B\leaders\hbox{$\cdot$}\hfill 402
+
+\end{theindex}
diff --git a/indexing/upmendex/source/tests/rule1.ist b/indexing/upmendex/source/tests/rule1.ist
new file mode 100644
index 0000000000..c986e39e96
--- /dev/null
+++ b/indexing/upmendex/source/tests/rule1.ist
@@ -0,0 +1,3 @@
+% -*- coding: utf-8 -*-
+
+icu_rules "&a<<<A<C<<Ç<<c<<ç<u<<U<<ü<<Ü<b<<B"
diff --git a/indexing/upmendex/source/tests/rule2.ist b/indexing/upmendex/source/tests/rule2.ist
new file mode 100644
index 0000000000..a61af0a546
--- /dev/null
+++ b/indexing/upmendex/source/tests/rule2.ist
@@ -0,0 +1,3 @@
+% -*- coding: utf-8 -*-
+
+icu_rules "&a<<<A<C<<\U000000c7<<c<<\u00E7<u<<U<<\x{fc}<<\xDC<b<<B"
diff --git a/indexing/upmendex/source/tests/rule3.ist b/indexing/upmendex/source/tests/rule3.ist
new file mode 100644
index 0000000000..7ab835a801
--- /dev/null
+++ b/indexing/upmendex/source/tests/rule3.ist
@@ -0,0 +1,4 @@
+% -*- coding: latin-1 -*-
+% To test malformed input
+
+icu_rules "&a<<<A<C<<<<c<<<u<<U<<<<<b<<B"
diff --git a/indexing/upmendex/source/tests/rule4.ist b/indexing/upmendex/source/tests/rule4.ist
new file mode 100644
index 0000000000..0c30dceba2
--- /dev/null
+++ b/indexing/upmendex/source/tests/rule4.ist
@@ -0,0 +1,4 @@
+% -*- coding: utf-8 -*-
+% To test malformed input
+
+icu_rules "&a<<<A<C<<\U000000c7<<c<<\uE7<u<<U<<\x{fc}<<\xDC<b<<B"
diff --git a/indexing/upmendex/source/tests/upmendex.test b/indexing/upmendex/source/tests/upmendex.test
index b64c045af9..84dfbd17be 100755
--- a/indexing/upmendex/source/tests/upmendex.test
+++ b/indexing/upmendex/source/tests/upmendex.test
@@ -1,5 +1,5 @@
#! /bin/sh -vx
-# $Id$
+# $Id: upmendex.test 60549 2021-09-19 08:57:18Z takuji $
# Copyright 2017 Karl Berry <tex-live@tug.org>
# Copyright 2013 Peter Breitenlohner <tex-live@tug.org>
# You may freely use, modify and/or distribute this file.
@@ -7,7 +7,7 @@
TEXMFCNF=$srcdir/../kpathsea
export TEXMFCNF
-rm -f foo.* uni.* pprec*.* rangetwo.*
+rm -f foo.* uni.* range*.* pprec*.* locale*.* attr*.* rule*.*
./upmendex $srcdir/tests/foo.idx -o foo.ind1 -t foo.ilg1 \
&& diff $srcdir/tests/foo.ind foo.ind1 || exit 1
@@ -28,10 +28,18 @@ cat $srcdir/tests/uni.idx | \
$srcdir/tests/uni.idx -o uni.ind3 -t uni.ilg3 \
&& diff $srcdir/tests/uni.ind uni.ind3 || exit 1
-# test for two-element range suffix_2p
-./upmendex -s $srcdir/tests/rangetwo.ist $srcdir/tests/rangetwo.idx \
- -o rangetwo.ind1 -t rangetwo.ilg \
- && diff $srcdir/tests/rangetwo.ind rangetwo.ind1 || exit 1
+# test for range suffix_2p, suffix_3p, suffix_mp
+./upmendex -s $srcdir/tests/range1.ist $srcdir/tests/range.idx \
+ -o range1.ind1 -t range1.ilg \
+ && diff $srcdir/tests/range1.ind range1.ind1 || exit 1
+
+./upmendex -s $srcdir/tests/range2.ist $srcdir/tests/range.idx \
+ -o range2.ind1 -t range2.ilg \
+ && diff $srcdir/tests/range2.ind range2.ind1 || exit 1
+
+./upmendex -s $srcdir/tests/range3.ist $srcdir/tests/range.idx \
+ -o range3.ind1 -t range3.ilg \
+ && diff $srcdir/tests/range3.ind range3.ind1 || exit 1
# test for page_precedence and suffix_3p
./upmendex -s $srcdir/tests/pprec0.ist $srcdir/tests/pprecA.idx \
@@ -46,3 +54,57 @@ cat $srcdir/tests/uni.idx | \
-o pprecA-2.ind1 -t pprecA-2.ilg \
&& diff $srcdir/tests/pprecA-2.ind pprecA-2.ind1 || exit 1
+./upmendex -s $srcdir/tests/pprec0.ist $srcdir/tests/pprecB.idx \
+ -o pprecB-0.ind1 -t pprecB-0.ilg \
+ && diff $srcdir/tests/pprecB-0.ind pprecB-0.ind1 || exit 1
+
+./upmendex -s $srcdir/tests/pprec3.ist $srcdir/tests/pprecB.idx \
+ -o pprecB-3.ind1 -t pprecB-3.ilg \
+ && diff $srcdir/tests/pprecB-3.ind pprecB-3.ind1 || exit 1
+
+./upmendex -s $srcdir/tests/pprec4.ist $srcdir/tests/pprecB.idx \
+ -o pprecB-4.ind1 -t pprecB-4.ilg \
+ && diff $srcdir/tests/pprecB-4.ind pprecB-4.ind1 || exit 1
+
+# test for icu_locale
+./upmendex -s $srcdir/tests/uni.ist -s $srcdir/tests/locale1.ist $srcdir/tests/rule.idx \
+ -o locale1.ind1 -t locale1.ilg \
+ && diff $srcdir/tests/locale1.ind locale1.ind1 || exit 1
+
+# test for icu_attributes
+./upmendex -s $srcdir/tests/uni.ist -s $srcdir/tests/attr1.ist $srcdir/tests/rule.idx \
+ -o attr1.ind1 -t attr1.ilg \
+ && diff $srcdir/tests/attr1.ind attr1.ind1 || exit 1
+
+# test for icu_rules
+./upmendex -s $srcdir/tests/uni.ist -s $srcdir/tests/rule1.ist $srcdir/tests/rule.idx \
+ -o rule1.ind1 -t rule1.ilg \
+ && diff $srcdir/tests/rule.ind rule1.ind1 || exit 1
+
+./upmendex -s $srcdir/tests/uni.ist -s $srcdir/tests/rule2.ist $srcdir/tests/rule.idx \
+ -o rule2.ind1 -t rule2.ilg \
+ && diff $srcdir/tests/rule.ind rule2.ind1 || exit 1
+
+# malformed input
+./upmendex -s $srcdir/tests/uni.ist -s $srcdir/tests/rule3.ist $srcdir/tests/rule.idx \
+ -o rule3.ind1 -t rule3.ilg
+if [ $? -eq 254 ]; then
+ echo "Error check: OK"
+else
+ echo "Error check: NG"
+ exit 1
+fi
+
+./upmendex -s $srcdir/tests/uni.ist -s $srcdir/tests/rule4.ist $srcdir/tests/rule.idx \
+ -o rule4.ind1 -t rule4.ilg
+if [ $? -eq 254 ]; then
+ echo "Error check: OK"
+else
+ echo "Error check: NG"
+ exit 1
+fi
+
+# test for icu_locale + icu_rules
+./upmendex -s $srcdir/tests/uni.ist -s $srcdir/tests/locale2.ist -s $srcdir/tests/rule1.ist $srcdir/tests/rule.idx \
+ -o locale2-rule1.ind1 -t locale2-rule1.ilg \
+ && diff $srcdir/tests/locale2-rule.ind locale2-rule1.ind1 || exit 1
diff --git a/indexing/upmendex/source/var.h b/indexing/upmendex/source/var.h
index 02e2714910..c9275fcc91 100644
--- a/indexing/upmendex/source/var.h
+++ b/indexing/upmendex/source/var.h
@@ -4,7 +4,7 @@ FILE *efp;
int lines,idxcount,acc=0,reject=0;
int prange=1,fsti=0,lorder=0,bcomp=0,force=0,fpage=0,gflg=0,verb=1,debug=0;
-int warn=0,scount=0,pattr[3]={0,0,0};
+int warn=0,scount=0,pattr[PAGE_COMPOSIT_DEPTH]={0};
/* static char roman[]={"ivxlcdm"},Roman[]={"IVXLCDM"}; */
struct index *ind;
@@ -27,7 +27,7 @@ int line_max=72;
char indent_space[STYBUFSIZE]={"\t\t"};
int indent_length=16;
int priority=0;
-char symbol[STYBUFSIZE]={""};
+char symhead[STYBUFSIZE]={""},numhead[STYBUFSIZE]={""};
char symhead_positive[STYBUFSIZE]={"Symbols"},symhead_negative[STYBUFSIZE]={"symbols"};
char numhead_positive[STYBUFSIZE]={"Numbers"},numhead_negative[STYBUFSIZE]={"numbers"};
int symbol_flag=1;
@@ -35,7 +35,8 @@ int letter_head=1;
UChar atama[STYBUFSIZE],hangul_head[STYBUFSIZE],hanzi_head[STYBUFSIZE]={L'\0'},kana_head[STYBUFSIZE]={L'\0'};
UChar devanagari_head[STYBUFSIZE],thai_head[STYBUFSIZE];
char page_compositor[STYBUFSIZE]={"-"},page_precedence[STYBUFSIZE]={"rnaRA"};
-char character_order[STYBUFSIZE]={"SNLGCJKHDT"};
+char character_order[STYBUFSIZE]={"SNLGCJKHDTah"};
+char script_preamble[11][STYBUFSIZE],script_postamble[11][STYBUFSIZE];
char icu_locale[STYBUFSIZE]={"root"},icu_rules[STYBUFSIZE]={""};
int icu_attributes[UCOL_ATTRIBUTE_COUNT];