summaryrefslogtreecommitdiff
path: root/indexing/upmendex/source/fwrite.c
diff options
context:
space:
mode:
Diffstat (limited to 'indexing/upmendex/source/fwrite.c')
-rw-r--r--indexing/upmendex/source/fwrite.c199
1 files changed, 141 insertions, 58 deletions
diff --git a/indexing/upmendex/source/fwrite.c b/indexing/upmendex/source/fwrite.c
index d1685ede1c..9ce382589e 100644
--- a/indexing/upmendex/source/fwrite.c
+++ b/indexing/upmendex/source/fwrite.c
@@ -13,12 +13,12 @@ int line_length=0;
static void printpage(struct index *ind, FILE *fp, int num, char *lbuff);
static int range_check(struct index ind, int count, char *lbuff);
-static void linecheck(char *lbuff, char *tmpbuff);
+static void linecheck(char *lbuff, char *tmpbuff, int force);
static void crcheck(char *lbuff, FILE *fp);
static void index_normalize(UChar *istr, UChar *ini, int *chset);
static int initial_cmp_char(UChar *ini, UChar ch);
static int init_hanzi_header(void);
-static const UNormalizer2* unormalizer_NFD;
+static const UNormalizer2 *unormalizer_NFD, *unormalizer_NFKD;
static int turkish_i;
#define M_NONE 0
@@ -93,7 +93,7 @@ static void fprint_uchar(FILE *fp, const UChar *a, const int mode, const int len
} else
olen=wclen;
if (olen>INITIALLENGTH) {
- warn_printf(efp, "\nWarning, Too long (%d) header.\n", olen);
+ warn_printf(efp, "\nWarning: Too long (%d) header.\n", olen);
wclen=INITIALLENGTH;
} else
wclen=olen;
@@ -128,7 +128,7 @@ void warn_printf(FILE *fp, const char *format, ...)
vsnprintf(print_buff, sizeof print_buff, format, argptr);
va_end(argptr);
- warn++;
+ warn++;
fputs(print_buff, stderr);
if (fp!=stderr) fputs(print_buff, fp);
}
@@ -146,11 +146,27 @@ void verb_printf(FILE *fp, const char *format, ...)
if (fp!=stderr) fputs(print_buff, fp);
}
+static int pnumconv2(struct page *p)
+{
+ int j,k,cc,pclen;
+
+ pclen=strlen(page_compositor);
+ for (j=k=cc=0;j<strlen(p->page);j++) {
+ if (strncmp(p->page+j,page_compositor,pclen)==0) {
+ j+=pclen;
+ k=j;
+ cc++;
+ continue;
+ }
+ }
+ return pnumconv(p->page+k,p->attr[cc]);
+}
+
/* write ind file */
void indwrite(char *filename, struct index *ind, int pagenum)
{
- int i,j,hpoint=0,tpoint=0,ipoint=0,jpoint=0;
+ int i,j,hpoint=0,tpoint=0,ipoint=0,jpoint=0,block_open=0;
char lbuff[BUFFERLEN],obuff[BUFFERLEN];
UChar datama[256],initial[INITIALLENGTH],initial_prev[INITIALLENGTH];
int chset,chset_prev;
@@ -173,12 +189,34 @@ void indwrite(char *filename, struct index *ind, int pagenum)
fprintf(fp,"%s%d%s",setpage_prefix,pagenum,setpage_suffix);
}
perr=U_ZERO_ERROR;
- unormalizer_NFD=unorm2_getInstance(NULL, "nfc", UNORM2_DECOMPOSE, &perr);
+ unormalizer_NFD =unorm2_getInstance(NULL, "nfc", UNORM2_DECOMPOSE, &perr);
+ unormalizer_NFKD=unorm2_getInstance(NULL, "nfkc", UNORM2_DECOMPOSE, &perr);
+
+ if (strlen(symhead)==0) {
+ if (lethead_flag>0) {
+ strcpy(symhead, symhead_positive);
+ }
+ else if (lethead_flag<0) {
+ strcpy(symhead, symhead_negative);
+ }
+ }
+ {
+ if (lethead_flag>0) {
+ strcpy(numhead, numhead_positive);
+ }
+ else if (lethead_flag<0) {
+ strcpy(numhead, numhead_negative);
+ }
+ }
for (i=line_length=0;i<lines;i++) {
index_normalize(ind[i].dic[0], initial, &chset);
if (i==0) {
- if ((CH_LATIN<=chset&&chset<=CH_GREEK) || chset==CH_HANZI) {
+ if (is_any_script(chset) && strlen(script_preamble[chset])) {
+ fputs(script_preamble[chset],fp);
+ block_open=chset;
+ }
+ if ((CH_LATIN<=chset&&chset<=CH_GREEK) || chset==CH_HANZI || (CH_ARABIC<=chset&&chset<=CH_HEBREW)) {
if (lethead_flag!=0) {
fputs(lethead_prefix,fp);
fprint_uchar(fp,initial,lethead_flag,-1);
@@ -280,16 +318,11 @@ void indwrite(char *filename, struct index *ind, int pagenum)
}
}
else {
- if (lethead_flag!=0) {
- if (symbol_flag && strlen(symbol)) {
- fprintf(fp,"%s%s%s",lethead_prefix,symbol,lethead_suffix);
- }
- else if (lethead_flag>0) {
- fprintf(fp,"%s%s%s",lethead_prefix,symhead_positive,lethead_suffix);
- }
- else if (lethead_flag<0) {
- fprintf(fp,"%s%s%s",lethead_prefix,symhead_negative,lethead_suffix);
- }
+ if (lethead_flag!=0 && symbol_flag==2 && chset==CH_NUMERIC) {
+ fprintf(fp,"%s%s%s",lethead_prefix,numhead,lethead_suffix);
+ }
+ if (lethead_flag!=0 && (symbol_flag==1 || (symbol_flag==2 && chset!=CH_NUMERIC))) {
+ fprintf(fp,"%s%s%s",lethead_prefix,symhead,lethead_suffix);
}
widechar_to_multibyte(obuff,BUFFERLEN,ind[i].idx[0]);
SPRINTF(lbuff,"%s%s",item_0,obuff);
@@ -323,7 +356,19 @@ void indwrite(char *filename, struct index *ind, int pagenum)
}
else {
index_normalize(ind[i-1].dic[0], initial_prev, &chset_prev);
- if ((CH_LATIN<=chset&&chset<=CH_GREEK) || chset==CH_HANZI) {
+ if (chset!=chset_prev && is_any_script(chset_prev) && block_open) {
+ if (strlen(script_postamble[chset_prev])) {
+ fputs(script_postamble[chset_prev],fp);
+ }
+ block_open=0;
+ }
+ if (chset!=chset_prev && is_any_script(chset)) {
+ if (strlen(script_preamble[chset])) {
+ fputs(script_preamble[chset],fp);
+ block_open=chset;
+ }
+ }
+ if ((CH_LATIN<=chset&&chset<=CH_GREEK) || chset==CH_HANZI || (CH_ARABIC<=chset&&chset<=CH_HEBREW)) {
if (chset!=chset_prev || ss_comp(initial,initial_prev)) {
fputs(group_skip,fp);
if (lethead_flag!=0) {
@@ -398,18 +443,15 @@ void indwrite(char *filename, struct index *ind, int pagenum)
}
}
else {
- if (CH_LATIN<=chset_prev&&chset_prev<=CH_THAI){
- fputs(group_skip,fp);
- if (lethead_flag!=0 && symbol_flag) {
- if (strlen(symbol)) {
- fprintf(fp,"%s%s%s",lethead_prefix,symbol,lethead_suffix);
- }
- else if (lethead_flag>0) {
- fprintf(fp,"%s%s%s",lethead_prefix,symhead_positive,lethead_suffix);
- }
- else if (lethead_flag<0) {
- fprintf(fp,"%s%s%s",lethead_prefix,symhead_negative,lethead_suffix);
- }
+ if (chset_prev!=chset) {
+ if (is_any_script(chset_prev) || symbol_flag==2)
+ fputs(group_skip,fp);
+ if (lethead_flag!=0 && symbol_flag==2 && chset==CH_NUMERIC) {
+ fprintf(fp,"%s%s%s",lethead_prefix,numhead,lethead_suffix);
+ }
+ if (lethead_flag!=0 && (symbol_flag==1 && is_any_script(chset_prev) ||
+ symbol_flag==2 && chset!=CH_NUMERIC) ) {
+ fprintf(fp,"%s%s%s",lethead_prefix,symhead,lethead_suffix);
}
}
}
@@ -479,6 +521,9 @@ void indwrite(char *filename, struct index *ind, int pagenum)
printpage(ind,fp,i,lbuff);
}
}
+ if (is_any_script(chset) && strlen(script_postamble[chset]) && block_open) {
+ fputs(script_postamble[chset],fp);
+ }
fputs(postamble,fp);
if (fp!=stdout) fclose(fp);
@@ -498,7 +543,9 @@ static void printpage(struct index *ind, FILE *fp, int num, char *lbuff)
for(j=0;j<ind[num].num;j++) {
cc=range_check(ind[num],j,lbuff);
if (cc>j) {
- if (pnumconv(ind[num].p[j].page,ind[num].p[j].attr[0])==pnumconv(ind[num].p[cc].page,ind[num].p[cc].attr[0])) {
+ int epage = pnumconv2(&ind[num].p[cc]);
+ int bpage = pnumconv2(&ind[num].p[j]);
+ if (epage==bpage) {
j=cc-1;
continue;
}
@@ -509,20 +556,18 @@ static void printpage(struct index *ind, FILE *fp, int num, char *lbuff)
if (strlen(ind[num].p[j].enc)>0) {
SPRINTF(buff,"%s%s%s",encap_prefix,ind[num].p[j].enc,encap_infix);
}
- if (strlen(suffix_3p)>0 && (pnumconv(ind[num].p[cc].page,ind[num].p[cc].attr[0])-pnumconv(ind[num].p[j].page,ind[num].p[j].attr[0]))==2) {
- SAPPENDF(buff,"%s",ind[num].p[j].page);
+ /* print beginning of range */
+ SAPPENDF(buff,"%s",ind[num].p[j].page);
+ if (strlen(suffix_3p)>0 && epage-bpage==2) {
SAPPENDF(buff,"%s",suffix_3p);
}
- else if (strlen(suffix_mp)>0 && (pnumconv(ind[num].p[cc].page,ind[num].p[cc].attr[0])-pnumconv(ind[num].p[j].page,ind[num].p[j].attr[0]))>=2) {
- SAPPENDF(buff,"%s",ind[num].p[j].page);
+ else if (strlen(suffix_mp)>0 && epage-bpage>=2) {
SAPPENDF(buff,"%s",suffix_mp);
}
- else if (strlen(suffix_2p)>0 && (pnumconv(ind[num].p[cc].page,ind[num].p[cc].attr[0])-pnumconv(ind[num].p[j].page,ind[num].p[j].attr[0]))==1) {
- SAPPENDF(buff,"%s",ind[num].p[j].page);
+ else if (strlen(suffix_2p)>0 && epage-bpage==1) {
SAPPENDF(buff,"%s",suffix_2p);
}
else {
- SAPPENDF(buff,"%s",ind[num].p[j].page);
SAPPENDF(buff,"%s",delim_r);
SAPPENDF(buff,"%s",ind[num].p[cc].page);
}
@@ -531,14 +576,14 @@ static void printpage(struct index *ind, FILE *fp, int num, char *lbuff)
if (strlen(ind[num].p[j].enc)>0) {
SAPPENDF(tmpbuff,"%s",encap_suffix);
}
- linecheck(lbuff,tmpbuff);
+ linecheck(lbuff,tmpbuff, FALSE);
j=cc;
if (j==ind[num].num) {
goto PRINT;
}
else {
SAPPENDF(tmpbuff,"%s",delim_n);
- linecheck(lbuff,tmpbuff);
+ linecheck(lbuff,tmpbuff, TRUE);
}
}
else if (strlen(ind[num].p[j].enc)>0) {
@@ -560,19 +605,19 @@ static void printpage(struct index *ind, FILE *fp, int num, char *lbuff)
SAPPENDF(tmpbuff,"%s",ind[num].p[j].page);
SAPPENDF(tmpbuff,"%s",encap_suffix);
SAPPENDF(tmpbuff,"%s",delim_n);
- linecheck(lbuff,tmpbuff);
+ linecheck(lbuff,tmpbuff, FALSE);
}
else {
SAPPENDF(tmpbuff,"%s",ind[num].p[j].page);
SAPPENDF(tmpbuff,"%s",delim_n);
- linecheck(lbuff,tmpbuff);
+ linecheck(lbuff,tmpbuff, FALSE);
}
}
else {
/* no encap */
SAPPENDF(tmpbuff,"%s",ind[num].p[j].page);
SAPPENDF(tmpbuff,"%s",delim_n);
- linecheck(lbuff,tmpbuff);
+ linecheck(lbuff,tmpbuff, FALSE);
}
}
@@ -606,7 +651,7 @@ static void printpage(struct index *ind, FILE *fp, int num, char *lbuff)
else {
SAPPENDF(tmpbuff,"%s",ind[num].p[j].page);
}
- linecheck(lbuff,tmpbuff);
+ linecheck(lbuff,tmpbuff, FALSE);
PRINT:
fputs(lbuff,fp);
@@ -652,7 +697,7 @@ static int range_check(struct index ind, int count, char *lbuff)
if (strlen(ind.p[j].enc)>0) {
SPRINTF(tmpbuff,"%s%s%s%s%s%s",encap_prefix,ind.p[j].enc,encap_infix
,ind.p[j].page,encap_suffix,delim_n);
- linecheck(lbuff,tmpbuff);
+ linecheck(lbuff,tmpbuff, FALSE);
}
}
}
@@ -682,8 +727,8 @@ static int range_check(struct index ind, int count, char *lbuff)
break;
}
}
- cc1=pnumconv(ind.p[i-1].page,ind.p[i-1].attr[0]);
- cc2=pnumconv(ind.p[count].page,ind.p[count].attr[0]);
+ cc1=pnumconv2(&ind.p[i-1]);
+ cc2=pnumconv2(&ind.p[count]);
if (cc1>=cc2+2 || (cc1>=cc2+1 && strlen(suffix_2p)) || force) {
return i-1;
}
@@ -691,9 +736,9 @@ static int range_check(struct index ind, int count, char *lbuff)
}
/* check line length */
-static void linecheck(char *lbuff, char *tmpbuff)
+static void linecheck(char *lbuff, char *tmpbuff, int force)
{
- if (line_length+strlen(tmpbuff)>line_max) {
+ if (line_length+strlen(tmpbuff)>line_max && !force) {
SAPPENDF(lbuff,"\n");
SAPPENDF(lbuff,"%s",indent_space);
SAPPENDF(lbuff,"%s",tmpbuff);
@@ -731,12 +776,20 @@ static void index_normalize(UChar *istr, UChar *ini, int *chset)
UChar32 c32;
UErrorCode perr;
UCollationResult order;
- static int hanzi_mode=0;
+ UCollationStrength strgth;
+ static int hanzi_mode=0, i_y_mode=0;
ch=istr[0];
*chset=charset(istr);
ini[1]=L'\0';
+ if (is_circkana(ch) || is_hankana(ch) || is_sqkana(ch) || is_circlatin(ch)) { /* ㋐㋑㋒.. アイウ.. ㌀㌁㌂.. */
+ src[0]=ch; src[1]=0x00;
+ perr=U_ZERO_ERROR;
+ unorm2_normalize(unormalizer_NFKD, src, 1, dest, 8, &perr);
+ if (U_SUCCESS(perr))
+ ch=dest[0];
+ }
if (is_hiragana(ch)) {
ch+=KATATOP-HIRATOP; /* hiragana -> katakana */
}
@@ -748,15 +801,15 @@ static void index_normalize(UChar *istr, UChar *ini, int *chset)
ini[0]=extkanatable[ch-EXKANATOP];
return;
}
- if (is_circkana(ch)) { /* ㋐㋑㋒㋓㋔ .. ㋻㋼㋽㋾ */
- ini[0]=circkanatable[ch-CRKANATOP];
- return;
- }
- else if (ch==0x309F) { ini[0]=0x3088; return; } /* HIRAGANA YORI -> よ */
+ if (ch==0x309F) { ini[0]=0x3088; return; } /* HIRAGANA YORI -> よ */
else if (ch==0x30FF) { ini[0]=0x3053; return; } /* KATAKANA KOTO -> こ */
else if (is_jpn_kana(istr)==2) {
c32=U16_GET_SUPPLEMENTARY(istr[0],istr[1]);
switch (c32) {
+ case 0x1F200: /* 🈀 */
+ ini[0]=0x307B; break; /* ほ */
+ case 0x1B000: /* 𛀀 */
+ ini[0]=0x3048; break; /* え */
case 0x1B150: case 0x1B164:
ini[0]=0x3090; break; /* ゐ */
case 0x1B151: case 0x1B165:
@@ -852,17 +905,29 @@ static void index_normalize(UChar *istr, UChar *ini, int *chset)
u_strcpy(ini,hz_index[lo-1].idx);
return;
}
- else if (is_devanagari(&ch)||is_thai(&ch)) {
- if (ch==0x929||0x931||0x934||(0x958<=ch&&ch<=0x95F)) {
+ else if (is_devanagari(&ch)||is_thai(&ch)||is_arabic(&ch)||is_hebrew(&ch)) {
+ if (ch==0x929||ch==0x931||ch==0x934||(0x958<=ch&&ch<=0x95F) /* Devanagary */
+ ||(0x622<=ch&&ch<=0x626)||ch==0x6C0||ch==0x6C2||ch==0x6D3 /* Arabic */
+ ||(0xFB50<=ch&&ch<=0xFDFF) /* Arabic Presentation Forms-A */
+ ||(0xFE70<=ch&&ch<=0xFEFF) /* Arabic Presentation Forms-B */
+ ||(0xFB1D<=ch&&ch<=0xFB4F) /* Hebrew presentation forms */
+ ) {
src[0]=ch; src[1]=0x00;
perr=U_ZERO_ERROR;
unorm2_normalize(unormalizer_NFD, src, 1, dest, 8, &perr);
if (U_SUCCESS(perr))
ch=dest[0]; /* without modifier */
}
+ else if (ch==0x5DA||ch==0x5DD||ch==0x05DF||ch==0x5E3||ch==0x05E5) { /* Hebrew letter final */
+ ch++;
+ }
ini[0]=ch;
return;
}
+ if (ch>=0xFF21&&ch<=0xFF3A || ch>=0xFF41&&ch<=0xFF5A) {
+ /* Fullwidth latin letter */
+ ch-=0xFF21-0x0041;
+ }
if (ch==0x049||ch==0x069||ch==0x130||ch==0x131||ch==0x0CE||ch==0x0EE) {
/* check dotted/dotless İ,I,i,ı and Î,î for Turkish */
strX[0] = 0x131; strX[1] = 0x5A; strX[2] = 0x00; /* ıZ */
@@ -874,6 +939,22 @@ static void index_normalize(UChar *istr, UChar *ini, int *chset)
return;
}
}
+ if (ch==0x059||ch==0x079) {
+ /* check Y versus I for Lithuanian */
+ if (i_y_mode==0) {
+ strgth = ucol_getStrength(icu_collator);
+ ucol_setStrength(icu_collator, UCOL_PRIMARY);
+ strX[0] = 0x059; strX[1] = 0x00; /* Y */
+ strZ[0] = 0x049; strZ[1] = 0x00; /* I */
+ order = ucol_strcoll(icu_collator, strZ, -1, strX, -1);
+ if (order==UCOL_EQUAL) i_y_mode=2; else i_y_mode=1;
+ ucol_setStrength(icu_collator, strgth);
+ }
+ if (i_y_mode==2) {
+ ini[0] = 0x049; /* I */
+ return;
+ }
+ }
if (ch==0x0C6||ch==0x0E6||ch==0x152||ch==0x153||ch==0x132||ch==0x133
||ch==0x0DF||ch==0x1E9E||ch==0x13F||ch==0x140||ch==0x490||ch==0x491) {
strX[0] = u_toupper(ch); strX[1] = 0x00; /* ex. "Æ" "Œ" */
@@ -976,6 +1057,8 @@ static void index_normalize(UChar *istr, UChar *ini, int *chset)
}
}
}
+ if (ch==0x0AA) ch=L'A';
+ if (ch==0x0BA) ch=L'O';
ini[0]=u_toupper(ch);
return;
}
@@ -1033,7 +1116,7 @@ static int init_hanzi_header(void)
len=u_strlen(pch0);
}
if (len>=INITIALLENGTH) {
- warn_printf(efp, "\nWarning, Too long (%d) hanzi header.\n", len);
+ warn_printf(efp, "\nWarning: Too long (%d) hanzi header.\n", len);
len=INITIALLENGTH-1;
}
u_strncpy(hz_index[k].idx,pch0,len);