diff options
Diffstat (limited to 'indexing/upmendex/source/fwrite.c')
-rw-r--r-- | indexing/upmendex/source/fwrite.c | 203 |
1 files changed, 139 insertions, 64 deletions
diff --git a/indexing/upmendex/source/fwrite.c b/indexing/upmendex/source/fwrite.c index 9ce382589e..f33230df8e 100644 --- a/indexing/upmendex/source/fwrite.c +++ b/indexing/upmendex/source/fwrite.c @@ -12,11 +12,11 @@ int line_length=0; static void printpage(struct index *ind, FILE *fp, int num, char *lbuff); -static int range_check(struct index ind, int count, char *lbuff); -static void linecheck(char *lbuff, char *tmpbuff, int force); +static int range_check(struct index ind, int count, char *lbuff, FILE *fp); +static void linecheck(char *lbuff, char *tmpbuff, FILE *fp, int force); static void crcheck(char *lbuff, FILE *fp); static void index_normalize(UChar *istr, UChar *ini, int *chset); -static int initial_cmp_char(UChar *ini, UChar ch); +static int initial_cmp_char(UChar *ini, UChar *ch); static int init_hanzi_header(void); static const UNormalizer2 *unormalizer_NFD, *unormalizer_NFKD; static int turkish_i; @@ -86,7 +86,7 @@ static void fprint_uchar(FILE *fp, const UChar *a, const int mode, const int len olen=u_strToUpper(istr,INITIALLENGTH,istr,wclen,"",&perr); } else if (mode==M_TO_LOWER) { perr=U_ZERO_ERROR; - olen=u_strToLower(istr,INITIALLENGTH,istr,wclen, istr[0]==0x130&&turkish_i?"tr":"", &perr); + olen=u_strToLower(istr,INITIALLENGTH,istr,wclen, istr[0]==0x130&&turkish_i==2?"tr":"", &perr); } else if (mode==M_TO_TITLE) { perr=U_ZERO_ERROR; olen=u_strToTitle(istr,INITIALLENGTH,istr,wclen,NULL,"",&perr); @@ -166,9 +166,9 @@ static int pnumconv2(struct page *p) /* write ind file */ void indwrite(char *filename, struct index *ind, int pagenum) { - int i,j,hpoint=0,tpoint=0,ipoint=0,jpoint=0,block_open=0; + int i,j,k,hpoint=0,tpoint=0,ipoint=0,jpoint=0,block_open=0; char lbuff[BUFFERLEN],obuff[BUFFERLEN]; - UChar datama[256],initial[INITIALLENGTH],initial_prev[INITIALLENGTH]; + UChar initial[INITIALLENGTH],initial_prev[INITIALLENGTH]; int chset,chset_prev; FILE *fp=NULL; UErrorCode perr; @@ -182,7 +182,6 @@ void indwrite(char *filename, struct index *ind, int pagenum) #endif } - convert(atama,datama); fputs(preamble,fp); if (fpage>0) { @@ -228,31 +227,35 @@ void indwrite(char *filename, struct index *ind, int pagenum) else if (chset==CH_KANA) { if (lethead_flag!=0) { fputs(lethead_prefix,fp); - for (j=hpoint;j<(u_strlen(datama));j++) { - if (initial_cmp_char(initial,datama[j])) { - fprint_uchar(fp,&atama[j-1],M_NONE,1); + for (j=hpoint;j<(u_strlen(kana_head));) { + if (initial_cmp_char(initial,&kana_head[j])) { + k=j; U16_BACK_1(kana_head, 0, k); + fprint_uchar(fp,&kana_head[k],M_NONE,1); hpoint=j; break; } + U16_FWD_1(kana_head, j, -1); } - if (j==(u_strlen(datama))) { - fprint_uchar(fp,&atama[j-1],M_NONE,1); + if (j==(u_strlen(kana_head))) { + k=j; U16_BACK_1(kana_head, 0, k); + fprint_uchar(fp,&kana_head[k],M_NONE,1); } fputs(lethead_suffix,fp); } widechar_to_multibyte(obuff,BUFFERLEN,ind[i].idx[0]); SPRINTF(lbuff,"%s%s",item_0,obuff); - for (hpoint=0;hpoint<(u_strlen(datama));hpoint++) { - if (initial_cmp_char(initial,datama[hpoint])) { + for (hpoint=0;hpoint<(u_strlen(kana_head));) { + if (initial_cmp_char(initial,&kana_head[hpoint])) { break; } + U16_FWD_1(kana_head, hpoint, -1); } } else if (chset==CH_HANGUL) { if (lethead_flag!=0) { fputs(lethead_prefix,fp); for (j=tpoint;j<(u_strlen(hangul_head));j++) { - if (initial_cmp_char(initial,hangul_head[j])) { + if (initial_cmp_char(initial,&hangul_head[j])) { fprint_uchar(fp,&hangul_head[j-1],M_NONE,1); tpoint=j; break; @@ -266,7 +269,7 @@ void indwrite(char *filename, struct index *ind, int pagenum) widechar_to_multibyte(obuff,BUFFERLEN,ind[i].idx[0]); SPRINTF(lbuff,"%s%s",item_0,obuff); for (tpoint=0;tpoint<(u_strlen(hangul_head));tpoint++) { - if (initial_cmp_char(initial,hangul_head[tpoint])) { + if (initial_cmp_char(initial,&hangul_head[tpoint])) { break; } } @@ -274,31 +277,35 @@ void indwrite(char *filename, struct index *ind, int pagenum) else if (chset==CH_DEVANAGARI) { if (lethead_flag!=0) { fputs(lethead_prefix,fp); - for (j=jpoint;j<(u_strlen(devanagari_head));j++) { - if (initial_cmp_char(initial,devanagari_head[j])) { - fprint_uchar(fp,&devanagari_head[j-1],M_NONE,1); + for (j=jpoint;j<(u_strlen(devanagari_head));) { + if (initial_cmp_char(initial,&devanagari_head[j])) { + k=j; U16_BACK_1(devanagari_head, 0, k); + fprint_uchar(fp,&devanagari_head[k],M_NONE,1); jpoint=j; break; } + U16_FWD_1(devanagari_head, j, -1); } if (j==(u_strlen(devanagari_head))) { - fprint_uchar(fp,&devanagari_head[j-1],M_NONE,1); + k=j; U16_BACK_1(devanagari_head, 0, k); + fprint_uchar(fp,&devanagari_head[k],M_NONE,1); } fputs(lethead_suffix,fp); } widechar_to_multibyte(obuff,BUFFERLEN,ind[i].idx[0]); SPRINTF(lbuff,"%s%s",item_0,obuff); - for (jpoint=0;jpoint<(u_strlen(devanagari_head));jpoint++) { - if (initial_cmp_char(initial,devanagari_head[jpoint])) { + for (jpoint=0;jpoint<(u_strlen(devanagari_head));) { + if (initial_cmp_char(initial,&devanagari_head[jpoint])) { break; } + U16_FWD_1(devanagari_head, jpoint, -1); } } else if (chset==CH_THAI) { if (lethead_flag!=0) { fputs(lethead_prefix,fp); for (j=ipoint;j<(u_strlen(thai_head));j++) { - if (initial_cmp_char(initial,thai_head[j])) { + if (initial_cmp_char(initial,&thai_head[j])) { fprint_uchar(fp,&thai_head[j-1],M_NONE,1); ipoint=j; break; @@ -312,7 +319,7 @@ void indwrite(char *filename, struct index *ind, int pagenum) widechar_to_multibyte(obuff,BUFFERLEN,ind[i].idx[0]); SPRINTF(lbuff,"%s%s",item_0,obuff); for (ipoint=0;ipoint<(u_strlen(thai_head));ipoint++) { - if (initial_cmp_char(initial,thai_head[ipoint])) { + if (initial_cmp_char(initial,&thai_head[ipoint])) { break; } } @@ -379,24 +386,26 @@ void indwrite(char *filename, struct index *ind, int pagenum) } } else if (chset==CH_KANA) { - for (j=hpoint;j<(u_strlen(datama));j++) { - if (initial_cmp_char(initial,datama[j])) { + for (j=hpoint;j<(u_strlen(kana_head));) { + if (initial_cmp_char(initial,&kana_head[j])) { break; } + U16_FWD_1(kana_head, j, -1); } if ((j!=hpoint)||(j==0)) { hpoint=j; fputs(group_skip,fp); if (lethead_flag!=0) { + k=j; U16_BACK_1(kana_head, 0, k); fputs(lethead_prefix,fp); - fprint_uchar(fp,&atama[j-1],M_NONE,1); + fprint_uchar(fp,&kana_head[k],M_NONE,1); fputs(lethead_suffix,fp); } } } else if (chset==CH_HANGUL) { for (j=tpoint;j<(u_strlen(hangul_head));j++) { - if (initial_cmp_char(initial,hangul_head[j])) { + if (initial_cmp_char(initial,&hangul_head[j])) { break; } } @@ -411,24 +420,26 @@ void indwrite(char *filename, struct index *ind, int pagenum) } } else if (chset==CH_DEVANAGARI) { - for (j=jpoint;j<(u_strlen(devanagari_head));j++) { - if (initial_cmp_char(initial,devanagari_head[j])) { + for (j=jpoint;j<(u_strlen(devanagari_head));) { + if (initial_cmp_char(initial,&devanagari_head[j])) { break; } + U16_FWD_1(devanagari_head, j, -1); } if ((j!=jpoint)||(j==0)) { jpoint=j; fputs(group_skip,fp); if (lethead_flag!=0) { + k=j; U16_BACK_1(devanagari_head, 0, k); fputs(lethead_prefix,fp); - fprint_uchar(fp,&devanagari_head[j-1],M_NONE,1); + fprint_uchar(fp,&devanagari_head[k],M_NONE,1); fputs(lethead_suffix,fp); } } } else if (chset==CH_THAI) { for (j=ipoint;j<(u_strlen(thai_head));j++) { - if (initial_cmp_char(initial,thai_head[j])) { + if (initial_cmp_char(initial,&thai_head[j])) { break; } } @@ -541,7 +552,7 @@ static void printpage(struct index *ind, FILE *fp, int num, char *lbuff) line_length=strlen(lbuff); for(j=0;j<ind[num].num;j++) { - cc=range_check(ind[num],j,lbuff); + cc=range_check(ind[num],j,lbuff,fp); if (cc>j) { int epage = pnumconv2(&ind[num].p[cc]); int bpage = pnumconv2(&ind[num].p[j]); @@ -576,14 +587,14 @@ static void printpage(struct index *ind, FILE *fp, int num, char *lbuff) if (strlen(ind[num].p[j].enc)>0) { SAPPENDF(tmpbuff,"%s",encap_suffix); } - linecheck(lbuff,tmpbuff, FALSE); + linecheck(lbuff,tmpbuff,fp, FALSE); j=cc; if (j==ind[num].num) { goto PRINT; } else { SAPPENDF(tmpbuff,"%s",delim_n); - linecheck(lbuff,tmpbuff, TRUE); + linecheck(lbuff,tmpbuff,fp, TRUE); } } else if (strlen(ind[num].p[j].enc)>0) { @@ -605,19 +616,19 @@ static void printpage(struct index *ind, FILE *fp, int num, char *lbuff) SAPPENDF(tmpbuff,"%s",ind[num].p[j].page); SAPPENDF(tmpbuff,"%s",encap_suffix); SAPPENDF(tmpbuff,"%s",delim_n); - linecheck(lbuff,tmpbuff, FALSE); + linecheck(lbuff,tmpbuff,fp, FALSE); } else { SAPPENDF(tmpbuff,"%s",ind[num].p[j].page); SAPPENDF(tmpbuff,"%s",delim_n); - linecheck(lbuff,tmpbuff, FALSE); + linecheck(lbuff,tmpbuff,fp, FALSE); } } else { /* no encap */ SAPPENDF(tmpbuff,"%s",ind[num].p[j].page); SAPPENDF(tmpbuff,"%s",delim_n); - linecheck(lbuff,tmpbuff, FALSE); + linecheck(lbuff,tmpbuff,fp, FALSE); } } @@ -651,7 +662,7 @@ static void printpage(struct index *ind, FILE *fp, int num, char *lbuff) else { SAPPENDF(tmpbuff,"%s",ind[num].p[j].page); } - linecheck(lbuff,tmpbuff, FALSE); + linecheck(lbuff,tmpbuff,fp, FALSE); PRINT: fputs(lbuff,fp); @@ -659,7 +670,7 @@ PRINT: lbuff[0]='\0'; } -static int range_check(struct index ind, int count, char *lbuff) +static int range_check(struct index ind, int count, char *lbuff, FILE *fp) { int i,j,k,cc1,cc2,start,force=0; char tmpbuff[BUFFERLEN],errbuff[BUFFERLEN],obuff[BUFFERLEN]; @@ -697,7 +708,7 @@ static int range_check(struct index ind, int count, char *lbuff) if (strlen(ind.p[j].enc)>0) { SPRINTF(tmpbuff,"%s%s%s%s%s%s",encap_prefix,ind.p[j].enc,encap_infix ,ind.p[j].page,encap_suffix,delim_n); - linecheck(lbuff,tmpbuff, FALSE); + linecheck(lbuff,tmpbuff,fp, FALSE); } } } @@ -736,10 +747,12 @@ static int range_check(struct index ind, int count, char *lbuff) } /* check line length */ -static void linecheck(char *lbuff, char *tmpbuff, int force) +static void linecheck(char *lbuff, char *tmpbuff, FILE *fp, int force) { if (line_length+strlen(tmpbuff)>line_max && !force) { SAPPENDF(lbuff,"\n"); + fputs(lbuff,fp); + lbuff[0]='\0'; SAPPENDF(lbuff,"%s",indent_space); SAPPENDF(lbuff,"%s",tmpbuff); line_length=indent_length+strlen(tmpbuff); @@ -771,13 +784,13 @@ static void crcheck(char *lbuff, FILE *fp) static void index_normalize(UChar *istr, UChar *ini, int *chset) { - int k, len, hi, lo, mi; + int k, hi, lo, mi; UChar ch,src[2],dest[8],strX[4],strY[4],strZ[4]; UChar32 c32; UErrorCode perr; UCollationResult order; UCollationStrength strgth; - static int hanzi_mode=0, i_y_mode=0; + static int i_y_mode=0; ch=istr[0]; *chset=charset(istr); @@ -810,6 +823,15 @@ static void index_normalize(UChar *istr, UChar *ini, int *chset) ini[0]=0x307B; break; /* ほ */ case 0x1B000: /* 𛀀 */ ini[0]=0x3048; break; /* え */ + case 0x1B11F: case 0x1B122: /* Archaic WU 𛄟 𛄢 */ + ini[0]=0xD82C; ini[1]=0xDD1F; ini[2]=L'\0'; + break; + case 0x1B120: /* Archaic YI 𛄠 */ + ini[0]=0xD82C; ini[1]=0xDD20; ini[2]=L'\0'; break; + case 0x1B121: case 0x1B001: /* Archaic YE 𛄡 𛀁 */ + ini[0]=0xD82C; ini[1]=0xDD21; ini[2]=L'\0'; break; + case 0x1B132: case 0x1B155: + ini[0]=0x3053; break; /* こ */ case 0x1B150: case 0x1B164: ini[0]=0x3090; break; /* ゐ */ case 0x1B151: case 0x1B165: @@ -885,27 +907,58 @@ static void index_normalize(UChar *istr, UChar *ini, int *chset) ini[0]=ch; return; } - else if ((len=is_hanzi(istr))>0) { + else if (is_hanzi(istr)) { if (hanzi_mode==0) hanzi_mode=init_hanzi_header(); if (hanzi_mode==HANZI_UNKNOWN) { u_strcpy(ini, hz_index[0].idx); return; } - strX[0] =istr[0]; - if (len==2){ strX[1] =istr[1]; } - strX[len]=L'\0'; lo=0; hi=hz_index_len; while (lo<hi) { mi = (lo+hi)/2; u_strcpy(strZ,hz_index[mi].threshold); - order = ucol_strcoll(icu_collator, strZ, -1, strX, -1); + order = ucol_strcoll(icu_collator, strZ, -1, istr, -1); if (order!=UCOL_GREATER) lo=mi+1; else hi=mi; } u_strcpy(ini,hz_index[lo-1].idx); return; } - else if (is_devanagari(&ch)||is_thai(&ch)||is_arabic(&ch)||is_hebrew(&ch)) { + else if (is_thai(&ch)) { + if ((istr[0]>=0x0E40 && istr[0]<=0x0E44) && (istr[1]>=0x0E01 && istr[1]<=0x0E2E)) { + /* Thai reordering :: Vowel followed by Consonant */ + /* https://unicode-org.github.io/icu/userguide/collation/concepts.html#thailao-reordering */ + ini[0]=istr[1]; + } else { + ini[0]=istr[0]; + } + return; + } + else if (is_devanagari(istr)==2) { + ini[0]=istr[0]; ini[1]=istr[1]; ini[2]=L'\0'; + return; + } + else if (is_devanagari(&ch)||is_arabic(&ch)||is_hebrew(&ch)) { + if (ch==0x626) { /* Arabic Letter Yeh with Hamza Above for Uyghur */ + strY[0]=0x626; strY[1]=L'\0'; /* Yeh with Hamza Above */ + strZ[0]=0x628; strZ[1]=L'\0'; /* Beh */ + order = ucol_strcoll(icu_collator, strZ, -1, strY, -1); + if (order==UCOL_LESS) { + ini[0]=strY[0]; ini[1]=strY[1]; + return; + } + } + /* Devanagary, Marathi */ + if ( (istr[0]==0x915 && istr[1]==0x94D && istr[2]== 0x937) /* KSSA क्ष */ + || (istr[0]==0x91C && istr[1]==0x94D && istr[2]== 0x91E)) { /* JNYA ज्ञ */ + strY[0]=istr[0]; strY[1]=istr[1]; strY[2]=istr[2]; strY[3]=L'\0'; + strZ[0]=0x939; strZ[1]=L'\0'; /* HA ह */ + order = ucol_strcoll(icu_collator, strZ, -1, strY, -1); + if (order==UCOL_LESS) { + u_strcpy(ini,strY); + return; + } + } if (ch==0x929||ch==0x931||ch==0x934||(0x958<=ch&&ch<=0x95F) /* Devanagary */ ||(0x622<=ch&&ch<=0x626)||ch==0x6C0||ch==0x6C2||ch==0x6D3 /* Arabic */ ||(0xFB50<=ch&&ch<=0xFDFF) /* Arabic Presentation Forms-A */ @@ -930,12 +983,17 @@ static void index_normalize(UChar *istr, UChar *ini, int *chset) } if (ch==0x049||ch==0x069||ch==0x130||ch==0x131||ch==0x0CE||ch==0x0EE) { /* check dotted/dotless İ,I,i,ı and Î,î for Turkish */ - strX[0] = 0x131; strX[1] = 0x5A; strX[2] = 0x00; /* ıZ */ - strZ[0] = 0x069; strZ[1] = 0x00; /* i */ - order = ucol_strcoll(icu_collator, strZ, -1, strX, -1); - if (order==UCOL_GREATER) { + if (turkish_i==0) { + strgth = ucol_getStrength(icu_collator); + ucol_setStrength(icu_collator, UCOL_SECONDARY); + strX[0] = 0x131; strX[1] = 0x069; strX[2] = 0x00; /* ıi */ + strZ[0] = 0x049; strZ[1] = 0x130; strZ[2] = 0x00; /* Iİ */ + order = ucol_strcoll(icu_collator, strZ, -1, strX, -1); + turkish_i = (order==UCOL_EQUAL) ? 2 : 1; + ucol_setStrength(icu_collator, strgth); + } + if (turkish_i==2) { ini[0] = (ch==0x049||ch==0x131) ? 0x131 : 0x130; /* ı or İ */ - turkish_i=1; return; } } @@ -947,7 +1005,7 @@ static void index_normalize(UChar *istr, UChar *ini, int *chset) strX[0] = 0x059; strX[1] = 0x00; /* Y */ strZ[0] = 0x049; strZ[1] = 0x00; /* I */ order = ucol_strcoll(icu_collator, strZ, -1, strX, -1); - if (order==UCOL_EQUAL) i_y_mode=2; else i_y_mode=1; + i_y_mode = (order==UCOL_EQUAL) ? 2 : 1; ucol_setStrength(icu_collator, strgth); } if (i_y_mode==2) { @@ -956,7 +1014,7 @@ static void index_normalize(UChar *istr, UChar *ini, int *chset) } } if (ch==0x0C6||ch==0x0E6||ch==0x152||ch==0x153||ch==0x132||ch==0x133 - ||ch==0x0DF||ch==0x1E9E||ch==0x13F||ch==0x140||ch==0x490||ch==0x491) { + ||ch==0x0DF||ch==0x1E9E||ch==0x13F||ch==0x140||ch==0x149||ch==0x490||ch==0x491) { strX[0] = u_toupper(ch); strX[1] = 0x00; /* ex. "Æ" "Œ" */ switch (ch) { case 0x0C6: case 0x0E6: /* Æ æ */ @@ -972,6 +1030,8 @@ static void index_normalize(UChar *istr, UChar *ini, int *chset) strZ[0] = 0x49; break; /* I */ case 0x13F: case 0x140: /* Ŀ ŀ */ strZ[0] = 0x4C; break; /* L */ + case 0x149: /* ʼn */ + strZ[0] = 0x4E; break; /* N */ case 0x490: case 0x491: /* Ґ ґ */ strZ[0] = 0x413; break; /* Г */ } @@ -1039,9 +1099,22 @@ static void index_normalize(UChar *istr, UChar *ini, int *chset) } } } + /* NG for Welsh */ + if (strX[0]==0x4E && strX[1]==0x47) { /* NG */ + strY[0]=0x4E; strY[1]=L'\0'; /* N */ + strZ[0]=0x4E; strZ[1]=0x47; strZ[2]=0x5A; strZ[3]=L'\0'; /* NGZ */ + order = ucol_strcoll(icu_collator, strZ, -1, strY, -1); + if (order==UCOL_LESS) { + ini[0]=strX[0]; ini[1]=strX[1]; /* NG */ + ini[2]=L'\0'; + return; + } + } /* other digraphs */ - if(((strX[0]==0x43 || strX[0]==0x44 || strX[0]==0x53 || strX[0]==0x54 || strX[0]==0x58 || strX[0]==0x5A) - && strX[1]==0x48) || /* CH DH SH TH XH ZH */ + if(((strX[0]==0x43 || strX[0]==0x44 || strX[0]==0x50 || strX[0]==0x52 || strX[0]==0x53 || strX[0]==0x54 || + strX[0]==0x58 || strX[0]==0x5A) && strX[1]==0x48) || /* CH DH PH RH SH TH XH ZH */ + (strX[0]==0x44 && strX[1]==0x44) || /* DD */ + (strX[0]==0x46 && strX[1]==0x46) || /* FF */ (strX[0]==0x4C && strX[1]==0x4C) || /* LL */ ((strX[0]==0x47 || strX[0]==0x4C || strX[0]==0x4E) && strX[1]==0x4A) || /* GJ LJ NJ */ (strX[0]==0x52 && strX[1]==0x52) || /* RR */ @@ -1063,12 +1136,14 @@ static void index_normalize(UChar *istr, UChar *ini, int *chset) return; } -static int initial_cmp_char(UChar *ini, UChar ch) +static int initial_cmp_char(UChar *ini, UChar *ch) { - UChar initial_tmp[INITIALLENGTH],istr[2]; - int chset; - istr[0]=ch; - istr[1]=L'\0'; + UChar initial_tmp[INITIALLENGTH],istr[3]; + int chset, l; + l = is_surrogate_pair(ch) ? 2 : 1; + istr[0]=ch[0]; + if (l==2) istr[1]=ch[1]; + istr[l]=L'\0'; index_normalize(istr, initial_tmp, &chset); return (ss_comp(ini, initial_tmp)<0); |