diff options
Diffstat (limited to 'indexing/upmendex/source/fwrite.c')
-rw-r--r-- | indexing/upmendex/source/fwrite.c | 199 |
1 files changed, 141 insertions, 58 deletions
diff --git a/indexing/upmendex/source/fwrite.c b/indexing/upmendex/source/fwrite.c index d1685ede1c..9ce382589e 100644 --- a/indexing/upmendex/source/fwrite.c +++ b/indexing/upmendex/source/fwrite.c @@ -13,12 +13,12 @@ int line_length=0; static void printpage(struct index *ind, FILE *fp, int num, char *lbuff); static int range_check(struct index ind, int count, char *lbuff); -static void linecheck(char *lbuff, char *tmpbuff); +static void linecheck(char *lbuff, char *tmpbuff, int force); static void crcheck(char *lbuff, FILE *fp); static void index_normalize(UChar *istr, UChar *ini, int *chset); static int initial_cmp_char(UChar *ini, UChar ch); static int init_hanzi_header(void); -static const UNormalizer2* unormalizer_NFD; +static const UNormalizer2 *unormalizer_NFD, *unormalizer_NFKD; static int turkish_i; #define M_NONE 0 @@ -93,7 +93,7 @@ static void fprint_uchar(FILE *fp, const UChar *a, const int mode, const int len } else olen=wclen; if (olen>INITIALLENGTH) { - warn_printf(efp, "\nWarning, Too long (%d) header.\n", olen); + warn_printf(efp, "\nWarning: Too long (%d) header.\n", olen); wclen=INITIALLENGTH; } else wclen=olen; @@ -128,7 +128,7 @@ void warn_printf(FILE *fp, const char *format, ...) vsnprintf(print_buff, sizeof print_buff, format, argptr); va_end(argptr); - warn++; + warn++; fputs(print_buff, stderr); if (fp!=stderr) fputs(print_buff, fp); } @@ -146,11 +146,27 @@ void verb_printf(FILE *fp, const char *format, ...) if (fp!=stderr) fputs(print_buff, fp); } +static int pnumconv2(struct page *p) +{ + int j,k,cc,pclen; + + pclen=strlen(page_compositor); + for (j=k=cc=0;j<strlen(p->page);j++) { + if (strncmp(p->page+j,page_compositor,pclen)==0) { + j+=pclen; + k=j; + cc++; + continue; + } + } + return pnumconv(p->page+k,p->attr[cc]); +} + /* write ind file */ void indwrite(char *filename, struct index *ind, int pagenum) { - int i,j,hpoint=0,tpoint=0,ipoint=0,jpoint=0; + int i,j,hpoint=0,tpoint=0,ipoint=0,jpoint=0,block_open=0; char lbuff[BUFFERLEN],obuff[BUFFERLEN]; UChar datama[256],initial[INITIALLENGTH],initial_prev[INITIALLENGTH]; int chset,chset_prev; @@ -173,12 +189,34 @@ void indwrite(char *filename, struct index *ind, int pagenum) fprintf(fp,"%s%d%s",setpage_prefix,pagenum,setpage_suffix); } perr=U_ZERO_ERROR; - unormalizer_NFD=unorm2_getInstance(NULL, "nfc", UNORM2_DECOMPOSE, &perr); + unormalizer_NFD =unorm2_getInstance(NULL, "nfc", UNORM2_DECOMPOSE, &perr); + unormalizer_NFKD=unorm2_getInstance(NULL, "nfkc", UNORM2_DECOMPOSE, &perr); + + if (strlen(symhead)==0) { + if (lethead_flag>0) { + strcpy(symhead, symhead_positive); + } + else if (lethead_flag<0) { + strcpy(symhead, symhead_negative); + } + } + { + if (lethead_flag>0) { + strcpy(numhead, numhead_positive); + } + else if (lethead_flag<0) { + strcpy(numhead, numhead_negative); + } + } for (i=line_length=0;i<lines;i++) { index_normalize(ind[i].dic[0], initial, &chset); if (i==0) { - if ((CH_LATIN<=chset&&chset<=CH_GREEK) || chset==CH_HANZI) { + if (is_any_script(chset) && strlen(script_preamble[chset])) { + fputs(script_preamble[chset],fp); + block_open=chset; + } + if ((CH_LATIN<=chset&&chset<=CH_GREEK) || chset==CH_HANZI || (CH_ARABIC<=chset&&chset<=CH_HEBREW)) { if (lethead_flag!=0) { fputs(lethead_prefix,fp); fprint_uchar(fp,initial,lethead_flag,-1); @@ -280,16 +318,11 @@ void indwrite(char *filename, struct index *ind, int pagenum) } } else { - if (lethead_flag!=0) { - if (symbol_flag && strlen(symbol)) { - fprintf(fp,"%s%s%s",lethead_prefix,symbol,lethead_suffix); - } - else if (lethead_flag>0) { - fprintf(fp,"%s%s%s",lethead_prefix,symhead_positive,lethead_suffix); - } - else if (lethead_flag<0) { - fprintf(fp,"%s%s%s",lethead_prefix,symhead_negative,lethead_suffix); - } + if (lethead_flag!=0 && symbol_flag==2 && chset==CH_NUMERIC) { + fprintf(fp,"%s%s%s",lethead_prefix,numhead,lethead_suffix); + } + if (lethead_flag!=0 && (symbol_flag==1 || (symbol_flag==2 && chset!=CH_NUMERIC))) { + fprintf(fp,"%s%s%s",lethead_prefix,symhead,lethead_suffix); } widechar_to_multibyte(obuff,BUFFERLEN,ind[i].idx[0]); SPRINTF(lbuff,"%s%s",item_0,obuff); @@ -323,7 +356,19 @@ void indwrite(char *filename, struct index *ind, int pagenum) } else { index_normalize(ind[i-1].dic[0], initial_prev, &chset_prev); - if ((CH_LATIN<=chset&&chset<=CH_GREEK) || chset==CH_HANZI) { + if (chset!=chset_prev && is_any_script(chset_prev) && block_open) { + if (strlen(script_postamble[chset_prev])) { + fputs(script_postamble[chset_prev],fp); + } + block_open=0; + } + if (chset!=chset_prev && is_any_script(chset)) { + if (strlen(script_preamble[chset])) { + fputs(script_preamble[chset],fp); + block_open=chset; + } + } + if ((CH_LATIN<=chset&&chset<=CH_GREEK) || chset==CH_HANZI || (CH_ARABIC<=chset&&chset<=CH_HEBREW)) { if (chset!=chset_prev || ss_comp(initial,initial_prev)) { fputs(group_skip,fp); if (lethead_flag!=0) { @@ -398,18 +443,15 @@ void indwrite(char *filename, struct index *ind, int pagenum) } } else { - if (CH_LATIN<=chset_prev&&chset_prev<=CH_THAI){ - fputs(group_skip,fp); - if (lethead_flag!=0 && symbol_flag) { - if (strlen(symbol)) { - fprintf(fp,"%s%s%s",lethead_prefix,symbol,lethead_suffix); - } - else if (lethead_flag>0) { - fprintf(fp,"%s%s%s",lethead_prefix,symhead_positive,lethead_suffix); - } - else if (lethead_flag<0) { - fprintf(fp,"%s%s%s",lethead_prefix,symhead_negative,lethead_suffix); - } + if (chset_prev!=chset) { + if (is_any_script(chset_prev) || symbol_flag==2) + fputs(group_skip,fp); + if (lethead_flag!=0 && symbol_flag==2 && chset==CH_NUMERIC) { + fprintf(fp,"%s%s%s",lethead_prefix,numhead,lethead_suffix); + } + if (lethead_flag!=0 && (symbol_flag==1 && is_any_script(chset_prev) || + symbol_flag==2 && chset!=CH_NUMERIC) ) { + fprintf(fp,"%s%s%s",lethead_prefix,symhead,lethead_suffix); } } } @@ -479,6 +521,9 @@ void indwrite(char *filename, struct index *ind, int pagenum) printpage(ind,fp,i,lbuff); } } + if (is_any_script(chset) && strlen(script_postamble[chset]) && block_open) { + fputs(script_postamble[chset],fp); + } fputs(postamble,fp); if (fp!=stdout) fclose(fp); @@ -498,7 +543,9 @@ static void printpage(struct index *ind, FILE *fp, int num, char *lbuff) for(j=0;j<ind[num].num;j++) { cc=range_check(ind[num],j,lbuff); if (cc>j) { - if (pnumconv(ind[num].p[j].page,ind[num].p[j].attr[0])==pnumconv(ind[num].p[cc].page,ind[num].p[cc].attr[0])) { + int epage = pnumconv2(&ind[num].p[cc]); + int bpage = pnumconv2(&ind[num].p[j]); + if (epage==bpage) { j=cc-1; continue; } @@ -509,20 +556,18 @@ static void printpage(struct index *ind, FILE *fp, int num, char *lbuff) if (strlen(ind[num].p[j].enc)>0) { SPRINTF(buff,"%s%s%s",encap_prefix,ind[num].p[j].enc,encap_infix); } - if (strlen(suffix_3p)>0 && (pnumconv(ind[num].p[cc].page,ind[num].p[cc].attr[0])-pnumconv(ind[num].p[j].page,ind[num].p[j].attr[0]))==2) { - SAPPENDF(buff,"%s",ind[num].p[j].page); + /* print beginning of range */ + SAPPENDF(buff,"%s",ind[num].p[j].page); + if (strlen(suffix_3p)>0 && epage-bpage==2) { SAPPENDF(buff,"%s",suffix_3p); } - else if (strlen(suffix_mp)>0 && (pnumconv(ind[num].p[cc].page,ind[num].p[cc].attr[0])-pnumconv(ind[num].p[j].page,ind[num].p[j].attr[0]))>=2) { - SAPPENDF(buff,"%s",ind[num].p[j].page); + else if (strlen(suffix_mp)>0 && epage-bpage>=2) { SAPPENDF(buff,"%s",suffix_mp); } - else if (strlen(suffix_2p)>0 && (pnumconv(ind[num].p[cc].page,ind[num].p[cc].attr[0])-pnumconv(ind[num].p[j].page,ind[num].p[j].attr[0]))==1) { - SAPPENDF(buff,"%s",ind[num].p[j].page); + else if (strlen(suffix_2p)>0 && epage-bpage==1) { SAPPENDF(buff,"%s",suffix_2p); } else { - SAPPENDF(buff,"%s",ind[num].p[j].page); SAPPENDF(buff,"%s",delim_r); SAPPENDF(buff,"%s",ind[num].p[cc].page); } @@ -531,14 +576,14 @@ static void printpage(struct index *ind, FILE *fp, int num, char *lbuff) if (strlen(ind[num].p[j].enc)>0) { SAPPENDF(tmpbuff,"%s",encap_suffix); } - linecheck(lbuff,tmpbuff); + linecheck(lbuff,tmpbuff, FALSE); j=cc; if (j==ind[num].num) { goto PRINT; } else { SAPPENDF(tmpbuff,"%s",delim_n); - linecheck(lbuff,tmpbuff); + linecheck(lbuff,tmpbuff, TRUE); } } else if (strlen(ind[num].p[j].enc)>0) { @@ -560,19 +605,19 @@ static void printpage(struct index *ind, FILE *fp, int num, char *lbuff) SAPPENDF(tmpbuff,"%s",ind[num].p[j].page); SAPPENDF(tmpbuff,"%s",encap_suffix); SAPPENDF(tmpbuff,"%s",delim_n); - linecheck(lbuff,tmpbuff); + linecheck(lbuff,tmpbuff, FALSE); } else { SAPPENDF(tmpbuff,"%s",ind[num].p[j].page); SAPPENDF(tmpbuff,"%s",delim_n); - linecheck(lbuff,tmpbuff); + linecheck(lbuff,tmpbuff, FALSE); } } else { /* no encap */ SAPPENDF(tmpbuff,"%s",ind[num].p[j].page); SAPPENDF(tmpbuff,"%s",delim_n); - linecheck(lbuff,tmpbuff); + linecheck(lbuff,tmpbuff, FALSE); } } @@ -606,7 +651,7 @@ static void printpage(struct index *ind, FILE *fp, int num, char *lbuff) else { SAPPENDF(tmpbuff,"%s",ind[num].p[j].page); } - linecheck(lbuff,tmpbuff); + linecheck(lbuff,tmpbuff, FALSE); PRINT: fputs(lbuff,fp); @@ -652,7 +697,7 @@ static int range_check(struct index ind, int count, char *lbuff) if (strlen(ind.p[j].enc)>0) { SPRINTF(tmpbuff,"%s%s%s%s%s%s",encap_prefix,ind.p[j].enc,encap_infix ,ind.p[j].page,encap_suffix,delim_n); - linecheck(lbuff,tmpbuff); + linecheck(lbuff,tmpbuff, FALSE); } } } @@ -682,8 +727,8 @@ static int range_check(struct index ind, int count, char *lbuff) break; } } - cc1=pnumconv(ind.p[i-1].page,ind.p[i-1].attr[0]); - cc2=pnumconv(ind.p[count].page,ind.p[count].attr[0]); + cc1=pnumconv2(&ind.p[i-1]); + cc2=pnumconv2(&ind.p[count]); if (cc1>=cc2+2 || (cc1>=cc2+1 && strlen(suffix_2p)) || force) { return i-1; } @@ -691,9 +736,9 @@ static int range_check(struct index ind, int count, char *lbuff) } /* check line length */ -static void linecheck(char *lbuff, char *tmpbuff) +static void linecheck(char *lbuff, char *tmpbuff, int force) { - if (line_length+strlen(tmpbuff)>line_max) { + if (line_length+strlen(tmpbuff)>line_max && !force) { SAPPENDF(lbuff,"\n"); SAPPENDF(lbuff,"%s",indent_space); SAPPENDF(lbuff,"%s",tmpbuff); @@ -731,12 +776,20 @@ static void index_normalize(UChar *istr, UChar *ini, int *chset) UChar32 c32; UErrorCode perr; UCollationResult order; - static int hanzi_mode=0; + UCollationStrength strgth; + static int hanzi_mode=0, i_y_mode=0; ch=istr[0]; *chset=charset(istr); ini[1]=L'\0'; + if (is_circkana(ch) || is_hankana(ch) || is_sqkana(ch) || is_circlatin(ch)) { /* ㋐㋑㋒.. アイウ.. ㌀㌁㌂.. */ + src[0]=ch; src[1]=0x00; + perr=U_ZERO_ERROR; + unorm2_normalize(unormalizer_NFKD, src, 1, dest, 8, &perr); + if (U_SUCCESS(perr)) + ch=dest[0]; + } if (is_hiragana(ch)) { ch+=KATATOP-HIRATOP; /* hiragana -> katakana */ } @@ -748,15 +801,15 @@ static void index_normalize(UChar *istr, UChar *ini, int *chset) ini[0]=extkanatable[ch-EXKANATOP]; return; } - if (is_circkana(ch)) { /* ㋐㋑㋒㋓㋔ .. ㋻㋼㋽㋾ */ - ini[0]=circkanatable[ch-CRKANATOP]; - return; - } - else if (ch==0x309F) { ini[0]=0x3088; return; } /* HIRAGANA YORI -> よ */ + if (ch==0x309F) { ini[0]=0x3088; return; } /* HIRAGANA YORI -> よ */ else if (ch==0x30FF) { ini[0]=0x3053; return; } /* KATAKANA KOTO -> こ */ else if (is_jpn_kana(istr)==2) { c32=U16_GET_SUPPLEMENTARY(istr[0],istr[1]); switch (c32) { + case 0x1F200: /* 🈀 */ + ini[0]=0x307B; break; /* ほ */ + case 0x1B000: /* 𛀀 */ + ini[0]=0x3048; break; /* え */ case 0x1B150: case 0x1B164: ini[0]=0x3090; break; /* ゐ */ case 0x1B151: case 0x1B165: @@ -852,17 +905,29 @@ static void index_normalize(UChar *istr, UChar *ini, int *chset) u_strcpy(ini,hz_index[lo-1].idx); return; } - else if (is_devanagari(&ch)||is_thai(&ch)) { - if (ch==0x929||0x931||0x934||(0x958<=ch&&ch<=0x95F)) { + else if (is_devanagari(&ch)||is_thai(&ch)||is_arabic(&ch)||is_hebrew(&ch)) { + if (ch==0x929||ch==0x931||ch==0x934||(0x958<=ch&&ch<=0x95F) /* Devanagary */ + ||(0x622<=ch&&ch<=0x626)||ch==0x6C0||ch==0x6C2||ch==0x6D3 /* Arabic */ + ||(0xFB50<=ch&&ch<=0xFDFF) /* Arabic Presentation Forms-A */ + ||(0xFE70<=ch&&ch<=0xFEFF) /* Arabic Presentation Forms-B */ + ||(0xFB1D<=ch&&ch<=0xFB4F) /* Hebrew presentation forms */ + ) { src[0]=ch; src[1]=0x00; perr=U_ZERO_ERROR; unorm2_normalize(unormalizer_NFD, src, 1, dest, 8, &perr); if (U_SUCCESS(perr)) ch=dest[0]; /* without modifier */ } + else if (ch==0x5DA||ch==0x5DD||ch==0x05DF||ch==0x5E3||ch==0x05E5) { /* Hebrew letter final */ + ch++; + } ini[0]=ch; return; } + if (ch>=0xFF21&&ch<=0xFF3A || ch>=0xFF41&&ch<=0xFF5A) { + /* Fullwidth latin letter */ + ch-=0xFF21-0x0041; + } if (ch==0x049||ch==0x069||ch==0x130||ch==0x131||ch==0x0CE||ch==0x0EE) { /* check dotted/dotless İ,I,i,ı and Î,î for Turkish */ strX[0] = 0x131; strX[1] = 0x5A; strX[2] = 0x00; /* ıZ */ @@ -874,6 +939,22 @@ static void index_normalize(UChar *istr, UChar *ini, int *chset) return; } } + if (ch==0x059||ch==0x079) { + /* check Y versus I for Lithuanian */ + if (i_y_mode==0) { + strgth = ucol_getStrength(icu_collator); + ucol_setStrength(icu_collator, UCOL_PRIMARY); + strX[0] = 0x059; strX[1] = 0x00; /* Y */ + strZ[0] = 0x049; strZ[1] = 0x00; /* I */ + order = ucol_strcoll(icu_collator, strZ, -1, strX, -1); + if (order==UCOL_EQUAL) i_y_mode=2; else i_y_mode=1; + ucol_setStrength(icu_collator, strgth); + } + if (i_y_mode==2) { + ini[0] = 0x049; /* I */ + return; + } + } if (ch==0x0C6||ch==0x0E6||ch==0x152||ch==0x153||ch==0x132||ch==0x133 ||ch==0x0DF||ch==0x1E9E||ch==0x13F||ch==0x140||ch==0x490||ch==0x491) { strX[0] = u_toupper(ch); strX[1] = 0x00; /* ex. "Æ" "Œ" */ @@ -976,6 +1057,8 @@ static void index_normalize(UChar *istr, UChar *ini, int *chset) } } } + if (ch==0x0AA) ch=L'A'; + if (ch==0x0BA) ch=L'O'; ini[0]=u_toupper(ch); return; } @@ -1033,7 +1116,7 @@ static int init_hanzi_header(void) len=u_strlen(pch0); } if (len>=INITIALLENGTH) { - warn_printf(efp, "\nWarning, Too long (%d) hanzi header.\n", len); + warn_printf(efp, "\nWarning: Too long (%d) hanzi header.\n", len); len=INITIALLENGTH-1; } u_strncpy(hz_index[k].idx,pch0,len); |