diff options
Diffstat (limited to 'Build/source/texk/bibtexu/bibtex-2.c')
-rw-r--r-- | Build/source/texk/bibtexu/bibtex-2.c | 285 |
1 files changed, 266 insertions, 19 deletions
diff --git a/Build/source/texk/bibtexu/bibtex-2.c b/Build/source/texk/bibtexu/bibtex-2.c index 487277e9915..469d6a1a39a 100644 --- a/Build/source/texk/bibtexu/bibtex-2.c +++ b/Build/source/texk/bibtexu/bibtex-2.c @@ -138,8 +138,14 @@ #include "gblvars.h" #include "utils.h" #include "version.h" - - +/* +Include the ICU heads. 23/sep/2009 +*/ +#include "unicode/ustdio.h" +#include "unicode/uchar.h" +#include "unicode/ustring.h" +#include "unicode/ucnv.h" +#include "unicode/ucol.h" /*************************************************************************** * WEB section number: 325 * ~~~~~~~~~~~~~~~~~~~ @@ -492,8 +498,10 @@ End_While_Label: DO_NOTHING; END else BEGIN + INCR (wiz_ptr); push_lit_stk (wiz_functions[wiz_ptr], STK_FN); + END INCR (wiz_ptr); END @@ -613,9 +621,11 @@ End_While_Label: DO_NOTHING; glob_chr_ptr = 0; while (glob_chr_ptr < glb_str_end[str_glb_ptr]) BEGIN - APPEND_CHAR (global_strs[str_glb_ptr][glob_chr_ptr]); + APPEND_CHAR (global_strs[str_glb_ptr][glob_chr_ptr]); + INCR (glob_chr_ptr); END + push_lit_stk (make_string (), STK_STR); END END @@ -713,6 +723,7 @@ BEGIN BEGIN cur_token = first_start; last_token = first_end; + if (cur_token == last_token) BEGIN to_be_written = FALSE; @@ -891,6 +902,7 @@ BEGIN * Here we output all the characters in the token, verbatim. ***************************************************************************/ BEGIN + name_bf_ptr = name_tok[cur_token]; name_bf_xptr = name_tok[cur_token + 1]; if (ex_buf_length + (name_bf_xptr - name_bf_ptr) @@ -920,12 +932,45 @@ BEGIN name_bf_xptr = name_tok[cur_token + 1]; while (name_bf_ptr < name_bf_xptr) BEGIN - if (lex_class[NAME_BUF[name_bf_ptr]] == ALPHA) - BEGIN - APPEND_EX_BUF_CHAR_AND_CHECK ( - NAME_BUF[name_bf_ptr]); - goto Loop_Exit_Label; - END +/* +For output the first character which is encodage UTF-8, we sould discuter different length of character. 23/sep/2009 +*/ + if((lex_class[NAME_BUF[name_bf_ptr]] != WHITE_SPACE) && (NAME_BUF[name_bf_ptr] != LEFT_BRACE)) + BEGIN + if(NAME_BUF[name_bf_ptr] <= 0x7F) + BEGIN + APPEND_EX_BUF_CHAR_AND_CHECK (NAME_BUF[name_bf_ptr]); + END + else if((NAME_BUF[name_bf_ptr] >= 0xC2) && (NAME_BUF[name_bf_ptr] <= 0xDF)) + BEGIN + APPEND_EX_BUF_CHAR_AND_CHECK (NAME_BUF[name_bf_ptr]); + APPEND_EX_BUF_CHAR_AND_CHECK (NAME_BUF[name_bf_ptr+1]); + END + else if((NAME_BUF[name_bf_ptr] >= 0xE0) && (NAME_BUF[name_bf_ptr] <= 0xEF)) + BEGIN + APPEND_EX_BUF_CHAR_AND_CHECK (NAME_BUF[name_bf_ptr]); + APPEND_EX_BUF_CHAR_AND_CHECK (NAME_BUF[name_bf_ptr+1]); + APPEND_EX_BUF_CHAR_AND_CHECK (NAME_BUF[name_bf_ptr+2]); + END + else if((NAME_BUF[name_bf_ptr] >= 0xF0) && (NAME_BUF[name_bf_ptr] <= 0xF4)) + BEGIN + APPEND_EX_BUF_CHAR_AND_CHECK (NAME_BUF[name_bf_ptr]); + APPEND_EX_BUF_CHAR_AND_CHECK (NAME_BUF[name_bf_ptr+1]); + APPEND_EX_BUF_CHAR_AND_CHECK (NAME_BUF[name_bf_ptr+2]); + APPEND_EX_BUF_CHAR_AND_CHECK (NAME_BUF[name_bf_ptr+3]); + END + else + BEGIN + printf("this isn't a right UTF-8 char!"); + END + + goto Loop_Exit_Label; + END + + + + + else if ((NAME_BUF[name_bf_ptr] == LEFT_BRACE) && ((name_bf_ptr + 1) < name_bf_xptr)) BEGIN @@ -1248,6 +1293,7 @@ BEGIN while ( ! scan1 (AT_SIGN)) BEGIN if ( ! input_ln (CUR_BIB_FILE)) + BEGIN goto Exit_Label; END @@ -2786,7 +2832,6 @@ END /*^^^^^^^^^^^^^^^^^^^^^^^^^^ END OF SECTION 47 ^^^^^^^^^^^^^^^^^^^^^^^^^^^*/ - /*************************************************************************** * WEB section number: 198 * ~~~~~~~~~~~~~~~~~~~ @@ -2881,7 +2926,15 @@ END * |ASCII_code|s to compare, so it might give ``interesting'' results * when handling nonletters. ***************************************************************************/ -Boolean_T less_than (CiteNumber_T arg1, CiteNumber_T arg2) +/* +This is original code of "less_than". Now it's less_than_uni. +Actually, it desn't be used in the program. We just resever the original +code. We use the code "less_than" after the code "less_than_uni". This new +"less_than" is for the UTF-8. Of couse, it's also avaible for ASCII. +23/sep/2009 +*/ + +Boolean_T less_than_uni (CiteNumber_T arg1, CiteNumber_T arg2) BEGIN Boolean_T less_than; Integer_T char_ptr; @@ -2897,6 +2950,9 @@ BEGIN BEGIN char1 = ENTRY_STRS(ptr1, char_ptr); char2 = ENTRY_STRS(ptr2, char_ptr); + + printf("%c ",char1);printf("%c",char2); + if (char1 == END_OF_STRING) BEGIN if (char2 == END_OF_STRING) @@ -2912,7 +2968,7 @@ BEGIN else BEGIN CONFUSION ("Duplicate sort key"); - END + END END else BEGIN @@ -2947,11 +3003,56 @@ BEGIN INCR (char_ptr); END Exit_Label: + printf("%d\n",less_than); return (less_than); END /*^^^^^^^^^^^^^^^^^^^^^^^^^^ END OF SECTION 301 ^^^^^^^^^^^^^^^^^^^^^^^^^^^*/ +/* +We use ICU libs to processing UTF-8. First, we have to transform UTF-8 to +Unicode/UChar with the fonction icu_UCHars. Then we use the UCollator +in the ICU libs to conparer the Unicode. There is an option "location", +we use "-o" to indicate the rule of conpare. 23/sep/2009 +*/ +Boolean_T less_than (CiteNumber_T arg1, CiteNumber_T arg2) +BEGIN + StrEntLoc_T ptr1, + ptr2; + Integer_T lenk1, lenk2; + UChar uch1[BUF_SIZE+1], uch2[BUF_SIZE+1]; + UBool u_less; + UCollator * ucol1; + int32_t ucap = BUF_SIZE+1; + int32_t uchlen1, uchlen2; + UErrorCode err1 = U_ZERO_ERROR; + + ptr1 = (arg1 * num_ent_strs) + sort_key_num; + ptr2 = (arg2 * num_ent_strs) + sort_key_num; + + lenk1= strlen(&ENTRY_STRS(ptr1, 0)); + lenk2= strlen(&ENTRY_STRS(ptr2, 0)); + + uchlen1 = icu_toUChars(entry_strs, (ptr1 * (ENT_STR_SIZE+1)), lenk1, uch1, ucap); + uchlen2 = icu_toUChars(entry_strs, (ptr2 * (ENT_STR_SIZE+1)), lenk2, uch2, ucap); + + if(Flag_location) + { + ucol1 = ucol_open(Str_location, &err1); + } + else + { + ucol1 = ucol_open(NULL, &err1); + } + if (!U_SUCCESS(err1)) + { printf("there is a error: U_ZERO_ERROR, open a ucol."); + } + u_less = !ucol_greaterOrEqual(ucol1, uch1, uchlen1, uch2, uchlen2); + + ucol_close(ucol1); + + return u_less; +END /*************************************************************************** @@ -2961,7 +3062,13 @@ END * lower case for the specified part of |buf|. It is system independent * because it uses only the internal representation for characters. ***************************************************************************/ -void lower_case (BufType_T buf, BufPointer_T bf_ptr, + +/* +Because the bibtex use "lower_case" a lot of time, but for processing the charcater +not many. We reserver the original code "lower_case" for other cases. 23/sep/2009 +*/ + +void lower_case (BufType_T buf, BufPointer_T bf_ptr, BufPointer_T len) BEGIN BufPointer_T i; @@ -2970,24 +3077,164 @@ BEGIN BEGIN for (i = bf_ptr; i <= (bf_ptr + len - 1); i++) BEGIN - #ifdef SUPPORT_8BIT - if (IsUpper (buf[i])) + if (IsUpper (buf[i])) BEGIN - buf[i] = ToLower (buf[i]); + buf[i] = ToLower (buf[i]); END #else /* NOT SUPPORT_8BIT */ - if ((buf[i] >= 'A') && (buf[i] <= 'Z')) + if ((buf[i] >= 'A') && (buf[i] <= 'Z')) BEGIN - buf[i] = buf[i] + CASE_DIFFERENCE; + buf[i] = buf[i] + CASE_DIFFERENCE; END #endif /* SUPPORT_8BIT */ - END END +return; END /*^^^^^^^^^^^^^^^^^^^^^^^^^^ END OF SECTION 62 ^^^^^^^^^^^^^^^^^^^^^^^^^^^*/ +/* +"lower_case_uni" is the fonction for processing the characters, actually the UTF-8. +We transform UTF-8 to Unicode, then to low case, then back to UTF-8 for output. +When we transform the character, the length have been changed. So we have do +some job for the length. And the output of this fonction we should be careful +to the length. 23/sep/2009 +*/ +BufPointer_T lower_case_uni (BufType_T buf, BufPointer_T bf_ptr, + BufPointer_T len) +BEGIN + + UChar target[BUF_SIZE+1]; + int32_t tarcap=BUF_SIZE+1; + int32_t tarlen = icu_toUChars(buf, bf_ptr, len, target, tarcap); + + UChar tarlow[BUF_SIZE+1]; + int32_t tlcap=BUF_SIZE+1; + int32_t tllen=icu_strToLower(tarlow, tlcap,target, tarlen); + + unsigned char dest[BUF_SIZE+1]; + int32_t destcap= BUF_SIZE-bf_ptr; + + int32_t tblen=icu_fromUChars(dest, destcap, (const UChar *) tarlow, tllen); + + BufPointer_T i; + if (tblen > 0) + BEGIN + if (len!=tblen) + BEGIN + unsigned char tmp[BUF_SIZE+1]; + BufPointer_T tmppr=0; + for (i=bf_ptr+len;i<=(BUF_SIZE-tblen+len);i++) + BEGIN + tmp[tmppr]=buf[i]; + tmppr++; + END + i=bf_ptr+tblen; + tmppr=0; + for (tmppr=0;tmppr<=(BUF_SIZE-bf_ptr-tblen);tmppr++) + BEGIN + buf[i]=tmp[tmppr]; + i++; + END + END + for (i = 0; i <= (tblen - 1); i++) + BEGIN + + buf[i+bf_ptr]=dest[i]; + + END + END + + return tblen; +END + + +/* +This fonction is for transform UTF-8 to Unicode with ICU libs. 23/sep/2009 +*/ + +int32_t icu_toUChars(BufType_T buf, BufPointer_T bf_ptr,BufPointer_T len,UChar * target, int32_t tarcap) +BEGIN + int32_t tarlen; + UConverter * ucon1; + UErrorCode err1 = U_ZERO_ERROR; + ucon1 = ucnv_open(NULL, &err1); + if (!U_SUCCESS(err1)) + BEGIN + printf("1there is a error: U_ZERO_ERROR"); + END + tarlen = ucnv_toUChars(ucon1, target, tarcap, &buf[bf_ptr], len, &err1); + if (!U_SUCCESS(err1)) + BEGIN + printf("2there is a error: U_ZERO_ERROR"); + END + + return len; +END + +/* +This fonction is for transform Unicode string to low case. 23/sep/2009 +*/ + +int32_t icu_strToLower(UChar * tarlow, int32_t tlcap, UChar * target, int32_t tarlen) +BEGIN + int32_t tllen; + UErrorCode err1 = U_ZERO_ERROR; + if (!U_SUCCESS(err1)) + BEGIN + printf("3there is a error: U_ZERO_ERROR"); + END + if (Flag_language) + { + tllen=u_strToLower(tarlow,tlcap, target,tarlen,Str_language,&err1); + } + else + { + tllen=u_strToLower(tarlow,tlcap, target,tarlen,NULL,&err1); + } + if (!U_SUCCESS(err1)) + BEGIN + int16_t i=0; + printf("4there is a error: U_ZERO_ERROR"); + for (i=0;i<tarlen;i++) + BEGIN + printf("%X", target[i]); + END + printf("\n"); + for (i=0;i<tllen;i++) + BEGIN + printf("%X", tarlow[i]); + END + printf("\n"); + END + + return tllen; +END + + +/* +This fonction is for transform Unicode to UTF-8. 23/sep/2009 +*/ + +int32_t icu_fromUChars(unsigned char * dest, int32_t destcap, const UChar * src, int32_t srclen) +BEGIN + UConverter * ucon2; + UErrorCode err2 = U_ZERO_ERROR; + int32_t tblen; + ucon2 = ucnv_open(NULL, &err2); + if (!U_SUCCESS(err2)) + BEGIN + printf("5there is a error: U_ZERO_ERROR"); + END + tblen=ucnv_fromUChars(ucon2, dest, destcap, src, srclen, &err2); + if (!U_SUCCESS(err2)) + BEGIN + printf("6there is a error: U_ZERO_ERROR"); + END + + return tblen; +END |