diff options
Diffstat (limited to 'Build/source/libs/icu-xetex/common/ustrtrns.c')
-rw-r--r-- | Build/source/libs/icu-xetex/common/ustrtrns.c | 920 |
1 files changed, 845 insertions, 75 deletions
diff --git a/Build/source/libs/icu-xetex/common/ustrtrns.c b/Build/source/libs/icu-xetex/common/ustrtrns.c index e64476996b0..1801d8aeb64 100644 --- a/Build/source/libs/icu-xetex/common/ustrtrns.c +++ b/Build/source/libs/icu-xetex/common/ustrtrns.c @@ -1,7 +1,7 @@ /* ****************************************************************************** * -* Copyright (C) 2001-2004, International Business Machines +* Copyright (C) 2001-2006, International Business Machines * Corporation and others. All Rights Reserved. * ****************************************************************************** @@ -186,69 +186,687 @@ u_strToUTF32(UChar32 *dest, return dest; } +/* for utf8_nextCharSafeBodyTerminated() */ +static const UChar32 +utf8_minLegal[4]={ 0, 0x80, 0x800, 0x10000 }; + +/* + * Version of utf8_nextCharSafeBody() with the following differences: + * - checks for NUL termination instead of length + * - works with pointers instead of indexes + * - always strict (strict==-1) + * + * *ps points to after the lead byte and will be moved to after the last trail byte. + * c is the lead byte. + * @return the code point, or U_SENTINEL + */ +static UChar32 +utf8_nextCharSafeBodyTerminated(const uint8_t **ps, UChar32 c) { + const uint8_t *s=*ps; + uint8_t trail, illegal=0; + uint8_t count=UTF8_COUNT_TRAIL_BYTES(c); + UTF8_MASK_LEAD_BYTE((c), count); + /* count==0 for illegally leading trail bytes and the illegal bytes 0xfe and 0xff */ + switch(count) { + /* each branch falls through to the next one */ + case 5: + case 4: + /* count>=4 is always illegal: no more than 3 trail bytes in Unicode's UTF-8 */ + illegal=1; + break; + case 3: + trail=(uint8_t)(*s++ - 0x80); + c=(c<<6)|trail; + if(trail>0x3f || c>=0x110) { + /* not a trail byte, or code point>0x10ffff (outside Unicode) */ + illegal=1; + break; + } + case 2: + trail=(uint8_t)(*s++ - 0x80); + if(trail>0x3f) { + /* not a trail byte */ + illegal=1; + break; + } + c=(c<<6)|trail; + case 1: + trail=(uint8_t)(*s++ - 0x80); + if(trail>0x3f) { + /* not a trail byte */ + illegal=1; + } + c=(c<<6)|trail; + break; + case 0: + return U_SENTINEL; + /* no default branch to optimize switch() - all values are covered */ + } + + /* correct sequence - all trail bytes have (b7..b6)==(10)? */ + /* illegal is also set if count>=4 */ + if(illegal || c<utf8_minLegal[count] || UTF_IS_SURROGATE(c)) { + /* error handling */ + /* don't go beyond this sequence */ + s=*ps; + while(count>0 && UTF8_IS_TRAIL(*s)) { + ++s; + --count; + } + c=U_SENTINEL; + } + *ps=s; + return c; +} + +/* + * Version of utf8_nextCharSafeBody() with the following differences: + * - works with pointers instead of indexes + * - always strict (strict==-1) + * + * *ps points to after the lead byte and will be moved to after the last trail byte. + * c is the lead byte. + * @return the code point, or U_SENTINEL + */ +static UChar32 +utf8_nextCharSafeBodyPointer(const uint8_t **ps, const uint8_t *limit, UChar32 c) { + const uint8_t *s=*ps; + uint8_t trail, illegal=0; + uint8_t count=UTF8_COUNT_TRAIL_BYTES(c); + if((limit-s)>=count) { + UTF8_MASK_LEAD_BYTE((c), count); + /* count==0 for illegally leading trail bytes and the illegal bytes 0xfe and 0xff */ + switch(count) { + /* each branch falls through to the next one */ + case 5: + case 4: + /* count>=4 is always illegal: no more than 3 trail bytes in Unicode's UTF-8 */ + illegal=1; + break; + case 3: + trail=*s++; + c=(c<<6)|(trail&0x3f); + if(c<0x110) { + illegal|=(trail&0xc0)^0x80; + } else { + /* code point>0x10ffff, outside Unicode */ + illegal=1; + break; + } + case 2: + trail=*s++; + c=(c<<6)|(trail&0x3f); + illegal|=(trail&0xc0)^0x80; + case 1: + trail=*s++; + c=(c<<6)|(trail&0x3f); + illegal|=(trail&0xc0)^0x80; + break; + case 0: + return U_SENTINEL; + /* no default branch to optimize switch() - all values are covered */ + } + } else { + illegal=1; /* too few bytes left */ + } + + /* correct sequence - all trail bytes have (b7..b6)==(10)? */ + /* illegal is also set if count>=4 */ + if(illegal || c<utf8_minLegal[count] || UTF_IS_SURROGATE(c)) { + /* error handling */ + /* don't go beyond this sequence */ + s=*ps; + while(count>0 && s<limit && UTF8_IS_TRAIL(*s)) { + ++s; + --count; + } + c=U_SENTINEL; + } + *ps=s; + return c; +} + U_CAPI UChar* U_EXPORT2 -u_strFromUTF8(UChar *dest, +u_strFromUTF8WithSub(UChar *dest, int32_t destCapacity, int32_t *pDestLength, - const char* src, + const char* src, int32_t srcLength, + UChar32 subchar, int32_t *pNumSubstitutions, UErrorCode *pErrorCode){ UChar *pDest = dest; UChar *pDestLimit = dest+destCapacity; - UChar32 ch=0; - int32_t index = 0; + UChar32 ch; int32_t reqLength = 0; - uint8_t* pSrc = (uint8_t*) src; + const uint8_t* pSrc = (const uint8_t*) src; + uint8_t t1, t2; /* trail bytes */ + int32_t numSubstitutions; /* args check */ if(pErrorCode==NULL || U_FAILURE(*pErrorCode)){ return NULL; } - if((src==NULL) || (srcLength < -1) || (destCapacity<0) || (!dest && destCapacity > 0)){ + if( (src==NULL) || (srcLength < -1) || (destCapacity<0) || (!dest && destCapacity > 0) || + subchar > 0x10ffff || U_IS_SURROGATE(subchar) + ) { *pErrorCode = U_ILLEGAL_ARGUMENT_ERROR; return NULL; } - if(srcLength == -1){ - srcLength = (int32_t)uprv_strlen((char*)pSrc); - } - - while((index < srcLength)&&(pDest<pDestLimit)){ - ch = pSrc[index++]; - if(ch <=0x7f){ - *pDest++=(UChar)ch; - }else{ - ch=utf8_nextCharSafeBody(pSrc, &index, srcLength, ch, -1); - if(ch<0){ - *pErrorCode = U_INVALID_CHAR_FOUND; - return NULL; - }else if(ch<=0xFFFF){ - *(pDest++)=(UChar)ch; - }else{ - *(pDest++)=UTF16_LEAD(ch); - if(pDest<pDestLimit){ - *(pDest++)=UTF16_TRAIL(ch); + numSubstitutions=0; + + /* + * Inline processing of UTF-8 byte sequences: + * + * Byte sequences for the most common characters are handled inline in + * the conversion loops. In order to reduce the path lengths for those + * characters, the tests are arranged in a kind of binary search. + * ASCII (<=0x7f) is checked first, followed by the dividing point + * between 2- and 3-byte sequences (0xe0). + * The 3-byte branch is tested first to speed up CJK text. + * The compiler should combine the subtractions for the two tests for 0xe0. + * Each branch then tests for the other end of its range. + */ + + if(srcLength < 0){ + /* + * Transform a NUL-terminated string. + * The code explicitly checks for NULs only in the lead byte position. + * A NUL byte in the trail byte position fails the trail byte range check anyway. + */ + while(((ch = *pSrc) != 0) && (pDest < pDestLimit)) { + if(ch <= 0x7f){ + *pDest++=(UChar)ch; + ++pSrc; + } else { + if(ch > 0xe0) { + if( /* handle U+1000..U+CFFF inline */ + ch <= 0xec && + (t1 = (uint8_t)(pSrc[1] - 0x80)) <= 0x3f && + (t2 = (uint8_t)(pSrc[2] - 0x80)) <= 0x3f + ) { + /* no need for (ch & 0xf) because the upper bits are truncated after <<12 in the cast to (UChar) */ + *pDest++ = (UChar)((ch << 12) | (t1 << 6) | t2); + pSrc += 3; + continue; + } + } else if(ch < 0xe0) { + if( /* handle U+0080..U+07FF inline */ + ch >= 0xc2 && + (t1 = (uint8_t)(pSrc[1] - 0x80)) <= 0x3f + ) { + *pDest++ = (UChar)(((ch & 0x1f) << 6) | t1); + pSrc += 2; + continue; + } + } + + /* function call for "complicated" and error cases */ + ++pSrc; /* continue after the lead byte */ + ch=utf8_nextCharSafeBodyTerminated(&pSrc, ch); + if(ch<0 && (++numSubstitutions, ch = subchar) < 0) { + *pErrorCode = U_INVALID_CHAR_FOUND; + return NULL; + } else if(ch<=0xFFFF) { + *(pDest++)=(UChar)ch; + } else { + *(pDest++)=UTF16_LEAD(ch); + if(pDest<pDestLimit) { + *(pDest++)=UTF16_TRAIL(ch); + } else { + reqLength++; + break; + } + } + } + } + + /* Pre-flight the rest of the string. */ + while((ch = *pSrc) != 0) { + if(ch <= 0x7f){ + ++reqLength; + ++pSrc; + } else { + if(ch > 0xe0) { + if( /* handle U+1000..U+CFFF inline */ + ch <= 0xec && + (uint8_t)(pSrc[1] - 0x80) <= 0x3f && + (uint8_t)(pSrc[2] - 0x80) <= 0x3f + ) { + ++reqLength; + pSrc += 3; + continue; + } + } else if(ch < 0xe0) { + if( /* handle U+0080..U+07FF inline */ + ch >= 0xc2 && + (uint8_t)(pSrc[1] - 0x80) <= 0x3f + ) { + ++reqLength; + pSrc += 2; + continue; + } + } + + /* function call for "complicated" and error cases */ + ++pSrc; /* continue after the lead byte */ + ch=utf8_nextCharSafeBodyTerminated(&pSrc, ch); + if(ch<0 && (++numSubstitutions, ch = subchar) < 0) { + *pErrorCode = U_INVALID_CHAR_FOUND; + return NULL; + } + reqLength += U16_LENGTH(ch); + } + } + } else /* srcLength >= 0 */ { + const uint8_t *pSrcLimit = pSrc + srcLength; + int32_t count; + + /* Faster loop without ongoing checking for pSrcLimit and pDestLimit. */ + for(;;) { + /* + * Each iteration of the inner loop progresses by at most 3 UTF-8 + * bytes and one UChar, for most characters. + * For supplementary code points (4 & 2), which are rare, + * there is an additional adjustment. + */ + count = (int32_t)(pDestLimit - pDest); + srcLength = (int32_t)((pSrcLimit - pSrc) / 3); + if(count > srcLength) { + count = srcLength; /* min(remaining dest, remaining src/3) */ + } + if(count < 3) { + /* + * Too much overhead if we get near the end of the string, + * continue with the next loop. + */ + break; + } + + do { + ch = *pSrc; + if(ch <= 0x7f){ + *pDest++=(UChar)ch; + ++pSrc; + } else { + if(ch > 0xe0) { + if( /* handle U+1000..U+CFFF inline */ + ch <= 0xec && + (t1 = (uint8_t)(pSrc[1] - 0x80)) <= 0x3f && + (t2 = (uint8_t)(pSrc[2] - 0x80)) <= 0x3f + ) { + /* no need for (ch & 0xf) because the upper bits are truncated after <<12 in the cast to (UChar) */ + *pDest++ = (UChar)((ch << 12) | (t1 << 6) | t2); + pSrc += 3; + continue; + } + } else if(ch < 0xe0) { + if( /* handle U+0080..U+07FF inline */ + ch >= 0xc2 && + (t1 = (uint8_t)(pSrc[1] - 0x80)) <= 0x3f + ) { + *pDest++ = (UChar)(((ch & 0x1f) << 6) | t1); + pSrc += 2; + continue; + } + } + + if(ch >= 0xf0 || subchar > 0xffff) { + /* + * We may read up to six bytes and write up to two UChars, + * which we didn't account for with computing count, + * so we adjust it here. + */ + if(--count == 0) { + break; + } + } + + /* function call for "complicated" and error cases */ + ++pSrc; /* continue after the lead byte */ + ch=utf8_nextCharSafeBodyPointer(&pSrc, pSrcLimit, ch); + if(ch<0 && (++numSubstitutions, ch = subchar) < 0){ + *pErrorCode = U_INVALID_CHAR_FOUND; + return NULL; + }else if(ch<=0xFFFF){ + *(pDest++)=(UChar)ch; + }else{ + *(pDest++)=UTF16_LEAD(ch); + if(pDest<pDestLimit){ + *(pDest++)=UTF16_TRAIL(ch); + }else{ + reqLength++; + break; + } + } + } + } while(--count > 0); + } + + while((pSrc<pSrcLimit) && (pDest<pDestLimit)) { + ch = *pSrc; + if(ch <= 0x7f){ + *pDest++=(UChar)ch; + ++pSrc; + } else { + if(ch > 0xe0) { + if( /* handle U+1000..U+CFFF inline */ + ch <= 0xec && + ((pSrcLimit - pSrc) >= 3) && + (t1 = (uint8_t)(pSrc[1] - 0x80)) <= 0x3f && + (t2 = (uint8_t)(pSrc[2] - 0x80)) <= 0x3f + ) { + /* no need for (ch & 0xf) because the upper bits are truncated after <<12 in the cast to (UChar) */ + *pDest++ = (UChar)((ch << 12) | (t1 << 6) | t2); + pSrc += 3; + continue; + } + } else if(ch < 0xe0) { + if( /* handle U+0080..U+07FF inline */ + ch >= 0xc2 && + ((pSrcLimit - pSrc) >= 2) && + (t1 = (uint8_t)(pSrc[1] - 0x80)) <= 0x3f + ) { + *pDest++ = (UChar)(((ch & 0x1f) << 6) | t1); + pSrc += 2; + continue; + } + } + + /* function call for "complicated" and error cases */ + ++pSrc; /* continue after the lead byte */ + ch=utf8_nextCharSafeBodyPointer(&pSrc, pSrcLimit, ch); + if(ch<0 && (++numSubstitutions, ch = subchar) < 0){ + *pErrorCode = U_INVALID_CHAR_FOUND; + return NULL; + }else if(ch<=0xFFFF){ + *(pDest++)=(UChar)ch; }else{ - reqLength++; - break; + *(pDest++)=UTF16_LEAD(ch); + if(pDest<pDestLimit){ + *(pDest++)=UTF16_TRAIL(ch); + }else{ + reqLength++; + break; + } } } } + /* donot fill the dest buffer just count the UChars needed */ + while(pSrc < pSrcLimit){ + ch = *pSrc; + if(ch <= 0x7f){ + reqLength++; + ++pSrc; + } else { + if(ch > 0xe0) { + if( /* handle U+1000..U+CFFF inline */ + ch <= 0xec && + ((pSrcLimit - pSrc) >= 3) && + (uint8_t)(pSrc[1] - 0x80) <= 0x3f && + (uint8_t)(pSrc[2] - 0x80) <= 0x3f + ) { + reqLength++; + pSrc += 3; + continue; + } + } else if(ch < 0xe0) { + if( /* handle U+0080..U+07FF inline */ + ch >= 0xc2 && + ((pSrcLimit - pSrc) >= 2) && + (uint8_t)(pSrc[1] - 0x80) <= 0x3f + ) { + reqLength++; + pSrc += 2; + continue; + } + } + + /* function call for "complicated" and error cases */ + ++pSrc; /* continue after the lead byte */ + ch=utf8_nextCharSafeBodyPointer(&pSrc, pSrcLimit, ch); + if(ch<0 && (++numSubstitutions, ch = subchar) < 0){ + *pErrorCode = U_INVALID_CHAR_FOUND; + return NULL; + } + reqLength+=UTF_CHAR_LENGTH(ch); + } + } } - /* donot fill the dest buffer just count the UChars needed */ - while(index < srcLength){ - ch = pSrc[index++]; - if(ch <= 0x7f){ - reqLength++; - }else{ - ch=utf8_nextCharSafeBody(pSrc, &index, srcLength, ch, -1); - if(ch<0){ - *pErrorCode = U_INVALID_CHAR_FOUND; - return NULL; + + reqLength+=(int32_t)(pDest - dest); + + if(pNumSubstitutions!=NULL) { + *pNumSubstitutions=numSubstitutions; + } + + if(pDestLength){ + *pDestLength = reqLength; + } + + /* Terminate the buffer */ + u_terminateUChars(dest,destCapacity,reqLength,pErrorCode); + + return dest; +} + +U_CAPI UChar* U_EXPORT2 +u_strFromUTF8(UChar *dest, + int32_t destCapacity, + int32_t *pDestLength, + const char* src, + int32_t srcLength, + UErrorCode *pErrorCode){ + return u_strFromUTF8WithSub( + dest, destCapacity, pDestLength, + src, srcLength, + U_SENTINEL, NULL, + pErrorCode); +} + +U_CAPI UChar * U_EXPORT2 +u_strFromUTF8Lenient(UChar *dest, + int32_t destCapacity, + int32_t *pDestLength, + const char *src, + int32_t srcLength, + UErrorCode *pErrorCode) { + + UChar *pDest = dest; + UChar32 ch; + int32_t reqLength = 0; + uint8_t* pSrc = (uint8_t*) src; + + /* args check */ + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)){ + return NULL; + } + + if((src==NULL) || (srcLength < -1) || (destCapacity<0) || (!dest && destCapacity > 0)) { + *pErrorCode = U_ILLEGAL_ARGUMENT_ERROR; + return NULL; + } + + if(srcLength < 0) { + /* Transform a NUL-terminated string. */ + UChar *pDestLimit = dest+destCapacity; + uint8_t t1, t2, t3; /* trail bytes */ + + while(((ch = *pSrc) != 0) && (pDest < pDestLimit)) { + if(ch < 0xc0) { + /* + * ASCII, or a trail byte in lead position which is treated like + * a single-byte sequence for better character boundary + * resynchronization after illegal sequences. + */ + *pDest++=(UChar)ch; + ++pSrc; + continue; + } else if(ch < 0xe0) { /* U+0080..U+07FF */ + if((t1 = pSrc[1]) != 0) { + /* 0x3080 = (0xc0 << 6) + 0x80 */ + *pDest++ = (UChar)((ch << 6) + t1 - 0x3080); + pSrc += 2; + continue; + } + } else if(ch < 0xf0) { /* U+0800..U+FFFF */ + if((t1 = pSrc[1]) != 0 && (t2 = pSrc[2]) != 0) { + /* no need for (ch & 0xf) because the upper bits are truncated after <<12 in the cast to (UChar) */ + /* 0x2080 = (0x80 << 6) + 0x80 */ + *pDest++ = (UChar)((ch << 12) + (t1 << 6) + t2 - 0x2080); + pSrc += 3; + continue; + } + } else /* f0..f4 */ { /* U+10000..U+10FFFF */ + if((t1 = pSrc[1]) != 0 && (t2 = pSrc[2]) != 0 && (t3 = pSrc[3]) != 0) { + pSrc += 4; + /* 0x3c82080 = (0xf0 << 18) + (0x80 << 12) + (0x80 << 6) + 0x80 */ + ch = (ch << 18) + (t1 << 12) + (t2 << 6) + t3 - 0x3c82080; + *(pDest++) = U16_LEAD(ch); + if(pDest < pDestLimit) { + *(pDest++) = U16_TRAIL(ch); + } else { + reqLength = 1; + break; + } + continue; + } } - reqLength+=UTF_CHAR_LENGTH(ch); + + /* truncated character at the end */ + *pDest++ = 0xfffd; + while(*++pSrc != 0) {} + break; + } + + /* Pre-flight the rest of the string. */ + while((ch = *pSrc) != 0) { + if(ch < 0xc0) { + /* + * ASCII, or a trail byte in lead position which is treated like + * a single-byte sequence for better character boundary + * resynchronization after illegal sequences. + */ + ++reqLength; + ++pSrc; + continue; + } else if(ch < 0xe0) { /* U+0080..U+07FF */ + if(pSrc[1] != 0) { + ++reqLength; + pSrc += 2; + continue; + } + } else if(ch < 0xf0) { /* U+0800..U+FFFF */ + if(pSrc[1] != 0 && pSrc[2] != 0) { + ++reqLength; + pSrc += 3; + continue; + } + } else /* f0..f4 */ { /* U+10000..U+10FFFF */ + if(pSrc[1] != 0 && pSrc[2] != 0 && pSrc[3] != 0) { + reqLength += 2; + pSrc += 4; + continue; + } + } + + /* truncated character at the end */ + ++reqLength; + break; + } + } else /* srcLength >= 0 */ { + const uint8_t *pSrcLimit = pSrc + srcLength; + + /* + * This function requires that if srcLength is given, then it must be + * destCapatity >= srcLength so that we need not check for + * destination buffer overflow in the loop. + */ + if(destCapacity < srcLength) { + if(pDestLength != NULL) { + *pDestLength = srcLength; /* this likely overestimates the true destLength! */ + } + *pErrorCode = U_BUFFER_OVERFLOW_ERROR; + return NULL; + } + + if((pSrcLimit - pSrc) >= 4) { + pSrcLimit -= 3; /* temporarily reduce pSrcLimit */ + + /* in this loop, we can always access at least 4 bytes, up to pSrc+3 */ + do { + ch = *pSrc++; + if(ch < 0xc0) { + /* + * ASCII, or a trail byte in lead position which is treated like + * a single-byte sequence for better character boundary + * resynchronization after illegal sequences. + */ + *pDest++=(UChar)ch; + } else if(ch < 0xe0) { /* U+0080..U+07FF */ + /* 0x3080 = (0xc0 << 6) + 0x80 */ + *pDest++ = (UChar)((ch << 6) + *pSrc++ - 0x3080); + } else if(ch < 0xf0) { /* U+0800..U+FFFF */ + /* no need for (ch & 0xf) because the upper bits are truncated after <<12 in the cast to (UChar) */ + /* 0x2080 = (0x80 << 6) + 0x80 */ + ch = (ch << 12) + (*pSrc++ << 6); + *pDest++ = (UChar)(ch + *pSrc++ - 0x2080); + } else /* f0..f4 */ { /* U+10000..U+10FFFF */ + /* 0x3c82080 = (0xf0 << 18) + (0x80 << 12) + (0x80 << 6) + 0x80 */ + ch = (ch << 18) + (*pSrc++ << 12); + ch += *pSrc++ << 6; + ch += *pSrc++ - 0x3c82080; + *(pDest++) = U16_LEAD(ch); + *(pDest++) = U16_TRAIL(ch); + } + } while(pSrc < pSrcLimit); + + pSrcLimit += 3; /* restore original pSrcLimit */ + } + + while(pSrc < pSrcLimit) { + ch = *pSrc++; + if(ch < 0xc0) { + /* + * ASCII, or a trail byte in lead position which is treated like + * a single-byte sequence for better character boundary + * resynchronization after illegal sequences. + */ + *pDest++=(UChar)ch; + continue; + } else if(ch < 0xe0) { /* U+0080..U+07FF */ + if(pSrc < pSrcLimit) { + /* 0x3080 = (0xc0 << 6) + 0x80 */ + *pDest++ = (UChar)(((ch & 0x1f) << 6) + *pSrc++ - 0x3080); + continue; + } + } else if(ch < 0xf0) { /* U+0800..U+FFFF */ + if((pSrcLimit - pSrc) >= 2) { + /* no need for (ch & 0xf) because the upper bits are truncated after <<12 in the cast to (UChar) */ + /* 0x2080 = (0x80 << 6) + 0x80 */ + ch = (ch << 12) + (*pSrc++ << 6); + *pDest++ = (UChar)(ch + *pSrc++ - 0x2080); + pSrc += 3; + continue; + } + } else /* f0..f4 */ { /* U+10000..U+10FFFF */ + if((pSrcLimit - pSrc) >= 3) { + /* 0x3c82080 = (0xf0 << 18) + (0x80 << 12) + (0x80 << 6) + 0x80 */ + ch = (ch << 18) + (*pSrc++ << 12); + ch += *pSrc++ << 6; + ch += *pSrc++ - 0x3c82080; + *(pDest++) = U16_LEAD(ch); + *(pDest++) = U16_TRAIL(ch); + pSrc += 4; + continue; + } + } + + /* truncated character at the end */ + *pDest++ = 0xfffd; + break; } } @@ -266,11 +884,13 @@ u_strFromUTF8(UChar *dest, static U_INLINE uint8_t * _appendUTF8(uint8_t *pDest, UChar32 c) { - /* c<=0x7f is handled by the caller, here it is 0x80<=c<=0x10ffff */ - if((c)<=0x7ff) { + /* it is 0<=c<=0x10ffff and not a surrogate if called by a validating function */ + if((c)<=0x7f) { + *pDest++=(uint8_t)c; + } else if(c<=0x7ff) { *pDest++=(uint8_t)((c>>6)|0xc0); *pDest++=(uint8_t)((c&0x3f)|0x80); - } else if((uint32_t)(c)<=0xffff) { + } else if(c<=0xffff) { *pDest++=(uint8_t)((c>>12)|0xe0); *pDest++=(uint8_t)(((c>>6)&0x3f)|0x80); *pDest++=(uint8_t)(((c)&0x3f)|0x80); @@ -285,57 +905,86 @@ _appendUTF8(uint8_t *pDest, UChar32 c) { U_CAPI char* U_EXPORT2 -u_strToUTF8(char *dest, +u_strToUTF8WithSub(char *dest, int32_t destCapacity, int32_t *pDestLength, - const UChar *pSrc, + const UChar *pSrc, int32_t srcLength, + UChar32 subchar, int32_t *pNumSubstitutions, UErrorCode *pErrorCode){ int32_t reqLength=0; - const UChar *pSrcLimit; uint32_t ch=0,ch2=0; uint8_t *pDest = (uint8_t *)dest; uint8_t *pDestLimit = pDest + destCapacity; - + int32_t numSubstitutions; /* args check */ if(pErrorCode==NULL || U_FAILURE(*pErrorCode)){ return NULL; } - if((pSrc==NULL) || (srcLength < -1) || (destCapacity<0) || (!dest && destCapacity > 0)){ + if( (pSrc==NULL) || (srcLength < -1) || (destCapacity<0) || (!dest && destCapacity > 0) || + subchar > 0x10ffff || U_IS_SURROGATE(subchar) + ) { *pErrorCode = U_ILLEGAL_ARGUMENT_ERROR; return NULL; } + numSubstitutions=0; + if(srcLength==-1) { - while((ch=*pSrc)!=0 && pDest!=pDestLimit) { + while((ch=*pSrc)!=0) { ++pSrc; if(ch <= 0x7f) { - *pDest++ = (char)ch; - ++reqLength; - continue; - } + if(pDest<pDestLimit) { + *pDest++ = (char)ch; + } else { + reqLength = 1; + break; + } + } else if(ch <= 0x7ff) { + if((pDestLimit - pDest) >= 2) { + *pDest++=(uint8_t)((ch>>6)|0xc0); + *pDest++=(uint8_t)((ch&0x3f)|0x80); + } else { + reqLength = 2; + break; + } + } else if(ch <= 0xd7ff || ch >= 0xe000) { + if((pDestLimit - pDest) >= 3) { + *pDest++=(uint8_t)((ch>>12)|0xe0); + *pDest++=(uint8_t)(((ch>>6)&0x3f)|0x80); + *pDest++=(uint8_t)((ch&0x3f)|0x80); + } else { + reqLength = 3; + break; + } + } else /* ch is a surrogate */ { + int32_t length; - /*need not check for NUL because NUL fails UTF_IS_TRAIL() anyway*/ - if(UTF_IS_SURROGATE(ch)) { + /*need not check for NUL because NUL fails UTF_IS_TRAIL() anyway*/ if(UTF_IS_SURROGATE_FIRST(ch) && UTF_IS_TRAIL(ch2=*pSrc)) { ++pSrc; ch=UTF16_GET_PAIR_VALUE(ch, ch2); + } else if(subchar>=0) { + ch=subchar; + ++numSubstitutions; } else { /* Unicode 3.2 forbids surrogate code points in UTF-8 */ *pErrorCode = U_INVALID_CHAR_FOUND; return NULL; } + + length = U8_LENGTH(ch); + if((pDestLimit - pDest) >= length) { + /* convert and append*/ + pDest=_appendUTF8(pDest, ch); + } else { + reqLength = length; + break; + } } - reqLength += UTF8_CHAR_LENGTH(ch); - /* do we have enough room in destination? */ - if(destCapacity< reqLength){ - break; - } - /* convert and append*/ - pDest=_appendUTF8(pDest, ch); } while((ch=*pSrc++)!=0) { if(ch<=0x7f) { @@ -347,6 +996,9 @@ u_strToUTF8(char *dest, } else if(UTF_IS_SURROGATE_FIRST(ch) && UTF_IS_TRAIL(ch2=*pSrc)) { ++pSrc; reqLength+=4; + } else if(subchar>=0) { + reqLength+=U8_LENGTH(subchar); + ++numSubstitutions; } else { /* Unicode 3.2 forbids surrogate code points in UTF-8 */ *pErrorCode = U_INVALID_CHAR_FOUND; @@ -354,32 +1006,127 @@ u_strToUTF8(char *dest, } } } else { - pSrcLimit = pSrc+srcLength; - while(pSrc<pSrcLimit && pDest<pDestLimit) { + const UChar *pSrcLimit = pSrc+srcLength; + int32_t count; + + /* Faster loop without ongoing checking for pSrcLimit and pDestLimit. */ + for(;;) { + /* + * Each iteration of the inner loop progresses by at most 3 UTF-8 + * bytes and one UChar, for most characters. + * For supplementary code points (4 & 2), which are rare, + * there is an additional adjustment. + */ + count = (int32_t)((pDestLimit - pDest) / 3); + srcLength = (int32_t)(pSrcLimit - pSrc); + if(count > srcLength) { + count = srcLength; /* min(remaining dest/3, remaining src) */ + } + if(count < 3) { + /* + * Too much overhead if we get near the end of the string, + * continue with the next loop. + */ + break; + } + do { + ch=*pSrc++; + if(ch <= 0x7f) { + *pDest++ = (char)ch; + } else if(ch <= 0x7ff) { + *pDest++=(uint8_t)((ch>>6)|0xc0); + *pDest++=(uint8_t)((ch&0x3f)|0x80); + } else if(ch <= 0xd7ff || ch >= 0xe000) { + *pDest++=(uint8_t)((ch>>12)|0xe0); + *pDest++=(uint8_t)(((ch>>6)&0x3f)|0x80); + *pDest++=(uint8_t)((ch&0x3f)|0x80); + } else /* ch is a surrogate */ { + /* + * We will read two UChars and probably output four bytes, + * which we didn't account for with computing count, + * so we adjust it here. + */ + if(--count == 0) { + --pSrc; /* undo ch=*pSrc++ for the lead surrogate */ + break; /* recompute count */ + } + + if(UTF_IS_SURROGATE_FIRST(ch) && UTF_IS_TRAIL(ch2=*pSrc)) { + ++pSrc; + ch=UTF16_GET_PAIR_VALUE(ch, ch2); + + /* writing 4 bytes per 2 UChars is ok */ + *pDest++=(uint8_t)((ch>>18)|0xf0); + *pDest++=(uint8_t)(((ch>>12)&0x3f)|0x80); + *pDest++=(uint8_t)(((ch>>6)&0x3f)|0x80); + *pDest++=(uint8_t)((ch&0x3f)|0x80); + } else { + /* Unicode 3.2 forbids surrogate code points in UTF-8 */ + if(subchar>=0) { + ch=subchar; + ++numSubstitutions; + } else { + *pErrorCode = U_INVALID_CHAR_FOUND; + return NULL; + } + + /* convert and append*/ + pDest=_appendUTF8(pDest, ch); + } + } + } while(--count > 0); + } + + while(pSrc<pSrcLimit) { ch=*pSrc++; if(ch <= 0x7f) { - *pDest++ = (char)ch; - ++reqLength; - continue; - } + if(pDest<pDestLimit) { + *pDest++ = (char)ch; + } else { + reqLength = 1; + break; + } + } else if(ch <= 0x7ff) { + if((pDestLimit - pDest) >= 2) { + *pDest++=(uint8_t)((ch>>6)|0xc0); + *pDest++=(uint8_t)((ch&0x3f)|0x80); + } else { + reqLength = 2; + break; + } + } else if(ch <= 0xd7ff || ch >= 0xe000) { + if((pDestLimit - pDest) >= 3) { + *pDest++=(uint8_t)((ch>>12)|0xe0); + *pDest++=(uint8_t)(((ch>>6)&0x3f)|0x80); + *pDest++=(uint8_t)((ch&0x3f)|0x80); + } else { + reqLength = 3; + break; + } + } else /* ch is a surrogate */ { + int32_t length; - if(UTF_IS_SURROGATE(ch)) { if(UTF_IS_SURROGATE_FIRST(ch) && pSrc<pSrcLimit && UTF_IS_TRAIL(ch2=*pSrc)) { ++pSrc; ch=UTF16_GET_PAIR_VALUE(ch, ch2); + } else if(subchar>=0) { + ch=subchar; + ++numSubstitutions; } else { /* Unicode 3.2 forbids surrogate code points in UTF-8 */ *pErrorCode = U_INVALID_CHAR_FOUND; return NULL; } + + length = U8_LENGTH(ch); + if((pDestLimit - pDest) >= length) { + /* convert and append*/ + pDest=_appendUTF8(pDest, ch); + } else { + reqLength = length; + break; + } } - reqLength += UTF8_CHAR_LENGTH(ch); - /* do we have enough room in destination? */ - if(destCapacity< reqLength){ - break; - } - /* convert and append*/ - pDest=_appendUTF8(pDest, ch); } while(pSrc<pSrcLimit) { ch=*pSrc++; @@ -392,6 +1139,9 @@ u_strToUTF8(char *dest, } else if(UTF_IS_SURROGATE_FIRST(ch) && pSrc<pSrcLimit && UTF_IS_TRAIL(ch2=*pSrc)) { ++pSrc; reqLength+=4; + } else if(subchar>=0) { + reqLength+=U8_LENGTH(subchar); + ++numSubstitutions; } else { /* Unicode 3.2 forbids surrogate code points in UTF-8 */ *pErrorCode = U_INVALID_CHAR_FOUND; @@ -400,6 +1150,12 @@ u_strToUTF8(char *dest, } } + reqLength+=(int32_t)(pDest - (uint8_t *)dest); + + if(pNumSubstitutions!=NULL) { + *pNumSubstitutions=numSubstitutions; + } + if(pDestLength){ *pDestLength = reqLength; } @@ -409,3 +1165,17 @@ u_strToUTF8(char *dest, return (char*)dest; } + +U_CAPI char* U_EXPORT2 +u_strToUTF8(char *dest, + int32_t destCapacity, + int32_t *pDestLength, + const UChar *pSrc, + int32_t srcLength, + UErrorCode *pErrorCode){ + return u_strToUTF8WithSub( + dest, destCapacity, pDestLength, + pSrc, srcLength, + U_SENTINEL, NULL, + pErrorCode); +} |