diff options
Diffstat (limited to 'Build/source/libs/icu/icu-4.8.1/test/cintltst/bocu1tst.c')
-rw-r--r-- | Build/source/libs/icu/icu-4.8.1/test/cintltst/bocu1tst.c | 1032 |
1 files changed, 1032 insertions, 0 deletions
diff --git a/Build/source/libs/icu/icu-4.8.1/test/cintltst/bocu1tst.c b/Build/source/libs/icu/icu-4.8.1/test/cintltst/bocu1tst.c new file mode 100644 index 00000000000..d089c561031 --- /dev/null +++ b/Build/source/libs/icu/icu-4.8.1/test/cintltst/bocu1tst.c @@ -0,0 +1,1032 @@ +/* +****************************************************************************** +* +* Copyright (C) 2002-2011, International Business Machines +* Corporation and others. All Rights Reserved. +* +****************************************************************************** +* file name: bocu1tst.c +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2002may27 +* created by: Markus W. Scherer +* +* This is the reference implementation of BOCU-1, +* the MIME-friendly form of the Binary Ordered Compression for Unicode, +* taken directly from ### http://source.icu-project.org/repos/icu/icuhtml/trunk/design/conversion/bocu1/ +* The files bocu1.h and bocu1.c from the design folder are taken +* verbatim (minus copyright and #include) and copied together into this file. +* The reference code and some of the reference bocu1tst.c +* is modified to run as part of the ICU cintltst +* test framework (minus main(), log_ln() etc. instead of printf()). +* +* This reference implementation is used here to verify +* the ICU BOCU-1 implementation, which is +* adapted for ICU conversion APIs and optimized. +* ### links in design doc to here and to ucnvbocu.c +*/ + +#include "unicode/utypes.h" +#include "unicode/ustring.h" +#include "unicode/ucnv.h" +#include "cmemory.h" +#include "cintltst.h" + +#define LENGTHOF(array) (sizeof(array)/sizeof((array)[0])) + +/* icuhtml/design/conversion/bocu1/bocu1.h ---------------------------------- */ + +/* BOCU-1 constants and macros ---------------------------------------------- */ + +/* + * BOCU-1 encodes the code points of a Unicode string as + * a sequence of byte-encoded differences (slope detection), + * preserving lexical order. + * + * Optimize the difference-taking for runs of Unicode text within + * small scripts: + * + * Most small scripts are allocated within aligned 128-blocks of Unicode + * code points. Lexical order is preserved if the "previous code point" state + * is always moved into the middle of such a block. + * + * Additionally, "prev" is moved from anywhere in the Unihan and Hangul + * areas into the middle of those areas. + * + * C0 control codes and space are encoded with their US-ASCII bytes. + * "prev" is reset for C0 controls but not for space. + */ + +/* initial value for "prev": middle of the ASCII range */ +#define BOCU1_ASCII_PREV 0x40 + +/* bounding byte values for differences */ +#define BOCU1_MIN 0x21 +#define BOCU1_MIDDLE 0x90 +#define BOCU1_MAX_LEAD 0xfe + +/* add the L suffix to make computations with BOCU1_MAX_TRAIL work on 16-bit compilers */ +#define BOCU1_MAX_TRAIL 0xffL +#define BOCU1_RESET 0xff + +/* number of lead bytes */ +#define BOCU1_COUNT (BOCU1_MAX_LEAD-BOCU1_MIN+1) + +/* adjust trail byte counts for the use of some C0 control byte values */ +#define BOCU1_TRAIL_CONTROLS_COUNT 20 +#define BOCU1_TRAIL_BYTE_OFFSET (BOCU1_MIN-BOCU1_TRAIL_CONTROLS_COUNT) + +/* number of trail bytes */ +#define BOCU1_TRAIL_COUNT ((BOCU1_MAX_TRAIL-BOCU1_MIN+1)+BOCU1_TRAIL_CONTROLS_COUNT) + +/* + * number of positive and negative single-byte codes + * (counting 0==BOCU1_MIDDLE among the positive ones) + */ +#define BOCU1_SINGLE 64 + +/* number of lead bytes for positive and negative 2/3/4-byte sequences */ +#define BOCU1_LEAD_2 43 +#define BOCU1_LEAD_3 3 +#define BOCU1_LEAD_4 1 + +/* The difference value range for single-byters. */ +#define BOCU1_REACH_POS_1 (BOCU1_SINGLE-1) +#define BOCU1_REACH_NEG_1 (-BOCU1_SINGLE) + +/* The difference value range for double-byters. */ +#define BOCU1_REACH_POS_2 (BOCU1_REACH_POS_1+BOCU1_LEAD_2*BOCU1_TRAIL_COUNT) +#define BOCU1_REACH_NEG_2 (BOCU1_REACH_NEG_1-BOCU1_LEAD_2*BOCU1_TRAIL_COUNT) + +/* The difference value range for 3-byters. */ +#define BOCU1_REACH_POS_3 \ + (BOCU1_REACH_POS_2+BOCU1_LEAD_3*BOCU1_TRAIL_COUNT*BOCU1_TRAIL_COUNT) + +#define BOCU1_REACH_NEG_3 (BOCU1_REACH_NEG_2-BOCU1_LEAD_3*BOCU1_TRAIL_COUNT*BOCU1_TRAIL_COUNT) + +/* The lead byte start values. */ +#define BOCU1_START_POS_2 (BOCU1_MIDDLE+BOCU1_REACH_POS_1+1) +#define BOCU1_START_POS_3 (BOCU1_START_POS_2+BOCU1_LEAD_2) +#define BOCU1_START_POS_4 (BOCU1_START_POS_3+BOCU1_LEAD_3) + /* ==BOCU1_MAX_LEAD */ + +#define BOCU1_START_NEG_2 (BOCU1_MIDDLE+BOCU1_REACH_NEG_1) +#define BOCU1_START_NEG_3 (BOCU1_START_NEG_2-BOCU1_LEAD_2) +#define BOCU1_START_NEG_4 (BOCU1_START_NEG_3-BOCU1_LEAD_3) + /* ==BOCU1_MIN+1 */ + +/* The length of a byte sequence, according to the lead byte (!=BOCU1_RESET). */ +#define BOCU1_LENGTH_FROM_LEAD(lead) \ + ((BOCU1_START_NEG_2<=(lead) && (lead)<BOCU1_START_POS_2) ? 1 : \ + (BOCU1_START_NEG_3<=(lead) && (lead)<BOCU1_START_POS_3) ? 2 : \ + (BOCU1_START_NEG_4<=(lead) && (lead)<BOCU1_START_POS_4) ? 3 : 4) + +/* The length of a byte sequence, according to its packed form. */ +#define BOCU1_LENGTH_FROM_PACKED(packed) \ + ((uint32_t)(packed)<0x04000000 ? (packed)>>24 : 4) + +/* + * 12 commonly used C0 control codes (and space) are only used to encode + * themselves directly, + * which makes BOCU-1 MIME-usable and reasonably safe for + * ASCII-oriented software. + * + * These controls are + * 0 NUL + * + * 7 BEL + * 8 BS + * + * 9 TAB + * a LF + * b VT + * c FF + * d CR + * + * e SO + * f SI + * + * 1a SUB + * 1b ESC + * + * The other 20 C0 controls are also encoded directly (to preserve order) + * but are also used as trail bytes in difference encoding + * (for better compression). + */ +#define BOCU1_TRAIL_TO_BYTE(t) ((t)>=BOCU1_TRAIL_CONTROLS_COUNT ? (t)+BOCU1_TRAIL_BYTE_OFFSET : bocu1TrailToByte[t]) + +/* + * Byte value map for control codes, + * from external byte values 0x00..0x20 + * to trail byte values 0..19 (0..0x13) as used in the difference calculation. + * External byte values that are illegal as trail bytes are mapped to -1. + */ +static const int8_t +bocu1ByteToTrail[BOCU1_MIN]={ +/* 0 1 2 3 4 5 6 7 */ + -1, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, -1, + +/* 8 9 a b c d e f */ + -1, -1, -1, -1, -1, -1, -1, -1, + +/* 10 11 12 13 14 15 16 17 */ + 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, + +/* 18 19 1a 1b 1c 1d 1e 1f */ + 0x0e, 0x0f, -1, -1, 0x10, 0x11, 0x12, 0x13, + +/* 20 */ + -1 +}; + +/* + * Byte value map for control codes, + * from trail byte values 0..19 (0..0x13) as used in the difference calculation + * to external byte values 0x00..0x20. + */ +static const int8_t +bocu1TrailToByte[BOCU1_TRAIL_CONTROLS_COUNT]={ +/* 0 1 2 3 4 5 6 7 */ + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x10, 0x11, + +/* 8 9 a b c d e f */ + 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, + +/* 10 11 12 13 */ + 0x1c, 0x1d, 0x1e, 0x1f +}; + +/** + * Integer division and modulo with negative numerators + * yields negative modulo results and quotients that are one more than + * what we need here. + * This macro adjust the results so that the modulo-value m is always >=0. + * + * For positive n, the if() condition is always FALSE. + * + * @param n Number to be split into quotient and rest. + * Will be modified to contain the quotient. + * @param d Divisor. + * @param m Output variable for the rest (modulo result). + */ +#define NEGDIVMOD(n, d, m) { \ + (m)=(n)%(d); \ + (n)/=(d); \ + if((m)<0) { \ + --(n); \ + (m)+=(d); \ + } \ +} + +/* State for BOCU-1 decoder function. */ +struct Bocu1Rx { + int32_t prev, count, diff; +}; + +typedef struct Bocu1Rx Bocu1Rx; + +/* Function prototypes ------------------------------------------------------ */ + +/* see bocu1.c */ +U_CFUNC int32_t +packDiff(int32_t diff); + +U_CFUNC int32_t +encodeBocu1(int32_t *pPrev, int32_t c); + +U_CFUNC int32_t +decodeBocu1(Bocu1Rx *pRx, uint8_t b); + +/* icuhtml/design/conversion/bocu1/bocu1.c ---------------------------------- */ + +/* BOCU-1 implementation functions ------------------------------------------ */ + +/** + * Compute the next "previous" value for differencing + * from the current code point. + * + * @param c current code point, 0..0x10ffff + * @return "previous code point" state value + */ +static U_INLINE int32_t +bocu1Prev(int32_t c) { + /* compute new prev */ + if(0x3040<=c && c<=0x309f) { + /* Hiragana is not 128-aligned */ + return 0x3070; + } else if(0x4e00<=c && c<=0x9fa5) { + /* CJK Unihan */ + return 0x4e00-BOCU1_REACH_NEG_2; + } else if(0xac00<=c && c<=0xd7a3) { + /* Korean Hangul (cast to int32_t to avoid wraparound on 16-bit compilers) */ + return ((int32_t)0xd7a3+(int32_t)0xac00)/2; + } else { + /* mostly small scripts */ + return (c&~0x7f)+BOCU1_ASCII_PREV; + } +} + +/** + * Encode a difference -0x10ffff..0x10ffff in 1..4 bytes + * and return a packed integer with them. + * + * The encoding favors small absolut differences with short encodings + * to compress runs of same-script characters. + * + * @param diff difference value -0x10ffff..0x10ffff + * @return + * 0x010000zz for 1-byte sequence zz + * 0x0200yyzz for 2-byte sequence yy zz + * 0x03xxyyzz for 3-byte sequence xx yy zz + * 0xwwxxyyzz for 4-byte sequence ww xx yy zz (ww>0x03) + */ +U_CFUNC int32_t +packDiff(int32_t diff) { + int32_t result, m, lead, count, shift; + + if(diff>=BOCU1_REACH_NEG_1) { + /* mostly positive differences, and single-byte negative ones */ + if(diff<=BOCU1_REACH_POS_1) { + /* single byte */ + return 0x01000000|(BOCU1_MIDDLE+diff); + } else if(diff<=BOCU1_REACH_POS_2) { + /* two bytes */ + diff-=BOCU1_REACH_POS_1+1; + lead=BOCU1_START_POS_2; + count=1; + } else if(diff<=BOCU1_REACH_POS_3) { + /* three bytes */ + diff-=BOCU1_REACH_POS_2+1; + lead=BOCU1_START_POS_3; + count=2; + } else { + /* four bytes */ + diff-=BOCU1_REACH_POS_3+1; + lead=BOCU1_START_POS_4; + count=3; + } + } else { + /* two- and four-byte negative differences */ + if(diff>=BOCU1_REACH_NEG_2) { + /* two bytes */ + diff-=BOCU1_REACH_NEG_1; + lead=BOCU1_START_NEG_2; + count=1; + } else if(diff>=BOCU1_REACH_NEG_3) { + /* three bytes */ + diff-=BOCU1_REACH_NEG_2; + lead=BOCU1_START_NEG_3; + count=2; + } else { + /* four bytes */ + diff-=BOCU1_REACH_NEG_3; + lead=BOCU1_START_NEG_4; + count=3; + } + } + + /* encode the length of the packed result */ + if(count<3) { + result=(count+1)<<24; + } else /* count==3, MSB used for the lead byte */ { + result=0; + } + + /* calculate trail bytes like digits in itoa() */ + shift=0; + do { + NEGDIVMOD(diff, BOCU1_TRAIL_COUNT, m); + result|=BOCU1_TRAIL_TO_BYTE(m)<<shift; + shift+=8; + } while(--count>0); + + /* add lead byte */ + result|=(lead+diff)<<shift; + + return result; +} + +/** + * BOCU-1 encoder function. + * + * @param pPrev pointer to the integer that holds + * the "previous code point" state; + * the initial value should be 0 which + * encodeBocu1 will set to the actual BOCU-1 initial state value + * @param c the code point to encode + * @return the packed 1/2/3/4-byte encoding, see packDiff(), + * or 0 if an error occurs + * + * @see packDiff + */ +U_CFUNC int32_t +encodeBocu1(int32_t *pPrev, int32_t c) { + int32_t prev; + + if(pPrev==NULL || c<0 || c>0x10ffff) { + /* illegal argument */ + return 0; + } + + prev=*pPrev; + if(prev==0) { + /* lenient handling of initial value 0 */ + prev=*pPrev=BOCU1_ASCII_PREV; + } + + if(c<=0x20) { + /* + * ISO C0 control & space: + * Encode directly for MIME compatibility, + * and reset state except for space, to not disrupt compression. + */ + if(c!=0x20) { + *pPrev=BOCU1_ASCII_PREV; + } + return 0x01000000|c; + } + + /* + * all other Unicode code points c==U+0021..U+10ffff + * are encoded with the difference c-prev + * + * a new prev is computed from c, + * placed in the middle of a 0x80-block (for most small scripts) or + * in the middle of the Unihan and Hangul blocks + * to statistically minimize the following difference + */ + *pPrev=bocu1Prev(c); + return packDiff(c-prev); +} + +/** + * Function for BOCU-1 decoder; handles multi-byte lead bytes. + * + * @param pRx pointer to the decoder state structure + * @param b lead byte; + * BOCU1_MIN<=b<BOCU1_START_NEG_2 or BOCU1_START_POS_2<=b<=BOCU1_MAX_LEAD + * @return -1 (state change only) + * + * @see decodeBocu1 + */ +static int32_t +decodeBocu1LeadByte(Bocu1Rx *pRx, uint8_t b) { + int32_t c, count; + + if(b>=BOCU1_START_NEG_2) { + /* positive difference */ + if(b<BOCU1_START_POS_3) { + /* two bytes */ + c=((int32_t)b-BOCU1_START_POS_2)*BOCU1_TRAIL_COUNT+BOCU1_REACH_POS_1+1; + count=1; + } else if(b<BOCU1_START_POS_4) { + /* three bytes */ + c=((int32_t)b-BOCU1_START_POS_3)*BOCU1_TRAIL_COUNT*BOCU1_TRAIL_COUNT+BOCU1_REACH_POS_2+1; + count=2; + } else { + /* four bytes */ + c=BOCU1_REACH_POS_3+1; + count=3; + } + } else { + /* negative difference */ + if(b>=BOCU1_START_NEG_3) { + /* two bytes */ + c=((int32_t)b-BOCU1_START_NEG_2)*BOCU1_TRAIL_COUNT+BOCU1_REACH_NEG_1; + count=1; + } else if(b>BOCU1_MIN) { + /* three bytes */ + c=((int32_t)b-BOCU1_START_NEG_3)*BOCU1_TRAIL_COUNT*BOCU1_TRAIL_COUNT+BOCU1_REACH_NEG_2; + count=2; + } else { + /* four bytes */ + c=-BOCU1_TRAIL_COUNT*BOCU1_TRAIL_COUNT*BOCU1_TRAIL_COUNT+BOCU1_REACH_NEG_3; + count=3; + } + } + + /* set the state for decoding the trail byte(s) */ + pRx->diff=c; + pRx->count=count; + return -1; +} + +/** + * Function for BOCU-1 decoder; handles multi-byte trail bytes. + * + * @param pRx pointer to the decoder state structure + * @param b trail byte + * @return result value, same as decodeBocu1 + * + * @see decodeBocu1 + */ +static int32_t +decodeBocu1TrailByte(Bocu1Rx *pRx, uint8_t b) { + int32_t t, c, count; + + if(b<=0x20) { + /* skip some C0 controls and make the trail byte range contiguous */ + t=bocu1ByteToTrail[b]; + if(t<0) { + /* illegal trail byte value */ + pRx->prev=BOCU1_ASCII_PREV; + pRx->count=0; + return -99; + } +#if BOCU1_MAX_TRAIL<0xff + } else if(b>BOCU1_MAX_TRAIL) { + return -99; +#endif + } else { + t=(int32_t)b-BOCU1_TRAIL_BYTE_OFFSET; + } + + /* add trail byte into difference and decrement count */ + c=pRx->diff; + count=pRx->count; + + if(count==1) { + /* final trail byte, deliver a code point */ + c=pRx->prev+c+t; + if(0<=c && c<=0x10ffff) { + /* valid code point result */ + pRx->prev=bocu1Prev(c); + pRx->count=0; + return c; + } else { + /* illegal code point result */ + pRx->prev=BOCU1_ASCII_PREV; + pRx->count=0; + return -99; + } + } + + /* intermediate trail byte */ + if(count==2) { + pRx->diff=c+t*BOCU1_TRAIL_COUNT; + } else /* count==3 */ { + pRx->diff=c+t*BOCU1_TRAIL_COUNT*BOCU1_TRAIL_COUNT; + } + pRx->count=count-1; + return -1; +} + +/** + * BOCU-1 decoder function. + * + * @param pRx pointer to the decoder state structure; + * the initial values should be 0 which + * decodeBocu1 will set to actual initial state values + * @param b an input byte + * @return + * 0..0x10ffff for a result code point + * -1 if only the state changed without code point output + * <-1 if an error occurs + */ +U_CFUNC int32_t +decodeBocu1(Bocu1Rx *pRx, uint8_t b) { + int32_t prev, c, count; + + if(pRx==NULL) { + /* illegal argument */ + return -99; + } + + prev=pRx->prev; + if(prev==0) { + /* lenient handling of initial 0 values */ + prev=pRx->prev=BOCU1_ASCII_PREV; + count=pRx->count=0; + } else { + count=pRx->count; + } + + if(count==0) { + /* byte in lead position */ + if(b<=0x20) { + /* + * Direct-encoded C0 control code or space. + * Reset prev for C0 control codes but not for space. + */ + if(b!=0x20) { + pRx->prev=BOCU1_ASCII_PREV; + } + return b; + } + + /* + * b is a difference lead byte. + * + * Return a code point directly from a single-byte difference. + * + * For multi-byte difference lead bytes, set the decoder state + * with the partial difference value from the lead byte and + * with the number of trail bytes. + * + * For four-byte differences, the signedness also affects the + * first trail byte, which has special handling farther below. + */ + if(b>=BOCU1_START_NEG_2 && b<BOCU1_START_POS_2) { + /* single-byte difference */ + c=prev+((int32_t)b-BOCU1_MIDDLE); + pRx->prev=bocu1Prev(c); + return c; + } else if(b==BOCU1_RESET) { + /* only reset the state, no code point */ + pRx->prev=BOCU1_ASCII_PREV; + return -1; + } else { + return decodeBocu1LeadByte(pRx, b); + } + } else { + /* trail byte in any position */ + return decodeBocu1TrailByte(pRx, b); + } +} + +/* icuhtml/design/conversion/bocu1/bocu1tst.c ------------------------------- */ + +/* test code ---------------------------------------------------------------- */ + +/* test code options */ + +/* ignore comma when processing name lists in testText() */ +#define TEST_IGNORE_COMMA 1 + +/** + * Write a packed BOCU-1 byte sequence into a byte array, + * without overflow check. + * Test function. + * + * @param packed packed BOCU-1 byte sequence, see packDiff() + * @param p pointer to byte array + * @return number of bytes + * + * @see packDiff + */ +static int32_t +writePacked(int32_t packed, uint8_t *p) { + int32_t count=BOCU1_LENGTH_FROM_PACKED(packed); + switch(count) { + case 4: + *p++=(uint8_t)(packed>>24); + case 3: + *p++=(uint8_t)(packed>>16); + case 2: + *p++=(uint8_t)(packed>>8); + case 1: + *p++=(uint8_t)packed; + default: + break; + } + + return count; +} + +/** + * Unpack a packed BOCU-1 non-C0/space byte sequence and get + * the difference to initialPrev. + * Used only for round-trip testing of the difference encoding and decoding. + * Test function. + * + * @param initialPrev bogus "previous code point" value to make sure that + * the resulting code point is in the range 0..0x10ffff + * @param packed packed BOCU-1 byte sequence + * @return the difference to initialPrev + * + * @see packDiff + * @see writeDiff + */ +static int32_t +unpackDiff(int32_t initialPrev, int32_t packed) { + Bocu1Rx rx={ 0, 0, 0 }; + int32_t count; + + rx.prev=initialPrev; + count=BOCU1_LENGTH_FROM_PACKED(packed); + switch(count) { + case 4: + decodeBocu1(&rx, (uint8_t)(packed>>24)); + case 3: + decodeBocu1(&rx, (uint8_t)(packed>>16)); + case 2: + decodeBocu1(&rx, (uint8_t)(packed>>8)); + case 1: + /* subtract initial prev */ + return decodeBocu1(&rx, (uint8_t)packed)-initialPrev; + default: + return -0x7fffffff; + } +} + +/** + * Encode one difference value -0x10ffff..+0x10ffff in 1..4 bytes, + * preserving lexical order. + * Also checks for roundtripping of the difference encoding. + * Test function. + * + * @param diff difference value to test, -0x10ffff..0x10ffff + * @param p pointer to output byte array + * @return p advanced by number of bytes output + * + * @see unpackDiff + */ +static uint8_t * +writeDiff(int32_t diff, uint8_t *p) { + /* generate the difference as a packed value and serialize it */ + int32_t packed, initialPrev; + + packed=packDiff(diff); + + /* + * bogus initial "prev" to work around + * code point range check in decodeBocu1() + */ + if(diff<=0) { + initialPrev=0x10ffff; + } else { + initialPrev=-1; + } + + if(diff!=unpackDiff(initialPrev, packed)) { + log_err("error: unpackDiff(packDiff(diff=%ld)=0x%08lx)=%ld!=diff\n", + diff, packed, unpackDiff(initialPrev, packed)); + } + return p+writePacked(packed, p); +} + +/** + * Encode a UTF-16 string in BOCU-1. + * Does not check for overflows, but otherwise useful function. + * + * @param s input UTF-16 string + * @param length number of UChar code units in s + * @param p pointer to output byte array + * @return number of bytes output + */ +static int32_t +writeString(const UChar *s, int32_t length, uint8_t *p) { + uint8_t *p0; + int32_t c, prev, i; + + prev=0; + p0=p; + i=0; + while(i<length) { + UTF_NEXT_CHAR(s, i, length, c); + p+=writePacked(encodeBocu1(&prev, c), p); + } + return (int32_t)(p-p0); +} + +/** + * Decode a BOCU-1 byte sequence to a UTF-16 string. + * Does not check for overflows, but otherwise useful function. + * + * @param p pointer to input BOCU-1 bytes + * @param length number of input bytes + * @param s point to output UTF-16 string array + * @return number of UChar code units output + */ +static int32_t +readString(const uint8_t *p, int32_t length, UChar *s) { + Bocu1Rx rx={ 0, 0, 0 }; + int32_t c, i, sLength; + + i=sLength=0; + while(i<length) { + c=decodeBocu1(&rx, p[i++]); + if(c<-1) { + log_err("error: readString detects encoding error at string index %ld\n", i); + return -1; + } + if(c>=0) { + UTF_APPEND_CHAR_UNSAFE(s, sLength, c); + } + } + return sLength; +} + +static U_INLINE char +hexDigit(uint8_t digit) { + return digit<=9 ? (char)('0'+digit) : (char)('a'-10+digit); +} + +/** + * Pretty-print 0-terminated byte values. + * Helper function for test output. + * + * @param bytes 0-terminated byte array to print + */ +static void +printBytes(uint8_t *bytes, char *out) { + int i; + uint8_t b; + + i=0; + while((b=*bytes++)!=0) { + *out++=' '; + *out++=hexDigit((uint8_t)(b>>4)); + *out++=hexDigit((uint8_t)(b&0xf)); + ++i; + } + i=3*(5-i); + while(i>0) { + *out++=' '; + --i; + } + *out=0; +} + +/** + * Basic BOCU-1 test function, called when there are no command line arguments. + * Prints some of the #define values and performs round-trip tests of the + * difference encoding and decoding. + */ +static void +TestBOCU1RefDiff(void) { + char buf1[80], buf2[80]; + uint8_t prev[5], level[5]; + int32_t i, cmp, countErrors; + + log_verbose("reach of single bytes: %ld\n", 1+BOCU1_REACH_POS_1-BOCU1_REACH_NEG_1); + log_verbose("reach of 2 bytes : %ld\n", 1+BOCU1_REACH_POS_2-BOCU1_REACH_NEG_2); + log_verbose("reach of 3 bytes : %ld\n\n", 1+BOCU1_REACH_POS_3-BOCU1_REACH_NEG_3); + + log_verbose(" BOCU1_REACH_NEG_1 %8ld BOCU1_REACH_POS_1 %8ld\n", BOCU1_REACH_NEG_1, BOCU1_REACH_POS_1); + log_verbose(" BOCU1_REACH_NEG_2 %8ld BOCU1_REACH_POS_2 %8ld\n", BOCU1_REACH_NEG_2, BOCU1_REACH_POS_2); + log_verbose(" BOCU1_REACH_NEG_3 %8ld BOCU1_REACH_POS_3 %8ld\n\n", BOCU1_REACH_NEG_3, BOCU1_REACH_POS_3); + + log_verbose(" BOCU1_MIDDLE 0x%02x\n", BOCU1_MIDDLE); + log_verbose(" BOCU1_START_NEG_2 0x%02x BOCU1_START_POS_2 0x%02x\n", BOCU1_START_NEG_2, BOCU1_START_POS_2); + log_verbose(" BOCU1_START_NEG_3 0x%02x BOCU1_START_POS_3 0x%02x\n\n", BOCU1_START_NEG_3, BOCU1_START_POS_3); + + /* test packDiff() & unpackDiff() with some specific values */ + writeDiff(0, level); + writeDiff(1, level); + writeDiff(65, level); + writeDiff(130, level); + writeDiff(30000, level); + writeDiff(1000000, level); + writeDiff(-65, level); + writeDiff(-130, level); + writeDiff(-30000, level); + writeDiff(-1000000, level); + + /* test that each value is smaller than any following one */ + countErrors=0; + i=-0x10ffff; + *writeDiff(i, prev)=0; + + /* show first number and bytes */ + printBytes(prev, buf1); + log_verbose(" wD(%8ld) %s\n", i, buf1); + + for(++i; i<=0x10ffff; ++i) { + *writeDiff(i, level)=0; + cmp=strcmp((const char *)prev, (const char *)level); + if(BOCU1_LENGTH_FROM_LEAD(level[0])!=(int32_t)strlen((const char *)level)) { + log_verbose("BOCU1_LENGTH_FROM_LEAD(0x%02x)=%ld!=%ld=strlen(writeDiff(%ld))\n", + level[0], BOCU1_LENGTH_FROM_LEAD(level[0]), strlen((const char *)level), i); + } + if(cmp<0) { + if(i==0 || i==1 || strlen((const char *)prev)!=strlen((const char *)level)) { + /* + * if the result is good, then print only if the length changed + * to get little but interesting output + */ + printBytes(prev, buf1); + printBytes(level, buf2); + log_verbose("ok: strcmp(wD(%8ld), wD(%8ld))=%2d %s%s\n", i-1, i, cmp, buf1, buf2); + } + } else { + ++countErrors; + printBytes(prev, buf1); + printBytes(level, buf2); + log_verbose("wrong: strcmp(wD(%8ld), wD(%8ld))=%2d %s%s\n", i-1, i, cmp, buf1, buf2); + } + /* remember the previous bytes */ + memcpy(prev, level, 4); + } + + /* show last number and bytes */ + printBytes((uint8_t *)"", buf1); + printBytes(prev, buf2); + log_verbose(" wD(%8ld) %s%s\n", i-1, buf1, buf2); + + if(countErrors==0) { + log_verbose("writeDiff(-0x10ffff..0x10ffff) works fine\n"); + } else { + log_err("writeDiff(-0x10ffff..0x10ffff) violates lexical ordering in %d cases\n", countErrors); + } + + /* output signature byte sequence */ + i=0; + writePacked(encodeBocu1(&i, 0xfeff), level); + log_verbose("\nBOCU-1 signature byte sequence: %02x %02x %02x\n", + level[0], level[1], level[2]); +} + +/* cintltst code ------------------------------------------------------------ */ + +static const int32_t DEFAULT_BUFFER_SIZE = 30000; + + +/* test one string with the ICU and the reference BOCU-1 implementations */ +static void +roundtripBOCU1(UConverter *bocu1, int32_t number, const UChar *text, int32_t length) { + UChar *roundtripRef, *roundtripICU; + char *bocu1Ref, *bocu1ICU; + + int32_t bocu1RefLength, bocu1ICULength, roundtripRefLength, roundtripICULength; + UErrorCode errorCode; + + roundtripRef = malloc(DEFAULT_BUFFER_SIZE * sizeof(UChar)); + roundtripICU = malloc(DEFAULT_BUFFER_SIZE * sizeof(UChar)); + bocu1Ref = malloc(DEFAULT_BUFFER_SIZE); + bocu1ICU = malloc(DEFAULT_BUFFER_SIZE); + + /* Unicode -> BOCU-1 */ + bocu1RefLength=writeString(text, length, (uint8_t *)bocu1Ref); + + errorCode=U_ZERO_ERROR; + bocu1ICULength=ucnv_fromUChars(bocu1, bocu1ICU, DEFAULT_BUFFER_SIZE, text, length, &errorCode); + if(U_FAILURE(errorCode)) { + log_err("ucnv_fromUChars(BOCU-1, text(%d)[%d]) failed: %s\n", number, length, u_errorName(errorCode)); + goto cleanup; + } + + if(bocu1RefLength!=bocu1ICULength || 0!=uprv_memcmp(bocu1Ref, bocu1ICU, bocu1RefLength)) { + log_err("Unicode(%d)[%d] -> BOCU-1: reference[%d]!=ICU[%d]\n", number, length, bocu1RefLength, bocu1ICULength); + goto cleanup; + } + + /* BOCU-1 -> Unicode */ + roundtripRefLength=readString((uint8_t *)bocu1Ref, bocu1RefLength, roundtripRef); + if(roundtripRefLength<0) { + goto cleanup; /* readString() found an error and reported it */ + } + + roundtripICULength=ucnv_toUChars(bocu1, roundtripICU, DEFAULT_BUFFER_SIZE, bocu1ICU, bocu1ICULength, &errorCode); + if(U_FAILURE(errorCode)) { + log_err("ucnv_toUChars(BOCU-1, text(%d)[%d]) failed: %s\n", number, length, u_errorName(errorCode)); + goto cleanup; + } + + if(length!=roundtripRefLength || 0!=u_memcmp(text, roundtripRef, length)) { + log_err("BOCU-1 -> Unicode: original(%d)[%d]!=reference[%d]\n", number, length, roundtripRefLength); + goto cleanup; + } + if(roundtripRefLength!=roundtripICULength || 0!=u_memcmp(roundtripRef, roundtripICU, roundtripRefLength)) { + log_err("BOCU-1 -> Unicode: reference(%d)[%d]!=ICU[%d]\n", number, roundtripRefLength, roundtripICULength); + goto cleanup; + } +cleanup: + free(roundtripRef); + free(roundtripICU); + free(bocu1Ref); + free(bocu1ICU); +} + +static const UChar feff[]={ 0xfeff }; +static const UChar ascii[]={ 0x61, 0x62, 0x20, 0x63, 0x61 }; +static const UChar crlf[]={ 0xd, 0xa, 0x20 }; +static const UChar nul[]={ 0 }; +static const UChar latin[]={ 0xdf, 0xe6 }; +static const UChar devanagari[]={ 0x930, 0x20, 0x918, 0x909 }; +static const UChar hiragana[]={ 0x3086, 0x304d, 0x20, 0x3053, 0x4000 }; +static const UChar unihan[]={ 0x4e00, 0x7777, 0x20, 0x9fa5, 0x4e00 }; +static const UChar hangul[]={ 0xac00, 0xbcde, 0x20, 0xd7a3 }; +static const UChar surrogates[]={ 0xdc00, 0xd800 }; /* single surrogates, unmatched! */ +static const UChar plane1[]={ 0xd800, 0xdc00 }; +static const UChar plane2[]={ 0xd845, 0xdddd }; +static const UChar plane15[]={ 0xdbbb, 0xddee, 0x20 }; +static const UChar plane16[]={ 0xdbff, 0xdfff }; +static const UChar c0[]={ 1, 0xe40, 0x20, 9 }; + +static const struct { + const UChar *s; + int32_t length; +} strings[]={ + { feff, LENGTHOF(feff) }, + { ascii, LENGTHOF(ascii) }, + { crlf, LENGTHOF(crlf) }, + { nul, LENGTHOF(nul) }, + { latin, LENGTHOF(latin) }, + { devanagari, LENGTHOF(devanagari) }, + { hiragana, LENGTHOF(hiragana) }, + { unihan, LENGTHOF(unihan) }, + { hangul, LENGTHOF(hangul) }, + { surrogates, LENGTHOF(surrogates) }, + { plane1, LENGTHOF(plane1) }, + { plane2, LENGTHOF(plane2) }, + { plane15, LENGTHOF(plane15) }, + { plane16, LENGTHOF(plane16) }, + { c0, LENGTHOF(c0) } +}; + +/* + * Verify that the ICU BOCU-1 implementation produces the same results as + * the reference implementation from the design folder. + * Generate some texts and convert them with both converters, verifying + * identical results and roundtripping. + */ +static void +TestBOCU1(void) { + UChar *text; + int32_t i, length; + + UConverter *bocu1; + UErrorCode errorCode; + + errorCode=U_ZERO_ERROR; + bocu1=ucnv_open("BOCU-1", &errorCode); + if(U_FAILURE(errorCode)) { + log_err("error: unable to open BOCU-1 converter: %s\n", u_errorName(errorCode)); + return; + } + + text = malloc(DEFAULT_BUFFER_SIZE * sizeof(UChar)); + + /* text 1: each of strings[] once */ + length=0; + for(i=0; i<LENGTHOF(strings); ++i) { + u_memcpy(text+length, strings[i].s, strings[i].length); + length+=strings[i].length; + } + roundtripBOCU1(bocu1, 1, text, length); + + /* text 2: each of strings[] twice */ + length=0; + for(i=0; i<LENGTHOF(strings); ++i) { + u_memcpy(text+length, strings[i].s, strings[i].length); + length+=strings[i].length; + u_memcpy(text+length, strings[i].s, strings[i].length); + length+=strings[i].length; + } + roundtripBOCU1(bocu1, 2, text, length); + + /* text 3: each of strings[] many times (set step vs. |strings| so that all strings are used) */ + length=0; + for(i=1; length<5000; i+=7) { + if(i>=LENGTHOF(strings)) { + i-=LENGTHOF(strings); + } + u_memcpy(text+length, strings[i].s, strings[i].length); + length+=strings[i].length; + } + roundtripBOCU1(bocu1, 3, text, length); + + ucnv_close(bocu1); + free(text); +} + +U_CFUNC void addBOCU1Tests(TestNode** root); + +U_CFUNC void +addBOCU1Tests(TestNode** root) { + addTest(root, TestBOCU1RefDiff, "tsconv/bocu1tst/TestBOCU1RefDiff"); + addTest(root, TestBOCU1, "tsconv/bocu1tst/TestBOCU1"); +} |