diff options
Diffstat (limited to 'Build/source/libs/icu/icu-xetex/test/cintltst/bocu1tst.c')
-rw-r--r-- | Build/source/libs/icu/icu-xetex/test/cintltst/bocu1tst.c | 1031 |
1 files changed, 0 insertions, 1031 deletions
diff --git a/Build/source/libs/icu/icu-xetex/test/cintltst/bocu1tst.c b/Build/source/libs/icu/icu-xetex/test/cintltst/bocu1tst.c deleted file mode 100644 index 0616b1cff9e..00000000000 --- a/Build/source/libs/icu/icu-xetex/test/cintltst/bocu1tst.c +++ /dev/null @@ -1,1031 +0,0 @@ -/* -****************************************************************************** -* -* Copyright (C) 2002-2007, International Business Machines -* Corporation and others. All Rights Reserved. -* -****************************************************************************** -* file name: bocu1tst.c -* encoding: US-ASCII -* tab size: 8 (not used) -* indentation:4 -* -* created on: 2002may27 -* created by: Markus W. Scherer -* -* This is the reference implementation of BOCU-1, -* the MIME-friendly form of the Binary Ordered Compression for Unicode, -* taken directly from ### http://source.icu-project.org/repos/icu/icuhtml/trunk/design/conversion/bocu1/ -* The files bocu1.h and bocu1.c from the design folder are taken -* verbatim (minus copyright and #include) and copied together into this file. -* The reference code and some of the reference bocu1tst.c -* is modified to run as part of the ICU cintltst -* test framework (minus main(), log_ln() etc. instead of printf()). -* -* This reference implementation is used here to verify -* the ICU BOCU-1 implementation, which is -* adapted for ICU conversion APIs and optimized. -* ### links in design doc to here and to ucnvbocu.c -*/ - -#include "unicode/utypes.h" -#include "unicode/ustring.h" -#include "unicode/ucnv.h" -#include "cmemory.h" -#include "cintltst.h" - -#define LENGTHOF(array) (sizeof(array)/sizeof((array)[0])) - -/* icuhtml/design/conversion/bocu1/bocu1.h ---------------------------------- */ - -/* BOCU-1 constants and macros ---------------------------------------------- */ - -/* - * BOCU-1 encodes the code points of a Unicode string as - * a sequence of byte-encoded differences (slope detection), - * preserving lexical order. - * - * Optimize the difference-taking for runs of Unicode text within - * small scripts: - * - * Most small scripts are allocated within aligned 128-blocks of Unicode - * code points. Lexical order is preserved if the "previous code point" state - * is always moved into the middle of such a block. - * - * Additionally, "prev" is moved from anywhere in the Unihan and Hangul - * areas into the middle of those areas. - * - * C0 control codes and space are encoded with their US-ASCII bytes. - * "prev" is reset for C0 controls but not for space. - */ - -/* initial value for "prev": middle of the ASCII range */ -#define BOCU1_ASCII_PREV 0x40 - -/* bounding byte values for differences */ -#define BOCU1_MIN 0x21 -#define BOCU1_MIDDLE 0x90 -#define BOCU1_MAX_LEAD 0xfe - -/* add the L suffix to make computations with BOCU1_MAX_TRAIL work on 16-bit compilers */ -#define BOCU1_MAX_TRAIL 0xffL -#define BOCU1_RESET 0xff - -/* number of lead bytes */ -#define BOCU1_COUNT (BOCU1_MAX_LEAD-BOCU1_MIN+1) - -/* adjust trail byte counts for the use of some C0 control byte values */ -#define BOCU1_TRAIL_CONTROLS_COUNT 20 -#define BOCU1_TRAIL_BYTE_OFFSET (BOCU1_MIN-BOCU1_TRAIL_CONTROLS_COUNT) - -/* number of trail bytes */ -#define BOCU1_TRAIL_COUNT ((BOCU1_MAX_TRAIL-BOCU1_MIN+1)+BOCU1_TRAIL_CONTROLS_COUNT) - -/* - * number of positive and negative single-byte codes - * (counting 0==BOCU1_MIDDLE among the positive ones) - */ -#define BOCU1_SINGLE 64 - -/* number of lead bytes for positive and negative 2/3/4-byte sequences */ -#define BOCU1_LEAD_2 43 -#define BOCU1_LEAD_3 3 -#define BOCU1_LEAD_4 1 - -/* The difference value range for single-byters. */ -#define BOCU1_REACH_POS_1 (BOCU1_SINGLE-1) -#define BOCU1_REACH_NEG_1 (-BOCU1_SINGLE) - -/* The difference value range for double-byters. */ -#define BOCU1_REACH_POS_2 (BOCU1_REACH_POS_1+BOCU1_LEAD_2*BOCU1_TRAIL_COUNT) -#define BOCU1_REACH_NEG_2 (BOCU1_REACH_NEG_1-BOCU1_LEAD_2*BOCU1_TRAIL_COUNT) - -/* The difference value range for 3-byters. */ -#define BOCU1_REACH_POS_3 \ - (BOCU1_REACH_POS_2+BOCU1_LEAD_3*BOCU1_TRAIL_COUNT*BOCU1_TRAIL_COUNT) - -#define BOCU1_REACH_NEG_3 (BOCU1_REACH_NEG_2-BOCU1_LEAD_3*BOCU1_TRAIL_COUNT*BOCU1_TRAIL_COUNT) - -/* The lead byte start values. */ -#define BOCU1_START_POS_2 (BOCU1_MIDDLE+BOCU1_REACH_POS_1+1) -#define BOCU1_START_POS_3 (BOCU1_START_POS_2+BOCU1_LEAD_2) -#define BOCU1_START_POS_4 (BOCU1_START_POS_3+BOCU1_LEAD_3) - /* ==BOCU1_MAX_LEAD */ - -#define BOCU1_START_NEG_2 (BOCU1_MIDDLE+BOCU1_REACH_NEG_1) -#define BOCU1_START_NEG_3 (BOCU1_START_NEG_2-BOCU1_LEAD_2) -#define BOCU1_START_NEG_4 (BOCU1_START_NEG_3-BOCU1_LEAD_3) - /* ==BOCU1_MIN+1 */ - -/* The length of a byte sequence, according to the lead byte (!=BOCU1_RESET). */ -#define BOCU1_LENGTH_FROM_LEAD(lead) \ - ((BOCU1_START_NEG_2<=(lead) && (lead)<BOCU1_START_POS_2) ? 1 : \ - (BOCU1_START_NEG_3<=(lead) && (lead)<BOCU1_START_POS_3) ? 2 : \ - (BOCU1_START_NEG_4<=(lead) && (lead)<BOCU1_START_POS_4) ? 3 : 4) - -/* The length of a byte sequence, according to its packed form. */ -#define BOCU1_LENGTH_FROM_PACKED(packed) \ - ((uint32_t)(packed)<0x04000000 ? (packed)>>24 : 4) - -/* - * 12 commonly used C0 control codes (and space) are only used to encode - * themselves directly, - * which makes BOCU-1 MIME-usable and reasonably safe for - * ASCII-oriented software. - * - * These controls are - * 0 NUL - * - * 7 BEL - * 8 BS - * - * 9 TAB - * a LF - * b VT - * c FF - * d CR - * - * e SO - * f SI - * - * 1a SUB - * 1b ESC - * - * The other 20 C0 controls are also encoded directly (to preserve order) - * but are also used as trail bytes in difference encoding - * (for better compression). - */ -#define BOCU1_TRAIL_TO_BYTE(t) ((t)>=BOCU1_TRAIL_CONTROLS_COUNT ? (t)+BOCU1_TRAIL_BYTE_OFFSET : bocu1TrailToByte[t]) - -/* - * Byte value map for control codes, - * from external byte values 0x00..0x20 - * to trail byte values 0..19 (0..0x13) as used in the difference calculation. - * External byte values that are illegal as trail bytes are mapped to -1. - */ -static const int8_t -bocu1ByteToTrail[BOCU1_MIN]={ -/* 0 1 2 3 4 5 6 7 */ - -1, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, -1, - -/* 8 9 a b c d e f */ - -1, -1, -1, -1, -1, -1, -1, -1, - -/* 10 11 12 13 14 15 16 17 */ - 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, - -/* 18 19 1a 1b 1c 1d 1e 1f */ - 0x0e, 0x0f, -1, -1, 0x10, 0x11, 0x12, 0x13, - -/* 20 */ - -1 -}; - -/* - * Byte value map for control codes, - * from trail byte values 0..19 (0..0x13) as used in the difference calculation - * to external byte values 0x00..0x20. - */ -static const int8_t -bocu1TrailToByte[BOCU1_TRAIL_CONTROLS_COUNT]={ -/* 0 1 2 3 4 5 6 7 */ - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x10, 0x11, - -/* 8 9 a b c d e f */ - 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, - -/* 10 11 12 13 */ - 0x1c, 0x1d, 0x1e, 0x1f -}; - -/** - * Integer division and modulo with negative numerators - * yields negative modulo results and quotients that are one more than - * what we need here. - * This macro adjust the results so that the modulo-value m is always >=0. - * - * For positive n, the if() condition is always FALSE. - * - * @param n Number to be split into quotient and rest. - * Will be modified to contain the quotient. - * @param d Divisor. - * @param m Output variable for the rest (modulo result). - */ -#define NEGDIVMOD(n, d, m) { \ - (m)=(n)%(d); \ - (n)/=(d); \ - if((m)<0) { \ - --(n); \ - (m)+=(d); \ - } \ -} - -/* State for BOCU-1 decoder function. */ -struct Bocu1Rx { - int32_t prev, count, diff; -}; - -typedef struct Bocu1Rx Bocu1Rx; - -/* Function prototypes ------------------------------------------------------ */ - -/* see bocu1.c */ -U_CFUNC int32_t -packDiff(int32_t diff); - -U_CFUNC int32_t -encodeBocu1(int32_t *pPrev, int32_t c); - -U_CFUNC int32_t -decodeBocu1(Bocu1Rx *pRx, uint8_t b); - -/* icuhtml/design/conversion/bocu1/bocu1.c ---------------------------------- */ - -/* BOCU-1 implementation functions ------------------------------------------ */ - -/** - * Compute the next "previous" value for differencing - * from the current code point. - * - * @param c current code point, 0..0x10ffff - * @return "previous code point" state value - */ -static U_INLINE int32_t -bocu1Prev(int32_t c) { - /* compute new prev */ - if(0x3040<=c && c<=0x309f) { - /* Hiragana is not 128-aligned */ - return 0x3070; - } else if(0x4e00<=c && c<=0x9fa5) { - /* CJK Unihan */ - return 0x4e00-BOCU1_REACH_NEG_2; - } else if(0xac00<=c && c<=0xd7a3) { - /* Korean Hangul (cast to int32_t to avoid wraparound on 16-bit compilers) */ - return ((int32_t)0xd7a3+(int32_t)0xac00)/2; - } else { - /* mostly small scripts */ - return (c&~0x7f)+BOCU1_ASCII_PREV; - } -} - -/** - * Encode a difference -0x10ffff..0x10ffff in 1..4 bytes - * and return a packed integer with them. - * - * The encoding favors small absolut differences with short encodings - * to compress runs of same-script characters. - * - * @param diff difference value -0x10ffff..0x10ffff - * @return - * 0x010000zz for 1-byte sequence zz - * 0x0200yyzz for 2-byte sequence yy zz - * 0x03xxyyzz for 3-byte sequence xx yy zz - * 0xwwxxyyzz for 4-byte sequence ww xx yy zz (ww>0x03) - */ -U_CFUNC int32_t -packDiff(int32_t diff) { - int32_t result, m, lead, count, shift; - - if(diff>=BOCU1_REACH_NEG_1) { - /* mostly positive differences, and single-byte negative ones */ - if(diff<=BOCU1_REACH_POS_1) { - /* single byte */ - return 0x01000000|(BOCU1_MIDDLE+diff); - } else if(diff<=BOCU1_REACH_POS_2) { - /* two bytes */ - diff-=BOCU1_REACH_POS_1+1; - lead=BOCU1_START_POS_2; - count=1; - } else if(diff<=BOCU1_REACH_POS_3) { - /* three bytes */ - diff-=BOCU1_REACH_POS_2+1; - lead=BOCU1_START_POS_3; - count=2; - } else { - /* four bytes */ - diff-=BOCU1_REACH_POS_3+1; - lead=BOCU1_START_POS_4; - count=3; - } - } else { - /* two- and four-byte negative differences */ - if(diff>=BOCU1_REACH_NEG_2) { - /* two bytes */ - diff-=BOCU1_REACH_NEG_1; - lead=BOCU1_START_NEG_2; - count=1; - } else if(diff>=BOCU1_REACH_NEG_3) { - /* three bytes */ - diff-=BOCU1_REACH_NEG_2; - lead=BOCU1_START_NEG_3; - count=2; - } else { - /* four bytes */ - diff-=BOCU1_REACH_NEG_3; - lead=BOCU1_START_NEG_4; - count=3; - } - } - - /* encode the length of the packed result */ - if(count<3) { - result=(count+1)<<24; - } else /* count==3, MSB used for the lead byte */ { - result=0; - } - - /* calculate trail bytes like digits in itoa() */ - shift=0; - do { - NEGDIVMOD(diff, BOCU1_TRAIL_COUNT, m); - result|=BOCU1_TRAIL_TO_BYTE(m)<<shift; - shift+=8; - } while(--count>0); - - /* add lead byte */ - result|=(lead+diff)<<shift; - - return result; -} - -/** - * BOCU-1 encoder function. - * - * @param pPrev pointer to the integer that holds - * the "previous code point" state; - * the initial value should be 0 which - * encodeBocu1 will set to the actual BOCU-1 initial state value - * @param c the code point to encode - * @return the packed 1/2/3/4-byte encoding, see packDiff(), - * or 0 if an error occurs - * - * @see packDiff - */ -U_CFUNC int32_t -encodeBocu1(int32_t *pPrev, int32_t c) { - int32_t prev; - - if(pPrev==NULL || c<0 || c>0x10ffff) { - /* illegal argument */ - return 0; - } - - prev=*pPrev; - if(prev==0) { - /* lenient handling of initial value 0 */ - prev=*pPrev=BOCU1_ASCII_PREV; - } - - if(c<=0x20) { - /* - * ISO C0 control & space: - * Encode directly for MIME compatibility, - * and reset state except for space, to not disrupt compression. - */ - if(c!=0x20) { - *pPrev=BOCU1_ASCII_PREV; - } - return 0x01000000|c; - } - - /* - * all other Unicode code points c==U+0021..U+10ffff - * are encoded with the difference c-prev - * - * a new prev is computed from c, - * placed in the middle of a 0x80-block (for most small scripts) or - * in the middle of the Unihan and Hangul blocks - * to statistically minimize the following difference - */ - *pPrev=bocu1Prev(c); - return packDiff(c-prev); -} - -/** - * Function for BOCU-1 decoder; handles multi-byte lead bytes. - * - * @param pRx pointer to the decoder state structure - * @param b lead byte; - * BOCU1_MIN<=b<BOCU1_START_NEG_2 or BOCU1_START_POS_2<=b<=BOCU1_MAX_LEAD - * @return -1 (state change only) - * - * @see decodeBocu1 - */ -static int32_t -decodeBocu1LeadByte(Bocu1Rx *pRx, uint8_t b) { - int32_t c, count; - - if(b>=BOCU1_START_NEG_2) { - /* positive difference */ - if(b<BOCU1_START_POS_3) { - /* two bytes */ - c=((int32_t)b-BOCU1_START_POS_2)*BOCU1_TRAIL_COUNT+BOCU1_REACH_POS_1+1; - count=1; - } else if(b<BOCU1_START_POS_4) { - /* three bytes */ - c=((int32_t)b-BOCU1_START_POS_3)*BOCU1_TRAIL_COUNT*BOCU1_TRAIL_COUNT+BOCU1_REACH_POS_2+1; - count=2; - } else { - /* four bytes */ - c=BOCU1_REACH_POS_3+1; - count=3; - } - } else { - /* negative difference */ - if(b>=BOCU1_START_NEG_3) { - /* two bytes */ - c=((int32_t)b-BOCU1_START_NEG_2)*BOCU1_TRAIL_COUNT+BOCU1_REACH_NEG_1; - count=1; - } else if(b>BOCU1_MIN) { - /* three bytes */ - c=((int32_t)b-BOCU1_START_NEG_3)*BOCU1_TRAIL_COUNT*BOCU1_TRAIL_COUNT+BOCU1_REACH_NEG_2; - count=2; - } else { - /* four bytes */ - c=-BOCU1_TRAIL_COUNT*BOCU1_TRAIL_COUNT*BOCU1_TRAIL_COUNT+BOCU1_REACH_NEG_3; - count=3; - } - } - - /* set the state for decoding the trail byte(s) */ - pRx->diff=c; - pRx->count=count; - return -1; -} - -/** - * Function for BOCU-1 decoder; handles multi-byte trail bytes. - * - * @param pRx pointer to the decoder state structure - * @param b trail byte - * @return result value, same as decodeBocu1 - * - * @see decodeBocu1 - */ -static int32_t -decodeBocu1TrailByte(Bocu1Rx *pRx, uint8_t b) { - int32_t t, c, count; - - if(b<=0x20) { - /* skip some C0 controls and make the trail byte range contiguous */ - t=bocu1ByteToTrail[b]; - if(t<0) { - /* illegal trail byte value */ - pRx->prev=BOCU1_ASCII_PREV; - pRx->count=0; - return -99; - } -#if BOCU1_MAX_TRAIL<0xff - } else if(b>BOCU1_MAX_TRAIL) { - return -99; -#endif - } else { - t=(int32_t)b-BOCU1_TRAIL_BYTE_OFFSET; - } - - /* add trail byte into difference and decrement count */ - c=pRx->diff; - count=pRx->count; - - if(count==1) { - /* final trail byte, deliver a code point */ - c=pRx->prev+c+t; - if(0<=c && c<=0x10ffff) { - /* valid code point result */ - pRx->prev=bocu1Prev(c); - pRx->count=0; - return c; - } else { - /* illegal code point result */ - pRx->prev=BOCU1_ASCII_PREV; - pRx->count=0; - return -99; - } - } - - /* intermediate trail byte */ - if(count==2) { - pRx->diff=c+t*BOCU1_TRAIL_COUNT; - } else /* count==3 */ { - pRx->diff=c+t*BOCU1_TRAIL_COUNT*BOCU1_TRAIL_COUNT; - } - pRx->count=count-1; - return -1; -} - -/** - * BOCU-1 decoder function. - * - * @param pRx pointer to the decoder state structure; - * the initial values should be 0 which - * decodeBocu1 will set to actual initial state values - * @param b an input byte - * @return - * 0..0x10ffff for a result code point - * -1 if only the state changed without code point output - * <-1 if an error occurs - */ -U_CFUNC int32_t -decodeBocu1(Bocu1Rx *pRx, uint8_t b) { - int32_t prev, c, count; - - if(pRx==NULL) { - /* illegal argument */ - return -99; - } - - prev=pRx->prev; - if(prev==0) { - /* lenient handling of initial 0 values */ - prev=pRx->prev=BOCU1_ASCII_PREV; - count=pRx->count=0; - } else { - count=pRx->count; - } - - if(count==0) { - /* byte in lead position */ - if(b<=0x20) { - /* - * Direct-encoded C0 control code or space. - * Reset prev for C0 control codes but not for space. - */ - if(b!=0x20) { - pRx->prev=BOCU1_ASCII_PREV; - } - return b; - } - - /* - * b is a difference lead byte. - * - * Return a code point directly from a single-byte difference. - * - * For multi-byte difference lead bytes, set the decoder state - * with the partial difference value from the lead byte and - * with the number of trail bytes. - * - * For four-byte differences, the signedness also affects the - * first trail byte, which has special handling farther below. - */ - if(b>=BOCU1_START_NEG_2 && b<BOCU1_START_POS_2) { - /* single-byte difference */ - c=prev+((int32_t)b-BOCU1_MIDDLE); - pRx->prev=bocu1Prev(c); - return c; - } else if(b==BOCU1_RESET) { - /* only reset the state, no code point */ - pRx->prev=BOCU1_ASCII_PREV; - return -1; - } else { - return decodeBocu1LeadByte(pRx, b); - } - } else { - /* trail byte in any position */ - return decodeBocu1TrailByte(pRx, b); - } -} - -/* icuhtml/design/conversion/bocu1/bocu1tst.c ------------------------------- */ - -/* test code ---------------------------------------------------------------- */ - -/* test code options */ - -/* ignore comma when processing name lists in testText() */ -#define TEST_IGNORE_COMMA 1 - -/** - * Write a packed BOCU-1 byte sequence into a byte array, - * without overflow check. - * Test function. - * - * @param packed packed BOCU-1 byte sequence, see packDiff() - * @param p pointer to byte array - * @return number of bytes - * - * @see packDiff - */ -static int32_t -writePacked(int32_t packed, uint8_t *p) { - int32_t count=BOCU1_LENGTH_FROM_PACKED(packed); - switch(count) { - case 4: - *p++=(uint8_t)(packed>>24); - case 3: - *p++=(uint8_t)(packed>>16); - case 2: - *p++=(uint8_t)(packed>>8); - case 1: - *p++=(uint8_t)packed; - default: - break; - } - - return count; -} - -/** - * Unpack a packed BOCU-1 non-C0/space byte sequence and get - * the difference to initialPrev. - * Used only for round-trip testing of the difference encoding and decoding. - * Test function. - * - * @param initialPrev bogus "previous code point" value to make sure that - * the resulting code point is in the range 0..0x10ffff - * @param packed packed BOCU-1 byte sequence - * @return the difference to initialPrev - * - * @see packDiff - * @see writeDiff - */ -static int32_t -unpackDiff(int32_t initialPrev, int32_t packed) { - Bocu1Rx rx={ 0, 0, 0 }; - int32_t count; - - rx.prev=initialPrev; - count=BOCU1_LENGTH_FROM_PACKED(packed); - switch(count) { - case 4: - decodeBocu1(&rx, (uint8_t)(packed>>24)); - case 3: - decodeBocu1(&rx, (uint8_t)(packed>>16)); - case 2: - decodeBocu1(&rx, (uint8_t)(packed>>8)); - case 1: - /* subtract initial prev */ - return decodeBocu1(&rx, (uint8_t)packed)-initialPrev; - default: - return -0x7fffffff; - } -} - -/** - * Encode one difference value -0x10ffff..+0x10ffff in 1..4 bytes, - * preserving lexical order. - * Also checks for roundtripping of the difference encoding. - * Test function. - * - * @param diff difference value to test, -0x10ffff..0x10ffff - * @param p pointer to output byte array - * @return p advanced by number of bytes output - * - * @see unpackDiff - */ -static uint8_t * -writeDiff(int32_t diff, uint8_t *p) { - /* generate the difference as a packed value and serialize it */ - int32_t packed, initialPrev; - - packed=packDiff(diff); - - /* - * bogus initial "prev" to work around - * code point range check in decodeBocu1() - */ - if(diff<=0) { - initialPrev=0x10ffff; - } else { - initialPrev=-1; - } - - if(diff!=unpackDiff(initialPrev, packed)) { - log_err("error: unpackDiff(packDiff(diff=%ld)=0x%08lx)=%ld!=diff\n", - diff, packed, unpackDiff(initialPrev, packed)); - } - return p+writePacked(packed, p); -} - -/** - * Encode a UTF-16 string in BOCU-1. - * Does not check for overflows, but otherwise useful function. - * - * @param s input UTF-16 string - * @param length number of UChar code units in s - * @param p pointer to output byte array - * @return number of bytes output - */ -static int32_t -writeString(const UChar *s, int32_t length, uint8_t *p) { - uint8_t *p0; - int32_t c, prev, i; - - prev=0; - p0=p; - i=0; - while(i<length) { - UTF_NEXT_CHAR(s, i, length, c); - p+=writePacked(encodeBocu1(&prev, c), p); - } - return (int32_t)(p-p0); -} - -/** - * Decode a BOCU-1 byte sequence to a UTF-16 string. - * Does not check for overflows, but otherwise useful function. - * - * @param p pointer to input BOCU-1 bytes - * @param length number of input bytes - * @param s point to output UTF-16 string array - * @return number of UChar code units output - */ -static int32_t -readString(const uint8_t *p, int32_t length, UChar *s) { - Bocu1Rx rx={ 0, 0, 0 }; - int32_t c, i, sLength; - - i=sLength=0; - while(i<length) { - c=decodeBocu1(&rx, p[i++]); - if(c<-1) { - log_err("error: readString detects encoding error at string index %ld\n", i); - return -1; - } - if(c>=0) { - UTF_APPEND_CHAR_UNSAFE(s, sLength, c); - } - } - return sLength; -} - -static U_INLINE char -hexDigit(uint8_t digit) { - return digit<=9 ? (char)('0'+digit) : (char)('a'-10+digit); -} - -/** - * Pretty-print 0-terminated byte values. - * Helper function for test output. - * - * @param bytes 0-terminated byte array to print - */ -static void -printBytes(uint8_t *bytes, char *out) { - int i; - uint8_t b; - - i=0; - while((b=*bytes++)!=0) { - *out++=' '; - *out++=hexDigit((uint8_t)(b>>4)); - *out++=hexDigit((uint8_t)(b&0xf)); - ++i; - } - i=3*(5-i); - while(i>0) { - *out++=' '; - --i; - } - *out=0; -} - -/** - * Basic BOCU-1 test function, called when there are no command line arguments. - * Prints some of the #define values and performs round-trip tests of the - * difference encoding and decoding. - */ -static void -TestBOCU1RefDiff(void) { - char buf1[80], buf2[80]; - uint8_t prev[5], level[5]; - int32_t i, cmp, countErrors; - - log_verbose("reach of single bytes: %ld\n", 1+BOCU1_REACH_POS_1-BOCU1_REACH_NEG_1); - log_verbose("reach of 2 bytes : %ld\n", 1+BOCU1_REACH_POS_2-BOCU1_REACH_NEG_2); - log_verbose("reach of 3 bytes : %ld\n\n", 1+BOCU1_REACH_POS_3-BOCU1_REACH_NEG_3); - - log_verbose(" BOCU1_REACH_NEG_1 %8ld BOCU1_REACH_POS_1 %8ld\n", BOCU1_REACH_NEG_1, BOCU1_REACH_POS_1); - log_verbose(" BOCU1_REACH_NEG_2 %8ld BOCU1_REACH_POS_2 %8ld\n", BOCU1_REACH_NEG_2, BOCU1_REACH_POS_2); - log_verbose(" BOCU1_REACH_NEG_3 %8ld BOCU1_REACH_POS_3 %8ld\n\n", BOCU1_REACH_NEG_3, BOCU1_REACH_POS_3); - - log_verbose(" BOCU1_MIDDLE 0x%02x\n", BOCU1_MIDDLE); - log_verbose(" BOCU1_START_NEG_2 0x%02x BOCU1_START_POS_2 0x%02x\n", BOCU1_START_NEG_2, BOCU1_START_POS_2); - log_verbose(" BOCU1_START_NEG_3 0x%02x BOCU1_START_POS_3 0x%02x\n\n", BOCU1_START_NEG_3, BOCU1_START_POS_3); - - /* test packDiff() & unpackDiff() with some specific values */ - writeDiff(0, level); - writeDiff(1, level); - writeDiff(65, level); - writeDiff(130, level); - writeDiff(30000, level); - writeDiff(1000000, level); - writeDiff(-65, level); - writeDiff(-130, level); - writeDiff(-30000, level); - writeDiff(-1000000, level); - - /* test that each value is smaller than any following one */ - countErrors=0; - i=-0x10ffff; - *writeDiff(i, prev)=0; - - /* show first number and bytes */ - printBytes(prev, buf1); - log_verbose(" wD(%8ld) %s\n", i, buf1); - - for(++i; i<=0x10ffff; ++i) { - *writeDiff(i, level)=0; - cmp=strcmp((const char *)prev, (const char *)level); - if(BOCU1_LENGTH_FROM_LEAD(level[0])!=(int32_t)strlen((const char *)level)) { - log_verbose("BOCU1_LENGTH_FROM_LEAD(0x%02x)=%ld!=%ld=strlen(writeDiff(%ld))\n", - level[0], BOCU1_LENGTH_FROM_LEAD(level[0]), strlen((const char *)level), i); - } - if(cmp<0) { - if(i==0 || i==1 || strlen((const char *)prev)!=strlen((const char *)level)) { - /* - * if the result is good, then print only if the length changed - * to get little but interesting output - */ - printBytes(prev, buf1); - printBytes(level, buf2); - log_verbose("ok: strcmp(wD(%8ld), wD(%8ld))=%2d %s%s\n", i-1, i, cmp, buf1, buf2); - } - } else { - ++countErrors; - printBytes(prev, buf1); - printBytes(level, buf2); - log_verbose("wrong: strcmp(wD(%8ld), wD(%8ld))=%2d %s%s\n", i-1, i, cmp, buf1, buf2); - } - /* remember the previous bytes */ - memcpy(prev, level, 4); - } - - /* show last number and bytes */ - printBytes((uint8_t *)"", buf1); - printBytes(prev, buf2); - log_verbose(" wD(%8ld) %s%s\n", i-1, buf1, buf2); - - if(countErrors==0) { - log_verbose("writeDiff(-0x10ffff..0x10ffff) works fine\n"); - } else { - log_err("writeDiff(-0x10ffff..0x10ffff) violates lexical ordering in %d cases\n", countErrors); - } - - /* output signature byte sequence */ - i=0; - writePacked(encodeBocu1(&i, 0xfeff), level); - log_verbose("\nBOCU-1 signature byte sequence: %02x %02x %02x\n", - level[0], level[1], level[2]); -} - -/* cintltst code ------------------------------------------------------------ */ - -static const int32_t DEFAULT_BUFFER_SIZE = 30000; - - -/* test one string with the ICU and the reference BOCU-1 implementations */ -static void -roundtripBOCU1(UConverter *bocu1, int32_t number, const UChar *text, int32_t length) { - UChar *roundtripRef, *roundtripICU; - char *bocu1Ref, *bocu1ICU; - - int32_t bocu1RefLength, bocu1ICULength, roundtripRefLength, roundtripICULength; - UErrorCode errorCode; - - roundtripRef = malloc(DEFAULT_BUFFER_SIZE * sizeof(UChar)); - roundtripICU = malloc(DEFAULT_BUFFER_SIZE * sizeof(UChar)); - bocu1Ref = malloc(DEFAULT_BUFFER_SIZE); - bocu1ICU = malloc(DEFAULT_BUFFER_SIZE); - - /* Unicode -> BOCU-1 */ - bocu1RefLength=writeString(text, length, (uint8_t *)bocu1Ref); - - errorCode=U_ZERO_ERROR; - bocu1ICULength=ucnv_fromUChars(bocu1, bocu1ICU, DEFAULT_BUFFER_SIZE, text, length, &errorCode); - if(U_FAILURE(errorCode)) { - log_err("ucnv_fromUChars(BOCU-1, text(%d)[%d]) failed: %s\n", number, length, u_errorName(errorCode)); - return; - } - - if(bocu1RefLength!=bocu1ICULength || 0!=uprv_memcmp(bocu1Ref, bocu1ICU, bocu1RefLength)) { - log_err("Unicode(%d)[%d] -> BOCU-1: reference[%d]!=ICU[%d]\n", number, length, bocu1RefLength, bocu1ICULength); - return; - } - - /* BOCU-1 -> Unicode */ - roundtripRefLength=readString((uint8_t *)bocu1Ref, bocu1RefLength, roundtripRef); - if(roundtripRefLength<0) { - return; /* readString() found an error and reported it */ - } - - roundtripICULength=ucnv_toUChars(bocu1, roundtripICU, DEFAULT_BUFFER_SIZE, bocu1ICU, bocu1ICULength, &errorCode); - if(U_FAILURE(errorCode)) { - log_err("ucnv_toUChars(BOCU-1, text(%d)[%d]) failed: %s\n", number, length, u_errorName(errorCode)); - return; - } - - if(length!=roundtripRefLength || 0!=u_memcmp(text, roundtripRef, length)) { - log_err("BOCU-1 -> Unicode: original(%d)[%d]!=reference[%d]\n", number, length, roundtripRefLength); - return; - } - if(roundtripRefLength!=roundtripICULength || 0!=u_memcmp(roundtripRef, roundtripICU, roundtripRefLength)) { - log_err("BOCU-1 -> Unicode: reference(%d)[%d]!=ICU[%d]\n", number, roundtripRefLength, roundtripICULength); - return; - } - free(roundtripRef); - free(roundtripICU); - free(bocu1Ref); - free(bocu1ICU); -} - -static const UChar feff[]={ 0xfeff }; -static const UChar ascii[]={ 0x61, 0x62, 0x20, 0x63, 0x61 }; -static const UChar crlf[]={ 0xd, 0xa, 0x20 }; -static const UChar nul[]={ 0 }; -static const UChar latin[]={ 0xdf, 0xe6 }; -static const UChar devanagari[]={ 0x930, 0x20, 0x918, 0x909 }; -static const UChar hiragana[]={ 0x3086, 0x304d, 0x20, 0x3053, 0x4000 }; -static const UChar unihan[]={ 0x4e00, 0x7777, 0x20, 0x9fa5, 0x4e00 }; -static const UChar hangul[]={ 0xac00, 0xbcde, 0x20, 0xd7a3 }; -static const UChar surrogates[]={ 0xdc00, 0xd800 }; /* single surrogates, unmatched! */ -static const UChar plane1[]={ 0xd800, 0xdc00 }; -static const UChar plane2[]={ 0xd845, 0xdddd }; -static const UChar plane15[]={ 0xdbbb, 0xddee, 0x20 }; -static const UChar plane16[]={ 0xdbff, 0xdfff }; -static const UChar c0[]={ 1, 0xe40, 0x20, 9 }; - -static const struct { - const UChar *s; - int32_t length; -} strings[]={ - { feff, LENGTHOF(feff) }, - { ascii, LENGTHOF(ascii) }, - { crlf, LENGTHOF(crlf) }, - { nul, LENGTHOF(nul) }, - { latin, LENGTHOF(latin) }, - { devanagari, LENGTHOF(devanagari) }, - { hiragana, LENGTHOF(hiragana) }, - { unihan, LENGTHOF(unihan) }, - { hangul, LENGTHOF(hangul) }, - { surrogates, LENGTHOF(surrogates) }, - { plane1, LENGTHOF(plane1) }, - { plane2, LENGTHOF(plane2) }, - { plane15, LENGTHOF(plane15) }, - { plane16, LENGTHOF(plane16) }, - { c0, LENGTHOF(c0) } -}; - -/* - * Verify that the ICU BOCU-1 implementation produces the same results as - * the reference implementation from the design folder. - * Generate some texts and convert them with both converters, verifying - * identical results and roundtripping. - */ -static void -TestBOCU1(void) { - UChar *text; - int32_t i, length; - - UConverter *bocu1; - UErrorCode errorCode; - - errorCode=U_ZERO_ERROR; - bocu1=ucnv_open("BOCU-1", &errorCode); - if(U_FAILURE(errorCode)) { - log_err("error: unable to open BOCU-1 converter: %s\n", u_errorName(errorCode)); - return; - } - - text = malloc(DEFAULT_BUFFER_SIZE * sizeof(UChar)); - - /* text 1: each of strings[] once */ - length=0; - for(i=0; i<LENGTHOF(strings); ++i) { - u_memcpy(text+length, strings[i].s, strings[i].length); - length+=strings[i].length; - } - roundtripBOCU1(bocu1, 1, text, length); - - /* text 2: each of strings[] twice */ - length=0; - for(i=0; i<LENGTHOF(strings); ++i) { - u_memcpy(text+length, strings[i].s, strings[i].length); - length+=strings[i].length; - u_memcpy(text+length, strings[i].s, strings[i].length); - length+=strings[i].length; - } - roundtripBOCU1(bocu1, 2, text, length); - - /* text 3: each of strings[] many times (set step vs. |strings| so that all strings are used) */ - length=0; - for(i=1; length<5000; i+=7) { - if(i>=LENGTHOF(strings)) { - i-=LENGTHOF(strings); - } - u_memcpy(text+length, strings[i].s, strings[i].length); - length+=strings[i].length; - } - roundtripBOCU1(bocu1, 3, text, length); - - ucnv_close(bocu1); - free(text); -} - -U_CFUNC void addBOCU1Tests(TestNode** root); - -U_CFUNC void -addBOCU1Tests(TestNode** root) { - addTest(root, TestBOCU1RefDiff, "tsconv/bocu1tst/TestBOCU1RefDiff"); - addTest(root, TestBOCU1, "tsconv/bocu1tst/TestBOCU1"); -} |