diff options
Diffstat (limited to 'Build/source/libs/icu/icu-xetex/common/unorm.cpp')
-rw-r--r-- | Build/source/libs/icu/icu-xetex/common/unorm.cpp | 4099 |
1 files changed, 4099 insertions, 0 deletions
diff --git a/Build/source/libs/icu/icu-xetex/common/unorm.cpp b/Build/source/libs/icu/icu-xetex/common/unorm.cpp new file mode 100644 index 00000000000..00ee9ec3e7d --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/common/unorm.cpp @@ -0,0 +1,4099 @@ +/* +****************************************************************************** +* Copyright (c) 1996-2007, International Business Machines +* Corporation and others. All Rights Reserved. +****************************************************************************** +* File unorm.cpp +* +* Created by: Vladimir Weinstein 12052000 +* +* Modification history : +* +* Date Name Description +* 02/01/01 synwee Added normalization quickcheck enum and method. +* 02/12/01 synwee Commented out quickcheck util api has been approved +* Added private method for doing FCD checks +* 02/23/01 synwee Modified quickcheck and checkFCE to run through +* string for codepoints < 0x300 for the normalization +* mode NFC. +* 05/25/01+ Markus Scherer total rewrite, implement all normalization here +* instead of just wrappers around normlzr.cpp, +* load unorm.dat, support Unicode 3.1 with +* supplementary code points, etc. +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_NORMALIZATION + +#include "unicode/udata.h" +#include "unicode/uchar.h" +#include "unicode/ustring.h" +#include "unicode/uiter.h" +#include "unicode/uniset.h" +#include "unicode/usetiter.h" +#include "unicode/unorm.h" +#include "ucln_cmn.h" +#include "unormimp.h" +#include "ucase.h" +#include "cmemory.h" +#include "umutex.h" +#include "utrie.h" +#include "unicode/uset.h" +#include "udataswp.h" +#include "putilimp.h" + +/* + * Status of tailored normalization + * + * This was done initially for investigation on Unicode public review issue 7 + * (http://www.unicode.org/review/). See Jitterbug 2481. + * While the UTC at meeting #94 (2003mar) did not take up the issue, this is + * a permanent feature in ICU 2.6 in support of IDNA which requires true + * Unicode 3.2 normalization. + * (NormalizationCorrections are rolled into IDNA mapping tables.) + * + * Tailored normalization as implemented here allows to "normalize less" + * than full Unicode normalization would. + * Based internally on a UnicodeSet of code points that are + * "excluded from normalization", the normalization functions leave those + * code points alone ("inert"). This means that tailored normalization + * still transforms text into a canonically equivalent form. + * It does not add decompositions to code points that do not have any or + * change decomposition results. + * + * Any function that searches for a safe boundary has not been touched, + * which means that these functions will be over-pessimistic when + * exclusions are applied. + * This should not matter because subsequent checks and normalizations + * do apply the exclusions; only a little more of the text may be processed + * than necessary under exclusions. + * + * Normalization exclusions have the following effect on excluded code points c: + * - c is not decomposed + * - c is not a composition target + * - c does not combine forward or backward for composition + * except that this is not implemented for Jamo + * - c is treated as having a combining class of 0 + */ +#define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0])) + +U_NAMESPACE_USE + +/* + * This new implementation of the normalization code loads its data from + * unorm.dat, which is generated with the gennorm tool. + * The format of that file is described in unormimp.h . + */ + +/* -------------------------------------------------------------------------- */ + +enum { + _STACK_BUFFER_CAPACITY=100 +}; + +/* + * Constants for the bit fields in the options bit set parameter. + * These need not be public. + * A user only needs to know the currently assigned values. + * The number and positions of reserved bits per field can remain private + * and may change in future implementations. + */ +enum { + _NORM_OPTIONS_NX_MASK=0x1f, + _NORM_OPTIONS_UNICODE_MASK=0x60, + _NORM_OPTIONS_SETS_MASK=0x7f, + + _NORM_OPTIONS_UNICODE_SHIFT=5, + + /* + * The following options are used only in some composition functions. + * They use bits 12 and up to preserve lower bits for the available options + * space in unorm_compare() - + * see documentation for UNORM_COMPARE_NORM_OPTIONS_SHIFT. + */ + + /** Options bit 12, for compatibility vs. canonical decomposition. */ + _NORM_OPTIONS_COMPAT=0x1000, + /** Options bit 13, no discontiguous composition (FCC vs. NFC). */ + _NORM_OPTIONS_COMPOSE_CONTIGUOUS=0x2000 +}; + +U_CDECL_BEGIN +static inline UBool +isHangulWithoutJamoT(UChar c) { + c-=HANGUL_BASE; + return c<HANGUL_COUNT && c%JAMO_T_COUNT==0; +} + +/* norm32 helpers */ + +/* is this a norm32 with a regular index? */ +static inline UBool +isNorm32Regular(uint32_t norm32) { + return norm32<_NORM_MIN_SPECIAL; +} + +/* is this a norm32 with a special index for a lead surrogate? */ +static inline UBool +isNorm32LeadSurrogate(uint32_t norm32) { + return _NORM_MIN_SPECIAL<=norm32 && norm32<_NORM_SURROGATES_TOP; +} + +/* is this a norm32 with a special index for a Hangul syllable or a Jamo? */ +static inline UBool +isNorm32HangulOrJamo(uint32_t norm32) { + return norm32>=_NORM_MIN_HANGUL; +} + +/* + * Given isNorm32HangulOrJamo(), + * is this a Hangul syllable or a Jamo? + */ +/*static inline UBool +isHangulJamoNorm32HangulOrJamoL(uint32_t norm32) { + return norm32<_NORM_MIN_JAMO_V; +}*/ + +/* + * Given norm32 for Jamo V or T, + * is this a Jamo V? + */ +static inline UBool +isJamoVTNorm32JamoV(uint32_t norm32) { + return norm32<_NORM_JAMO_V_TOP; +} + +/* load unorm.dat ----------------------------------------------------------- */ + +/* normTrie: 32-bit trie result may contain a special extraData index with the folding offset */ +static int32_t U_CALLCONV +getFoldingNormOffset(uint32_t norm32) { + if(isNorm32LeadSurrogate(norm32)) { + return + UTRIE_BMP_INDEX_LENGTH+ + (((int32_t)norm32>>(_NORM_EXTRA_SHIFT-UTRIE_SURROGATE_BLOCK_BITS))& + (0x3ff<<UTRIE_SURROGATE_BLOCK_BITS)); + } else { + return 0; + } +} + +/* auxTrie: the folding offset is in bits 9..0 of the 16-bit trie result */ +static int32_t U_CALLCONV +getFoldingAuxOffset(uint32_t data) { + return (int32_t)(data&_NORM_AUX_FNC_MASK)<<UTRIE_SURROGATE_BLOCK_BITS; +} +U_CDECL_END + +#define UNORM_HARDCODE_DATA 1 + +#if UNORM_HARDCODE_DATA + +/* unorm_props_data.c is machine-generated by gennorm --csource */ +#include "unorm_props_data.c" + +static const UBool formatVersion_2_2=TRUE; + +#else + +#define DATA_NAME "unorm" +#define DATA_TYPE "icu" + +static UDataMemory *normData=NULL; +static UErrorCode dataErrorCode=U_ZERO_ERROR; +static int8_t haveNormData=0; + +static int32_t indexes[_NORM_INDEX_TOP]={ 0 }; +static UTrie normTrie={ 0,0,0,0,0,0,0 }, fcdTrie={ 0,0,0,0,0,0,0 }, auxTrie={ 0,0,0,0,0,0,0 }; + +/* + * pointers into the memory-mapped unorm.icu + */ +static const uint16_t *extraData=NULL, + *combiningTable=NULL, + *canonStartSets=NULL; + +static uint8_t formatVersion[4]={ 0, 0, 0, 0 }; +static UBool formatVersion_2_1=FALSE, formatVersion_2_2=FALSE; + +/* the Unicode version of the normalization data */ +static UVersionInfo dataVersion={ 0, 0, 0, 0 }; + +#endif + +/* cache UnicodeSets for each combination of exclusion flags */ +static UnicodeSet *nxCache[_NORM_OPTIONS_SETS_MASK+1]={ NULL }; + +U_CDECL_BEGIN + +static UBool U_CALLCONV +unorm_cleanup(void) { + int32_t i; + +#if !UNORM_HARDCODE_DATA + if(normData!=NULL) { + udata_close(normData); + normData=NULL; + } + dataErrorCode=U_ZERO_ERROR; + haveNormData=0; +#endif + + for(i=0; i<(int32_t)LENGTHOF(nxCache); ++i) { + if (nxCache[i]) { + delete nxCache[i]; + nxCache[i] = 0; + } + } + + return TRUE; +} + +#if !UNORM_HARDCODE_DATA + +static UBool U_CALLCONV +isAcceptable(void * /* context */, + const char * /* type */, const char * /* name */, + const UDataInfo *pInfo) { + if( + pInfo->size>=20 && + pInfo->isBigEndian==U_IS_BIG_ENDIAN && + pInfo->charsetFamily==U_CHARSET_FAMILY && + pInfo->dataFormat[0]==0x4e && /* dataFormat="Norm" */ + pInfo->dataFormat[1]==0x6f && + pInfo->dataFormat[2]==0x72 && + pInfo->dataFormat[3]==0x6d && + pInfo->formatVersion[0]==2 && + pInfo->formatVersion[2]==UTRIE_SHIFT && + pInfo->formatVersion[3]==UTRIE_INDEX_SHIFT + ) { + uprv_memcpy(formatVersion, pInfo->formatVersion, 4); + uprv_memcpy(dataVersion, pInfo->dataVersion, 4); + return TRUE; + } else { + return FALSE; + } +} + +#endif + +static UBool U_CALLCONV +_enumPropertyStartsRange(const void *context, UChar32 start, UChar32 /*limit*/, uint32_t /*value*/) { + /* add the start code point to the USet */ + const USetAdder *sa=(const USetAdder *)context; + sa->add(sa->set, start); + return TRUE; +} + +U_CDECL_END + +#if !UNORM_HARDCODE_DATA + +static int8_t +loadNormData(UErrorCode &errorCode) { + /* load Unicode normalization data from file */ + + /* + * This lazy intialization with double-checked locking (without mutex protection for + * haveNormData==0) is transiently unsafe under certain circumstances. + * Check the readme and use u_init() if necessary. + * + * While u_init() initializes the main normalization data via this functions, + * it does not do so for exclusion sets (which are fully mutexed). + * This is because + * - there can be many exclusion sets + * - they are rarely used + * - they are not usually used in execution paths that are + * as performance-sensitive as others + * (e.g., IDNA takes more time than unorm_quickCheck() anyway) + */ + if(haveNormData==0) { + UTrie _normTrie={ 0,0,0,0,0,0,0 }, _fcdTrie={ 0,0,0,0,0,0,0 }, _auxTrie={ 0,0,0,0,0,0,0 }; + UDataMemory *data; + + const int32_t *p=NULL; + const uint8_t *pb; + + if(&errorCode==NULL || U_FAILURE(errorCode)) { + return 0; + } + + /* open the data outside the mutex block */ + data=udata_openChoice(NULL, DATA_TYPE, DATA_NAME, isAcceptable, NULL, &errorCode); + dataErrorCode=errorCode; + if(U_FAILURE(errorCode)) { + return haveNormData=-1; + } + + p=(const int32_t *)udata_getMemory(data); + pb=(const uint8_t *)(p+_NORM_INDEX_TOP); + utrie_unserialize(&_normTrie, pb, p[_NORM_INDEX_TRIE_SIZE], &errorCode); + _normTrie.getFoldingOffset=getFoldingNormOffset; + + pb+=p[_NORM_INDEX_TRIE_SIZE]+p[_NORM_INDEX_UCHAR_COUNT]*2+p[_NORM_INDEX_COMBINE_DATA_COUNT]*2; + if(p[_NORM_INDEX_FCD_TRIE_SIZE]!=0) { + utrie_unserialize(&_fcdTrie, pb, p[_NORM_INDEX_FCD_TRIE_SIZE], &errorCode); + } + pb+=p[_NORM_INDEX_FCD_TRIE_SIZE]; + + if(p[_NORM_INDEX_AUX_TRIE_SIZE]!=0) { + utrie_unserialize(&_auxTrie, pb, p[_NORM_INDEX_AUX_TRIE_SIZE], &errorCode); + _auxTrie.getFoldingOffset=getFoldingAuxOffset; + } + + if(U_FAILURE(errorCode)) { + dataErrorCode=errorCode; + udata_close(data); + return haveNormData=-1; + } + + /* in the mutex block, set the data for this process */ + umtx_lock(NULL); + if(normData==NULL) { + normData=data; + data=NULL; + + uprv_memcpy(&indexes, p, sizeof(indexes)); + uprv_memcpy(&normTrie, &_normTrie, sizeof(UTrie)); + uprv_memcpy(&fcdTrie, &_fcdTrie, sizeof(UTrie)); + uprv_memcpy(&auxTrie, &_auxTrie, sizeof(UTrie)); + } else { + p=(const int32_t *)udata_getMemory(normData); + } + + /* initialize some variables */ + extraData=(uint16_t *)((uint8_t *)(p+_NORM_INDEX_TOP)+indexes[_NORM_INDEX_TRIE_SIZE]); + combiningTable=extraData+indexes[_NORM_INDEX_UCHAR_COUNT]; + formatVersion_2_1=formatVersion[0]>2 || (formatVersion[0]==2 && formatVersion[1]>=1); + formatVersion_2_2=formatVersion[0]>2 || (formatVersion[0]==2 && formatVersion[1]>=2); + if(formatVersion_2_1) { + canonStartSets=combiningTable+ + indexes[_NORM_INDEX_COMBINE_DATA_COUNT]+ + (indexes[_NORM_INDEX_FCD_TRIE_SIZE]+indexes[_NORM_INDEX_AUX_TRIE_SIZE])/2; + } + haveNormData=1; + ucln_common_registerCleanup(UCLN_COMMON_UNORM, unorm_cleanup); + umtx_unlock(NULL); + + /* if a different thread set it first, then close the extra data */ + if(data!=NULL) { + udata_close(data); /* NULL if it was set correctly */ + } + } + + return haveNormData; +} + +#endif + +static inline UBool +_haveData(UErrorCode &errorCode) { +#if UNORM_HARDCODE_DATA + return U_SUCCESS(errorCode); +#else + if(U_FAILURE(errorCode)) { + return FALSE; + } else if(haveNormData>0) { + return TRUE; + } else if(haveNormData<0) { + errorCode=dataErrorCode; + return FALSE; + } else /* haveNormData==0 */ { + return (UBool)(loadNormData(errorCode)>0); + } +#endif +} + +U_CAPI UBool U_EXPORT2 +unorm_haveData(UErrorCode *pErrorCode) { + return _haveData(*pErrorCode); +} + +U_CAPI const uint16_t * U_EXPORT2 +unorm_getFCDTrie(UErrorCode *pErrorCode) { + if(_haveData(*pErrorCode)) { + return fcdTrie.index; + } else { + return NULL; + } +} + +/* data access primitives --------------------------------------------------- */ + +static inline uint32_t +_getNorm32(UChar c) { + return UTRIE_GET32_FROM_LEAD(&normTrie, c); +} + +static inline uint32_t +_getNorm32FromSurrogatePair(uint32_t norm32, UChar c2) { + /* + * the surrogate index in norm32 stores only the number of the surrogate index block + * see gennorm/store.c/getFoldedNormValue() + */ + norm32= + UTRIE_BMP_INDEX_LENGTH+ + ((norm32>>(_NORM_EXTRA_SHIFT-UTRIE_SURROGATE_BLOCK_BITS))& + (0x3ff<<UTRIE_SURROGATE_BLOCK_BITS)); + return UTRIE_GET32_FROM_OFFSET_TRAIL(&normTrie, norm32, c2); +} + +/* + * get a norm32 from text with complete code points + * (like from decompositions) + */ +static inline uint32_t +_getNorm32(const UChar *p, uint32_t mask) { + uint32_t norm32=_getNorm32(*p); + if((norm32&mask) && isNorm32LeadSurrogate(norm32)) { + /* *p is a lead surrogate, get the real norm32 */ + norm32=_getNorm32FromSurrogatePair(norm32, *(p+1)); + } + return norm32; +} + +static inline uint16_t +_getFCD16(UChar c) { + return UTRIE_GET16_FROM_LEAD(&fcdTrie, c); +} + +static inline uint16_t +_getFCD16FromSurrogatePair(uint16_t fcd16, UChar c2) { + /* the surrogate index in fcd16 is an absolute offset over the start of stage 1 */ + return UTRIE_GET16_FROM_OFFSET_TRAIL(&fcdTrie, fcd16, c2); +} + +static inline const uint16_t * +_getExtraData(uint32_t norm32) { + return extraData+(norm32>>_NORM_EXTRA_SHIFT); +} + +#if 0 +/* + * It is possible to get the FCD data from the main trie if unorm.icu + * was built without the FCD trie, although it is slower. + * This is not implemented because it is hard to test, and because it seems + * unusual to want to use FCD and not build the data file for it. + * + * Untested sample code: + */ +static inline uint16_t +_getFCD16FromNormData(UChar32 c) { + uint32_t norm32, fcd; + + norm32=_getNorm32(c); + if((norm32&_NORM_QC_NFD) && isNorm32Regular(norm32)) { + /* get the lead/trail cc from the decomposition data */ + const uint16_t *nfd=_getExtraData(norm32); + if(*nfd&_NORM_DECOMP_FLAG_LENGTH_HAS_CC) { + fcd=nfd[1]; + } + } else { + fcd=norm32&_NORM_CC_MASK; + if(fcd!=0) { + /* use the code point cc value for both lead and trail cc's */ + fcd|=fcd>>_NORM_CC_SHIFT; /* assume that the cc is in bits 15..8 */ + } + } + + return (uint16_t)fcd; +} +#endif + +/* normalization exclusion sets --------------------------------------------- */ + +/* + * Normalization exclusion UnicodeSets are used for tailored normalization; + * see the comment near the beginning of this file. + * + * By specifying one or several sets of code points, + * those code points become inert for normalization. + */ + +static const UnicodeSet * +internalGetNXHangul(UErrorCode &errorCode) { + /* internal function, does not check for incoming U_FAILURE */ + UBool isCached; + + UMTX_CHECK(NULL, (UBool)(nxCache[UNORM_NX_HANGUL]!=NULL), isCached); + + if(!isCached) { + UnicodeSet *set=new UnicodeSet(0xac00, 0xd7a3); + if(set==NULL) { + errorCode=U_MEMORY_ALLOCATION_ERROR; + return NULL; + } + // Compact the set for caching. + set->compact(); + + umtx_lock(NULL); + if(nxCache[UNORM_NX_HANGUL]==NULL) { + nxCache[UNORM_NX_HANGUL]=set; + set=NULL; + ucln_common_registerCleanup(UCLN_COMMON_UNORM, unorm_cleanup); + } + umtx_unlock(NULL); + + delete set; + } + + return nxCache[UNORM_NX_HANGUL]; +} + +/* unorm.cpp 1.116 had and used +static const UnicodeSet * +internalGetNXFromPattern(int32_t options, const char *pattern, UErrorCode &errorCode) { + ... +} +*/ + +/* get and set an exclusion set from a serialized UnicodeSet */ +static const UnicodeSet * +internalGetSerializedNX(int32_t options, int32_t nxIndex, UErrorCode &errorCode) { + /* internal function, does not check for incoming U_FAILURE */ + UBool isCached; + + UMTX_CHECK(NULL, (UBool)(nxCache[options]!=NULL), isCached); + + if( !isCached && + canonStartSets!=NULL && + canonStartSets[nxIndex]!=0 && canonStartSets[nxIndex+1]>canonStartSets[nxIndex] + ) { + USerializedSet sset; + UnicodeSet *set; + UChar32 start, end; + int32_t i; + + if( !uset_getSerializedSet( + &sset, + canonStartSets+canonStartSets[nxIndex], + canonStartSets[nxIndex+1]-canonStartSets[nxIndex]) + ) { + errorCode=U_INVALID_FORMAT_ERROR; + return NULL; + } + + /* turn the serialized set into a UnicodeSet */ + set=new UnicodeSet(); + if(set==NULL) { + errorCode=U_MEMORY_ALLOCATION_ERROR; + return NULL; + } + for(i=0; uset_getSerializedRange(&sset, i, &start, &end); ++i) { + set->add(start, end); + } + // Compact the set for caching. + set->compact(); + + umtx_lock(NULL); + if(nxCache[options]==NULL) { + nxCache[options]=set; + set=NULL; + ucln_common_registerCleanup(UCLN_COMMON_UNORM, unorm_cleanup); + } + umtx_unlock(NULL); + + delete set; + } + + return nxCache[options]; +} + +static const UnicodeSet * +internalGetNXCJKCompat(UErrorCode &errorCode) { + /* build a set from [[:Ideographic:]&[:NFD_QC=No:]]=[CJK Ideographs]&[has canonical decomposition] */ + return internalGetSerializedNX( + UNORM_NX_CJK_COMPAT, + _NORM_SET_INDEX_NX_CJK_COMPAT_OFFSET, + errorCode); +} + +static const UnicodeSet * +internalGetNXUnicode(uint32_t options, UErrorCode &errorCode) { + /* internal function, does not check for incoming U_FAILURE */ + int32_t nxIndex; + + options&=_NORM_OPTIONS_UNICODE_MASK; + switch(options) { + case 0: + return NULL; + case UNORM_UNICODE_3_2: + /* [:^Age=3.2:] */ + nxIndex=_NORM_SET_INDEX_NX_UNICODE32_OFFSET; + break; + default: + errorCode=U_ILLEGAL_ARGUMENT_ERROR; + return NULL; + } + + /* build a set with all code points that were not designated by the specified Unicode version */ + return internalGetSerializedNX(options, nxIndex, errorCode); +} + +/* Get a decomposition exclusion set. The data must be loaded. */ +static const UnicodeSet * +internalGetNX(int32_t options, UErrorCode &errorCode) { + options&=_NORM_OPTIONS_SETS_MASK; + + UBool isCached; + + UMTX_CHECK(NULL, (UBool)(nxCache[options]!=NULL), isCached); + + if(!isCached) { + /* return basic sets */ + if(options==UNORM_NX_HANGUL) { + return internalGetNXHangul(errorCode); + } + if(options==UNORM_NX_CJK_COMPAT) { + return internalGetNXCJKCompat(errorCode); + } + if((options&_NORM_OPTIONS_UNICODE_MASK)!=0 && (options&_NORM_OPTIONS_NX_MASK)==0) { + return internalGetNXUnicode(options, errorCode); + } + + /* build a set from multiple subsets */ + UnicodeSet *set; + const UnicodeSet *other; + + set=new UnicodeSet(); + if(set==NULL) { + errorCode=U_MEMORY_ALLOCATION_ERROR; + return NULL; + } + + if((options&UNORM_NX_HANGUL)!=0 && NULL!=(other=internalGetNXHangul(errorCode))) { + set->addAll(*other); + } + if((options&UNORM_NX_CJK_COMPAT)!=0 && NULL!=(other=internalGetNXCJKCompat(errorCode))) { + set->addAll(*other); + } + if((options&_NORM_OPTIONS_UNICODE_MASK)!=0 && NULL!=(other=internalGetNXUnicode(options, errorCode))) { + set->addAll(*other); + } + + if(U_FAILURE(errorCode)) { + delete set; + return NULL; + } + // Compact the set for caching. + set->compact(); + + umtx_lock(NULL); + if(nxCache[options]==NULL) { + nxCache[options]=set; + set=NULL; + ucln_common_registerCleanup(UCLN_COMMON_UNORM, unorm_cleanup); + } + umtx_unlock(NULL); + + delete set; + } + + return nxCache[options]; +} + +static inline const UnicodeSet * +getNX(int32_t options, UErrorCode &errorCode) { + if(U_FAILURE(errorCode) || (options&=_NORM_OPTIONS_SETS_MASK)==0) { + /* incoming failure, or no decomposition exclusions requested */ + return NULL; + } else { + return internalGetNX(options, errorCode); + } +} + +U_CFUNC const UnicodeSet * +unorm_getNX(int32_t options, UErrorCode *pErrorCode) { + return getNX(options, *pErrorCode); +} + +static inline UBool +nx_contains(const UnicodeSet *nx, UChar32 c) { + return nx!=NULL && nx->contains(c); +} + +static inline UBool +nx_contains(const UnicodeSet *nx, UChar c, UChar c2) { + return nx!=NULL && nx->contains(c2==0 ? c : U16_GET_SUPPLEMENTARY(c, c2)); +} + +/* other normalization primitives ------------------------------------------- */ + +/* get the canonical or compatibility decomposition for one character */ +static inline const UChar * +_decompose(uint32_t norm32, uint32_t qcMask, int32_t &length, + uint8_t &cc, uint8_t &trailCC) { + const UChar *p=(const UChar *)_getExtraData(norm32); + length=*p++; + + if((norm32&qcMask&_NORM_QC_NFKD)!=0 && length>=0x100) { + /* use compatibility decomposition, skip canonical data */ + p+=((length>>7)&1)+(length&_NORM_DECOMP_LENGTH_MASK); + length>>=8; + } + + if(length&_NORM_DECOMP_FLAG_LENGTH_HAS_CC) { + /* get the lead and trail cc's */ + UChar bothCCs=*p++; + cc=(uint8_t)(bothCCs>>8); + trailCC=(uint8_t)bothCCs; + } else { + /* lead and trail cc's are both 0 */ + cc=trailCC=0; + } + + length&=_NORM_DECOMP_LENGTH_MASK; + return p; +} + +/* get the canonical decomposition for one character */ +static inline const UChar * +_decompose(uint32_t norm32, int32_t &length, + uint8_t &cc, uint8_t &trailCC) { + const UChar *p=(const UChar *)_getExtraData(norm32); + length=*p++; + + if(length&_NORM_DECOMP_FLAG_LENGTH_HAS_CC) { + /* get the lead and trail cc's */ + UChar bothCCs=*p++; + cc=(uint8_t)(bothCCs>>8); + trailCC=(uint8_t)bothCCs; + } else { + /* lead and trail cc's are both 0 */ + cc=trailCC=0; + } + + length&=_NORM_DECOMP_LENGTH_MASK; + return p; +} + +/** + * Get the canonical decomposition for one code point. + * @param c code point + * @param buffer out-only buffer for algorithmic decompositions of Hangul + * @param length out-only, takes the length of the decomposition, if any + * @return pointer to decomposition, or 0 if none + * @internal + */ +U_CFUNC const UChar * +unorm_getCanonicalDecomposition(UChar32 c, UChar buffer[4], int32_t *pLength) { + uint32_t norm32; + + if(c<indexes[_NORM_INDEX_MIN_NFD_NO_MAYBE]) { + /* trivial case */ + return NULL; + } + + UTRIE_GET32(&normTrie, c, norm32); + if(norm32&_NORM_QC_NFD) { + if(isNorm32HangulOrJamo(norm32)) { + /* Hangul syllable: decompose algorithmically */ + UChar c2; + + c-=HANGUL_BASE; + + c2=(UChar)(c%JAMO_T_COUNT); + c/=JAMO_T_COUNT; + if(c2>0) { + buffer[2]=(UChar)(JAMO_T_BASE+c2); + *pLength=3; + } else { + *pLength=2; + } + + buffer[1]=(UChar)(JAMO_V_BASE+c%JAMO_V_COUNT); + buffer[0]=(UChar)(JAMO_L_BASE+c/JAMO_V_COUNT); + return buffer; + } else { + /* normal decomposition */ + uint8_t cc, trailCC; + return _decompose(norm32, *pLength, cc, trailCC); + } + } else { + return 0; + } +} + +/* + * get the combining class of (c, c2)=*p++ + * before: p<limit after: p<=limit + * if only one code unit is used, then c2==0 + */ +static inline uint8_t +_getNextCC(const UChar *&p, const UChar *limit, UChar &c, UChar &c2) { + uint32_t norm32; + + c=*p++; + norm32=_getNorm32(c); + if((norm32&_NORM_CC_MASK)==0) { + c2=0; + return 0; + } else { + if(!isNorm32LeadSurrogate(norm32)) { + c2=0; + } else { + /* c is a lead surrogate, get the real norm32 */ + if(p!=limit && UTF_IS_SECOND_SURROGATE(c2=*p)) { + ++p; + norm32=_getNorm32FromSurrogatePair(norm32, c2); + } else { + c2=0; + return 0; + } + } + + return (uint8_t)(norm32>>_NORM_CC_SHIFT); + } +} + +/* + * read backwards and get norm32 + * return 0 if the character is <minC + * if c2!=0 then (c2, c) is a surrogate pair (reversed - c2 is first surrogate but read second!) + */ +static inline uint32_t +_getPrevNorm32(const UChar *start, const UChar *&src, + uint32_t minC, uint32_t mask, + UChar &c, UChar &c2) { + uint32_t norm32; + + c=*--src; + c2=0; + + /* check for a surrogate before getting norm32 to see if we need to predecrement further */ + if(c<minC) { + return 0; + } else if(!UTF_IS_SURROGATE(c)) { + return _getNorm32(c); + } else if(UTF_IS_SURROGATE_FIRST(c)) { + /* unpaired first surrogate */ + return 0; + } else if(src!=start && UTF_IS_FIRST_SURROGATE(c2=*(src-1))) { + --src; + norm32=_getNorm32(c2); + + if((norm32&mask)==0) { + /* all surrogate pairs with this lead surrogate have only irrelevant data */ + return 0; + } else { + /* norm32 must be a surrogate special */ + return _getNorm32FromSurrogatePair(norm32, c); + } + } else { + /* unpaired second surrogate */ + c2=0; + return 0; + } +} + +/* + * get the combining class of (c, c2)=*--p + * before: start<p after: start<=p + */ +static inline uint8_t +_getPrevCC(const UChar *start, const UChar *&p) { + UChar c, c2; + + return (uint8_t)(_getPrevNorm32(start, p, _NORM_MIN_WITH_LEAD_CC, _NORM_CC_MASK, c, c2)>>_NORM_CC_SHIFT); +} + +/* + * is this a safe boundary character for NF*D? + * (lead cc==0) + */ +static inline UBool +_isNFDSafe(uint32_t norm32, uint32_t ccOrQCMask, uint32_t decompQCMask) { + if((norm32&ccOrQCMask)==0) { + return TRUE; /* cc==0 and no decomposition: this is NF*D safe */ + } + + /* inspect its decomposition - maybe a Hangul but not a surrogate here */ + if(isNorm32Regular(norm32) && (norm32&decompQCMask)!=0) { + int32_t length; + uint8_t cc, trailCC; + + /* decomposes, get everything from the variable-length extra data */ + _decompose(norm32, decompQCMask, length, cc, trailCC); + return cc==0; + } else { + /* no decomposition (or Hangul), test the cc directly */ + return (norm32&_NORM_CC_MASK)==0; + } +} + +/* + * is this (or does its decomposition begin with) a "true starter"? + * (cc==0 and NF*C_YES) + */ +static inline UBool +_isTrueStarter(uint32_t norm32, uint32_t ccOrQCMask, uint32_t decompQCMask) { + if((norm32&ccOrQCMask)==0) { + return TRUE; /* this is a true starter (could be Hangul or Jamo L) */ + } + + /* inspect its decomposition - not a Hangul or a surrogate here */ + if((norm32&decompQCMask)!=0) { + const UChar *p; + int32_t length; + uint8_t cc, trailCC; + + /* decomposes, get everything from the variable-length extra data */ + p=_decompose(norm32, decompQCMask, length, cc, trailCC); + if(cc==0) { + uint32_t qcMask=ccOrQCMask&_NORM_QC_MASK; + + /* does it begin with NFC_YES? */ + if((_getNorm32(p, qcMask)&qcMask)==0) { + /* yes, the decomposition begins with a true starter */ + return TRUE; + } + } + } + return FALSE; +} + +/* uchar.h */ +U_CAPI uint8_t U_EXPORT2 +u_getCombiningClass(UChar32 c) { +#if !UNORM_HARDCODE_DATA + UErrorCode errorCode=U_ZERO_ERROR; + if(_haveData(errorCode)) { +#endif + uint32_t norm32; + + UTRIE_GET32(&normTrie, c, norm32); + return (uint8_t)(norm32>>_NORM_CC_SHIFT); +#if !UNORM_HARDCODE_DATA + } else { + return 0; + } +#endif +} + +U_CFUNC UBool U_EXPORT2 +unorm_internalIsFullCompositionExclusion(UChar32 c) { +#if UNORM_HARDCODE_DATA + if(auxTrie.index!=NULL) { +#else + UErrorCode errorCode=U_ZERO_ERROR; + if(_haveData(errorCode) && auxTrie.index!=NULL) { +#endif + uint16_t aux; + + UTRIE_GET16(&auxTrie, c, aux); + return (UBool)((aux&_NORM_AUX_COMP_EX_MASK)!=0); + } else { + return FALSE; + } +} + +U_CFUNC UBool U_EXPORT2 +unorm_isCanonSafeStart(UChar32 c) { +#if UNORM_HARDCODE_DATA + if(auxTrie.index!=NULL) { +#else + UErrorCode errorCode=U_ZERO_ERROR; + if(_haveData(errorCode) && auxTrie.index!=NULL) { +#endif + uint16_t aux; + + UTRIE_GET16(&auxTrie, c, aux); + return (UBool)((aux&_NORM_AUX_UNSAFE_MASK)==0); + } else { + return FALSE; + } +} + +U_CAPI void U_EXPORT2 +unorm_getUnicodeVersion(UVersionInfo *versionInfo, UErrorCode *pErrorCode){ + if(unorm_haveData(pErrorCode)){ + uprv_memcpy(*versionInfo, dataVersion, 4); + } +} + + +U_CAPI UBool U_EXPORT2 +unorm_getCanonStartSet(UChar32 c, USerializedSet *fillSet) { +#if !UNORM_HARDCODE_DATA + UErrorCode errorCode=U_ZERO_ERROR; +#endif + if( fillSet!=NULL && (uint32_t)c<=0x10ffff && +#if !UNORM_HARDCODE_DATA + _haveData(errorCode) && +#endif + canonStartSets!=NULL + ) { + const uint16_t *table; + int32_t i, start, limit; + + /* + * binary search for c + * + * There are two search tables, + * one for BMP code points and one for supplementary ones. + * See unormimp.h for details. + */ + if(c<=0xffff) { + table=canonStartSets+canonStartSets[_NORM_SET_INDEX_CANON_SETS_LENGTH]; + start=0; + limit=canonStartSets[_NORM_SET_INDEX_CANON_BMP_TABLE_LENGTH]; + + /* each entry is a pair { c, result } */ + while(start<limit-2) { + i=(uint16_t)(((start+limit)/4)*2); /* (start+limit)/2 and address pairs */ + if(c<table[i]) { + limit=i; + } else { + start=i; + } + } + + /* found? */ + if(c==table[start]) { + i=table[start+1]; + if((i&_NORM_CANON_SET_BMP_MASK)==_NORM_CANON_SET_BMP_IS_INDEX) { + /* result 01xxxxxx xxxxxx contains index x to a USerializedSet */ + i&=(_NORM_MAX_CANON_SETS-1); + return uset_getSerializedSet(fillSet, + canonStartSets+i, + canonStartSets[_NORM_SET_INDEX_CANON_SETS_LENGTH]-i); + } else { + /* other result values are BMP code points for single-code point sets */ + uset_setSerializedToOne(fillSet, (UChar32)i); + return TRUE; + } + } + } else { + uint16_t high, low, h; + + table=canonStartSets+canonStartSets[_NORM_SET_INDEX_CANON_SETS_LENGTH]+ + canonStartSets[_NORM_SET_INDEX_CANON_BMP_TABLE_LENGTH]; + start=0; + limit=canonStartSets[_NORM_SET_INDEX_CANON_SUPP_TABLE_LENGTH]; + + high=(uint16_t)(c>>16); + low=(uint16_t)c; + + /* each entry is a triplet { high(c), low(c), result } */ + while(start<limit-3) { + i=(uint16_t)(((start+limit)/6)*3); /* (start+limit)/2 and address triplets */ + h=table[i]&0x1f; /* high word */ + if(high<h || (high==h && low<table[i+1])) { + limit=i; + } else { + start=i; + } + } + + /* found? */ + h=table[start]; + if(high==(h&0x1f) && low==table[start+1]) { + i=table[start+2]; + if((h&0x8000)==0) { + /* the result is an index to a USerializedSet */ + return uset_getSerializedSet(fillSet, + canonStartSets+i, + canonStartSets[_NORM_SET_INDEX_CANON_SETS_LENGTH]-i); + } else { + /* + * single-code point set {x} in + * triplet { 100xxxxx 000hhhhh llllllll llllllll xxxxxxxx xxxxxxxx } + */ + i|=((int32_t)h&0x1f00)<<8; /* add high bits from high(c) */ + uset_setSerializedToOne(fillSet, (UChar32)i); + return TRUE; + } + } + } + } + + return FALSE; /* not found */ +} + +U_CAPI int32_t U_EXPORT2 +u_getFC_NFKC_Closure(UChar32 c, UChar *dest, int32_t destCapacity, UErrorCode *pErrorCode) { + uint16_t aux; + + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return 0; + } + if(destCapacity<0 || (dest==NULL && destCapacity>0)) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + if(!_haveData(*pErrorCode) || auxTrie.index==NULL) { + return 0; + } + + UTRIE_GET16(&auxTrie, c, aux); + aux&=_NORM_AUX_FNC_MASK; + if(aux!=0) { + const UChar *s; + int32_t length; + + s=(const UChar *)(extraData+aux); + if(*s<0xff00) { + /* s points to the single-unit string */ + length=1; + } else { + length=*s&0xff; + ++s; + } + if(0<length && length<=destCapacity) { + uprv_memcpy(dest, s, length*U_SIZEOF_UCHAR); + } + return u_terminateUChars(dest, destCapacity, length, pErrorCode); + } else { + return u_terminateUChars(dest, destCapacity, 0, pErrorCode); + } +} + +/* Is c an NF<mode>-skippable code point? See unormimp.h. */ +U_CAPI UBool U_EXPORT2 +unorm_isNFSkippable(UChar32 c, UNormalizationMode mode) { + uint32_t norm32, mask; + uint16_t aux, fcd; + +#if !UNORM_HARDCODE_DATA + UErrorCode errorCode=U_ZERO_ERROR; + if(!_haveData(errorCode)) { + return FALSE; + } +#endif + + /* handle trivial cases; set the comparison mask for the normal ones */ + switch(mode) { + case UNORM_NONE: + return TRUE; + case UNORM_NFD: + mask=_NORM_CC_MASK|_NORM_QC_NFD; + break; + case UNORM_NFKD: + mask=_NORM_CC_MASK|_NORM_QC_NFKD; + break; + case UNORM_NFC: + /* case UNORM_FCC: */ + mask=_NORM_CC_MASK|_NORM_COMBINES_ANY|(_NORM_QC_NFC&_NORM_QC_ANY_NO); + break; + case UNORM_NFKC: + mask=_NORM_CC_MASK|_NORM_COMBINES_ANY|(_NORM_QC_NFKC&_NORM_QC_ANY_NO); + break; + case UNORM_FCD: + /* FCD: skippable if lead cc==0 and trail cc<=1 */ + if(fcdTrie.index!=NULL) { + UTRIE_GET16(&fcdTrie, c, fcd); + return fcd<=1; + } else { + return FALSE; + } + default: + return FALSE; + } + + /* check conditions (a)..(e), see unormimp.h */ + UTRIE_GET32(&normTrie, c, norm32); + if((norm32&mask)!=0) { + return FALSE; /* fails (a)..(e), not skippable */ + } + + if(mode<UNORM_NFC) { + return TRUE; /* NF*D, passed (a)..(c), is skippable */ + } + + /* NF*C/FCC, passed (a)..(e) */ + if((norm32&_NORM_QC_NFD)==0) { + return TRUE; /* no canonical decomposition, is skippable */ + } + + /* check Hangul syllables algorithmically */ + if(isNorm32HangulOrJamo(norm32)) { + /* Jamo passed (a)..(e) above, must be Hangul */ + return !isHangulWithoutJamoT((UChar)c); /* LVT are skippable, LV are not */ + } + + /* if(mode<=UNORM_NFKC) { -- enable when implementing FCC */ + /* NF*C, test (f) flag */ + if(!formatVersion_2_2 || auxTrie.index==NULL) { + return FALSE; /* no (f) data, say not skippable to be safe */ + } + + UTRIE_GET16(&auxTrie, c, aux); + return (aux&_NORM_AUX_NFC_SKIP_F_MASK)==0; /* TRUE=skippable if the (f) flag is not set */ + + /* } else { FCC, test fcd<=1 instead of the above } */ +} + +U_CAPI void U_EXPORT2 +unorm_addPropertyStarts(const USetAdder *sa, UErrorCode *pErrorCode) { + UChar c; + + if(!_haveData(*pErrorCode)) { + return; + } + + /* add the start code point of each same-value range of each trie */ + utrie_enum(&normTrie, NULL, _enumPropertyStartsRange, sa); + if(fcdTrie.index!=NULL) { + utrie_enum(&fcdTrie, NULL, _enumPropertyStartsRange, sa); + } + if(auxTrie.index!=NULL) { + utrie_enum(&auxTrie, NULL, _enumPropertyStartsRange, sa); + } + + /* add Hangul LV syllables and LV+1 because of skippables */ + for(c=HANGUL_BASE; c<HANGUL_BASE+HANGUL_COUNT; c+=JAMO_T_COUNT) { + sa->add(sa->set, c); + sa->add(sa->set, c+1); + } + sa->add(sa->set, HANGUL_BASE+HANGUL_COUNT); /* add Hangul+1 to continue with other properties */ +} + +U_CFUNC UNormalizationCheckResult U_EXPORT2 +unorm_getQuickCheck(UChar32 c, UNormalizationMode mode) { + static const uint32_t qcMask[UNORM_MODE_COUNT]={ + 0, 0, _NORM_QC_NFD, _NORM_QC_NFKD, _NORM_QC_NFC, _NORM_QC_NFKC + }; + + uint32_t norm32; + +#if !UNORM_HARDCODE_DATA + UErrorCode errorCode=U_ZERO_ERROR; + if(!_haveData(errorCode)) { + return UNORM_YES; + } +#endif + + UTRIE_GET32(&normTrie, c, norm32); + norm32&=qcMask[mode]; + + if(norm32==0) { + return UNORM_YES; + } else if(norm32&_NORM_QC_ANY_NO) { + return UNORM_NO; + } else /* _NORM_QC_ANY_MAYBE */ { + return UNORM_MAYBE; + } +} + +U_CFUNC uint16_t U_EXPORT2 +unorm_getFCD16FromCodePoint(UChar32 c) { + uint16_t fcd; +#if !UNORM_HARDCODE_DATA + UErrorCode errorCode; + errorCode=U_ZERO_ERROR; +#endif + + if( +#if !UNORM_HARDCODE_DATA + !_haveData(errorCode) || +#endif + fcdTrie.index==NULL + ) { + return 0; + } + + UTRIE_GET16(&fcdTrie, c, fcd); + return fcd; +} + +/* reorder UTF-16 in-place -------------------------------------------------- */ + +/* + * simpler, single-character version of _mergeOrdered() - + * bubble-insert one single code point into the preceding string + * which is already canonically ordered + * (c, c2) may or may not yet have been inserted at [current..p[ + * + * it must be p=current+lengthof(c, c2) i.e. p=current+(c2==0 ? 1 : 2) + * + * before: [start..current[ is already ordered, and + * [current..p[ may or may not hold (c, c2) but + * must be exactly the same length as (c, c2) + * after: [start..p[ is ordered + * + * returns the trailing combining class + */ +static uint8_t +_insertOrdered(const UChar *start, UChar *current, UChar *p, + UChar c, UChar c2, uint8_t cc) { + const UChar *pBack, *pPreBack; + UChar *r; + uint8_t prevCC, trailCC=cc; + + if(start<current && cc!=0) { + /* search for the insertion point where cc>=prevCC */ + pPreBack=pBack=current; + prevCC=_getPrevCC(start, pPreBack); + if(cc<prevCC) { + /* this will be the last code point, so keep its cc */ + trailCC=prevCC; + pBack=pPreBack; + while(start<pPreBack) { + prevCC=_getPrevCC(start, pPreBack); + if(cc>=prevCC) { + break; + } + pBack=pPreBack; + } + + /* + * this is where we are right now with all these pointers: + * [start..pPreBack[ 0..? code points that we can ignore + * [pPreBack..pBack[ 0..1 code points with prevCC<=cc + * [pBack..current[ 0..n code points with >cc, move up to insert (c, c2) + * [current..p[ 1 code point (c, c2) with cc + */ + + /* move the code units in between up */ + r=p; + do { + *--r=*--current; + } while(pBack!=current); + } + } + + /* insert (c, c2) */ + *current=c; + if(c2!=0) { + *(current+1)=c2; + } + + /* we know the cc of the last code point */ + return trailCC; +} + +/* + * merge two UTF-16 string parts together + * to canonically order (order by combining classes) their concatenation + * + * the two strings may already be adjacent, so that the merging is done in-place + * if the two strings are not adjacent, then the buffer holding the first one + * must be large enough + * the second string may or may not be ordered in itself + * + * before: [start..current[ is already ordered, and + * [next..limit[ may be ordered in itself, but + * is not in relation to [start..current[ + * after: [start..current+(limit-next)[ is ordered + * + * the algorithm is a simple bubble-sort that takes the characters from *next++ + * and inserts them in correct combining class order into the preceding part + * of the string + * + * since this function is called much less often than the single-code point + * _insertOrdered(), it just uses that for easier maintenance + * (see file version from before 2001aug31 for a more optimized version) + * + * returns the trailing combining class + */ +static uint8_t +_mergeOrdered(UChar *start, UChar *current, + const UChar *next, const UChar *limit, UBool isOrdered=TRUE) { + UChar *r; + UChar c, c2; + uint8_t cc, trailCC=0; + UBool adjacent; + + adjacent= current==next; + + if(start!=current || !isOrdered) { + while(next<limit) { + cc=_getNextCC(next, limit, c, c2); + if(cc==0) { + /* does not bubble back */ + trailCC=0; + if(adjacent) { + current=(UChar *)next; + } else { + *current++=c; + if(c2!=0) { + *current++=c2; + } + } + if(isOrdered) { + break; + } else { + start=current; + } + } else { + r=current+(c2==0 ? 1 : 2); + trailCC=_insertOrdered(start, current, r, c, c2, cc); + current=r; + } + } + } + + if(next==limit) { + /* we know the cc of the last code point */ + return trailCC; + } else { + if(!adjacent) { + /* copy the second string part */ + do { + *current++=*next++; + } while(next!=limit); + limit=current; + } + return _getPrevCC(start, limit); + } +} + +/* find the last true starter in [start..src[ and return the pointer to it */ +static const UChar * +_findPreviousStarter(const UChar *start, const UChar *src, + uint32_t ccOrQCMask, uint32_t decompQCMask, UChar minNoMaybe) { + uint32_t norm32; + UChar c, c2; + + while(start<src) { + norm32=_getPrevNorm32(start, src, minNoMaybe, ccOrQCMask|decompQCMask, c, c2); + if(_isTrueStarter(norm32, ccOrQCMask, decompQCMask)) { + break; + } + } + return src; +} + +/* find the first true starter in [src..limit[ and return the pointer to it */ +static const UChar * +_findNextStarter(const UChar *src, const UChar *limit, + uint32_t qcMask, uint32_t decompQCMask, UChar minNoMaybe) { + const UChar *p; + uint32_t norm32, ccOrQCMask; + int32_t length; + UChar c, c2; + uint8_t cc, trailCC; + + ccOrQCMask=_NORM_CC_MASK|qcMask; + + for(;;) { + if(src==limit) { + break; /* end of string */ + } + c=*src; + if(c<minNoMaybe) { + break; /* catches NUL terminater, too */ + } + + norm32=_getNorm32(c); + if((norm32&ccOrQCMask)==0) { + break; /* true starter */ + } + + if(isNorm32LeadSurrogate(norm32)) { + /* c is a lead surrogate, get the real norm32 */ + if((src+1)==limit || !UTF_IS_SECOND_SURROGATE(c2=*(src+1))) { + break; /* unmatched first surrogate: counts as a true starter */ + } + norm32=_getNorm32FromSurrogatePair(norm32, c2); + + if((norm32&ccOrQCMask)==0) { + break; /* true starter */ + } + } else { + c2=0; + } + + /* (c, c2) is not a true starter but its decomposition may be */ + if(norm32&decompQCMask) { + /* (c, c2) decomposes, get everything from the variable-length extra data */ + p=_decompose(norm32, decompQCMask, length, cc, trailCC); + + /* get the first character's norm32 to check if it is a true starter */ + if(cc==0 && (_getNorm32(p, qcMask)&qcMask)==0) { + break; /* true starter */ + } + } + + src+= c2==0 ? 1 : 2; /* not a true starter, continue */ + } + + return src; +} + +/* make NFD & NFKD ---------------------------------------------------------- */ + +U_CAPI int32_t U_EXPORT2 +unorm_getDecomposition(UChar32 c, UBool compat, + UChar *dest, int32_t destCapacity) { +#if !UNORM_HARDCODE_DATA + UErrorCode errorCode=U_ZERO_ERROR; +#endif + if( (uint32_t)c<=0x10ffff && +#if !UNORM_HARDCODE_DATA + _haveData(errorCode) && +#endif + ((dest!=NULL && destCapacity>0) || destCapacity==0) + ) { + uint32_t norm32, qcMask; + UChar32 minNoMaybe; + int32_t length; + + /* initialize */ + if(!compat) { + minNoMaybe=(UChar32)indexes[_NORM_INDEX_MIN_NFD_NO_MAYBE]; + qcMask=_NORM_QC_NFD; + } else { + minNoMaybe=(UChar32)indexes[_NORM_INDEX_MIN_NFKD_NO_MAYBE]; + qcMask=_NORM_QC_NFKD; + } + + if(c<minNoMaybe) { + /* trivial case */ + if(destCapacity>0) { + dest[0]=(UChar)c; + } + return -1; + } + + /* data lookup */ + UTRIE_GET32(&normTrie, c, norm32); + if((norm32&qcMask)==0) { + /* simple case: no decomposition */ + if(c<=0xffff) { + if(destCapacity>0) { + dest[0]=(UChar)c; + } + return -1; + } else { + if(destCapacity>=2) { + dest[0]=UTF16_LEAD(c); + dest[1]=UTF16_TRAIL(c); + } + return -2; + } + } else if(isNorm32HangulOrJamo(norm32)) { + /* Hangul syllable: decompose algorithmically */ + UChar c2; + + c-=HANGUL_BASE; + + c2=(UChar)(c%JAMO_T_COUNT); + c/=JAMO_T_COUNT; + if(c2>0) { + if(destCapacity>=3) { + dest[2]=(UChar)(JAMO_T_BASE+c2); + } + length=3; + } else { + length=2; + } + + if(destCapacity>=2) { + dest[1]=(UChar)(JAMO_V_BASE+c%JAMO_V_COUNT); + dest[0]=(UChar)(JAMO_L_BASE+c/JAMO_V_COUNT); + } + return length; + } else { + /* c decomposes, get everything from the variable-length extra data */ + const UChar *p, *limit; + uint8_t cc, trailCC; + + p=_decompose(norm32, qcMask, length, cc, trailCC); + if(length<=destCapacity) { + limit=p+length; + do { + *dest++=*p++; + } while(p<limit); + } + return length; + } + } else { + return 0; + } +} + +static int32_t +_decompose(UChar *dest, int32_t destCapacity, + const UChar *src, int32_t srcLength, + UBool compat, const UnicodeSet *nx, + uint8_t &outTrailCC) { + UChar buffer[3]; + const UChar *limit, *prevSrc, *p; + uint32_t norm32, ccOrQCMask, qcMask; + int32_t destIndex, reorderStartIndex, length; + UChar c, c2, minNoMaybe; + uint8_t cc, prevCC, trailCC; + + if(!compat) { + minNoMaybe=(UChar)indexes[_NORM_INDEX_MIN_NFD_NO_MAYBE]; + qcMask=_NORM_QC_NFD; + } else { + minNoMaybe=(UChar)indexes[_NORM_INDEX_MIN_NFKD_NO_MAYBE]; + qcMask=_NORM_QC_NFKD; + } + + /* initialize */ + ccOrQCMask=_NORM_CC_MASK|qcMask; + destIndex=reorderStartIndex=0; + prevCC=0; + + /* avoid compiler warnings */ + norm32=0; + c=0; + cc=0; + trailCC=0; + + if(srcLength>=0) { + /* string with length */ + limit=src+srcLength; + } else /* srcLength==-1 */ { + /* zero-terminated string */ + limit=NULL; + } + + U_ALIGN_CODE(16); + + for(;;) { + /* count code units below the minimum or with irrelevant data for the quick check */ + prevSrc=src; + if(limit==NULL) { + while((c=*src)<minNoMaybe ? c!=0 : ((norm32=_getNorm32(c))&ccOrQCMask)==0) { + prevCC=0; + ++src; + } + } else { + while(src!=limit && ((c=*src)<minNoMaybe || ((norm32=_getNorm32(c))&ccOrQCMask)==0)) { + prevCC=0; + ++src; + } + } + + /* copy these code units all at once */ + if(src!=prevSrc) { + length=(int32_t)(src-prevSrc); + if((destIndex+length)<=destCapacity) { + uprv_memcpy(dest+destIndex, prevSrc, length*U_SIZEOF_UCHAR); + } + destIndex+=length; + reorderStartIndex=destIndex; + } + + /* end of source reached? */ + if(limit==NULL ? c==0 : src==limit) { + break; + } + + /* c already contains *src and norm32 is set for it, increment src */ + ++src; + + /* check one above-minimum, relevant code unit */ + /* + * generally, set p and length to the decomposition string + * in simple cases, p==NULL and (c, c2) will hold the length code units to append + * in all cases, set cc to the lead and trailCC to the trail combining class + * + * the following merge-sort of the current character into the preceding, + * canonically ordered result text will use the optimized _insertOrdered() + * if there is only one single code point to process; + * this is indicated with p==NULL, and (c, c2) is the character to insert + * ((c, 0) for a BMP character and (lead surrogate, trail surrogate) + * for a supplementary character) + * otherwise, p[length] is merged in with _mergeOrdered() + */ + if(isNorm32HangulOrJamo(norm32)) { + if(nx_contains(nx, c)) { + c2=0; + p=NULL; + length=1; + } else { + /* Hangul syllable: decompose algorithmically */ + p=buffer; + cc=trailCC=0; + + c-=HANGUL_BASE; + + c2=(UChar)(c%JAMO_T_COUNT); + c/=JAMO_T_COUNT; + if(c2>0) { + buffer[2]=(UChar)(JAMO_T_BASE+c2); + length=3; + } else { + length=2; + } + + buffer[1]=(UChar)(JAMO_V_BASE+c%JAMO_V_COUNT); + buffer[0]=(UChar)(JAMO_L_BASE+c/JAMO_V_COUNT); + } + } else { + if(isNorm32Regular(norm32)) { + c2=0; + length=1; + } else { + /* c is a lead surrogate, get the real norm32 */ + if(src!=limit && UTF_IS_SECOND_SURROGATE(c2=*src)) { + ++src; + length=2; + norm32=_getNorm32FromSurrogatePair(norm32, c2); + } else { + c2=0; + length=1; + norm32=0; + } + } + + /* get the decomposition and the lead and trail cc's */ + if(nx_contains(nx, c, c2)) { + /* excluded: norm32==0 */ + cc=trailCC=0; + p=NULL; + } else if((norm32&qcMask)==0) { + /* c does not decompose */ + cc=trailCC=(uint8_t)(norm32>>_NORM_CC_SHIFT); + p=NULL; + } else { + /* c decomposes, get everything from the variable-length extra data */ + p=_decompose(norm32, qcMask, length, cc, trailCC); + if(length==1) { + /* fastpath a single code unit from decomposition */ + c=*p; + c2=0; + p=NULL; + } + } + } + + /* append the decomposition to the destination buffer, assume length>0 */ + if((destIndex+length)<=destCapacity) { + UChar *reorderSplit=dest+destIndex; + if(p==NULL) { + /* fastpath: single code point */ + if(cc!=0 && cc<prevCC) { + /* (c, c2) is out of order with respect to the preceding text */ + destIndex+=length; + trailCC=_insertOrdered(dest+reorderStartIndex, reorderSplit, dest+destIndex, c, c2, cc); + } else { + /* just append (c, c2) */ + dest[destIndex++]=c; + if(c2!=0) { + dest[destIndex++]=c2; + } + } + } else { + /* general: multiple code points (ordered by themselves) from decomposition */ + if(cc!=0 && cc<prevCC) { + /* the decomposition is out of order with respect to the preceding text */ + destIndex+=length; + trailCC=_mergeOrdered(dest+reorderStartIndex, reorderSplit, p, p+length); + } else { + /* just append the decomposition */ + do { + dest[destIndex++]=*p++; + } while(--length>0); + } + } + } else { + /* buffer overflow */ + /* keep incrementing the destIndex for preflighting */ + destIndex+=length; + } + + prevCC=trailCC; + if(prevCC==0) { + reorderStartIndex=destIndex; + } + } + + outTrailCC=prevCC; + return destIndex; +} + +U_CAPI int32_t U_EXPORT2 +unorm_decompose(UChar *dest, int32_t destCapacity, + const UChar *src, int32_t srcLength, + UBool compat, int32_t options, + UErrorCode *pErrorCode) { + const UnicodeSet *nx; + int32_t destIndex; + uint8_t trailCC; + + if(!_haveData(*pErrorCode)) { + return 0; + } + + nx=getNX(options, *pErrorCode); + if(U_FAILURE(*pErrorCode)) { + return 0; + } + + destIndex=_decompose(dest, destCapacity, + src, srcLength, + compat, nx, + trailCC); + + return u_terminateUChars(dest, destCapacity, destIndex, pErrorCode); +} + +/* make NFC & NFKC ---------------------------------------------------------- */ + +/* get the composition properties of the next character */ +static inline uint32_t +_getNextCombining(UChar *&p, const UChar *limit, + UChar &c, UChar &c2, + uint16_t &combiningIndex, uint8_t &cc, + const UnicodeSet *nx) { + uint32_t norm32, combineFlags; + + /* get properties */ + c=*p++; + norm32=_getNorm32(c); + + /* preset output values for most characters */ + c2=0; + combiningIndex=0; + cc=0; + + if((norm32&(_NORM_CC_MASK|_NORM_COMBINES_ANY))==0) { + return 0; + } else { + if(isNorm32Regular(norm32)) { + /* set cc etc. below */ + } else if(isNorm32HangulOrJamo(norm32)) { + /* a compatibility decomposition contained Jamos */ + combiningIndex=(uint16_t)(0xfff0|(norm32>>_NORM_EXTRA_SHIFT)); + return norm32&_NORM_COMBINES_ANY; + } else { + /* c is a lead surrogate, get the real norm32 */ + if(p!=limit && UTF_IS_SECOND_SURROGATE(c2=*p)) { + ++p; + norm32=_getNorm32FromSurrogatePair(norm32, c2); + } else { + c2=0; + return 0; + } + } + + if(nx_contains(nx, c, c2)) { + return 0; /* excluded: norm32==0 */ + } + + cc=(uint8_t)(norm32>>_NORM_CC_SHIFT); + + combineFlags=norm32&_NORM_COMBINES_ANY; + if(combineFlags!=0) { + combiningIndex=*(_getExtraData(norm32)-1); + } + return combineFlags; + } +} + +/* + * given a composition-result starter (c, c2) - which means its cc==0, + * it combines forward, it has extra data, its norm32!=0, + * it is not a Hangul or Jamo, + * get just its combineFwdIndex + * + * norm32(c) is special if and only if c2!=0 + */ +static inline uint16_t +_getCombiningIndexFromStarter(UChar c, UChar c2) { + uint32_t norm32; + + norm32=_getNorm32(c); + if(c2!=0) { + norm32=_getNorm32FromSurrogatePair(norm32, c2); + } + return *(_getExtraData(norm32)-1); +} + +/* + * Find the recomposition result for + * a forward-combining character + * (specified with a pointer to its part of the combiningTable[]) + * and a backward-combining character + * (specified with its combineBackIndex). + * + * If these two characters combine, then set (value, value2) + * with the code unit(s) of the composition character. + * + * Return value: + * 0 do not combine + * 1 combine + * >1 combine, and the composition is a forward-combining starter + * + * See unormimp.h for a description of the composition table format. + */ +static inline uint16_t +_combine(const uint16_t *table, uint16_t combineBackIndex, + uint16_t &value, uint16_t &value2) { + uint16_t key; + + /* search in the starter's composition table */ + for(;;) { + key=*table++; + if(key>=combineBackIndex) { + break; + } + table+= *table&0x8000 ? 2 : 1; + } + + /* mask off bit 15, the last-entry-in-the-list flag */ + if((key&0x7fff)==combineBackIndex) { + /* found! combine! */ + value=*table; + + /* is the composition a starter that combines forward? */ + key=(uint16_t)((value&0x2000)+1); + + /* get the composition result code point from the variable-length result value */ + if(value&0x8000) { + if(value&0x4000) { + /* surrogate pair composition result */ + value=(uint16_t)((value&0x3ff)|0xd800); + value2=*(table+1); + } else { + /* BMP composition result U+2000..U+ffff */ + value=*(table+1); + value2=0; + } + } else { + /* BMP composition result U+0000..U+1fff */ + value&=0x1fff; + value2=0; + } + + return key; + } else { + /* not found */ + return 0; + } +} + +static inline UBool +_composeHangul(UChar prev, UChar c, uint32_t norm32, const UChar *&src, const UChar *limit, + UBool compat, UChar *dest, const UnicodeSet *nx) { + if(isJamoVTNorm32JamoV(norm32)) { + /* c is a Jamo V, compose with previous Jamo L and following Jamo T */ + prev=(UChar)(prev-JAMO_L_BASE); + if(prev<JAMO_L_COUNT) { + c=(UChar)(HANGUL_BASE+(prev*JAMO_V_COUNT+(c-JAMO_V_BASE))*JAMO_T_COUNT); + + /* check if the next character is a Jamo T (normal or compatibility) */ + if(src!=limit) { + UChar next, t; + + next=*src; + if((t=(UChar)(next-JAMO_T_BASE))<JAMO_T_COUNT) { + /* normal Jamo T */ + ++src; + c+=t; + } else if(compat) { + /* if NFKC, then check for compatibility Jamo T (BMP only) */ + norm32=_getNorm32(next); + if(isNorm32Regular(norm32) && (norm32&_NORM_QC_NFKD)) { + const UChar *p; + int32_t length; + uint8_t cc, trailCC; + + p=_decompose(norm32, _NORM_QC_NFKD, length, cc, trailCC); + if(length==1 && (t=(UChar)(*p-JAMO_T_BASE))<JAMO_T_COUNT) { + /* compatibility Jamo T */ + ++src; + c+=t; + } + } + } + } + if(nx_contains(nx, c)) { + if(!isHangulWithoutJamoT(c)) { + --src; /* undo ++src from reading the Jamo T */ + } + return FALSE; + } + if(dest!=0) { + *dest=c; + } + return TRUE; + } + } else if(isHangulWithoutJamoT(prev)) { + /* c is a Jamo T, compose with previous Hangul LV that does not contain a Jamo T */ + c=(UChar)(prev+(c-JAMO_T_BASE)); + if(nx_contains(nx, c)) { + return FALSE; + } + if(dest!=0) { + *dest=c; + } + return TRUE; + } + return FALSE; +} + +/* + * recompose the characters in [p..limit[ + * (which is in NFD - decomposed and canonically ordered), + * adjust limit, and return the trailing cc + * + * since for NFKC we may get Jamos in decompositions, we need to + * recompose those too + * + * note that recomposition never lengthens the text: + * any character consists of either one or two code units; + * a composition may contain at most one more code unit than the original starter, + * while the combining mark that is removed has at least one code unit + */ +static uint8_t +_recompose(UChar *p, UChar *&limit, int32_t options, const UnicodeSet *nx) { + UChar *starter, *pRemove, *q, *r; + uint32_t combineFlags; + UChar c, c2; + uint16_t combineFwdIndex, combineBackIndex; + uint16_t result, value, value2; + uint8_t cc, prevCC; + UBool starterIsSupplementary; + + starter=NULL; /* no starter */ + combineFwdIndex=0; /* will not be used until starter!=NULL - avoid compiler warnings */ + combineBackIndex=0; /* will always be set if combineFlags!=0 - avoid compiler warnings */ + value=value2=0; /* always set by _combine() before used - avoid compiler warnings */ + starterIsSupplementary=FALSE; /* will not be used until starter!=NULL - avoid compiler warnings */ + prevCC=0; + + for(;;) { + combineFlags=_getNextCombining(p, limit, c, c2, combineBackIndex, cc, nx); + if((combineFlags&_NORM_COMBINES_BACK) && starter!=NULL) { + if(combineBackIndex&0x8000) { + /* c is a Jamo V/T, see if we can compose it with the previous character */ + /* for the PRI #29 fix, check that there is no intervening combining mark */ + if((options&UNORM_BEFORE_PRI_29) || prevCC==0) { + pRemove=NULL; /* NULL while no Hangul composition */ + combineFlags=0; + c2=*starter; + if(combineBackIndex==0xfff2) { + /* Jamo V, compose with previous Jamo L and following Jamo T */ + c2=(UChar)(c2-JAMO_L_BASE); + if(c2<JAMO_L_COUNT) { + pRemove=p-1; + c=(UChar)(HANGUL_BASE+(c2*JAMO_V_COUNT+(c-JAMO_V_BASE))*JAMO_T_COUNT); + if(p!=limit && (c2=(UChar)(*p-JAMO_T_BASE))<JAMO_T_COUNT) { + ++p; + c+=c2; + } else { + /* the result is an LV syllable, which is a starter (unlike LVT) */ + combineFlags=_NORM_COMBINES_FWD; + } + if(!nx_contains(nx, c)) { + *starter=c; + } else { + /* excluded */ + if(!isHangulWithoutJamoT(c)) { + --p; /* undo the ++p from reading the Jamo T */ + } + /* c is modified but not used any more -- c=*(p-1); -- re-read the Jamo V/T */ + pRemove=NULL; + } + } + + /* + * Normally, the following can not occur: + * Since the input is in NFD, there are no Hangul LV syllables that + * a Jamo T could combine with. + * All Jamo Ts are combined above when handling Jamo Vs. + * + * However, before the PRI #29 fix, this can occur due to + * an intervening combining mark between the Hangul LV and the Jamo T. + */ + } else { + /* Jamo T, compose with previous Hangul that does not have a Jamo T */ + if(isHangulWithoutJamoT(c2)) { + c2+=(UChar)(c-JAMO_T_BASE); + if(!nx_contains(nx, c2)) { + pRemove=p-1; + *starter=c2; + } + } + } + + if(pRemove!=NULL) { + /* remove the Jamo(s) */ + q=pRemove; + r=p; + while(r<limit) { + *q++=*r++; + } + p=pRemove; + limit=q; + } + + c2=0; /* c2 held *starter temporarily */ + + if(combineFlags!=0) { + /* + * not starter=NULL because the composition is a Hangul LV syllable + * and might combine once more (but only before the PRI #29 fix) + */ + + /* done? */ + if(p==limit) { + return prevCC; + } + + /* the composition is a Hangul LV syllable which is a starter that combines forward */ + combineFwdIndex=0xfff0; + + /* we combined; continue with looking for compositions */ + continue; + } + } + + /* + * now: cc==0 and the combining index does not include "forward" -> + * the rest of the loop body will reset starter to NULL; + * technically, a composed Hangul syllable is a starter, but it + * does not combine forward now that we have consumed all eligible Jamos; + * for Jamo V/T, combineFlags does not contain _NORM_COMBINES_FWD + */ + + } else if( + /* the starter is not a Hangul LV or Jamo V/T and */ + !(combineFwdIndex&0x8000) && + /* the combining mark is not blocked and */ + ((options&UNORM_BEFORE_PRI_29) ? + (prevCC!=cc || prevCC==0) : + (prevCC<cc || prevCC==0)) && + /* the starter and the combining mark (c, c2) do combine and */ + 0!=(result=_combine(combiningTable+combineFwdIndex, combineBackIndex, value, value2)) && + /* the composition result is not excluded */ + !nx_contains(nx, value, value2) + ) { + /* replace the starter with the composition, remove the combining mark */ + pRemove= c2==0 ? p-1 : p-2; /* pointer to the combining mark */ + + /* replace the starter with the composition */ + *starter=(UChar)value; + if(starterIsSupplementary) { + if(value2!=0) { + /* both are supplementary */ + *(starter+1)=(UChar)value2; + } else { + /* the composition is shorter than the starter, move the intermediate characters forward one */ + starterIsSupplementary=FALSE; + q=starter+1; + r=q+1; + while(r<pRemove) { + *q++=*r++; + } + --pRemove; + } + } else if(value2!=0) { + /* the composition is longer than the starter, move the intermediate characters back one */ + starterIsSupplementary=TRUE; + ++starter; /* temporarily increment for the loop boundary */ + q=pRemove; + r=++pRemove; + while(starter<q) { + *--r=*--q; + } + *starter=(UChar)value2; + --starter; /* undo the temporary increment */ + /* } else { both are on the BMP, nothing more to do */ + } + + /* remove the combining mark by moving the following text over it */ + if(pRemove<p) { + q=pRemove; + r=p; + while(r<limit) { + *q++=*r++; + } + p=pRemove; + limit=q; + } + + /* keep prevCC because we removed the combining mark */ + + /* done? */ + if(p==limit) { + return prevCC; + } + + /* is the composition a starter that combines forward? */ + if(result>1) { + combineFwdIndex=_getCombiningIndexFromStarter((UChar)value, (UChar)value2); + } else { + starter=NULL; + } + + /* we combined; continue with looking for compositions */ + continue; + } + } + + /* no combination this time */ + prevCC=cc; + if(p==limit) { + return prevCC; + } + + /* if (c, c2) did not combine, then check if it is a starter */ + if(cc==0) { + /* found a new starter; combineFlags==0 if (c, c2) is excluded */ + if(combineFlags&_NORM_COMBINES_FWD) { + /* it may combine with something, prepare for it */ + if(c2==0) { + starterIsSupplementary=FALSE; + starter=p-1; + } else { + starterIsSupplementary=TRUE; + starter=p-2; + } + combineFwdIndex=combineBackIndex; + } else { + /* it will not combine with anything */ + starter=NULL; + } + } else if(options&_NORM_OPTIONS_COMPOSE_CONTIGUOUS) { + /* FCC: no discontiguous compositions; any intervening character blocks */ + starter=NULL; + } + } +} + +/* decompose and recompose [prevStarter..src[ */ +static const UChar * +_composePart(UChar *stackBuffer, UChar *&buffer, int32_t &bufferCapacity, int32_t &length, + const UChar *prevStarter, const UChar *src, + uint8_t &prevCC, + int32_t options, const UnicodeSet *nx, + UErrorCode *pErrorCode) { + UChar *recomposeLimit; + uint8_t trailCC; + UBool compat; + + compat=(UBool)((options&_NORM_OPTIONS_COMPAT)!=0); + + /* decompose [prevStarter..src[ */ + length=_decompose(buffer, bufferCapacity, + prevStarter, (int32_t)(src-prevStarter), + compat, nx, + trailCC); + if(length>bufferCapacity) { + if(!u_growBufferFromStatic(stackBuffer, &buffer, &bufferCapacity, 2*length, 0)) { + *pErrorCode=U_MEMORY_ALLOCATION_ERROR; + return NULL; + } + length=_decompose(buffer, bufferCapacity, + prevStarter, (int32_t)(src-prevStarter), + compat, nx, + trailCC); + } + + /* recompose the decomposition */ + recomposeLimit=buffer+length; + if(length>=2) { + prevCC=_recompose(buffer, recomposeLimit, options, nx); + } + + /* return with a pointer to the recomposition and its length */ + length=(int32_t)(recomposeLimit-buffer); + return buffer; +} + +static int32_t +_compose(UChar *dest, int32_t destCapacity, + const UChar *src, int32_t srcLength, + int32_t options, const UnicodeSet *nx, + UErrorCode *pErrorCode) { + UChar stackBuffer[_STACK_BUFFER_CAPACITY]; + UChar *buffer; + int32_t bufferCapacity; + + const UChar *limit, *prevSrc, *prevStarter; + uint32_t norm32, ccOrQCMask, qcMask; + int32_t destIndex, reorderStartIndex, length; + UChar c, c2, minNoMaybe; + uint8_t cc, prevCC; + + if(options&_NORM_OPTIONS_COMPAT) { + minNoMaybe=(UChar)indexes[_NORM_INDEX_MIN_NFKC_NO_MAYBE]; + qcMask=_NORM_QC_NFKC; + } else { + minNoMaybe=(UChar)indexes[_NORM_INDEX_MIN_NFC_NO_MAYBE]; + qcMask=_NORM_QC_NFC; + } + + /* initialize */ + buffer=stackBuffer; + bufferCapacity=_STACK_BUFFER_CAPACITY; + + /* + * prevStarter points to the last character before the current one + * that is a "true" starter with cc==0 and quick check "yes". + * + * prevStarter will be used instead of looking for a true starter + * while incrementally decomposing [prevStarter..prevSrc[ + * in _composePart(). Having a good prevStarter allows to just decompose + * the entire [prevStarter..prevSrc[. + * + * When _composePart() backs out from prevSrc back to prevStarter, + * then it also backs out destIndex by the same amount. + * Therefore, at all times, the (prevSrc-prevStarter) source units + * must correspond 1:1 to destination units counted with destIndex, + * except for reordering. + * This is true for the qc "yes" characters copied in the fast loop, + * and for pure reordering. + * prevStarter must be set forward to src when this is not true: + * In _composePart() and after composing a Hangul syllable. + * + * This mechanism relies on the assumption that the decomposition of a true starter + * also begins with a true starter. gennorm/store.c checks for this. + */ + prevStarter=src; + + ccOrQCMask=_NORM_CC_MASK|qcMask; + destIndex=reorderStartIndex=0; + prevCC=0; + + /* avoid compiler warnings */ + norm32=0; + c=0; + + if(srcLength>=0) { + /* string with length */ + limit=src+srcLength; + } else /* srcLength==-1 */ { + /* zero-terminated string */ + limit=NULL; + } + + U_ALIGN_CODE(16); + + for(;;) { + /* count code units below the minimum or with irrelevant data for the quick check */ + prevSrc=src; + if(limit==NULL) { + while((c=*src)<minNoMaybe ? c!=0 : ((norm32=_getNorm32(c))&ccOrQCMask)==0) { + prevCC=0; + ++src; + } + } else { + while(src!=limit && ((c=*src)<minNoMaybe || ((norm32=_getNorm32(c))&ccOrQCMask)==0)) { + prevCC=0; + ++src; + } + } + + /* copy these code units all at once */ + if(src!=prevSrc) { + length=(int32_t)(src-prevSrc); + if((destIndex+length)<=destCapacity) { + uprv_memcpy(dest+destIndex, prevSrc, length*U_SIZEOF_UCHAR); + } + destIndex+=length; + reorderStartIndex=destIndex; + + /* set prevStarter to the last character in the quick check loop */ + prevStarter=src-1; + if(UTF_IS_SECOND_SURROGATE(*prevStarter) && prevSrc<prevStarter && UTF_IS_FIRST_SURROGATE(*(prevStarter-1))) { + --prevStarter; + } + + prevSrc=src; + } + + /* end of source reached? */ + if(limit==NULL ? c==0 : src==limit) { + break; + } + + /* c already contains *src and norm32 is set for it, increment src */ + ++src; + + /* + * source buffer pointers: + * + * all done quick check current char not yet + * "yes" but (c, c2) processed + * may combine + * forward + * [-------------[-------------[-------------[-------------[ + * | | | | | + * start prevStarter prevSrc src limit + * + * + * destination buffer pointers and indexes: + * + * all done might take not filled yet + * characters for + * reordering + * [-------------[-------------[-------------[ + * | | | | + * dest reorderStartIndex destIndex destCapacity + */ + + /* check one above-minimum, relevant code unit */ + /* + * norm32 is for c=*(src-1), and the quick check flag is "no" or "maybe", and/or cc!=0 + * check for Jamo V/T, then for surrogates and regular characters + * c is not a Hangul syllable or Jamo L because + * they are not marked with no/maybe for NFC & NFKC (and their cc==0) + */ + if(isNorm32HangulOrJamo(norm32)) { + /* + * c is a Jamo V/T: + * try to compose with the previous character, Jamo V also with a following Jamo T, + * and set values here right now in case we just continue with the main loop + */ + prevCC=cc=0; + reorderStartIndex=destIndex; + + if( + destIndex>0 && + _composeHangul( + *(prevSrc-1), c, norm32, src, limit, (UBool)((options&_NORM_OPTIONS_COMPAT)!=0), + destIndex<=destCapacity ? dest+(destIndex-1) : 0, + nx) + ) { + prevStarter=src; + continue; + } + + /* the Jamo V/T did not compose into a Hangul syllable, just append to dest */ + c2=0; + length=1; + prevStarter=prevSrc; + } else { + if(isNorm32Regular(norm32)) { + c2=0; + length=1; + } else { + /* c is a lead surrogate, get the real norm32 */ + if(src!=limit && UTF_IS_SECOND_SURROGATE(c2=*src)) { + ++src; + length=2; + norm32=_getNorm32FromSurrogatePair(norm32, c2); + } else { + /* c is an unpaired lead surrogate, nothing to do */ + c2=0; + length=1; + norm32=0; + } + } + + /* we are looking at the character (c, c2) at [prevSrc..src[ */ + if(nx_contains(nx, c, c2)) { + /* excluded: norm32==0 */ + cc=0; + } else if((norm32&qcMask)==0) { + cc=(uint8_t)(norm32>>_NORM_CC_SHIFT); + } else { + const UChar *p; + uint32_t decompQCMask; + + /* + * find appropriate boundaries around this character, + * decompose the source text from between the boundaries, + * and recompose it + * + * this puts the intermediate text into the side buffer because + * it might be longer than the recomposition end result, + * or the destination buffer may be too short or missing + * + * note that destIndex may be adjusted backwards to account + * for source text that passed the quick check but needed to + * take part in the recomposition + */ + decompQCMask=(qcMask<<2)&0xf; /* decomposition quick check mask */ + + /* + * find the last true starter in [prevStarter..src[ + * it is either the decomposition of the current character (at prevSrc), + * or prevStarter + */ + if(_isTrueStarter(norm32, ccOrQCMask, decompQCMask)) { + prevStarter=prevSrc; + } else { + /* adjust destIndex: back out what had been copied with qc "yes" */ + destIndex-=(int32_t)(prevSrc-prevStarter); + } + + /* find the next true starter in [src..limit[ - modifies src to point to the next starter */ + src=_findNextStarter(src, limit, qcMask, decompQCMask, minNoMaybe); + + /* compose [prevStarter..src[ */ + p=_composePart(stackBuffer, buffer, bufferCapacity, + length, /* output */ + prevStarter, src, + prevCC, /* output */ + options, nx, + pErrorCode); + + if(p==NULL) { + destIndex=0; /* an error occurred (out of memory) */ + break; + } + + /* append the recomposed buffer contents to the destination buffer */ + if((destIndex+length)<=destCapacity) { + while(length>0) { + dest[destIndex++]=*p++; + --length; + } + } else { + /* buffer overflow */ + /* keep incrementing the destIndex for preflighting */ + destIndex+=length; + } + + /* set the next starter */ + prevStarter=src; + + continue; + } + } + + /* append the single code point (c, c2) to the destination buffer */ + if((destIndex+length)<=destCapacity) { + if(cc!=0 && cc<prevCC) { + /* (c, c2) is out of order with respect to the preceding text */ + UChar *reorderSplit=dest+destIndex; + destIndex+=length; + prevCC=_insertOrdered(dest+reorderStartIndex, reorderSplit, dest+destIndex, c, c2, cc); + } else { + /* just append (c, c2) */ + dest[destIndex++]=c; + if(c2!=0) { + dest[destIndex++]=c2; + } + prevCC=cc; + } + } else { + /* buffer overflow */ + /* keep incrementing the destIndex for preflighting */ + destIndex+=length; + prevCC=cc; + } + } + + /* cleanup */ + if(buffer!=stackBuffer) { + uprv_free(buffer); + } + + return destIndex; +} + +U_CAPI int32_t U_EXPORT2 +unorm_compose(UChar *dest, int32_t destCapacity, + const UChar *src, int32_t srcLength, + UBool compat, int32_t options, + UErrorCode *pErrorCode) { + const UnicodeSet *nx; + int32_t destIndex; + + if(!_haveData(*pErrorCode)) { + return 0; + } + + nx=getNX(options, *pErrorCode); + if(U_FAILURE(*pErrorCode)) { + return 0; + } + + /* reset options bits that should only be set here or inside _compose() */ + options&=~(_NORM_OPTIONS_SETS_MASK|_NORM_OPTIONS_COMPAT|_NORM_OPTIONS_COMPOSE_CONTIGUOUS); + + if(compat) { + options|=_NORM_OPTIONS_COMPAT; + } + + destIndex=_compose(dest, destCapacity, + src, srcLength, + options, nx, + pErrorCode); + + return u_terminateUChars(dest, destCapacity, destIndex, pErrorCode); +} + +/* make FCD ----------------------------------------------------------------- */ + +static const UChar * +_findSafeFCD(const UChar *src, const UChar *limit, uint16_t fcd16) { + UChar c, c2; + + /* + * find the first position in [src..limit[ after some cc==0 according to FCD data + * + * at the beginning of the loop, we have fcd16 from before src + * + * stop at positions: + * - after trail cc==0 + * - at the end of the source + * - before lead cc==0 + */ + for(;;) { + /* stop if trail cc==0 for the previous character */ + if((fcd16&0xff)==0) { + break; + } + + /* get c=*src - stop at end of string */ + if(src==limit) { + break; + } + c=*src; + + /* stop if lead cc==0 for this character */ + if(c<_NORM_MIN_WITH_LEAD_CC || (fcd16=_getFCD16(c))==0) { + break; /* catches terminating NUL, too */ + } + + if(!UTF_IS_FIRST_SURROGATE(c)) { + if(fcd16<=0xff) { + break; + } + ++src; + } else if((src+1)!=limit && (c2=*(src+1), UTF_IS_SECOND_SURROGATE(c2))) { + /* c is a lead surrogate, get the real fcd16 */ + fcd16=_getFCD16FromSurrogatePair(fcd16, c2); + if(fcd16<=0xff) { + break; + } + src+=2; + } else { + /* c is an unpaired first surrogate, lead cc==0 */ + break; + } + } + + return src; +} + +static uint8_t +_decomposeFCD(const UChar *src, const UChar *decompLimit, + UChar *dest, int32_t &destIndex, int32_t destCapacity, + const UnicodeSet *nx) { + const UChar *p; + uint32_t norm32; + int32_t reorderStartIndex, length; + UChar c, c2; + uint8_t cc, prevCC, trailCC; + + /* + * canonically decompose [src..decompLimit[ + * + * all characters in this range have some non-zero cc, + * directly or in decomposition, + * so that we do not need to check in the following for quick-check limits etc. + * + * there _are_ _no_ Hangul syllables or Jamos in here because they are FCD-safe (cc==0)! + * + * we also do not need to check for c==0 because we have an established decompLimit + */ + reorderStartIndex=destIndex; + prevCC=0; + + while(src<decompLimit) { + c=*src++; + norm32=_getNorm32(c); + if(isNorm32Regular(norm32)) { + c2=0; + length=1; + } else { + /* + * reminder: this function is called with [src..decompLimit[ + * not containing any Hangul/Jamo characters, + * therefore the only specials are lead surrogates + */ + /* c is a lead surrogate, get the real norm32 */ + if(src!=decompLimit && UTF_IS_SECOND_SURROGATE(c2=*src)) { + ++src; + length=2; + norm32=_getNorm32FromSurrogatePair(norm32, c2); + } else { + c2=0; + length=1; + norm32=0; + } + } + + /* get the decomposition and the lead and trail cc's */ + if(nx_contains(nx, c, c2)) { + /* excluded: norm32==0 */ + cc=trailCC=0; + p=NULL; + } else if((norm32&_NORM_QC_NFD)==0) { + /* c does not decompose */ + cc=trailCC=(uint8_t)(norm32>>_NORM_CC_SHIFT); + p=NULL; + } else { + /* c decomposes, get everything from the variable-length extra data */ + p=_decompose(norm32, length, cc, trailCC); + if(length==1) { + /* fastpath a single code unit from decomposition */ + c=*p; + c2=0; + p=NULL; + } + } + + /* append the decomposition to the destination buffer, assume length>0 */ + if((destIndex+length)<=destCapacity) { + UChar *reorderSplit=dest+destIndex; + if(p==NULL) { + /* fastpath: single code point */ + if(cc!=0 && cc<prevCC) { + /* (c, c2) is out of order with respect to the preceding text */ + destIndex+=length; + trailCC=_insertOrdered(dest+reorderStartIndex, reorderSplit, dest+destIndex, c, c2, cc); + } else { + /* just append (c, c2) */ + dest[destIndex++]=c; + if(c2!=0) { + dest[destIndex++]=c2; + } + } + } else { + /* general: multiple code points (ordered by themselves) from decomposition */ + if(cc!=0 && cc<prevCC) { + /* the decomposition is out of order with respect to the preceding text */ + destIndex+=length; + trailCC=_mergeOrdered(dest+reorderStartIndex, reorderSplit, p, p+length); + } else { + /* just append the decomposition */ + do { + dest[destIndex++]=*p++; + } while(--length>0); + } + } + } else { + /* buffer overflow */ + /* keep incrementing the destIndex for preflighting */ + destIndex+=length; + } + + prevCC=trailCC; + if(prevCC==0) { + reorderStartIndex=destIndex; + } + } + + return prevCC; +} + +static int32_t +unorm_makeFCD(UChar *dest, int32_t destCapacity, + const UChar *src, int32_t srcLength, + const UnicodeSet *nx, + UErrorCode *pErrorCode) { + const UChar *limit, *prevSrc, *decompStart; + int32_t destIndex, length; + UChar c, c2; + uint16_t fcd16; + int16_t prevCC, cc; + + if(!_haveData(*pErrorCode)) { + return 0; + } + + /* initialize */ + decompStart=src; + destIndex=0; + prevCC=0; + + /* avoid compiler warnings */ + c=0; + fcd16=0; + + if(srcLength>=0) { + /* string with length */ + limit=src+srcLength; + } else /* srcLength==-1 */ { + /* zero-terminated string */ + limit=NULL; + } + + U_ALIGN_CODE(16); + + for(;;) { + /* skip a run of code units below the minimum or with irrelevant data for the FCD check */ + prevSrc=src; + if(limit==NULL) { + for(;;) { + c=*src; + if(c<_NORM_MIN_WITH_LEAD_CC) { + if(c==0) { + break; + } + prevCC=(int16_t)-c; + } else if((fcd16=_getFCD16(c))==0) { + prevCC=0; + } else { + break; + } + ++src; + } + } else { + for(;;) { + if(src==limit) { + break; + } else if((c=*src)<_NORM_MIN_WITH_LEAD_CC) { + prevCC=(int16_t)-c; + } else if((fcd16=_getFCD16(c))==0) { + prevCC=0; + } else { + break; + } + ++src; + } + } + + /* + * prevCC has values from the following ranges: + * 0..0xff - the previous trail combining class + * <0 - the negative value of the previous code unit; + * that code unit was <_NORM_MIN_WITH_LEAD_CC and its _getFCD16() + * was deferred so that average text is checked faster + */ + + /* copy these code units all at once */ + if(src!=prevSrc) { + length=(int32_t)(src-prevSrc); + if((destIndex+length)<=destCapacity) { + uprv_memcpy(dest+destIndex, prevSrc, length*U_SIZEOF_UCHAR); + } + destIndex+=length; + prevSrc=src; + + /* prevCC<0 is only possible from the above loop, i.e., only if prevSrc<src */ + if(prevCC<0) { + /* the previous character was <_NORM_MIN_WITH_LEAD_CC, we need to get its trail cc */ + if(!nx_contains(nx, (UChar32)-prevCC)) { + prevCC=(int16_t)(_getFCD16((UChar)-prevCC)&0xff); + } else { + prevCC=0; /* excluded: fcd16==0 */ + } + + /* + * set a pointer to this below-U+0300 character; + * if prevCC==0 then it will moved to after this character below + */ + decompStart=prevSrc-1; + } + } + /* + * now: + * prevSrc==src - used later to adjust destIndex before decomposition + * prevCC>=0 + */ + + /* end of source reached? */ + if(limit==NULL ? c==0 : src==limit) { + break; + } + + /* set a pointer to after the last source position where prevCC==0 */ + if(prevCC==0) { + decompStart=prevSrc; + } + + /* c already contains *src and fcd16 is set for it, increment src */ + ++src; + + /* check one above-minimum, relevant code unit */ + if(UTF_IS_FIRST_SURROGATE(c)) { + /* c is a lead surrogate, get the real fcd16 */ + if(src!=limit && UTF_IS_SECOND_SURROGATE(c2=*src)) { + ++src; + fcd16=_getFCD16FromSurrogatePair(fcd16, c2); + } else { + c2=0; + fcd16=0; + } + } else { + c2=0; + } + + /* we are looking at the character (c, c2) at [prevSrc..src[ */ + if(nx_contains(nx, c, c2)) { + fcd16=0; /* excluded: fcd16==0 */ + } + + /* check the combining order, get the lead cc */ + cc=(int16_t)(fcd16>>8); + if(cc==0 || cc>=prevCC) { + /* the order is ok */ + if(cc==0) { + decompStart=prevSrc; + } + prevCC=(int16_t)(fcd16&0xff); + + /* just append (c, c2) */ + length= c2==0 ? 1 : 2; + if((destIndex+length)<=destCapacity) { + dest[destIndex++]=c; + if(c2!=0) { + dest[destIndex++]=c2; + } + } else { + destIndex+=length; + } + } else { + /* + * back out the part of the source that we copied already but + * is now going to be decomposed; + * prevSrc is set to after what was copied + */ + destIndex-=(int32_t)(prevSrc-decompStart); + + /* + * find the part of the source that needs to be decomposed; + * to be safe and simple, decompose to before the next character with lead cc==0 + */ + src=_findSafeFCD(src, limit, fcd16); + + /* + * the source text does not fulfill the conditions for FCD; + * decompose and reorder a limited piece of the text + */ + prevCC=_decomposeFCD(decompStart, src, + dest, destIndex, destCapacity, + nx); + decompStart=src; + } + } + + return u_terminateUChars(dest, destCapacity, destIndex, pErrorCode); +} + +/* quick check functions ---------------------------------------------------- */ + +static UBool +unorm_checkFCD(const UChar *src, int32_t srcLength, const UnicodeSet *nx) { + const UChar *limit; + UChar c, c2; + uint16_t fcd16; + int16_t prevCC, cc; + + /* initialize */ + prevCC=0; + + if(srcLength>=0) { + /* string with length */ + limit=src+srcLength; + } else /* srcLength==-1 */ { + /* zero-terminated string */ + limit=NULL; + } + + U_ALIGN_CODE(16); + + for(;;) { + /* skip a run of code units below the minimum or with irrelevant data for the FCD check */ + if(limit==NULL) { + for(;;) { + c=*src++; + if(c<_NORM_MIN_WITH_LEAD_CC) { + if(c==0) { + return TRUE; + } + /* + * delay _getFCD16(c) for any character <_NORM_MIN_WITH_LEAD_CC + * because chances are good that the next one will have + * a leading cc of 0; + * _getFCD16(-prevCC) is later called when necessary - + * -c fits into int16_t because it is <_NORM_MIN_WITH_LEAD_CC==0x300 + */ + prevCC=(int16_t)-c; + } else if((fcd16=_getFCD16(c))==0) { + prevCC=0; + } else { + break; + } + } + } else { + for(;;) { + if(src==limit) { + return TRUE; + } else if((c=*src++)<_NORM_MIN_WITH_LEAD_CC) { + prevCC=(int16_t)-c; + } else if((fcd16=_getFCD16(c))==0) { + prevCC=0; + } else { + break; + } + } + } + + /* check one above-minimum, relevant code unit */ + if(UTF_IS_FIRST_SURROGATE(c)) { + /* c is a lead surrogate, get the real fcd16 */ + if(src!=limit && UTF_IS_SECOND_SURROGATE(c2=*src)) { + ++src; + fcd16=_getFCD16FromSurrogatePair(fcd16, c2); + } else { + c2=0; + fcd16=0; + } + } else { + c2=0; + } + + if(nx_contains(nx, c, c2)) { + prevCC=0; /* excluded: fcd16==0 */ + continue; + } + + /* + * prevCC has values from the following ranges: + * 0..0xff - the previous trail combining class + * <0 - the negative value of the previous code unit; + * that code unit was <_NORM_MIN_WITH_LEAD_CC and its _getFCD16() + * was deferred so that average text is checked faster + */ + + /* check the combining order */ + cc=(int16_t)(fcd16>>8); + if(cc!=0) { + if(prevCC<0) { + /* the previous character was <_NORM_MIN_WITH_LEAD_CC, we need to get its trail cc */ + if(!nx_contains(nx, (UChar32)-prevCC)) { + prevCC=(int16_t)(_getFCD16((UChar)-prevCC)&0xff); + } else { + prevCC=0; /* excluded: fcd16==0 */ + } + } + + if(cc<prevCC) { + return FALSE; + } + } + prevCC=(int16_t)(fcd16&0xff); + } +} + +static UNormalizationCheckResult +_quickCheck(const UChar *src, + int32_t srcLength, + UNormalizationMode mode, + UBool allowMaybe, + const UnicodeSet *nx, + UErrorCode *pErrorCode) { + UChar stackBuffer[_STACK_BUFFER_CAPACITY]; + UChar *buffer; + int32_t bufferCapacity; + + const UChar *start, *limit; + uint32_t norm32, qcNorm32, ccOrQCMask, qcMask; + int32_t options; + UChar c, c2, minNoMaybe; + uint8_t cc, prevCC; + UNormalizationCheckResult result; + + /* check arguments */ + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return UNORM_MAYBE; + } + + if(src==NULL || srcLength<-1) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return UNORM_MAYBE; + } + + if(!_haveData(*pErrorCode)) { + return UNORM_MAYBE; + } + + /* check for a valid mode and set the quick check minimum and mask */ + switch(mode) { + case UNORM_NFC: + minNoMaybe=(UChar)indexes[_NORM_INDEX_MIN_NFC_NO_MAYBE]; + qcMask=_NORM_QC_NFC; + options=0; + break; + case UNORM_NFKC: + minNoMaybe=(UChar)indexes[_NORM_INDEX_MIN_NFKC_NO_MAYBE]; + qcMask=_NORM_QC_NFKC; + options=_NORM_OPTIONS_COMPAT; + break; + case UNORM_NFD: + minNoMaybe=(UChar)indexes[_NORM_INDEX_MIN_NFD_NO_MAYBE]; + qcMask=_NORM_QC_NFD; + options=0; + break; + case UNORM_NFKD: + minNoMaybe=(UChar)indexes[_NORM_INDEX_MIN_NFKD_NO_MAYBE]; + qcMask=_NORM_QC_NFKD; + options=_NORM_OPTIONS_COMPAT; + break; + case UNORM_FCD: + if(fcdTrie.index==NULL) { + *pErrorCode=U_UNSUPPORTED_ERROR; + return UNORM_MAYBE; + } + return unorm_checkFCD(src, srcLength, nx) ? UNORM_YES : UNORM_NO; + default: + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return UNORM_MAYBE; + } + + /* initialize */ + buffer=stackBuffer; + bufferCapacity=_STACK_BUFFER_CAPACITY; + + ccOrQCMask=_NORM_CC_MASK|qcMask; + result=UNORM_YES; + prevCC=0; + + start=src; + if(srcLength>=0) { + /* string with length */ + limit=src+srcLength; + } else /* srcLength==-1 */ { + /* zero-terminated string */ + limit=NULL; + } + + U_ALIGN_CODE(16); + + for(;;) { + /* skip a run of code units below the minimum or with irrelevant data for the quick check */ + if(limit==NULL) { + for(;;) { + c=*src++; + if(c<minNoMaybe) { + if(c==0) { + goto endloop; /* break out of outer loop */ + } + } else if(((norm32=_getNorm32(c))&ccOrQCMask)!=0) { + break; + } + prevCC=0; + } + } else { + for(;;) { + if(src==limit) { + goto endloop; /* break out of outer loop */ + } else if((c=*src++)>=minNoMaybe && ((norm32=_getNorm32(c))&ccOrQCMask)!=0) { + break; + } + prevCC=0; + } + } + + /* check one above-minimum, relevant code unit */ + if(isNorm32LeadSurrogate(norm32)) { + /* c is a lead surrogate, get the real norm32 */ + if(src!=limit && UTF_IS_SECOND_SURROGATE(c2=*src)) { + ++src; + norm32=_getNorm32FromSurrogatePair(norm32, c2); + } else { + c2=0; + norm32=0; + } + } else { + c2=0; + } + + if(nx_contains(nx, c, c2)) { + /* excluded: norm32==0 */ + norm32=0; + } + + /* check the combining order */ + cc=(uint8_t)(norm32>>_NORM_CC_SHIFT); + if(cc!=0 && cc<prevCC) { + result=UNORM_NO; + break; + } + prevCC=cc; + + /* check for "no" or "maybe" quick check flags */ + qcNorm32=norm32&qcMask; + if(qcNorm32&_NORM_QC_ANY_NO) { + result=UNORM_NO; + break; + } else if(qcNorm32!=0) { + /* "maybe" can only occur for NFC and NFKC */ + if(allowMaybe) { + result=UNORM_MAYBE; + } else { + /* normalize a section around here to see if it is really normalized or not */ + const UChar *prevStarter; + uint32_t decompQCMask; + int32_t length; + + decompQCMask=(qcMask<<2)&0xf; /* decomposition quick check mask */ + + /* find the previous starter */ + prevStarter=src-1; /* set prevStarter to the beginning of the current character */ + if(UTF_IS_TRAIL(*prevStarter)) { + --prevStarter; /* safe because unpaired surrogates do not result in "maybe" */ + } + prevStarter=_findPreviousStarter(start, prevStarter, ccOrQCMask, decompQCMask, minNoMaybe); + + /* find the next true starter in [src..limit[ - modifies src to point to the next starter */ + src=_findNextStarter(src, limit, qcMask, decompQCMask, minNoMaybe); + + /* decompose and recompose [prevStarter..src[ */ + _composePart(stackBuffer, buffer, bufferCapacity, + length, + prevStarter, + src, + prevCC, + options, nx, pErrorCode); + if(U_FAILURE(*pErrorCode)) { + result=UNORM_MAYBE; /* error (out of memory) */ + break; + } + + /* compare the normalized version with the original */ + if(0!=uprv_strCompare(prevStarter, (int32_t)(src-prevStarter), buffer, length, FALSE, FALSE)) { + result=UNORM_NO; /* normalization differs */ + break; + } + + /* continue after the next starter */ + } + } + } +endloop: + + if(buffer!=stackBuffer) { + uprv_free(buffer); + } + + return result; +} + +U_CAPI UNormalizationCheckResult U_EXPORT2 +unorm_quickCheck(const UChar *src, + int32_t srcLength, + UNormalizationMode mode, + UErrorCode *pErrorCode) { + return _quickCheck(src, srcLength, mode, TRUE, NULL, pErrorCode); +} + +U_CAPI UNormalizationCheckResult U_EXPORT2 +unorm_quickCheckWithOptions(const UChar *src, int32_t srcLength, + UNormalizationMode mode, int32_t options, + UErrorCode *pErrorCode) { + return _quickCheck(src, srcLength, mode, TRUE, getNX(options, *pErrorCode), pErrorCode); +} + +U_CFUNC UNormalizationCheckResult +unorm_internalQuickCheck(const UChar *src, + int32_t srcLength, + UNormalizationMode mode, + UBool allowMaybe, + const UnicodeSet *nx, + UErrorCode *pErrorCode) { + return _quickCheck(src, srcLength, mode, allowMaybe, nx, pErrorCode); +} + +U_CAPI UBool U_EXPORT2 +unorm_isNormalized(const UChar *src, int32_t srcLength, + UNormalizationMode mode, + UErrorCode *pErrorCode) { + return (UBool)(UNORM_YES==_quickCheck(src, srcLength, mode, FALSE, NULL, pErrorCode)); +} + +U_CAPI UBool U_EXPORT2 +unorm_isNormalizedWithOptions(const UChar *src, int32_t srcLength, + UNormalizationMode mode, int32_t options, + UErrorCode *pErrorCode) { + return (UBool)(UNORM_YES==_quickCheck(src, srcLength, mode, FALSE, getNX(options, *pErrorCode), pErrorCode)); +} + +/* normalize() API ---------------------------------------------------------- */ + +/** + * Internal API for normalizing. + * Does not check for bad input. + * Requires _haveData() to be true. + * @internal + */ +U_CFUNC int32_t +unorm_internalNormalizeWithNX(UChar *dest, int32_t destCapacity, + const UChar *src, int32_t srcLength, + UNormalizationMode mode, int32_t options, const UnicodeSet *nx, + UErrorCode *pErrorCode) { + int32_t destLength; + uint8_t trailCC; + + switch(mode) { + case UNORM_NFD: + destLength=_decompose(dest, destCapacity, + src, srcLength, + FALSE, nx, trailCC); + break; + case UNORM_NFKD: + destLength=_decompose(dest, destCapacity, + src, srcLength, + TRUE, nx, trailCC); + break; + case UNORM_NFC: + destLength=_compose(dest, destCapacity, + src, srcLength, + options, nx, pErrorCode); + break; + case UNORM_NFKC: + destLength=_compose(dest, destCapacity, + src, srcLength, + options|_NORM_OPTIONS_COMPAT, nx, pErrorCode); + break; + case UNORM_FCD: + if(fcdTrie.index==NULL) { + *pErrorCode=U_UNSUPPORTED_ERROR; + return 0; + } + return unorm_makeFCD(dest, destCapacity, + src, srcLength, + nx, + pErrorCode); +#if 0 + case UNORM_FCC: + destLength=_compose(dest, destCapacity, + src, srcLength, + options|_NORM_OPTIONS_COMPOSE_CONTIGUOUS, nx, pErrorCode); + break; +#endif + case UNORM_NONE: + /* just copy the string */ + if(srcLength==-1) { + srcLength=u_strlen(src); + } + if(srcLength>0 && srcLength<=destCapacity) { + uprv_memcpy(dest, src, srcLength*U_SIZEOF_UCHAR); + } + destLength=srcLength; + break; + default: + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + return u_terminateUChars(dest, destCapacity, destLength, pErrorCode); +} + +/** + * Internal API for normalizing. + * Does not check for bad input. + * @internal + */ +U_CAPI int32_t U_EXPORT2 +unorm_internalNormalize(UChar *dest, int32_t destCapacity, + const UChar *src, int32_t srcLength, + UNormalizationMode mode, int32_t options, + UErrorCode *pErrorCode) { + const UnicodeSet *nx; + + if(!_haveData(*pErrorCode)) { + return 0; + } + + nx=getNX(options, *pErrorCode); + if(U_FAILURE(*pErrorCode)) { + return 0; + } + + /* reset options bits that should only be set inside unorm_internalNormalizeWithNX() */ + options&=~(_NORM_OPTIONS_SETS_MASK|_NORM_OPTIONS_COMPAT|_NORM_OPTIONS_COMPOSE_CONTIGUOUS); + + return unorm_internalNormalizeWithNX(dest, destCapacity, + src, srcLength, + mode, options, nx, + pErrorCode); +} + +/** Public API for normalizing. */ +U_CAPI int32_t U_EXPORT2 +unorm_normalize(const UChar *src, int32_t srcLength, + UNormalizationMode mode, int32_t options, + UChar *dest, int32_t destCapacity, + UErrorCode *pErrorCode) { + /* check argument values */ + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return 0; + } + + if( destCapacity<0 || (dest==NULL && destCapacity>0) || + src==NULL || srcLength<-1 + ) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + /* check for overlapping src and destination */ + if( dest!=NULL && + ((src>=dest && src<(dest+destCapacity)) || + (srcLength>0 && dest>=src && dest<(src+srcLength))) + ) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + return unorm_internalNormalize(dest, destCapacity, + src, srcLength, + mode, options, + pErrorCode); +} + + +/* iteration functions ------------------------------------------------------ */ + +/* + * These iteration functions are the core implementations of the + * Normalizer class iteration API. + * They read from a UCharIterator into their own buffer + * and normalize into the Normalizer iteration buffer. + * Normalizer itself then iterates over its buffer until that needs to be + * filled again. + */ + +/* + * ### TODO: + * Now that UCharIterator.next/previous return (int32_t)-1 not (UChar)0xffff + * if iteration bounds are reached, + * try to not call hasNext/hasPrevious and instead check for >=0. + */ + +/* backward iteration ------------------------------------------------------- */ + +/* + * read backwards and get norm32 + * return 0 if the character is <minC + * if c2!=0 then (c2, c) is a surrogate pair (reversed - c2 is first surrogate but read second!) + */ +static inline uint32_t +_getPrevNorm32(UCharIterator &src, uint32_t minC, uint32_t mask, UChar &c, UChar &c2) { + uint32_t norm32; + + /* need src.hasPrevious() */ + c=(UChar)src.previous(&src); + c2=0; + + /* check for a surrogate before getting norm32 to see if we need to predecrement further */ + if(c<minC) { + return 0; + } else if(!UTF_IS_SURROGATE(c)) { + return _getNorm32(c); + } else if(UTF_IS_SURROGATE_FIRST(c) || !src.hasPrevious(&src)) { + /* unpaired surrogate */ + return 0; + } else if(UTF_IS_FIRST_SURROGATE(c2=(UChar)src.previous(&src))) { + norm32=_getNorm32(c2); + if((norm32&mask)==0) { + /* all surrogate pairs with this lead surrogate have irrelevant data */ + return 0; + } else { + /* norm32 must be a surrogate special */ + return _getNorm32FromSurrogatePair(norm32, c); + } + } else { + /* unpaired second surrogate, undo the c2=src.previous() movement */ + src.move(&src, 1, UITER_CURRENT); + c2=0; + return 0; + } +} + +/* + * read backwards and check if the character is a previous-iteration boundary + * if c2!=0 then (c2, c) is a surrogate pair (reversed - c2 is first surrogate but read second!) + */ +typedef UBool +IsPrevBoundaryFn(UCharIterator &src, uint32_t minC, uint32_t mask, UChar &c, UChar &c2); + +/* + * for NF*D: + * read backwards and check if the lead combining class is 0 + * if c2!=0 then (c2, c) is a surrogate pair (reversed - c2 is first surrogate but read second!) + */ +static UBool +_isPrevNFDSafe(UCharIterator &src, uint32_t minC, uint32_t ccOrQCMask, UChar &c, UChar &c2) { + return _isNFDSafe(_getPrevNorm32(src, minC, ccOrQCMask, c, c2), ccOrQCMask, ccOrQCMask&_NORM_QC_MASK); +} + +/* + * read backwards and check if the character is (or its decomposition begins with) + * a "true starter" (cc==0 and NF*C_YES) + * if c2!=0 then (c2, c) is a surrogate pair (reversed - c2 is first surrogate but read second!) + */ +static UBool +_isPrevTrueStarter(UCharIterator &src, uint32_t minC, uint32_t ccOrQCMask, UChar &c, UChar &c2) { + uint32_t norm32, decompQCMask; + + decompQCMask=(ccOrQCMask<<2)&0xf; /* decomposition quick check mask */ + norm32=_getPrevNorm32(src, minC, ccOrQCMask|decompQCMask, c, c2); + return _isTrueStarter(norm32, ccOrQCMask, decompQCMask); +} + +static int32_t +_findPreviousIterationBoundary(UCharIterator &src, + IsPrevBoundaryFn *isPrevBoundary, uint32_t minC, uint32_t mask, + UChar *&buffer, int32_t &bufferCapacity, + int32_t &startIndex, + UErrorCode *pErrorCode) { + UChar *stackBuffer; + UChar c, c2; + UBool isBoundary; + + /* initialize */ + stackBuffer=buffer; + startIndex=bufferCapacity; /* fill the buffer from the end backwards */ + + while(src.hasPrevious(&src)) { + isBoundary=isPrevBoundary(src, minC, mask, c, c2); + + /* always write this character to the front of the buffer */ + /* make sure there is enough space in the buffer */ + if(startIndex < (c2==0 ? 1 : 2)) { + int32_t bufferLength=bufferCapacity; + + if(!u_growBufferFromStatic(stackBuffer, &buffer, &bufferCapacity, 2*bufferCapacity, bufferLength)) { + *pErrorCode=U_MEMORY_ALLOCATION_ERROR; + src.move(&src, 0, UITER_START); + return 0; + } + + /* move the current buffer contents up */ + uprv_memmove(buffer+(bufferCapacity-bufferLength), buffer, bufferLength*U_SIZEOF_UCHAR); + startIndex+=bufferCapacity-bufferLength; + } + + buffer[--startIndex]=c; + if(c2!=0) { + buffer[--startIndex]=c2; + } + + /* stop if this just-copied character is a boundary */ + if(isBoundary) { + break; + } + } + + /* return the length of the buffer contents */ + return bufferCapacity-startIndex; +} + +U_CAPI int32_t U_EXPORT2 +unorm_previous(UCharIterator *src, + UChar *dest, int32_t destCapacity, + UNormalizationMode mode, int32_t options, + UBool doNormalize, UBool *pNeededToNormalize, + UErrorCode *pErrorCode) { + UChar stackBuffer[100]; + UChar *buffer=NULL; + IsPrevBoundaryFn *isPreviousBoundary=NULL; + uint32_t mask=0; + int32_t startIndex=0, bufferLength=0, bufferCapacity=0, destLength=0; + int32_t c=0, c2=0; + UChar minC=0; + + /* check argument values */ + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return 0; + } + + if( destCapacity<0 || (dest==NULL && destCapacity>0) || + src==NULL + ) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + if(!_haveData(*pErrorCode)) { + return 0; + } + + if(pNeededToNormalize!=NULL) { + *pNeededToNormalize=FALSE; + } + + switch(mode) { + case UNORM_FCD: + if(fcdTrie.index==NULL) { + *pErrorCode=U_UNSUPPORTED_ERROR; + return 0; + } + /* fall through to NFD */ + case UNORM_NFD: + isPreviousBoundary=_isPrevNFDSafe; + minC=_NORM_MIN_WITH_LEAD_CC; + mask=_NORM_CC_MASK|_NORM_QC_NFD; + break; + case UNORM_NFKD: + isPreviousBoundary=_isPrevNFDSafe; + minC=_NORM_MIN_WITH_LEAD_CC; + mask=_NORM_CC_MASK|_NORM_QC_NFKD; + break; + case UNORM_NFC: + isPreviousBoundary=_isPrevTrueStarter; + minC=(UChar)indexes[_NORM_INDEX_MIN_NFC_NO_MAYBE]; + mask=_NORM_CC_MASK|_NORM_QC_NFC; + break; + case UNORM_NFKC: + isPreviousBoundary=_isPrevTrueStarter; + minC=(UChar)indexes[_NORM_INDEX_MIN_NFKC_NO_MAYBE]; + mask=_NORM_CC_MASK|_NORM_QC_NFKC; + break; + case UNORM_NONE: + destLength=0; + if((c=src->previous(src))>=0) { + destLength=1; + if(UTF_IS_TRAIL(c) && (c2=src->previous(src))>=0) { + if(UTF_IS_LEAD(c2)) { + if(destCapacity>=2) { + dest[1]=(UChar)c; /* trail surrogate */ + destLength=2; + } + c=c2; /* lead surrogate to be written below */ + } else { + src->move(src, 1, UITER_CURRENT); + } + } + + if(destCapacity>0) { + dest[0]=(UChar)c; + } + } + return u_terminateUChars(dest, destCapacity, destLength, pErrorCode); + default: + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + buffer=stackBuffer; + bufferCapacity=(int32_t)(sizeof(stackBuffer)/U_SIZEOF_UCHAR); + bufferLength=_findPreviousIterationBoundary(*src, + isPreviousBoundary, minC, mask, + buffer, bufferCapacity, + startIndex, + pErrorCode); + if(bufferLength>0) { + if(doNormalize) { + destLength=unorm_internalNormalize(dest, destCapacity, + buffer+startIndex, bufferLength, + mode, options, + pErrorCode); + if(pNeededToNormalize!=0 && U_SUCCESS(*pErrorCode)) { + *pNeededToNormalize= + (UBool)(destLength!=bufferLength || + 0!=uprv_memcmp(dest, buffer+startIndex, destLength*U_SIZEOF_UCHAR)); + } + } else { + /* just copy the source characters */ + if(destCapacity>0) { + uprv_memcpy(dest, buffer+startIndex, uprv_min(bufferLength, destCapacity)*U_SIZEOF_UCHAR); + } + destLength=u_terminateUChars(dest, destCapacity, bufferLength, pErrorCode); + } + } else { + destLength=u_terminateUChars(dest, destCapacity, 0, pErrorCode); + } + + /* cleanup */ + if(buffer!=stackBuffer) { + uprv_free(buffer); + } + + return destLength; +} + +/* forward iteration -------------------------------------------------------- */ + +/* + * read forward and get norm32 + * return 0 if the character is <minC + * if c2!=0 then (c2, c) is a surrogate pair + * always reads complete characters + */ +static inline uint32_t +_getNextNorm32(UCharIterator &src, uint32_t minC, uint32_t mask, UChar &c, UChar &c2) { + uint32_t norm32; + + /* need src.hasNext() to be true */ + c=(UChar)src.next(&src); + c2=0; + + if(c<minC) { + return 0; + } + + norm32=_getNorm32(c); + if(UTF_IS_FIRST_SURROGATE(c)) { + if(src.hasNext(&src) && UTF_IS_SECOND_SURROGATE(c2=(UChar)src.current(&src))) { + src.move(&src, 1, UITER_CURRENT); /* skip the c2 surrogate */ + if((norm32&mask)==0) { + /* irrelevant data */ + return 0; + } else { + /* norm32 must be a surrogate special */ + return _getNorm32FromSurrogatePair(norm32, c2); + } + } else { + /* unmatched surrogate */ + c2=0; + return 0; + } + } + return norm32; +} + +/* + * read forward and check if the character is a next-iteration boundary + * if c2!=0 then (c, c2) is a surrogate pair + */ +typedef UBool +IsNextBoundaryFn(UCharIterator &src, uint32_t minC, uint32_t mask, UChar &c, UChar &c2); + +/* + * for NF*D: + * read forward and check if the lead combining class is 0 + * if c2!=0 then (c, c2) is a surrogate pair + */ +static UBool +_isNextNFDSafe(UCharIterator &src, uint32_t minC, uint32_t ccOrQCMask, UChar &c, UChar &c2) { + return _isNFDSafe(_getNextNorm32(src, minC, ccOrQCMask, c, c2), ccOrQCMask, ccOrQCMask&_NORM_QC_MASK); +} + +/* + * for NF*C: + * read forward and check if the character is (or its decomposition begins with) + * a "true starter" (cc==0 and NF*C_YES) + * if c2!=0 then (c, c2) is a surrogate pair + */ +static UBool +_isNextTrueStarter(UCharIterator &src, uint32_t minC, uint32_t ccOrQCMask, UChar &c, UChar &c2) { + uint32_t norm32, decompQCMask; + + decompQCMask=(ccOrQCMask<<2)&0xf; /* decomposition quick check mask */ + norm32=_getNextNorm32(src, minC, ccOrQCMask|decompQCMask, c, c2); + return _isTrueStarter(norm32, ccOrQCMask, decompQCMask); +} + +static int32_t +_findNextIterationBoundary(UCharIterator &src, + IsNextBoundaryFn *isNextBoundary, uint32_t minC, uint32_t mask, + UChar *&buffer, int32_t &bufferCapacity, + UErrorCode *pErrorCode) { + UChar *stackBuffer; + int32_t bufferIndex; + UChar c, c2; + + if(!src.hasNext(&src)) { + return 0; + } + + /* initialize */ + stackBuffer=buffer; + + /* get one character and ignore its properties */ + buffer[0]=c=(UChar)src.next(&src); + bufferIndex=1; + if(UTF_IS_FIRST_SURROGATE(c) && src.hasNext(&src)) { + if(UTF_IS_SECOND_SURROGATE(c2=(UChar)src.next(&src))) { + buffer[bufferIndex++]=c2; + } else { + src.move(&src, -1, UITER_CURRENT); /* back out the non-trail-surrogate */ + } + } + + /* get all following characters until we see a boundary */ + /* checking hasNext() instead of c!=DONE on the off-chance that U+ffff is part of the string */ + while(src.hasNext(&src)) { + if(isNextBoundary(src, minC, mask, c, c2)) { + /* back out the latest movement to stop at the boundary */ + src.move(&src, c2==0 ? -1 : -2, UITER_CURRENT); + break; + } else { + if(bufferIndex+(c2==0 ? 1 : 2)<=bufferCapacity || + /* attempt to grow the buffer */ + u_growBufferFromStatic(stackBuffer, &buffer, &bufferCapacity, + 2*bufferCapacity, + bufferIndex) + ) { + buffer[bufferIndex++]=c; + if(c2!=0) { + buffer[bufferIndex++]=c2; + } + } else { + *pErrorCode=U_MEMORY_ALLOCATION_ERROR; + src.move(&src, 0, UITER_LIMIT); + return 0; + } + } + } + + /* return the length of the buffer contents */ + return bufferIndex; +} + +U_CAPI int32_t U_EXPORT2 +unorm_next(UCharIterator *src, + UChar *dest, int32_t destCapacity, + UNormalizationMode mode, int32_t options, + UBool doNormalize, UBool *pNeededToNormalize, + UErrorCode *pErrorCode) { + UChar stackBuffer[100]; + UChar *buffer; + IsNextBoundaryFn *isNextBoundary; + uint32_t mask; + int32_t bufferLength, bufferCapacity, destLength; + int32_t c, c2; + UChar minC; + + /* check argument values */ + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return 0; + } + + if( destCapacity<0 || (dest==NULL && destCapacity>0) || + src==NULL + ) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + if(!_haveData(*pErrorCode)) { + return 0; + } + + if(pNeededToNormalize!=NULL) { + *pNeededToNormalize=FALSE; + } + + switch(mode) { + case UNORM_FCD: + if(fcdTrie.index==NULL) { + *pErrorCode=U_UNSUPPORTED_ERROR; + return 0; + } + /* fall through to NFD */ + case UNORM_NFD: + isNextBoundary=_isNextNFDSafe; + minC=_NORM_MIN_WITH_LEAD_CC; + mask=_NORM_CC_MASK|_NORM_QC_NFD; + break; + case UNORM_NFKD: + isNextBoundary=_isNextNFDSafe; + minC=_NORM_MIN_WITH_LEAD_CC; + mask=_NORM_CC_MASK|_NORM_QC_NFKD; + break; + case UNORM_NFC: + isNextBoundary=_isNextTrueStarter; + minC=(UChar)indexes[_NORM_INDEX_MIN_NFC_NO_MAYBE]; + mask=_NORM_CC_MASK|_NORM_QC_NFC; + break; + case UNORM_NFKC: + isNextBoundary=_isNextTrueStarter; + minC=(UChar)indexes[_NORM_INDEX_MIN_NFKC_NO_MAYBE]; + mask=_NORM_CC_MASK|_NORM_QC_NFKC; + break; + case UNORM_NONE: + destLength=0; + if((c=src->next(src))>=0) { + destLength=1; + if(UTF_IS_LEAD(c) && (c2=src->next(src))>=0) { + if(UTF_IS_TRAIL(c2)) { + if(destCapacity>=2) { + dest[1]=(UChar)c2; /* trail surrogate */ + destLength=2; + } + /* lead surrogate to be written below */ + } else { + src->move(src, -1, UITER_CURRENT); + } + } + + if(destCapacity>0) { + dest[0]=(UChar)c; + } + } + return u_terminateUChars(dest, destCapacity, destLength, pErrorCode); + default: + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + buffer=stackBuffer; + bufferCapacity=(int32_t)(sizeof(stackBuffer)/U_SIZEOF_UCHAR); + bufferLength=_findNextIterationBoundary(*src, + isNextBoundary, minC, mask, + buffer, bufferCapacity, + pErrorCode); + if(bufferLength>0) { + if(doNormalize) { + destLength=unorm_internalNormalize(dest, destCapacity, + buffer, bufferLength, + mode, options, + pErrorCode); + if(pNeededToNormalize!=0 && U_SUCCESS(*pErrorCode)) { + *pNeededToNormalize= + (UBool)(destLength!=bufferLength || + 0!=uprv_memcmp(dest, buffer, destLength*U_SIZEOF_UCHAR)); + } + } else { + /* just copy the source characters */ + if(destCapacity>0) { + uprv_memcpy(dest, buffer, uprv_min(bufferLength, destCapacity)*U_SIZEOF_UCHAR); + } + destLength=u_terminateUChars(dest, destCapacity, bufferLength, pErrorCode); + } + } else { + destLength=u_terminateUChars(dest, destCapacity, 0, pErrorCode); + } + + /* cleanup */ + if(buffer!=stackBuffer) { + uprv_free(buffer); + } + + return destLength; +} + +/* + * ### TODO: check if NF*D and FCD iteration finds optimal boundaries + * and if not, how hard it would be to improve it. + * For example, see _findSafeFCD(). + */ + +/* Concatenation of normalized strings -------------------------------------- */ + +U_CAPI int32_t U_EXPORT2 +unorm_concatenate(const UChar *left, int32_t leftLength, + const UChar *right, int32_t rightLength, + UChar *dest, int32_t destCapacity, + UNormalizationMode mode, int32_t options, + UErrorCode *pErrorCode) { + UChar stackBuffer[100]; + UChar *buffer; + int32_t bufferLength, bufferCapacity; + + UCharIterator iter; + int32_t leftBoundary, rightBoundary, destLength; + + /* check argument values */ + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return 0; + } + + if( destCapacity<0 || (dest==NULL && destCapacity>0) || + left==NULL || leftLength<-1 || + right==NULL || rightLength<-1 + ) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + /* check for overlapping right and destination */ + if( dest!=NULL && + ((right>=dest && right<(dest+destCapacity)) || + (rightLength>0 && dest>=right && dest<(right+rightLength))) + ) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + /* allow left==dest */ + + /* set up intermediate buffer */ + buffer=stackBuffer; + bufferCapacity=(int32_t)(sizeof(stackBuffer)/U_SIZEOF_UCHAR); + + /* + * Input: left[0..leftLength[ + right[0..rightLength[ + * + * Find normalization-safe boundaries leftBoundary and rightBoundary + * and copy the end parts together: + * buffer=left[leftBoundary..leftLength[ + right[0..rightBoundary[ + * + * dest=left[0..leftBoundary[ + + * normalize(buffer) + + * right[rightBoundary..rightLength[ + */ + + /* + * find a normalization boundary at the end of the left string + * and copy the end part into the buffer + */ + uiter_setString(&iter, left, leftLength); + iter.index=leftLength=iter.length; /* end of left string */ + + bufferLength=unorm_previous(&iter, buffer, bufferCapacity, + mode, options, + FALSE, NULL, + pErrorCode); + leftBoundary=iter.index; + if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) { + *pErrorCode=U_ZERO_ERROR; + if(!u_growBufferFromStatic(stackBuffer, &buffer, &bufferCapacity, 2*bufferLength, 0)) { + *pErrorCode=U_MEMORY_ALLOCATION_ERROR; + /* dont need to cleanup here since + * u_growBufferFromStatic frees buffer if(buffer!=stackBuffer) + */ + return 0; + } + + /* just copy from the left string: we know the boundary already */ + uprv_memcpy(buffer, left+leftBoundary, bufferLength*U_SIZEOF_UCHAR); + } + + /* + * find a normalization boundary at the beginning of the right string + * and concatenate the beginning part to the buffer + */ + uiter_setString(&iter, right, rightLength); + rightLength=iter.length; /* in case it was -1 */ + + rightBoundary=unorm_next(&iter, buffer+bufferLength, bufferCapacity-bufferLength, + mode, options, + FALSE, NULL, + pErrorCode); + if(*pErrorCode==U_BUFFER_OVERFLOW_ERROR) { + *pErrorCode=U_ZERO_ERROR; + if(!u_growBufferFromStatic(stackBuffer, &buffer, &bufferCapacity, bufferLength+rightBoundary, 0)) { + *pErrorCode=U_MEMORY_ALLOCATION_ERROR; + /* dont need to cleanup here since + * u_growBufferFromStatic frees buffer if(buffer!=stackBuffer) + */ + return 0; + } + + /* just copy from the right string: we know the boundary already */ + uprv_memcpy(buffer+bufferLength, right, rightBoundary*U_SIZEOF_UCHAR); + } + + bufferLength+=rightBoundary; + + /* copy left[0..leftBoundary[ to dest */ + if(left!=dest && leftBoundary>0 && destCapacity>0) { + uprv_memcpy(dest, left, uprv_min(leftBoundary, destCapacity)*U_SIZEOF_UCHAR); + } + destLength=leftBoundary; + + /* concatenate the normalization of the buffer to dest */ + if(destCapacity>destLength) { + destLength+=unorm_internalNormalize(dest+destLength, destCapacity-destLength, + buffer, bufferLength, + mode, options, + pErrorCode); + } else { + destLength+=unorm_internalNormalize(NULL, 0, + buffer, bufferLength, + mode, options, + pErrorCode); + } + /* + * only errorCode that is expected is a U_BUFFER_OVERFLOW_ERROR + * so we dont check for the error code here..just let it pass through + */ + /* concatenate right[rightBoundary..rightLength[ to dest */ + right+=rightBoundary; + rightLength-=rightBoundary; + if(rightLength>0 && destCapacity>destLength) { + uprv_memcpy(dest+destLength, right, uprv_min(rightLength, destCapacity-destLength)*U_SIZEOF_UCHAR); + } + destLength+=rightLength; + + /* cleanup */ + if(buffer!=stackBuffer) { + uprv_free(buffer); + } + + return u_terminateUChars(dest, destCapacity, destLength, pErrorCode); +} + +#endif /* #if !UCONFIG_NO_NORMALIZATION */ |