diff options
author | Peter Breitenlohner <peb@mppmu.mpg.de> | 2009-04-20 18:13:48 +0000 |
---|---|---|
committer | Peter Breitenlohner <peb@mppmu.mpg.de> | 2009-04-20 18:13:48 +0000 |
commit | e5786f530f9555469c01435f86039b06aa53feba (patch) | |
tree | c868e8d32282422d7d445b729db95c3fcdafb6d0 /Build/source/libs/icu/icu-xetex/tools/gencase/store.c | |
parent | ebaa1768b43c8606d923d2e861b5286b74207b3e (diff) |
new build system: build icu libs and xetex plus misc updates
git-svn-id: svn://tug.org/texlive/trunk@12759 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Build/source/libs/icu/icu-xetex/tools/gencase/store.c')
-rw-r--r-- | Build/source/libs/icu/icu-xetex/tools/gencase/store.c | 1169 |
1 files changed, 1169 insertions, 0 deletions
diff --git a/Build/source/libs/icu/icu-xetex/tools/gencase/store.c b/Build/source/libs/icu/icu-xetex/tools/gencase/store.c new file mode 100644 index 00000000000..90d3abf8d71 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/tools/gencase/store.c @@ -0,0 +1,1169 @@ +/* +******************************************************************************* +* +* Copyright (C) 2004-2007, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: store.c +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2004aug28 +* created by: Markus W. Scherer +* +* Store Unicode case mapping properties efficiently for +* random access. +*/ + +#include <stdio.h> +#include <stdlib.h> +#include "unicode/utypes.h" +#include "unicode/uchar.h" +#include "unicode/ustring.h" +#include "cmemory.h" +#include "cstring.h" +#include "filestrm.h" +#include "utrie.h" +#include "uarrsort.h" +#include "unicode/udata.h" +#include "unewdata.h" +#include "propsvec.h" +#include "writesrc.h" +#include "gencase.h" + +#define LENGTHOF(array) (sizeof(array)/sizeof((array)[0])) + +/* Unicode case mapping properties file format --------------------------------- + +The file format prepared and written here contains several data +structures that store indexes or data. + +Before the data contents described below, there are the headers required by +the udata API for loading ICU data. Especially, a UDataInfo structure +precedes the actual data. It contains platform properties values and the +file format version. + +The following is a description of format version 1.1 . + +Format version 1.1 adds data for case closure. + +The file contains the following structures: + + const int32_t indexes[i0] with values i0, i1, ...: + (see UCASE_IX_... constants for names of indexes) + + i0 indexLength; -- length of indexes[] (UCASE_IX_TOP) + i1 dataLength; -- length in bytes of the post-header data (incl. indexes[]) + i2 trieSize; -- size in bytes of the case mapping properties trie + i3 exceptionsLength; -- length in uint16_t of the exceptions array + i4 unfoldLength; -- length in uint16_t of the reverse-folding array (new in format version 1.1) + + i5..i14 reservedIndexes; -- reserved values; 0 for now + + i15 maxFullLength; -- maximum length of a full case mapping/folding string + + + Serialized trie, see utrie.h; + + const uint16_t exceptions[exceptionsLength]; + + const UChar unfold[unfoldLength]; + + +Trie data word: +Bits +if(exception) { + 15..4 unsigned exception index +} else { + if(not uncased) { + 15..6 signed delta to simple case mapping code point + (add delta to input code point) + } else { + 6 the code point is case-ignorable + (U+0307 is also case-ignorable but has an exception) + } + 5..4 0 normal character with cc=0 + 1 soft-dotted character + 2 cc=230 + 3 other cc +} + 3 exception + 2 case sensitive + 1..0 0 uncased + 1 lowercase + 2 uppercase + 3 titlecase + + +Exceptions: +A sub-array of the exceptions array is indexed by the exception index in a +trie word. +The sub-array consists of the following fields: + uint16_t excWord; + uint16_t optional values []; + UTF-16 strings for full (string) mappings for lowercase, case folding, uppercase, titlecase + +excWord: (see UCASE_EXC_...) +Bits + 15 conditional case folding + 14 conditional special casing +13..12 same as non-exception trie data bits 5..4 + moved here because the exception index needs more bits than the delta + 0 normal character with cc=0 + 1 soft-dotted character + 2 cc=230 + 3 other cc +11.. 9 reserved + 8 if set, then for each optional-value slot there are 2 uint16_t values + (high and low parts of 32-bit values) + instead of single ones + 7.. 0 bits for which optional value is present + +Optional-value slots: +0 lowercase mapping (code point) +1 case folding (code point) +2 uppercase mapping (code point) +3 titlecase mapping (code point) +4 reserved +5 reserved +6 closure mappings (new in format version 1.1) +7 there is at least one full (string) case mapping + the length of each is encoded in a nibble of this optional value, + and the strings follow this optional value in the same order: + lower/fold/upper/title + +The optional closure mappings value is used as follows: +Bits 0..3 contain the length of a string of code points for case closure. +The string immediately follows the full case mappings, or the closure value +slot if there are no full case mappings. +Bits 4..15 are reserved and could be used in the future to indicate the +number of strings for case closure. +Complete case closure for a code point is given by the union of all simple +and full case mappings and foldings, plus the case closure code points +(and potentially, in the future, case closure strings). + +For space saving, some values are not stored. Lookups are as follows: +- If special casing is conditional, then no full lower/upper/title mapping + strings are stored. +- If case folding is conditional, then no simple or full case foldings are + stored. +- Fall back in this order: + full (string) mapping -- if full mappings are used + simple (code point) mapping of the same type + simple fold->simple lower + simple title->simple upper + finally, the original code point (no mapping) + +This fallback order is strict: +In particular, the fallback from full case folding is to simple case folding, +not to full lowercase mapping. + +Reverse case folding data ("unfold") array: (new in format version 1.1) + +This array stores some miscellaneous values followed by a table. The data maps +back from multi-character strings to their original code points, for use +in case closure. + +The table contains two columns of strings. +The string in the first column is the case folding of each of the code points +in the second column. The strings are terminated with NUL or by the end of the +column, whichever comes first. + +The miscellaneous data takes up one pseudo-row and includes: +- number of rows +- number of UChars per row +- number of UChars in the left (folding string) column + +The table is sorted by its first column. Values in the first column are unique. + +----------------------------------------------------------------------------- */ + +/* UDataInfo cf. udata.h */ +static UDataInfo dataInfo={ + sizeof(UDataInfo), + 0, + + U_IS_BIG_ENDIAN, + U_CHARSET_FAMILY, + U_SIZEOF_UCHAR, + 0, + + /* dataFormat="cAsE" */ + { UCASE_FMT_0, UCASE_FMT_1, UCASE_FMT_2, UCASE_FMT_3 }, + { 1, 1, UTRIE_SHIFT, UTRIE_INDEX_SHIFT }, /* formatVersion */ + { 4, 0, 1, 0 } /* dataVersion */ +}; + +enum { + /* maximum number of exceptions expected */ + MAX_EXC_COUNT=1000 +}; + +/* exceptions values */ +static uint16_t exceptions[UCASE_MAX_EXCEPTIONS+100]; +static uint16_t exceptionsTop=0; +static Props excProps[MAX_EXC_COUNT]; +static uint16_t exceptionsCount=0; + +/* becomes indexes[UCASE_IX_MAX_FULL_LENGTH] */ +static int32_t maxFullLength=U16_MAX_LENGTH; + +/* reverse case folding ("unfold") data */ +static UChar unfold[UGENCASE_UNFOLD_MAX_ROWS*UGENCASE_UNFOLD_WIDTH]={ + 0, UGENCASE_UNFOLD_WIDTH, UGENCASE_UNFOLD_STRING_WIDTH, 0, 0 +}; +static uint16_t unfoldRows=0; +static uint16_t unfoldTop=UGENCASE_UNFOLD_WIDTH; + +/* Unicode versions --------------------------------------------------------- */ + +static const UVersionInfo +unicodeVersions[]={ + { 1, 0, 0, 0 }, + { 1, 1, 0, 0 }, + { 2, 0, 0, 0 }, + { 3, 0, 0, 0 }, + { 3, 1, 0, 0 }, + { 3, 2, 0, 0 }, + { 4, 0, 0, 0 }, + { 4, 0, 1, 0 }, + { 4, 1, 0, 0 } +}; + +int32_t ucdVersion=UNI_4_1; + +static int32_t +findUnicodeVersion(const UVersionInfo version) { + int32_t i; + + for(i=0; /* while(version>unicodeVersions[i]) {} */ + i<UNI_VER_COUNT && uprv_memcmp(version, unicodeVersions[i], 4)>0; + ++i) {} + if(0<i && i<UNI_VER_COUNT && uprv_memcmp(version, unicodeVersions[i], 4)<0) { + --i; /* fix 4.0.2 to land before 4.1, for valid x>=ucdVersion comparisons */ + } + return i; /* version>=unicodeVersions[i] && version<unicodeVersions[i+1]; possible: i==UNI_VER_COUNT */ +} + +extern void +setUnicodeVersion(const char *v) { + UVersionInfo version; + u_versionFromString(version, v); + uprv_memcpy(dataInfo.dataVersion, version, 4); + ucdVersion=findUnicodeVersion(version); +} + +/* -------------------------------------------------------------------------- */ + +static void +addUnfolding(UChar32 c, const UChar *s, int32_t length) { + int32_t i; + + if(length>UGENCASE_UNFOLD_STRING_WIDTH) { + fprintf(stderr, "gencase error: case folding too long (length=%ld>%d=UGENCASE_UNFOLD_STRING_WIDTH)\n", + (long)length, UGENCASE_UNFOLD_STRING_WIDTH); + exit(U_INTERNAL_PROGRAM_ERROR); + } + if(unfoldTop >= (LENGTHOF(unfold) - UGENCASE_UNFOLD_STRING_WIDTH)) { + fprintf(stderr, "gencase error: too many multi-character case foldings\n"); + exit(U_BUFFER_OVERFLOW_ERROR); + } + u_memset(unfold+unfoldTop, 0, UGENCASE_UNFOLD_WIDTH); + u_memcpy(unfold+unfoldTop, s, length); + + i=unfoldTop+UGENCASE_UNFOLD_STRING_WIDTH; + U16_APPEND_UNSAFE(unfold, i, c); + + ++unfoldRows; + unfoldTop+=UGENCASE_UNFOLD_WIDTH; +} + +/* store a character's properties ------------------------------------------- */ + +extern void +setProps(Props *p) { + UErrorCode errorCode; + uint32_t value, oldValue; + int32_t delta; + UBool isCaseIgnorable; + + /* get the non-UnicodeData.txt properties */ + value=oldValue=upvec_getValue(pv, p->code, 0); + + /* default: map to self */ + delta=0; + + if(p->gc==U_TITLECASE_LETTER) { + /* the Titlecase property is read late, from UnicodeData.txt */ + value|=UCASE_TITLE; + } + + if(p->upperCase!=0) { + /* uppercase mapping as delta if the character is lowercase */ + if((value&UCASE_TYPE_MASK)==UCASE_LOWER) { + delta=p->upperCase-p->code; + } else { + value|=UCASE_EXCEPTION; + } + } + if(p->lowerCase!=0) { + /* lowercase mapping as delta if the character is uppercase or titlecase */ + if((value&UCASE_TYPE_MASK)>=UCASE_UPPER) { + delta=p->lowerCase-p->code; + } else { + value|=UCASE_EXCEPTION; + } + } + if(p->upperCase!=p->titleCase) { + value|=UCASE_EXCEPTION; + } + if(p->closure[0]!=0) { + value|=UCASE_EXCEPTION; + } + if(p->specialCasing!=NULL) { + value|=UCASE_EXCEPTION; + } + if(p->caseFolding!=NULL) { + value|=UCASE_EXCEPTION; + } + + if(delta<UCASE_MIN_DELTA || UCASE_MAX_DELTA<delta) { + value|=UCASE_EXCEPTION; + } + + if(p->cc!=0) { + if(value&UCASE_DOT_MASK) { + fprintf(stderr, "gencase: a soft-dotted character has cc!=0\n"); + exit(U_INTERNAL_PROGRAM_ERROR); + } + if(p->cc==230) { + value|=UCASE_ABOVE; + } else { + value|=UCASE_OTHER_ACCENT; + } + } + + /* encode case-ignorable as delta==1 on uncased characters */ + isCaseIgnorable=FALSE; + if((value&UCASE_TYPE_MASK)==UCASE_NONE) { + if(ucdVersion>=UNI_4_1) { + /* Unicode 4.1 and up: (D47a) Word_Break=MidLetter or Mn, Me, Cf, Lm, Sk */ + if( + (U_MASK(p->gc)&(U_GC_MN_MASK|U_GC_ME_MASK|U_GC_CF_MASK|U_GC_LM_MASK|U_GC_SK_MASK))!=0 || + ((upvec_getValue(pv, p->code, 1)>>UGENCASE_IS_MID_LETTER_SHIFT)&1)!=0 + ) { + isCaseIgnorable=TRUE; + } + } else { + /* before Unicode 4.1: Mn, Me, Cf, Lm, Sk or 0027 or 00AD or 2019 */ + if( + (U_MASK(p->gc)&(U_GC_MN_MASK|U_GC_ME_MASK|U_GC_CF_MASK|U_GC_LM_MASK|U_GC_SK_MASK))!=0 || + p->code==0x27 || p->code==0xad || p->code==0x2019 + ) { + isCaseIgnorable=TRUE; + } + } + } + + if(isCaseIgnorable && p->code!=0x307) { + /* + * We use one of the delta/exception bits, which works because we only + * store the case-ignorable flag for uncased characters. + * There is no delta for uncased characters (see checks above). + * If there is an exception for an uncased, case-ignorable character + * (although there should not be any case mappings if it's uncased) + * then we have a problem. + * There is one character which is case-ignorable but has an exception: + * U+0307 is uncased, Mn, has conditional special casing and + * is therefore handled in code instead. + */ + if(value&UCASE_EXCEPTION) { + fprintf(stderr, "gencase error: unable to encode case-ignorable for U+%04lx with exceptions\n", + (unsigned long)p->code); + exit(U_INTERNAL_PROGRAM_ERROR); + } + + delta=1; + } + + /* handle exceptions */ + if(value&UCASE_EXCEPTION) { + /* simply store exceptions for later processing and encoding */ + value|=(uint32_t)exceptionsCount<<UGENCASE_EXC_SHIFT; + uprv_memcpy(excProps+exceptionsCount, p, sizeof(*p)); + if(++exceptionsCount==MAX_EXC_COUNT) { + fprintf(stderr, "gencase: too many exceptions\n"); + exit(U_INDEX_OUTOFBOUNDS_ERROR); + } + } else { + /* store the simple case mapping delta */ + value|=((uint32_t)delta<<UCASE_DELTA_SHIFT)&UCASE_DELTA_MASK; + } + + errorCode=U_ZERO_ERROR; + if( value!=oldValue && + !upvec_setValue(pv, p->code, p->code+1, 0, value, 0xffffffff, &errorCode) + ) { + fprintf(stderr, "gencase error: unable to set case mapping values, code: %s\n", + u_errorName(errorCode)); + exit(errorCode); + } + + /* add the multi-character case folding to the "unfold" data */ + if(p->caseFolding!=NULL) { + int32_t length=p->caseFolding->full[0]; + if(length>1 && u_strHasMoreChar32Than(p->caseFolding->full+1, length, 1)) { + addUnfolding(p->code, p->caseFolding->full+1, length); + } + } +} + +extern void +addCaseSensitive(UChar32 first, UChar32 last) { + UErrorCode errorCode=U_ZERO_ERROR; + if(!upvec_setValue(pv, first, last+1, 0, UCASE_SENSITIVE, UCASE_SENSITIVE, &errorCode)) { + fprintf(stderr, "gencase error: unable to set UCASE_SENSITIVE, code: %s\n", + u_errorName(errorCode)); + exit(errorCode); + } +} + +/* finalize reverse case folding ("unfold") data ---------------------------- */ + +static int32_t U_CALLCONV +compareUnfold(const void *context, const void *left, const void *right) { + return u_memcmp((const UChar *)left, (const UChar *)right, UGENCASE_UNFOLD_WIDTH); +} + +static void +makeUnfoldData() { + static const UChar + iDot[2]= { 0x69, 0x307 }; + + UChar *p, *q; + int32_t i, j, k; + UErrorCode errorCode; + + /* + * add a case folding that we missed because it's conditional: + * 0130; F; 0069 0307; # LATIN CAPITAL LETTER I WITH DOT ABOVE + */ + addUnfolding(0x130, iDot, 2); + + /* sort the data */ + errorCode=U_ZERO_ERROR; + uprv_sortArray(unfold+UGENCASE_UNFOLD_WIDTH, unfoldRows, UGENCASE_UNFOLD_WIDTH*2, + compareUnfold, NULL, FALSE, &errorCode); + + /* make unique-string rows by merging adjacent ones' code point columns */ + + /* make p point to row i-1 */ + p=(UChar *)unfold+UGENCASE_UNFOLD_WIDTH; + + for(i=1; i<unfoldRows;) { + if(0==u_memcmp(p, p+UGENCASE_UNFOLD_WIDTH, UGENCASE_UNFOLD_STRING_WIDTH)) { + /* concatenate code point columns */ + q=p+UGENCASE_UNFOLD_STRING_WIDTH; + for(j=1; j<UGENCASE_UNFOLD_CP_WIDTH && q[j]!=0; ++j) {} + for(k=0; k<UGENCASE_UNFOLD_CP_WIDTH && q[UGENCASE_UNFOLD_WIDTH+k]!=0; ++j, ++k) { + q[j]=q[UGENCASE_UNFOLD_WIDTH+k]; + } + if(j>UGENCASE_UNFOLD_CP_WIDTH) { + fprintf(stderr, "gencase error: too many code points in unfold[]: %ld>%d=UGENCASE_UNFOLD_CP_WIDTH\n", + (long)j, UGENCASE_UNFOLD_CP_WIDTH); + exit(U_BUFFER_OVERFLOW_ERROR); + } + + /* move following rows up one */ + --unfoldRows; + unfoldTop-=UGENCASE_UNFOLD_WIDTH; + u_memmove(p+UGENCASE_UNFOLD_WIDTH, p+UGENCASE_UNFOLD_WIDTH*2, (unfoldRows-i)*UGENCASE_UNFOLD_WIDTH); + } else { + p+=UGENCASE_UNFOLD_WIDTH; + ++i; + } + } + + unfold[UCASE_UNFOLD_ROWS]=(UChar)unfoldRows; + + if(beVerbose) { + puts("unfold data:"); + + p=(UChar *)unfold; + for(i=0; i<unfoldRows; ++i) { + p+=UGENCASE_UNFOLD_WIDTH; + printf("[%2d] %04x %04x %04x <- %04x %04x\n", + (int)i, p[0], p[1], p[2], p[3], p[4]); + } + } +} + +/* case closure ------------------------------------------------------------- */ + +static void +addClosureMapping(UChar32 src, UChar32 dest) { + uint32_t value; + + if(beVerbose) { + printf("add closure mapping U+%04lx->U+%04lx\n", + (unsigned long)src, (unsigned long)dest); + } + + value=upvec_getValue(pv, src, 0); + if(value&UCASE_EXCEPTION) { + Props *p=excProps+(value>>UGENCASE_EXC_SHIFT); + int32_t i; + + /* append dest to src's closure array */ + for(i=0;; ++i) { + if(i==LENGTHOF(p->closure)) { + fprintf(stderr, "closure[] overflow for U+%04lx->U+%04lx\n", + (unsigned long)src, (unsigned long)dest); + exit(U_BUFFER_OVERFLOW_ERROR); + } else if(p->closure[i]==dest) { + break; /* do not store duplicates */ + } else if(p->closure[i]==0) { + p->closure[i]=dest; + break; + } + } + } else { + Props p2={ 0 }; + UChar32 next; + UErrorCode errorCode; + + /* + * decode value into p2 (enough for makeException() to work properly), + * add the closure mapping, + * and set the new exception for src + */ + p2.code=src; + p2.closure[0]=dest; + + if((value&UCASE_TYPE_MASK)>UCASE_NONE) { + /* one simple case mapping, don't care which one */ + next=src+((int16_t)value>>UCASE_DELTA_SHIFT); + if(next!=src) { + if((value&UCASE_TYPE_MASK)==UCASE_LOWER) { + p2.upperCase=p2.titleCase=next; + } else { + p2.lowerCase=next; + } + } + } else if(value&UCASE_DELTA_MASK) { + fprintf(stderr, "gencase error: unable to add case closure exception to case-ignorable U+%04lx\n", + (unsigned long)src); + exit(U_INTERNAL_PROGRAM_ERROR); + } + + value&=~(UGENCASE_EXC_MASK|UCASE_DELTA_MASK); /* remove previous simple mapping */ + value|=(uint32_t)exceptionsCount<<UGENCASE_EXC_SHIFT; + value|=UCASE_EXCEPTION; + uprv_memcpy(excProps+exceptionsCount, &p2, sizeof(p2)); + if(++exceptionsCount==MAX_EXC_COUNT) { + fprintf(stderr, "gencase: too many exceptions\n"); + exit(U_INDEX_OUTOFBOUNDS_ERROR); + } + + errorCode=U_ZERO_ERROR; + if(!upvec_setValue(pv, src, src+1, 0, value, 0xffffffff, &errorCode)) { + fprintf(stderr, "gencase error: unable to set case mapping values, code: %s\n", + u_errorName(errorCode)); + exit(errorCode); + } + } +} + +/* + * Find missing case mapping relationships and add mappings for case closure. + * This function starts from an "original" code point and recursively + * finds its case mappings and the case mappings of where it maps to. + * + * The recursion depth is capped at 3 nested calls of this function. + * In each call, the current code point is c, and the function enumerates + * all of c's simple (single-code point) case mappings. + * prev is the code point that case-mapped to c. + * prev2 is the code point that case-mapped to prev. + * + * The initial function call has prev2<0, prev<0, and c==orig + * (marking no code points). + * It enumerates c's case mappings and recurses without further action. + * + * The second-level function call has prev2<0, prev==orig, and c is + * the destination code point of one of prev's case mappings. + * The function checks if any of c's case mappings go back to orig + * and adds a closure mapping if not. + * In other words, it turns a case mapping relationship of + * orig->c + * into + * orig<->c + * + * The third-level function call has prev2==orig, prev>=0, and c is + * the destination code point of one of prev's case mappings. + * (And prev is the destination of one of prev2's case mappings.) + * The function checks if any of c's case mappings go back to orig + * and adds a closure mapping if not. + * In other words, it turns case mapping relationships of + * orig->prev->c or orig->prev<->c + * into + * orig->prev->c->orig or orig->prev<->c->orig + * etc. + * (Graphically, this closes a triangle.) + * + * With repeated application on all code points until no more closure mappings + * are added, all case equivalence groups get complete mappings. + * That is, in each group of code points with case relationships + * each code point will in the end have some mapping to each other + * code point in the group. + * + * @return TRUE if a closure mapping was added + */ +static UBool +addClosure(UChar32 orig, UChar32 prev2, UChar32 prev, UChar32 c, uint32_t value) { + UChar32 next; + UBool someMappingsAdded=FALSE; + + if(c!=orig) { + /* get the properties for c */ + value=upvec_getValue(pv, c, 0); + } + /* else if c==orig then c's value was passed in */ + + if(value&UCASE_EXCEPTION) { + UChar32 set[32]; + int32_t i, count=0; + + Props *p=excProps+(value>>UGENCASE_EXC_SHIFT); + + /* + * marker for whether any of c's mappings goes to orig + * c==orig: prevent adding a closure mapping when getting orig's own, direct mappings + */ + UBool mapsToOrig=(UBool)(c==orig); + + /* collect c's case mapping destinations in set[] */ + if((next=p->upperCase)!=0 && next!=c) { + set[count++]=next; + } + if((next=p->lowerCase)!=0 && next!=c) { + set[count++]=next; + } + if(p->upperCase!=(next=p->titleCase) && next!=c) { + set[count++]=next; + } + if(p->caseFolding!=NULL && (next=p->caseFolding->simple)!=0 && next!=c) { + set[count++]=next; + } + + /* append c's current closure mappings to set[] */ + for(i=0; i<LENGTHOF(p->closure) && (next=p->closure[i])!=0; ++i) { + set[count++]=next; + } + + /* process all code points to which c case-maps */ + for(i=0; i<count; ++i) { + next=set[i]; /* next!=c */ + + if(next==orig) { + mapsToOrig=TRUE; /* remember that we map to orig */ + } else if(prev2<0 && next!=prev) { + /* + * recurse unless + * we have reached maximum depth (prev2>=0) or + * this is a mapping to one of the previous code points (orig, prev, c) + */ + someMappingsAdded|=addClosure(orig, prev, c, next, 0); + } + } + + if(!mapsToOrig) { + addClosureMapping(c, orig); + return TRUE; + } + } else { + if((value&UCASE_TYPE_MASK)>UCASE_NONE) { + /* one simple case mapping, don't care which one */ + next=c+((int16_t)value>>UCASE_DELTA_SHIFT); + if(next!=c) { + /* + * recurse unless + * we have reached maximum depth (prev2>=0) or + * this is a mapping to one of the previous code points (orig, prev, c) + */ + if(prev2<0 && next!=orig && next!=prev) { + someMappingsAdded|=addClosure(orig, prev, c, next, 0); + } + + if(c!=orig && next!=orig) { + /* c does not map to orig, add a closure mapping c->orig */ + addClosureMapping(c, orig); + return TRUE; + } + } + } + } + + return someMappingsAdded; +} + +extern void +makeCaseClosure() { + UChar *p; + uint32_t *row; + uint32_t value; + UChar32 start, limit, c, c2; + int32_t i, j; + UBool someMappingsAdded; + + /* + * finalize the "unfold" data because we need to use it to add closure mappings + * for situations like FB05->"st"<-FB06 + * where we would otherwise miss the FB05<->FB06 relationship + */ + makeUnfoldData(); + + /* use the "unfold" data to add mappings */ + + /* p always points to the code points; this loop ignores the strings completely */ + p=unfold+UGENCASE_UNFOLD_WIDTH+UGENCASE_UNFOLD_STRING_WIDTH; + + for(i=0; i<unfoldRows; p+=UGENCASE_UNFOLD_WIDTH, ++i) { + j=0; + U16_NEXT_UNSAFE(p, j, c); + while(j<UGENCASE_UNFOLD_CP_WIDTH && p[j]!=0) { + U16_NEXT_UNSAFE(p, j, c2); + addClosure(c, U_SENTINEL, c, c2, 0); + } + } + + if(beVerbose) { + puts("---- ---- ---- ---- (done with closures from unfolding)"); + } + + /* add further closure mappings from analyzing simple mappings */ + do { + someMappingsAdded=FALSE; + + i=0; + while((row=upvec_getRow(pv, i, &start, &limit))!=NULL) { + value=*row; + if(value!=0) { + while(start<limit) { + if(addClosure(start, U_SENTINEL, U_SENTINEL, start, value)) { + someMappingsAdded=TRUE; + + /* + * stop this loop because pv was changed and row is not valid any more + * skip all rows below the current start + */ + while((row=upvec_getRow(pv, i, NULL, &limit))!=NULL && start>=limit) { + ++i; + } + row=NULL; /* signal to continue with outer loop, without further ++i */ + break; + } + ++start; + } + if(row==NULL) { + continue; /* see row=NULL above */ + } + } + ++i; + } + + if(beVerbose && someMappingsAdded) { + puts("---- ---- ---- ----"); + } + } while(someMappingsAdded); +} + +/* exceptions --------------------------------------------------------------- */ + +/* get the string length from zero-terminated code points in a limited-length array */ +static int32_t +getLengthOfCodePoints(const UChar32 *s, int32_t maxLength) { + int32_t i, length; + + for(i=length=0; i<maxLength && s[i]!=0; ++i) { + length+=U16_LENGTH(s[i]); + } + return length; +} + +static UBool +fullMappingEqualsSimple(const UChar *s, UChar32 simple, UChar32 c) { + int32_t i, length; + UChar32 full; + + length=*s++; + if(length==0 || length>U16_MAX_LENGTH) { + return FALSE; + } + i=0; + U16_NEXT(s, i, length, full); + + if(simple==0) { + simple=c; /* UCD has no simple mapping if it's the same as the code point itself */ + } + return (UBool)(i==length && full==simple); +} + +static uint16_t +makeException(uint32_t value, Props *p) { + uint32_t slots[8]; + uint32_t slotBits; + uint16_t excWord, excIndex, excTop, i, count, length, fullLengths; + UBool doubleSlots; + + /* excIndex will be returned for storing in the trie word */ + excIndex=exceptionsTop; + if(excIndex>=UCASE_MAX_EXCEPTIONS) { + fprintf(stderr, "gencase error: too many exceptions words\n"); + exit(U_BUFFER_OVERFLOW_ERROR); + } + + excTop=excIndex+1; /* +1 for excWord which will be stored at excIndex */ + + /* copy and shift the soft-dotted bits */ + excWord=((uint16_t)value&UCASE_DOT_MASK)<<UCASE_EXC_DOT_SHIFT; + + /* update maxFullLength */ + if(p->specialCasing!=NULL) { + length=p->specialCasing->lowerCase[0]; + if(length>maxFullLength) { + maxFullLength=length; + } + length=p->specialCasing->upperCase[0]; + if(length>maxFullLength) { + maxFullLength=length; + } + length=p->specialCasing->titleCase[0]; + if(length>maxFullLength) { + maxFullLength=length; + } + } + if(p->caseFolding!=NULL) { + length=p->caseFolding->full[0]; + if(length>maxFullLength) { + maxFullLength=length; + } + } + + /* set the bits for conditional mappings */ + if(p->specialCasing!=NULL && p->specialCasing->isComplex) { + excWord|=UCASE_EXC_CONDITIONAL_SPECIAL; + p->specialCasing=NULL; + } + if(p->caseFolding!=NULL && p->caseFolding->simple==0 && p->caseFolding->full[0]==0) { + excWord|=UCASE_EXC_CONDITIONAL_FOLD; + p->caseFolding=NULL; + } + + /* + * Note: + * UCD stores no simple mappings when they are the same as the code point itself. + * SpecialCasing and CaseFolding do store simple mappings even if they are + * the same as the code point itself. + * Comparisons between simple regular mappings and simple special/folding + * mappings need to compensate for the difference by comparing with the + * original code point if a simple UCD mapping is missing (0). + */ + + /* remove redundant data */ + if(p->specialCasing!=NULL) { + /* do not store full mappings if they are the same as the simple ones */ + if(fullMappingEqualsSimple(p->specialCasing->lowerCase, p->lowerCase, p->code)) { + p->specialCasing->lowerCase[0]=0; + } + if(fullMappingEqualsSimple(p->specialCasing->upperCase, p->upperCase, p->code)) { + p->specialCasing->upperCase[0]=0; + } + if(fullMappingEqualsSimple(p->specialCasing->titleCase, p->titleCase, p->code)) { + p->specialCasing->titleCase[0]=0; + } + } + if( p->caseFolding!=NULL && + fullMappingEqualsSimple(p->caseFolding->full, p->caseFolding->simple, p->code) + ) { + p->caseFolding->full[0]=0; + } + + /* write the optional slots */ + slotBits=0; + count=0; + + if(p->lowerCase!=0) { + slots[count]=(uint32_t)p->lowerCase; + slotBits|=slots[count]; + ++count; + excWord|=U_MASK(UCASE_EXC_LOWER); + } + if( p->caseFolding!=NULL && + p->caseFolding->simple!=0 && + (p->lowerCase!=0 ? + p->caseFolding->simple!=p->lowerCase : + p->caseFolding->simple!=p->code) + ) { + slots[count]=(uint32_t)p->caseFolding->simple; + slotBits|=slots[count]; + ++count; + excWord|=U_MASK(UCASE_EXC_FOLD); + } + if(p->upperCase!=0) { + slots[count]=(uint32_t)p->upperCase; + slotBits|=slots[count]; + ++count; + excWord|=U_MASK(UCASE_EXC_UPPER); + } + if(p->upperCase!=p->titleCase) { + if(p->titleCase!=0) { + slots[count]=(uint32_t)p->titleCase; + } else { + slots[count]=(uint32_t)p->code; + } + slotBits|=slots[count]; + ++count; + excWord|=U_MASK(UCASE_EXC_TITLE); + } + + /* length of case closure */ + if(p->closure[0]!=0) { + length=getLengthOfCodePoints(p->closure, LENGTHOF(p->closure)); + slots[count]=(uint32_t)length; /* must be 1..UCASE_CLOSURE_MAX_LENGTH */ + slotBits|=slots[count]; + ++count; + excWord|=U_MASK(UCASE_EXC_CLOSURE); + } + + /* lengths of full case mapping strings, stored in the last slot */ + fullLengths=0; + if(p->specialCasing!=NULL) { + fullLengths=p->specialCasing->lowerCase[0]; + fullLengths|=p->specialCasing->upperCase[0]<<8; + fullLengths|=p->specialCasing->titleCase[0]<<12; + } + if(p->caseFolding!=NULL) { + fullLengths|=p->caseFolding->full[0]<<4; + } + if(fullLengths!=0) { + slots[count]=fullLengths; + slotBits|=slots[count]; + ++count; + excWord|=U_MASK(UCASE_EXC_FULL_MAPPINGS); + } + + /* write slots */ + doubleSlots=(UBool)(slotBits>0xffff); + if(!doubleSlots) { + for(i=0; i<count; ++i) { + exceptions[excTop++]=(uint16_t)slots[i]; + } + } else { + excWord|=UCASE_EXC_DOUBLE_SLOTS; + for(i=0; i<count; ++i) { + exceptions[excTop++]=(uint16_t)(slots[i]>>16); + exceptions[excTop++]=(uint16_t)slots[i]; + } + } + + /* write the full case mapping strings */ + if(p->specialCasing!=NULL) { + length=(uint16_t)p->specialCasing->lowerCase[0]; + u_memcpy((UChar *)exceptions+excTop, p->specialCasing->lowerCase+1, length); + excTop+=length; + } + if(p->caseFolding!=NULL) { + length=(uint16_t)p->caseFolding->full[0]; + u_memcpy((UChar *)exceptions+excTop, p->caseFolding->full+1, length); + excTop+=length; + } + if(p->specialCasing!=NULL) { + length=(uint16_t)p->specialCasing->upperCase[0]; + u_memcpy((UChar *)exceptions+excTop, p->specialCasing->upperCase+1, length); + excTop+=length; + + length=(uint16_t)p->specialCasing->titleCase[0]; + u_memcpy((UChar *)exceptions+excTop, p->specialCasing->titleCase+1, length); + excTop+=length; + } + + /* write the closure data */ + if(p->closure[0]!=0) { + UChar32 c; + + for(i=0; i<LENGTHOF(p->closure) && (c=p->closure[i])!=0; ++i) { + U16_APPEND_UNSAFE((UChar *)exceptions, excTop, c); + } + } + + exceptionsTop=excTop; + + /* write the main exceptions word */ + exceptions[excIndex]=excWord; + + return excIndex; +} + +extern void +makeExceptions() { + uint32_t *row; + uint32_t value; + int32_t i; + uint16_t excIndex; + + i=0; + while((row=upvec_getRow(pv, i, NULL, NULL))!=NULL) { + value=*row; + if(value&UCASE_EXCEPTION) { + excIndex=makeException(value, excProps+(value>>UGENCASE_EXC_SHIFT)); + *row=(value&~(UGENCASE_EXC_MASK|UCASE_EXC_MASK))|(excIndex<<UCASE_EXC_SHIFT); + } + ++i; + } +} + +/* generate output data ----------------------------------------------------- */ + +extern void +generateData(const char *dataDir, UBool csource) { + static int32_t indexes[UCASE_IX_TOP]={ + UCASE_IX_TOP + }; + static uint8_t trieBlock[40000]; + + const uint32_t *row; + UChar32 start, limit; + int32_t i; + + UNewDataMemory *pData; + UNewTrie *pTrie; + UErrorCode errorCode=U_ZERO_ERROR; + int32_t trieSize; + long dataLength; + + pTrie=utrie_open(NULL, NULL, 20000, 0, 0, TRUE); + if(pTrie==NULL) { + fprintf(stderr, "gencase error: unable to create a UNewTrie\n"); + exit(U_MEMORY_ALLOCATION_ERROR); + } + + for(i=0; (row=upvec_getRow(pv, i, &start, &limit))!=NULL; ++i) { + if(!utrie_setRange32(pTrie, start, limit, *row, TRUE)) { + fprintf(stderr, "gencase error: unable to set trie value (overflow)\n"); + exit(U_BUFFER_OVERFLOW_ERROR); + } + } + + trieSize=utrie_serialize(pTrie, trieBlock, sizeof(trieBlock), NULL, TRUE, &errorCode); + if(U_FAILURE(errorCode)) { + fprintf(stderr, "error: utrie_serialize failed: %s (length %ld)\n", u_errorName(errorCode), (long)trieSize); + exit(errorCode); + } + + indexes[UCASE_IX_EXC_LENGTH]=exceptionsTop; + indexes[UCASE_IX_TRIE_SIZE]=trieSize; + indexes[UCASE_IX_UNFOLD_LENGTH]=unfoldTop; + indexes[UCASE_IX_LENGTH]=(int32_t)sizeof(indexes)+trieSize+2*exceptionsTop+2*unfoldTop; + + indexes[UCASE_IX_MAX_FULL_LENGTH]=maxFullLength; + + if(beVerbose) { + printf("trie size in bytes: %5d\n", (int)trieSize); + printf("number of code points with exceptions: %5d\n", exceptionsCount); + printf("size in bytes of exceptions: %5d\n", 2*exceptionsTop); + printf("size in bytes of reverse foldings: %5d\n", 2*unfoldTop); + printf("data size: %5d\n", (int)indexes[UCASE_IX_LENGTH]); + } + + if(csource) { + /* write .c file for hardcoded data */ + UTrie trie={ NULL }; + FILE *f; + + utrie_unserialize(&trie, trieBlock, trieSize, &errorCode); + if(U_FAILURE(errorCode)) { + fprintf( + stderr, + "gencase error: failed to utrie_unserialize(ucase.icu trie) - %s\n", + u_errorName(errorCode)); + return; + } + + f=usrc_create(dataDir, "ucase_props_data.c"); + if(f!=NULL) { + usrc_writeArray(f, + "static const UVersionInfo ucase_props_dataVersion={", + dataInfo.dataVersion, 8, 4, + "};\n\n"); + usrc_writeArray(f, + "static const int32_t ucase_props_indexes[UCASE_IX_TOP]={", + indexes, 32, UCASE_IX_TOP, + "};\n\n"); + usrc_writeUTrieArrays(f, + "static const uint16_t ucase_props_trieIndex[%ld]={\n", NULL, + &trie, + "\n};\n\n"); + usrc_writeArray(f, + "static const uint16_t ucase_props_exceptions[%ld]={\n", + exceptions, 16, exceptionsTop, + "\n};\n\n"); + usrc_writeArray(f, + "static const uint16_t ucase_props_unfold[%ld]={\n", + unfold, 16, unfoldTop, + "\n};\n\n"); + fputs( + "static const UCaseProps ucase_props_singleton={\n" + " NULL,\n" + " ucase_props_indexes,\n" + " ucase_props_exceptions,\n" + " ucase_props_unfold,\n", + f); + usrc_writeUTrieStruct(f, + " {\n", + &trie, "ucase_props_trieIndex", NULL, NULL, + " },\n"); + usrc_writeArray(f, " { ", dataInfo.formatVersion, 8, 4, " }\n"); + fputs("};\n", f); + fclose(f); + } + } else { + /* write the data */ + pData=udata_create(dataDir, UCASE_DATA_TYPE, UCASE_DATA_NAME, &dataInfo, + haveCopyright ? U_COPYRIGHT_STRING : NULL, &errorCode); + if(U_FAILURE(errorCode)) { + fprintf(stderr, "gencase: unable to create data memory, %s\n", u_errorName(errorCode)); + exit(errorCode); + } + + udata_writeBlock(pData, indexes, sizeof(indexes)); + udata_writeBlock(pData, trieBlock, trieSize); + udata_writeBlock(pData, exceptions, 2*exceptionsTop); + udata_writeBlock(pData, unfold, 2*unfoldTop); + + /* finish up */ + dataLength=udata_finish(pData, &errorCode); + if(U_FAILURE(errorCode)) { + fprintf(stderr, "gencase: error %d writing the output file\n", errorCode); + exit(errorCode); + } + + if(dataLength!=indexes[UCASE_IX_LENGTH]) { + fprintf(stderr, "gencase: data length %ld != calculated size %d\n", + dataLength, (int)indexes[UCASE_IX_LENGTH]); + exit(U_INTERNAL_PROGRAM_ERROR); + } + } + + utrie_close(pTrie); +} + +/* + * Hey, Emacs, please set the following: + * + * Local Variables: + * indent-tabs-mode: nil + * End: + * + */ |