summaryrefslogtreecommitdiff
path: root/Build/source/libs/icu/icu-xetex/tools/gennorm/store.c
diff options
context:
space:
mode:
Diffstat (limited to 'Build/source/libs/icu/icu-xetex/tools/gennorm/store.c')
-rw-r--r--Build/source/libs/icu/icu-xetex/tools/gennorm/store.c2107
1 files changed, 2107 insertions, 0 deletions
diff --git a/Build/source/libs/icu/icu-xetex/tools/gennorm/store.c b/Build/source/libs/icu/icu-xetex/tools/gennorm/store.c
new file mode 100644
index 00000000000..e0e4051b9fe
--- /dev/null
+++ b/Build/source/libs/icu/icu-xetex/tools/gennorm/store.c
@@ -0,0 +1,2107 @@
+/*
+*******************************************************************************
+*
+* Copyright (C) 1999-2006, International Business Machines
+* Corporation and others. All Rights Reserved.
+*
+*******************************************************************************
+* file name: store.c
+* encoding: US-ASCII
+* tab size: 8 (not used)
+* indentation:4
+*
+* created on: 2001may25
+* created by: Markus W. Scherer
+*
+* Store Unicode normalization data in a memory-mappable file.
+*/
+
+#include <stdio.h>
+#include <stdlib.h>
+#include "unicode/utypes.h"
+#include "unicode/uchar.h"
+#include "unicode/ustring.h"
+#include "cmemory.h"
+#include "cstring.h"
+#include "filestrm.h"
+#include "unicode/udata.h"
+#include "utrie.h"
+#include "unicode/uset.h"
+#include "toolutil.h"
+#include "unewdata.h"
+#include "writesrc.h"
+#include "unormimp.h"
+#include "gennorm.h"
+
+#define DO_DEBUG_OUT 0
+
+#define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0]))
+
+/*
+ * The new implementation of the normalization code loads its data from
+ * unorm.icu, which is generated with this gennorm tool.
+ * The format of that file is described in unormimp.h .
+ */
+
+/* file data ---------------------------------------------------------------- */
+
+#if UCONFIG_NO_NORMALIZATION
+
+/* dummy UDataInfo cf. udata.h */
+static UDataInfo dataInfo = {
+ sizeof(UDataInfo),
+ 0,
+
+ U_IS_BIG_ENDIAN,
+ U_CHARSET_FAMILY,
+ U_SIZEOF_UCHAR,
+ 0,
+
+ { 0, 0, 0, 0 }, /* dummy dataFormat */
+ { 0, 0, 0, 0 }, /* dummy formatVersion */
+ { 0, 0, 0, 0 } /* dummy dataVersion */
+};
+
+#else
+
+/* UDataInfo cf. udata.h */
+static UDataInfo dataInfo={
+ sizeof(UDataInfo),
+ 0,
+
+ U_IS_BIG_ENDIAN,
+ U_CHARSET_FAMILY,
+ U_SIZEOF_UCHAR,
+ 0,
+
+ { 0x4e, 0x6f, 0x72, 0x6d }, /* dataFormat="Norm" */
+ { 2, 3, UTRIE_SHIFT, UTRIE_INDEX_SHIFT }, /* formatVersion */
+ { 3, 2, 0, 0 } /* dataVersion (Unicode version) */
+};
+
+extern void
+setUnicodeVersion(const char *v) {
+ UVersionInfo version;
+ u_versionFromString(version, v);
+ uprv_memcpy(dataInfo.dataVersion, version, 4);
+}
+
+static int32_t indexes[_NORM_INDEX_TOP]={ 0 };
+
+/* builder data ------------------------------------------------------------- */
+
+/* modularization flags, see gennorm.h (default to "store everything") */
+uint32_t gStoreFlags=0xffffffff;
+
+typedef void EnumTrieFn(void *context, uint32_t code, Norm *norm);
+
+static UNewTrie
+ *normTrie,
+ *norm32Trie,
+ *fcdTrie,
+ *auxTrie;
+
+static UToolMemory *normMem, *utf32Mem, *extraMem, *combiningTriplesMem;
+
+static Norm *norms;
+
+/*
+ * set a flag for each code point that was seen in decompositions -
+ * avoid to decompose ones that have not been used before
+ */
+static uint32_t haveSeenFlags[256];
+
+/* set of characters with NFD_QC=No (i.e., those with canonical decompositions) */
+static USet *nfdQCNoSet;
+
+/* see addCombiningCP() for details */
+static uint32_t combiningCPs[2000];
+
+/*
+ * after processCombining() this contains for each code point in combiningCPs[]
+ * the runtime combining index
+ */
+static uint16_t combiningIndexes[2000];
+
+/* section limits for combiningCPs[], see addCombiningCP() */
+static uint16_t combineFwdTop=0, combineBothTop=0, combineBackTop=0;
+
+/**
+ * Structure for a triple of code points, stored in combiningTriplesMem.
+ * The lead and trail code points combine into the the combined one,
+ * i.e., there is a canonical decomposition of combined-> <lead, trail>.
+ *
+ * Before processCombining() is called, leadIndex and trailIndex are 0.
+ * After processCombining(), they contain the indexes of the lead and trail
+ * code point in the combiningCPs[] array.
+ * They are then sorted by leadIndex, then trailIndex.
+ * They are not sorted by code points.
+ */
+typedef struct CombiningTriple {
+ uint16_t leadIndex, trailIndex;
+ uint32_t lead, trail, combined;
+} CombiningTriple;
+
+/* 15b in the combining index -> <=0x8000 uint16_t values in the combining table */
+static uint16_t combiningTable[0x8000];
+static uint16_t combiningTableTop=0;
+
+#define _NORM_MAX_SET_SEARCH_TABLE_LENGTH 0x4000
+static uint16_t canonStartSets[_NORM_MAX_CANON_SETS+2*_NORM_MAX_SET_SEARCH_TABLE_LENGTH
+ +10000]; /* +10000 for exclusion sets */
+static int32_t canonStartSetsTop=_NORM_SET_INDEX_TOP;
+static int32_t canonSetsCount=0;
+
+/* allocate and initialize a Norm unit */
+static Norm *
+allocNorm() {
+ /* allocate Norm */
+ Norm *p=(Norm *)utm_alloc(normMem);
+ /*
+ * The combiningIndex must not be initialized to 0 because 0 is the
+ * combiningIndex of the first forward-combining character.
+ */
+ p->combiningIndex=0xffff;
+ return p;
+}
+
+extern void
+init() {
+ uint16_t *p16;
+
+ normTrie = (UNewTrie *)uprv_malloc(sizeof(UNewTrie));
+ uprv_memset(normTrie, 0, sizeof(UNewTrie));
+ norm32Trie = (UNewTrie *)uprv_malloc(sizeof(UNewTrie));
+ uprv_memset(norm32Trie, 0, sizeof(UNewTrie));
+ fcdTrie = (UNewTrie *)uprv_malloc(sizeof(UNewTrie));
+ uprv_memset(fcdTrie, 0, sizeof(UNewTrie));
+ auxTrie = (UNewTrie *)uprv_malloc(sizeof(UNewTrie));
+ uprv_memset(auxTrie, 0, sizeof(UNewTrie));
+
+ /* initialize the two tries */
+ if(NULL==utrie_open(normTrie, NULL, 30000, 0, 0, FALSE)) {
+ fprintf(stderr, "error: failed to initialize tries\n");
+ exit(U_MEMORY_ALLOCATION_ERROR);
+ }
+
+ /* allocate Norm structures and reset the first one */
+ normMem=utm_open("gennorm normalization structs", 20000, 20000, sizeof(Norm));
+ norms=allocNorm();
+
+ /* allocate UTF-32 string memory */
+ utf32Mem=utm_open("gennorm UTF-32 strings", 30000, 30000, 4);
+
+ /* reset all "have seen" flags */
+ uprv_memset(haveSeenFlags, 0, sizeof(haveSeenFlags));
+
+ /* open an empty set */
+ nfdQCNoSet=uset_open(1, 0);
+
+ /* allocate extra data memory for UTF-16 decomposition strings and other values */
+ extraMem=utm_open("gennorm extra 16-bit memory", _NORM_EXTRA_INDEX_TOP, _NORM_EXTRA_INDEX_TOP, 2);
+ /* initialize the extraMem counter for the top of FNC strings */
+ p16=(uint16_t *)utm_alloc(extraMem);
+ *p16=1;
+
+ /* allocate temporary memory for combining triples */
+ combiningTriplesMem=utm_open("gennorm combining triples", 0x4000, 0x4000, sizeof(CombiningTriple));
+
+ /* set the minimum code points for no/maybe quick check values to the end of the BMP */
+ indexes[_NORM_INDEX_MIN_NFC_NO_MAYBE]=0xffff;
+ indexes[_NORM_INDEX_MIN_NFKC_NO_MAYBE]=0xffff;
+ indexes[_NORM_INDEX_MIN_NFD_NO_MAYBE]=0xffff;
+ indexes[_NORM_INDEX_MIN_NFKD_NO_MAYBE]=0xffff;
+
+ /* preset the indexes portion of canonStartSets */
+ uprv_memset(canonStartSets, 0, _NORM_SET_INDEX_TOP*2);
+}
+
+/*
+ * get or create a Norm unit;
+ * get or create the intermediate trie entries for it as well
+ */
+static Norm *
+createNorm(uint32_t code) {
+ Norm *p;
+ uint32_t i;
+
+ i=utrie_get32(normTrie, (UChar32)code, NULL);
+ if(i!=0) {
+ p=norms+i;
+ } else {
+ /* allocate Norm */
+ p=allocNorm();
+ if(!utrie_set32(normTrie, (UChar32)code, (uint32_t)(p-norms))) {
+ fprintf(stderr, "error: too many normalization entries\n");
+ exit(U_BUFFER_OVERFLOW_ERROR);
+ }
+ }
+ return p;
+}
+
+/* get an existing Norm unit */
+static Norm *
+getNorm(uint32_t code) {
+ uint32_t i;
+
+ i=utrie_get32(normTrie, (UChar32)code, NULL);
+ if(i==0) {
+ return NULL;
+ }
+ return norms+i;
+}
+
+/* get the canonical combining class of a character */
+static uint8_t
+getCCFromCP(uint32_t code) {
+ Norm *norm=getNorm(code);
+ if(norm==NULL) {
+ return 0;
+ } else {
+ return norm->udataCC;
+ }
+}
+
+/*
+ * enumerate all code points with their Norm structs and call a function for each
+ * return the number of code points with data
+ */
+static uint32_t
+enumTrie(EnumTrieFn *fn, void *context) {
+ uint32_t count, i;
+ UChar32 code;
+ UBool isInBlockZero;
+
+ count=0;
+ for(code=0; code<=0x10ffff;) {
+ i=utrie_get32(normTrie, code, &isInBlockZero);
+ if(isInBlockZero) {
+ code+=UTRIE_DATA_BLOCK_LENGTH;
+ } else {
+ if(i!=0) {
+ fn(context, (uint32_t)code, norms+i);
+ ++count;
+ }
+ ++code;
+ }
+ }
+ return count;
+}
+
+static void
+setHaveSeenString(const uint32_t *s, int32_t length) {
+ uint32_t c;
+
+ while(length>0) {
+ c=*s++;
+ haveSeenFlags[(c>>5)&0xff]|=(1<<(c&0x1f));
+ --length;
+ }
+}
+
+#define HAVE_SEEN(c) (haveSeenFlags[((c)>>5)&0xff]&(1<<((c)&0x1f)))
+
+/* handle combining data ---------------------------------------------------- */
+
+/*
+ * Insert an entry into combiningCPs[] for the new code point code with its flags.
+ * The flags indicate if code combines forward, backward, or both.
+ *
+ * combiningCPs[] contains three sections:
+ * 1. code points that combine forward
+ * 2. code points that combine forward and backward
+ * 3. code points that combine backward
+ *
+ * Search for code in the entire array.
+ * If it is found and already is in the right section (old flags==new flags)
+ * then we are done.
+ * If it is found but the flags are different, then remove it,
+ * union the old and new flags, and reinsert it into its correct section.
+ * If it is not found, then just insert it.
+ *
+ * Within each section, the code points are not sorted.
+ */
+static void
+addCombiningCP(uint32_t code, uint8_t flags) {
+ uint32_t newEntry;
+ uint16_t i;
+
+ newEntry=code|((uint32_t)flags<<24);
+
+ /* search for this code point */
+ for(i=0; i<combineBackTop; ++i) {
+ if(code==(combiningCPs[i]&0xffffff)) {
+ /* found it */
+ if(newEntry==combiningCPs[i]) {
+ return; /* no change */
+ }
+
+ /* combine the flags, remove the old entry from the old place, and insert the new one */
+ newEntry|=combiningCPs[i];
+ if(i!=--combineBackTop) {
+ uprv_memmove(combiningCPs+i, combiningCPs+i+1, (combineBackTop-i)*4);
+ }
+ if(i<combineBothTop) {
+ --combineBothTop;
+ }
+ if(i<combineFwdTop) {
+ --combineFwdTop;
+ }
+ break;
+ }
+ }
+
+ /* not found or modified, insert it */
+ if(combineBackTop>=sizeof(combiningCPs)/4) {
+ fprintf(stderr, "error: gennorm combining code points - trying to use more than %ld units\n",
+ (long)(sizeof(combiningCPs)/4));
+ exit(U_MEMORY_ALLOCATION_ERROR);
+ }
+
+ /* set i to the insertion point */
+ flags=(uint8_t)(newEntry>>24);
+ if(flags==1) {
+ i=combineFwdTop++;
+ ++combineBothTop;
+ } else if(flags==3) {
+ i=combineBothTop++;
+ } else /* flags==2 */ {
+ i=combineBackTop;
+ }
+
+ /* move the following code points up one and insert newEntry at i */
+ if(i<combineBackTop) {
+ uprv_memmove(combiningCPs+i+1, combiningCPs+i, (combineBackTop-i)*4);
+ }
+ combiningCPs[i]=newEntry;
+
+ /* finally increment the total counter */
+ ++combineBackTop;
+}
+
+/**
+ * Find the index in combiningCPs[] where code point code is stored.
+ * @param code code point to look for
+ * @param isLead is code a forward combining code point?
+ * @return index in combiningCPs[] where code is stored
+ */
+static uint16_t
+findCombiningCP(uint32_t code, UBool isLead) {
+ uint16_t i, limit;
+
+ if(isLead) {
+ i=0;
+ limit=combineBothTop;
+ } else {
+ i=combineFwdTop;
+ limit=combineBackTop;
+ }
+
+ /* search for this code point */
+ for(; i<limit; ++i) {
+ if(code==(combiningCPs[i]&0xffffff)) {
+ /* found it */
+ return i;
+ }
+ }
+
+ /* not found */
+ return 0xffff;
+}
+
+static void
+addCombiningTriple(uint32_t lead, uint32_t trail, uint32_t combined) {
+ CombiningTriple *triple;
+
+ if(DO_NOT_STORE(UGENNORM_STORE_COMPOSITION)) {
+ return;
+ }
+
+ /*
+ * set combiningFlags for the two code points
+ * do this after decomposition so that getNorm() above returns NULL
+ * if we do not have actual sub-decomposition data for the initial NFD here
+ */
+ createNorm(lead)->combiningFlags|=1; /* combines forward */
+ createNorm(trail)->combiningFlags|=2; /* combines backward */
+
+ addCombiningCP(lead, 1);
+ addCombiningCP(trail, 2);
+
+ triple=(CombiningTriple *)utm_alloc(combiningTriplesMem);
+ triple->lead=lead;
+ triple->trail=trail;
+ triple->combined=combined;
+}
+
+static int
+compareTriples(const void *l, const void *r) {
+ int diff;
+ diff=(int)((CombiningTriple *)l)->leadIndex-
+ (int)((CombiningTriple *)r)->leadIndex;
+ if(diff==0) {
+ diff=(int)((CombiningTriple *)l)->trailIndex-
+ (int)((CombiningTriple *)r)->trailIndex;
+ }
+ return diff;
+}
+
+static void
+processCombining() {
+ CombiningTriple *triples;
+ uint16_t *p;
+ uint32_t combined;
+ uint16_t i, j, count, tableTop, finalIndex, combinesFwd;
+
+ triples=utm_getStart(combiningTriplesMem);
+
+ /* add lead and trail indexes to the triples for sorting */
+ count=(uint16_t)utm_countItems(combiningTriplesMem);
+ for(i=0; i<count; ++i) {
+ /* findCombiningCP() must always find the code point */
+ triples[i].leadIndex=findCombiningCP(triples[i].lead, TRUE);
+ triples[i].trailIndex=findCombiningCP(triples[i].trail, FALSE);
+ }
+
+ /* sort them by leadIndex, trailIndex */
+ qsort(triples, count, sizeof(CombiningTriple), compareTriples);
+
+ /* calculate final combining indexes and store them in the Norm entries */
+ tableTop=0;
+ j=0; /* triples counter */
+
+ /* first, combining indexes of fwd/both characters are indexes into the combiningTable */
+ for(i=0; i<combineBothTop; ++i) {
+ /* start a new table */
+
+ /* assign combining index */
+ createNorm(combiningCPs[i]&0xffffff)->combiningIndex=combiningIndexes[i]=tableTop;
+
+ /* calculate the length of the combining data for this lead code point in the combiningTable */
+ while(j<count && i==triples[j].leadIndex) {
+ /* count 2 to 3 16-bit units per composition entry (back-index, code point) */
+ combined=triples[j++].combined;
+ if(combined<=0x1fff) {
+ tableTop+=2;
+ } else {
+ tableTop+=3;
+ }
+ }
+ }
+
+ /* second, combining indexes of back-only characters are simply incremented from here to be unique */
+ finalIndex=tableTop;
+ for(; i<combineBackTop; ++i) {
+ createNorm(combiningCPs[i]&0xffffff)->combiningIndex=combiningIndexes[i]=finalIndex++;
+ }
+
+ /* it must be finalIndex<=0x8000 because bit 15 is used in combiningTable as an end-for-this-lead marker */
+ if(finalIndex>0x8000) {
+ fprintf(stderr, "error: gennorm combining table - trying to use %u units, more than the %ld units available\n",
+ tableTop, (long)(sizeof(combiningTable)/4));
+ exit(U_MEMORY_ALLOCATION_ERROR);
+ }
+
+ combiningTableTop=tableTop;
+
+ /* store the combining data in the combiningTable, with the final indexes from above */
+ p=combiningTable;
+ j=0; /* triples counter */
+
+ /*
+ * this is essentially the same loop as above, but
+ * it writes the table data instead of calculating and setting the final indexes;
+ * it is necessary to have two passes so that all the final indexes are known before
+ * they are written into the table
+ */
+ for(i=0; i<combineBothTop; ++i) {
+ /* start a new table */
+
+ combined=0; /* avoid compiler warning */
+
+ /* store the combining data for this lead code point in the combiningTable */
+ while(j<count && i==triples[j].leadIndex) {
+ finalIndex=combiningIndexes[triples[j].trailIndex];
+ combined=triples[j++].combined;
+
+ /* is combined a starter? (i.e., cc==0 && combines forward) */
+ combinesFwd=(uint16_t)((getNorm(combined)->combiningFlags&1)<<13);
+
+ *p++=finalIndex;
+ if(combined<=0x1fff) {
+ *p++=(uint16_t)(combinesFwd|combined);
+ } else if(combined<=0xffff) {
+ *p++=(uint16_t)(0x8000|combinesFwd);
+ *p++=(uint16_t)combined;
+ } else {
+ *p++=(uint16_t)(0xc000|combinesFwd|((combined-0x10000)>>10));
+ *p++=(uint16_t)(0xdc00|(combined&0x3ff));
+ }
+ }
+
+ /* set a marker on the last final trail index in this lead's table */
+ if(combined<=0x1fff) {
+ *(p-2)|=0x8000;
+ } else {
+ *(p-3)|=0x8000;
+ }
+ }
+
+ /* post condition: tableTop==(p-combiningTable) */
+}
+
+/* processing incoming normalization data ----------------------------------- */
+
+/*
+ * Decompose Hangul syllables algorithmically and fill a pseudo-Norm struct.
+ * c must be a Hangul syllable code point.
+ */
+static void
+getHangulDecomposition(uint32_t c, Norm *pHangulNorm, uint32_t hangulBuffer[3]) {
+ /* Hangul syllable: decompose algorithmically */
+ uint32_t c2;
+ uint8_t length;
+
+ uprv_memset(pHangulNorm, 0, sizeof(Norm));
+
+ c-=HANGUL_BASE;
+
+ c2=c%JAMO_T_COUNT;
+ c/=JAMO_T_COUNT;
+ if(c2>0) {
+ hangulBuffer[2]=JAMO_T_BASE+c2;
+ length=3;
+ } else {
+ hangulBuffer[2]=0;
+ length=2;
+ }
+
+ hangulBuffer[1]=JAMO_V_BASE+c%JAMO_V_COUNT;
+ hangulBuffer[0]=JAMO_L_BASE+c/JAMO_V_COUNT;
+
+ pHangulNorm->nfd=hangulBuffer;
+ pHangulNorm->lenNFD=length;
+ if(DO_STORE(UGENNORM_STORE_COMPAT)) {
+ pHangulNorm->nfkd=hangulBuffer;
+ pHangulNorm->lenNFKD=length;
+ }
+}
+
+/*
+ * decompose the one decomposition further, may generate two decompositions
+ * apply all previous characters' decompositions to this one
+ */
+static void
+decompStoreNewNF(uint32_t code, Norm *norm) {
+ uint32_t nfd[40], nfkd[40], hangulBuffer[3];
+ Norm hangulNorm;
+
+ uint32_t *s32;
+ Norm *p;
+ uint32_t c;
+ int32_t i, length;
+ uint8_t lenNFD=0, lenNFKD=0;
+ UBool changedNFD=FALSE, changedNFKD=FALSE;
+
+ if((length=norm->lenNFD)!=0) {
+ /* always allocate the original string */
+ changedNFD=TRUE;
+ s32=norm->nfd;
+ } else if((length=norm->lenNFKD)!=0) {
+ /* always allocate the original string */
+ changedNFKD=TRUE;
+ s32=norm->nfkd;
+ } else {
+ /* no decomposition here, nothing to do */
+ return;
+ }
+
+ /* decompose each code point */
+ for(i=0; i<length; ++i) {
+ c=s32[i];
+ p=getNorm(c);
+ if(p==NULL) {
+ if(HANGUL_BASE<=c && c<(HANGUL_BASE+HANGUL_COUNT)) {
+ getHangulDecomposition(c, &hangulNorm, hangulBuffer);
+ p=&hangulNorm;
+ } else {
+ /* no data, no decomposition */
+ nfd[lenNFD++]=c;
+ nfkd[lenNFKD++]=c;
+ continue;
+ }
+ }
+
+ /* canonically decompose c */
+ if(changedNFD) {
+ if(p->lenNFD!=0) {
+ uprv_memcpy(nfd+lenNFD, p->nfd, p->lenNFD*4);
+ lenNFD+=p->lenNFD;
+ } else {
+ nfd[lenNFD++]=c;
+ }
+ }
+
+ /* compatibility-decompose c */
+ if(p->lenNFKD!=0) {
+ uprv_memcpy(nfkd+lenNFKD, p->nfkd, p->lenNFKD*4);
+ lenNFKD+=p->lenNFKD;
+ changedNFKD=TRUE;
+ } else if(p->lenNFD!=0) {
+ uprv_memcpy(nfkd+lenNFKD, p->nfd, p->lenNFD*4);
+ lenNFKD+=p->lenNFD;
+ /*
+ * not changedNFKD=TRUE;
+ * so that we do not store a new nfkd if there was no nfkd string before
+ * and we only see canonical decompositions
+ */
+ } else {
+ nfkd[lenNFKD++]=c;
+ }
+ }
+
+ /* assume that norm->lenNFD==1 or ==2 */
+ if(norm->lenNFD==2 && !(norm->combiningFlags&0x80)) {
+ addCombiningTriple(s32[0], s32[1], code);
+ }
+
+ if(changedNFD) {
+ if(lenNFD!=0) {
+ s32=utm_allocN(utf32Mem, lenNFD);
+ uprv_memcpy(s32, nfd, lenNFD*4);
+ } else {
+ s32=NULL;
+ }
+ norm->lenNFD=lenNFD;
+ norm->nfd=s32;
+ setHaveSeenString(nfd, lenNFD);
+ }
+ if(changedNFKD) {
+ if(lenNFKD!=0) {
+ s32=utm_allocN(utf32Mem, lenNFKD);
+ uprv_memcpy(s32, nfkd, lenNFKD*4);
+ } else {
+ s32=NULL;
+ }
+ norm->lenNFKD=lenNFKD;
+ norm->nfkd=s32;
+ setHaveSeenString(nfkd, lenNFKD);
+ }
+}
+
+typedef struct DecompSingle {
+ uint32_t c;
+ Norm *norm;
+} DecompSingle;
+
+/*
+ * apply this one character's decompositions (there is at least one!) to
+ * all previous characters' decompositions to decompose them further
+ */
+static void
+decompWithSingleFn(void *context, uint32_t code, Norm *norm) {
+ uint32_t nfd[40], nfkd[40];
+ uint32_t *s32;
+ DecompSingle *me=(DecompSingle *)context;
+ uint32_t c, myC;
+ int32_t i, length;
+ uint8_t lenNFD=0, lenNFKD=0, myLenNFD, myLenNFKD;
+ UBool changedNFD=FALSE, changedNFKD=FALSE;
+
+ /* get the new character's data */
+ myC=me->c;
+ myLenNFD=me->norm->lenNFD;
+ myLenNFKD=me->norm->lenNFKD;
+ /* assume that myC has at least one decomposition */
+
+ if((length=norm->lenNFD)!=0 && myLenNFD!=0) {
+ /* apply NFD(myC) to norm->nfd */
+ s32=norm->nfd;
+ for(i=0; i<length; ++i) {
+ c=s32[i];
+ if(c==myC) {
+ uprv_memcpy(nfd+lenNFD, me->norm->nfd, myLenNFD*4);
+ lenNFD+=myLenNFD;
+ changedNFD=TRUE;
+ } else {
+ nfd[lenNFD++]=c;
+ }
+ }
+ }
+
+ if((length=norm->lenNFKD)!=0) {
+ /* apply NFD(myC) and NFKD(myC) to norm->nfkd */
+ s32=norm->nfkd;
+ for(i=0; i<length; ++i) {
+ c=s32[i];
+ if(c==myC) {
+ if(myLenNFKD!=0) {
+ uprv_memcpy(nfkd+lenNFKD, me->norm->nfkd, myLenNFKD*4);
+ lenNFKD+=myLenNFKD;
+ } else /* assume myLenNFD!=0 */ {
+ uprv_memcpy(nfkd+lenNFKD, me->norm->nfd, myLenNFD*4);
+ lenNFKD+=myLenNFD;
+ }
+ changedNFKD=TRUE;
+ } else {
+ nfkd[lenNFKD++]=c;
+ }
+ }
+ } else if((length=norm->lenNFD)!=0 && myLenNFKD!=0) {
+ /* apply NFKD(myC) to norm->nfd, forming a new norm->nfkd */
+ s32=norm->nfd;
+ for(i=0; i<length; ++i) {
+ c=s32[i];
+ if(c==myC) {
+ uprv_memcpy(nfkd+lenNFKD, me->norm->nfkd, myLenNFKD*4);
+ lenNFKD+=myLenNFKD;
+ changedNFKD=TRUE;
+ } else {
+ nfkd[lenNFKD++]=c;
+ }
+ }
+ }
+
+ /* set the new decompositions, forget the old ones */
+ if(changedNFD) {
+ if(lenNFD!=0) {
+ if(lenNFD>norm->lenNFD) {
+ s32=utm_allocN(utf32Mem, lenNFD);
+ } else {
+ s32=norm->nfd;
+ }
+ uprv_memcpy(s32, nfd, lenNFD*4);
+ } else {
+ s32=NULL;
+ }
+ norm->lenNFD=lenNFD;
+ norm->nfd=s32;
+ }
+ if(changedNFKD) {
+ if(lenNFKD!=0) {
+ if(lenNFKD>norm->lenNFKD) {
+ s32=utm_allocN(utf32Mem, lenNFKD);
+ } else {
+ s32=norm->nfkd;
+ }
+ uprv_memcpy(s32, nfkd, lenNFKD*4);
+ } else {
+ s32=NULL;
+ }
+ norm->lenNFKD=lenNFKD;
+ norm->nfkd=s32;
+ }
+}
+
+/*
+ * process the data for one code point listed in UnicodeData;
+ * UnicodeData itself never maps a code point to both NFD and NFKD
+ */
+extern void
+storeNorm(uint32_t code, Norm *norm) {
+ DecompSingle decompSingle;
+ Norm *p;
+
+ if(DO_NOT_STORE(UGENNORM_STORE_COMPAT)) {
+ /* ignore compatibility decomposition */
+ norm->lenNFKD=0;
+ }
+
+ /* copy existing derived normalization properties */
+ p=createNorm(code);
+ norm->qcFlags=p->qcFlags;
+ norm->combiningFlags=p->combiningFlags;
+ norm->fncIndex=p->fncIndex;
+
+ /* process the decomposition if there is one here */
+ if((norm->lenNFD|norm->lenNFKD)!=0) {
+ /* decompose this one decomposition further, may generate two decompositions */
+ decompStoreNewNF(code, norm);
+
+ /* has this code point been used in previous decompositions? */
+ if(HAVE_SEEN(code)) {
+ /* use this decomposition to decompose other decompositions further */
+ decompSingle.c=code;
+ decompSingle.norm=norm;
+ enumTrie(decompWithSingleFn, &decompSingle);
+ }
+ }
+
+ /* store the data */
+ uprv_memcpy(p, norm, sizeof(Norm));
+}
+
+extern void
+setQCFlags(uint32_t code, uint8_t qcFlags) {
+ if(DO_NOT_STORE(UGENNORM_STORE_COMPAT)) {
+ /* ignore compatibility decomposition: unset the KC/KD flags */
+ qcFlags&=~(_NORM_QC_NFKC|_NORM_QC_NFKD);
+
+ /* set the KC/KD flags to the same values as the C/D flags */
+ qcFlags|=qcFlags<<1;
+ }
+ if(DO_NOT_STORE(UGENNORM_STORE_COMPOSITION)) {
+ /* ignore composition data: unset the C/KC flags */
+ qcFlags&=~(_NORM_QC_NFC|_NORM_QC_NFKC);
+
+ /* set the C/KC flags to the same values as the D/KD flags */
+ qcFlags|=qcFlags>>2;
+ }
+
+ createNorm(code)->qcFlags|=qcFlags;
+
+ /* adjust the minimum code point for quick check no/maybe */
+ if(code<0xffff) {
+ if((qcFlags&_NORM_QC_NFC) && (uint16_t)code<indexes[_NORM_INDEX_MIN_NFC_NO_MAYBE]) {
+ indexes[_NORM_INDEX_MIN_NFC_NO_MAYBE]=(uint16_t)code;
+ }
+ if((qcFlags&_NORM_QC_NFKC) && (uint16_t)code<indexes[_NORM_INDEX_MIN_NFKC_NO_MAYBE]) {
+ indexes[_NORM_INDEX_MIN_NFKC_NO_MAYBE]=(uint16_t)code;
+ }
+ if((qcFlags&_NORM_QC_NFD) && (uint16_t)code<indexes[_NORM_INDEX_MIN_NFD_NO_MAYBE]) {
+ indexes[_NORM_INDEX_MIN_NFD_NO_MAYBE]=(uint16_t)code;
+ }
+ if((qcFlags&_NORM_QC_NFKD) && (uint16_t)code<indexes[_NORM_INDEX_MIN_NFKD_NO_MAYBE]) {
+ indexes[_NORM_INDEX_MIN_NFKD_NO_MAYBE]=(uint16_t)code;
+ }
+ }
+
+ if(qcFlags&_NORM_QC_NFD) {
+ uset_add(nfdQCNoSet, (UChar32)code);
+ }
+}
+
+extern void
+setCompositionExclusion(uint32_t code) {
+ if(DO_STORE(UGENNORM_STORE_COMPOSITION)) {
+ createNorm(code)->combiningFlags|=0x80;
+ }
+}
+
+static void
+setHangulJamoSpecials() {
+ Norm *norm;
+ uint32_t c, hangul;
+
+ /*
+ * Hangul syllables are algorithmically decomposed into Jamos,
+ * and Jamos are algorithmically composed into Hangul syllables.
+ * The quick check flags are parsed, except for Hangul.
+ */
+
+ /* set Jamo L specials */
+ hangul=0xac00;
+ for(c=0x1100; c<=0x1112; ++c) {
+ norm=createNorm(c);
+ norm->specialTag=_NORM_EXTRA_INDEX_TOP+_NORM_EXTRA_JAMO_L;
+ if(DO_STORE(UGENNORM_STORE_COMPOSITION)) {
+ norm->combiningFlags=1;
+ }
+
+ /* for each Jamo L create a set with its associated Hangul block */
+ norm->canonStart=uset_open(hangul, hangul+21*28-1);
+ hangul+=21*28;
+ }
+
+ /* set Jamo V specials */
+ for(c=0x1161; c<=0x1175; ++c) {
+ norm=createNorm(c);
+ norm->specialTag=_NORM_EXTRA_INDEX_TOP+_NORM_EXTRA_JAMO_V;
+ if(DO_STORE(UGENNORM_STORE_COMPOSITION)) {
+ norm->combiningFlags=2;
+ }
+ norm->unsafeStart=TRUE;
+ }
+
+ /* set Jamo T specials */
+ for(c=0x11a8; c<=0x11c2; ++c) {
+ norm=createNorm(c);
+ norm->specialTag=_NORM_EXTRA_INDEX_TOP+_NORM_EXTRA_JAMO_T;
+ if(DO_STORE(UGENNORM_STORE_COMPOSITION)) {
+ norm->combiningFlags=2;
+ }
+ norm->unsafeStart=TRUE;
+ }
+
+ /* set Hangul specials, precompacted */
+ norm=allocNorm();
+ norm->specialTag=_NORM_EXTRA_INDEX_TOP+_NORM_EXTRA_HANGUL;
+ if(DO_STORE(UGENNORM_STORE_COMPAT)) {
+ norm->qcFlags=_NORM_QC_NFD|_NORM_QC_NFKD;
+ } else {
+ norm->qcFlags=_NORM_QC_NFD;
+ }
+
+ if(!utrie_setRange32(normTrie, 0xac00, 0xd7a4, (uint32_t)(norm-norms), TRUE)) {
+ fprintf(stderr, "error: too many normalization entries (setting Hangul)\n");
+ exit(U_BUFFER_OVERFLOW_ERROR);
+ }
+}
+
+/*
+ * set FC-NFKC-Closure string
+ * s contains the closure string; s[0]==length, s[1..length] is the actual string
+ * may modify s[0]
+ */
+U_CFUNC void
+setFNC(uint32_t c, UChar *s) {
+ uint16_t *p;
+ int32_t length, i, count;
+ UChar first;
+
+ if( DO_NOT_STORE(UGENNORM_STORE_COMPAT) ||
+ DO_NOT_STORE(UGENNORM_STORE_COMPOSITION) ||
+ DO_NOT_STORE(UGENNORM_STORE_AUX)
+ ) {
+ return;
+ }
+
+ count=utm_countItems(extraMem);
+ length=s[0];
+ first=s[1];
+
+ /* try to overlay single-unit strings with existing ones */
+ if(length==1 && first<0xff00) {
+ p=utm_getStart(extraMem);
+ for(i=1; i<count; ++i) {
+ if(first==p[i]) {
+ break;
+ }
+ }
+ } else {
+ i=count;
+ }
+
+ /* append the new string if it cannot be overlayed with an old one */
+ if(i==count) {
+ if(count>_NORM_AUX_MAX_FNC) {
+ fprintf(stderr, "gennorm error: too many FNC strings\n");
+ exit(U_INDEX_OUTOFBOUNDS_ERROR);
+ }
+
+ /* prepend 0xffxx with xx==length */
+ s[0]=(uint16_t)(0xff00+length);
+ ++length;
+ p=(uint16_t *)utm_allocN(extraMem, length);
+ uprv_memcpy(p, s, length*2);
+
+ /* update the top index in extraMem[0] */
+ count+=length;
+ ((uint16_t *)utm_getStart(extraMem))[0]=(uint16_t)count;
+ }
+
+ /* store the index to the string */
+ createNorm(c)->fncIndex=i;
+}
+
+/* build runtime structures ------------------------------------------------- */
+
+/* canonically reorder a UTF-32 string; return { leadCC, trailCC } */
+static uint16_t
+reorderString(uint32_t *s, int32_t length) {
+ uint8_t ccs[40];
+ uint32_t c;
+ int32_t i, j;
+ uint8_t cc, prevCC;
+
+ if(length<=0) {
+ return 0;
+ }
+
+ for(i=0; i<length; ++i) {
+ /* get the i-th code point and its combining class */
+ c=s[i];
+ cc=getCCFromCP(c);
+ if(cc!=0 && i!=0) {
+ /* it is a combining mark, see if it needs to be moved back */
+ j=i;
+ do {
+ prevCC=ccs[j-1];
+ if(prevCC<=cc) {
+ break; /* found the right place */
+ }
+ /* move the previous code point here and go back */
+ s[j]=s[j-1];
+ ccs[j]=prevCC;
+ } while(--j!=0);
+ s[j]=c;
+ ccs[j]=cc;
+ } else {
+ /* just store the combining class */
+ ccs[i]=cc;
+ }
+ }
+
+ return (uint16_t)(((uint16_t)ccs[0]<<8)|ccs[length-1]);
+}
+
+#if 0
+static UBool combineAndQC[64]={ 0 };
+#endif
+
+/*
+ * canonically reorder the up to two decompositions
+ * and store the leading and trailing combining classes accordingly
+ *
+ * also process canonical decompositions for canonical closure
+ */
+static void
+postParseFn(void *context, uint32_t code, Norm *norm) {
+ int32_t length;
+
+ /* canonically order the NFD */
+ length=norm->lenNFD;
+ if(length>0) {
+ norm->canonBothCCs=reorderString(norm->nfd, length);
+ }
+
+ /* canonically reorder the NFKD */
+ length=norm->lenNFKD;
+ if(length>0) {
+ norm->compatBothCCs=reorderString(norm->nfkd, length);
+ }
+
+ /* verify that code has a decomposition if and only if the quick check flags say "no" on NF(K)D */
+ if((norm->lenNFD!=0) != ((norm->qcFlags&_NORM_QC_NFD)!=0)) {
+ fprintf(stderr, "gennorm warning: U+%04lx has NFD[%d] but quick check 0x%02x\n", (long)code, norm->lenNFD, norm->qcFlags);
+ }
+ if(((norm->lenNFD|norm->lenNFKD)!=0) != ((norm->qcFlags&(_NORM_QC_NFD|_NORM_QC_NFKD))!=0)) {
+ fprintf(stderr, "gennorm warning: U+%04lx has NFD[%d] NFKD[%d] but quick check 0x%02x\n", (long)code, norm->lenNFD, norm->lenNFKD, norm->qcFlags);
+ }
+
+ /* see which combinations of combiningFlags and qcFlags are used for NFC/NFKC */
+#if 0
+ combineAndQC[(norm->qcFlags&0x33)|((norm->combiningFlags&3)<<2)]=1;
+#endif
+
+ if(norm->combiningFlags&1) {
+ if(norm->udataCC!=0) {
+ /* illegal - data-derivable composition exclusion */
+ fprintf(stderr, "gennorm warning: U+%04lx combines forward but udataCC==%u\n", (long)code, norm->udataCC);
+ }
+ }
+ if(norm->combiningFlags&2) {
+ if((norm->qcFlags&0x11)==0) {
+ fprintf(stderr, "gennorm warning: U+%04lx combines backward but qcNF?C==0\n", (long)code);
+ }
+#if 0
+ /* occurs sometimes, this one is ok (therefore #if 0) - still here for documentation */
+ if(norm->udataCC==0) {
+ printf("U+%04lx combines backward but udataCC==0\n", (long)code);
+ }
+#endif
+ }
+ if((norm->combiningFlags&3)==3 && beVerbose) {
+ printf("U+%04lx combines both ways\n", (long)code);
+ }
+
+ /*
+ * process canonical decompositions for canonical closure
+ *
+ * in each canonical decomposition:
+ * add the current character (code) to the set of canonical starters of its norm->nfd[0]
+ * set the "unsafe starter" flag for each norm->nfd[1..]
+ */
+ length=norm->lenNFD;
+ if(length>0) {
+ Norm *otherNorm;
+ UChar32 c;
+ int32_t i;
+
+ /* nfd[0].canonStart.add(code) */
+ c=norm->nfd[0];
+ otherNorm=createNorm(c);
+ if(otherNorm->canonStart==NULL) {
+ otherNorm->canonStart=uset_open(code, code);
+ if(otherNorm->canonStart==NULL) {
+ fprintf(stderr, "gennorm error: out of memory in uset_open()\n");
+ exit(U_MEMORY_ALLOCATION_ERROR);
+ }
+ } else {
+ uset_add(otherNorm->canonStart, code);
+ if(!uset_contains(otherNorm->canonStart, code)) {
+ fprintf(stderr, "gennorm error: uset_add(setOf(U+%4x), U+%4x)\n", (int)c, (int)code);
+ exit(U_INTERNAL_PROGRAM_ERROR);
+ }
+ }
+
+ /* for(i=1..length-1) nfd[i].unsafeStart=TRUE */
+ for(i=1; i<length; ++i) {
+ createNorm(norm->nfd[i])->unsafeStart=TRUE;
+ }
+ }
+}
+
+static uint32_t
+make32BitNorm(Norm *norm) {
+ UChar extra[100];
+ const Norm *other;
+ uint32_t word;
+ int32_t i, length, beforeZero=0, count, start;
+
+ /*
+ * Check for assumptions:
+ *
+ * Test that if a "true starter" (cc==0 && NF*C_YES) decomposes,
+ * then the decomposition also begins with a true starter.
+ */
+ if(norm->udataCC==0) {
+ /* this is a starter */
+ if((norm->qcFlags&_NORM_QC_NFC)==0 && norm->lenNFD>0) {
+ /* a "true" NFC starter with a canonical decomposition */
+ if( norm->canonBothCCs>=0x100 || /* lead cc!=0 or */
+ ((other=getNorm(norm->nfd[0]))!=NULL && (other->qcFlags&_NORM_QC_NFC)!=0) /* nfd[0] not NFC_YES */
+ ) {
+ fprintf(stderr,
+ "error: true NFC starter canonical decomposition[%u] does not begin\n"
+ " with a true NFC starter: U+%04lx U+%04lx%s\n",
+ norm->lenNFD, (long)norm->nfd[0], (long)norm->nfd[1],
+ norm->lenNFD<=2 ? "" : " ...");
+ exit(U_INVALID_TABLE_FILE);
+ }
+ }
+
+ if((norm->qcFlags&_NORM_QC_NFKC)==0) {
+ if(norm->lenNFKD>0) {
+ /* a "true" NFKC starter with a compatibility decomposition */
+ if( norm->compatBothCCs>=0x100 || /* lead cc!=0 or */
+ ((other=getNorm(norm->nfkd[0]))!=NULL && (other->qcFlags&_NORM_QC_NFKC)!=0) /* nfkd[0] not NFKC_YES */
+ ) {
+ fprintf(stderr,
+ "error: true NFKC starter compatibility decomposition[%u] does not begin\n"
+ " with a true NFKC starter: U+%04lx U+%04lx%s\n",
+ norm->lenNFKD, (long)norm->nfkd[0], (long)norm->nfkd[1],
+ norm->lenNFKD<=2 ? "" : " ...");
+ exit(U_INVALID_TABLE_FILE);
+ }
+ } else if(norm->lenNFD>0) {
+ /* a "true" NFKC starter with only a canonical decomposition */
+ if( norm->canonBothCCs>=0x100 || /* lead cc!=0 or */
+ ((other=getNorm(norm->nfd[0]))!=NULL && (other->qcFlags&_NORM_QC_NFKC)!=0) /* nfd[0] not NFKC_YES */
+ ) {
+ fprintf(stderr,
+ "error: true NFKC starter canonical decomposition[%u] does not begin\n"
+ " with a true NFKC starter: U+%04lx U+%04lx%s\n",
+ norm->lenNFD, (long)norm->nfd[0], (long)norm->nfd[1],
+ norm->lenNFD<=2 ? "" : " ...");
+ exit(U_INVALID_TABLE_FILE);
+ }
+ }
+ }
+ }
+
+ /* reset the 32-bit word and set the quick check flags */
+ word=norm->qcFlags;
+
+ /* set the UnicodeData combining class */
+ word|=(uint32_t)norm->udataCC<<_NORM_CC_SHIFT;
+
+ /* set the combining flag and index */
+ if(norm->combiningFlags&3) {
+ word|=(uint32_t)(norm->combiningFlags&3)<<6;
+ }
+
+ /* set the combining index value into the extra data */
+ /* 0xffff: no combining index; 0..0x7fff: combining index */
+ if(norm->combiningIndex!=0xffff) {
+ extra[0]=norm->combiningIndex;
+ beforeZero=1;
+ }
+
+ count=beforeZero;
+
+ /* write the decompositions */
+ if((norm->lenNFD|norm->lenNFKD)!=0) {
+ extra[count++]=0; /* set the pieces when available, into extra[beforeZero] */
+
+ length=norm->lenNFD;
+ if(length>0) {
+ if(norm->canonBothCCs!=0) {
+ extra[beforeZero]|=0x80;
+ extra[count++]=norm->canonBothCCs;
+ }
+ start=count;
+ for(i=0; i<length; ++i) {
+ UTF_APPEND_CHAR_UNSAFE(extra, count, norm->nfd[i]);
+ }
+ extra[beforeZero]|=(UChar)(count-start); /* set the decomp length as the number of UTF-16 code units */
+ }
+
+ length=norm->lenNFKD;
+ if(length>0) {
+ if(norm->compatBothCCs!=0) {
+ extra[beforeZero]|=0x8000;
+ extra[count++]=norm->compatBothCCs;
+ }
+ start=count;
+ for(i=0; i<length; ++i) {
+ UTF_APPEND_CHAR_UNSAFE(extra, count, norm->nfkd[i]);
+ }
+ extra[beforeZero]|=(UChar)((count-start)<<8); /* set the decomp length as the number of UTF-16 code units */
+ }
+ }
+
+ /* allocate and copy the extra data */
+ if(count!=0) {
+ UChar *p;
+
+ if(norm->specialTag!=0) {
+ fprintf(stderr, "error: gennorm - illegal to have both extra data and a special tag (0x%x)\n", norm->specialTag);
+ exit(U_ILLEGAL_ARGUMENT_ERROR);
+ }
+
+ p=(UChar *)utm_allocN(extraMem, count);
+ uprv_memcpy(p, extra, count*2);
+
+ /* set the extra index, offset by beforeZero */
+ word|=(uint32_t)(beforeZero+(p-(UChar *)utm_getStart(extraMem)))<<_NORM_EXTRA_SHIFT;
+ } else if(norm->specialTag!=0) {
+ /* set a special tag instead of an extra index */
+ word|=(uint32_t)norm->specialTag<<_NORM_EXTRA_SHIFT;
+ }
+
+ return word;
+}
+
+/* turn all Norm structs into corresponding 32-bit norm values */
+static void
+makeAll32() {
+ uint32_t *pNormData;
+ uint32_t n;
+ int32_t i, normLength, count;
+
+ count=(int32_t)utm_countItems(normMem);
+ for(i=0; i<count; ++i) {
+ norms[i].value32=make32BitNorm(norms+i);
+ }
+
+ pNormData=utrie_getData(norm32Trie, &normLength);
+
+ count=0; /* count is now just used for debugging */
+ for(i=0; i<normLength; ++i) {
+ n=pNormData[i];
+ if(0!=(pNormData[i]=norms[n].value32)) {
+ ++count;
+ }
+ }
+}
+
+/*
+ * extract all Norm.canonBothCCs into the FCD table
+ * set 32-bit values to use the common fold and compact functions
+ */
+static void
+makeFCD() {
+ uint32_t *pFCDData;
+ uint32_t n;
+ int32_t i, count, fcdLength;
+ uint16_t bothCCs;
+
+ count=utm_countItems(normMem);
+ for(i=0; i<count; ++i) {
+ bothCCs=norms[i].canonBothCCs;
+ if(bothCCs==0) {
+ /* if there are no decomposition cc's then use the udataCC twice */
+ bothCCs=norms[i].udataCC;
+ bothCCs|=bothCCs<<8;
+ }
+ norms[i].value32=bothCCs;
+ }
+
+ pFCDData=utrie_getData(fcdTrie, &fcdLength);
+
+ for(i=0; i<fcdLength; ++i) {
+ n=pFCDData[i];
+ pFCDData[i]=norms[n].value32;
+ }
+}
+
+/**
+ * If the given set contains exactly one character, then return it.
+ * Otherwise return -1.
+ */
+static int32_t
+usetContainsOne(const USet* set) {
+ if(uset_getItemCount(set)==1) {
+ /* there is a single item (a single range) */
+ UChar32 start, end;
+ UErrorCode ec=U_ZERO_ERROR;
+ int32_t len=uset_getItem(set, 0, &start, &end, NULL, 0, &ec);
+ if (len==0 && start==end) { /* a range (len==0) with a single code point */
+ return start;
+ }
+ }
+ return -1;
+}
+
+static void
+makeCanonSetFn(void *context, uint32_t code, Norm *norm) {
+ if(norm->canonStart!=NULL && !uset_isEmpty(norm->canonStart)) {
+ uint16_t *table;
+ int32_t c, tableLength;
+ UErrorCode errorCode=U_ZERO_ERROR;
+
+ /* does the set contain exactly one code point? */
+ c=usetContainsOne(norm->canonStart);
+
+ /* add an entry to the BMP or supplementary search table */
+ if(code<=0xffff) {
+ table=canonStartSets+_NORM_MAX_CANON_SETS;
+ tableLength=canonStartSets[_NORM_SET_INDEX_CANON_BMP_TABLE_LENGTH];
+
+ table[tableLength++]=(uint16_t)code;
+
+ if(c>=0 && c<=0xffff && (c&_NORM_CANON_SET_BMP_MASK)!=_NORM_CANON_SET_BMP_IS_INDEX) {
+ /* single-code point BMP result for BMP code point */
+ table[tableLength++]=(uint16_t)c;
+ } else {
+ table[tableLength++]=(uint16_t)(_NORM_CANON_SET_BMP_IS_INDEX|canonStartSetsTop);
+ c=-1;
+ }
+ canonStartSets[_NORM_SET_INDEX_CANON_BMP_TABLE_LENGTH]=(uint16_t)tableLength;
+ } else {
+ table=canonStartSets+_NORM_MAX_CANON_SETS+_NORM_MAX_SET_SEARCH_TABLE_LENGTH;
+ tableLength=canonStartSets[_NORM_SET_INDEX_CANON_SUPP_TABLE_LENGTH];
+
+ table[tableLength++]=(uint16_t)(code>>16);
+ table[tableLength++]=(uint16_t)code;
+
+ if(c>=0) {
+ /* single-code point result for supplementary code point */
+ table[tableLength-2]|=(uint16_t)(0x8000|((c>>8)&0x1f00));
+ table[tableLength++]=(uint16_t)c;
+ } else {
+ table[tableLength++]=(uint16_t)canonStartSetsTop;
+ }
+ canonStartSets[_NORM_SET_INDEX_CANON_SUPP_TABLE_LENGTH]=(uint16_t)tableLength;
+ }
+
+ if(c<0) {
+ /* write a USerializedSet */
+ ++canonSetsCount;
+ canonStartSetsTop+=
+ uset_serialize(norm->canonStart,
+ canonStartSets+canonStartSetsTop,
+ _NORM_MAX_CANON_SETS-canonStartSetsTop,
+ &errorCode);
+ }
+ canonStartSets[_NORM_SET_INDEX_CANON_SETS_LENGTH]=(uint16_t)canonStartSetsTop;
+
+ if(U_FAILURE(errorCode)) {
+ fprintf(stderr, "gennorm error: uset_serialize()->%s (canonStartSetsTop=%d)\n", u_errorName(errorCode), (int)canonStartSetsTop);
+ exit(errorCode);
+ }
+ if(tableLength>_NORM_MAX_SET_SEARCH_TABLE_LENGTH) {
+ fprintf(stderr, "gennorm error: search table for canonical starter sets too long\n");
+ exit(U_INDEX_OUTOFBOUNDS_ERROR);
+ }
+ }
+}
+
+/* for getSkippableFlags ---------------------------------------------------- */
+
+/* combine the lead and trail code points; return <0 if they do not combine */
+static int32_t
+combine(uint32_t lead, uint32_t trail) {
+ CombiningTriple *triples;
+ uint32_t i, count;
+
+ /* search for all triples with c as lead code point */
+ triples=utm_getStart(combiningTriplesMem);
+ count=utm_countItems(combiningTriplesMem);
+
+ /* triples are not sorted by code point but for each lead CP there is one contiguous block */
+ for(i=0; i<count && lead!=triples[i].lead; ++i) {}
+
+ /* check each triple for this code point */
+ for(; i<count && lead==triples[i].lead; ++i) {
+ if(trail==triples[i].trail) {
+ return (int32_t)triples[i].combined;
+ }
+ }
+
+ return -1;
+}
+
+/*
+ * Starting from the canonical decomposition s[0..length[ of a single code point,
+ * is the code point c consumed in an NFC/FCC recomposition?
+ *
+ * No need to handle discontiguous composition because that would not consume some
+ * intermediate character, so would not compose back to the original character.
+ * See comments in canChangeWithFollowing().
+ *
+ * No need to compose beyond where c canonically orders because if it is consumed
+ * then the result differs from the original anyway.
+ *
+ * Possible optimization:
+ * - Verify that there are no cases of the same combining mark stacking twice.
+ * - return FALSE right away if c inserts after a copy of itself
+ * without attempting to recompose; will happen because each mark in
+ * the decomposition will be enumerated and passed in as c.
+ * More complicated and fragile though than it is already.
+ *
+ * markus 2002nov04
+ */
+static UBool
+doesComposeConsume(const uint32_t *s, int32_t length, uint32_t c, uint8_t cc) {
+ int32_t starter, i;
+
+ /* ignore trailing characters where cc<prevCC */
+ while(length>1 && cc<getCCFromCP(s[length-1])) {
+ --length;
+ }
+
+ /* start consuming/combining from the beginning */
+ starter=(int32_t)s[0];
+ for(i=1; i<length; ++i) {
+ starter=combine((uint32_t)starter, s[i]);
+ if(starter<0) {
+ fprintf(stderr, "error: unable to consume normal decomposition in doesComposeConsume(<%04x, %04x, ...>[%d], U+%04x, %u)\n",
+ (int)s[0], (int)s[1], (int)length, (int)c, cc);
+ exit(U_INTERNAL_PROGRAM_ERROR);
+ }
+ }
+
+ /* try to combine/consume c, return TRUE if it is consumed */
+ return combine((uint32_t)starter, c)>=0;
+}
+
+/* does the starter s[0] combine forward with another char that is below trailCC? */
+static UBool
+canChangeWithFollowing(const uint32_t *s, int32_t length, uint8_t trailCC) {
+ if(trailCC<=1) {
+ /* no character will combine ahead of the trailing char of the decomposition */
+ return FALSE;
+ }
+
+ /*
+ * We are only checking skippable condition (f).
+ * Therefore, the original character does not have quick check flag NFC_NO (c),
+ * i.e., the decomposition recomposes completely back into the original code point.
+ * So s[0] must be a true starter with cc==0 and
+ * combining with following code points.
+ *
+ * Similarly, length==1 is not possible because that would be a singleton
+ * decomposition which is marked with NFC_NO and does not pass (c).
+ *
+ * Only a character with cc<trailCC can change the composition.
+ * Reason: A char with cc>=trailCC would order after decomposition s[],
+ * composition would consume all of the decomposition, and here we know that
+ * the original char passed check d), i.e., it does not combine forward,
+ * therefore does not combine with anything after the decomposition is consumed.
+ *
+ * Now see if there is a character that
+ * 1. combines backward
+ * 2. has cc<trailCC
+ * 3. is consumed in recomposition
+ *
+ * length==2 is simple:
+ *
+ * Characters that fulfill these conditions are exactly the ones that combine directly
+ * with the starter c==s[0] because there is no intervening character after
+ * reordering.
+ * We can just enumerate all chars with which c combines (they all pass 1. and 3.)
+ * and see if one has cc<trailCC (passes 2.).
+ *
+ * length>2 is a little harder:
+ *
+ * Since we will get different starters during recomposition, we need to
+ * enumerate each backward-combining character (1.)
+ * with cc<trailCC (2.) and
+ * see if it gets consumed in recomposition. (3.)
+ * No need to enumerate both-ways combining characters because they must have cc==0.
+ */
+ if(length==2) {
+ /* enumerate all chars that combine with this one and check their cc */
+ CombiningTriple *triples;
+ uint32_t c, i, count;
+ uint8_t cc;
+
+ /* search for all triples with c as lead code point */
+ triples=utm_getStart(combiningTriplesMem);
+ count=utm_countItems(combiningTriplesMem);
+ c=s[0];
+
+ /* triples are not sorted by code point but for each lead CP there is one contiguous block */
+ for(i=0; i<count && c!=triples[i].lead; ++i) {}
+
+ /* check each triple for this code point */
+ for(; i<count && c==triples[i].lead; ++i) {
+ cc=getCCFromCP(triples[i].trail);
+ if(cc>0 && cc<trailCC) {
+ /* this trail code point combines with c and has cc<trailCC */
+ return TRUE;
+ }
+ }
+ } else {
+ /* enumerate all chars that combine backward */
+ uint32_t c2;
+ uint16_t i;
+ uint8_t cc;
+
+ for(i=combineBothTop; i<combineBackTop; ++i) {
+ c2=combiningCPs[i]&0xffffff;
+ cc=getCCFromCP(c2);
+ /* pass in length-1 because we already know that c2 will insert before the last character with trailCC */
+ if(cc>0 && cc<trailCC && doesComposeConsume(s, length-1, c2, cc)) {
+ return TRUE;
+ }
+ }
+ }
+
+ /* this decomposition is not modified by any appended character */
+ return FALSE;
+}
+
+/* see unormimp.h for details on NF*C Skippable flags */
+static uint32_t
+getSkippableFlags(const Norm *norm) {
+ /* ignore NF*D skippable properties because they are covered by norm32, test at runtime */
+
+ /* ignore Hangul, test those at runtime (LV Hangul are not skippable) */
+ if(norm->specialTag==_NORM_EXTRA_INDEX_TOP+_NORM_EXTRA_HANGUL) {
+ return 0;
+ }
+
+ /* ### TODO check other data generation functions whether they should & do ignore Hangul/Jamo specials */
+
+ /*
+ * Note:
+ * This function returns a non-zero flag only if (a)..(e) indicate skippable but (f) does not.
+ *
+ * This means that (a)..(e) must always be derived from the runtime norm32 value,
+ * and (f) be checked from the auxTrie if the character is skippable per (a)..(e),
+ * the form is NF*C and there is a canonical decomposition (NFD_NO).
+ *
+ * (a) unassigned code points get "not skippable"==false because they
+ * don't have a Norm struct so they won't get here
+ */
+
+ /* (b) not skippable if cc!=0 */
+ if(norm->udataCC!=0) {
+ return 0; /* non-zero flag for (f) only */
+ }
+
+ /*
+ * not NFC_Skippable if
+ * (c) quick check flag == NO or
+ * (d) combines forward or
+ * (e) combines back or
+ * (f) can change if another character is added
+ *
+ * for (f):
+ * For NF*C: Get corresponding decomposition, get its last starter (cc==0),
+ * check its composition list,
+ * see if any of the second code points in the list
+ * has cc less than the trailCC of the decomposition.
+ *
+ * For FCC: Test at runtime if the decomposition has a trailCC>1
+ * -> there are characters with cc==1, they would order before the trail char
+ * and prevent contiguous combination with the trail char.
+ */
+ if( (norm->qcFlags&(_NORM_QC_NFC&_NORM_QC_ANY_NO))!=0 ||
+ (norm->combiningFlags&3)!=0) {
+ return 0; /* non-zero flag for (f) only */
+ }
+ if(norm->lenNFD!=0 && canChangeWithFollowing(norm->nfd, norm->lenNFD, (uint8_t)norm->canonBothCCs)) {
+ return _NORM_AUX_NFC_SKIP_F_MASK;
+ }
+
+ return 0; /* skippable */
+}
+
+static void
+makeAux() {
+ Norm *norm;
+ uint32_t *pData;
+ int32_t i, length;
+
+ pData=utrie_getData(auxTrie, &length);
+
+ for(i=0; i<length; ++i) {
+ norm=norms+pData[i];
+ /*
+ * 16-bit auxiliary normalization properties
+ * see unormimp.h
+ */
+ pData[i]=
+ ((uint32_t)(norm->combiningFlags&0x80)<<(_NORM_AUX_COMP_EX_SHIFT-7))|
+ (uint32_t)norm->fncIndex;
+
+ if(norm->unsafeStart || norm->udataCC!=0) {
+ pData[i]|=_NORM_AUX_UNSAFE_MASK;
+ }
+
+ pData[i]|=getSkippableFlags(norm);
+ }
+}
+
+/* folding value for normalization: just store the offset (16 bits) if there is any non-0 entry */
+static uint32_t U_CALLCONV
+getFoldedNormValue(UNewTrie *trie, UChar32 start, int32_t offset) {
+ uint32_t value, leadNorm32=0;
+ UChar32 limit;
+ UBool inBlockZero;
+
+ limit=start+0x400;
+ while(start<limit) {
+ value=utrie_get32(trie, start, &inBlockZero);
+ if(inBlockZero) {
+ start+=UTRIE_DATA_BLOCK_LENGTH;
+ } else {
+ if(value!=0) {
+ leadNorm32|=value;
+ }
+ ++start;
+ }
+ }
+
+ /* turn multi-bit fields into the worst-case value */
+ if(leadNorm32&_NORM_CC_MASK) {
+ leadNorm32|=_NORM_CC_MASK;
+ }
+
+ /* clean up unnecessarily ored bit fields */
+ leadNorm32&=~((uint32_t)0xffffffff<<_NORM_EXTRA_SHIFT);
+
+ if(leadNorm32==0) {
+ /* nothing to do (only composition exclusions?) */
+ return 0;
+ }
+
+ /* add the extra surrogate index, offset by the BMP top, for the new stage 1 location */
+ leadNorm32|=(
+ (uint32_t)_NORM_EXTRA_INDEX_TOP+
+ (uint32_t)((offset-UTRIE_BMP_INDEX_LENGTH)>>UTRIE_SURROGATE_BLOCK_BITS)
+ )<<_NORM_EXTRA_SHIFT;
+
+ return leadNorm32;
+}
+
+/* folding value for FCD: use default function (just store the offset (16 bits) if there is any non-0 entry) */
+
+/*
+ * folding value for auxiliary data:
+ * store the non-zero offset in bits 9..0 (FNC bits)
+ * if there is any non-0 entry;
+ * "or" [verb!] together data bits 15..10 of all of the 1024 supplementary code points
+ */
+static uint32_t U_CALLCONV
+getFoldedAuxValue(UNewTrie *trie, UChar32 start, int32_t offset) {
+ uint32_t value, oredValues;
+ UChar32 limit;
+ UBool inBlockZero;
+
+ oredValues=0;
+ limit=start+0x400;
+ while(start<limit) {
+ value=utrie_get32(trie, start, &inBlockZero);
+ if(inBlockZero) {
+ start+=UTRIE_DATA_BLOCK_LENGTH;
+ } else {
+ oredValues|=value;
+ ++start;
+ }
+ }
+
+ if(oredValues!=0) {
+ /* move the 10 significant offset bits into bits 9..0 */
+ offset>>=UTRIE_SURROGATE_BLOCK_BITS;
+ if(offset>_NORM_AUX_FNC_MASK) {
+ fprintf(stderr, "gennorm error: folding offset too large (auxTrie)\n");
+ exit(U_INDEX_OUTOFBOUNDS_ERROR);
+ }
+ return (uint32_t)offset|(oredValues&~_NORM_AUX_FNC_MASK);
+ } else {
+ return 0;
+ }
+}
+
+extern void
+processData() {
+#if 0
+ uint16_t i;
+#endif
+
+ processCombining();
+
+ /* canonically reorder decompositions and assign combining classes for decompositions */
+ enumTrie(postParseFn, NULL);
+
+#if 0
+ for(i=1; i<64; ++i) {
+ if(combineAndQC[i]) {
+ printf("combiningFlags==0x%02x qcFlags(NF?C)==0x%02x\n", (i&0xc)>>2, i&0x33);
+ }
+ }
+#endif
+
+ /* add hangul/jamo specials */
+ setHangulJamoSpecials();
+
+ /* set this value; will be updated as makeCanonSetFn() adds sets (if there are any, see gStoreFlags) */
+ canonStartSets[_NORM_SET_INDEX_CANON_SETS_LENGTH]=(uint16_t)canonStartSetsTop;
+
+ /* store search tables and USerializedSets for canonical starters (after Hangul/Jamo specials!) */
+ if(DO_STORE(UGENNORM_STORE_AUX) && DO_STORE(UGENNORM_STORE_COMPOSITION)) {
+ enumTrie(makeCanonSetFn, NULL);
+ }
+
+ /* clone the normalization builder trie to make the final data tries */
+ if( NULL==utrie_clone(norm32Trie, normTrie, NULL, 0) ||
+ NULL==utrie_clone(fcdTrie, normTrie, NULL, 0) ||
+ NULL==utrie_clone(auxTrie, normTrie, NULL, 0)
+ ) {
+ fprintf(stderr, "error: unable to clone the normalization trie\n");
+ exit(U_MEMORY_ALLOCATION_ERROR);
+ }
+
+ /* --- finalize data for quick checks & normalization --- */
+
+ /* turn the Norm structs (stage2, norms) into 32-bit data words */
+ makeAll32();
+
+ /* --- finalize data for FCD checks --- */
+
+ /* FCD data: take Norm.canonBothCCs and store them in the FCD table */
+ makeFCD();
+
+ /* --- finalize auxiliary normalization data --- */
+ makeAux();
+
+ if(beVerbose) {
+#if 0
+ printf("number of stage 2 entries: %ld\n", stage2Mem->index);
+ printf("size of stage 1 (BMP) & 2 (uncompacted) + extra data: %ld bytes\n", _NORM_STAGE_1_BMP_COUNT*2+stage2Mem->index*4+extraMem->index*2);
+#endif
+ printf("combining CPs tops: fwd %u both %u back %u\n", combineFwdTop, combineBothTop, combineBackTop);
+ printf("combining table count: %u\n", combiningTableTop);
+ }
+}
+
+#endif /* #if !UCONFIG_NO_NORMALIZATION */
+
+extern void
+generateData(const char *dataDir, UBool csource) {
+ static uint8_t normTrieBlock[100000], fcdTrieBlock[100000], auxTrieBlock[100000];
+
+ UNewDataMemory *pData;
+ UErrorCode errorCode=U_ZERO_ERROR;
+ int32_t size, dataLength;
+
+#if UCONFIG_NO_NORMALIZATION
+
+ size=0;
+
+#else
+
+ U_STRING_DECL(nxCJKCompatPattern, "[:Ideographic:]", 15);
+ U_STRING_DECL(nxUnicode32Pattern, "[:^Age=3.2:]", 12);
+ USet *set;
+ int32_t normTrieSize, fcdTrieSize, auxTrieSize;
+
+ normTrieSize=utrie_serialize(norm32Trie, normTrieBlock, sizeof(normTrieBlock), getFoldedNormValue, FALSE, &errorCode);
+ if(U_FAILURE(errorCode)) {
+ fprintf(stderr, "error: utrie_serialize(normalization properties) failed, %s\n", u_errorName(errorCode));
+ exit(errorCode);
+ }
+
+ if(DO_STORE(UGENNORM_STORE_FCD)) {
+ fcdTrieSize=utrie_serialize(fcdTrie, fcdTrieBlock, sizeof(fcdTrieBlock), NULL, TRUE, &errorCode);
+ if(U_FAILURE(errorCode)) {
+ fprintf(stderr, "error: utrie_serialize(FCD data) failed, %s\n", u_errorName(errorCode));
+ exit(errorCode);
+ }
+ } else {
+ fcdTrieSize=0;
+ }
+
+ if(DO_STORE(UGENNORM_STORE_AUX)) {
+ auxTrieSize=utrie_serialize(auxTrie, auxTrieBlock, sizeof(auxTrieBlock), getFoldedAuxValue, TRUE, &errorCode);
+ if(U_FAILURE(errorCode)) {
+ fprintf(stderr, "error: utrie_serialize(auxiliary data) failed, %s\n", u_errorName(errorCode));
+ exit(errorCode);
+ }
+ } else {
+ auxTrieSize=0;
+ }
+
+ /* move the parts of canonStartSets[] together into a contiguous block */
+ if( canonStartSetsTop<_NORM_MAX_CANON_SETS &&
+ canonStartSets[_NORM_SET_INDEX_CANON_BMP_TABLE_LENGTH]!=0
+ ) {
+ uprv_memmove(canonStartSets+canonStartSetsTop,
+ canonStartSets+_NORM_MAX_CANON_SETS,
+ canonStartSets[_NORM_SET_INDEX_CANON_BMP_TABLE_LENGTH]*2);
+ }
+ canonStartSetsTop+=canonStartSets[_NORM_SET_INDEX_CANON_BMP_TABLE_LENGTH];
+
+ if( canonStartSetsTop<(_NORM_MAX_CANON_SETS+_NORM_MAX_SET_SEARCH_TABLE_LENGTH) &&
+ canonStartSets[_NORM_SET_INDEX_CANON_SUPP_TABLE_LENGTH]!=0
+ ) {
+ uprv_memmove(canonStartSets+canonStartSetsTop,
+ canonStartSets+_NORM_MAX_CANON_SETS+_NORM_MAX_SET_SEARCH_TABLE_LENGTH,
+ canonStartSets[_NORM_SET_INDEX_CANON_SUPP_TABLE_LENGTH]*2);
+ }
+ canonStartSetsTop+=canonStartSets[_NORM_SET_INDEX_CANON_SUPP_TABLE_LENGTH];
+
+ /* create the normalization exclusion sets */
+ /*
+ * nxCJKCompatPattern should be [[:Ideographic:]&[:NFD_QC=No:]]
+ * but we cannot use NFD_QC from the pattern because that would require
+ * unorm.icu which we are just going to generate.
+ * Therefore we have manually collected nfdQCNoSet and intersect Ideographic
+ * with that.
+ */
+ U_STRING_INIT(nxCJKCompatPattern, "[:Ideographic:]", 15);
+ U_STRING_INIT(nxUnicode32Pattern, "[:^Age=3.2:]", 12);
+
+ canonStartSets[_NORM_SET_INDEX_NX_CJK_COMPAT_OFFSET]=canonStartSetsTop;
+ set=uset_openPattern(nxCJKCompatPattern, -1, &errorCode);
+ if(U_FAILURE(errorCode)) {
+ fprintf(stderr, "error: uset_openPattern([:Ideographic:]&[:NFD_QC=No:]) failed, %s\n", u_errorName(errorCode));
+ exit(errorCode);
+ }
+ uset_retainAll(set, nfdQCNoSet);
+ if(DO_NOT_STORE(UGENNORM_STORE_EXCLUSIONS)) {
+ uset_clear(set);
+ }
+ canonStartSetsTop+=uset_serialize(set, canonStartSets+canonStartSetsTop, LENGTHOF(canonStartSets)-canonStartSetsTop, &errorCode);
+ if(U_FAILURE(errorCode)) {
+ fprintf(stderr, "error: uset_serialize([:Ideographic:]&[:NFD_QC=No:]) failed, %s\n", u_errorName(errorCode));
+ exit(errorCode);
+ }
+ uset_close(set);
+
+ canonStartSets[_NORM_SET_INDEX_NX_UNICODE32_OFFSET]=canonStartSetsTop;
+ set=uset_openPattern(nxUnicode32Pattern, -1, &errorCode);
+ if(U_FAILURE(errorCode)) {
+ fprintf(stderr, "error: uset_openPattern([:^Age=3.2:]) failed, %s\n", u_errorName(errorCode));
+ exit(errorCode);
+ }
+ if(DO_NOT_STORE(UGENNORM_STORE_EXCLUSIONS)) {
+ uset_clear(set);
+ }
+ canonStartSetsTop+=uset_serialize(set, canonStartSets+canonStartSetsTop, LENGTHOF(canonStartSets)-canonStartSetsTop, &errorCode);
+ if(U_FAILURE(errorCode)) {
+ fprintf(stderr, "error: uset_serialize([:^Age=3.2:]) failed, %s\n", u_errorName(errorCode));
+ exit(errorCode);
+ }
+ uset_close(set);
+
+ canonStartSets[_NORM_SET_INDEX_NX_RESERVED_OFFSET]=canonStartSetsTop;
+
+ /* make sure that the FCD trie is 4-aligned */
+ if((utm_countItems(extraMem)+combiningTableTop)&1) {
+ combiningTable[combiningTableTop++]=0x1234; /* add one 16-bit word for an even number */
+ }
+
+ /* pad canonStartSets to 4-alignment, too */
+ if(canonStartSetsTop&1) {
+ canonStartSets[canonStartSetsTop++]=0x1235;
+ }
+
+ size=
+ _NORM_INDEX_TOP*4+
+ normTrieSize+
+ utm_countItems(extraMem)*2+
+ combiningTableTop*2+
+ fcdTrieSize+
+ auxTrieSize+
+ canonStartSetsTop*2;
+
+ if(beVerbose) {
+ printf("size of normalization trie %5u bytes\n", (int)normTrieSize);
+ printf("size of 16-bit extra memory %5u UChars/uint16_t\n", (int)utm_countItems(extraMem));
+ printf(" of that: FC_NFKC_Closure size %5u UChars/uint16_t\n", ((uint16_t *)utm_getStart(extraMem))[0]);
+ printf("size of combining table %5u uint16_t\n", combiningTableTop);
+ printf("size of FCD trie %5u bytes\n", (int)fcdTrieSize);
+ printf("size of auxiliary trie %5u bytes\n", (int)auxTrieSize);
+ printf("size of canonStartSets[] %5u uint16_t\n", (int)canonStartSetsTop);
+ printf(" number of indexes %5u uint16_t\n", _NORM_SET_INDEX_TOP);
+ printf(" size of sets %5u uint16_t\n", canonStartSets[_NORM_SET_INDEX_CANON_SETS_LENGTH]-_NORM_SET_INDEX_TOP);
+ printf(" number of sets %5d\n", (int)canonSetsCount);
+ printf(" size of BMP search table %5u uint16_t\n", canonStartSets[_NORM_SET_INDEX_CANON_BMP_TABLE_LENGTH]);
+ printf(" size of supplementary search table %5u uint16_t\n", canonStartSets[_NORM_SET_INDEX_CANON_SUPP_TABLE_LENGTH]);
+ printf(" length of exclusion sets %5u uint16_t\n", canonStartSets[_NORM_SET_INDEX_NX_RESERVED_OFFSET]-canonStartSets[_NORM_SET_INDEX_NX_CJK_COMPAT_OFFSET]);
+ printf("size of " U_ICUDATA_NAME "_" DATA_NAME "." DATA_TYPE " contents: %ld bytes\n", (long)size);
+ }
+
+ indexes[_NORM_INDEX_TRIE_SIZE]=normTrieSize;
+ indexes[_NORM_INDEX_UCHAR_COUNT]=(uint16_t)utm_countItems(extraMem);
+
+ indexes[_NORM_INDEX_COMBINE_DATA_COUNT]=combiningTableTop;
+ indexes[_NORM_INDEX_COMBINE_FWD_COUNT]=combineFwdTop;
+ indexes[_NORM_INDEX_COMBINE_BOTH_COUNT]=(uint16_t)(combineBothTop-combineFwdTop);
+ indexes[_NORM_INDEX_COMBINE_BACK_COUNT]=(uint16_t)(combineBackTop-combineBothTop);
+
+ /* the quick check minimum code points are already set */
+
+ indexes[_NORM_INDEX_FCD_TRIE_SIZE]=fcdTrieSize;
+ indexes[_NORM_INDEX_AUX_TRIE_SIZE]=auxTrieSize;
+ indexes[_NORM_INDEX_CANON_SET_COUNT]=canonStartSetsTop;
+
+#endif
+
+ if(csource) {
+#if UCONFIG_NO_NORMALIZATION
+ /* no csource for dummy mode..? */
+ fprintf(stderr, "gennorm error: UCONFIG_NO_NORMALIZATION is on in csource mode.\n");
+ exit(1);
+#else
+ /* write .c file for hardcoded data */
+ UTrie normTrie2={ NULL }, fcdTrie2={ NULL }, auxTrie2={ NULL };
+ FILE *f;
+
+ utrie_unserialize(&normTrie2, normTrieBlock, normTrieSize, &errorCode);
+ if(fcdTrieSize>0) {
+ utrie_unserialize(&fcdTrie2, fcdTrieBlock, fcdTrieSize, &errorCode);
+ }
+ if(auxTrieSize>0) {
+ utrie_unserialize(&auxTrie2, auxTrieBlock, auxTrieSize, &errorCode);
+ }
+ if(U_FAILURE(errorCode)) {
+ fprintf(
+ stderr,
+ "gennorm error: failed to utrie_unserialize() one of the tries - %s\n",
+ u_errorName(errorCode));
+ exit(errorCode);
+ }
+
+ f=usrc_create(dataDir, "unorm_props_data.c");
+ if(f!=NULL) {
+ usrc_writeArray(f,
+ "static const UVersionInfo formatVersion={ ",
+ dataInfo.formatVersion, 8, 4,
+ " };\n\n");
+ usrc_writeArray(f,
+ "static const UVersionInfo dataVersion={ ",
+ dataInfo.dataVersion, 8, 4,
+ " };\n\n");
+ usrc_writeArray(f,
+ "static const int32_t indexes[_NORM_INDEX_TOP]={\n",
+ indexes, 32, _NORM_INDEX_TOP,
+ "\n};\n\n");
+ usrc_writeUTrieArrays(f,
+ "static const uint16_t normTrie_index[%ld]={\n",
+ "static const uint32_t normTrie_data32[%ld]={\n",
+ &normTrie2,
+ "\n};\n\n");
+ usrc_writeUTrieStruct(f,
+ "static const UTrie normTrie={\n",
+ &normTrie2, "normTrie_index", "normTrie_data32", "getFoldingNormOffset",
+ "};\n\n");
+ usrc_writeArray(f,
+ "static const uint16_t extraData[%ld]={\n",
+ utm_getStart(extraMem), 16, utm_countItems(extraMem),
+ "\n};\n\n");
+ usrc_writeArray(f,
+ "static const uint16_t combiningTable[%ld]={\n",
+ combiningTable, 16, combiningTableTop,
+ "\n};\n\n");
+ if(fcdTrieSize>0) {
+ usrc_writeUTrieArrays(f,
+ "static const uint16_t fcdTrie_index[%ld]={\n", NULL,
+ &fcdTrie2,
+ "\n};\n\n");
+ usrc_writeUTrieStruct(f,
+ "static const UTrie fcdTrie={\n",
+ &fcdTrie2, "fcdTrie_index", NULL, NULL,
+ "};\n\n");
+ } else {
+ fputs( "static const UTrie fcdTrie={ NULL };\n\n", f);
+ }
+ if(auxTrieSize>0) {
+ usrc_writeUTrieArrays(f,
+ "static const uint16_t auxTrie_index[%ld]={\n", NULL,
+ &auxTrie2,
+ "\n};\n\n");
+ usrc_writeUTrieStruct(f,
+ "static const UTrie auxTrie={\n",
+ &auxTrie2, "auxTrie_index", NULL, "getFoldingAuxOffset",
+ "};\n\n");
+ } else {
+ fputs( "static const UTrie auxTrie={ NULL };\n\n", f);
+ }
+ usrc_writeArray(f,
+ "static const uint16_t canonStartSets[%ld]={\n",
+ canonStartSets, 16, canonStartSetsTop,
+ "\n};\n\n");
+ fclose(f);
+ }
+#endif
+ } else {
+ /* write the data */
+ pData=udata_create(dataDir, DATA_TYPE, DATA_NAME, &dataInfo,
+ haveCopyright ? U_COPYRIGHT_STRING : NULL, &errorCode);
+ if(U_FAILURE(errorCode)) {
+ fprintf(stderr, "gennorm: unable to create the output file, error %d\n", errorCode);
+ exit(errorCode);
+ }
+
+#if !UCONFIG_NO_NORMALIZATION
+
+ udata_writeBlock(pData, indexes, sizeof(indexes));
+ udata_writeBlock(pData, normTrieBlock, normTrieSize);
+ udata_writeBlock(pData, utm_getStart(extraMem), utm_countItems(extraMem)*2);
+ udata_writeBlock(pData, combiningTable, combiningTableTop*2);
+ udata_writeBlock(pData, fcdTrieBlock, fcdTrieSize);
+ udata_writeBlock(pData, auxTrieBlock, auxTrieSize);
+ udata_writeBlock(pData, canonStartSets, canonStartSetsTop*2);
+
+#endif
+
+ /* finish up */
+ dataLength=udata_finish(pData, &errorCode);
+ if(U_FAILURE(errorCode)) {
+ fprintf(stderr, "gennorm: error %d writing the output file\n", errorCode);
+ exit(errorCode);
+ }
+
+ if(dataLength!=size) {
+ fprintf(stderr, "gennorm error: data length %ld != calculated size %ld\n",
+ (long)dataLength, (long)size);
+ exit(U_INTERNAL_PROGRAM_ERROR);
+ }
+ }
+}
+
+#if !UCONFIG_NO_NORMALIZATION
+
+extern void
+cleanUpData(void) {
+ int32_t i, count;
+
+ count=utm_countItems(normMem);
+ for(i=0; i<count; ++i) {
+ uset_close(norms[i].canonStart);
+ }
+
+ utm_close(normMem);
+ utm_close(utf32Mem);
+ utm_close(extraMem);
+ utm_close(combiningTriplesMem);
+ utrie_close(normTrie);
+ utrie_close(norm32Trie);
+ utrie_close(fcdTrie);
+ utrie_close(auxTrie);
+
+ uset_close(nfdQCNoSet);
+
+ uprv_free(normTrie);
+ uprv_free(norm32Trie);
+ uprv_free(fcdTrie);
+ uprv_free(auxTrie);
+}
+
+#endif /* #if !UCONFIG_NO_NORMALIZATION */
+
+/*
+ * Hey, Emacs, please set the following:
+ *
+ * Local Variables:
+ * indent-tabs-mode: nil
+ * End:
+ *
+ */