From 53790d2a0c50915d8cec71df3e89cd24d887b2fe Mon Sep 17 00:00:00 2001 From: Jonathan Kew Date: Tue, 4 Mar 2008 13:26:36 +0000 Subject: update icu-xetex to 3.8.1-based code from xetex repository git-svn-id: svn://tug.org/texlive/trunk@6842 c570f23f-e606-0410-a88d-b1316a301751 --- .../libs/icu-xetex/tools/gennames/Makefile.in | 97 ++ .../libs/icu-xetex/tools/gennames/gennames.c | 1386 ++++++++++++++++++++ .../libs/icu-xetex/tools/gennames/gennames.vcproj | 226 ++++ 3 files changed, 1709 insertions(+) create mode 100644 Build/source/libs/icu-xetex/tools/gennames/Makefile.in create mode 100644 Build/source/libs/icu-xetex/tools/gennames/gennames.c create mode 100644 Build/source/libs/icu-xetex/tools/gennames/gennames.vcproj (limited to 'Build/source/libs/icu-xetex/tools/gennames') diff --git a/Build/source/libs/icu-xetex/tools/gennames/Makefile.in b/Build/source/libs/icu-xetex/tools/gennames/Makefile.in new file mode 100644 index 00000000000..ed2f88f3fd8 --- /dev/null +++ b/Build/source/libs/icu-xetex/tools/gennames/Makefile.in @@ -0,0 +1,97 @@ +## Makefile.in for ICU - tools/gennames +## Copyright (c) 1999-2005, International Business Machines Corporation and +## others. All Rights Reserved. +## Steven R. Loomis + +## Source directory information +srcdir = @srcdir@ +top_srcdir = @top_srcdir@ + +top_builddir = ../.. + +include $(top_builddir)/icudefs.mk + +## Build directory information +subdir = tools/gennames + +TARGET_STUB_NAME = gennames + +SECTION = 8 + +#MAN_FILES = $(TARGET_STUB_NAME).$(SECTION) + + +## Extra files to remove for 'make clean' +CLEANFILES = *~ $(DEPS) $(MAN_FILES) + +## Target information +TARGET = $(BINDIR)/$(TARGET_STUB_NAME)$(EXEEXT) + +ifneq ($(top_builddir),$(top_srcdir)) +CPPFLAGS += -I$(top_builddir)/common +endif +CPPFLAGS += -I$(top_srcdir)/common -I$(srcdir)/../toolutil +LIBS = $(LIBICUTOOLUTIL) $(LIBICUI18N) $(LIBICUUC) $(DEFAULT_LIBS) $(LIB_M) + +OBJECTS = gennames.o + +DEPS = $(OBJECTS:.o=.d) + +## List of phony targets +.PHONY : all all-local install install-local clean clean-local \ +distclean distclean-local dist dist-local check check-local install-man + +## Clear suffix list +.SUFFIXES : + +## List of standard targets +all: all-local +install: install-local +clean: clean-local +distclean : distclean-local +dist: dist-local +check: all check-local + +all-local: $(TARGET) $(MAN_FILES) + +install-local: all-local install-man +# $(MKINSTALLDIRS) $(DESTDIR)$(sbindir) +# $(INSTALL) $(TARGET) $(DESTDIR)$(sbindir) + +install-man: $(MAN_FILES) +# $(MKINSTALLDIRS) $(DESTDIR)$(mandir)/man$(SECTION) +# $(INSTALL_DATA) $? $(DESTDIR)$(mandir)/man$(SECTION) + +dist-local: + +clean-local: + test -z "$(CLEANFILES)" || $(RMV) $(CLEANFILES) + $(RMV) $(TARGET) $(OBJECTS) + +distclean-local: clean-local + $(RMV) Makefile + +check-local: all-local + +Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status + cd $(top_builddir) \ + && CONFIG_FILES=$(subdir)/$@ CONFIG_HEADERS= $(SHELL) ./config.status + +$(TARGET) : $(OBJECTS) + $(LINK.cc) $(OUTOPT)$@ $^ $(LIBS) + $(POST_BUILD_STEP) + + +%.$(SECTION): $(srcdir)/%.$(SECTION).in + cd $(top_builddir) \ + && CONFIG_FILES=$(subdir)/$@ CONFIG_HEADERS= $(SHELL) ./config.status + + +ifeq (,$(MAKECMDGOALS)) +-include $(DEPS) +else +ifneq ($(patsubst %clean,,$(MAKECMDGOALS)),) +-include $(DEPS) +endif +endif + diff --git a/Build/source/libs/icu-xetex/tools/gennames/gennames.c b/Build/source/libs/icu-xetex/tools/gennames/gennames.c new file mode 100644 index 00000000000..8a50acc0efc --- /dev/null +++ b/Build/source/libs/icu-xetex/tools/gennames/gennames.c @@ -0,0 +1,1386 @@ +/* +******************************************************************************* +* +* Copyright (C) 1999-2005, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: gennames.c +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 1999sep30 +* created by: Markus W. Scherer +* +* This program reads the Unicode character database text file, +* parses it, and extracts the character code, +* the "modern" character name, and optionally the +* Unicode 1.0 character name, and (starting with ICU 2.2) the ISO 10646 comment. +* It then tokenizes and compresses the names and builds +* compact binary tables for random-access lookup +* in a u_charName() API function. +* +* unames.icu file format (after UDataInfo header etc. - see udata.c) +* (all data is static const) +* +* UDataInfo fields: +* dataFormat "unam" +* formatVersion 1.0 +* dataVersion = Unicode version from -u or --unicode command line option, defaults to 3.0.0 +* +* -- data-based names +* uint32_t tokenStringOffset, +* groupsOffset, +* groupStringOffset, +* algNamesOffset; +* +* uint16_t tokenCount; +* uint16_t tokenTable[tokenCount]; +* +* char tokenStrings[]; -- padded to even count +* +* -- strings (groupStrings) are tokenized as follows: +* for each character c +* if(c>=tokenCount) write that character c directly +* else +* token=tokenTable[c]; +* if(token==0xfffe) -- lead byte of double-byte token +* token=tokenTable[c<<8|next character]; +* if(token==-1) +* write c directly +* else +* tokenString=tokenStrings+token; (tokenStrings=start of names data + tokenStringOffset;) +* append zero-terminated tokenString; +* +* Different strings for a code point - normal name, 1.0 name, and ISO comment - +* are separated by ';'. +* +* uint16_t groupCount; +* struct { +* uint16_t groupMSB; -- for a group of 32 character names stored, this is code point>>5 +* uint16_t offsetHigh; -- group strings are at start of names data + groupStringsOffset + this 32 bit-offset +* uint16_t offsetLow; +* } groupTable[groupCount]; +* +* char groupStrings[]; -- padded to 4-count +* +* -- The actual, tokenized group strings are not zero-terminated because +* that would take up too much space. +* Instead, they are preceeded by their length, written in a variable-length sequence: +* For each of the 32 group strings, one or two nibbles are stored for its length. +* Nibbles (4-bit values, half-bytes) are read MSB first. +* A nibble with a value of 0..11 directly indicates the length of the name string. +* A nibble n with a value of 12..15 is a lead nibble and forms a value with the following nibble m +* by (((n-12)<<4)|m)+12, reaching values of 12..75. +* These lengths are sequentially for each tokenized string, not for the de-tokenized result. +* For the de-tokenizing, see token description above; the strings immediately follow the +* 32 lengths. +* +* -- algorithmic names +* +* typedef struct AlgorithmicRange { +* uint32_t rangeStart, rangeEnd; +* uint8_t algorithmType, algorithmVariant; +* uint16_t rangeSize; +* } AlgorithmicRange; +* +* uint32_t algRangesCount; -- number of data blocks for ranges of +* algorithmic names (Unicode 3.0.0: 3, hardcoded in gennames) +* +* struct { +* AlgorithmicRange algRange; +* uint8_t algRangeData[]; -- padded to 4-count except in last range +* } algRanges[algNamesCount]; +* -- not a real array because each part has a different size +* of algRange.rangeSize (including AlgorithmicRange) +* +* -- algorithmic range types: +* +* 0 Names are formed from a string prefix that is stored in +* the algRangeData (zero-terminated), followed by the Unicode code point +* of the character in hexadecimal digits; +* algRange.algorithmVariant digits are written +* +* 1 Names are formed by calculating modulo-factors of the code point value as follows: +* algRange.algorithmVariant is the count of modulo factors +* algRangeData contains +* uint16_t factors[algRange.algorithmVariant]; +* char strings[]; +* the first zero-terminated string is written as the prefix; then: +* +* The rangeStart is subtracted; with the difference, here "code": +* for(i=algRange.algorithmVariant-1 to 0 step -1) +* index[i]=code%factor[i]; +* code/=factor[i]; +* +* The strings after the prefix are short pieces that are then appended to the result +* according to index[0..algRange.algorithmVariant-1]. +*/ + +#include +#include "unicode/utypes.h" +#include "unicode/putil.h" +#include "unicode/uclean.h" +#include "unicode/udata.h" +#include "cmemory.h" +#include "cstring.h" +#include "uarrsort.h" +#include "unewdata.h" +#include "uoptions.h" +#include "uparse.h" + +#define STRING_STORE_SIZE 1000000 +#define GROUP_STORE_SIZE 5000 + +#define GROUP_SHIFT 5 +#define LINES_PER_GROUP (1UL<unicodeVersions[i]) {} */ + i0; + ++i) {} + if(0=ucdVersion comparisons */ + } + return i; /* version>=unicodeVersions[i] && version=2 ? argv[1] : "-", store10Names); + compress(); + generateData(options[5].value); + + u_cleanup(); + return 0; +} + +static void +init() { + int i; + + for(i=0; i<256; ++i) { + tokens[i]=0; + } +} + +/* parsing ------------------------------------------------------------------ */ + +/* get a name, strip leading and trailing whitespace */ +static int16_t +getName(char **pStart, char *limit) { + /* strip leading whitespace */ + char *start=(char *)u_skipWhitespace(*pStart); + + /* strip trailing whitespace */ + while(start brackets */ + lengths[0]=0; + } + + /* store 1.0 names */ + /* get the second character name, the one from Unicode 1.0 */ + /* do not store pseudo-names in <> brackets */ + names[1]=fields[10][0]; + lengths[1]=getName(names+1, fields[10][1]); + if(*(UBool *)context && names[1][0]!='<') { + /* keep the name */ + } else { + lengths[1]=0; + } + + /* get the ISO 10646 comment */ + names[2]=fields[11][0]; + lengths[2]=getName(names+2, fields[11][1]); + + if(lengths[0]+lengths[1]+lengths[2]==0) { + return; + } + + /* check for non-character code points */ + if(!UTF_IS_UNICODE_CHAR(code)) { + fprintf(stderr, "gennames: error - properties for non-character code point U+%04lx\n", + (unsigned long)code); + *pErrorCode=U_PARSE_ERROR; + exit(U_PARSE_ERROR); + } + + /* check that the code points (code) are in ascending order */ + if(code<=prevCode && code>0) { + fprintf(stderr, "gennames: error - UnicodeData entries out of order, U+%04lx after U+%04lx\n", + (unsigned long)code, (unsigned long)prevCode); + *pErrorCode=U_PARSE_ERROR; + exit(U_PARSE_ERROR); + } + prevCode=code; + + parseName(names[0], lengths[0]); + parseName(names[1], lengths[1]); + parseName(names[2], lengths[2]); + + /* + * set the count argument to + * 1: only store regular names + * 2: store regular and 1.0 names + * 3: store names and ISO 10646 comment + */ + addLine(code, names, lengths, 3); +} + +static void +parseDB(const char *filename, UBool store10Names) { + char *fields[15][2]; + UErrorCode errorCode=U_ZERO_ERROR; + + u_parseDelimitedFile(filename, ';', fields, 15, lineFn, &store10Names, &errorCode); + if(U_FAILURE(errorCode)) { + fprintf(stderr, "gennames parse error: %s\n", u_errorName(errorCode)); + exit(errorCode); + } + + if(!beQuiet) { + printf("size of all names in the database: %lu\n", + (unsigned long)lineTop); + printf("number of named Unicode characters: %lu\n", + (unsigned long)lineCount); + printf("number of words in the dictionary from these names: %lu\n", + (unsigned long)wordCount); + } +} + +static void +parseName(char *name, int16_t length) { + int16_t start=0, limit, wordLength/*, prevStart=-1*/; + Word *word; + + while(start1) { + word=findWord(name+start, wordLength); + if(word==NULL) { + word=addWord(name+start, wordLength); + } + countWord(word); + } + +#if 0 + /* + * if there was a word before this + * (with no noise in between), then add the pair of words, too + */ + if(prevStart!=-1) { + wordLength=limit-prevStart; + word=findWord(name+prevStart, wordLength); + if(word==NULL) { + word=addWord(name+prevStart, wordLength); + } + countWord(word); + } +#endif + + /*prevStart=start;*/ + start=limit; + } +} + +static UBool U_INLINE +isWordChar(char c) { + return ('A'<=c && c<='I') || /* EBCDIC-safe check for letters */ + ('J'<=c && c<='R') || + ('S'<=c && c<='Z') || + + ('a'<=c && c<='i') || /* lowercase letters for ISO comments */ + ('j'<=c && c<='r') || + ('s'<=c && c<='z') || + + ('0'<=c && c<='9'); +} + +static int16_t +skipNoise(char *line, int16_t start, int16_t limit) { + /* skip anything that is not part of a word in this sense */ + while(start0 && words[wordCount-1].weight<1) { + --wordCount; + } + + /* count the letters in the token range */ + letterCount=0; + for(i=LEADBYTE_LIMIT; i<256; ++i) { + if(tokens[i]==-1) { + ++letterCount; + } + } + if(!beQuiet) { + printf("number of letters used in the names: %d\n", (int)letterCount); + } + + /* do we need double-byte tokens? */ + if(wordCount+letterCount<=256) { + /* no, single-byte tokens are enough */ + leadByteCount=0; + for(i=0, wordNumber=0; wordNumber<(int16_t)wordCount; ++i) { + if(tokens[i]!=-1) { + tokens[i]=wordNumber; + if(beVerbose) { + printf("tokens[0x%03x]: word%8ld \"%.*s\"\n", + (int)i, (long)words[wordNumber].weight, + words[wordNumber].length, words[wordNumber].s); + } + ++wordNumber; + } + } + tokenCount=i; + } else { + /* + * The tokens that need two token bytes + * get their weight reduced by their count + * because they save less. + */ + tokenCount=256-letterCount; + for(i=tokenCount; i0 && words[wordCount-1].weight<1) { + --wordCount; + } + + /* how many tokens and lead bytes do we have now? */ + tokenCount=wordCount+letterCount+(LEADBYTE_LIMIT-1); + /* + * adjust upwards to take into account that + * double-byte tokens must not + * use NAME_SEPARATOR_CHAR as a second byte + */ + tokenCount+=(tokenCount-256+254)/255; + + leadByteCount=(int16_t)(tokenCount>>8); + if(leadByteCountcode; + + /* segment the lines to groups of 32 */ + if(inLine>>GROUP_SHIFT!=groupMSB) { + /* finish the current group with empty lines */ + while((++outLine&GROUP_MASK)!=0) { + appendLineLength(0); + } + + /* store the group like a line */ + if(groupTop>0) { + if(groupTop>GROUP_STORE_SIZE) { + fprintf(stderr, "gennames: group store overflow\n"); + exit(U_BUFFER_OVERFLOW_ERROR); + } + addGroup(groupMSB, groupStore, groupTop); + if(lineTop>(uint32_t)(line->s-stringStore)) { + fprintf(stderr, "gennames: group store runs into string store\n"); + exit(U_INTERNAL_PROGRAM_ERROR); + } + } + + /* start the new group */ + lineLengthsTop=0; + groupTop=0; + groupMSB=inLine>>GROUP_SHIFT; + outLine=(inLine&~GROUP_MASK)-1; + } + + /* write empty lines between the previous line in the group and this one */ + while(++outLines, line->length, &groupTop)); + } + + /* finish and store the last group */ + if(line && groupMSB!=0xffff) { + /* finish the current group with empty lines */ + while((++outLine&GROUP_MASK)!=0) { + appendLineLength(0); + } + + /* store the group like a line */ + if(groupTop>0) { + if(groupTop>GROUP_STORE_SIZE) { + fprintf(stderr, "gennames: group store overflow\n"); + exit(U_BUFFER_OVERFLOW_ERROR); + } + addGroup(groupMSB, groupStore, groupTop); + if(lineTop>(uint32_t)(line->s-stringStore)) { + fprintf(stderr, "gennames: group store runs into string store\n"); + exit(U_INTERNAL_PROGRAM_ERROR); + } + } + } + + if(!beQuiet) { + printf("number of groups: %lu\n", (unsigned long)lineCount); + } +} + +static int16_t +compressLine(uint8_t *s, int16_t length, int16_t *pGroupTop) { + int16_t start, limit, token, groupTop=*pGroupTop; + + start=0; + do { + /* write any "noise" characters */ + limit=skipNoise((char *)s, start, length); + while(start0xff) { + groupStore[groupTop++]=(uint8_t)(token>>8); + } + groupStore[groupTop++]=(uint8_t)token; + start=limit; + } else { + while(startweight-((Word *)word1)->weight; +} + +/* generate output data ----------------------------------------------------- */ + +static void +generateData(const char *dataDir) { + UNewDataMemory *pData; + UErrorCode errorCode=U_ZERO_ERROR; + uint16_t groupWords[3]; + uint32_t i, groupTop=lineTop, offset, size, + tokenStringOffset, groupsOffset, groupStringOffset, algNamesOffset; + long dataLength; + int16_t token; + + pData=udata_create(dataDir, DATA_TYPE,DATA_NAME, &dataInfo, + haveCopyright ? U_COPYRIGHT_STRING : NULL, &errorCode); + if(U_FAILURE(errorCode)) { + fprintf(stderr, "gennames: unable to create data memory, error %d\n", errorCode); + exit(errorCode); + } + + /* first, see how much space we need, and prepare the token strings */ + for(i=0; i256, then a semicolon (NAME_SEPARATOR_CHAR) is used + * and will be swapped between ASCII and EBCDIC between + * positions 0x3b (ASCII semicolon) and 0x5e (EBCDIC semicolon). + * This should be the only -1 entry in tokens[256..511] on which the data + * swapper bases its trail byte permutation map (trailMap[]). + * + * It would be sufficient to increase tokenCount so that its lower 8 bits + * are at least 0x5e+1 to make room for swapping between the two semicolons. + * For values higher than 0x5e, the trail byte permutation map (trailMap[]) + * should always be an identity map, where we do not need additional room. + */ + i=tokenCount; + tokenCount=(tokenCount+0xff)&~0xff; + if(!beQuiet && i>16); + groupWords[2]=(uint16_t)(offset); + udata_writeBlock(pData, groupWords, 6); + } + + /* group strings */ + udata_writeBlock(pData, stringStore, groupTop); + + /* 4-align the algorithmic names data */ + udata_writePadding(pData, algNamesOffset-(groupStringOffset+groupTop)); + + generateAlgorithmicData(pData); + + /* finish up */ + dataLength=udata_finish(pData, &errorCode); + if(U_FAILURE(errorCode)) { + fprintf(stderr, "gennames: error %d writing the output file\n", errorCode); + exit(errorCode); + } + + if(dataLength!=(long)size) { + fprintf(stderr, "gennames: data length %ld != calculated size %lu\n", +dataLength, (unsigned long)size); + exit(U_INTERNAL_PROGRAM_ERROR); + } +} + +/* the structure for algorithmic names needs to be 4-aligned */ +typedef struct AlgorithmicRange { + uint32_t rangeStart, rangeEnd; + uint8_t algorithmType, algorithmVariant; + uint16_t rangeSize; +} AlgorithmicRange; + +static uint32_t +generateAlgorithmicData(UNewDataMemory *pData) { + static char prefix[] = "CJK UNIFIED IDEOGRAPH-"; +# define PREFIX_LENGTH 23 +# define PREFIX_LENGTH_4 24 + uint32_t countAlgRanges; + + static AlgorithmicRange cjkExtA={ + 0x3400, 0x4db5, + 0, 4, + sizeof(AlgorithmicRange)+PREFIX_LENGTH_4 + }; + static AlgorithmicRange cjk={ + 0x4e00, 0x9fa5, + 0, 4, + sizeof(AlgorithmicRange)+PREFIX_LENGTH_4 + }; + static AlgorithmicRange cjkExtB={ + 0x20000, 0x2a6d6, + 0, 5, + sizeof(AlgorithmicRange)+PREFIX_LENGTH_4 + }; + + static char jamo[]= + "HANGUL SYLLABLE \0" + + "G\0GG\0N\0D\0DD\0R\0M\0B\0BB\0" + "S\0SS\0\0J\0JJ\0C\0K\0T\0P\0H\0" + + "A\0AE\0YA\0YAE\0EO\0E\0YEO\0YE\0O\0" + "WA\0WAE\0OE\0YO\0U\0WEO\0WE\0WI\0" + "YU\0EU\0YI\0I\0" + + "\0G\0GG\0GS\0N\0NJ\0NH\0D\0L\0LG\0LM\0" + "LB\0LS\0LT\0LP\0LH\0M\0B\0BS\0" + "S\0SS\0NG\0J\0C\0K\0T\0P\0H" + ; + + static AlgorithmicRange hangul={ + 0xac00, 0xd7a3, + 1, 3, + sizeof(AlgorithmicRange)+6+sizeof(jamo) + }; + + /* modulo factors, maximum 8 */ + /* 3 factors: 19, 21, 28, most-to-least-significant */ + static uint16_t hangulFactors[3]={ + 19, 21, 28 + }; + + uint32_t size; + + size=0; + + if(ucdVersion>=UNI_4_1) { + /* Unicode 4.1 and up has a longer CJK Unihan range than before */ + cjk.rangeEnd=0x9FBB; + } + + /* number of ranges of algorithmic names */ + if(ucdVersion>=UNI_3_1) { + /* Unicode 3.1 and up has 4 ranges including CJK Extension B */ + countAlgRanges=4; + } else if(ucdVersion>=UNI_3_0) { + /* Unicode 3.0 has 3 ranges including CJK Extension A */ + countAlgRanges=3; + } else { + /* Unicode 2.0 has 2 ranges including Hangul and CJK Unihan */ + countAlgRanges=2; + } + + if(pData!=NULL) { + udata_write32(pData, countAlgRanges); + } else { + size+=4; + } + + /* + * each range: + * uint32_t rangeStart + * uint32_t rangeEnd + * uint8_t algorithmType + * uint8_t algorithmVariant + * uint16_t size of range data + * uint8_t[size] data + */ + + /* range 0: cjk extension a */ + if(countAlgRanges>=3) { + if(pData!=NULL) { + udata_writeBlock(pData, &cjkExtA, sizeof(AlgorithmicRange)); + udata_writeString(pData, prefix, PREFIX_LENGTH); + if(PREFIX_LENGTH=4) { + if(pData!=NULL) { + udata_writeBlock(pData, &cjkExtB, sizeof(AlgorithmicRange)); + udata_writeString(pData, prefix, PREFIX_LENGTH); + if(PREFIX_LENGTH=0 && length==words[token].length && 0==uprv_memcmp(s, words[token].s, length)) { + return i; + } + } + + return -1; +} + +static Word * +findWord(char *s, int16_t length) { + uint32_t i; + + for(i=0; iweight=-(length+1+2); + word->count=0; + word->length=length; + word->s=stringStart; + + ++wordCount; + + return word; +} + +static void +countWord(Word *word) { + /* add to the weight the savings: the length of the word minus 1 byte for the token */ + word->weight+=word->length-1; + ++word->count; +} + +static void +addLine(uint32_t code, char *names[], int16_t lengths[], int16_t count) { + uint8_t *stringStart; + Line *line; + int16_t i, length; + + if(lineCount==MAX_LINE_COUNT) { + fprintf(stderr, "gennames: too many lines\n"); + exit(U_BUFFER_OVERFLOW_ERROR); + } + + /* find the last non-empty name */ + while(count>0 && lengths[count-1]==0) { + --count; + } + if(count==0) { + return; /* should not occur: caller should not have called */ + } + + /* there will be (count-1) separator characters */ + i=count; + length=count-1; + + /* add lengths of strings */ + while(i>0) { + length+=lengths[--i]; + } + + /* allocate line memory */ + stringStart=allocLine(length); + + /* copy all strings into the line memory */ + length=0; /* number of chars copied so far */ + for(i=0; i0) { + stringStart[length++]=NAME_SEPARATOR_CHAR; + } + if(lengths[i]>0) { + uprv_memcpy(stringStart+length, names[i], lengths[i]); + length+=lengths[i]; + } + } + + line=lines+lineCount; + + line->code=code; + line->length=length; + line->s=stringStart; + + ++lineCount; + + /* prevent a character value that is actually in a name from becoming a token */ + while(length>0) { + tokens[stringStart[--length]]=-1; + } +} + +static void +addGroup(uint32_t groupMSB, uint8_t *strings, int16_t length) { + uint8_t *stringStart; + Line *line; + + if(lineCount==MAX_LINE_COUNT) { + fprintf(stderr, "gennames: too many groups\n"); + exit(U_BUFFER_OVERFLOW_ERROR); + } + + /* store the line lengths first, then the strings */ + lineLengthsTop=(lineLengthsTop+1)/2; + stringStart=allocLine(lineLengthsTop+length); + uprv_memcpy(stringStart, lineLengths, lineLengthsTop); + uprv_memcpy(stringStart+lineLengthsTop, strings, length); + + line=lines+lineCount; + + line->code=groupMSB; + line->length=length; + line->s=stringStart; + + ++lineCount; +} + +static uint32_t +addToken(uint8_t *s, int16_t length) { + uint8_t *stringStart; + + stringStart=allocLine(length+1); + uprv_memcpy(stringStart, s, length); + stringStart[length]=0; + + return (uint32_t)(stringStart - stringStore); +} + +static void +appendLineLength(int16_t length) { + if(length>=76) { + fprintf(stderr, "gennames: compressed line too long\n"); + exit(U_BUFFER_OVERFLOW_ERROR); + } + if(length>=12) { + length-=12; + appendLineLengthNibble((uint8_t)((length>>4)|12)); + } + appendLineLengthNibble((uint8_t)length); +} + +static void +appendLineLengthNibble(uint8_t nibble) { + if((lineLengthsTop&1)==0) { + lineLengths[lineLengthsTop/2]=(uint8_t)(nibble<<4); + } else { + lineLengths[lineLengthsTop/2]|=nibble&0xf; + } + ++lineLengthsTop; +} + +static uint8_t * +allocLine(int32_t length) { + uint32_t top=lineTop+length; + uint8_t *p; + + if(top>wordBottom) { + fprintf(stderr, "gennames: out of memory\n"); + exit(U_MEMORY_ALLOCATION_ERROR); + } + p=stringStore+lineTop; + lineTop=top; + return p; +} + +static uint8_t * +allocWord(uint32_t length) { + uint32_t bottom=wordBottom-length; + + if(lineTop>bottom) { + fprintf(stderr, "gennames: out of memory\n"); + exit(U_MEMORY_ALLOCATION_ERROR); + } + wordBottom=bottom; + return stringStore+bottom; +} + +/* + * Hey, Emacs, please set the following: + * + * Local Variables: + * indent-tabs-mode: nil + * End: + * + */ diff --git a/Build/source/libs/icu-xetex/tools/gennames/gennames.vcproj b/Build/source/libs/icu-xetex/tools/gennames/gennames.vcproj new file mode 100644 index 00000000000..1ac7db3c1eb --- /dev/null +++ b/Build/source/libs/icu-xetex/tools/gennames/gennames.vcproj @@ -0,0 +1,226 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + -- cgit v1.2.3