diff options
Diffstat (limited to 'Build/source/libs/icu/icu-xetex/tools/toolutil')
29 files changed, 10260 insertions, 0 deletions
diff --git a/Build/source/libs/icu/icu-xetex/tools/toolutil/Makefile.in b/Build/source/libs/icu/icu-xetex/tools/toolutil/Makefile.in new file mode 100644 index 00000000000..be483c88702 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/tools/toolutil/Makefile.in @@ -0,0 +1,129 @@ +#****************************************************************************** +# +# Copyright (C) 1999-2007, International Business Machines +# Corporation and others. All Rights Reserved. +# +#****************************************************************************** +## Makefile.in for ICU - tools/toolutil +## Steven R. Loomis + +## Source directory information +srcdir = @srcdir@ +top_srcdir = @top_srcdir@ + +top_builddir = ../.. + +## All the flags and other definitions are included here. +include $(top_builddir)/icudefs.mk + +## Build directory information +subdir = tools/toolutil + +## Extra files to remove for 'make clean' +CLEANFILES = *~ $(DEPS) $(IMPORT_LIB) $(MIDDLE_IMPORT_LIB) $(FINAL_IMPORT_LIB) + +## Target information + +TARGET_STUBNAME=$(TOOLUTIL_STUBNAME) + +ifneq ($(ENABLE_STATIC),) +TARGET = $(LIBDIR)/$(LIBSICU)$(TARGET_STUBNAME)$(ICULIBSUFFIX).$(A) +endif + +ifneq ($(ENABLE_SHARED),) +SO_TARGET = $(LIBDIR)/$(LIBICU)$(TARGET_STUBNAME)$(ICULIBSUFFIX).$(SO) +ALL_SO_TARGETS = $(SO_TARGET) $(MIDDLE_SO_TARGET) $(FINAL_SO_TARGET) $(SHARED_OBJECT) +endif + +ALL_TARGETS = $(TARGET) $(ALL_SO_TARGETS) + +DYNAMICCPPFLAGS = $(SHAREDLIBCPPFLAGS) +DYNAMICCFLAGS = $(SHAREDLIBCFLAGS) +DYNAMICCXXFLAGS = $(SHAREDLIBCXXFLAGS) +CFLAGS += $(LIBCFLAGS) +CXXFLAGS += $(LIBCXXFLAGS) + +ifneq ($(top_builddir),$(top_srcdir)) +CPPFLAGS += -I$(top_builddir)/common +endif +CPPFLAGS += -I$(top_srcdir)/common -I$(top_srcdir)/i18n $(LIBCPPFLAGS) +DEFS += -DU_TOOLUTIL_IMPLEMENTATION +LDFLAGS += $(LDFLAGSICUTOOLUTIL) +LIBS = $(LIBICUI18N) $(LIBICUUC) $(DEFAULT_LIBS) + +OBJECTS = filestrm.o package.o pkgitems.o propsvec.o swapimpl.o toolutil.o unewdata.o \ +ucm.o ucmstate.o uoptions.o uparse.o \ +ucbuf.o xmlparser.o writesrc.o + +STATIC_OBJECTS = $(OBJECTS:.o=.$(STATIC_O)) + +DEPS = $(OBJECTS:.o=.d) + +-include Makefile.local + +## List of phony targets +.PHONY : all all-local install install-local clean clean-local \ +distclean distclean-local install-library dist \ +dist-local check check-local + +## Clear suffix list +.SUFFIXES : + +## List of standard targets +all: all-local +install: install-local +clean: clean-local +distclean : distclean-local +dist: dist-local +check: all check-local + +all-local: $(ALL_TARGETS) + +install-local: install-library + +install-library: all-local + $(MKINSTALLDIRS) $(DESTDIR)$(libdir) +ifneq ($(ENABLE_SHARED),) + $(INSTALL-L) $(FINAL_SO_TARGET) $(DESTDIR)$(libdir) +ifneq ($(FINAL_SO_TARGET),$(SO_TARGET)) + cd $(DESTDIR)$(libdir) && $(RM) $(notdir $(SO_TARGET)) && ln -s $(notdir $(FINAL_SO_TARGET)) $(notdir $(SO_TARGET)) +ifneq ($(FINAL_SO_TARGET),$(MIDDLE_SO_TARGET)) + cd $(DESTDIR)$(libdir) && $(RM) $(notdir $(MIDDLE_SO_TARGET)) && ln -s $(notdir $(FINAL_SO_TARGET)) $(notdir $(MIDDLE_SO_TARGET)) +endif +endif +endif + +dist-local: + +clean-local: + test -z "$(CLEANFILES)" || $(RMV) $(CLEANFILES) + $(RMV) $(OBJECTS) $(STATIC_OBJECTS) $(ALL_TARGETS) + +distclean-local: clean-local + $(RMV) Makefile + +check-local: all-local + +Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status + cd $(top_builddir) \ + && CONFIG_FILES=$(subdir)/$@ CONFIG_HEADERS= $(SHELL) ./config.status + +ifneq ($(ENABLE_STATIC),) +$(TARGET): $(STATIC_OBJECTS) + $(AR) $(ARFLAGS) $(AR_OUTOPT)$@ $^ + $(RANLIB) $@ +endif + +ifneq ($(ENABLE_SHARED),) +$(SHARED_OBJECT): $(OBJECTS) + $(SHLIB.cc) $(LD_SONAME) $(OUTOPT)$@ $^ $(LIBS) +endif + +ifeq (,$(MAKECMDGOALS)) +-include $(DEPS) +else +ifneq ($(patsubst %clean,,$(MAKECMDGOALS)),) +-include $(DEPS) +endif +endif + diff --git a/Build/source/libs/icu/icu-xetex/tools/toolutil/filestrm.c b/Build/source/libs/icu/icu-xetex/tools/toolutil/filestrm.c new file mode 100644 index 00000000000..bf646916f61 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/tools/toolutil/filestrm.c @@ -0,0 +1,225 @@ +/* +****************************************************************************** +* +* Copyright (C) 1997-2005, International Business Machines +* Corporation and others. All Rights Reserved. +* +****************************************************************************** +* +* File FILESTRM.C +* +* @author Glenn Marcy +* +* Modification History: +* +* Date Name Description +* 5/8/98 gm Created +* 03/02/99 stephen Reordered params in ungetc to match stdio +* Added wopen +* 3/29/99 helena Merged Stephen and Bertrand's changes. +* +****************************************************************************** +*/ + +#include "filestrm.h" + +#include "cmemory.h" + +#include <stdio.h> + +U_CAPI FileStream* U_EXPORT2 +T_FileStream_open(const char* filename, const char* mode) +{ + if(filename != NULL && *filename != 0 && mode != NULL && *mode != 0) { + FILE *file = fopen(filename, mode); + return (FileStream*)file; + } else { + return NULL; + } +} + +/* +U_CAPI FileStream* U_EXPORT2 +T_FileStream_wopen(const wchar_t* filename, const wchar_t* mode) +{ + // TBD: _wfopen is believed to be MS-specific? +#if defined(U_WINDOWS) + FILE* result = _wfopen(filename, mode); + return (FileStream*)result; +#else + size_t fnMbsSize, mdMbsSize; + char *fn, *md; + FILE *result; + + // convert from wchar_t to char + fnMbsSize = wcstombs(NULL, filename, ((size_t)-1) >> 1); + fn = (char*)uprv_malloc(fnMbsSize+2); + wcstombs(fn, filename, fnMbsSize); + fn[fnMbsSize] = 0; + + mdMbsSize = wcstombs(NULL, mode, ((size_t)-1) >> 1); + md = (char*)uprv_malloc(mdMbsSize+2); + wcstombs(md, mode, mdMbsSize); + md[mdMbsSize] = 0; + + result = fopen(fn, md); + uprv_free(fn); + uprv_free(md); + return (FileStream*)result; +#endif +} +*/ +U_CAPI void U_EXPORT2 +T_FileStream_close(FileStream* fileStream) +{ + if (fileStream != 0) + fclose((FILE*)fileStream); +} + +U_CAPI UBool U_EXPORT2 +T_FileStream_file_exists(const char* filename) +{ + FILE* temp = fopen(filename, "r"); + if (temp) { + fclose(temp); + return TRUE; + } else + return FALSE; +} + +/*static const int32_t kEOF; +const int32_t FileStream::kEOF = EOF;*/ + +/* +U_CAPI FileStream* +T_FileStream_tmpfile() +{ + FILE* file = tmpfile(); + return (FileStream*)file; +} +*/ + +U_CAPI int32_t U_EXPORT2 +T_FileStream_read(FileStream* fileStream, void* addr, int32_t len) +{ + return fread(addr, 1, len, (FILE*)fileStream); +} + +U_CAPI int32_t U_EXPORT2 +T_FileStream_write(FileStream* fileStream, const void* addr, int32_t len) +{ + + return fwrite(addr, 1, len, (FILE*)fileStream); +} + +U_CAPI void U_EXPORT2 +T_FileStream_rewind(FileStream* fileStream) +{ + rewind((FILE*)fileStream); +} + +U_CAPI int32_t U_EXPORT2 +T_FileStream_putc(FileStream* fileStream, int32_t ch) +{ + int32_t c = fputc(ch, (FILE*)fileStream); + return c; +} + +U_CAPI int U_EXPORT2 +T_FileStream_getc(FileStream* fileStream) +{ + int c = fgetc((FILE*)fileStream); + return c; +} + +U_CAPI int32_t U_EXPORT2 +T_FileStream_ungetc(int32_t ch, FileStream* fileStream) +{ + + int32_t c = ungetc(ch, (FILE*)fileStream); + return c; +} + +U_CAPI int32_t U_EXPORT2 +T_FileStream_peek(FileStream* fileStream) +{ + int32_t c = fgetc((FILE*)fileStream); + return ungetc(c, (FILE*)fileStream); +} + +U_CAPI char* U_EXPORT2 +T_FileStream_readLine(FileStream* fileStream, char* buffer, int32_t length) +{ + return fgets(buffer, length, (FILE*)fileStream); +} + +U_CAPI int32_t U_EXPORT2 +T_FileStream_writeLine(FileStream* fileStream, const char* buffer) +{ + return fputs(buffer, (FILE*)fileStream); +} + +U_CAPI int32_t U_EXPORT2 +T_FileStream_size(FileStream* fileStream) +{ + int32_t savedPos = ftell((FILE*)fileStream); + int32_t size = 0; + + /*Changes by Bertrand A. D. doesn't affect the current position + goes to the end of the file before ftell*/ + fseek((FILE*)fileStream, 0, SEEK_END); + size = (int32_t)ftell((FILE*)fileStream); + fseek((FILE*)fileStream, savedPos, SEEK_SET); + return size; +} + +U_CAPI int U_EXPORT2 +T_FileStream_eof(FileStream* fileStream) +{ + return feof((FILE*)fileStream); +} + +/* + Warning + This function may not work consistently on all platforms + (e.g. HP-UX, FreeBSD and MacOSX don't return an error when + putc is used on a file opened as readonly) +*/ +U_CAPI int U_EXPORT2 +T_FileStream_error(FileStream* fileStream) +{ + return (fileStream == 0 || ferror((FILE*)fileStream)); +} + +/* This function doesn't work. */ +/* force the stream to set its error flag*/ +/*U_CAPI void U_EXPORT2 +T_FileStream_setError(FileStream* fileStream) +{ + fseek((FILE*)fileStream, 99999, SEEK_SET); +} +*/ + +U_CAPI FileStream* U_EXPORT2 +T_FileStream_stdin(void) +{ + return (FileStream*)stdin; +} + +U_CAPI FileStream* U_EXPORT2 +T_FileStream_stdout(void) +{ + return (FileStream*)stdout; +} + + +U_CAPI FileStream* U_EXPORT2 +T_FileStream_stderr(void) +{ + return (FileStream*)stderr; +} + +U_CAPI UBool U_EXPORT2 +T_FileStream_remove(const char* fileName){ + return (remove(fileName) == 0); +} diff --git a/Build/source/libs/icu/icu-xetex/tools/toolutil/filestrm.h b/Build/source/libs/icu/icu-xetex/tools/toolutil/filestrm.h new file mode 100644 index 00000000000..0eeba98b661 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/tools/toolutil/filestrm.h @@ -0,0 +1,104 @@ +/* +****************************************************************************** +* +* Copyright (C) 1997-2005, International Business Machines +* Corporation and others. All Rights Reserved. +* +****************************************************************************** +* +* File FILESTRM.H +* +* Contains FileStream interface +* +* @author Glenn Marcy +* +* Modification History: +* +* Date Name Description +* 5/8/98 gm Created. +* 03/02/99 stephen Reordered params in ungetc to match stdio +* Added wopen +* +****************************************************************************** +*/ + +#ifndef FILESTRM_H +#define FILESTRM_H + +#include "unicode/utypes.h" + +typedef struct _FileStream FileStream; + +U_CAPI FileStream* U_EXPORT2 +T_FileStream_open(const char* filename, const char* mode); + +/* +U_CAPI FileStream* U_EXPORT2 +T_FileStream_wopen(const wchar_t* filename, const wchar_t* mode); +*/ +U_CAPI void U_EXPORT2 +T_FileStream_close(FileStream* fileStream); + +U_CAPI UBool U_EXPORT2 +T_FileStream_file_exists(const char* filename); + +/* +U_CAPI FileStream* U_EXPORT2 +T_FileStream_tmpfile(void); +*/ + +U_CAPI int32_t U_EXPORT2 +T_FileStream_read(FileStream* fileStream, void* addr, int32_t len); + +U_CAPI int32_t U_EXPORT2 +T_FileStream_write(FileStream* fileStream, const void* addr, int32_t len); + +U_CAPI void U_EXPORT2 +T_FileStream_rewind(FileStream* fileStream); + +/*Added by Bertrand A. D. */ +U_CAPI char * U_EXPORT2 +T_FileStream_readLine(FileStream* fileStream, char* buffer, int32_t length); + +U_CAPI int32_t U_EXPORT2 +T_FileStream_writeLine(FileStream* fileStream, const char* buffer); + +U_CAPI int32_t U_EXPORT2 +T_FileStream_putc(FileStream* fileStream, int32_t ch); + +U_CAPI int U_EXPORT2 +T_FileStream_getc(FileStream* fileStream); + +U_CAPI int32_t U_EXPORT2 +T_FileStream_ungetc(int32_t ch, FileStream *fileStream); + +U_CAPI int32_t U_EXPORT2 +T_FileStream_peek(FileStream* fileStream); + +U_CAPI int32_t U_EXPORT2 +T_FileStream_size(FileStream* fileStream); + +U_CAPI int U_EXPORT2 +T_FileStream_eof(FileStream* fileStream); + +U_CAPI int U_EXPORT2 +T_FileStream_error(FileStream* fileStream); + +/* +U_CAPI void U_EXPORT2 +T_FileStream_setError(FileStream* fileStream); +*/ + +U_CAPI FileStream* U_EXPORT2 +T_FileStream_stdin(void); + +U_CAPI FileStream* U_EXPORT2 +T_FileStream_stdout(void); + +U_CAPI FileStream* U_EXPORT2 +T_FileStream_stderr(void); + +U_CAPI UBool U_EXPORT2 +T_FileStream_remove(const char* fileName); + +#endif /* _FILESTRM*/ diff --git a/Build/source/libs/icu/icu-xetex/tools/toolutil/package.cpp b/Build/source/libs/icu/icu-xetex/tools/toolutil/package.cpp new file mode 100644 index 00000000000..c5db3017338 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/tools/toolutil/package.cpp @@ -0,0 +1,1217 @@ +/* +******************************************************************************* +* +* Copyright (C) 1999-2007, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: package.cpp +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2005aug25 +* created by: Markus W. Scherer +* +* Read, modify, and write ICU .dat data package files. +* This is an integral part of the icupkg tool, moved to the toolutil library +* because parts of tool implementations tend to be later shared by +* other tools. +* Subsumes functionality and implementation code from +* gencmn, decmn, and icuswap tools. +*/ + +#include "unicode/utypes.h" +#include "unicode/putil.h" +#include "unicode/udata.h" +#include "cstring.h" +#include "uarrsort.h" +#include "ucmndata.h" +#include "udataswp.h" +#include "swapimpl.h" +#include "toolutil.h" +#include "package.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +// general definitions ----------------------------------------------------- *** + +#define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0])) + +/* UDataInfo cf. udata.h */ +static const UDataInfo dataInfo={ + (uint16_t)sizeof(UDataInfo), + 0, + + U_IS_BIG_ENDIAN, + U_CHARSET_FAMILY, + (uint8_t)sizeof(UChar), + 0, + + {0x43, 0x6d, 0x6e, 0x44}, /* dataFormat="CmnD" */ + {1, 0, 0, 0}, /* formatVersion */ + {3, 0, 0, 0} /* dataVersion */ +}; + +U_CDECL_BEGIN +static void U_CALLCONV +printPackageError(void *context, const char *fmt, va_list args) { + vfprintf((FILE *)context, fmt, args); +} +U_CDECL_END + +static uint16_t +readSwapUInt16(uint16_t x) { + return (uint16_t)((x<<8)|(x>>8)); +} + +// platform types ---------------------------------------------------------- *** + +static const char *types="lb?e"; + +enum { TYPE_L, TYPE_B, TYPE_LE, TYPE_E, TYPE_COUNT }; + +static inline int32_t +makeTypeEnum(uint8_t charset, UBool isBigEndian) { + return 2*(int32_t)charset+isBigEndian; +} + +static inline int32_t +makeTypeEnum(char type) { + return + type == 'l' ? TYPE_L : + type == 'b' ? TYPE_B : + type == 'e' ? TYPE_E : + -1; +} + +static inline char +makeTypeLetter(uint8_t charset, UBool isBigEndian) { + return types[makeTypeEnum(charset, isBigEndian)]; +} + +static inline char +makeTypeLetter(int32_t typeEnum) { + return types[typeEnum]; +} + +static void +makeTypeProps(char type, uint8_t &charset, UBool &isBigEndian) { + int32_t typeEnum=makeTypeEnum(type); + charset=(uint8_t)(typeEnum>>1); + isBigEndian=(UBool)(typeEnum&1); +} + +U_CFUNC const UDataInfo * +getDataInfo(const uint8_t *data, int32_t length, + int32_t &infoLength, int32_t &headerLength, + UErrorCode *pErrorCode) { + const DataHeader *pHeader; + const UDataInfo *pInfo; + + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return NULL; + } + if( data==NULL || + (length>=0 && length<(int32_t)sizeof(DataHeader)) + ) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return NULL; + } + + pHeader=(const DataHeader *)data; + pInfo=&pHeader->info; + if( (length>=0 && length<(int32_t)sizeof(DataHeader)) || + pHeader->dataHeader.magic1!=0xda || + pHeader->dataHeader.magic2!=0x27 || + pInfo->sizeofUChar!=2 + ) { + *pErrorCode=U_UNSUPPORTED_ERROR; + return NULL; + } + + if(pInfo->isBigEndian==U_IS_BIG_ENDIAN) { + headerLength=pHeader->dataHeader.headerSize; + infoLength=pInfo->size; + } else { + headerLength=readSwapUInt16(pHeader->dataHeader.headerSize); + infoLength=readSwapUInt16(pInfo->size); + } + + if( headerLength<(int32_t)sizeof(DataHeader) || + infoLength<(int32_t)sizeof(UDataInfo) || + headerLength<(int32_t)(sizeof(pHeader->dataHeader)+infoLength) || + (length>=0 && length<headerLength) + ) { + *pErrorCode=U_UNSUPPORTED_ERROR; + return NULL; + } + + return pInfo; +} + +static int32_t +getTypeEnumForInputData(const uint8_t *data, int32_t length, + UErrorCode *pErrorCode) { + const UDataInfo *pInfo; + int32_t infoLength, headerLength; + + /* getDataInfo() checks for illegal arguments */ + pInfo=getDataInfo(data, length, infoLength, headerLength, pErrorCode); + if(pInfo==NULL) { + return -1; + } + + return makeTypeEnum(pInfo->charsetFamily, (UBool)pInfo->isBigEndian); +} + +// file handling ----------------------------------------------------------- *** + +static void +extractPackageName(const char *filename, + char pkg[], int32_t capacity) { + const char *basename; + int32_t len; + + basename=findBasename(filename); + len=(int32_t)strlen(basename)-4; /* -4: subtract the length of ".dat" */ + + if(len<=0 || 0!=strcmp(basename+len, ".dat")) { + fprintf(stderr, "icupkg: \"%s\" is not recognized as a package filename (must end with .dat)\n", + basename); + exit(U_ILLEGAL_ARGUMENT_ERROR); + } + + if(len>=capacity) { + fprintf(stderr, "icupkg: the package name \"%s\" is too long (>=%ld)\n", + basename, (long)capacity); + exit(U_ILLEGAL_ARGUMENT_ERROR); + } + + memcpy(pkg, basename, len); + pkg[len]=0; +} + +static int32_t +getFileLength(FILE *f) { + int32_t length; + + fseek(f, 0, SEEK_END); + length=(int32_t)ftell(f); + fseek(f, 0, SEEK_SET); + return length; +} + +/* + * Turn tree separators and alternate file separators into normal file separators. + */ +#if U_TREE_ENTRY_SEP_CHAR==U_FILE_SEP_CHAR && U_FILE_ALT_SEP_CHAR==U_FILE_SEP_CHAR +#define treeToPath(s) +#else +static void +treeToPath(char *s) { + char *t; + + for(t=s; *t!=0; ++t) { + if(*t==U_TREE_ENTRY_SEP_CHAR || *t==U_FILE_ALT_SEP_CHAR) { + *t=U_FILE_SEP_CHAR; + } + } +} +#endif + +/* + * Turn file separators into tree separators. + */ +#if U_TREE_ENTRY_SEP_CHAR==U_FILE_SEP_CHAR && U_FILE_ALT_SEP_CHAR==U_FILE_SEP_CHAR +#define pathToTree(s) +#else +static void +pathToTree(char *s) { + char *t; + + for(t=s; *t!=0; ++t) { + if(*t==U_FILE_SEP_CHAR || *t==U_FILE_ALT_SEP_CHAR) { + *t=U_TREE_ENTRY_SEP_CHAR; + } + } +} +#endif + +/* + * Prepend the path (if any) to the name and run the name through treeToName(). + */ +static void +makeFullFilename(const char *path, const char *name, + char *filename, int32_t capacity) { + char *s; + + // prepend the path unless NULL or empty + if(path!=NULL && path[0]!=0) { + if((int32_t)(strlen(path)+1)>=capacity) { + fprintf(stderr, "pathname too long: \"%s\"\n", path); + exit(U_BUFFER_OVERFLOW_ERROR); + } + strcpy(filename, path); + + // make sure the path ends with a file separator + s=strchr(filename, 0); + if(*(s-1)!=U_FILE_SEP_CHAR && *(s-1)!=U_FILE_ALT_SEP_CHAR) { + *s++=U_FILE_SEP_CHAR; + } + } else { + s=filename; + } + + // turn the name into a filename, turn tree separators into file separators + if((int32_t)((s-filename)+strlen(name))>=capacity) { + fprintf(stderr, "path/filename too long: \"%s%s\"\n", filename, name); + exit(U_BUFFER_OVERFLOW_ERROR); + } + strcpy(s, name); + treeToPath(s); +} + +static void +makeFullFilenameAndDirs(const char *path, const char *name, + char *filename, int32_t capacity) { + char *sep; + UErrorCode errorCode; + + makeFullFilename(path, name, filename, capacity); + + // make tree directories + errorCode=U_ZERO_ERROR; + sep=strchr(filename, 0)-strlen(name); + while((sep=strchr(sep, U_FILE_SEP_CHAR))!=NULL) { + if(sep!=filename) { + *sep=0; // truncate temporarily + uprv_mkdir(filename, &errorCode); + if(U_FAILURE(errorCode)) { + fprintf(stderr, "icupkg: unable to create tree directory \"%s\"\n", filename); + exit(U_FILE_ACCESS_ERROR); + } + } + *sep++=U_FILE_SEP_CHAR; // restore file separator character + } +} + +static uint8_t * +readFile(const char *path, const char *name, int32_t &length, char &type) { + char filename[1024]; + FILE *file; + uint8_t *data; + UErrorCode errorCode; + int32_t fileLength, typeEnum; + + makeFullFilename(path, name, filename, (int32_t)sizeof(filename)); + + /* open the input file, get its length, allocate memory for it, read the file */ + file=fopen(filename, "rb"); + if(file==NULL) { + fprintf(stderr, "icupkg: unable to open input file \"%s\"\n", filename); + exit(U_FILE_ACCESS_ERROR); + } + + /* get the file length */ + fileLength=getFileLength(file); + if(ferror(file) || fileLength<=0) { + fprintf(stderr, "icupkg: empty input file \"%s\"\n", filename); + fclose(file); + exit(U_FILE_ACCESS_ERROR); + } + + /* allocate the buffer, pad to multiple of 16 */ + length=(fileLength+0xf)&~0xf; + data=(uint8_t *)malloc(length); + if(data==NULL) { + fclose(file); + exit(U_MEMORY_ALLOCATION_ERROR); + } + + /* read the file */ + if(fileLength!=(int32_t)fread(data, 1, fileLength, file)) { + fprintf(stderr, "icupkg: error reading \"%s\"\n", filename); + fclose(file); + free(data); + exit(U_FILE_ACCESS_ERROR); + } + + /* pad the file to a multiple of 16 using the usual padding byte */ + if(fileLength<length) { + memset(data+fileLength, 0xaa, length-fileLength); + } + + fclose(file); + + // minimum check for ICU-format data + errorCode=U_ZERO_ERROR; + typeEnum=getTypeEnumForInputData(data, length, &errorCode); + if(typeEnum<0 || U_FAILURE(errorCode)) { + fprintf(stderr, "icupkg: not an ICU data file: \"%s\"\n", filename); + free(data); + exit(U_INVALID_FORMAT_ERROR); + } + type=makeTypeLetter(typeEnum); + + return data; +} + +// .dat package file representation ---------------------------------------- *** + +U_CDECL_BEGIN + +static int32_t U_CALLCONV +compareItems(const void * /*context*/, const void *left, const void *right) { + U_NAMESPACE_USE + + return (int32_t)strcmp(((Item *)left)->name, ((Item *)right)->name); +} + +U_CDECL_END + +U_NAMESPACE_BEGIN + +Package::Package() { + inPkgName[0]=0; + inData=NULL; + inLength=0; + inCharset=U_CHARSET_FAMILY; + inIsBigEndian=U_IS_BIG_ENDIAN; + + itemCount=0; + inStringTop=outStringTop=0; + + matchMode=0; + findPrefix=findSuffix=NULL; + findPrefixLength=findSuffixLength=0; + findNextIndex=-1; + + // create a header for an empty package + DataHeader *pHeader; + pHeader=(DataHeader *)header; + pHeader->dataHeader.magic1=0xda; + pHeader->dataHeader.magic2=0x27; + memcpy(&pHeader->info, &dataInfo, sizeof(dataInfo)); + headerLength=(int32_t)(4+sizeof(dataInfo)); + if(headerLength&0xf) { + /* NUL-pad the header to a multiple of 16 */ + int32_t length=(headerLength+0xf)&~0xf; + memset(header+headerLength, 0, length-headerLength); + headerLength=length; + } + pHeader->dataHeader.headerSize=(uint16_t)headerLength; +} + +Package::~Package() { + int32_t index; + + free(inData); + + for(index=0; index<itemCount; ++index) { + if(items[index].isDataOwned) { + free(items[index].data); + } + } +} + +void +Package::readPackage(const char *filename) { + UDataSwapper *ds; + const UDataInfo *pInfo; + UErrorCode errorCode; + + const uint8_t *inBytes; + + int32_t length, offset, i; + int32_t itemLength, typeEnum; + char type; + + const UDataOffsetTOCEntry *inEntries; + + extractPackageName(filename, inPkgName, (int32_t)sizeof(inPkgName)); + + /* read the file */ + inData=readFile(NULL, filename, inLength, type); + length=inLength; + + /* + * swap the header - even if the swapping itself is a no-op + * because it tells us the header length + */ + errorCode=U_ZERO_ERROR; + makeTypeProps(type, inCharset, inIsBigEndian); + ds=udata_openSwapper(inIsBigEndian, inCharset, U_IS_BIG_ENDIAN, U_CHARSET_FAMILY, &errorCode); + if(U_FAILURE(errorCode)) { + fprintf(stderr, "icupkg: udata_openSwapper(\"%s\") failed - %s\n", + filename, u_errorName(errorCode)); + exit(errorCode); + } + + ds->printError=printPackageError; + ds->printErrorContext=stderr; + + headerLength=sizeof(header); + if(length<headerLength) { + headerLength=length; + } + headerLength=udata_swapDataHeader(ds, inData, headerLength, header, &errorCode); + if(U_FAILURE(errorCode)) { + exit(errorCode); + } + + /* check data format and format version */ + pInfo=(const UDataInfo *)((const char *)inData+4); + if(!( + pInfo->dataFormat[0]==0x43 && /* dataFormat="CmnD" */ + pInfo->dataFormat[1]==0x6d && + pInfo->dataFormat[2]==0x6e && + pInfo->dataFormat[3]==0x44 && + pInfo->formatVersion[0]==1 + )) { + fprintf(stderr, "icupkg: data format %02x.%02x.%02x.%02x (format version %02x) is not recognized as an ICU .dat package\n", + pInfo->dataFormat[0], pInfo->dataFormat[1], + pInfo->dataFormat[2], pInfo->dataFormat[3], + pInfo->formatVersion[0]); + exit(U_UNSUPPORTED_ERROR); + } + inIsBigEndian=(UBool)pInfo->isBigEndian; + inCharset=pInfo->charsetFamily; + + inBytes=(const uint8_t *)inData+headerLength; + inEntries=(const UDataOffsetTOCEntry *)(inBytes+4); + + /* check that the itemCount fits, then the ToC table, then at least the header of the last item */ + length-=headerLength; + if(length<4) { + /* itemCount does not fit */ + offset=0x7fffffff; + } else { + itemCount=udata_readInt32(ds, *(const int32_t *)inBytes); + if(itemCount==0) { + offset=4; + } else if(length<(4+8*itemCount)) { + /* ToC table does not fit */ + offset=0x7fffffff; + } else { + /* offset of the last item plus at least 20 bytes for its header */ + offset=20+(int32_t)ds->readUInt32(inEntries[itemCount-1].dataOffset); + } + } + if(length<offset) { + fprintf(stderr, "icupkg: too few bytes (%ld after header) for a .dat package\n", + (long)length); + exit(U_INDEX_OUTOFBOUNDS_ERROR); + } + /* do not modify the package length variable until the last item's length is set */ + + if(itemCount>0) { + char prefix[MAX_PKG_NAME_LENGTH+4]; + char *s, *inItemStrings; + int32_t inPkgNameLength, prefixLength, stringsOffset; + + if(itemCount>MAX_FILE_COUNT) { + fprintf(stderr, "icupkg: too many items, maximum is %d\n", MAX_FILE_COUNT); + exit(U_BUFFER_OVERFLOW_ERROR); + } + + /* swap the item name strings */ + stringsOffset=4+8*itemCount; + itemLength=(int32_t)(ds->readUInt32(inEntries[0].dataOffset))-stringsOffset; + + // don't include padding bytes at the end of the item names + while(itemLength>0 && inBytes[stringsOffset+itemLength-1]!=0) { + --itemLength; + } + + if((inStringTop+itemLength)>STRING_STORE_SIZE) { + fprintf(stderr, "icupkg: total length of item name strings too long\n"); + exit(U_BUFFER_OVERFLOW_ERROR); + } + + inItemStrings=inStrings+inStringTop; + ds->swapInvChars(ds, inBytes+stringsOffset, itemLength, inItemStrings, &errorCode); + if(U_FAILURE(errorCode)) { + fprintf(stderr, "icupkg failed to swap the input .dat package item name strings\n"); + exit(U_INVALID_FORMAT_ERROR); + } + inStringTop+=itemLength; + + // reset the Item entries + memset(items, 0, itemCount*sizeof(Item)); + + inPkgNameLength=strlen(inPkgName); + memcpy(prefix, inPkgName, inPkgNameLength); + prefixLength=inPkgNameLength; + + /* + * Get the common prefix of the items. + * New-style ICU .dat packages use tree separators ('/') between package names, + * tree names, and item names, + * while old-style ICU .dat packages (before multi-tree support) + * use an underscore ('_') between package and item names. + */ + offset=(int32_t)ds->readUInt32(inEntries[0].nameOffset)-stringsOffset; + s=inItemStrings+offset; + if( (int32_t)strlen(s)>=(inPkgNameLength+2) && + 0==memcmp(s, inPkgName, inPkgNameLength) && + s[inPkgNameLength]=='_' + ) { + // old-style .dat package + prefix[prefixLength++]='_'; + } else { + // new-style .dat package + prefix[prefixLength++]=U_TREE_ENTRY_SEP_CHAR; + // if it turns out to not contain U_TREE_ENTRY_SEP_CHAR + // then the test in the loop below will fail + } + prefix[prefixLength]=0; + + /* read the ToC table */ + for(i=0; i<itemCount; ++i) { + // skip the package part of the item name, error if it does not match the actual package name + // or if nothing follows the package name + offset=(int32_t)ds->readUInt32(inEntries[i].nameOffset)-stringsOffset; + s=inItemStrings+offset; + if(0!=strncmp(s, prefix, prefixLength) || s[prefixLength]==0) { + fprintf(stderr, "icupkg: input .dat item name \"%s\" does not start with \"%s\"\n", + s, prefix); + exit(U_UNSUPPORTED_ERROR); + } + items[i].name=s+prefixLength; + + // set the item's data + items[i].data=(uint8_t *)inBytes+ds->readUInt32(inEntries[i].dataOffset); + if(i>0) { + items[i-1].length=(int32_t)(items[i].data-items[i-1].data); + + // set the previous item's platform type + typeEnum=getTypeEnumForInputData(items[i-1].data, items[i-1].length, &errorCode); + if(typeEnum<0 || U_FAILURE(errorCode)) { + fprintf(stderr, "icupkg: not an ICU data file: item \"%s\" in \"%s\"\n", items[i-1].name, filename); + exit(U_INVALID_FORMAT_ERROR); + } + items[i-1].type=makeTypeLetter(typeEnum); + } + items[i].isDataOwned=FALSE; + } + // set the last item's length + items[itemCount-1].length=length-ds->readUInt32(inEntries[itemCount-1].dataOffset); + + // set the last item's platform type + typeEnum=getTypeEnumForInputData(items[itemCount-1].data, items[itemCount-1].length, &errorCode); + if(typeEnum<0 || U_FAILURE(errorCode)) { + fprintf(stderr, "icupkg: not an ICU data file: item \"%s\" in \"%s\"\n", items[i-1].name, filename); + exit(U_INVALID_FORMAT_ERROR); + } + items[itemCount-1].type=makeTypeLetter(typeEnum); + + if(type!=U_ICUDATA_TYPE_LETTER[0]) { + // sort the item names for the local charset + sortItems(); + } + } + + udata_closeSwapper(ds); +} + +char +Package::getInType() { + return makeTypeLetter(inCharset, inIsBigEndian); +} + +void +Package::writePackage(const char *filename, char outType, const char *comment) { + char prefix[MAX_PKG_NAME_LENGTH+4]; + UDataOffsetTOCEntry entry; + UDataSwapper *dsLocalToOut, *ds[TYPE_COUNT]; + FILE *file; + Item *pItem; + char *name; + UErrorCode errorCode; + int32_t i, length, prefixLength, maxItemLength, basenameOffset, offset, outInt32; + uint8_t outCharset; + UBool outIsBigEndian; + + extractPackageName(filename, prefix, MAX_PKG_NAME_LENGTH); + + // if there is an explicit comment, then use it, else use what's in the current header + if(comment!=NULL) { + /* get the header size minus the current comment */ + DataHeader *pHeader; + int32_t length; + + pHeader=(DataHeader *)header; + headerLength=4+pHeader->info.size; + length=(int32_t)strlen(comment); + if((int32_t)(headerLength+length)>=(int32_t)sizeof(header)) { + fprintf(stderr, "icupkg: comment too long\n"); + exit(U_BUFFER_OVERFLOW_ERROR); + } + memcpy(header+headerLength, comment, length+1); + headerLength+=length; + if(headerLength&0xf) { + /* NUL-pad the header to a multiple of 16 */ + length=(headerLength+0xf)&~0xf; + memset(header+headerLength, 0, length-headerLength); + headerLength=length; + } + pHeader->dataHeader.headerSize=(uint16_t)headerLength; + } + + makeTypeProps(outType, outCharset, outIsBigEndian); + + // open (TYPE_COUNT-2) swappers + // one is a no-op for local type==outType + // one type (TYPE_LE) is bogus + errorCode=U_ZERO_ERROR; + i=makeTypeEnum(outType); + ds[TYPE_B]= i==TYPE_B ? NULL : udata_openSwapper(TRUE, U_ASCII_FAMILY, outIsBigEndian, outCharset, &errorCode); + ds[TYPE_L]= i==TYPE_L ? NULL : udata_openSwapper(FALSE, U_ASCII_FAMILY, outIsBigEndian, outCharset, &errorCode); + ds[TYPE_LE]=NULL; + ds[TYPE_E]= i==TYPE_E ? NULL : udata_openSwapper(TRUE, U_EBCDIC_FAMILY, outIsBigEndian, outCharset, &errorCode); + if(U_FAILURE(errorCode)) { + fprintf(stderr, "icupkg: udata_openSwapper() failed - %s\n", u_errorName(errorCode)); + exit(errorCode); + } + for(i=0; i<TYPE_COUNT; ++i) { + if(ds[i]!=NULL) { + ds[i]->printError=printPackageError; + ds[i]->printErrorContext=stderr; + } + } + + dsLocalToOut=ds[makeTypeEnum(U_CHARSET_FAMILY, U_IS_BIG_ENDIAN)]; + + // create the file and write its contents + file=fopen(filename, "wb"); + if(file==NULL) { + fprintf(stderr, "icupkg: unable to create file \"%s\"\n", filename); + exit(U_FILE_ACCESS_ERROR); + } + + // swap and write the header + if(dsLocalToOut!=NULL) { + udata_swapDataHeader(dsLocalToOut, header, headerLength, header, &errorCode); + if(U_FAILURE(errorCode)) { + fprintf(stderr, "icupkg: udata_swapDataHeader(local to out) failed - %s\n", u_errorName(errorCode)); + exit(errorCode); + } + } + length=(int32_t)fwrite(header, 1, headerLength, file); + if(length!=headerLength) { + fprintf(stderr, "icupkg: unable to write complete header to file \"%s\"\n", filename); + exit(U_FILE_ACCESS_ERROR); + } + + // prepare and swap the package name with a tree separator + // for prepending to item names + strcat(prefix, U_TREE_ENTRY_SEP_STRING); + prefixLength=(int32_t)strlen(prefix); + if(dsLocalToOut!=NULL) { + dsLocalToOut->swapInvChars(dsLocalToOut, prefix, prefixLength, prefix, &errorCode); + if(U_FAILURE(errorCode)) { + fprintf(stderr, "icupkg: swapInvChars(output package name) failed - %s\n", u_errorName(errorCode)); + exit(errorCode); + } + + // swap and sort the item names (sorting needs to be done in the output charset) + dsLocalToOut->swapInvChars(dsLocalToOut, inStrings, inStringTop, inStrings, &errorCode); + if(U_FAILURE(errorCode)) { + fprintf(stderr, "icupkg: swapInvChars(item names) failed - %s\n", u_errorName(errorCode)); + exit(errorCode); + } + sortItems(); + } + + // create the output item names in sorted order, with the package name prepended to each + for(i=0; i<itemCount; ++i) { + length=(int32_t)strlen(items[i].name); + name=allocString(FALSE, length+prefixLength); + memcpy(name, prefix, prefixLength); + memcpy(name+prefixLength, items[i].name, length+1); + items[i].name=name; + } + + // calculate offsets for item names and items, pad to 16-align items + // align only the first item; each item's length is a multiple of 16 + basenameOffset=4+8*itemCount; + offset=basenameOffset+outStringTop; + if((length=(offset&15))!=0) { + length=16-length; + memset(allocString(FALSE, length-1), 0xaa, length); + offset+=length; + } + + // write the table of contents + // first the itemCount + outInt32=itemCount; + if(dsLocalToOut!=NULL) { + dsLocalToOut->swapArray32(dsLocalToOut, &outInt32, 4, &outInt32, &errorCode); + if(U_FAILURE(errorCode)) { + fprintf(stderr, "icupkg: swapArray32(item count) failed - %s\n", u_errorName(errorCode)); + exit(errorCode); + } + } + length=(int32_t)fwrite(&outInt32, 1, 4, file); + if(length!=4) { + fprintf(stderr, "icupkg: unable to write complete item count to file \"%s\"\n", filename); + exit(U_FILE_ACCESS_ERROR); + } + + // then write the item entries (and collect the maxItemLength) + maxItemLength=0; + for(i=0; i<itemCount; ++i) { + entry.nameOffset=(uint32_t)(basenameOffset+(items[i].name-outStrings)); + entry.dataOffset=(uint32_t)offset; + if(dsLocalToOut!=NULL) { + dsLocalToOut->swapArray32(dsLocalToOut, &entry, 8, &entry, &errorCode); + if(U_FAILURE(errorCode)) { + fprintf(stderr, "icupkg: swapArray32(item entry %ld) failed - %s\n", (long)i, u_errorName(errorCode)); + exit(errorCode); + } + } + length=(int32_t)fwrite(&entry, 1, 8, file); + if(length!=8) { + fprintf(stderr, "icupkg: unable to write complete item entry %ld to file \"%s\"\n", (long)i, filename); + exit(U_FILE_ACCESS_ERROR); + } + + length=items[i].length; + if(length>maxItemLength) { + maxItemLength=length; + } + offset+=length; + } + + // write the item names + length=(int32_t)fwrite(outStrings, 1, outStringTop, file); + if(length!=outStringTop) { + fprintf(stderr, "icupkg: unable to write complete item names to file \"%s\"\n", filename); + exit(U_FILE_ACCESS_ERROR); + } + + // write the items + for(pItem=items, i=0; i<itemCount; ++pItem, ++i) { + int32_t type=makeTypeEnum(pItem->type); + if(ds[type]!=NULL) { + // swap each item from its platform properties to the desired ones + udata_swap( + ds[type], + pItem->data, pItem->length, pItem->data, + &errorCode); + if(U_FAILURE(errorCode)) { + fprintf(stderr, "icupkg: udata_swap(item %ld) failed - %s\n", (long)i, u_errorName(errorCode)); + exit(errorCode); + } + } + length=(int32_t)fwrite(pItem->data, 1, pItem->length, file); + if(length!=pItem->length) { + fprintf(stderr, "icupkg: unable to write complete item %ld to file \"%s\"\n", (long)i, filename); + exit(U_FILE_ACCESS_ERROR); + } + } + + if(ferror(file)) { + fprintf(stderr, "icupkg: unable to write complete file \"%s\"\n", filename); + exit(U_FILE_ACCESS_ERROR); + } + + fclose(file); + for(i=0; i<TYPE_COUNT; ++i) { + udata_closeSwapper(ds[i]); + } +} + +int32_t +Package::findItem(const char *name, int32_t length) const { + int32_t i, start, limit; + int result; + + /* do a binary search for the string */ + start=0; + limit=itemCount; + while(start<limit) { + i=(start+limit)/2; + if(length>=0) { + result=strncmp(name, items[i].name, length); + } else { + result=strcmp(name, items[i].name); + } + + if(result==0) { + /* found */ + if(length>=0) { + /* + * if we compared just prefixes, then we may need to back up + * to the first item with this prefix + */ + while(i>0 && 0==strncmp(name, items[i-1].name, length)) { + --i; + } + } + return i; + } else if(result<0) { + limit=i; + } else /* result>0 */ { + start=i+1; + } + } + + return ~start; /* not found, return binary-not of the insertion point */ +} + +void +Package::findItems(const char *pattern) { + const char *wild; + + if(pattern==NULL || *pattern==0) { + findNextIndex=-1; + return; + } + + findPrefix=pattern; + findSuffix=NULL; + findSuffixLength=0; + + wild=strchr(pattern, '*'); + if(wild==NULL) { + // no wildcard + findPrefixLength=(int32_t)strlen(pattern); + } else { + // one wildcard + findPrefixLength=(int32_t)(wild-pattern); + findSuffix=wild+1; + findSuffixLength=(int32_t)strlen(findSuffix); + if(NULL!=strchr(findSuffix, '*')) { + // two or more wildcards + fprintf(stderr, "icupkg: syntax error (more than one '*') in item pattern \"%s\"\n", pattern); + exit(U_PARSE_ERROR); + } + } + + if(findPrefixLength==0) { + findNextIndex=0; + } else { + findNextIndex=findItem(findPrefix, findPrefixLength); + } +} + +int32_t +Package::findNextItem() { + const char *name, *middle, *treeSep; + int32_t index, nameLength, middleLength; + + if(findNextIndex<0) { + return -1; + } + + while(findNextIndex<itemCount) { + index=findNextIndex++; + name=items[index].name; + nameLength=(int32_t)strlen(name); + if(nameLength<(findPrefixLength+findSuffixLength)) { + // item name too short for prefix & suffix + continue; + } + if(findPrefixLength>0 && 0!=memcmp(findPrefix, name, findPrefixLength)) { + // left the range of names with this prefix + break; + } + middle=name+findPrefixLength; + middleLength=nameLength-findPrefixLength-findSuffixLength; + if(findSuffixLength>0 && 0!=memcmp(findSuffix, name+(nameLength-findSuffixLength), findSuffixLength)) { + // suffix does not match + continue; + } + // prefix & suffix match + + if(matchMode&MATCH_NOSLASH) { + treeSep=strchr(middle, U_TREE_ENTRY_SEP_CHAR); + if(treeSep!=NULL && (treeSep-middle)<middleLength) { + // the middle (matching the * wildcard) contains a tree separator / + continue; + } + } + + // found a matching item + return index; + } + + // no more items + findNextIndex=-1; + return -1; +} + +void +Package::setMatchMode(uint32_t mode) { + matchMode=mode; +} + +void +Package::addItem(const char *name) { + addItem(name, NULL, 0, FALSE, U_ICUDATA_TYPE_LETTER[0]); +} + +void +Package::addItem(const char *name, uint8_t *data, int32_t length, UBool isDataOwned, char type) { + int32_t index; + + index=findItem(name); + if(index<0) { + // new item, make space at the insertion point + if(itemCount>=MAX_FILE_COUNT) { + fprintf(stderr, "icupkg: too many items, maximum is %d\n", MAX_FILE_COUNT); + exit(U_BUFFER_OVERFLOW_ERROR); + } + // move the following items down + index=~index; + if(index<itemCount) { + memmove(items+index+1, items+index, (itemCount-index)*sizeof(Item)); + } + ++itemCount; + + // reset this Item entry + memset(items+index, 0, sizeof(Item)); + + // copy the item's name + items[index].name=allocString(TRUE, strlen(name)); + strcpy(items[index].name, name); + pathToTree(items[index].name); + } else { + // same-name item found, replace it + if(items[index].isDataOwned) { + free(items[index].data); + } + + // keep the item's name since it is the same + } + + // set the item's data + items[index].data=data; + items[index].length=length; + items[index].isDataOwned=isDataOwned; + items[index].type=type; +} + +void +Package::addFile(const char *filesPath, const char *name) { + uint8_t *data; + int32_t length; + char type; + + data=readFile(filesPath, name, length, type); + // readFile() exits the tool if it fails + addItem(name, data, length, TRUE, type); +} + +void +Package::addItems(const Package &listPkg) { + const Item *pItem; + int32_t i; + + for(pItem=listPkg.items, i=0; i<listPkg.itemCount; ++pItem, ++i) { + addItem(pItem->name, pItem->data, pItem->length, FALSE, pItem->type); + } +} + +void +Package::removeItem(int32_t index) { + if(index>=0) { + // remove the item + if(items[index].isDataOwned) { + free(items[index].data); + } + + // move the following items up + if((index+1)<itemCount) { + memmove(items+index, items+index+1, (itemCount-(index+1))*sizeof(Item)); + } + --itemCount; + + if(index<=findNextIndex) { + --findNextIndex; + } + } +} + +void +Package::removeItems(const char *pattern) { + int32_t index; + + findItems(pattern); + while((index=findNextItem())>=0) { + removeItem(index); + } +} + +void +Package::removeItems(const Package &listPkg) { + const Item *pItem; + int32_t i; + + for(pItem=listPkg.items, i=0; i<listPkg.itemCount; ++pItem, ++i) { + removeItems(pItem->name); + } +} + +void +Package::extractItem(const char *filesPath, const char *outName, int32_t index, char outType) { + char filename[1024]; + UDataSwapper *ds; + FILE *file; + Item *pItem; + int32_t fileLength; + uint8_t itemCharset, outCharset; + UBool itemIsBigEndian, outIsBigEndian; + + if(index<0 || itemCount<=index) { + return; + } + pItem=items+index; + + // swap the data to the outType + // outType==0: don't swap + if(outType!=0 && pItem->type!=outType) { + // open the swapper + UErrorCode errorCode=U_ZERO_ERROR; + makeTypeProps(pItem->type, itemCharset, itemIsBigEndian); + makeTypeProps(outType, outCharset, outIsBigEndian); + ds=udata_openSwapper(itemIsBigEndian, itemCharset, outIsBigEndian, outCharset, &errorCode); + if(U_FAILURE(errorCode)) { + fprintf(stderr, "icupkg: udata_openSwapper(item %ld) failed - %s\n", + (long)index, u_errorName(errorCode)); + exit(errorCode); + } + + ds->printError=printPackageError; + ds->printErrorContext=stderr; + + // swap the item from its platform properties to the desired ones + udata_swap(ds, pItem->data, pItem->length, pItem->data, &errorCode); + if(U_FAILURE(errorCode)) { + fprintf(stderr, "icupkg: udata_swap(item %ld) failed - %s\n", (long)index, u_errorName(errorCode)); + exit(errorCode); + } + udata_closeSwapper(ds); + } + + // create the file and write its contents + makeFullFilenameAndDirs(filesPath, outName, filename, (int32_t)sizeof(filename)); + file=fopen(filename, "wb"); + if(file==NULL) { + fprintf(stderr, "icupkg: unable to create file \"%s\"\n", filename); + exit(U_FILE_ACCESS_ERROR); + } + fileLength=(int32_t)fwrite(pItem->data, 1, pItem->length, file); + + if(ferror(file) || fileLength!=pItem->length) { + fprintf(stderr, "icupkg: unable to write complete file \"%s\"\n", filename); + exit(U_FILE_ACCESS_ERROR); + } + fclose(file); +} + +void +Package::extractItem(const char *filesPath, int32_t index, char outType) { + extractItem(filesPath, items[index].name, index, outType); +} + +void +Package::extractItems(const char *filesPath, const char *pattern, char outType) { + int32_t index; + + findItems(pattern); + while((index=findNextItem())>=0) { + extractItem(filesPath, index, outType); + } +} + +void +Package::extractItems(const char *filesPath, const Package &listPkg, char outType) { + const Item *pItem; + int32_t i; + + for(pItem=listPkg.items, i=0; i<listPkg.itemCount; ++pItem, ++i) { + extractItems(filesPath, pItem->name, outType); + } +} + +int32_t +Package::getItemCount() const { + return itemCount; +} + +const Item * +Package::getItem(int32_t idx) const { + if (0 <= idx && idx < itemCount) { + return &items[idx]; + } + return NULL; +} + +void +Package::checkDependency(void *context, const char *itemName, const char *targetName) { + // check dependency: make sure the target item is in the package + Package *me=(Package *)context; + if(me->findItem(targetName)<0) { + me->isMissingItems=TRUE; + fprintf(stderr, "Item %s depends on missing item %s\n", itemName, targetName); + } +} + +UBool +Package::checkDependencies() { + isMissingItems=FALSE; + enumDependencies(this, checkDependency); + return (UBool)!isMissingItems; +} + +void +Package::enumDependencies(void *context, CheckDependency check) { + int32_t i; + + for(i=0; i<itemCount; ++i) { + enumDependencies(items+i, context, check); + } +} + +char * +Package::allocString(UBool in, int32_t length) { + char *p; + int32_t top; + + if(in) { + top=inStringTop; + p=inStrings+top; + } else { + top=outStringTop; + p=outStrings+top; + } + top+=length+1; + + if(top>STRING_STORE_SIZE) { + fprintf(stderr, "icupkg: string storage overflow\n"); + exit(U_BUFFER_OVERFLOW_ERROR); + } + if(in) { + inStringTop=top; + } else { + outStringTop=top; + } + return p; +} + +void +Package::sortItems() { + UErrorCode errorCode=U_ZERO_ERROR; + uprv_sortArray(items, itemCount, (int32_t)sizeof(Item), compareItems, NULL, FALSE, &errorCode); + if(U_FAILURE(errorCode)) { + fprintf(stderr, "icupkg: sorting item names failed - %s\n", u_errorName(errorCode)); + exit(errorCode); + } +} + +U_NAMESPACE_END diff --git a/Build/source/libs/icu/icu-xetex/tools/toolutil/package.h b/Build/source/libs/icu/icu-xetex/tools/toolutil/package.h new file mode 100644 index 00000000000..067397d8b33 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/tools/toolutil/package.h @@ -0,0 +1,167 @@ +/* +******************************************************************************* +* +* Copyright (C) 2005-2007, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: package.h +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2005aug25 +* created by: Markus W. Scherer +* +* Read, modify, and write ICU .dat data package files. +*/ + +#ifndef __PACKAGE_H__ +#define __PACKAGE_H__ + +#include "unicode/utypes.h" + +#include <stdio.h> + +// .dat package file representation ---------------------------------------- *** + +#define STRING_STORE_SIZE 100000 +#define MAX_FILE_COUNT 2000 +#define MAX_PKG_NAME_LENGTH 32 + +typedef void CheckDependency(void *context, const char *itemName, const char *targetName); + +U_NAMESPACE_BEGIN + +struct Item { + char *name; + uint8_t *data; + int32_t length; + UBool isDataOwned; + char type; +}; + +class U_TOOLUTIL_API Package { +public: + /* + * Constructor. + * Prepare this object for a new, empty package. + */ + Package(); + + /* Destructor. */ + ~Package(); + + /* + * Read an existing .dat package file. + * The header and item name strings are swapped into this object, + * but the items are left unswapped. + */ + void readPackage(const char *filename); + /* + * Write a .dat package file with the items in this object. + * Swap all pieces to the desired output platform properties. + * The package becomes unusable: + * The item names are swapped and sorted in the outCharset rather than the local one. + * Also, the items themselves are swapped in-place + */ + void writePackage(const char *filename, char outType, const char *comment); + + /* + * Return the input data type letter (l, b, or e). + */ + char getInType(); + + // find the item in items[], return the non-negative index if found, else the binary-not of the insertion point + int32_t findItem(const char *name, int32_t length=-1) const; + + /* + * Set internal state for following calls to findNextItem() which will return + * indexes for items whose names match the pattern. + */ + void findItems(const char *pattern); + int32_t findNextItem(); + /* + * Set the match mode for findItems() & findNextItem(). + * @param mode 0=default + * MATCH_NOSLASH * does not match a '/' + */ + void setMatchMode(uint32_t mode); + + enum { + MATCH_NOSLASH=1 + }; + + void addItem(const char *name); + void addItem(const char *name, uint8_t *data, int32_t length, UBool isDataOwned, char type); + void addFile(const char *filesPath, const char *name); + void addItems(const Package &listPkg); + + void removeItem(int32_t index); + void removeItems(const char *pattern); + void removeItems(const Package &listPkg); + + /* The extractItem() functions accept outputType=0 to mean "don't swap the item". */ + void extractItem(const char *filesPath, int32_t index, char outType); + void extractItems(const char *filesPath, const char *pattern, char outType); + void extractItems(const char *filesPath, const Package &listPkg, char outType); + + /* This variant extracts an item to a specific filename. */ + void extractItem(const char *filesPath, const char *outName, int32_t index, char outType); + + int32_t getItemCount() const; + const Item *getItem(int32_t idx) const; + + /* + * Check dependencies and return TRUE if all dependencies are fulfilled. + */ + UBool checkDependencies(); + + /* + * Enumerate all the dependencies and give the results to context and check + */ + void enumDependencies(void *context, CheckDependency check); + +private: + void enumDependencies(Item *pItem, void *context, CheckDependency check); + + static void checkDependency(void *context, const char *itemName, const char *targetName); + + /* + * Allocate a string in inStrings or outStrings. + * The length does not include the terminating NUL. + */ + char *allocString(UBool in, int32_t length); + + void sortItems(); + + // data fields + char inPkgName[MAX_PKG_NAME_LENGTH]; + + uint8_t *inData; + uint8_t header[1024]; + int32_t inLength, headerLength; + uint8_t inCharset; + UBool inIsBigEndian; + + int32_t itemCount; + Item items[MAX_FILE_COUNT]; + + int32_t inStringTop, outStringTop; + char inStrings[STRING_STORE_SIZE], outStrings[STRING_STORE_SIZE]; + + // match mode for findItems(pattern) and findNextItem() + uint32_t matchMode; + + // state for findItems(pattern) and findNextItem() + const char *findPrefix, *findSuffix; + int32_t findPrefixLength, findSuffixLength; + int32_t findNextIndex; + + // state for checkDependencies() + UBool isMissingItems; +}; + +U_NAMESPACE_END + +#endif diff --git a/Build/source/libs/icu/icu-xetex/tools/toolutil/pkg_imp.h b/Build/source/libs/icu/icu-xetex/tools/toolutil/pkg_imp.h new file mode 100644 index 00000000000..0b0c2b740f8 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/tools/toolutil/pkg_imp.h @@ -0,0 +1,35 @@ +/* +******************************************************************************* +* +* Copyright (C) 2005, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: pkg_imp.h +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2005sep18 +* created by: Markus W. Scherer +* +* Implementation definitions for data package functions in toolutil. +*/ + +#ifndef __PKG_IMP_H__ +#define __PKG_IMP_H__ + +#include "unicode/utypes.h" + +/* + * Read an ICU data item with any platform type, + * return the pointer to the UDataInfo in its header, + * and set the lengths of the UDataInfo and of the whole header. + * All data remains in its platform type. + */ +U_CFUNC const UDataInfo * +getDataInfo(const uint8_t *data, int32_t length, + int32_t &infoLength, int32_t &headerLength, + UErrorCode *pErrorCode); + +#endif diff --git a/Build/source/libs/icu/icu-xetex/tools/toolutil/pkgitems.cpp b/Build/source/libs/icu/icu-xetex/tools/toolutil/pkgitems.cpp new file mode 100644 index 00000000000..6a93769c0d2 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/tools/toolutil/pkgitems.cpp @@ -0,0 +1,636 @@ +/* +******************************************************************************* +* +* Copyright (C) 2003-2007, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: pkgitems.cpp +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2005sep18 +* created by: Markus W. Scherer +* +* Companion file to package.cpp. Deals with details of ICU data item formats. +* Used for item dependencies. +* Contains adapted code from uresdata.c and ucnv_bld.c (swapper code from 2003). +*/ + +#include "unicode/utypes.h" +#include "unicode/ures.h" +#include "unicode/putil.h" +#include "unicode/udata.h" +#include "cstring.h" +#include "ucmndata.h" +#include "udataswp.h" +#include "swapimpl.h" +#include "toolutil.h" +#include "package.h" +#include "pkg_imp.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +/* item formats in common */ + +#include "uresdata.h" +#include "ucnv_bld.h" +#include "ucnv_io.h" + +// general definitions ----------------------------------------------------- *** + +#define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0])) + +U_CDECL_BEGIN + +static void U_CALLCONV +printError(void *context, const char *fmt, va_list args) { + vfprintf((FILE *)context, fmt, args); +} + +U_CDECL_END + +// check a dependency ------------------------------------------------------ *** + +/* + * assemble the target item name from the source item name, an ID + * and a suffix + */ +static void +checkIDSuffix(const char *itemName, const char *id, int32_t idLength, const char *suffix, + CheckDependency check, void *context, + UErrorCode *pErrorCode) { + char target[200]; + const char *itemID; + int32_t treeLength, suffixLength, targetLength; + + // get the item basename + itemID=strrchr(itemName, '/'); + if(itemID!=NULL) { + ++itemID; + } else { + itemID=itemName; + } + + // build the target string + treeLength=(int32_t)(itemID-itemName); + if(idLength<0) { + idLength=(int32_t)strlen(id); + } + suffixLength=(int32_t)strlen(suffix); + targetLength=treeLength+idLength+suffixLength; + if(targetLength>=(int32_t)sizeof(target)) { + fprintf(stderr, "icupkg/checkIDSuffix(%s) alias target item name length %ld too long\n", + itemName, (long)targetLength); + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + return; + } + + memcpy(target, itemName, treeLength); + memcpy(target+treeLength, id, idLength); + memcpy(target+treeLength+idLength, suffix, suffixLength+1); // +1 includes the terminating NUL + + check(context, itemName, target); +} + +/* assemble the target item name from the item's parent item name */ +static void +checkParent(const char *itemName, CheckDependency check, void *context, + UErrorCode *pErrorCode) { + const char *itemID, *parent, *parentLimit, *suffix; + int32_t parentLength; + + // get the item basename + itemID=strrchr(itemName, '/'); + if(itemID!=NULL) { + ++itemID; + } else { + itemID=itemName; + } + + // get the item suffix + suffix=strrchr(itemID, '.'); + if(suffix==NULL) { + // empty suffix, point to the end of the string + suffix=strrchr(itemID, 0); + } + + // get the position of the last '_' + for(parentLimit=suffix; parentLimit>itemID && *--parentLimit!='_';) {} + + if(parentLimit!=itemID) { + // get the parent item name by truncating the last part of this item's name */ + parent=itemID; + parentLength=(int32_t)(parentLimit-itemID); + } else { + // no '_' in the item name: the parent is the root bundle + parent="root"; + parentLength=4; + if((suffix-itemID)==parentLength && 0==memcmp(itemID, parent, parentLength)) { + // the item itself is "root", which does not depend on a parent + return; + } + } + checkIDSuffix(itemName, parent, parentLength, suffix, check, context, pErrorCode); +} + +// get dependencies from resource bundles ---------------------------------- *** + +static const char *const gAliasKey="%%ALIAS"; +enum { gAliasKeyLength=7 }; + +/* + * Enumerate one resource item and its children and extract dependencies from + * aliases. + * Code adapted from ures_preflightResource() and ures_swapResource(). + */ +static void +ures_enumDependencies(const UDataSwapper *ds, + const char *itemName, + const Resource *inBundle, int32_t length, + Resource res, const char *inKey, int32_t depth, + CheckDependency check, void *context, + UErrorCode *pErrorCode) { + const Resource *p; + int32_t offset; + + if(res==0 || RES_GET_TYPE(res)==URES_INT) { + /* empty string or integer, nothing to do */ + return; + } + + /* all other types use an offset to point to their data */ + offset=(int32_t)RES_GET_OFFSET(res); + if(0<=length && length<=offset) { + udata_printError(ds, "icupkg/ures_enumDependencies(%s res=%08x) resource offset exceeds bundle length %d\n", + itemName, res, length); + *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; + return; + } + p=inBundle+offset; + + switch(RES_GET_TYPE(res)) { + /* strings and aliases have physically the same value layout */ + case URES_STRING: + // we ignore all strings except top-level strings with a %%ALIAS key + if(depth!=1) { + break; + } else { + char key[8]; + int32_t keyLength; + + keyLength=(int32_t)strlen(inKey); + if(keyLength!=gAliasKeyLength) { + break; + } + ds->swapInvChars(ds, inKey, gAliasKeyLength+1, key, pErrorCode); + if(U_FAILURE(*pErrorCode)) { + udata_printError(ds, "icupkg/ures_enumDependencies(%s res=%08x) string key contains variant characters\n", + itemName, res); + return; + } + if(0!=strcmp(key, gAliasKey)) { + break; + } + } + // for the top-level %%ALIAS string fall through to URES_ALIAS + case URES_ALIAS: + { + char localeID[32]; + const uint16_t *p16; + int32_t i, stringLength; + uint16_t u16, ored16; + + stringLength=udata_readInt32(ds, (int32_t)*p); + + /* top=offset+1+(string length +1)/2 rounded up */ + offset+=1+((stringLength+1)+1)/2; + if(offset>length) { + break; // the resource does not fit into the bundle, print error below + } + + // extract the locale ID from alias strings like + // locale_ID/key1/key2/key3 + // locale_ID + if(U_IS_BIG_ENDIAN==ds->inIsBigEndian) { + u16=0x2f; // slash in local endianness + } else { + u16=0x2f00; // slash in opposite endianness + } + p16=(const uint16_t *)(p+1); // Unicode string contents + + // search for the first slash + for(i=0; i<stringLength && p16[i]!=u16; ++i) {} + + if(RES_GET_TYPE(res)==URES_ALIAS) { + // ignore aliases with an initial slash: + // /ICUDATA/... and /pkgname/... go to a different package + // /LOCALE/... are for dynamic sideways fallbacks and don't go to a fixed bundle + if(i==0) { + break; // initial slash ('/') + } + + // ignore the intra-bundle path starting from the first slash ('/') + stringLength=i; + } else /* URES_STRING */ { + // the whole string should only consist of a locale ID + if(i!=stringLength) { + udata_printError(ds, "icupkg/ures_enumDependencies(%s res=%08x) %%ALIAS contains a '/'\n", + itemName, res); + *pErrorCode=U_UNSUPPORTED_ERROR; + return; + } + } + + // convert the Unicode string to char * and + // check that it has a bundle path but no package + if(stringLength>=(int32_t)sizeof(localeID)) { + udata_printError(ds, "icupkg/ures_enumDependencies(%s res=%08x) alias locale ID length %ld too long\n", + itemName, res, stringLength); + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + return; + } + + // convert the alias Unicode string to US-ASCII + ored16=0; + if(U_IS_BIG_ENDIAN==ds->inIsBigEndian) { + for(i=0; i<stringLength; ++i) { + u16=p16[i]; + ored16|=u16; + localeID[i]=(char)u16; + } + } else { + for(i=0; i<stringLength; ++i) { + u16=p16[i]; + ored16|=u16; + localeID[i]=(char)(u16>>8); + } + ored16=(uint16_t)((ored16<<8)|(ored16>>8)); + } + localeID[stringLength]=0; + if(ored16>0x7f) { + udata_printError(ds, "icupkg/ures_enumDependencies(%s res=%08x) alias string contains non-ASCII characters\n", + itemName, res); + *pErrorCode=U_INVALID_CHAR_FOUND; + return; + } + +#if (U_CHARSET_FAMILY==U_EBCDIC_FAMILY) + // swap to EBCDIC + // our swapper is probably not the right one, but + // the function uses it only for printing errors + uprv_ebcdicFromAscii(ds, localeID, stringLength, localeID, pErrorCode); + if(U_FAILURE(*pErrorCode)) { + return; + } +#endif +#if U_CHARSET_FAMILY!=U_ASCII_FAMILY && U_CHARSET_FAMILY!=U_EBCDIC_FAMILY +# error Unknown U_CHARSET_FAMILY value! +#endif + + checkIDSuffix(itemName, localeID, -1, ".res", check, context, pErrorCode); + } + break; + case URES_TABLE: + case URES_TABLE32: + { + const uint16_t *pKey16; + const int32_t *pKey32; + + Resource item; + int32_t i, count; + + if(RES_GET_TYPE(res)==URES_TABLE) { + /* get table item count */ + pKey16=(const uint16_t *)p; + count=ds->readUInt16(*pKey16++); + + pKey32=NULL; + + /* top=((1+ table item count)/2 rounded up)+(table item count) */ + offset+=((1+count)+1)/2; + } else { + /* get table item count */ + pKey32=(const int32_t *)p; + count=udata_readInt32(ds, *pKey32++); + + pKey16=NULL; + + /* top=(1+ table item count)+(table item count) */ + offset+=1+count; + } + + p=inBundle+offset; /* pointer to table resources */ + offset+=count; + + if(offset>length) { + break; // the resource does not fit into the bundle, print error below + } + + /* recurse */ + for(i=0; i<count; ++i) { + item=ds->readUInt32(*p++); + ures_enumDependencies( + ds, itemName, inBundle, length, item, + ((const char *)inBundle)+ + (pKey16!=NULL ? + ds->readUInt16(pKey16[i]) : + udata_readInt32(ds, pKey32[i])), + depth+1, + check, context, + pErrorCode); + if(U_FAILURE(*pErrorCode)) { + udata_printError(ds, "icupkg/ures_enumDependencies(%s table res=%08x)[%d].recurse(%08x) failed\n", + itemName, res, i, item); + break; + } + } + } + break; + case URES_ARRAY: + { + Resource item; + int32_t i, count; + + /* top=offset+1+(array length) */ + count=udata_readInt32(ds, (int32_t)*p++); + offset+=1+count; + + if(offset>length) { + break; // the resource does not fit into the bundle, print error below + } + + /* recurse */ + for(i=0; i<count; ++i) { + item=ds->readUInt32(*p++); + ures_enumDependencies( + ds, itemName, inBundle, length, + item, NULL, depth+1, + check, context, + pErrorCode); + if(U_FAILURE(*pErrorCode)) { + udata_printError(ds, "icupkg/ures_enumDependencies(%s array res=%08x)[%d].recurse(%08x) failed\n", + itemName, res, i, item); + break; + } + } + } + break; + default: + break; + } + + if(U_FAILURE(*pErrorCode)) { + /* nothing to do */ + } else if(0<=length && length<offset) { + udata_printError(ds, "icupkg/ures_enumDependencies(%s res=%08x) resource limit exceeds bundle length %d\n", + itemName, res, length); + *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; + } +} + +/* code adapted from ures_swap() */ +static void +ures_enumDependencies(const UDataSwapper *ds, + const char *itemName, const UDataInfo *pInfo, + const uint8_t *inBytes, int32_t length, + CheckDependency check, void *context, + UErrorCode *pErrorCode) { + const Resource *inBundle; + Resource rootRes; + + /* the following integers count Resource item offsets (4 bytes each), not bytes */ + int32_t bundleLength; + + /* check format version */ + if(pInfo->formatVersion[0]!=1) { + fprintf(stderr, "icupkg: .res format version %02x not supported\n", + pInfo->formatVersion[0]); + exit(U_UNSUPPORTED_ERROR); + } + + /* a resource bundle must contain at least one resource item */ + bundleLength=length/4; + + /* formatVersion 1.1 must have a root item and at least 5 indexes */ + if( bundleLength< + (pInfo->formatVersion[1]==0 ? 1 : 1+5) + ) { + fprintf(stderr, "icupkg: too few bytes (%d after header) for a resource bundle\n", + length); + exit(U_INDEX_OUTOFBOUNDS_ERROR); + } + + inBundle=(const Resource *)inBytes; + rootRes=ds->readUInt32(*inBundle); + + ures_enumDependencies( + ds, itemName, inBundle, bundleLength, + rootRes, NULL, 0, + check, context, + pErrorCode); + + /* + * if the bundle attributes are present and the nofallback flag is not set, + * then add the parent bundle as a dependency + */ + if(pInfo->formatVersion[1]>=1) { + int32_t indexes[URES_INDEX_TOP]; + const int32_t *inIndexes; + + inIndexes=(const int32_t *)inBundle+1; + indexes[URES_INDEX_LENGTH]=udata_readInt32(ds, inIndexes[URES_INDEX_LENGTH]); + if(indexes[URES_INDEX_LENGTH]>URES_INDEX_ATTRIBUTES) { + indexes[URES_INDEX_ATTRIBUTES]=udata_readInt32(ds, inIndexes[URES_INDEX_ATTRIBUTES]); + if(0==(indexes[URES_INDEX_ATTRIBUTES]&URES_ATT_NO_FALLBACK)) { + /* this bundle participates in locale fallback */ + checkParent(itemName, check, context, pErrorCode); + } + } + } +} + +// get dependencies from conversion tables --------------------------------- *** + +/* code adapted from ucnv_swap() */ +static void +ucnv_enumDependencies(const UDataSwapper *ds, + const char *itemName, const UDataInfo *pInfo, + const uint8_t *inBytes, int32_t length, + CheckDependency check, void *context, + UErrorCode *pErrorCode) { + uint32_t staticDataSize; + + const UConverterStaticData *inStaticData; + + const _MBCSHeader *inMBCSHeader; + uint8_t outputType; + + /* check format version */ + if(!( + pInfo->formatVersion[0]==6 && + pInfo->formatVersion[1]>=2 + )) { + fprintf(stderr, "icupkg/ucnv_enumDependencies(): .cnv format version %02x.%02x not supported\n", + pInfo->formatVersion[0], pInfo->formatVersion[1]); + exit(U_UNSUPPORTED_ERROR); + } + + /* read the initial UConverterStaticData structure after the UDataInfo header */ + inStaticData=(const UConverterStaticData *)inBytes; + + if( length<(int32_t)sizeof(UConverterStaticData) || + (uint32_t)length<(staticDataSize=ds->readUInt32(inStaticData->structSize)) + ) { + udata_printError(ds, "icupkg/ucnv_enumDependencies(): too few bytes (%d after header) for an ICU .cnv conversion table\n", + length); + *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; + return; + } + + inBytes+=staticDataSize; + length-=(int32_t)staticDataSize; + + /* check for supported conversionType values */ + if(inStaticData->conversionType==UCNV_MBCS) { + /* MBCS data */ + uint32_t mbcsHeaderFlags; + int32_t extOffset; + + inMBCSHeader=(const _MBCSHeader *)inBytes; + + if(length<(int32_t)sizeof(_MBCSHeader)) { + udata_printError(ds, "icupkg/ucnv_enumDependencies(): too few bytes (%d after headers) for an ICU MBCS .cnv conversion table\n", + length); + *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; + return; + } + if(!(inMBCSHeader->version[0]==4 && inMBCSHeader->version[1]>=1)) { + udata_printError(ds, "icupkg/ucnv_enumDependencies(): unsupported _MBCSHeader.version %d.%d\n", + inMBCSHeader->version[0], inMBCSHeader->version[1]); + *pErrorCode=U_UNSUPPORTED_ERROR; + return; + } + + mbcsHeaderFlags=ds->readUInt32(inMBCSHeader->flags); + extOffset=(int32_t)(mbcsHeaderFlags>>8); + outputType=(uint8_t)mbcsHeaderFlags; + + if(outputType==MBCS_OUTPUT_EXT_ONLY) { + /* + * extension-only file, + * contains a base name instead of normal base table data + */ + char baseName[32]; + int32_t baseNameLength; + + /* there is extension data after the base data, see ucnv_ext.h */ + if(length<(extOffset+UCNV_EXT_INDEXES_MIN_LENGTH*4)) { + udata_printError(ds, "icupkg/ucnv_enumDependencies(): too few bytes (%d after headers) for an ICU MBCS .cnv conversion table with extension data\n", + length); + *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; + return; + } + + /* swap the base name, between the header and the extension data */ + baseNameLength=(int32_t)strlen((const char *)(inMBCSHeader+1)); + if(baseNameLength>=(int32_t)sizeof(baseName)) { + udata_printError(ds, "icupkg/ucnv_enumDependencies(%s): base name length %ld too long\n", + itemName, baseNameLength); + *pErrorCode=U_UNSUPPORTED_ERROR; + return; + } + ds->swapInvChars(ds, inMBCSHeader+1, baseNameLength+1, baseName, pErrorCode); + + checkIDSuffix(itemName, baseName, -1, ".cnv", check, context, pErrorCode); + } + } +} + +// ICU data formats -------------------------------------------------------- *** + +static const struct { + uint8_t dataFormat[4]; +} dataFormats[]={ + { { 0x52, 0x65, 0x73, 0x42 } }, /* dataFormat="ResB" */ + { { 0x63, 0x6e, 0x76, 0x74 } }, /* dataFormat="cnvt" */ + { { 0x43, 0x76, 0x41, 0x6c } } /* dataFormat="CvAl" */ +}; + +enum { + FMT_RES, + FMT_CNV, + FMT_ALIAS, + FMT_COUNT +}; + +static int32_t +getDataFormat(const uint8_t dataFormat[4]) { + int32_t i; + + for(i=0; i<FMT_COUNT; ++i) { + if(0==memcmp(dataFormats[i].dataFormat, dataFormat, 4)) { + return i; + } + } + return -1; +} + +// enumerate dependencies of a package item -------------------------------- *** + +U_NAMESPACE_BEGIN + +void +Package::enumDependencies(Item *pItem, void *context, CheckDependency check) { + const UDataInfo *pInfo; + const uint8_t *inBytes; + int32_t format, length, infoLength, itemHeaderLength; + UErrorCode errorCode; + + errorCode=U_ZERO_ERROR; + pInfo=getDataInfo(pItem->data,pItem->length, infoLength, itemHeaderLength, &errorCode); + if(U_FAILURE(errorCode)) { + return; // should not occur because readFile() checks headers + } + + // find the data format and call the corresponding function, if any + format=getDataFormat(pInfo->dataFormat); + if(format>=0) { + UDataSwapper *ds; + + // TODO: share/cache swappers + ds=udata_openSwapper((UBool)pInfo->isBigEndian, pInfo->charsetFamily, U_IS_BIG_ENDIAN, U_CHARSET_FAMILY, &errorCode); + if(U_FAILURE(errorCode)) { + fprintf(stderr, "icupkg: udata_openSwapper(\"%s\") failed - %s\n", + pItem->name, u_errorName(errorCode)); + exit(errorCode); + } + + ds->printError=printError; + ds->printErrorContext=stderr; + + inBytes=pItem->data+itemHeaderLength; + length=pItem->length-itemHeaderLength; + + switch(format) { + case FMT_RES: + ures_enumDependencies(ds, pItem->name, pInfo, inBytes, length, check, context, &errorCode); + break; + case FMT_CNV: + ucnv_enumDependencies(ds, pItem->name, pInfo, inBytes, length, check, context, &errorCode); + break; + default: + break; + } + + udata_closeSwapper(ds); + + if(U_FAILURE(errorCode)) { + exit(errorCode); + } + } +} +U_NAMESPACE_END diff --git a/Build/source/libs/icu/icu-xetex/tools/toolutil/propsvec.c b/Build/source/libs/icu/icu-xetex/tools/toolutil/propsvec.c new file mode 100644 index 00000000000..c4cb7b17293 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/tools/toolutil/propsvec.c @@ -0,0 +1,353 @@ +/* +******************************************************************************* +* +* Copyright (C) 2002-2005, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: propsvec.c +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2002feb22 +* created by: Markus W. Scherer +* +* Store additional Unicode character properties in bit set vectors. +*/ + +#include <stdlib.h> +#include "unicode/utypes.h" +#include "cmemory.h" +#include "utrie.h" +#include "uarrsort.h" +#include "propsvec.h" + +static uint32_t * +_findRow(uint32_t *pv, UChar32 rangeStart) { + uint32_t *row; + int32_t *hdr; + int32_t columns, i, start, limit, prevRow, rows; + + hdr=(int32_t *)pv; + columns=hdr[UPVEC_COLUMNS]; + limit=hdr[UPVEC_ROWS]; + prevRow=hdr[UPVEC_PREV_ROW]; + rows=hdr[UPVEC_ROWS]; + pv+=UPVEC_HEADER_LENGTH; + + /* check the vicinity of the last-seen row */ + if(prevRow<rows) { + row=pv+prevRow*columns; + if(rangeStart>=(UChar32)row[0]) { + if(rangeStart<(UChar32)row[1]) { + /* same row as last seen */ + return row; + } else if( + ++prevRow<rows && + rangeStart>=(UChar32)(row+=columns)[0] && rangeStart<(UChar32)row[1] + ) { + /* next row after the last one */ + hdr[UPVEC_PREV_ROW]=prevRow; + return row; + } + } + } + + /* do a binary search for the start of the range */ + start=0; + while(start<limit-1) { + i=(start+limit)/2; + row=pv+i*columns; + if(rangeStart<(UChar32)row[0]) { + limit=i; + } else if(rangeStart<(UChar32)row[1]) { + hdr[UPVEC_PREV_ROW]=i; + return row; + } else { + start=i; + } + } + + /* must be found because all ranges together always cover all of Unicode */ + hdr[UPVEC_PREV_ROW]=start; + return pv+start*columns; +} + +U_CAPI uint32_t * U_EXPORT2 +upvec_open(int32_t columns, int32_t maxRows) { + uint32_t *pv, *row; + int32_t length; + + if(columns<1 || maxRows<1) { + return NULL; + } + + columns+=2; /* count range start and limit columns */ + length=UPVEC_HEADER_LENGTH+maxRows*columns; + pv=(uint32_t *)uprv_malloc(length*4); + if(pv!=NULL) { + /* set header */ + pv[UPVEC_COLUMNS]=(uint32_t)columns; + pv[UPVEC_MAXROWS]=(uint32_t)maxRows; + pv[UPVEC_ROWS]=1; + pv[UPVEC_PREV_ROW]=0; + + /* set initial row */ + row=pv+UPVEC_HEADER_LENGTH; + *row++=0; + *row++=0x110000; + columns-=2; + do { + *row++=0; + } while(--columns>0); + } + return pv; +} + +U_CAPI void U_EXPORT2 +upvec_close(uint32_t *pv) { + if(pv!=NULL) { + uprv_free(pv); + } +} + +U_CAPI UBool U_EXPORT2 +upvec_setValue(uint32_t *pv, + UChar32 start, UChar32 limit, + int32_t column, + uint32_t value, uint32_t mask, + UErrorCode *pErrorCode) { + uint32_t *firstRow, *lastRow; + int32_t columns; + UBool splitFirstRow, splitLastRow; + + /* argument checking */ + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return FALSE; + } + + if( pv==NULL || + start<0 || start>limit || limit>0x110000 || + column<0 || (uint32_t)(column+1)>=pv[UPVEC_COLUMNS] + ) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return FALSE; + } + if(start==limit) { + /* empty range, nothing to do */ + return TRUE; + } + + /* initialize */ + columns=(int32_t)pv[UPVEC_COLUMNS]; + column+=2; /* skip range start and limit columns */ + value&=mask; + + /* find the rows whose ranges overlap with the input range */ + + /* find the first row, always successful */ + firstRow=_findRow(pv, start); + + /* find the last row, always successful */ + lastRow=firstRow; + while(limit>(UChar32)lastRow[1]) { + lastRow+=columns; + } + + /* + * Rows need to be split if they partially overlap with the + * input range (only possible for the first and last rows) + * and if their value differs from the input value. + */ + splitFirstRow= (UBool)(start!=(UChar32)firstRow[0] && value!=(firstRow[column]&mask)); + splitLastRow= (UBool)(limit!=(UChar32)lastRow[1] && value!=(lastRow[column]&mask)); + + /* split first/last rows if necessary */ + if(splitFirstRow || splitLastRow) { + int32_t count, rows; + + rows=(int32_t)pv[UPVEC_ROWS]; + if((rows+splitFirstRow+splitLastRow)>(int32_t)pv[UPVEC_MAXROWS]) { + *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; + return FALSE; + } + + /* count the number of row cells to move after the last row, and move them */ + count = (int32_t)((pv+UPVEC_HEADER_LENGTH+rows*columns)-(lastRow+columns)); + if(count>0) { + uprv_memmove( + lastRow+(1+splitFirstRow+splitLastRow)*columns, + lastRow+columns, + count*4); + } + pv[UPVEC_ROWS]=rows+splitFirstRow+splitLastRow; + + /* split the first row, and move the firstRow pointer to the second part */ + if(splitFirstRow) { + /* copy all affected rows up one and move the lastRow pointer */ + count = (int32_t)((lastRow-firstRow)+columns); + uprv_memmove(firstRow+columns, firstRow, count*4); + lastRow+=columns; + + /* split the range and move the firstRow pointer */ + firstRow[1]=firstRow[columns]=(uint32_t)start; + firstRow+=columns; + } + + /* split the last row */ + if(splitLastRow) { + /* copy the last row data */ + uprv_memcpy(lastRow+columns, lastRow, columns*4); + + /* split the range and move the firstRow pointer */ + lastRow[1]=lastRow[columns]=(uint32_t)limit; + } + } + + /* set the "row last seen" to the last row for the range */ + pv[UPVEC_PREV_ROW]=(uint32_t)((lastRow-(pv+UPVEC_HEADER_LENGTH))/columns); + + /* set the input value in all remaining rows */ + firstRow+=column; + lastRow+=column; + mask=~mask; + for(;;) { + *firstRow=(*firstRow&mask)|value; + if(firstRow==lastRow) { + break; + } + firstRow+=columns; + } + return TRUE; +} + +U_CAPI uint32_t U_EXPORT2 +upvec_getValue(uint32_t *pv, UChar32 c, int32_t column) { + uint32_t *row; + + if(pv==NULL || c<0 || c>=0x110000) { + return 0; + } + row=_findRow(pv, c); + return row[2+column]; +} + +U_CAPI uint32_t * U_EXPORT2 +upvec_getRow(uint32_t *pv, int32_t rowIndex, + UChar32 *pRangeStart, UChar32 *pRangeLimit) { + uint32_t *row; + int32_t columns; + + if(pv==NULL || rowIndex<0 || rowIndex>=(int32_t)pv[UPVEC_ROWS]) { + return NULL; + } + + columns=(int32_t)pv[UPVEC_COLUMNS]; + row=pv+UPVEC_HEADER_LENGTH+rowIndex*columns; + if(pRangeStart!=NULL) { + *pRangeStart=row[0]; + } + if(pRangeLimit!=NULL) { + *pRangeLimit=row[1]; + } + return row+2; +} + +static int32_t U_CALLCONV +upvec_compareRows(const void *context, const void *l, const void *r) { + const uint32_t *left=(const uint32_t *)l, *right=(const uint32_t *)r; + const uint32_t *pv=(const uint32_t *)context; + int32_t i, count, columns; + + count=columns=(int32_t)pv[UPVEC_COLUMNS]; /* includes start/limit columns */ + + /* start comparing after start/limit but wrap around to them */ + i=2; + do { + if(left[i]!=right[i]) { + return left[i]<right[i] ? -1 : 1; + } + if(++i==columns) { + i=0; + } + } while(--count>0); + + return 0; +} + +U_CAPI int32_t U_EXPORT2 +upvec_compact(uint32_t *pv, UPVecCompactHandler *handler, void *context, UErrorCode *pErrorCode) { + uint32_t *row; + int32_t columns, valueColumns, rows, count; + UChar32 start, limit; + + /* argument checking */ + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return 0; + } + + if(pv==NULL || handler==NULL) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + row=pv+UPVEC_HEADER_LENGTH; + columns=(int32_t)pv[UPVEC_COLUMNS]; + rows=(int32_t)pv[UPVEC_ROWS]; + + if(rows==0) { + return 0; + } + + /* sort the properties vectors to find unique vector values */ + if(rows>1) { + uprv_sortArray(pv+UPVEC_HEADER_LENGTH, rows, columns*4, + upvec_compareRows, pv, FALSE, pErrorCode); + } + if(U_FAILURE(*pErrorCode)) { + return 0; + } + + /* + * Move vector contents up to a contiguous array with only unique + * vector values, and call the handler function for each vector. + * + * This destroys the Properties Vector structure and replaces it + * with an array of just vector values. + */ + valueColumns=columns-2; /* not counting start & limit */ + count=-valueColumns; + + do { + /* fetch these first before memmove() may overwrite them */ + start=(UChar32)row[0]; + limit=(UChar32)row[1]; + + /* add a new values vector if it is different from the current one */ + if(count<0 || 0!=uprv_memcmp(row+2, pv+count, valueColumns*4)) { + count+=valueColumns; + uprv_memmove(pv+count, row+2, valueColumns*4); + } + + handler(context, start, limit, count, pv+count, valueColumns, pErrorCode); + if(U_FAILURE(*pErrorCode)) { + return 0; + } + + row+=columns; + } while(--rows>0); + + /* count is at the beginning of the last vector, add valueColumns to include that last vector */ + return count+valueColumns; +} + +U_CAPI void U_CALLCONV +upvec_compactToTrieHandler(void *context, + UChar32 start, UChar32 limit, + int32_t rowIndex, uint32_t *row, int32_t columns, + UErrorCode *pErrorCode) { + if(!utrie_setRange32((UNewTrie *)context, start, limit, (uint32_t)rowIndex, FALSE)) { + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + } +} diff --git a/Build/source/libs/icu/icu-xetex/tools/toolutil/propsvec.h b/Build/source/libs/icu/icu-xetex/tools/toolutil/propsvec.h new file mode 100644 index 00000000000..c899d5885e4 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/tools/toolutil/propsvec.h @@ -0,0 +1,103 @@ +/* +******************************************************************************* +* +* Copyright (C) 2002-2005, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: propsvec.h +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2002feb22 +* created by: Markus W. Scherer +* +* Store additional Unicode character properties in bit set vectors. +*/ + +#ifndef __UPROPSVEC_H__ +#define __UPROPSVEC_H__ + +#include "unicode/utypes.h" +#include "utrie.h" + +/* + * Unicode Properties Vectors associated with code point ranges. + * Stored in an array of uint32_t. + * + * The array starts with a header, then rows of integers store + * the range limits and the properties vectors. + * + * In each row, row[0] contains the start code point and + * row[1] contains the limit code point, + * which is the start of the next range. + * + * Initially, there is only one range [0..0x110000[ with values 0. + * + * It would be possible to store only one range boundary per row, + * but self-contained rows allow to later sort them by contents. + */ +enum { + /* stores number of columns, plus two for start & limit values */ + UPVEC_COLUMNS, + UPVEC_MAXROWS, + UPVEC_ROWS, + /* search optimization: remember last row seen */ + UPVEC_PREV_ROW, + UPVEC_HEADER_LENGTH +}; + +U_CAPI uint32_t * U_EXPORT2 +upvec_open(int32_t columns, int32_t maxRows); + +U_CAPI void U_EXPORT2 +upvec_close(uint32_t *pv); + +U_CAPI UBool U_EXPORT2 +upvec_setValue(uint32_t *pv, + UChar32 start, UChar32 limit, + int32_t column, + uint32_t value, uint32_t mask, + UErrorCode *pErrorCode); + +U_CAPI uint32_t U_EXPORT2 +upvec_getValue(uint32_t *pv, UChar32 c, int32_t column); + +/* + * pRangeStart and pRangeLimit can be NULL. + * @return NULL if rowIndex out of range and for illegal arguments + */ +U_CAPI uint32_t * U_EXPORT2 +upvec_getRow(uint32_t *pv, int32_t rowIndex, + UChar32 *pRangeStart, UChar32 *pRangeLimit); + +/* + * Compact the vectors: + * - modify the memory + * - keep only unique vectors + * - store them contiguously from the beginning of the memory + * - for each (non-unique) row, call the handler function + * + * The handler's rowIndex is the uint32_t index of the row in the compacted + * memory block. + * (Therefore, it starts at 0 increases in increments of the columns value.) + */ + +typedef void U_CALLCONV +UPVecCompactHandler(void *context, + UChar32 start, UChar32 limit, + int32_t rowIndex, uint32_t *row, int32_t columns, + UErrorCode *pErrorCode); + +U_CAPI int32_t U_EXPORT2 +upvec_compact(uint32_t *pv, UPVecCompactHandler *handler, void *context, UErrorCode *pErrorCode); + +/* context=UNewTrie, stores the rowIndex values into the trie */ +U_CAPI void U_CALLCONV +upvec_compactToTrieHandler(void *context, + UChar32 start, UChar32 limit, + int32_t rowIndex, uint32_t *row, int32_t columns, + UErrorCode *pErrorCode); + +#endif diff --git a/Build/source/libs/icu/icu-xetex/tools/toolutil/swapimpl.cpp b/Build/source/libs/icu/icu-xetex/tools/toolutil/swapimpl.cpp new file mode 100644 index 00000000000..fd0f7faf6ad --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/tools/toolutil/swapimpl.cpp @@ -0,0 +1,643 @@ +/* +******************************************************************************* +* +* Copyright (C) 2005-2006, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: swapimpl.cpp +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2005may05 +* created by: Markus W. Scherer +* +* Data file swapping functions moved here from the common library +* because some data is hardcoded in ICU4C and needs not be swapped any more. +* Moving the functions here simplifies testing (for code coverage) because +* we need not jump through hoops (like adding snapshots of these files +* to testdata). +* +* The declarations for these functions remain in the internal header files +* in icu/source/common/ +*/ + +#include "unicode/utypes.h" +#include "unicode/putil.h" +#include "unicode/udata.h" +#include "cmemory.h" +#include "cstring.h" +#include "uinvchar.h" +#include "uassert.h" +#include "uarrsort.h" +#include "ucmndata.h" +#include "udataswp.h" + +/* swapping implementations in common */ + +#include "uresdata.h" +#include "ucnv_io.h" +#include "uprops.h" +#include "ucase.h" +#include "ubidi_props.h" +#include "ucol_swp.h" +#include "ucnv_bld.h" +#include "unormimp.h" +#include "sprpimpl.h" +#include "propname.h" +#include "rbbidata.h" +#include "triedict.h" + +/* swapping implementations in i18n */ + +/* definitions */ + +#define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0])) + +/* Unicode properties data swapping ----------------------------------------- */ + +U_CAPI int32_t U_EXPORT2 +uprops_swap(const UDataSwapper *ds, + const void *inData, int32_t length, void *outData, + UErrorCode *pErrorCode) { + const UDataInfo *pInfo; + int32_t headerSize, i; + + int32_t dataIndexes[UPROPS_INDEX_COUNT]; + const int32_t *inData32; + + /* udata_swapDataHeader checks the arguments */ + headerSize=udata_swapDataHeader(ds, inData, length, outData, pErrorCode); + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return 0; + } + + /* check data format and format version */ + pInfo=(const UDataInfo *)((const char *)inData+4); + if(!( + pInfo->dataFormat[0]==0x55 && /* dataFormat="UPro" */ + pInfo->dataFormat[1]==0x50 && + pInfo->dataFormat[2]==0x72 && + pInfo->dataFormat[3]==0x6f && + (pInfo->formatVersion[0]==3 || pInfo->formatVersion[0]==4) && + pInfo->formatVersion[2]==UTRIE_SHIFT && + pInfo->formatVersion[3]==UTRIE_INDEX_SHIFT + )) { + udata_printError(ds, "uprops_swap(): data format %02x.%02x.%02x.%02x (format version %02x) is not a Unicode properties file\n", + pInfo->dataFormat[0], pInfo->dataFormat[1], + pInfo->dataFormat[2], pInfo->dataFormat[3], + pInfo->formatVersion[0]); + *pErrorCode=U_UNSUPPORTED_ERROR; + return 0; + } + + /* the properties file must contain at least the indexes array */ + if(length>=0 && (length-headerSize)<(int32_t)sizeof(dataIndexes)) { + udata_printError(ds, "uprops_swap(): too few bytes (%d after header) for a Unicode properties file\n", + length-headerSize); + *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; + return 0; + } + + /* read the indexes */ + inData32=(const int32_t *)((const char *)inData+headerSize); + for(i=0; i<UPROPS_INDEX_COUNT; ++i) { + dataIndexes[i]=udata_readInt32(ds, inData32[i]); + } + + /* + * comments are copied from the data format description in genprops/store.c + * indexes[] constants are in uprops.h + */ + if(length>=0) { + int32_t *outData32; + + if((length-headerSize)<(4*dataIndexes[UPROPS_RESERVED_INDEX])) { + udata_printError(ds, "uprops_swap(): too few bytes (%d after header) for a Unicode properties file\n", + length-headerSize); + *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; + return 0; + } + + outData32=(int32_t *)((char *)outData+headerSize); + + /* copy everything for inaccessible data (padding) */ + if(inData32!=outData32) { + uprv_memcpy(outData32, inData32, 4*dataIndexes[UPROPS_RESERVED_INDEX]); + } + + /* swap the indexes[16] */ + ds->swapArray32(ds, inData32, 4*UPROPS_INDEX_COUNT, outData32, pErrorCode); + + /* + * swap the main properties UTrie + * PT serialized properties trie, see utrie.h (byte size: 4*(i0-16)) + */ + utrie_swap(ds, + inData32+UPROPS_INDEX_COUNT, + 4*(dataIndexes[UPROPS_PROPS32_INDEX]-UPROPS_INDEX_COUNT), + outData32+UPROPS_INDEX_COUNT, + pErrorCode); + + /* + * swap the properties and exceptions words + * P const uint32_t props32[i1-i0]; + * E const uint32_t exceptions[i2-i1]; + */ + ds->swapArray32(ds, + inData32+dataIndexes[UPROPS_PROPS32_INDEX], + 4*(dataIndexes[UPROPS_EXCEPTIONS_TOP_INDEX]-dataIndexes[UPROPS_PROPS32_INDEX]), + outData32+dataIndexes[UPROPS_PROPS32_INDEX], + pErrorCode); + + /* + * swap the UChars + * U const UChar uchars[2*(i3-i2)]; + */ + ds->swapArray16(ds, + inData32+dataIndexes[UPROPS_EXCEPTIONS_TOP_INDEX], + 4*(dataIndexes[UPROPS_ADDITIONAL_TRIE_INDEX]-dataIndexes[UPROPS_EXCEPTIONS_TOP_INDEX]), + outData32+dataIndexes[UPROPS_EXCEPTIONS_TOP_INDEX], + pErrorCode); + + /* + * swap the additional UTrie + * i3 additionalTrieIndex; -- 32-bit unit index to the additional trie for more properties + */ + utrie_swap(ds, + inData32+dataIndexes[UPROPS_ADDITIONAL_TRIE_INDEX], + 4*(dataIndexes[UPROPS_ADDITIONAL_VECTORS_INDEX]-dataIndexes[UPROPS_ADDITIONAL_TRIE_INDEX]), + outData32+dataIndexes[UPROPS_ADDITIONAL_TRIE_INDEX], + pErrorCode); + + /* + * swap the properties vectors + * PV const uint32_t propsVectors[(i6-i4)/i5][i5]==uint32_t propsVectors[i6-i4]; + */ + ds->swapArray32(ds, + inData32+dataIndexes[UPROPS_ADDITIONAL_VECTORS_INDEX], + 4*(dataIndexes[UPROPS_RESERVED_INDEX]-dataIndexes[UPROPS_ADDITIONAL_VECTORS_INDEX]), + outData32+dataIndexes[UPROPS_ADDITIONAL_VECTORS_INDEX], + pErrorCode); + } + + /* i6 reservedItemIndex; -- 32-bit unit index to the top of the properties vectors table */ + return headerSize+4*dataIndexes[UPROPS_RESERVED_INDEX]; +} + +/* Unicode case mapping data swapping --------------------------------------- */ + +U_CAPI int32_t U_EXPORT2 +ucase_swap(const UDataSwapper *ds, + const void *inData, int32_t length, void *outData, + UErrorCode *pErrorCode) { + const UDataInfo *pInfo; + int32_t headerSize; + + const uint8_t *inBytes; + uint8_t *outBytes; + + const int32_t *inIndexes; + int32_t indexes[16]; + + int32_t i, offset, count, size; + + /* udata_swapDataHeader checks the arguments */ + headerSize=udata_swapDataHeader(ds, inData, length, outData, pErrorCode); + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return 0; + } + + /* check data format and format version */ + pInfo=(const UDataInfo *)((const char *)inData+4); + if(!( + pInfo->dataFormat[0]==UCASE_FMT_0 && /* dataFormat="cAsE" */ + pInfo->dataFormat[1]==UCASE_FMT_1 && + pInfo->dataFormat[2]==UCASE_FMT_2 && + pInfo->dataFormat[3]==UCASE_FMT_3 && + pInfo->formatVersion[0]==1 && + pInfo->formatVersion[2]==UTRIE_SHIFT && + pInfo->formatVersion[3]==UTRIE_INDEX_SHIFT + )) { + udata_printError(ds, "ucase_swap(): data format %02x.%02x.%02x.%02x (format version %02x) is not recognized as case mapping data\n", + pInfo->dataFormat[0], pInfo->dataFormat[1], + pInfo->dataFormat[2], pInfo->dataFormat[3], + pInfo->formatVersion[0]); + *pErrorCode=U_UNSUPPORTED_ERROR; + return 0; + } + + inBytes=(const uint8_t *)inData+headerSize; + outBytes=(uint8_t *)outData+headerSize; + + inIndexes=(const int32_t *)inBytes; + + if(length>=0) { + length-=headerSize; + if(length<16*4) { + udata_printError(ds, "ucase_swap(): too few bytes (%d after header) for case mapping data\n", + length); + *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; + return 0; + } + } + + /* read the first 16 indexes (ICU 3.2/format version 1: UCASE_IX_TOP==16, might grow) */ + for(i=0; i<16; ++i) { + indexes[i]=udata_readInt32(ds, inIndexes[i]); + } + + /* get the total length of the data */ + size=indexes[UCASE_IX_LENGTH]; + + if(length>=0) { + if(length<size) { + udata_printError(ds, "ucase_swap(): too few bytes (%d after header) for all of case mapping data\n", + length); + *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; + return 0; + } + + /* copy the data for inaccessible bytes */ + if(inBytes!=outBytes) { + uprv_memcpy(outBytes, inBytes, size); + } + + offset=0; + + /* swap the int32_t indexes[] */ + count=indexes[UCASE_IX_INDEX_TOP]*4; + ds->swapArray32(ds, inBytes, count, outBytes, pErrorCode); + offset+=count; + + /* swap the UTrie */ + count=indexes[UCASE_IX_TRIE_SIZE]; + utrie_swap(ds, inBytes+offset, count, outBytes+offset, pErrorCode); + offset+=count; + + /* swap the uint16_t exceptions[] and unfold[] */ + count=(indexes[UCASE_IX_EXC_LENGTH]+indexes[UCASE_IX_UNFOLD_LENGTH])*2; + ds->swapArray16(ds, inBytes+offset, count, outBytes+offset, pErrorCode); + offset+=count; + + U_ASSERT(offset==size); + } + + return headerSize+size; +} + +/* Unicode bidi/shaping data swapping --------------------------------------- */ + +U_CAPI int32_t U_EXPORT2 +ubidi_swap(const UDataSwapper *ds, + const void *inData, int32_t length, void *outData, + UErrorCode *pErrorCode) { + const UDataInfo *pInfo; + int32_t headerSize; + + const uint8_t *inBytes; + uint8_t *outBytes; + + const int32_t *inIndexes; + int32_t indexes[16]; + + int32_t i, offset, count, size; + + /* udata_swapDataHeader checks the arguments */ + headerSize=udata_swapDataHeader(ds, inData, length, outData, pErrorCode); + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return 0; + } + + /* check data format and format version */ + pInfo=(const UDataInfo *)((const char *)inData+4); + if(!( + pInfo->dataFormat[0]==UBIDI_FMT_0 && /* dataFormat="BiDi" */ + pInfo->dataFormat[1]==UBIDI_FMT_1 && + pInfo->dataFormat[2]==UBIDI_FMT_2 && + pInfo->dataFormat[3]==UBIDI_FMT_3 && + pInfo->formatVersion[0]==1 && + pInfo->formatVersion[2]==UTRIE_SHIFT && + pInfo->formatVersion[3]==UTRIE_INDEX_SHIFT + )) { + udata_printError(ds, "ubidi_swap(): data format %02x.%02x.%02x.%02x (format version %02x) is not recognized as bidi/shaping data\n", + pInfo->dataFormat[0], pInfo->dataFormat[1], + pInfo->dataFormat[2], pInfo->dataFormat[3], + pInfo->formatVersion[0]); + *pErrorCode=U_UNSUPPORTED_ERROR; + return 0; + } + + inBytes=(const uint8_t *)inData+headerSize; + outBytes=(uint8_t *)outData+headerSize; + + inIndexes=(const int32_t *)inBytes; + + if(length>=0) { + length-=headerSize; + if(length<16*4) { + udata_printError(ds, "ubidi_swap(): too few bytes (%d after header) for bidi/shaping data\n", + length); + *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; + return 0; + } + } + + /* read the first 16 indexes (ICU 3.4/format version 1: UBIDI_IX_TOP==16, might grow) */ + for(i=0; i<16; ++i) { + indexes[i]=udata_readInt32(ds, inIndexes[i]); + } + + /* get the total length of the data */ + size=indexes[UBIDI_IX_LENGTH]; + + if(length>=0) { + if(length<size) { + udata_printError(ds, "ubidi_swap(): too few bytes (%d after header) for all of bidi/shaping data\n", + length); + *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; + return 0; + } + + /* copy the data for inaccessible bytes */ + if(inBytes!=outBytes) { + uprv_memcpy(outBytes, inBytes, size); + } + + offset=0; + + /* swap the int32_t indexes[] */ + count=indexes[UBIDI_IX_INDEX_TOP]*4; + ds->swapArray32(ds, inBytes, count, outBytes, pErrorCode); + offset+=count; + + /* swap the UTrie */ + count=indexes[UBIDI_IX_TRIE_SIZE]; + utrie_swap(ds, inBytes+offset, count, outBytes+offset, pErrorCode); + offset+=count; + + /* swap the uint32_t mirrors[] */ + count=indexes[UBIDI_IX_MIRROR_LENGTH]*4; + ds->swapArray32(ds, inBytes+offset, count, outBytes+offset, pErrorCode); + offset+=count; + + /* just skip the uint8_t jgArray[] */ + count=indexes[UBIDI_IX_JG_LIMIT]-indexes[UBIDI_IX_JG_START]; + offset+=count; + + U_ASSERT(offset==size); + } + + return headerSize+size; +} + +/* Unicode normalization data swapping -------------------------------------- */ + +#if !UCONFIG_NO_NORMALIZATION + +U_CAPI int32_t U_EXPORT2 +unorm_swap(const UDataSwapper *ds, + const void *inData, int32_t length, void *outData, + UErrorCode *pErrorCode) { + const UDataInfo *pInfo; + int32_t headerSize; + + const uint8_t *inBytes; + uint8_t *outBytes; + + const int32_t *inIndexes; + int32_t indexes[32]; + + int32_t i, offset, count, size; + + /* udata_swapDataHeader checks the arguments */ + headerSize=udata_swapDataHeader(ds, inData, length, outData, pErrorCode); + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return 0; + } + + /* check data format and format version */ + pInfo=(const UDataInfo *)((const char *)inData+4); + if(!( + pInfo->dataFormat[0]==0x4e && /* dataFormat="Norm" */ + pInfo->dataFormat[1]==0x6f && + pInfo->dataFormat[2]==0x72 && + pInfo->dataFormat[3]==0x6d && + pInfo->formatVersion[0]==2 + )) { + udata_printError(ds, "unorm_swap(): data format %02x.%02x.%02x.%02x (format version %02x) is not recognized as unorm.icu\n", + pInfo->dataFormat[0], pInfo->dataFormat[1], + pInfo->dataFormat[2], pInfo->dataFormat[3], + pInfo->formatVersion[0]); + *pErrorCode=U_UNSUPPORTED_ERROR; + return 0; + } + + inBytes=(const uint8_t *)inData+headerSize; + outBytes=(uint8_t *)outData+headerSize; + + inIndexes=(const int32_t *)inBytes; + + if(length>=0) { + length-=headerSize; + if(length<32*4) { + udata_printError(ds, "unorm_swap(): too few bytes (%d after header) for unorm.icu\n", + length); + *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; + return 0; + } + } + + /* read the first 32 indexes (ICU 2.8/format version 2.2: _NORM_INDEX_TOP==32, might grow) */ + for(i=0; i<32; ++i) { + indexes[i]=udata_readInt32(ds, inIndexes[i]); + } + + /* calculate the total length of the data */ + size= + 32*4+ /* size of indexes[] */ + indexes[_NORM_INDEX_TRIE_SIZE]+ + indexes[_NORM_INDEX_UCHAR_COUNT]*2+ + indexes[_NORM_INDEX_COMBINE_DATA_COUNT]*2+ + indexes[_NORM_INDEX_FCD_TRIE_SIZE]+ + indexes[_NORM_INDEX_AUX_TRIE_SIZE]+ + indexes[_NORM_INDEX_CANON_SET_COUNT]*2; + + if(length>=0) { + if(length<size) { + udata_printError(ds, "unorm_swap(): too few bytes (%d after header) for all of unorm.icu\n", + length); + *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR; + return 0; + } + + /* copy the data for inaccessible bytes */ + if(inBytes!=outBytes) { + uprv_memcpy(outBytes, inBytes, size); + } + + offset=0; + + /* swap the indexes[] */ + count=32*4; + ds->swapArray32(ds, inBytes, count, outBytes, pErrorCode); + offset+=count; + + /* swap the main UTrie */ + count=indexes[_NORM_INDEX_TRIE_SIZE]; + utrie_swap(ds, inBytes+offset, count, outBytes+offset, pErrorCode); + offset+=count; + + /* swap the uint16_t extraData[] and the uint16_t combiningTable[] */ + count=(indexes[_NORM_INDEX_UCHAR_COUNT]+indexes[_NORM_INDEX_COMBINE_DATA_COUNT])*2; + ds->swapArray16(ds, inBytes+offset, count, outBytes+offset, pErrorCode); + offset+=count; + + /* swap the FCD UTrie */ + count=indexes[_NORM_INDEX_FCD_TRIE_SIZE]; + if(count!=0) { + utrie_swap(ds, inBytes+offset, count, outBytes+offset, pErrorCode); + offset+=count; + } + + /* swap the aux UTrie */ + count=indexes[_NORM_INDEX_AUX_TRIE_SIZE]; + if(count!=0) { + utrie_swap(ds, inBytes+offset, count, outBytes+offset, pErrorCode); + offset+=count; + } + + /* swap the uint16_t combiningTable[] */ + count=indexes[_NORM_INDEX_CANON_SET_COUNT]*2; + ds->swapArray16(ds, inBytes+offset, count, outBytes+offset, pErrorCode); + offset+=count; + } + + return headerSize+size; +} + +#endif + +/* swap any data (except a .dat package) ------------------------------------ */ + +static const struct { + uint8_t dataFormat[4]; + UDataSwapFn *swapFn; +} swapFns[]={ + { { 0x52, 0x65, 0x73, 0x42 }, ures_swap }, /* dataFormat="ResB" */ +#if !UCONFIG_NO_LEGACY_CONVERSION + { { 0x63, 0x6e, 0x76, 0x74 }, ucnv_swap }, /* dataFormat="cnvt" */ +#endif +#if !UCONFIG_NO_CONVERSION + { { 0x43, 0x76, 0x41, 0x6c }, ucnv_swapAliases }, /* dataFormat="CvAl" */ +#endif +#if !UCONFIG_NO_IDNA + { { 0x53, 0x50, 0x52, 0x50 }, usprep_swap }, /* dataFormat="SPRP" */ +#endif + /* insert data formats here, descending by expected frequency of occurrence */ + { { 0x55, 0x50, 0x72, 0x6f }, uprops_swap }, /* dataFormat="UPro" */ + + { { UCASE_FMT_0, UCASE_FMT_1, UCASE_FMT_2, UCASE_FMT_3 }, + ucase_swap }, /* dataFormat="cAsE" */ + + { { UBIDI_FMT_0, UBIDI_FMT_1, UBIDI_FMT_2, UBIDI_FMT_3 }, + ubidi_swap }, /* dataFormat="BiDi" */ + +#if !UCONFIG_NO_NORMALIZATION + { { 0x4e, 0x6f, 0x72, 0x6d }, unorm_swap }, /* dataFormat="Norm" */ +#endif +#if !UCONFIG_NO_COLLATION + { { 0x55, 0x43, 0x6f, 0x6c }, ucol_swap }, /* dataFormat="UCol" */ + { { 0x49, 0x6e, 0x76, 0x43 }, ucol_swapInverseUCA },/* dataFormat="InvC" */ +#endif +#if !UCONFIG_NO_BREAK_ITERATION + { { 0x42, 0x72, 0x6b, 0x20 }, ubrk_swap }, /* dataFormat="Brk " */ + { { 0x54, 0x72, 0x44, 0x63 }, triedict_swap }, /* dataFormat="TrDc " */ +#endif + { { 0x70, 0x6e, 0x61, 0x6d }, upname_swap }, /* dataFormat="pnam" */ + { { 0x75, 0x6e, 0x61, 0x6d }, uchar_swapNames } /* dataFormat="unam" */ +}; + +U_CAPI int32_t U_EXPORT2 +udata_swap(const UDataSwapper *ds, + const void *inData, int32_t length, void *outData, + UErrorCode *pErrorCode) { + char dataFormatChars[4]; + const UDataInfo *pInfo; + int32_t headerSize, i, swappedLength; + + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return 0; + } + + /* + * Preflight the header first; checks for illegal arguments, too. + * Do not swap the header right away because the format-specific swapper + * will swap it, get the headerSize again, and also use the header + * information. Otherwise we would have to pass some of the information + * and not be able to use the UDataSwapFn signature. + */ + headerSize=udata_swapDataHeader(ds, inData, -1, NULL, pErrorCode); + + /* + * If we wanted udata_swap() to also handle non-loadable data like a UTrie, + * then we could check here for further known magic values and structures. + */ + if(U_FAILURE(*pErrorCode)) { + return 0; /* the data format was not recognized */ + } + + pInfo=(const UDataInfo *)((const char *)inData+4); + + { + /* convert the data format from ASCII to Unicode to the system charset */ + UChar u[4]={ + pInfo->dataFormat[0], pInfo->dataFormat[1], + pInfo->dataFormat[2], pInfo->dataFormat[3] + }; + + if(uprv_isInvariantUString(u, 4)) { + u_UCharsToChars(u, dataFormatChars, 4); + } else { + dataFormatChars[0]=dataFormatChars[1]=dataFormatChars[2]=dataFormatChars[3]='?'; + } + } + + /* dispatch to the swap function for the dataFormat */ + for(i=0; i<LENGTHOF(swapFns); ++i) { + if(0==memcmp(swapFns[i].dataFormat, pInfo->dataFormat, 4)) { + swappedLength=swapFns[i].swapFn(ds, inData, length, outData, pErrorCode); + + if(U_FAILURE(*pErrorCode)) { + udata_printError(ds, "udata_swap(): failure swapping data format %02x.%02x.%02x.%02x (\"%c%c%c%c\") - %s\n", + pInfo->dataFormat[0], pInfo->dataFormat[1], + pInfo->dataFormat[2], pInfo->dataFormat[3], + dataFormatChars[0], dataFormatChars[1], + dataFormatChars[2], dataFormatChars[3], + u_errorName(*pErrorCode)); + } else if(swappedLength<(length-15)) { + /* swapped less than expected */ + udata_printError(ds, "udata_swap() warning: swapped only %d out of %d bytes - data format %02x.%02x.%02x.%02x (\"%c%c%c%c\")\n", + swappedLength, length, + pInfo->dataFormat[0], pInfo->dataFormat[1], + pInfo->dataFormat[2], pInfo->dataFormat[3], + dataFormatChars[0], dataFormatChars[1], + dataFormatChars[2], dataFormatChars[3], + u_errorName(*pErrorCode)); + } + + return swappedLength; + } + } + + /* the dataFormat was not recognized */ + udata_printError(ds, "udata_swap(): unknown data format %02x.%02x.%02x.%02x (\"%c%c%c%c\")\n", + pInfo->dataFormat[0], pInfo->dataFormat[1], + pInfo->dataFormat[2], pInfo->dataFormat[3], + dataFormatChars[0], dataFormatChars[1], + dataFormatChars[2], dataFormatChars[3]); + + *pErrorCode=U_UNSUPPORTED_ERROR; + return 0; +} diff --git a/Build/source/libs/icu/icu-xetex/tools/toolutil/swapimpl.h b/Build/source/libs/icu/icu-xetex/tools/toolutil/swapimpl.h new file mode 100644 index 00000000000..b36ec2609da --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/tools/toolutil/swapimpl.h @@ -0,0 +1,43 @@ +/* +******************************************************************************* +* +* Copyright (C) 2005, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: swapimpl.h +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2005jul29 +* created by: Markus W. Scherer +* +* Declarations for data file swapping functions not declared in internal +* library headers. +*/ + +#ifndef __SWAPIMPL_H__ +#define __SWAPIMPL_H__ + +#include "unicode/utypes.h" +#include "udataswp.h" + +/** + * Identifies and then transforms the ICU data piece in-place, or determines + * its length. See UDataSwapFn. + * This function handles single data pieces (but not .dat data packages) + * and internally dispatches to per-type swap functions. + * Sets a U_UNSUPPORTED_ERROR if the data format is not recognized. + * + * @see UDataSwapFn + * @see udata_openSwapper + * @see udata_openSwapperForInputData + * @internal ICU 2.8 + */ +U_CAPI int32_t U_EXPORT2 +udata_swap(const UDataSwapper *ds, + const void *inData, int32_t length, void *outData, + UErrorCode *pErrorCode); + +#endif diff --git a/Build/source/libs/icu/icu-xetex/tools/toolutil/toolutil.c b/Build/source/libs/icu/icu-xetex/tools/toolutil/toolutil.c new file mode 100644 index 00000000000..d4246a85680 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/tools/toolutil/toolutil.c @@ -0,0 +1,209 @@ +/* +******************************************************************************* +* +* Copyright (C) 1999-2007, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: toolutil.c +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 1999nov19 +* created by: Markus W. Scherer +* +* This file contains utility functions for ICU tools like genccode. +*/ + +#include <stdio.h> +#include "unicode/utypes.h" +#include "unicode/putil.h" +#include "cmemory.h" +#include "cstring.h" +#include "toolutil.h" + +#ifdef U_WINDOWS +# define VC_EXTRALEAN +# define WIN32_LEAN_AND_MEAN +# define NOUSER +# define NOSERVICE +# define NOIME +# define NOMCX +# include <windows.h> +# include <direct.h> +#else +# include <sys/stat.h> +# include <sys/types.h> +#endif +#include <errno.h> + +U_CAPI const char * U_EXPORT2 +getLongPathname(const char *pathname) { +#ifdef U_WINDOWS + /* anticipate problems with "short" pathnames */ + static WIN32_FIND_DATA info; + HANDLE file=FindFirstFileA(pathname, &info); + if(file!=INVALID_HANDLE_VALUE) { + if(info.cAlternateFileName[0]!=0) { + /* this file has a short name, get and use the long one */ + const char *basename=findBasename(pathname); + if(basename!=pathname) { + /* prepend the long filename with the original path */ + uprv_memmove(info.cFileName+(basename-pathname), info.cFileName, uprv_strlen(info.cFileName)+1); + uprv_memcpy(info.cFileName, pathname, basename-pathname); + } + pathname=info.cFileName; + } + FindClose(file); + } +#endif + return pathname; +} + +U_CAPI const char * U_EXPORT2 +findBasename(const char *filename) { + const char *basename=uprv_strrchr(filename, U_FILE_SEP_CHAR); + +#if U_FILE_ALT_SEP_CHAR!=U_FILE_SEP_CHAR + if(basename==NULL) { + /* Use lenient matching on Windows, which can accept either \ or / + This is useful for environments like Win32+CygWin which have both. + */ + basename=uprv_strrchr(filename, U_FILE_ALT_SEP_CHAR); + } +#endif + + if(basename!=NULL) { + return basename+1; + } else { + return filename; + } +} + +U_CAPI void U_EXPORT2 +uprv_mkdir(const char *pathname, UErrorCode *status) { + int retVal = 0; +#if defined(U_WINDOWS) + retVal = _mkdir(pathname); +#else + retVal = mkdir(pathname, S_IRWXU | (S_IROTH | S_IXOTH) | (S_IROTH | S_IXOTH)); +#endif + if (retVal && errno != EEXIST) { + *status = U_FILE_ACCESS_ERROR; + } +} + +/* tool memory helper ------------------------------------------------------- */ + +struct UToolMemory { + char name[64]; + int32_t capacity, maxCapacity, size, index; + void *array; + UAlignedMemory staticArray[1]; +}; + +U_CAPI UToolMemory * U_EXPORT2 +utm_open(const char *name, int32_t initialCapacity, int32_t maxCapacity, int32_t size) { + UToolMemory *mem; + + if(maxCapacity<initialCapacity) { + maxCapacity=initialCapacity; + } + + mem=(UToolMemory *)uprv_malloc(sizeof(UToolMemory)+initialCapacity*size); + if(mem==NULL) { + fprintf(stderr, "error: %s - out of memory\n", name); + exit(U_MEMORY_ALLOCATION_ERROR); + } + mem->array=mem->staticArray; + + uprv_strcpy(mem->name, name); + mem->capacity=initialCapacity; + mem->maxCapacity=maxCapacity; + mem->size=size; + mem->index=0; + return mem; +} + +U_CAPI void U_EXPORT2 +utm_close(UToolMemory *mem) { + if(mem!=NULL) { + if(mem->array!=mem->staticArray) { + uprv_free(mem->array); + } + uprv_free(mem); + } +} + + +U_CAPI void * U_EXPORT2 +utm_getStart(UToolMemory *mem) { + return (char *)mem->array; +} + +U_CAPI int32_t U_EXPORT2 +utm_countItems(UToolMemory *mem) { + return mem->index; +} + + +static UBool +utm_hasCapacity(UToolMemory *mem, int32_t capacity) { + if(mem->capacity<capacity) { + int32_t newCapacity; + + if(mem->maxCapacity<capacity) { + fprintf(stderr, "error: %s - trying to use more than maxCapacity=%ld units\n", + mem->name, (long)mem->maxCapacity); + exit(U_MEMORY_ALLOCATION_ERROR); + } + + /* try to allocate a larger array */ + if(capacity>=2*mem->capacity) { + newCapacity=capacity; + } else if(mem->capacity<=mem->maxCapacity/3) { + newCapacity=2*mem->capacity; + } else { + newCapacity=mem->maxCapacity; + } + + if(mem->array==mem->staticArray) { + mem->array=uprv_malloc(newCapacity*mem->size); + if(mem->array!=NULL) { + uprv_memcpy(mem->array, mem->staticArray, mem->index*mem->size); + } + } else { + mem->array=uprv_realloc(mem->array, newCapacity*mem->size); + } + + if(mem->array==NULL) { + fprintf(stderr, "error: %s - out of memory\n", mem->name); + exit(U_MEMORY_ALLOCATION_ERROR); + } + } + + return TRUE; +} + +U_CAPI void * U_EXPORT2 +utm_alloc(UToolMemory *mem) { + char *p=(char *)mem->array+mem->index*mem->size; + int32_t newIndex=mem->index+1; + if(utm_hasCapacity(mem, newIndex)) { + mem->index=newIndex; + uprv_memset(p, 0, mem->size); + } + return p; +} + +U_CAPI void * U_EXPORT2 +utm_allocN(UToolMemory *mem, int32_t n) { + char *p=(char *)mem->array+mem->index*mem->size; + int32_t newIndex=mem->index+n; + if(utm_hasCapacity(mem, newIndex)) { + mem->index=newIndex; + uprv_memset(p, 0, n*mem->size); + } + return p; +} diff --git a/Build/source/libs/icu/icu-xetex/tools/toolutil/toolutil.h b/Build/source/libs/icu/icu-xetex/tools/toolutil/toolutil.h new file mode 100644 index 00000000000..91c6c881ca0 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/tools/toolutil/toolutil.h @@ -0,0 +1,111 @@ +/* +******************************************************************************* +* +* Copyright (C) 1999-2005, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: toolutil.h +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 1999nov19 +* created by: Markus W. Scherer +* +* This file defines utility functions for ICU tools like genccode. +*/ + +#ifndef __TOOLUTIL_H__ +#define __TOOLUTIL_H__ + +#include "unicode/utypes.h" + +/* + * For Windows, a path/filename may be the short (8.3) version + * of the "real", long one. In this case, the short one + * is abbreviated and contains a tilde etc. + * This function returns a pointer to the original pathname + * if it is the "real" one itself, and a pointer to a static + * buffer (not thread-safe) containing the long version + * if the pathname is indeed abbreviated. + * + * On platforms other than Windows, this function always returns + * the input pathname pointer. + * + * This function is especially useful in tools that are called + * by a batch file for loop, which yields short pathnames on Win9x. + */ +U_CAPI const char * U_EXPORT2 +getLongPathname(const char *pathname); + +/* + * Find the basename at the end of a pathname, i.e., the part + * after the last file separator, and return a pointer + * to this part of the pathname. + * If the pathname only contains a basename and no file separator, + * then the pathname pointer itself is returned. + */ +U_CAPI const char * U_EXPORT2 +findBasename(const char *filename); + +/* + * Creates a diretory with pathname. + * + * @param status Set to an error code when mkdir failed. + */ +U_CAPI void U_EXPORT2 +uprv_mkdir(const char *pathname, UErrorCode *status); + +/* + * UToolMemory is used for generic, custom memory management. + * It is allocated with enough space for count*size bytes starting + * at array. + * The array is declared with a union of large data types so + * that its base address is aligned for any types. + * If size is a multiple of a data type size, then such items + * can be safely allocated inside the array, at offsets that + * are themselves multiples of size. + */ +struct UToolMemory; +typedef struct UToolMemory UToolMemory; + +/** + * Open a UToolMemory object for allocation of initialCapacity to maxCapacity + * items with size bytes each. + */ +U_CAPI UToolMemory * U_EXPORT2 +utm_open(const char *name, int32_t initialCapacity, int32_t maxCapacity, int32_t size); + +/** + * Close a UToolMemory object. + */ +U_CAPI void U_EXPORT2 +utm_close(UToolMemory *mem); + +/** + * Get the pointer to the beginning of the array of items. + * The pointer becomes invalid after allocation of new items. + */ +U_CAPI void * U_EXPORT2 +utm_getStart(UToolMemory *mem); + +/** + * Get the current number of items. + */ +U_CAPI int32_t U_EXPORT2 +utm_countItems(UToolMemory *mem); + +/** + * Allocate one more item and return the pointer to its start in the array. + */ +U_CAPI void * U_EXPORT2 +utm_alloc(UToolMemory *mem); + +/** + * Allocate n items and return the pointer to the start of the first one in the array. + */ +U_CAPI void * U_EXPORT2 +utm_allocN(UToolMemory *mem, int32_t n); + +#endif diff --git a/Build/source/libs/icu/icu-xetex/tools/toolutil/toolutil.vcproj b/Build/source/libs/icu/icu-xetex/tools/toolutil/toolutil.vcproj new file mode 100644 index 00000000000..1ada5504bec --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/tools/toolutil/toolutil.vcproj @@ -0,0 +1,356 @@ +<?xml version="1.0" encoding="Windows-1252"?> +<VisualStudioProject + ProjectType="Visual C++" + Version="8.00" + Name="toolutil" + ProjectGUID="{6B231032-3CB5-4EED-9210-810D666A23A0}" + > + <Platforms> + <Platform + Name="Win32" + /> + </Platforms> + <ToolFiles> + </ToolFiles> + <Configurations> + <Configuration + Name="Release|Win32" + OutputDirectory=".\..\..\..\lib" + IntermediateDirectory=".\Release" + ConfigurationType="2" + InheritedPropertySheets="$(VCInstallDir)VCProjectDefaults\UpgradeFromVC71.vsprops" + UseOfMFC="0" + ATLMinimizesCRunTimeLibraryUsage="false" + CharacterSet="2" + > + <Tool + Name="VCPreBuildEventTool" + /> + <Tool + Name="VCCustomBuildTool" + /> + <Tool + Name="VCXMLDataGeneratorTool" + /> + <Tool + Name="VCWebServiceProxyGeneratorTool" + /> + <Tool + Name="VCMIDLTool" + PreprocessorDefinitions="NDEBUG" + MkTypLibCompatible="true" + SuppressStartupBanner="true" + TargetEnvironment="1" + TypeLibraryName=".\..\..\..\lib\icutu.tlb" + /> + <Tool + Name="VCCLCompilerTool" + WholeProgramOptimization="true" + AdditionalIncludeDirectories="..\..\..\include,..\..\common" + PreprocessorDefinitions="WIN32;NDEBUG;_CRT_SECURE_NO_DEPRECATE;U_TOOLUTIL_IMPLEMENTATION" + StringPooling="true" + RuntimeLibrary="2" + EnableFunctionLevelLinking="true" + DisableLanguageExtensions="true" + TreatWChar_tAsBuiltInType="true" + PrecompiledHeaderFile=".\Release/toolutil.pch" + AssemblerListingLocation=".\Release/" + ObjectFile=".\Release/" + ProgramDataBaseFileName=".\Release/" + WarningLevel="3" + SuppressStartupBanner="true" + CompileAs="0" + /> + <Tool + Name="VCManagedResourceCompilerTool" + /> + <Tool + Name="VCResourceCompilerTool" + PreprocessorDefinitions="NDEBUG" + Culture="1033" + /> + <Tool + Name="VCPreLinkEventTool" + /> + <Tool + Name="VCLinkerTool" + OutputFile="..\..\..\bin\icutu38.dll" + LinkIncremental="1" + SuppressStartupBanner="true" + AdditionalLibraryDirectories="..\..\..\lib" + ProgramDatabaseFile=".\..\..\..\lib\icutu38.pdb" + EnableCOMDATFolding="2" + OptimizeForWindows98="1" + LinkTimeCodeGeneration="1" + BaseAddress="0x4ac00000" + ImportLibrary="..\..\..\lib\icutu.lib" + /> + <Tool + Name="VCALinkTool" + /> + <Tool + Name="VCManifestTool" + /> + <Tool + Name="VCXDCMakeTool" + /> + <Tool + Name="VCBscMakeTool" + /> + <Tool + Name="VCFxCopTool" + /> + <Tool + Name="VCAppVerifierTool" + /> + <Tool + Name="VCWebDeploymentTool" + /> + <Tool + Name="VCPostBuildEventTool" + /> + </Configuration> + <Configuration + Name="Debug|Win32" + OutputDirectory=".\..\..\..\lib" + IntermediateDirectory=".\Debug" + ConfigurationType="2" + InheritedPropertySheets="$(VCInstallDir)VCProjectDefaults\UpgradeFromVC71.vsprops" + UseOfMFC="0" + ATLMinimizesCRunTimeLibraryUsage="false" + CharacterSet="2" + > + <Tool + Name="VCPreBuildEventTool" + /> + <Tool + Name="VCCustomBuildTool" + /> + <Tool + Name="VCXMLDataGeneratorTool" + /> + <Tool + Name="VCWebServiceProxyGeneratorTool" + /> + <Tool + Name="VCMIDLTool" + PreprocessorDefinitions="_DEBUG" + MkTypLibCompatible="true" + SuppressStartupBanner="true" + TargetEnvironment="1" + TypeLibraryName=".\..\..\..\lib\icutud.tlb" + /> + <Tool + Name="VCCLCompilerTool" + Optimization="0" + AdditionalIncludeDirectories="..\..\..\include,..\..\common" + PreprocessorDefinitions="WIN32;_DEBUG;_CRT_SECURE_NO_DEPRECATE;U_TOOLUTIL_IMPLEMENTATION" + BasicRuntimeChecks="3" + RuntimeLibrary="3" + BufferSecurityCheck="true" + DisableLanguageExtensions="true" + TreatWChar_tAsBuiltInType="true" + PrecompiledHeaderFile=".\Debug/toolutil.pch" + AssemblerListingLocation=".\Debug/" + ObjectFile=".\Debug/" + ProgramDataBaseFileName=".\Debug/" + WarningLevel="3" + SuppressStartupBanner="true" + DebugInformationFormat="4" + CompileAs="0" + /> + <Tool + Name="VCManagedResourceCompilerTool" + /> + <Tool + Name="VCResourceCompilerTool" + PreprocessorDefinitions="_DEBUG" + Culture="1033" + /> + <Tool + Name="VCPreLinkEventTool" + /> + <Tool + Name="VCLinkerTool" + OutputFile="..\..\..\bin\icutu38d.dll" + LinkIncremental="2" + SuppressStartupBanner="true" + AdditionalLibraryDirectories="..\..\..\lib" + GenerateDebugInformation="true" + ProgramDatabaseFile=".\..\..\..\lib\icutu38d.pdb" + BaseAddress="0x4ac00000" + ImportLibrary="..\..\..\lib\icutud.lib" + /> + <Tool + Name="VCALinkTool" + /> + <Tool + Name="VCManifestTool" + UseFAT32Workaround="true" + /> + <Tool + Name="VCXDCMakeTool" + /> + <Tool + Name="VCBscMakeTool" + /> + <Tool + Name="VCFxCopTool" + /> + <Tool + Name="VCAppVerifierTool" + /> + <Tool + Name="VCWebDeploymentTool" + /> + <Tool + Name="VCPostBuildEventTool" + /> + </Configuration> + </Configurations> + <References> + </References> + <Files> + <Filter + Name="Source Files" + Filter="cpp;c;cxx;rc;def;r;odl;idl;hpj;bat" + > + <File + RelativePath=".\filestrm.c" + > + </File> + <File + RelativePath=".\package.cpp" + > + </File> + <File + RelativePath=".\pkgitems.cpp" + > + </File> + <File + RelativePath=".\propsvec.c" + > + </File> + <File + RelativePath=".\swapimpl.cpp" + > + </File> + <File + RelativePath=".\toolutil.c" + > + <FileConfiguration + Name="Release|Win32" + > + <Tool + Name="VCCLCompilerTool" + DisableLanguageExtensions="false" + /> + </FileConfiguration> + <FileConfiguration + Name="Debug|Win32" + > + <Tool + Name="VCCLCompilerTool" + DisableLanguageExtensions="false" + /> + </FileConfiguration> + </File> + <File + RelativePath=".\ucbuf.c" + > + </File> + <File + RelativePath=".\ucm.c" + > + </File> + <File + RelativePath=".\ucmstate.c" + > + </File> + <File + RelativePath=".\unewdata.c" + > + </File> + <File + RelativePath=".\uoptions.c" + > + </File> + <File + RelativePath=".\uparse.c" + > + </File> + <File + RelativePath=".\writesrc.c" + > + </File> + <File + RelativePath=".\xmlparser.cpp" + > + </File> + </Filter> + <Filter + Name="Header Files" + Filter="h;hpp;hxx;hm;inl" + > + <File + RelativePath=".\filestrm.h" + > + </File> + <File + RelativePath=".\package.h" + > + </File> + <File + RelativePath=".\pkg_imp.h" + > + </File> + <File + RelativePath=".\propsvec.h" + > + </File> + <File + RelativePath=".\swapimpl.h" + > + </File> + <File + RelativePath=".\toolutil.h" + > + </File> + <File + RelativePath=".\ucbuf.h" + > + </File> + <File + RelativePath=".\ucm.h" + > + </File> + <File + RelativePath=".\unewdata.h" + > + </File> + <File + RelativePath=".\uoptions.h" + > + </File> + <File + RelativePath=".\uparse.h" + > + </File> + <File + RelativePath=".\writesrc.h" + > + </File> + <File + RelativePath=".\xmlparser.h" + > + </File> + </Filter> + <Filter + Name="Resource Files" + Filter="ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe" + > + </Filter> + </Files> + <Globals> + </Globals> +</VisualStudioProject> diff --git a/Build/source/libs/icu/icu-xetex/tools/toolutil/ucbuf.c b/Build/source/libs/icu/icu-xetex/tools/toolutil/ucbuf.c new file mode 100644 index 00000000000..7d7556e6ddc --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/tools/toolutil/ucbuf.c @@ -0,0 +1,789 @@ +/* +******************************************************************************* +* +* Copyright (C) 1998-2007, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* +* File ucbuf.c +* +* Modification History: +* +* Date Name Description +* 05/10/01 Ram Creation. +******************************************************************************* +*/ + +#include "unicode/utypes.h" +#include "unicode/putil.h" +#include "unicode/ucnv.h" +#include "unicode/ucnv_err.h" +#include "filestrm.h" +#include "cstring.h" +#include "cmemory.h" +#include "ustrfmt.h" +#include "unicode/ustring.h" +#include "unicode/uchar.h" +#include "ucbuf.h" +#include <stdio.h> + +#if !UCONFIG_NO_CONVERSION + + +#define MAX_IN_BUF 1000 +#define MAX_U_BUF 1500 +#define CONTEXT_LEN 15 + +struct UCHARBUF { + UChar* buffer; + UChar* currentPos; + UChar* bufLimit; + int32_t bufCapacity; + int32_t remaining; + int32_t signatureLength; + FileStream* in; + UConverter* conv; + UBool showWarning; /* makes this API not produce any errors */ + UBool isBuffered; +}; + +U_CAPI UBool U_EXPORT2 +ucbuf_autodetect_fs(FileStream* in, const char** cp, UConverter** conv, int32_t* signatureLength, UErrorCode* error){ + char start[8]; + int32_t numRead; + + UChar target[1]={ 0 }; + UChar* pTarget; + const char* pStart; + + /* read a few bytes */ + numRead=T_FileStream_read(in, start, sizeof(start)); + + *cp = ucnv_detectUnicodeSignature(start, numRead, signatureLength, error); + + /* unread the bytes beyond what was consumed for U+FEFF */ + T_FileStream_rewind(in); + if (*signatureLength > 0) { + numRead = T_FileStream_read(in, start, *signatureLength); + } + + if(*cp==NULL){ + *conv =NULL; + return FALSE; + } + + /* open the converter for the detected Unicode charset */ + *conv = ucnv_open(*cp,error); + + /* convert and ignore initial U+FEFF, and the buffer overflow */ + pTarget = target; + pStart = start; + ucnv_toUnicode(*conv, &pTarget, target+1, &pStart, start+*signatureLength, NULL, FALSE, error); + *signatureLength = (int32_t)(pStart - start); + if(*error==U_BUFFER_OVERFLOW_ERROR) { + *error=U_ZERO_ERROR; + } + + /* verify that we successfully read exactly U+FEFF */ + if(U_SUCCESS(*error) && (pTarget!=(target+1) || target[0]!=0xfeff)) { + *error=U_INTERNAL_PROGRAM_ERROR; + } + + + return TRUE; +} +static UBool ucbuf_isCPKnown(const char* cp){ + if(ucnv_compareNames("UTF-8",cp)==0){ + return TRUE; + } + if(ucnv_compareNames("UTF-16BE",cp)==0){ + return TRUE; + } + if(ucnv_compareNames("UTF-16LE",cp)==0){ + return TRUE; + } + if(ucnv_compareNames("UTF-16",cp)==0){ + return TRUE; + } + if(ucnv_compareNames("UTF-32",cp)==0){ + return TRUE; + } + if(ucnv_compareNames("UTF-32BE",cp)==0){ + return TRUE; + } + if(ucnv_compareNames("UTF-32LE",cp)==0){ + return TRUE; + } + if(ucnv_compareNames("SCSU",cp)==0){ + return TRUE; + } + if(ucnv_compareNames("BOCU-1",cp)==0){ + return TRUE; + } + if(ucnv_compareNames("UTF-7",cp)==0){ + return TRUE; + } + return FALSE; +} + +U_CAPI FileStream * U_EXPORT2 +ucbuf_autodetect(const char* fileName, const char** cp,UConverter** conv, int32_t* signatureLength,UErrorCode* error){ + FileStream* in=NULL; + if(error==NULL || U_FAILURE(*error)){ + return NULL; + } + if(conv==NULL || cp==NULL || fileName==NULL){ + *error = U_ILLEGAL_ARGUMENT_ERROR; + return NULL; + } + /* open the file */ + in= T_FileStream_open(fileName,"rb"); + + if(in == NULL){ + *error=U_FILE_ACCESS_ERROR; + return NULL; + } + + if(ucbuf_autodetect_fs(in,cp,conv,signatureLength,error)) { + return in; + } else { + ucnv_close(*conv); + *conv=NULL; + T_FileStream_close(in); + return NULL; + } +} + +/* fill the uchar buffer */ +static UCHARBUF* +ucbuf_fillucbuf( UCHARBUF* buf,UErrorCode* error){ + UChar* pTarget=NULL; + UChar* target=NULL; + const char* source=NULL; + char carr[MAX_IN_BUF] = {'\0'}; + char* cbuf = carr; + int32_t inputRead=0; + int32_t outputWritten=0; + int32_t offset=0; + const char* sourceLimit =NULL; + int32_t cbufSize=0; + pTarget = buf->buffer; + /* check if we arrived here without exhausting the buffer*/ + if(buf->currentPos<buf->bufLimit){ + offset = (int32_t)(buf->bufLimit-buf->currentPos); + memmove(buf->buffer,buf->currentPos,offset* sizeof(UChar)); + } + +#if DEBUG + memset(pTarget+offset,0xff,sizeof(UChar)*(MAX_IN_BUF-offset)); +#endif + if(buf->isBuffered){ + cbufSize = MAX_IN_BUF; + /* read the file */ + inputRead=T_FileStream_read(buf->in,cbuf,cbufSize-offset); + buf->remaining-=inputRead; + + }else{ + cbufSize = T_FileStream_size(buf->in); + cbuf = (char*)uprv_malloc(cbufSize); + inputRead= T_FileStream_read(buf->in,cbuf,cbufSize); + buf->remaining-=inputRead; + } + + /* just to be sure...*/ + if ( 0 == inputRead ) + buf->remaining = 0; + + target=pTarget; + /* convert the bytes */ + if(buf->conv){ + /* set the callback to stop */ + UConverterToUCallback toUOldAction ; + void* toUOldContext; + void* toUNewContext=NULL; + ucnv_setToUCallBack(buf->conv, + UCNV_TO_U_CALLBACK_STOP, + toUNewContext, + &toUOldAction, + (const void**)&toUOldContext, + error); + /* since state is saved in the converter we add offset to source*/ + target = pTarget+offset; + source = cbuf; + sourceLimit = source + inputRead; + ucnv_toUnicode(buf->conv,&target,target+(buf->bufCapacity-offset), + &source,sourceLimit,NULL, + (UBool)(buf->remaining==0),error); + + if(U_FAILURE(*error)){ + char context[CONTEXT_LEN]; + char preContext[CONTEXT_LEN]; + char postContext[CONTEXT_LEN]; + int8_t len = CONTEXT_LEN; + int32_t start=0; + int32_t stop =0; + int32_t pos =0; + /* use erro1 to preserve the error code */ + UErrorCode error1 =U_ZERO_ERROR; + + if( buf->showWarning==TRUE){ + fprintf(stderr,"\n###WARNING: Encountered abnormal bytes while" + " converting input stream to target encoding: %s\n", + u_errorName(*error)); + } + + + /* now get the context chars */ + ucnv_getInvalidChars(buf->conv,context,&len,&error1); + context[len]= 0 ; /* null terminate the buffer */ + + pos = (int32_t)(source - cbuf - len); + + /* for pre-context */ + start = (pos <=CONTEXT_LEN)? 0 : (pos - (CONTEXT_LEN-1)); + stop = pos-len; + + memcpy(preContext,cbuf+start,stop-start); + /* null terminate the buffer */ + preContext[stop-start] = 0; + + /* for post-context */ + start = pos+len; + stop = (int32_t)(((pos+CONTEXT_LEN)<= (sourceLimit-cbuf) )? (pos+(CONTEXT_LEN-1)) : (sourceLimit-cbuf)); + + memcpy(postContext,source,stop-start); + /* null terminate the buffer */ + postContext[stop-start] = 0; + + if(buf->showWarning ==TRUE){ + /* print out the context */ + fprintf(stderr,"\tPre-context: %s\n",preContext); + fprintf(stderr,"\tContext: %s\n",context); + fprintf(stderr,"\tPost-context: %s\n", postContext); + } + + /* reset the converter */ + ucnv_reset(buf->conv); + + /* set the call back to substitute + * and restart conversion + */ + ucnv_setToUCallBack(buf->conv, + UCNV_TO_U_CALLBACK_SUBSTITUTE, + toUNewContext, + &toUOldAction, + (const void**)&toUOldContext, + &error1); + + /* reset source and target start positions */ + target = pTarget+offset; + source = cbuf; + + /* re convert */ + ucnv_toUnicode(buf->conv,&target,target+(buf->bufCapacity-offset), + &source,sourceLimit,NULL, + (UBool)(buf->remaining==0),&error1); + + } + outputWritten = (int32_t)(target - pTarget); + + +#if DEBUG + { + int i; + target = pTarget; + for(i=0;i<numRead;i++){ + /* printf("%c", (char)(*target++));*/ + } + } +#endif + + }else{ + u_charsToUChars(cbuf,target+offset,inputRead); + outputWritten=((buf->remaining>cbufSize)? cbufSize:inputRead+offset); + } + buf->currentPos = pTarget; + buf->bufLimit=pTarget+outputWritten; + *buf->bufLimit=0; /*NUL terminate*/ + if(cbuf!=carr){ + uprv_free(cbuf); + } + return buf; +} + + + +/* get a UChar from the stream*/ +U_CAPI int32_t U_EXPORT2 +ucbuf_getc(UCHARBUF* buf,UErrorCode* error){ + if(error==NULL || U_FAILURE(*error)){ + return FALSE; + } + if(buf->currentPos>=buf->bufLimit){ + if(buf->remaining==0){ + return U_EOF; + } + buf=ucbuf_fillucbuf(buf,error); + if(U_FAILURE(*error)){ + return U_EOF; + } + } + + return *(buf->currentPos++); +} + +/* get a UChar32 from the stream*/ +U_CAPI int32_t U_EXPORT2 +ucbuf_getc32(UCHARBUF* buf,UErrorCode* error){ + int32_t retVal = (int32_t)U_EOF; + if(error==NULL || U_FAILURE(*error)){ + return FALSE; + } + if(buf->currentPos+1>=buf->bufLimit){ + if(buf->remaining==0){ + return U_EOF; + } + buf=ucbuf_fillucbuf(buf,error); + if(U_FAILURE(*error)){ + return U_EOF; + } + } + if(UTF_IS_LEAD(*(buf->currentPos))){ + retVal=UTF16_GET_PAIR_VALUE(buf->currentPos[0],buf->currentPos[1]); + buf->currentPos+=2; + }else{ + retVal = *(buf->currentPos++); + } + return retVal; +} + +/* u_unescapeAt() callback to return a UChar*/ +static UChar U_CALLCONV +_charAt(int32_t offset, void *context) { + return ((UCHARBUF*) context)->currentPos[offset]; +} + +/* getc and escape it */ +U_CAPI int32_t U_EXPORT2 +ucbuf_getcx32(UCHARBUF* buf,UErrorCode* error) { + int32_t length; + int32_t offset; + UChar32 c32,c1,c2; + if(error==NULL || U_FAILURE(*error)){ + return FALSE; + } + /* Fill the buffer if it is empty */ + if (buf->currentPos >=buf->bufLimit-2) { + ucbuf_fillucbuf(buf,error); + } + + /* Get the next character in the buffer */ + if (buf->currentPos < buf->bufLimit) { + c1 = *(buf->currentPos)++; + } else { + c1 = U_EOF; + } + + c2 = *(buf->currentPos); + + /* If it isn't a backslash, return it */ + if (c1 != 0x005C) { + return c1; + } + + /* Determine the amount of data in the buffer */ + length = (int32_t)(buf->bufLimit - buf->currentPos); + + /* The longest escape sequence is \Uhhhhhhhh; make sure + we have at least that many characters */ + if (length < 10) { + + /* fill the buffer */ + ucbuf_fillucbuf(buf,error); + length = (int32_t)(buf->bufLimit - buf->buffer); + } + + /* Process the escape */ + offset = 0; + c32 = u_unescapeAt(_charAt, &offset, length, (void*)buf); + + /* check if u_unescapeAt unescaped and converted + * to c32 or not + */ + if(c32==0xFFFFFFFF){ + if(buf->showWarning) { + char context[20]; + int32_t len = 20; + if(length < len) { + len = length; + } + context[len]= 0 ; /* null terminate the buffer */ + u_UCharsToChars( buf->currentPos, context, len); + fprintf(stderr,"Bad escape: [%c%s]...\n", (int)c1, context); + } + *error= U_ILLEGAL_ESCAPE_SEQUENCE; + return c1; + }else if(c32!=c2 || (c32==0x0075 && c2==0x0075 && c1==0x005C) /* for \u0075 c2=0x0075 and c32==0x0075*/){ + /* Update the current buffer position */ + buf->currentPos += offset; + }else{ + /* unescaping failed so we just return + * c1 and not consume the buffer + * this is useful for rules with escapes + * in resouce bundles + * eg: \' \\ \" + */ + return c1; + } + + return c32; +} + +U_CAPI UCHARBUF* U_EXPORT2 +ucbuf_open(const char* fileName,const char** cp,UBool showWarning, UBool buffered, UErrorCode* error){ + + FileStream* in = NULL; + int32_t fileSize=0; + const char* knownCp; + if(error==NULL || U_FAILURE(*error)){ + return NULL; + } + if(cp==NULL || fileName==NULL){ + *error = U_ILLEGAL_ARGUMENT_ERROR; + return FALSE; + } + if (!uprv_strcmp(fileName, "-")) { + in = T_FileStream_stdin(); + }else{ + in = T_FileStream_open(fileName, "rb"); + } + + if(in!=NULL){ + UCHARBUF* buf =(UCHARBUF*) uprv_malloc(sizeof(UCHARBUF)); + fileSize = T_FileStream_size(in); + if(buf == NULL){ + *error = U_MEMORY_ALLOCATION_ERROR; + T_FileStream_close(in); + return NULL; + } + buf->in=in; + buf->conv=NULL; + buf->showWarning = showWarning; + buf->isBuffered = buffered; + buf->signatureLength=0; + if(*cp==NULL || **cp=='\0'){ + /* don't have code page name... try to autodetect */ + ucbuf_autodetect_fs(in,cp,&buf->conv,&buf->signatureLength,error); + }else if(ucbuf_isCPKnown(*cp)){ + /* discard BOM */ + ucbuf_autodetect_fs(in,&knownCp,&buf->conv,&buf->signatureLength,error); + } + if(U_SUCCESS(*error) && buf->conv==NULL) { + buf->conv=ucnv_open(*cp,error); + } + if(U_FAILURE(*error)){ + ucnv_close(buf->conv); + uprv_free(buf); + T_FileStream_close(in); + return NULL; + } + + if((buf->conv==NULL) && (buf->showWarning==TRUE)){ + fprintf(stderr,"###WARNING: No converter defined. Using codepage of system.\n"); + } + buf->remaining=fileSize-buf->signatureLength; + if(buf->isBuffered){ + buf->bufCapacity=MAX_U_BUF; + }else{ + buf->bufCapacity=buf->remaining+buf->signatureLength+1/*for terminating nul*/; + } + buf->buffer=(UChar*) uprv_malloc(U_SIZEOF_UCHAR * buf->bufCapacity ); + if (buf->buffer == NULL) { + *error = U_MEMORY_ALLOCATION_ERROR; + ucnv_close(buf->conv); + uprv_free(buf); + T_FileStream_close(in); + return NULL; + } + buf->currentPos=buf->buffer; + buf->bufLimit=buf->buffer; + if(U_FAILURE(*error)){ + fprintf(stderr, "Could not open codepage [%s]: %s\n", *cp, u_errorName(*error)); + ucnv_close(buf->conv); + uprv_free(buf); + T_FileStream_close(in); + return NULL; + } + buf=ucbuf_fillucbuf(buf,error); + return buf; + } + *error =U_FILE_ACCESS_ERROR; + return NULL; +} + + + +/* TODO: this method will fail if at the + * begining of buffer and the uchar to unget + * is from the previous buffer. Need to implement + * system to take care of that situation. + */ +U_CAPI void U_EXPORT2 +ucbuf_ungetc(int32_t c,UCHARBUF* buf){ + /* decrement currentPos pointer + * if not at the begining of buffer + */ + UChar escaped[8] ={'\0'}; + int32_t len =0; + if(c > 0xFFFF){ + len = uprv_itou(escaped,8,c,16,8); + }else{ + len=uprv_itou(escaped,8,c,16,4); + } + if(buf->currentPos!=buf->buffer){ + if(*(buf->currentPos-1)==c){ + buf->currentPos--; + }else if(u_strncmp(buf->currentPos-len,escaped,len) == 0){ + while(--len>0){ + buf->currentPos--; + } + } + } +} + +/* frees the resources of UChar* buffer */ +static void +ucbuf_closebuf(UCHARBUF* buf){ + uprv_free(buf->buffer); + buf->buffer = NULL; +} + +/* close the buf and release resources*/ +U_CAPI void U_EXPORT2 +ucbuf_close(UCHARBUF* buf){ + if(buf!=NULL){ + if(buf->conv){ + ucnv_close(buf->conv); + } + T_FileStream_close(buf->in); + ucbuf_closebuf(buf); + uprv_free(buf); + } +} + +/* rewind the buf and file stream */ +U_CAPI void U_EXPORT2 +ucbuf_rewind(UCHARBUF* buf,UErrorCode* error){ + if(error==NULL || U_FAILURE(*error)){ + return; + } + if(buf){ + buf->currentPos=buf->buffer; + buf->bufLimit=buf->buffer; + T_FileStream_rewind(buf->in); + buf->remaining=T_FileStream_size(buf->in)-buf->signatureLength; + + ucnv_resetToUnicode(buf->conv); + if(buf->signatureLength>0) { + UChar target[1]={ 0 }; + UChar* pTarget; + char start[8]; + const char* pStart; + int32_t numRead; + + /* read the signature bytes */ + numRead=T_FileStream_read(buf->in, start, buf->signatureLength); + + /* convert and ignore initial U+FEFF, and the buffer overflow */ + pTarget = target; + pStart = start; + ucnv_toUnicode(buf->conv, &pTarget, target+1, &pStart, start+numRead, NULL, FALSE, error); + if(*error==U_BUFFER_OVERFLOW_ERROR) { + *error=U_ZERO_ERROR; + } + + /* verify that we successfully read exactly U+FEFF */ + if(U_SUCCESS(*error) && (numRead!=buf->signatureLength || pTarget!=(target+1) || target[0]!=0xfeff)) { + *error=U_INTERNAL_PROGRAM_ERROR; + } + } + } +} + + +U_CAPI int32_t U_EXPORT2 +ucbuf_size(UCHARBUF* buf){ + if(buf){ + if(buf->isBuffered){ + return (T_FileStream_size(buf->in)-buf->signatureLength)/ucnv_getMinCharSize(buf->conv); + }else{ + return (int32_t)(buf->bufLimit - buf->buffer); + } + } + return 0; +} + +U_CAPI const UChar* U_EXPORT2 +ucbuf_getBuffer(UCHARBUF* buf,int32_t* len,UErrorCode* error){ + if(error==NULL || U_FAILURE(*error)){ + return NULL; + } + if(buf==NULL || len==NULL){ + *error = U_ILLEGAL_ARGUMENT_ERROR; + return NULL; + } + *len = (int32_t)(buf->bufLimit - buf->buffer); + return buf->buffer; +} + +U_CAPI const char* U_EXPORT2 +ucbuf_resolveFileName(const char* inputDir, const char* fileName, char* target, int32_t* len, UErrorCode* status){ + int32_t requiredLen = 0; + int32_t dirlen = 0; + int32_t filelen = 0; + if(status==NULL || U_FAILURE(*status)){ + return NULL; + } + + if(inputDir == NULL || fileName == NULL || len==NULL || (target==NULL && *len>0)){ + *status = U_ILLEGAL_ARGUMENT_ERROR; + return NULL; + } + + + dirlen = (int32_t)uprv_strlen(inputDir); + filelen = (int32_t)uprv_strlen(fileName); + if(inputDir[dirlen-1] != U_FILE_SEP_CHAR) { + requiredLen = dirlen + filelen + 2; + if((*len < requiredLen) || target==NULL){ + *len = requiredLen; + *status = U_BUFFER_OVERFLOW_ERROR; + return NULL; + } + + target[0] = '\0'; + /* + * append the input dir to openFileName if the first char in + * filename is not file seperation char and the last char input directory is not '.'. + * This is to support : + * genrb -s. /home/icu/data + * genrb -s. icu/data + * The user cannot mix notations like + * genrb -s. /icu/data --- the absolute path specified. -s redundant + * user should use + * genrb -s. icu/data --- start from CWD and look in icu/data dir + */ + if( (fileName[0] != U_FILE_SEP_CHAR) && (inputDir[dirlen-1] !='.')){ + uprv_strcpy(target, inputDir); + target[dirlen] = U_FILE_SEP_CHAR; + } + target[dirlen + 1] = '\0'; + } else { + requiredLen = dirlen + filelen + 1; + if((*len < requiredLen) || target==NULL){ + *len = requiredLen; + *status = U_BUFFER_OVERFLOW_ERROR; + return NULL; + } + + uprv_strcpy(target, inputDir); + } + + uprv_strcat(target, fileName); + return target; +} +/* + * Unicode TR 13 says any of the below chars is + * a new line char in a readline function in addition + * to CR+LF combination which needs to be + * handled seperately + */ +static UBool ucbuf_isCharNewLine(UChar c){ + switch(c){ + case 0x000A: /* LF */ + case 0x000D: /* CR */ + case 0x000C: /* FF */ + case 0x0085: /* NEL */ + case 0x2028: /* LS */ + case 0x2029: /* PS */ + return TRUE; + default: + return FALSE; + } +} + +U_CAPI const UChar* U_EXPORT2 +ucbuf_readline(UCHARBUF* buf,int32_t* len,UErrorCode* err){ + UChar* temp = buf->currentPos; + UChar* savePos =NULL; + UChar c=0x0000; + if(buf->isBuffered){ + /* The input is buffered we have to do more + * for returning a pointer U_TRUNCATED_CHAR_FOUND + */ + for(;;){ + c = *temp++; + if(buf->remaining==0){ + return NULL; /* end of file is reached return NULL */ + } + if(temp>=buf->bufLimit && buf->currentPos == buf->buffer){ + *err= U_TRUNCATED_CHAR_FOUND; + return NULL; + }else{ + ucbuf_fillucbuf(buf,err); + if(U_FAILURE(*err)){ + return NULL; + } + } + /* + * Accoding to TR 13 readLine functions must interpret + * CR, CR+LF, LF, NEL, PS, LS or FF as line seperators + */ + /* Windows CR LF */ + if(c ==0x0d && temp+1<=buf->bufLimit && *(temp+1) == 0x0a ){ + *len = (int32_t)(temp++ - buf->currentPos); + savePos = buf->currentPos; + buf->currentPos = temp; + return savePos; + } + /* else */ + + if (temp>=buf->bufLimit|| ucbuf_isCharNewLine(c)){ /* Unipad inserts 2028 line separators! */ + *len = (int32_t)(temp - buf->currentPos); + savePos = buf->currentPos; + buf->currentPos = temp; + return savePos; + } + } + }else{ + /* we know that all input is read into the internal + * buffer so we can safely return pointers + */ + for(;;){ + c = *temp++; + + if(buf->currentPos==buf->bufLimit){ + return NULL; /* end of file is reached return NULL */ + } + /* Windows CR LF */ + if(c ==0x0d && temp+1<=buf->bufLimit && *(temp+1) == 0x0a ){ + *len = (int32_t)(temp++ - buf->currentPos); + savePos = buf->currentPos; + buf->currentPos = temp; + return savePos; + } + /* else */ + if (temp>=buf->bufLimit|| ucbuf_isCharNewLine(c)) { /* Unipad inserts 2028 line separators! */ + *len = (int32_t)(temp - buf->currentPos); + savePos = buf->currentPos; + buf->currentPos = temp; + return savePos; + } + } + } + /* not reached */ + /* A compiler warning will appear if all paths don't contain a return statement. */ +/* return NULL;*/ +} +#endif diff --git a/Build/source/libs/icu/icu-xetex/tools/toolutil/ucbuf.h b/Build/source/libs/icu/icu-xetex/tools/toolutil/ucbuf.h new file mode 100644 index 00000000000..abb36c851ee --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/tools/toolutil/ucbuf.h @@ -0,0 +1,197 @@ +/* +******************************************************************************* +* +* Copyright (C) 1998-2005, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* +* File ucbuf.c +* +* Modification History: +* +* Date Name Description +* 05/10/01 Ram Creation. +* +* This API reads in files and returns UChars +******************************************************************************* +*/ + +#include "unicode/ucnv.h" +#include "filestrm.h" + +#if !UCONFIG_NO_CONVERSION + +#ifndef UCBUF_H +#define UCBUF_H 1 + +typedef struct UCHARBUF UCHARBUF; +/** + * End of file value + */ +#define U_EOF 0xFFFFFFFF +/** + * Error value if a sequence cannot be unescaped + */ +#define U_ERR 0xFFFFFFFE + +typedef struct ULine ULine; + +struct ULine { + UChar *name; + int32_t len; +}; + +/** + * Opens the UCHARBUF with the given file stream and code page for conversion + * @param fileName Name of the file to open. + * @param codepage The encoding of the file stream to convert to Unicode. + * If *codepoge is NULL on input the API will try to autodetect + * popular Unicode encodings + * @param showWarning Flag to print out warnings to STDOUT + * @param buffered If TRUE performs a buffered read of the input file. If FALSE reads + * the whole file into memory and converts it. + * @param err is a pointer to a valid <code>UErrorCode</code> value. If this value + * indicates a failure on entry, the function will immediately return. + * On exit the value will indicate the success of the operation. + * @return pointer to the newly opened UCHARBUF + */ +U_CAPI UCHARBUF* U_EXPORT2 +ucbuf_open(const char* fileName,const char** codepage,UBool showWarning, UBool buffered, UErrorCode* err); + +/** + * Gets a UTF-16 code unit at the current position from the converted buffer + * and increments the current position + * @param buf Pointer to UCHARBUF structure + * @param err is a pointer to a valid <code>UErrorCode</code> value. If this value + * indicates a failure on entry, the function will immediately return. + * On exit the value will indicate the success of the operation. + */ +U_CAPI int32_t U_EXPORT2 +ucbuf_getc(UCHARBUF* buf,UErrorCode* err); + +/** + * Gets a UTF-32 code point at the current position from the converted buffer + * and increments the current position + * @param buf Pointer to UCHARBUF structure + * @param err is a pointer to a valid <code>UErrorCode</code> value. If this value + * indicates a failure on entry, the function will immediately return. + * On exit the value will indicate the success of the operation. + */ +U_CAPI int32_t U_EXPORT2 +ucbuf_getc32(UCHARBUF* buf,UErrorCode* err); + +/** + * Gets a UTF-16 code unit at the current position from the converted buffer after + * unescaping and increments the current position. If the escape sequence is for UTF-32 + * code point (\\Uxxxxxxxx) then a UTF-32 codepoint is returned + * @param buf Pointer to UCHARBUF structure + * @param err is a pointer to a valid <code>UErrorCode</code> value. If this value + * indicates a failure on entry, the function will immediately return. + * On exit the value will indicate the success of the operation. + */ +U_CAPI int32_t U_EXPORT2 +ucbuf_getcx32(UCHARBUF* buf,UErrorCode* err); + +/** + * Gets a pointer to the current position in the internal buffer and length of the line. + * It imperative to make a copy of the returned buffere before performing operations on it. + * @param buf Pointer to UCHARBUF structure + * @param len Output param to receive the len of the buffer returned till end of the line + * @param err is a pointer to a valid <code>UErrorCode</code> value. If this value + * indicates a failure on entry, the function will immediately return. + * On exit the value will indicate the success of the operation. + * Error: U_TRUNCATED_CHAR_FOUND + * @return Pointer to the internal buffer, NULL if EOF + */ +U_CAPI const UChar* U_EXPORT2 +ucbuf_readline(UCHARBUF* buf,int32_t* len, UErrorCode* err); + + +/** + * Resets the buffers and the underlying file stream. + * @param buf Pointer to UCHARBUF structure + * @param err is a pointer to a valid <code>UErrorCode</code> value. If this value + * indicates a failure on entry, the function will immediately return. + * On exit the value will indicate the success of the operation. + */ +U_CAPI void U_EXPORT2 +ucbuf_rewind(UCHARBUF* buf,UErrorCode* err); + +/** + * Returns a pointer to the internal converted buffer + * @param buf Pointer to UCHARBUF structure + * @param len Pointer to int32_t to receive the lenth of buffer + * @param err is a pointer to a valid <code>UErrorCode</code> value. If this value + * indicates a failure on entry, the function will immediately return. + * On exit the value will indicate the success of the operation. + * @return Pointer to internal UChar buffer + */ +U_CAPI const UChar* U_EXPORT2 +ucbuf_getBuffer(UCHARBUF* buf,int32_t* len,UErrorCode* err); + +/** + * Closes the UCHARBUF structure members and cleans up the malloc'ed memory + * @param buf Pointer to UCHARBUF structure + */ +U_CAPI void U_EXPORT2 +ucbuf_close(UCHARBUF* buf); + +/** + * Rewinds the buffer by one codepoint + */ +U_CAPI void U_EXPORT2 +ucbuf_ungetc(int32_t ungetChar,UCHARBUF* buf); + + +/** + * Autodetects the encoding of the file stream. Only Unicode charsets are autodectected. + * Some Unicode charsets are stateful and need byte identifiers to be converted also to bring + * the converter to correct state for converting the rest of the stream. So the UConverter parameter + * is necessary. + * If the charset was autodetected, the caller must close both the input FileStream + * and the converter. + * + * @param fileName The file name to be opened and encoding autodected + * @param conv Output param to receive the opened converter if autodetected; NULL otherwise. + * @param cp Output param to receive the detected encoding + * @param err is a pointer to a valid <code>UErrorCode</code> value. If this value + * indicates a failure on entry, the function will immediately return. + * On exit the value will indicate the success of the operation. + * @return The input FileStream if its charset was autodetected; NULL otherwise. + */ +U_CAPI FileStream * U_EXPORT2 +ucbuf_autodetect(const char* fileName, const char** cp,UConverter** conv, +int32_t* signatureLength, UErrorCode* status); + +/** + * Autodetects the encoding of the file stream. Only Unicode charsets are autodectected. + * Some Unicode charsets are stateful and need byte identifiers to be converted also to bring + * the converter to correct state for converting the rest of the stream. So the UConverter parameter + * is necessary. + * If the charset was autodetected, the caller must close the converter. + * + * @param fileStream The file stream whose encoding is to be detected + * @param conv Output param to receive the opened converter if autodetected; NULL otherwise. + * @param cp Output param to receive the detected encoding + * @param err is a pointer to a valid <code>UErrorCode</code> value. If this value + * indicates a failure on entry, the function will immediately return. + * On exit the value will indicate the success of the operation. + * @return Boolean whether the Unicode charset was autodetected. + */ + +U_CAPI UBool U_EXPORT2 +ucbuf_autodetect_fs(FileStream* in, const char** cp, UConverter** conv, int32_t* signatureLength, UErrorCode* status); + +/** + * Returns the approximate size in UChars required for converting the file to UChars + */ +U_CAPI int32_t U_EXPORT2 +ucbuf_size(UCHARBUF* buf); + +U_CAPI const char* U_EXPORT2 +ucbuf_resolveFileName(const char* inputDir, const char* fileName, char* target, int32_t* len, UErrorCode* status); + +#endif +#endif + diff --git a/Build/source/libs/icu/icu-xetex/tools/toolutil/ucm.c b/Build/source/libs/icu/icu-xetex/tools/toolutil/ucm.c new file mode 100644 index 00000000000..bb7c3bb0edd --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/tools/toolutil/ucm.c @@ -0,0 +1,1188 @@ +/* +******************************************************************************* +* +* Copyright (C) 2003-2007, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: ucm.c +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2003jun20 +* created by: Markus W. Scherer +* +* This file reads a .ucm file, stores its mappings and sorts them. +* It implements handling of Unicode conversion mappings from .ucm files +* for makeconv, canonucm, rptp2ucm, etc. +* +* Unicode code point sequences with a length of more than 1, +* as well as byte sequences with more than 4 bytes or more than one complete +* character sequence are handled to support m:n mappings. +*/ + +#include "unicode/utypes.h" +#include "unicode/ustring.h" +#include "cstring.h" +#include "cmemory.h" +#include "filestrm.h" +#include "uarrsort.h" +#include "ucnvmbcs.h" +#include "ucnv_bld.h" +#include "ucnv_ext.h" +#include "uparse.h" +#include "ucm.h" +#include <stdio.h> + +#if !UCONFIG_NO_CONVERSION + +/* -------------------------------------------------------------------------- */ + +static void +printMapping(UCMapping *m, UChar32 *codePoints, uint8_t *bytes, FILE *f) { + int32_t j; + + for(j=0; j<m->uLen; ++j) { + fprintf(f, "<U%04lX>", (long)codePoints[j]); + } + + fputc(' ', f); + + for(j=0; j<m->bLen; ++j) { + fprintf(f, "\\x%02X", bytes[j]); + } + + if(m->f>=0) { + fprintf(f, " |%u\n", m->f); + } else { + fputs("\n", f); + } +} + +U_CAPI void U_EXPORT2 +ucm_printMapping(UCMTable *table, UCMapping *m, FILE *f) { + printMapping(m, UCM_GET_CODE_POINTS(table, m), UCM_GET_BYTES(table, m), f); +} + +U_CAPI void U_EXPORT2 +ucm_printTable(UCMTable *table, FILE *f, UBool byUnicode) { + UCMapping *m; + int32_t i, length; + + m=table->mappings; + length=table->mappingsLength; + if(byUnicode) { + for(i=0; i<length; ++m, ++i) { + ucm_printMapping(table, m, f); + } + } else { + const int32_t *map=table->reverseMap; + for(i=0; i<length; ++i) { + ucm_printMapping(table, m+map[i], f); + } + } +} + +/* mapping comparisons ------------------------------------------------------ */ + +static int32_t +compareUnicode(UCMTable *lTable, const UCMapping *l, + UCMTable *rTable, const UCMapping *r) { + const UChar32 *lu, *ru; + int32_t result, i, length; + + if(l->uLen==1 && r->uLen==1) { + /* compare two single code points */ + return l->u-r->u; + } + + /* get pointers to the code point sequences */ + lu=UCM_GET_CODE_POINTS(lTable, l); + ru=UCM_GET_CODE_POINTS(rTable, r); + + /* get the minimum length */ + if(l->uLen<=r->uLen) { + length=l->uLen; + } else { + length=r->uLen; + } + + /* compare the code points */ + for(i=0; i<length; ++i) { + result=lu[i]-ru[i]; + if(result!=0) { + return result; + } + } + + /* compare the lengths */ + return l->uLen-r->uLen; +} + +static int32_t +compareBytes(UCMTable *lTable, const UCMapping *l, + UCMTable *rTable, const UCMapping *r, + UBool lexical) { + const uint8_t *lb, *rb; + int32_t result, i, length; + + /* + * A lexical comparison is used for sorting in the builder, to allow + * an efficient search for a byte sequence that could be a prefix + * of a previously entered byte sequence. + * + * Comparing by lengths first is for compatibility with old .ucm tools + * like canonucm and rptp2ucm. + */ + if(lexical) { + /* get the minimum length and continue */ + if(l->bLen<=r->bLen) { + length=l->bLen; + } else { + length=r->bLen; + } + } else { + /* compare lengths first */ + result=l->bLen-r->bLen; + if(result!=0) { + return result; + } else { + length=l->bLen; + } + } + + /* get pointers to the byte sequences */ + lb=UCM_GET_BYTES(lTable, l); + rb=UCM_GET_BYTES(rTable, r); + + /* compare the bytes */ + for(i=0; i<length; ++i) { + result=lb[i]-rb[i]; + if(result!=0) { + return result; + } + } + + /* compare the lengths */ + return l->bLen-r->bLen; +} + +/* compare UCMappings for sorting */ +static int32_t +compareMappings(UCMTable *lTable, const UCMapping *l, + UCMTable *rTable, const UCMapping *r, + UBool uFirst) { + int32_t result; + + /* choose which side to compare first */ + if(uFirst) { + /* Unicode then bytes */ + result=compareUnicode(lTable, l, rTable, r); + if(result==0) { + result=compareBytes(lTable, l, rTable, r, FALSE); /* not lexically, like canonucm */ + } + } else { + /* bytes then Unicode */ + result=compareBytes(lTable, l, rTable, r, TRUE); /* lexically, for builder */ + if(result==0) { + result=compareUnicode(lTable, l, rTable, r); + } + } + + if(result!=0) { + return result; + } + + /* compare the flags */ + return l->f-r->f; +} + +/* sorting by Unicode first sorts mappings directly */ +static int32_t +compareMappingsUnicodeFirst(const void *context, const void *left, const void *right) { + return compareMappings( + (UCMTable *)context, (const UCMapping *)left, + (UCMTable *)context, (const UCMapping *)right, TRUE); +} + +/* sorting by bytes first sorts the reverseMap; use indirection to mappings */ +static int32_t +compareMappingsBytesFirst(const void *context, const void *left, const void *right) { + UCMTable *table=(UCMTable *)context; + int32_t l=*(const int32_t *)left, r=*(const int32_t *)right; + return compareMappings( + table, table->mappings+l, + table, table->mappings+r, FALSE); +} + +U_CAPI void U_EXPORT2 +ucm_sortTable(UCMTable *t) { + UErrorCode errorCode; + int32_t i; + + if(t->isSorted) { + return; + } + + errorCode=U_ZERO_ERROR; + + /* 1. sort by Unicode first */ + uprv_sortArray(t->mappings, t->mappingsLength, sizeof(UCMapping), + compareMappingsUnicodeFirst, t, + FALSE, &errorCode); + + /* build the reverseMap */ + if(t->reverseMap==NULL) { + /* + * allocate mappingsCapacity instead of mappingsLength so that + * if mappings are added, the reverseMap need not be + * reallocated each time + * (see ucm_moveMappings() and ucm_addMapping()) + */ + t->reverseMap=(int32_t *)uprv_malloc(t->mappingsCapacity*sizeof(int32_t)); + if(t->reverseMap==NULL) { + fprintf(stderr, "ucm error: unable to allocate reverseMap\n"); + exit(U_MEMORY_ALLOCATION_ERROR); + } + } + for(i=0; i<t->mappingsLength; ++i) { + t->reverseMap[i]=i; + } + + /* 2. sort reverseMap by mappings bytes first */ + uprv_sortArray(t->reverseMap, t->mappingsLength, sizeof(int32_t), + compareMappingsBytesFirst, t, + FALSE, &errorCode); + + if(U_FAILURE(errorCode)) { + fprintf(stderr, "ucm error: sortTable()/uprv_sortArray() fails - %s\n", + u_errorName(errorCode)); + exit(errorCode); + } + + t->isSorted=TRUE; +} + +/* + * remove mappings with their move flag set from the base table + * and move some of them (with UCM_MOVE_TO_EXT) to the extension table + */ +U_CAPI void U_EXPORT2 +ucm_moveMappings(UCMTable *base, UCMTable *ext) { + UCMapping *mb, *mbLimit; + int8_t flag; + + mb=base->mappings; + mbLimit=mb+base->mappingsLength; + + while(mb<mbLimit) { + flag=mb->moveFlag; + if(flag!=0) { + /* reset the move flag */ + mb->moveFlag=0; + + if(ext!=NULL && (flag&UCM_MOVE_TO_EXT)) { + /* add the mapping to the extension table */ + ucm_addMapping(ext, mb, UCM_GET_CODE_POINTS(base, mb), UCM_GET_BYTES(base, mb)); + } + + /* remove this mapping: move the last base mapping down and overwrite the current one */ + if(mb<(mbLimit-1)) { + uprv_memcpy(mb, mbLimit-1, sizeof(UCMapping)); + } + --mbLimit; + --base->mappingsLength; + base->isSorted=FALSE; + } else { + ++mb; + } + } +} + +enum { + NEEDS_MOVE=1, + HAS_ERRORS=2 +}; + +static uint8_t +checkBaseExtUnicode(UCMStates *baseStates, UCMTable *base, UCMTable *ext, + UBool moveToExt, UBool intersectBase) { + UCMapping *mb, *me, *mbLimit, *meLimit; + int32_t cmp; + uint8_t result; + + mb=base->mappings; + mbLimit=mb+base->mappingsLength; + + me=ext->mappings; + meLimit=me+ext->mappingsLength; + + result=0; + + for(;;) { + /* skip irrelevant mappings on both sides */ + for(;;) { + if(mb==mbLimit) { + return result; + } + + if(0<=mb->f && mb->f<=2) { + break; + } + + ++mb; + } + + for(;;) { + if(me==meLimit) { + return result; + } + + if(0<=me->f && me->f<=2) { + break; + } + + ++me; + } + + /* compare the base and extension mappings */ + cmp=compareUnicode(base, mb, ext, me); + if(cmp<0) { + if(intersectBase && (intersectBase!=2 || mb->bLen>1)) { + /* + * mapping in base but not in ext, move it + * + * if ext is DBCS, move DBCS mappings here + * and check SBCS ones for Unicode prefix below + */ + mb->moveFlag|=UCM_MOVE_TO_EXT; + result|=NEEDS_MOVE; + + /* does mb map from an input sequence that is a prefix of me's? */ + } else if( mb->uLen<me->uLen && + 0==uprv_memcmp(UCM_GET_CODE_POINTS(base, mb), UCM_GET_CODE_POINTS(ext, me), 4*mb->uLen) + ) { + if(moveToExt) { + /* mark this mapping to be moved to the extension table */ + mb->moveFlag|=UCM_MOVE_TO_EXT; + result|=NEEDS_MOVE; + } else { + fprintf(stderr, + "ucm error: the base table contains a mapping whose input sequence\n" + " is a prefix of the input sequence of an extension mapping\n"); + ucm_printMapping(base, mb, stderr); + ucm_printMapping(ext, me, stderr); + result|=HAS_ERRORS; + } + } + + ++mb; + } else if(cmp==0) { + /* + * same output: remove the extension mapping, + * otherwise treat as an error + */ + if( mb->f==me->f && mb->bLen==me->bLen && + 0==uprv_memcmp(UCM_GET_BYTES(base, mb), UCM_GET_BYTES(ext, me), mb->bLen) + ) { + me->moveFlag|=UCM_REMOVE_MAPPING; + result|=NEEDS_MOVE; + } else if(intersectBase) { + /* mapping in base but not in ext, move it */ + mb->moveFlag|=UCM_MOVE_TO_EXT; + result|=NEEDS_MOVE; + } else { + fprintf(stderr, + "ucm error: the base table contains a mapping whose input sequence\n" + " is the same as the input sequence of an extension mapping\n" + " but it maps differently\n"); + ucm_printMapping(base, mb, stderr); + ucm_printMapping(ext, me, stderr); + result|=HAS_ERRORS; + } + + ++mb; + } else /* cmp>0 */ { + ++me; + } + } +} + +static uint8_t +checkBaseExtBytes(UCMStates *baseStates, UCMTable *base, UCMTable *ext, + UBool moveToExt, UBool intersectBase) { + UCMapping *mb, *me; + int32_t *baseMap, *extMap; + int32_t b, e, bLimit, eLimit, cmp; + uint8_t result; + UBool isSISO; + + baseMap=base->reverseMap; + extMap=ext->reverseMap; + + b=e=0; + bLimit=base->mappingsLength; + eLimit=ext->mappingsLength; + + result=0; + + isSISO=(UBool)(baseStates->outputType==MBCS_OUTPUT_2_SISO); + + for(;;) { + /* skip irrelevant mappings on both sides */ + for(;; ++b) { + if(b==bLimit) { + return result; + } + mb=base->mappings+baseMap[b]; + + if(intersectBase==2 && mb->bLen==1) { + /* + * comparing a base against a DBCS extension: + * leave SBCS base mappings alone + */ + continue; + } + + if(mb->f==0 || mb->f==3) { + break; + } + } + + for(;;) { + if(e==eLimit) { + return result; + } + me=ext->mappings+extMap[e]; + + if(me->f==0 || me->f==3) { + break; + } + + ++e; + } + + /* compare the base and extension mappings */ + cmp=compareBytes(base, mb, ext, me, TRUE); + if(cmp<0) { + if(intersectBase) { + /* mapping in base but not in ext, move it */ + mb->moveFlag|=UCM_MOVE_TO_EXT; + result|=NEEDS_MOVE; + + /* + * does mb map from an input sequence that is a prefix of me's? + * for SI/SO tables, a single byte is never a prefix because it + * occurs in a separate single-byte state + */ + } else if( mb->bLen<me->bLen && + (!isSISO || mb->bLen>1) && + 0==uprv_memcmp(UCM_GET_BYTES(base, mb), UCM_GET_BYTES(ext, me), mb->bLen) + ) { + if(moveToExt) { + /* mark this mapping to be moved to the extension table */ + mb->moveFlag|=UCM_MOVE_TO_EXT; + result|=NEEDS_MOVE; + } else { + fprintf(stderr, + "ucm error: the base table contains a mapping whose input sequence\n" + " is a prefix of the input sequence of an extension mapping\n"); + ucm_printMapping(base, mb, stderr); + ucm_printMapping(ext, me, stderr); + result|=HAS_ERRORS; + } + } + + ++b; + } else if(cmp==0) { + /* + * same output: remove the extension mapping, + * otherwise treat as an error + */ + if( mb->f==me->f && mb->uLen==me->uLen && + 0==uprv_memcmp(UCM_GET_CODE_POINTS(base, mb), UCM_GET_CODE_POINTS(ext, me), 4*mb->uLen) + ) { + me->moveFlag|=UCM_REMOVE_MAPPING; + result|=NEEDS_MOVE; + } else if(intersectBase) { + /* mapping in base but not in ext, move it */ + mb->moveFlag|=UCM_MOVE_TO_EXT; + result|=NEEDS_MOVE; + } else { + fprintf(stderr, + "ucm error: the base table contains a mapping whose input sequence\n" + " is the same as the input sequence of an extension mapping\n" + " but it maps differently\n"); + ucm_printMapping(base, mb, stderr); + ucm_printMapping(ext, me, stderr); + result|=HAS_ERRORS; + } + + ++b; + } else /* cmp>0 */ { + ++e; + } + } +} + +U_CAPI UBool U_EXPORT2 +ucm_checkValidity(UCMTable *table, UCMStates *baseStates) { + UCMapping *m, *mLimit; + int32_t count; + UBool isOK; + + m=table->mappings; + mLimit=m+table->mappingsLength; + isOK=TRUE; + + while(m<mLimit) { + count=ucm_countChars(baseStates, UCM_GET_BYTES(table, m), m->bLen); + if(count<1) { + ucm_printMapping(table, m, stderr); + isOK=FALSE; + } + ++m; + } + + return isOK; +} + +U_CAPI UBool U_EXPORT2 +ucm_checkBaseExt(UCMStates *baseStates, + UCMTable *base, UCMTable *ext, UCMTable *moveTarget, + UBool intersectBase) { + uint8_t result; + + /* if we have an extension table, we must always use precision flags */ + if(base->flagsType&UCM_FLAGS_IMPLICIT) { + fprintf(stderr, "ucm error: the base table contains mappings without precision flags\n"); + return FALSE; + } + if(ext->flagsType&UCM_FLAGS_IMPLICIT) { + fprintf(stderr, "ucm error: extension table contains mappings without precision flags\n"); + return FALSE; + } + + /* checking requires both tables to be sorted */ + ucm_sortTable(base); + ucm_sortTable(ext); + + /* check */ + result= + checkBaseExtUnicode(baseStates, base, ext, (UBool)(moveTarget!=NULL), intersectBase)| + checkBaseExtBytes(baseStates, base, ext, (UBool)(moveTarget!=NULL), intersectBase); + + if(result&HAS_ERRORS) { + return FALSE; + } + + if(result&NEEDS_MOVE) { + ucm_moveMappings(ext, NULL); + ucm_moveMappings(base, moveTarget); + ucm_sortTable(base); + ucm_sortTable(ext); + if(moveTarget!=NULL) { + ucm_sortTable(moveTarget); + } + } + + return TRUE; +} + +/* merge tables for rptp2ucm ------------------------------------------------ */ + +U_CAPI void U_EXPORT2 +ucm_mergeTables(UCMTable *fromUTable, UCMTable *toUTable, + const uint8_t *subchar, int32_t subcharLength, + uint8_t subchar1) { + UCMapping *fromUMapping, *toUMapping; + int32_t fromUIndex, toUIndex, fromUTop, toUTop, cmp; + + ucm_sortTable(fromUTable); + ucm_sortTable(toUTable); + + fromUMapping=fromUTable->mappings; + toUMapping=toUTable->mappings; + + fromUTop=fromUTable->mappingsLength; + toUTop=toUTable->mappingsLength; + + fromUIndex=toUIndex=0; + + while(fromUIndex<fromUTop && toUIndex<toUTop) { + cmp=compareMappings(fromUTable, fromUMapping, toUTable, toUMapping, TRUE); + if(cmp==0) { + /* equal: roundtrip, nothing to do (flags are initially 0) */ + ++fromUMapping; + ++toUMapping; + + ++fromUIndex; + ++toUIndex; + } else if(cmp<0) { + /* + * the fromU mapping does not have a toU counterpart: + * fallback Unicode->codepage + */ + if( (fromUMapping->bLen==subcharLength && + 0==uprv_memcmp(UCM_GET_BYTES(fromUTable, fromUMapping), subchar, subcharLength)) || + (subchar1!=0 && fromUMapping->bLen==1 && fromUMapping->b.bytes[0]==subchar1) + ) { + fromUMapping->f=2; /* SUB mapping */ + } else { + fromUMapping->f=1; /* normal fallback */ + } + + ++fromUMapping; + ++fromUIndex; + } else { + /* + * the toU mapping does not have a fromU counterpart: + * (reverse) fallback codepage->Unicode, copy it to the fromU table + */ + + /* ignore reverse fallbacks to Unicode SUB */ + if(!(toUMapping->uLen==1 && (toUMapping->u==0xfffd || toUMapping->u==0x1a))) { + toUMapping->f=3; /* reverse fallback */ + ucm_addMapping(fromUTable, toUMapping, UCM_GET_CODE_POINTS(toUTable, toUMapping), UCM_GET_BYTES(toUTable, toUMapping)); + + /* the table may have been reallocated */ + fromUMapping=fromUTable->mappings+fromUIndex; + } + + ++toUMapping; + ++toUIndex; + } + } + + /* either one or both tables are exhausted */ + while(fromUIndex<fromUTop) { + /* leftover fromU mappings are fallbacks */ + if( (fromUMapping->bLen==subcharLength && + 0==uprv_memcmp(UCM_GET_BYTES(fromUTable, fromUMapping), subchar, subcharLength)) || + (subchar1!=0 && fromUMapping->bLen==1 && fromUMapping->b.bytes[0]==subchar1) + ) { + fromUMapping->f=2; /* SUB mapping */ + } else { + fromUMapping->f=1; /* normal fallback */ + } + + ++fromUMapping; + ++fromUIndex; + } + + while(toUIndex<toUTop) { + /* leftover toU mappings are reverse fallbacks */ + + /* ignore reverse fallbacks to Unicode SUB */ + if(!(toUMapping->uLen==1 && (toUMapping->u==0xfffd || toUMapping->u==0x1a))) { + toUMapping->f=3; /* reverse fallback */ + ucm_addMapping(fromUTable, toUMapping, UCM_GET_CODE_POINTS(toUTable, toUMapping), UCM_GET_BYTES(toUTable, toUMapping)); + } + + ++toUMapping; + ++toUIndex; + } + + fromUTable->isSorted=FALSE; +} + +/* separate extension mappings out of base table for rptp2ucm --------------- */ + +U_CAPI UBool U_EXPORT2 +ucm_separateMappings(UCMFile *ucm, UBool isSISO) { + UCMTable *table; + UCMapping *m, *mLimit; + int32_t type; + UBool needsMove, isOK; + + table=ucm->base; + m=table->mappings; + mLimit=m+table->mappingsLength; + + needsMove=FALSE; + isOK=TRUE; + + for(; m<mLimit; ++m) { + if(isSISO && m->bLen==1 && (m->b.bytes[0]==0xe || m->b.bytes[0]==0xf)) { + fprintf(stderr, "warning: removing illegal mapping from an SI/SO-stateful table\n"); + ucm_printMapping(table, m, stderr); + m->moveFlag|=UCM_REMOVE_MAPPING; + needsMove=TRUE; + continue; + } + + type=ucm_mappingType( + &ucm->states, m, + UCM_GET_CODE_POINTS(table, m), UCM_GET_BYTES(table, m)); + if(type<0) { + /* illegal byte sequence */ + printMapping(m, UCM_GET_CODE_POINTS(table, m), UCM_GET_BYTES(table, m), stderr); + isOK=FALSE; + } else if(type>0) { + m->moveFlag|=UCM_MOVE_TO_EXT; + needsMove=TRUE; + } + } + + if(!isOK) { + return FALSE; + } + if(needsMove) { + ucm_moveMappings(ucm->base, ucm->ext); + return ucm_checkBaseExt(&ucm->states, ucm->base, ucm->ext, ucm->ext, FALSE); + } else { + ucm_sortTable(ucm->base); + return TRUE; + } +} + +/* ucm parser --------------------------------------------------------------- */ + +U_CAPI int8_t U_EXPORT2 +ucm_parseBytes(uint8_t bytes[UCNV_EXT_MAX_BYTES], const char *line, const char **ps) { + const char *s=*ps; + char *end; + uint8_t byte; + int8_t bLen; + + bLen=0; + for(;;) { + /* skip an optional plus sign */ + if(bLen>0 && *s=='+') { + ++s; + } + if(*s!='\\') { + break; + } + + if( s[1]!='x' || + (byte=(uint8_t)uprv_strtoul(s+2, &end, 16), end)!=s+4 + ) { + fprintf(stderr, "ucm error: byte must be formatted as \\xXX (2 hex digits) - \"%s\"\n", line); + return -1; + } + + if(bLen==UCNV_EXT_MAX_BYTES) { + fprintf(stderr, "ucm error: too many bytes on \"%s\"\n", line); + return -1; + } + bytes[bLen++]=byte; + s=end; + } + + *ps=s; + return bLen; +} + +/* parse a mapping line; must not be empty */ +U_CAPI UBool U_EXPORT2 +ucm_parseMappingLine(UCMapping *m, + UChar32 codePoints[UCNV_EXT_MAX_UCHARS], + uint8_t bytes[UCNV_EXT_MAX_BYTES], + const char *line) { + const char *s; + char *end; + UChar32 cp; + int32_t u16Length; + int8_t uLen, bLen, f; + + s=line; + uLen=bLen=0; + + /* parse code points */ + for(;;) { + /* skip an optional plus sign */ + if(uLen>0 && *s=='+') { + ++s; + } + if(*s!='<') { + break; + } + + if( s[1]!='U' || + (cp=(UChar32)uprv_strtoul(s+2, &end, 16), end)==s+2 || + *end!='>' + ) { + fprintf(stderr, "ucm error: Unicode code point must be formatted as <UXXXX> (1..6 hex digits) - \"%s\"\n", line); + return FALSE; + } + if((uint32_t)cp>0x10ffff || U_IS_SURROGATE(cp)) { + fprintf(stderr, "ucm error: Unicode code point must be 0..d7ff or e000..10ffff - \"%s\"\n", line); + return FALSE; + } + + if(uLen==UCNV_EXT_MAX_UCHARS) { + fprintf(stderr, "ucm error: too many code points on \"%s\"\n", line); + return FALSE; + } + codePoints[uLen++]=cp; + s=end+1; + } + + if(uLen==0) { + fprintf(stderr, "ucm error: no Unicode code points on \"%s\"\n", line); + return FALSE; + } else if(uLen==1) { + m->u=codePoints[0]; + } else { + UErrorCode errorCode=U_ZERO_ERROR; + u_strFromUTF32(NULL, 0, &u16Length, codePoints, uLen, &errorCode); + if( (U_FAILURE(errorCode) && errorCode!=U_BUFFER_OVERFLOW_ERROR) || + u16Length>UCNV_EXT_MAX_UCHARS + ) { + fprintf(stderr, "ucm error: too many UChars on \"%s\"\n", line); + return FALSE; + } + } + + s=u_skipWhitespace(s); + + /* parse bytes */ + bLen=ucm_parseBytes(bytes, line, &s); + + if(bLen<0) { + return FALSE; + } else if(bLen==0) { + fprintf(stderr, "ucm error: no bytes on \"%s\"\n", line); + return FALSE; + } else if(bLen<=4) { + uprv_memcpy(m->b.bytes, bytes, bLen); + } + + /* skip everything until the fallback indicator, even the start of a comment */ + for(;;) { + if(*s==0) { + f=-1; /* no fallback indicator */ + break; + } else if(*s=='|') { + f=(int8_t)(s[1]-'0'); + if((uint8_t)f>3) { + fprintf(stderr, "ucm error: fallback indicator must be |0..|3 - \"%s\"\n", line); + return FALSE; + } + break; + } + ++s; + } + + m->uLen=uLen; + m->bLen=bLen; + m->f=f; + return TRUE; +} + +/* general APIs ------------------------------------------------------------- */ + +U_CAPI UCMTable * U_EXPORT2 +ucm_openTable() { + UCMTable *table=(UCMTable *)uprv_malloc(sizeof(UCMTable)); + if(table==NULL) { + fprintf(stderr, "ucm error: unable to allocate a UCMTable\n"); + exit(U_MEMORY_ALLOCATION_ERROR); + } + + memset(table, 0, sizeof(UCMTable)); + return table; +} + +U_CAPI void U_EXPORT2 +ucm_closeTable(UCMTable *table) { + if(table!=NULL) { + uprv_free(table->mappings); + uprv_free(table->codePoints); + uprv_free(table->bytes); + uprv_free(table->reverseMap); + uprv_free(table); + } +} + +U_CAPI void U_EXPORT2 +ucm_resetTable(UCMTable *table) { + if(table!=NULL) { + table->mappingsLength=0; + table->flagsType=0; + table->unicodeMask=0; + table->bytesLength=table->codePointsLength=0; + table->isSorted=FALSE; + } +} + +U_CAPI void U_EXPORT2 +ucm_addMapping(UCMTable *table, + UCMapping *m, + UChar32 codePoints[UCNV_EXT_MAX_UCHARS], + uint8_t bytes[UCNV_EXT_MAX_BYTES]) { + UCMapping *tm; + UChar32 c; + int32_t index; + + if(table->mappingsLength>=table->mappingsCapacity) { + /* make the mappings array larger */ + if(table->mappingsCapacity==0) { + table->mappingsCapacity=1000; + } else { + table->mappingsCapacity*=10; + } + table->mappings=(UCMapping *)uprv_realloc(table->mappings, + table->mappingsCapacity*sizeof(UCMapping)); + if(table->mappings==NULL) { + fprintf(stderr, "ucm error: unable to allocate %d UCMappings\n", + (int)table->mappingsCapacity); + exit(U_MEMORY_ALLOCATION_ERROR); + } + + if(table->reverseMap!=NULL) { + /* the reverseMap must be reallocated in a new sort */ + uprv_free(table->reverseMap); + table->reverseMap=NULL; + } + } + + if(m->uLen>1 && table->codePointsCapacity==0) { + table->codePointsCapacity=10000; + table->codePoints=(UChar32 *)uprv_malloc(table->codePointsCapacity*4); + if(table->codePoints==NULL) { + fprintf(stderr, "ucm error: unable to allocate %d UChar32s\n", + (int)table->codePointsCapacity); + exit(U_MEMORY_ALLOCATION_ERROR); + } + } + + if(m->bLen>4 && table->bytesCapacity==0) { + table->bytesCapacity=10000; + table->bytes=(uint8_t *)uprv_malloc(table->bytesCapacity); + if(table->bytes==NULL) { + fprintf(stderr, "ucm error: unable to allocate %d bytes\n", + (int)table->bytesCapacity); + exit(U_MEMORY_ALLOCATION_ERROR); + } + } + + if(m->uLen>1) { + index=table->codePointsLength; + table->codePointsLength+=m->uLen; + if(table->codePointsLength>table->codePointsCapacity) { + fprintf(stderr, "ucm error: too many code points in multiple-code point mappings\n"); + exit(U_MEMORY_ALLOCATION_ERROR); + } + + uprv_memcpy(table->codePoints+index, codePoints, m->uLen*4); + m->u=index; + } + + if(m->bLen>4) { + index=table->bytesLength; + table->bytesLength+=m->bLen; + if(table->bytesLength>table->bytesCapacity) { + fprintf(stderr, "ucm error: too many bytes in mappings with >4 charset bytes\n"); + exit(U_MEMORY_ALLOCATION_ERROR); + } + + uprv_memcpy(table->bytes+index, bytes, m->bLen); + m->b.index=index; + } + + /* set unicodeMask */ + for(index=0; index<m->uLen; ++index) { + c=codePoints[index]; + if(c>=0x10000) { + table->unicodeMask|=UCNV_HAS_SUPPLEMENTARY; /* there are supplementary code points */ + } else if(U_IS_SURROGATE(c)) { + table->unicodeMask|=UCNV_HAS_SURROGATES; /* there are surrogate code points */ + } + } + + /* set flagsType */ + if(m->f<0) { + table->flagsType|=UCM_FLAGS_IMPLICIT; + } else { + table->flagsType|=UCM_FLAGS_EXPLICIT; + } + + tm=table->mappings+table->mappingsLength++; + uprv_memcpy(tm, m, sizeof(UCMapping)); + + table->isSorted=FALSE; +} + +U_CAPI UCMFile * U_EXPORT2 +ucm_open() { + UCMFile *ucm=(UCMFile *)uprv_malloc(sizeof(UCMFile)); + if(ucm==NULL) { + fprintf(stderr, "ucm error: unable to allocate a UCMFile\n"); + exit(U_MEMORY_ALLOCATION_ERROR); + } + + memset(ucm, 0, sizeof(UCMFile)); + + ucm->base=ucm_openTable(); + ucm->ext=ucm_openTable(); + + ucm->states.stateFlags[0]=MBCS_STATE_FLAG_DIRECT; + ucm->states.conversionType=UCNV_UNSUPPORTED_CONVERTER; + ucm->states.outputType=-1; + ucm->states.minCharLength=ucm->states.maxCharLength=1; + + return ucm; +} + +U_CAPI void U_EXPORT2 +ucm_close(UCMFile *ucm) { + if(ucm!=NULL) { + uprv_free(ucm->base); + uprv_free(ucm->ext); + uprv_free(ucm); + } +} + +U_CAPI int32_t U_EXPORT2 +ucm_mappingType(UCMStates *baseStates, + UCMapping *m, + UChar32 codePoints[UCNV_EXT_MAX_UCHARS], + uint8_t bytes[UCNV_EXT_MAX_BYTES]) { + /* check validity of the bytes and count the characters in them */ + int32_t count=ucm_countChars(baseStates, bytes, m->bLen); + if(count<1) { + /* illegal byte sequence */ + return -1; + } + + /* + * Suitable for an ICU conversion base table means: + * - a 1:1 mapping (1 Unicode code point : 1 byte sequence) + * - SBCS: any 1:1 mapping + * (the table stores additional bits to distinguish mapping types) + * - MBCS: not a |2 SUB mapping for <subchar1> + * - MBCS: not a |1 fallback to 0x00 + * - MBCS: not a multi-byte mapping with leading 0x00 bytes + * + * Further restrictions for fromUnicode tables + * are enforced in makeconv (MBCSOkForBaseFromUnicode()). + * + * All of the MBCS fromUnicode specific tests could be removed from here, + * but the ones above are for unusual mappings, and removing the tests + * from here would change canonucm output which seems gratuitous. + * (Markus Scherer 2006-nov-28) + * + * Exception: All implicit mappings (f<0) that need to be moved + * because of fromUnicode restrictions _must_ be moved here because + * makeconv uses a hack for moving mappings only for the fromUnicode table + * that only works with non-negative values of f. + */ + if( m->uLen==1 && count==1 && + (baseStates->maxCharLength==1 || + !((m->f==2 && m->bLen==1) || + (m->f==1 && bytes[0]==0) || + (m->f<=1 && m->bLen>1 && bytes[0]==0))) + ) { + return 0; /* suitable for a base table */ + } else { + return 1; /* needs to go into an extension table */ + } +} + +U_CAPI UBool U_EXPORT2 +ucm_addMappingAuto(UCMFile *ucm, UBool forBase, UCMStates *baseStates, + UCMapping *m, + UChar32 codePoints[UCNV_EXT_MAX_UCHARS], + uint8_t bytes[UCNV_EXT_MAX_BYTES]) { + int32_t type; + + if(m->f==2 && m->uLen>1) { + fprintf(stderr, "ucm error: illegal <subchar1> |2 mapping from multiple code points\n"); + printMapping(m, codePoints, bytes, stderr); + return FALSE; + } + + if(baseStates!=NULL) { + /* check validity of the bytes and count the characters in them */ + type=ucm_mappingType(baseStates, m, codePoints, bytes); + if(type<0) { + /* illegal byte sequence */ + printMapping(m, codePoints, bytes, stderr); + return FALSE; + } + } else { + /* not used - adding a mapping for an extension-only table before its base table is read */ + type=1; + } + + /* + * Add the mapping to the base table if this is requested and suitable. + * Otherwise, add it to the extension table. + */ + if(forBase && type==0) { + ucm_addMapping(ucm->base, m, codePoints, bytes); + } else { + ucm_addMapping(ucm->ext, m, codePoints, bytes); + } + + return TRUE; +} + +U_CAPI UBool U_EXPORT2 +ucm_addMappingFromLine(UCMFile *ucm, const char *line, UBool forBase, UCMStates *baseStates) { + UCMapping m={ 0 }; + UChar32 codePoints[UCNV_EXT_MAX_UCHARS]; + uint8_t bytes[UCNV_EXT_MAX_BYTES]; + + const char *s; + + /* ignore empty and comment lines */ + if(line[0]=='#' || *(s=u_skipWhitespace(line))==0 || *s=='\n' || *s=='\r') { + return TRUE; + } + + return + ucm_parseMappingLine(&m, codePoints, bytes, line) && + ucm_addMappingAuto(ucm, forBase, baseStates, &m, codePoints, bytes); +} + +U_CAPI void U_EXPORT2 +ucm_readTable(UCMFile *ucm, FileStream* convFile, + UBool forBase, UCMStates *baseStates, + UErrorCode *pErrorCode) { + char line[500]; + char *end; + UBool isOK; + + if(U_FAILURE(*pErrorCode)) { + return; + } + + isOK=TRUE; + + for(;;) { + /* read the next line */ + if(!T_FileStream_readLine(convFile, line, sizeof(line))) { + fprintf(stderr, "incomplete charmap section\n"); + isOK=FALSE; + break; + } + + /* remove CR LF */ + end=uprv_strchr(line, 0); + while(line<end && (*(end-1)=='\r' || *(end-1)=='\n')) { + --end; + } + *end=0; + + /* ignore empty and comment lines */ + if(line[0]==0 || line[0]=='#') { + continue; + } + + /* stop at the end of the mapping table */ + if(0==uprv_strcmp(line, "END CHARMAP")) { + break; + } + + isOK&=ucm_addMappingFromLine(ucm, line, forBase, baseStates); + } + + if(!isOK) { + *pErrorCode=U_INVALID_TABLE_FORMAT; + } +} +#endif diff --git a/Build/source/libs/icu/icu-xetex/tools/toolutil/ucm.h b/Build/source/libs/icu/icu-xetex/tools/toolutil/ucm.h new file mode 100644 index 00000000000..6575721175e --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/tools/toolutil/ucm.h @@ -0,0 +1,299 @@ +/* + ******************************************************************************* + * Copyright (C) 2003-2007, International Business Machines + * Corporation and others. All Rights Reserved. + ******************************************************************************* + * file name: ucm.h + * encoding: US-ASCII + * tab size: 8 (not used) + * indentation:4 + * + * created on: 2003jun20 + * created by: Markus W. Scherer + * + * Definitions for the .ucm file parser and handler module ucm.c. + */ + +#ifndef __UCM_H__ +#define __UCM_H__ + +#include "unicode/utypes.h" +#include "ucnvmbcs.h" +#include "ucnv_ext.h" +#include "filestrm.h" +#include <stdio.h> + +#if !UCONFIG_NO_CONVERSION + +U_CDECL_BEGIN + +/* constants for UCMapping.moveFlag */ +enum { + UCM_MOVE_TO_EXT=1, + UCM_REMOVE_MAPPING=2 +}; + +/* + * Per-mapping data structure + * + * u if uLen==1: Unicode code point + * else index to uLen code points + * b if bLen<=4: up to 4 bytes + * else index to bLen bytes + * uLen number of code points + * bLen number of words containing left-justified bytes + * bIsMultipleChars indicates that the bytes contain more than one sequence + * according to the state table + * f flag for roundtrip (0), fallback (1), sub mapping (2), reverse fallback (3) + * same values as in the source file after | + */ +typedef struct UCMapping { + UChar32 u; + union { + uint32_t index; + uint8_t bytes[4]; + } b; + int8_t uLen, bLen, f, moveFlag; +} UCMapping; + +/* constants for UCMTable.flagsType */ +enum { + UCM_FLAGS_INITIAL, /* no mappings parsed yet */ + UCM_FLAGS_EXPLICIT, /* .ucm file has mappings with | fallback indicators */ + UCM_FLAGS_IMPLICIT, /* .ucm file has mappings without | fallback indicators, later wins */ + UCM_FLAGS_MIXED /* both implicit and explicit */ +}; + +typedef struct UCMTable { + UCMapping *mappings; + int32_t mappingsCapacity, mappingsLength; + + UChar32 *codePoints; + int32_t codePointsCapacity, codePointsLength; + + uint8_t *bytes; + int32_t bytesCapacity, bytesLength; + + /* index map for mapping by bytes first */ + int32_t *reverseMap; + + uint8_t unicodeMask; + int8_t flagsType; /* UCM_FLAGS_INITIAL etc. */ + UBool isSorted; +} UCMTable; + +enum { + MBCS_STATE_FLAG_DIRECT=1, + MBCS_STATE_FLAG_SURROGATES, + + MBCS_STATE_FLAG_READY=16 +}; + +typedef struct UCMStates { + int32_t stateTable[MBCS_MAX_STATE_COUNT][256]; + uint32_t stateFlags[MBCS_MAX_STATE_COUNT], + stateOffsetSum[MBCS_MAX_STATE_COUNT]; + + int32_t countStates, minCharLength, maxCharLength, countToUCodeUnits; + int8_t conversionType, outputType; +} UCMStates; + +typedef struct UCMFile { + UCMTable *base, *ext; + UCMStates states; + + char baseName[UCNV_MAX_CONVERTER_NAME_LENGTH]; +} UCMFile; + +/* simple accesses ---------------------------------------------------------- */ + +#define UCM_GET_CODE_POINTS(t, m) \ + (((m)->uLen==1) ? &(m)->u : (t)->codePoints+(m)->u) + +#define UCM_GET_BYTES(t, m) \ + (((m)->bLen<=4) ? (m)->b.bytes : (t)->bytes+(m)->b.index) + +/* APIs --------------------------------------------------------------------- */ + +U_CAPI UCMFile * U_EXPORT2 +ucm_open(void); + +U_CAPI void U_EXPORT2 +ucm_close(UCMFile *ucm); + +U_CAPI UBool U_EXPORT2 +ucm_parseHeaderLine(UCMFile *ucm, + char *line, char **pKey, char **pValue); + +/* @return -1 illegal bytes 0 suitable for base table 1 needs to go into extension table */ +U_CAPI int32_t U_EXPORT2 +ucm_mappingType(UCMStates *baseStates, + UCMapping *m, + UChar32 codePoints[UCNV_EXT_MAX_UCHARS], + uint8_t bytes[UCNV_EXT_MAX_BYTES]); + +/* add a mapping to the base or extension table as appropriate */ +U_CAPI UBool U_EXPORT2 +ucm_addMappingAuto(UCMFile *ucm, UBool forBase, UCMStates *baseStates, + UCMapping *m, + UChar32 codePoints[UCNV_EXT_MAX_UCHARS], + uint8_t bytes[UCNV_EXT_MAX_BYTES]); + +U_CAPI UBool U_EXPORT2 +ucm_addMappingFromLine(UCMFile *ucm, const char *line, UBool forBase, UCMStates *baseStates); + + +U_CAPI UCMTable * U_EXPORT2 +ucm_openTable(void); + +U_CAPI void U_EXPORT2 +ucm_closeTable(UCMTable *table); + +U_CAPI void U_EXPORT2 +ucm_resetTable(UCMTable *table); + +U_CAPI void U_EXPORT2 +ucm_sortTable(UCMTable *t); + +/* + * Remove mappings with their move flag set from the base table + * and move some of them (with UCM_MOVE_TO_EXT) to the extension table. + */ +U_CAPI void U_EXPORT2 +ucm_moveMappings(UCMTable *base, UCMTable *ext); + +/** + * Read a table from a .ucm file, from after the CHARMAP line to + * including the END CHARMAP line. + */ +U_CAPI void U_EXPORT2 +ucm_readTable(UCMFile *ucm, FileStream* convFile, + UBool forBase, UCMStates *baseStates, + UErrorCode *pErrorCode); + +/** + * Check the validity of mappings against a base table's states; + * necessary for extension-only tables that were read before their base tables. + */ +U_CAPI UBool U_EXPORT2 +ucm_checkValidity(UCMTable *ext, UCMStates *baseStates); + +/** + * Check a base table against an extension table. + * Set the moveTarget!=NULL if it is possible to move mappings from the base. + * This is the case where base and extension tables are parsed from a single file + * (moveTarget==ext) + * or when delta file mappings are subtracted from a base table. + * + * When a base table cannot be modified because a delta file is parsed in makeconv, + * then set moveTarget=NULL. + * + * if(intersectBase) then mappings that exist in the base table but not in + * the extension table are moved to moveTarget instead of showing an error. + * + * Special mode: + * If intersectBase==2 for a DBCS extension table, then SBCS mappings are + * not moved out of the base unless their Unicode input requires it. + * This helps ucmkbase generate base tables for DBCS-only extension .cnv files. + * + * For both tables in the same file, the extension table is automatically + * built. + * For separate files, the extension file can use a complete mapping table (.ucm file), + * so that common mappings need not be stripped out manually. + * + * + * Sort both tables, and then for each mapping direction: + * + * If intersectBase is TRUE and the base table contains a mapping + * that does not exist in the extension table, then this mapping is moved + * to moveTarget. + * + * - otherwise - + * + * If the base table contains a mapping for which the input sequence is + * the same as the extension input, then + * - if the output is the same: remove the extension mapping + * - else: error + * + * If the base table contains a mapping for which the input sequence is + * a prefix of the extension input, then + * - if moveTarget!=NULL: move the base mapping to the moveTarget table + * - else: error + * + * @return FALSE in case of an irreparable error + */ +U_CAPI UBool U_EXPORT2 +ucm_checkBaseExt(UCMStates *baseStates, UCMTable *base, UCMTable *ext, + UCMTable *moveTarget, UBool intersectBase); + +U_CAPI void U_EXPORT2 +ucm_printTable(UCMTable *table, FILE *f, UBool byUnicode); + +U_CAPI void U_EXPORT2 +ucm_printMapping(UCMTable *table, UCMapping *m, FILE *f); + + +U_CAPI void U_EXPORT2 +ucm_addState(UCMStates *states, const char *s); + +U_CAPI void U_EXPORT2 +ucm_processStates(UCMStates *states); + +U_CAPI int32_t U_EXPORT2 +ucm_countChars(UCMStates *states, + const uint8_t *bytes, int32_t length); + + +U_CAPI int8_t U_EXPORT2 +ucm_parseBytes(uint8_t bytes[UCNV_EXT_MAX_BYTES], const char *line, const char **ps); + +U_CAPI UBool U_EXPORT2 +ucm_parseMappingLine(UCMapping *m, + UChar32 codePoints[UCNV_EXT_MAX_UCHARS], + uint8_t bytes[UCNV_EXT_MAX_BYTES], + const char *line); + +U_CAPI void U_EXPORT2 +ucm_addMapping(UCMTable *table, + UCMapping *m, + UChar32 codePoints[UCNV_EXT_MAX_UCHARS], + uint8_t bytes[UCNV_EXT_MAX_BYTES]); + +/* very makeconv-specific functions ----------------------------------------- */ + +/* finalize and optimize states after the toUnicode mappings are processed */ +U_CAPI void U_EXPORT2 +ucm_optimizeStates(UCMStates *states, + uint16_t **pUnicodeCodeUnits, + _MBCSToUFallback *toUFallbacks, int32_t countToUFallbacks, + UBool verbose); + +/* moved here because it is used inside ucmstate.c */ +U_CAPI int32_t U_EXPORT2 +ucm_findFallback(_MBCSToUFallback *toUFallbacks, int32_t countToUFallbacks, + uint32_t offset); + +/* very rptp2ucm-specific functions ----------------------------------------- */ + +/* + * Input: Separate tables with mappings from/to Unicode, + * subchar and subchar1 (0 if none). + * All mappings must have flag 0. + * + * Output: fromUTable will contain the union of mappings with the correct + * precision flags, and be sorted. + */ +U_CAPI void U_EXPORT2 +ucm_mergeTables(UCMTable *fromUTable, UCMTable *toUTable, + const uint8_t *subchar, int32_t subcharLength, + uint8_t subchar1); + +U_CAPI UBool U_EXPORT2 +ucm_separateMappings(UCMFile *ucm, UBool isSISO); + +U_CDECL_END + +#endif + +#endif + diff --git a/Build/source/libs/icu/icu-xetex/tools/toolutil/ucmstate.c b/Build/source/libs/icu/icu-xetex/tools/toolutil/ucmstate.c new file mode 100644 index 00000000000..393d18b3781 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/tools/toolutil/ucmstate.c @@ -0,0 +1,1046 @@ +/* +******************************************************************************* +* +* Copyright (C) 2003-2005, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: ucmstate.c +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2003oct09 +* created by: Markus W. Scherer +* +* This file handles ICU .ucm file state information as part of the ucm module. +* Most of this code used to be in makeconv.c. +*/ + +#include "unicode/utypes.h" +#include "cstring.h" +#include "cmemory.h" +#include "uarrsort.h" +#include "ucnvmbcs.h" +#include "ucnv_ext.h" +#include "uparse.h" +#include "ucm.h" +#include <stdio.h> + +#if !UCONFIG_NO_CONVERSION + +/* MBCS state handling ------------------------------------------------------ */ + +/* + * state table row grammar (ebnf-style): + * (whitespace is allowed between all tokens) + * + * row=[[firstentry ','] entry (',' entry)*] + * firstentry="initial" | "surrogates" + * (initial state (default for state 0), output is all surrogate pairs) + * entry=range [':' nextstate] ['.' action] + * range=number ['-' number] + * nextstate=number + * (0..7f) + * action='u' | 's' | 'p' | 'i' + * (unassigned, state change only, surrogate pair, illegal) + * number=(1- or 2-digit hexadecimal number) + */ +static const char * +parseState(const char *s, int32_t state[256], uint32_t *pFlags) { + const char *t; + uint32_t start, end, i; + int32_t entry; + + /* initialize the state: all illegal with U+ffff */ + for(i=0; i<256; ++i) { + state[i]=MBCS_ENTRY_FINAL(0, MBCS_STATE_ILLEGAL, 0xffff); + } + + /* skip leading white space */ + s=u_skipWhitespace(s); + + /* is there an "initial" or "surrogates" directive? */ + if(uprv_strncmp("initial", s, 7)==0) { + *pFlags=MBCS_STATE_FLAG_DIRECT; + s=u_skipWhitespace(s+7); + if(*s++!=',') { + return s-1; + } + } else if(*pFlags==0 && uprv_strncmp("surrogates", s, 10)==0) { + *pFlags=MBCS_STATE_FLAG_SURROGATES; + s=u_skipWhitespace(s+10); + if(*s++!=',') { + return s-1; + } + } else if(*s==0) { + /* empty state row: all-illegal */ + return NULL; + } + + for(;;) { + /* read an entry, the start of the range first */ + s=u_skipWhitespace(s); + start=uprv_strtoul(s, (char **)&t, 16); + if(s==t || 0xff<start) { + return s; + } + s=u_skipWhitespace(t); + + /* read the end of the range if there is one */ + if(*s=='-') { + s=u_skipWhitespace(s+1); + end=uprv_strtoul(s, (char **)&t, 16); + if(s==t || end<start || 0xff<end) { + return s; + } + s=u_skipWhitespace(t); + } else { + end=start; + } + + /* determine the state entrys for this range */ + if(*s!=':' && *s!='.') { + /* the default is: final state with valid entries */ + entry=MBCS_ENTRY_FINAL(0, MBCS_STATE_VALID_16, 0); + } else { + entry=MBCS_ENTRY_TRANSITION(0, 0); + if(*s==':') { + /* get the next state, default to 0 */ + s=u_skipWhitespace(s+1); + i=uprv_strtoul(s, (char **)&t, 16); + if(s!=t) { + if(0x7f<i) { + return s; + } + s=u_skipWhitespace(t); + entry=MBCS_ENTRY_SET_STATE(entry, i); + } + } + + /* get the state action, default to valid */ + if(*s=='.') { + /* this is a final state */ + entry=MBCS_ENTRY_SET_FINAL(entry); + + s=u_skipWhitespace(s+1); + if(*s=='u') { + /* unassigned set U+fffe */ + entry=MBCS_ENTRY_FINAL_SET_ACTION_VALUE(entry, MBCS_STATE_UNASSIGNED, 0xfffe); + s=u_skipWhitespace(s+1); + } else if(*s=='p') { + if(*pFlags!=MBCS_STATE_FLAG_DIRECT) { + entry=MBCS_ENTRY_FINAL_SET_ACTION(entry, MBCS_STATE_VALID_16_PAIR); + } else { + entry=MBCS_ENTRY_FINAL_SET_ACTION(entry, MBCS_STATE_VALID_16); + } + s=u_skipWhitespace(s+1); + } else if(*s=='s') { + entry=MBCS_ENTRY_FINAL_SET_ACTION(entry, MBCS_STATE_CHANGE_ONLY); + s=u_skipWhitespace(s+1); + } else if(*s=='i') { + /* illegal set U+ffff */ + entry=MBCS_ENTRY_FINAL_SET_ACTION_VALUE(entry, MBCS_STATE_ILLEGAL, 0xffff); + s=u_skipWhitespace(s+1); + } else { + /* default to valid */ + entry=MBCS_ENTRY_FINAL_SET_ACTION(entry, MBCS_STATE_VALID_16); + } + } else { + /* this is an intermediate state, nothing to do */ + } + } + + /* adjust "final valid" states according to the state flags */ + if(MBCS_ENTRY_FINAL_ACTION(entry)==MBCS_STATE_VALID_16) { + switch(*pFlags) { + case 0: + /* no adjustment */ + break; + case MBCS_STATE_FLAG_DIRECT: + /* set the valid-direct code point to "unassigned"==0xfffe */ + entry=MBCS_ENTRY_FINAL_SET_ACTION_VALUE(entry, MBCS_STATE_VALID_DIRECT_16, 0xfffe); + break; + case MBCS_STATE_FLAG_SURROGATES: + entry=MBCS_ENTRY_FINAL_SET_ACTION_VALUE(entry, MBCS_STATE_VALID_16_PAIR, 0); + break; + default: + break; + } + } + + /* set this entry for the range */ + for(i=start; i<=end; ++i) { + state[i]=entry; + } + + if(*s==',') { + ++s; + } else { + return *s==0 ? NULL : s; + } + } +} + +U_CAPI void U_EXPORT2 +ucm_addState(UCMStates *states, const char *s) { + const char *error; + + if(states->countStates==MBCS_MAX_STATE_COUNT) { + fprintf(stderr, "ucm error: too many states (maximum %u)\n", MBCS_MAX_STATE_COUNT); + exit(U_INVALID_TABLE_FORMAT); + } + + error=parseState(s, states->stateTable[states->countStates], + &states->stateFlags[states->countStates]); + if(error!=NULL) { + fprintf(stderr, "ucm error: parse error in state definition at '%s'\n", error); + exit(U_INVALID_TABLE_FORMAT); + } + + ++states->countStates; +} + +U_CAPI UBool U_EXPORT2 +ucm_parseHeaderLine(UCMFile *ucm, + char *line, char **pKey, char **pValue) { + UCMStates *states; + char *s, *end; + char c; + + states=&ucm->states; + + /* remove comments and trailing CR and LF and remove whitespace from the end */ + for(end=line; (c=*end)!=0; ++end) { + if(c=='#' || c=='\r' || c=='\n') { + break; + } + } + while(end>line && (*(end-1)==' ' || *(end-1)=='\t')) { + --end; + } + *end=0; + + /* skip leading white space and ignore empty lines */ + s=(char *)u_skipWhitespace(line); + if(*s==0) { + return TRUE; + } + + /* stop at the beginning of the mapping section */ + if(uprv_memcmp(s, "CHARMAP", 7)==0) { + return FALSE; + } + + /* get the key name, bracketed in <> */ + if(*s!='<') { + fprintf(stderr, "ucm error: no header field <key> in line \"%s\"\n", line); + exit(U_INVALID_TABLE_FORMAT); + } + *pKey=++s; + while(*s!='>') { + if(*s==0) { + fprintf(stderr, "ucm error: incomplete header field <key> in line \"%s\"\n", line); + exit(U_INVALID_TABLE_FORMAT); + } + ++s; + } + *s=0; + + /* get the value string, possibly quoted */ + s=(char *)u_skipWhitespace(s+1); + if(*s!='"') { + *pValue=s; + } else { + /* remove the quotes */ + *pValue=s+1; + if(end>*pValue && *(end-1)=='"') { + *--end=0; + } + } + + /* collect the information from the header field, ignore unknown keys */ + if(uprv_strcmp(*pKey, "uconv_class")==0) { + if(uprv_strcmp(*pValue, "DBCS")==0) { + states->conversionType=UCNV_DBCS; + } else if(uprv_strcmp(*pValue, "SBCS")==0) { + states->conversionType = UCNV_SBCS; + } else if(uprv_strcmp(*pValue, "MBCS")==0) { + states->conversionType = UCNV_MBCS; + } else if(uprv_strcmp(*pValue, "EBCDIC_STATEFUL")==0) { + states->conversionType = UCNV_EBCDIC_STATEFUL; + } else { + fprintf(stderr, "ucm error: unknown <uconv_class> %s\n", *pValue); + exit(U_INVALID_TABLE_FORMAT); + } + return TRUE; + } else if(uprv_strcmp(*pKey, "mb_cur_max")==0) { + c=**pValue; + if('1'<=c && c<='4' && (*pValue)[1]==0) { + states->maxCharLength=(int8_t)(c-'0'); + states->outputType=(int8_t)(states->maxCharLength-1); + } else { + fprintf(stderr, "ucm error: illegal <mb_cur_max> %s\n", *pValue); + exit(U_INVALID_TABLE_FORMAT); + } + return TRUE; + } else if(uprv_strcmp(*pKey, "mb_cur_min")==0) { + c=**pValue; + if('1'<=c && c<='4' && (*pValue)[1]==0) { + states->minCharLength=(int8_t)(c-'0'); + } else { + fprintf(stderr, "ucm error: illegal <mb_cur_min> %s\n", *pValue); + exit(U_INVALID_TABLE_FORMAT); + } + return TRUE; + } else if(uprv_strcmp(*pKey, "icu:state")==0) { + /* if an SBCS/DBCS/EBCDIC_STATEFUL converter has icu:state, then turn it into MBCS */ + switch(states->conversionType) { + case UCNV_SBCS: + case UCNV_DBCS: + case UCNV_EBCDIC_STATEFUL: + states->conversionType=UCNV_MBCS; + break; + case UCNV_MBCS: + break; + default: + fprintf(stderr, "ucm error: <icu:state> entry for non-MBCS table or before the <uconv_class> line\n"); + exit(U_INVALID_TABLE_FORMAT); + } + + if(states->maxCharLength==0) { + fprintf(stderr, "ucm error: <icu:state> before the <mb_cur_max> line\n"); + exit(U_INVALID_TABLE_FORMAT); + } + ucm_addState(states, *pValue); + return TRUE; + } else if(uprv_strcmp(*pKey, "icu:base")==0) { + if(**pValue==0) { + fprintf(stderr, "ucm error: <icu:base> without a base table name\n"); + exit(U_INVALID_TABLE_FORMAT); + } + uprv_strcpy(ucm->baseName, *pValue); + return TRUE; + } + + return FALSE; +} + +/* post-processing ---------------------------------------------------------- */ + +static int32_t +sumUpStates(UCMStates *states) { + int32_t entry, sum, state, cell, count; + UBool allStatesReady; + + /* + * Sum up the offsets for all states. + * In each final state (where there are only final entries), + * the offsets add up directly. + * In all other state table rows, for each transition entry to another state, + * the offsets sum of that state needs to be added. + * This is achieved in at most countStates iterations. + */ + allStatesReady=FALSE; + for(count=states->countStates; !allStatesReady && count>=0; --count) { + allStatesReady=TRUE; + for(state=states->countStates-1; state>=0; --state) { + if(!(states->stateFlags[state]&MBCS_STATE_FLAG_READY)) { + allStatesReady=FALSE; + sum=0; + + /* at first, add up only the final delta offsets to keep them <512 */ + for(cell=0; cell<256; ++cell) { + entry=states->stateTable[state][cell]; + if(MBCS_ENTRY_IS_FINAL(entry)) { + switch(MBCS_ENTRY_FINAL_ACTION(entry)) { + case MBCS_STATE_VALID_16: + states->stateTable[state][cell]=MBCS_ENTRY_FINAL_SET_VALUE(entry, sum); + sum+=1; + break; + case MBCS_STATE_VALID_16_PAIR: + states->stateTable[state][cell]=MBCS_ENTRY_FINAL_SET_VALUE(entry, sum); + sum+=2; + break; + default: + /* no addition */ + break; + } + } + } + + /* now, add up the delta offsets for the transitional entries */ + for(cell=0; cell<256; ++cell) { + entry=states->stateTable[state][cell]; + if(MBCS_ENTRY_IS_TRANSITION(entry)) { + if(states->stateFlags[MBCS_ENTRY_TRANSITION_STATE(entry)]&MBCS_STATE_FLAG_READY) { + states->stateTable[state][cell]=MBCS_ENTRY_TRANSITION_SET_OFFSET(entry, sum); + sum+=states->stateOffsetSum[MBCS_ENTRY_TRANSITION_STATE(entry)]; + } else { + /* that next state does not have a sum yet, we cannot finish the one for this state */ + sum=-1; + break; + } + } + } + + if(sum!=-1) { + states->stateOffsetSum[state]=sum; + states->stateFlags[state]|=MBCS_STATE_FLAG_READY; + } + } + } + } + + if(!allStatesReady) { + fprintf(stderr, "ucm error: the state table contains loops\n"); + exit(U_INVALID_TABLE_FORMAT); + } + + /* + * For all "direct" (i.e., initial) states>0, + * the offsets need to be increased by the sum of + * the previous initial states. + */ + sum=states->stateOffsetSum[0]; + for(state=1; state<states->countStates; ++state) { + if((states->stateFlags[state]&0xf)==MBCS_STATE_FLAG_DIRECT) { + int32_t sum2=sum; + sum+=states->stateOffsetSum[state]; + for(cell=0; cell<256; ++cell) { + entry=states->stateTable[state][cell]; + if(MBCS_ENTRY_IS_TRANSITION(entry)) { + states->stateTable[state][cell]=MBCS_ENTRY_TRANSITION_ADD_OFFSET(entry, sum2); + } + } + } + } + + /* round up to the next even number to have the following data 32-bit-aligned */ + return states->countToUCodeUnits=(sum+1)&~1; +} + +U_CAPI void U_EXPORT2 +ucm_processStates(UCMStates *states) { + int32_t entry, state, cell, count; + + if(states->conversionType==UCNV_UNSUPPORTED_CONVERTER) { + fprintf(stderr, "ucm error: missing conversion type (<uconv_class>)\n"); + exit(U_INVALID_TABLE_FORMAT); + } + + if(states->countStates==0) { + switch(states->conversionType) { + case UCNV_SBCS: + /* SBCS: use MBCS data structure with a default state table */ + if(states->maxCharLength!=1) { + fprintf(stderr, "error: SBCS codepage with max B/char!=1\n"); + exit(U_INVALID_TABLE_FORMAT); + } + states->conversionType=UCNV_MBCS; + ucm_addState(states, "0-ff"); + break; + case UCNV_MBCS: + fprintf(stderr, "ucm error: missing state table information (<icu:state>) for MBCS\n"); + exit(U_INVALID_TABLE_FORMAT); + break; + case UCNV_EBCDIC_STATEFUL: + /* EBCDIC_STATEFUL: use MBCS data structure with a default state table */ + if(states->minCharLength!=1 || states->maxCharLength!=2) { + fprintf(stderr, "error: DBCS codepage with min B/char!=1 or max B/char!=2\n"); + exit(U_INVALID_TABLE_FORMAT); + } + states->conversionType=UCNV_MBCS; + ucm_addState(states, "0-ff, e:1.s, f:0.s"); + ucm_addState(states, "initial, 0-3f:4, e:1.s, f:0.s, 40:3, 41-fe:2, ff:4"); + ucm_addState(states, "0-40:1.i, 41-fe:1., ff:1.i"); + ucm_addState(states, "0-ff:1.i, 40:1."); + ucm_addState(states, "0-ff:1.i"); + break; + case UCNV_DBCS: + /* DBCS: use MBCS data structure with a default state table */ + if(states->minCharLength!=2 || states->maxCharLength!=2) { + fprintf(stderr, "error: DBCS codepage with min or max B/char!=2\n"); + exit(U_INVALID_TABLE_FORMAT); + } + states->conversionType = UCNV_MBCS; + ucm_addState(states, "0-3f:3, 40:2, 41-fe:1, ff:3"); + ucm_addState(states, "41-fe"); + ucm_addState(states, "40"); + ucm_addState(states, ""); + break; + default: + fprintf(stderr, "ucm error: unknown charset structure\n"); + exit(U_INVALID_TABLE_FORMAT); + break; + } + } + + /* + * check that the min/max character lengths are reasonable; + * to do this right, all paths through the state table would have to be + * recursively walked while keeping track of the sequence lengths, + * but these simple checks cover most state tables in practice + */ + if(states->maxCharLength<states->minCharLength) { + fprintf(stderr, "ucm error: max B/char < min B/char\n"); + exit(U_INVALID_TABLE_FORMAT); + } + + /* count non-direct states and compare with max B/char */ + count=0; + for(state=0; state<states->countStates; ++state) { + if((states->stateFlags[state]&0xf)!=MBCS_STATE_FLAG_DIRECT) { + ++count; + } + } + if(states->maxCharLength>count+1) { + fprintf(stderr, "ucm error: max B/char too large\n"); + exit(U_INVALID_TABLE_FORMAT); + } + + if(states->minCharLength==1) { + int32_t action; + + /* + * if there are single-byte characters, + * then the initial state must have direct result states + */ + for(cell=0; cell<256; ++cell) { + entry=states->stateTable[0][cell]; + if( MBCS_ENTRY_IS_FINAL(entry) && + ((action=MBCS_ENTRY_FINAL_ACTION(entry))==MBCS_STATE_VALID_DIRECT_16 || + action==MBCS_STATE_UNASSIGNED) + ) { + break; + } + } + + if(cell==256) { + fprintf(stderr, "ucm warning: min B/char too small\n"); + } + } + + /* + * make sure that all "next state" values are within limits + * and that all next states after final ones have the "direct" + * flag of initial states + */ + for(state=states->countStates-1; state>=0; --state) { + for(cell=0; cell<256; ++cell) { + entry=states->stateTable[state][cell]; + if((uint8_t)MBCS_ENTRY_STATE(entry)>=states->countStates) { + fprintf(stderr, "ucm error: state table entry [%x][%x] has a next state of %x that is too high\n", + (int)state, (int)cell, (int)MBCS_ENTRY_STATE(entry)); + exit(U_INVALID_TABLE_FORMAT); + } + if(MBCS_ENTRY_IS_FINAL(entry) && (states->stateFlags[MBCS_ENTRY_STATE(entry)]&0xf)!=MBCS_STATE_FLAG_DIRECT) { + fprintf(stderr, "ucm error: state table entry [%x][%x] is final but has a non-initial next state of %x\n", + (int)state, (int)cell, (int)MBCS_ENTRY_STATE(entry)); + exit(U_INVALID_TABLE_FORMAT); + } else if(MBCS_ENTRY_IS_TRANSITION(entry) && (states->stateFlags[MBCS_ENTRY_STATE(entry)]&0xf)==MBCS_STATE_FLAG_DIRECT) { + fprintf(stderr, "ucm error: state table entry [%x][%x] is not final but has an initial next state of %x\n", + (int)state, (int)cell, (int)MBCS_ENTRY_STATE(entry)); + exit(U_INVALID_TABLE_FORMAT); + } + } + } + + /* is this an SI/SO (like EBCDIC-stateful) state table? */ + if(states->countStates>=2 && (states->stateFlags[1]&0xf)==MBCS_STATE_FLAG_DIRECT) { + if(states->maxCharLength!=2) { + fprintf(stderr, "ucm error: SI/SO codepages must have max 2 bytes/char (not %x)\n", (int)states->maxCharLength); + exit(U_INVALID_TABLE_FORMAT); + } + if(states->countStates<3) { + fprintf(stderr, "ucm error: SI/SO codepages must have at least 3 states (not %x)\n", (int)states->countStates); + exit(U_INVALID_TABLE_FORMAT); + } + /* are the SI/SO all in the right places? */ + if( states->stateTable[0][0xe]==MBCS_ENTRY_FINAL(1, MBCS_STATE_CHANGE_ONLY, 0) && + states->stateTable[0][0xf]==MBCS_ENTRY_FINAL(0, MBCS_STATE_CHANGE_ONLY, 0) && + states->stateTable[1][0xe]==MBCS_ENTRY_FINAL(1, MBCS_STATE_CHANGE_ONLY, 0) && + states->stateTable[1][0xf]==MBCS_ENTRY_FINAL(0, MBCS_STATE_CHANGE_ONLY, 0) + ) { + states->outputType=MBCS_OUTPUT_2_SISO; + } else { + fprintf(stderr, "ucm error: SI/SO codepages must have in states 0 and 1 transitions e:1.s, f:0.s\n"); + exit(U_INVALID_TABLE_FORMAT); + } + state=2; + } else { + state=1; + } + + /* check that no unexpected state is a "direct" one */ + while(state<states->countStates) { + if((states->stateFlags[state]&0xf)==MBCS_STATE_FLAG_DIRECT) { + fprintf(stderr, "ucm error: state %d is 'initial' - not supported except for SI/SO codepages\n", (int)state); + exit(U_INVALID_TABLE_FORMAT); + } + ++state; + } + + sumUpStates(states); +} + +/* find a fallback for this offset; return the index or -1 if not found */ +U_CAPI int32_t U_EXPORT2 +ucm_findFallback(_MBCSToUFallback *toUFallbacks, int32_t countToUFallbacks, + uint32_t offset) { + int32_t i; + + if(countToUFallbacks==0) { + /* shortcut: most codepages do not have fallbacks from codepage to Unicode */ + return -1; + } + + /* do a linear search for the fallback mapping (the table is not yet sorted) */ + for(i=0; i<countToUFallbacks; ++i) { + if(offset==toUFallbacks[i].offset) { + return i; + } + } + return -1; +} + +/* + * This function tries to compact toUnicode tables for 2-byte codepages + * by finding lead bytes with all-unassigned trail bytes and adding another state + * for them. + */ +static void +compactToUnicode2(UCMStates *states, + uint16_t **pUnicodeCodeUnits, + _MBCSToUFallback *toUFallbacks, int32_t countToUFallbacks, + UBool verbose) { + int32_t (*oldStateTable)[256]; + uint16_t count[256]; + uint16_t *oldUnicodeCodeUnits; + int32_t entry, offset, oldOffset, trailOffset, oldTrailOffset, savings, sum; + int32_t i, j, leadState, trailState, newState, fallback; + uint16_t unit; + + /* find the lead state */ + if(states->outputType==MBCS_OUTPUT_2_SISO) { + /* use the DBCS lead state for SI/SO codepages */ + leadState=1; + } else { + leadState=0; + } + + /* find the main trail state: the most used target state */ + uprv_memset(count, 0, sizeof(count)); + for(i=0; i<256; ++i) { + entry=states->stateTable[leadState][i]; + if(MBCS_ENTRY_IS_TRANSITION(entry)) { + ++count[MBCS_ENTRY_TRANSITION_STATE(entry)]; + } + } + trailState=0; + for(i=1; i<states->countStates; ++i) { + if(count[i]>count[trailState]) { + trailState=i; + } + } + + /* count possible savings from lead bytes with all-unassigned results in all trail bytes */ + uprv_memset(count, 0, sizeof(count)); + savings=0; + /* for each lead byte */ + for(i=0; i<256; ++i) { + entry=states->stateTable[leadState][i]; + if(MBCS_ENTRY_IS_TRANSITION(entry) && (MBCS_ENTRY_TRANSITION_STATE(entry))==trailState) { + /* the offset is different for each lead byte */ + offset=MBCS_ENTRY_TRANSITION_OFFSET(entry); + /* for each trail byte for this lead byte */ + for(j=0; j<256; ++j) { + entry=states->stateTable[trailState][j]; + switch(MBCS_ENTRY_FINAL_ACTION(entry)) { + case MBCS_STATE_VALID_16: + entry=offset+MBCS_ENTRY_FINAL_VALUE_16(entry); + if((*pUnicodeCodeUnits)[entry]==0xfffe && ucm_findFallback(toUFallbacks, countToUFallbacks, entry)<0) { + ++count[i]; + } else { + j=999; /* do not count for this lead byte because there are assignments */ + } + break; + case MBCS_STATE_VALID_16_PAIR: + entry=offset+MBCS_ENTRY_FINAL_VALUE_16(entry); + if((*pUnicodeCodeUnits)[entry]==0xfffe) { + count[i]+=2; + } else { + j=999; /* do not count for this lead byte because there are assignments */ + } + break; + default: + break; + } + } + if(j==256) { + /* all trail bytes for this lead byte are unassigned */ + savings+=count[i]; + } else { + count[i]=0; + } + } + } + /* subtract from the possible savings the cost of an additional state */ + savings=savings*2-1024; /* count bytes, not 16-bit words */ + if(savings<=0) { + return; + } + if(verbose) { + printf("compacting toUnicode data saves %ld bytes\n", (long)savings); + } + if(states->countStates>=MBCS_MAX_STATE_COUNT) { + fprintf(stderr, "cannot compact toUnicode because the maximum number of states is reached\n"); + return; + } + + /* make a copy of the state table */ + oldStateTable=(int32_t (*)[256])uprv_malloc(states->countStates*1024); + if(oldStateTable==NULL) { + fprintf(stderr, "cannot compact toUnicode: out of memory\n"); + return; + } + uprv_memcpy(oldStateTable, states->stateTable, states->countStates*1024); + + /* add the new state */ + /* + * this function does not catch the degenerate case where all lead bytes + * have all-unassigned trail bytes and the lead state could be removed + */ + newState=states->countStates++; + states->stateFlags[newState]=0; + /* copy the old trail state, turning all assigned states into unassigned ones */ + for(i=0; i<256; ++i) { + entry=states->stateTable[trailState][i]; + switch(MBCS_ENTRY_FINAL_ACTION(entry)) { + case MBCS_STATE_VALID_16: + case MBCS_STATE_VALID_16_PAIR: + states->stateTable[newState][i]=MBCS_ENTRY_FINAL_SET_ACTION_VALUE(entry, MBCS_STATE_UNASSIGNED, 0xfffe); + break; + default: + states->stateTable[newState][i]=entry; + break; + } + } + + /* in the lead state, redirect all lead bytes with all-unassigned trail bytes to the new state */ + for(i=0; i<256; ++i) { + if(count[i]>0) { + states->stateTable[leadState][i]=MBCS_ENTRY_SET_STATE(states->stateTable[leadState][i], newState); + } + } + + /* sum up the new state table */ + for(i=0; i<states->countStates; ++i) { + states->stateFlags[i]&=~MBCS_STATE_FLAG_READY; + } + sum=sumUpStates(states); + + /* allocate a new, smaller code units array */ + oldUnicodeCodeUnits=*pUnicodeCodeUnits; + if(sum==0) { + *pUnicodeCodeUnits=NULL; + if(oldUnicodeCodeUnits!=NULL) { + uprv_free(oldUnicodeCodeUnits); + } + uprv_free(oldStateTable); + return; + } + *pUnicodeCodeUnits=(uint16_t *)uprv_malloc(sum*sizeof(uint16_t)); + if(*pUnicodeCodeUnits==NULL) { + fprintf(stderr, "cannot compact toUnicode: out of memory allocating %ld 16-bit code units\n", + (long)sum); + /* revert to the old state table */ + *pUnicodeCodeUnits=oldUnicodeCodeUnits; + --states->countStates; + uprv_memcpy(states->stateTable, oldStateTable, states->countStates*1024); + uprv_free(oldStateTable); + return; + } + for(i=0; i<sum; ++i) { + (*pUnicodeCodeUnits)[i]=0xfffe; + } + + /* copy the code units for all assigned characters */ + /* + * The old state table has the same lead _and_ trail states for assigned characters! + * The differences are in the offsets, and in the trail states for some unassigned characters. + * For each character with an assigned state in the new table, it was assigned in the old one. + * Only still-assigned characters are copied. + * Note that fallback mappings need to get their offset values adjusted. + */ + + /* for each initial state */ + for(leadState=0; leadState<states->countStates; ++leadState) { + if((states->stateFlags[leadState]&0xf)==MBCS_STATE_FLAG_DIRECT) { + /* for each lead byte from there */ + for(i=0; i<256; ++i) { + entry=states->stateTable[leadState][i]; + if(MBCS_ENTRY_IS_TRANSITION(entry)) { + trailState=(uint8_t)MBCS_ENTRY_TRANSITION_STATE(entry); + /* the new state does not have assigned states */ + if(trailState!=newState) { + trailOffset=MBCS_ENTRY_TRANSITION_OFFSET(entry); + oldTrailOffset=MBCS_ENTRY_TRANSITION_OFFSET(oldStateTable[leadState][i]); + /* for each trail byte */ + for(j=0; j<256; ++j) { + entry=states->stateTable[trailState][j]; + /* copy assigned-character code units and adjust fallback offsets */ + switch(MBCS_ENTRY_FINAL_ACTION(entry)) { + case MBCS_STATE_VALID_16: + offset=trailOffset+MBCS_ENTRY_FINAL_VALUE_16(entry); + /* find the old offset according to the old state table */ + oldOffset=oldTrailOffset+MBCS_ENTRY_FINAL_VALUE_16(oldStateTable[trailState][j]); + unit=(*pUnicodeCodeUnits)[offset]=oldUnicodeCodeUnits[oldOffset]; + if(unit==0xfffe && (fallback=ucm_findFallback(toUFallbacks, countToUFallbacks, oldOffset))>=0) { + toUFallbacks[fallback].offset=0x80000000|offset; + } + break; + case MBCS_STATE_VALID_16_PAIR: + offset=trailOffset+MBCS_ENTRY_FINAL_VALUE_16(entry); + /* find the old offset according to the old state table */ + oldOffset=oldTrailOffset+MBCS_ENTRY_FINAL_VALUE_16(oldStateTable[trailState][j]); + (*pUnicodeCodeUnits)[offset++]=oldUnicodeCodeUnits[oldOffset++]; + (*pUnicodeCodeUnits)[offset]=oldUnicodeCodeUnits[oldOffset]; + break; + default: + break; + } + } + } + } + } + } + } + + /* remove temporary flags from fallback offsets that protected them from being modified twice */ + for(i=0; i<countToUFallbacks; ++i) { + toUFallbacks[i].offset&=0x7fffffff; + } + + /* free temporary memory */ + uprv_free(oldUnicodeCodeUnits); + uprv_free(oldStateTable); +} + +/* + * recursive sub-function of compactToUnicodeHelper() + * returns: + * >0 number of bytes that are used in unicodeCodeUnits[] that could be saved, + * if all sequences from this state are unassigned, returns the + * <0 there are assignments in unicodeCodeUnits[] + * 0 no use of unicodeCodeUnits[] + */ +static int32_t +findUnassigned(UCMStates *states, + uint16_t *unicodeCodeUnits, + _MBCSToUFallback *toUFallbacks, int32_t countToUFallbacks, + int32_t state, int32_t offset, uint32_t b) { + int32_t i, entry, savings, localSavings, belowSavings; + UBool haveAssigned; + + localSavings=belowSavings=0; + haveAssigned=FALSE; + for(i=0; i<256; ++i) { + entry=states->stateTable[state][i]; + if(MBCS_ENTRY_IS_TRANSITION(entry)) { + savings=findUnassigned(states, + unicodeCodeUnits, + toUFallbacks, countToUFallbacks, + MBCS_ENTRY_TRANSITION_STATE(entry), + offset+MBCS_ENTRY_TRANSITION_OFFSET(entry), + (b<<8)|(uint32_t)i); + if(savings<0) { + haveAssigned=TRUE; + } else if(savings>0) { + printf(" all-unassigned sequences from prefix 0x%02lx state %ld use %ld bytes\n", + (unsigned long)((b<<8)|i), (long)state, (long)savings); + belowSavings+=savings; + } + } else if(!haveAssigned) { + switch(MBCS_ENTRY_FINAL_ACTION(entry)) { + case MBCS_STATE_VALID_16: + entry=offset+MBCS_ENTRY_FINAL_VALUE_16(entry); + if(unicodeCodeUnits[entry]==0xfffe && ucm_findFallback(toUFallbacks, countToUFallbacks, entry)<0) { + localSavings+=2; + } else { + haveAssigned=TRUE; + } + break; + case MBCS_STATE_VALID_16_PAIR: + entry=offset+MBCS_ENTRY_FINAL_VALUE_16(entry); + if(unicodeCodeUnits[entry]==0xfffe) { + localSavings+=4; + } else { + haveAssigned=TRUE; + } + break; + default: + break; + } + } + } + if(haveAssigned) { + return -1; + } else { + return localSavings+belowSavings; + } +} + +/* helper function for finding compaction opportunities */ +static void +compactToUnicodeHelper(UCMStates *states, + uint16_t *unicodeCodeUnits, + _MBCSToUFallback *toUFallbacks, int32_t countToUFallbacks) { + int32_t state, savings; + + /* for each initial state */ + for(state=0; state<states->countStates; ++state) { + if((states->stateFlags[state]&0xf)==MBCS_STATE_FLAG_DIRECT) { + savings=findUnassigned(states, + unicodeCodeUnits, + toUFallbacks, countToUFallbacks, + state, 0, 0); + if(savings>0) { + printf(" all-unassigned sequences from initial state %ld use %ld bytes\n", + (long)state, (long)savings); + } + } + } +} + +static int32_t +compareFallbacks(const void *context, const void *fb1, const void *fb2) { + return ((const _MBCSToUFallback *)fb1)->offset-((const _MBCSToUFallback *)fb2)->offset; +} + +U_CAPI void U_EXPORT2 +ucm_optimizeStates(UCMStates *states, + uint16_t **pUnicodeCodeUnits, + _MBCSToUFallback *toUFallbacks, int32_t countToUFallbacks, + UBool verbose) { + UErrorCode errorCode; + int32_t state, cell, entry; + + /* test each state table entry */ + for(state=0; state<states->countStates; ++state) { + for(cell=0; cell<256; ++cell) { + entry=states->stateTable[state][cell]; + /* + * if the entry is a final one with an MBCS_STATE_VALID_DIRECT_16 action code + * and the code point is "unassigned" (0xfffe), then change it to + * the "unassigned" action code with bits 26..23 set to zero and U+fffe. + */ + if(MBCS_ENTRY_SET_STATE(entry, 0)==MBCS_ENTRY_FINAL(0, MBCS_STATE_VALID_DIRECT_16, 0xfffe)) { + states->stateTable[state][cell]=MBCS_ENTRY_FINAL_SET_ACTION(entry, MBCS_STATE_UNASSIGNED); + } + } + } + + /* try to compact the toUnicode tables */ + if(states->maxCharLength==2) { + compactToUnicode2(states, pUnicodeCodeUnits, toUFallbacks, countToUFallbacks, verbose); + } else if(states->maxCharLength>2) { + if(verbose) { + compactToUnicodeHelper(states, *pUnicodeCodeUnits, toUFallbacks, countToUFallbacks); + } + } + + /* sort toUFallbacks */ + /* + * It should be safe to sort them before compactToUnicode2() is called, + * because it should not change the relative order of the offset values + * that it adjusts, but they need to be sorted at some point, and + * it is safest here. + */ + if(countToUFallbacks>0) { + errorCode=U_ZERO_ERROR; /* nothing bad will happen... */ + uprv_sortArray(toUFallbacks, countToUFallbacks, + sizeof(_MBCSToUFallback), + compareFallbacks, NULL, FALSE, &errorCode); + } +} + +/* use a complete state table ----------------------------------------------- */ + +U_CAPI int32_t U_EXPORT2 +ucm_countChars(UCMStates *states, + const uint8_t *bytes, int32_t length) { + uint32_t offset; + int32_t i, entry, count; + uint8_t state; + + offset=0; + i=count=0; + state=0; + + if(states->countStates==0) { + fprintf(stderr, "ucm error: there is no state information!\n"); + return -1; + } + + /* for SI/SO (like EBCDIC-stateful), double-byte sequences start in state 1 */ + if(length==2 && states->outputType==MBCS_OUTPUT_2_SISO) { + state=1; + } + + /* + * Walk down the state table like in conversion, + * much like getNextUChar(). + * We assume that c<=0x10ffff. + */ + for(i=0; i<length; ++i) { + entry=states->stateTable[state][bytes[i]]; + if(MBCS_ENTRY_IS_TRANSITION(entry)) { + state=(uint8_t)MBCS_ENTRY_TRANSITION_STATE(entry); + offset+=MBCS_ENTRY_TRANSITION_OFFSET(entry); + } else { + switch(MBCS_ENTRY_FINAL_ACTION(entry)) { + case MBCS_STATE_ILLEGAL: + fprintf(stderr, "ucm error: byte sequence ends in illegal state\n"); + return -1; + case MBCS_STATE_CHANGE_ONLY: + fprintf(stderr, "ucm error: byte sequence ends in state-change-only\n"); + return -1; + case MBCS_STATE_UNASSIGNED: + case MBCS_STATE_FALLBACK_DIRECT_16: + case MBCS_STATE_VALID_DIRECT_16: + case MBCS_STATE_FALLBACK_DIRECT_20: + case MBCS_STATE_VALID_DIRECT_20: + case MBCS_STATE_VALID_16: + case MBCS_STATE_VALID_16_PAIR: + /* count a complete character and prepare for a new one */ + ++count; + state=(uint8_t)MBCS_ENTRY_FINAL_STATE(entry); + offset=0; + break; + default: + /* reserved, must never occur */ + fprintf(stderr, "ucm error: byte sequence reached reserved action code, entry: 0x%02lx\n", (unsigned long)entry); + return -1; + } + } + } + + if(offset!=0) { + fprintf(stderr, "ucm error: byte sequence too short, ends in non-final state %hu\n", state); + return -1; + } + + /* + * for SI/SO (like EBCDIC-stateful), multiple-character results + * must consist of only double-byte sequences + */ + if(count>1 && states->outputType==MBCS_OUTPUT_2_SISO && length!=2*count) { + fprintf(stderr, "ucm error: SI/SO (like EBCDIC-stateful) result with %d characters does not contain all DBCS\n", (int)count); + return -1; + } + + return count; +} +#endif + diff --git a/Build/source/libs/icu/icu-xetex/tools/toolutil/unewdata.c b/Build/source/libs/icu/icu-xetex/tools/toolutil/unewdata.c new file mode 100644 index 00000000000..5666c87fda2 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/tools/toolutil/unewdata.c @@ -0,0 +1,219 @@ +/* +******************************************************************************* +* +* Copyright (C) 1999,2004, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: unewdata.c +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 1999oct25 +* created by: Markus W. Scherer +*/ + +#include "unicode/utypes.h" +#include "unicode/putil.h" +#include "unicode/ustring.h" +#include "cmemory.h" +#include "cstring.h" +#include "filestrm.h" +#include "unicode/udata.h" +#include "unewdata.h" + +struct UNewDataMemory { + FileStream *file; + uint16_t headerSize; + uint8_t magic1, magic2; +}; + +U_CAPI UNewDataMemory * U_EXPORT2 +udata_create(const char *dir, const char *type, const char *name, + const UDataInfo *pInfo, + const char *comment, + UErrorCode *pErrorCode) { + UNewDataMemory *pData; + uint16_t headerSize, commentLength; + char filename[512]; + uint8_t bytes[16]; + + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return NULL; + } else if(name==NULL || *name==0 || pInfo==NULL) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return NULL; + } + + /* allocate the data structure */ + pData=(UNewDataMemory *)uprv_malloc(sizeof(UNewDataMemory)); + if(pData==NULL) { + *pErrorCode=U_MEMORY_ALLOCATION_ERROR; + return NULL; + } + + /* open the output file */ + if(dir!=NULL && *dir!=0) { /* if dir has a value, we prepend it to the filename */ + char *p=filename+strlen(dir); + uprv_strcpy(filename, dir); + if (*(p-1)!=U_FILE_SEP_CHAR) { + *p++=U_FILE_SEP_CHAR; + *p=0; + } + } else { /* otherwise, we'll output to the current dir */ + filename[0]=0; + } + uprv_strcat(filename, name); + if(type!=NULL && *type!=0) { + uprv_strcat(filename, "."); + uprv_strcat(filename, type); + } + pData->file=T_FileStream_open(filename, "wb"); + if(pData->file==NULL) { + uprv_free(pData); + *pErrorCode=U_FILE_ACCESS_ERROR; + return NULL; + } + + /* write the header information */ + headerSize=(uint16_t)(pInfo->size+4); + if(comment!=NULL && *comment!=0) { + commentLength=(uint16_t)(uprv_strlen(comment)+1); + headerSize+=commentLength; + } else { + commentLength=0; + } + + /* write the size of the header, take padding into account */ + pData->headerSize=(uint16_t)((headerSize+15)&~0xf); + pData->magic1=0xda; + pData->magic2=0x27; + T_FileStream_write(pData->file, &pData->headerSize, 4); + + /* write the information data */ + T_FileStream_write(pData->file, pInfo, pInfo->size); + + /* write the comment */ + if(commentLength>0) { + T_FileStream_write(pData->file, comment, commentLength); + } + + /* write padding bytes to align the data section to 16 bytes */ + headerSize&=0xf; + if(headerSize!=0) { + headerSize=(uint16_t)(16-headerSize); + uprv_memset(bytes, 0, headerSize); + T_FileStream_write(pData->file, bytes, headerSize); + } + + return pData; +} + +U_CAPI uint32_t U_EXPORT2 +udata_finish(UNewDataMemory *pData, UErrorCode *pErrorCode) { + uint32_t fileLength=0; + + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return 0; + } + + if(pData!=NULL) { + if(pData->file!=NULL) { + /* fflush(pData->file);*/ + fileLength=T_FileStream_size(pData->file); + if(T_FileStream_error(pData->file)) { + *pErrorCode=U_FILE_ACCESS_ERROR; + } else { + fileLength-=pData->headerSize; + } + T_FileStream_close(pData->file); + } + uprv_free(pData); + } + + return fileLength; +} + +U_CAPI void U_EXPORT2 +udata_write8(UNewDataMemory *pData, uint8_t byte) { + if(pData!=NULL && pData->file!=NULL) { + T_FileStream_write(pData->file, &byte, 1); + } +} + +U_CAPI void U_EXPORT2 +udata_write16(UNewDataMemory *pData, uint16_t word) { + if(pData!=NULL && pData->file!=NULL) { + T_FileStream_write(pData->file, &word, 2); + } +} + +U_CAPI void U_EXPORT2 +udata_write32(UNewDataMemory *pData, uint32_t wyde) { + if(pData!=NULL && pData->file!=NULL) { + T_FileStream_write(pData->file, &wyde, 4); + } +} + +U_CAPI void U_EXPORT2 +udata_writeBlock(UNewDataMemory *pData, const void *s, int32_t length) { + if(pData!=NULL && pData->file!=NULL) { + if(length>0) { + T_FileStream_write(pData->file, s, length); + } + } +} + +U_CAPI void U_EXPORT2 +udata_writePadding(UNewDataMemory *pData, int32_t length) { + static const uint8_t padding[16]={ + 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa + }; + if(pData!=NULL && pData->file!=NULL) { + while(length>=16) { + T_FileStream_write(pData->file, padding, 16); + length-=16; + } + if(length>0) { + T_FileStream_write(pData->file, padding, length); + } + } +} + +U_CAPI void U_EXPORT2 +udata_writeString(UNewDataMemory *pData, const char *s, int32_t length) { + if(pData!=NULL && pData->file!=NULL) { + if(length==-1) { + length=(int32_t)uprv_strlen(s); + } + if(length>0) { + T_FileStream_write(pData->file, s, length); + } + } +} + +U_CAPI void U_EXPORT2 +udata_writeUString(UNewDataMemory *pData, const UChar *s, int32_t length) { + if(pData!=NULL && pData->file!=NULL) { + if(length==-1) { + length=u_strlen(s); + } + if(length>0) { + T_FileStream_write(pData->file, s, length*sizeof(UChar)); + } + } +} + +/* + * Hey, Emacs, please set the following: + * + * Local Variables: + * indent-tabs-mode: nil + * End: + * + */ + diff --git a/Build/source/libs/icu/icu-xetex/tools/toolutil/unewdata.h b/Build/source/libs/icu/icu-xetex/tools/toolutil/unewdata.h new file mode 100644 index 00000000000..fb190e6f89d --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/tools/toolutil/unewdata.h @@ -0,0 +1,107 @@ +/* +******************************************************************************* +* +* Copyright (C) 1999-2000, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: unewdata.h +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 1999oct25 +* created by: Markus W. Scherer +*/ + +#ifndef __UNEWDATA_H__ +#define __UNEWDATA_H__ + +#include "unicode/utypes.h" +#include "unicode/udata.h" + +/* API for writing data -----------------------------------------------------*/ + +/** @memo Forward declaration of the data memory creation type. */ +typedef struct UNewDataMemory UNewDataMemory; + +/** + * Create a new binary data file. + * The file-writing <code>udata_</code> functions facilitate writing + * binary data files that can be read by ICU's <code>udata</code> API. + * This function opens a new file with a filename determined from its + * parameters - of the form "name.type". + * It then writes a short header, followed by the <code>UDataInfo</code> + * structure and, optionally, by the comment string. + * It then writes padding bytes to round up to a multiple of 16 bytes. + * Subsequent write operations will thus start at an offset in the file + * that is a multiple of 16. <code>udata_getMemory()</code> will return + * a pointer to this same starting offset. + * + * See udata.h . + * + * @param dir A string that specifies the directory where the data will be + * written. If <code>NULL</code>, then + * <code>u_getDataDirectory</code> is used. + * @param type A string that specifies the type of data to be written. + * For example, resource bundles are written with type "res", + * conversion tables with type "cnv". + * This may be <code>NULL</code> or empty. + * @param name A string that specifies the name of the data. + * @param pInfo A pointer to a correctly filled <code>UDataInfo</code> + * structure that will be copied into the file. + * @param comment A string (e.g., a copyright statement) that will be + * copied into the file if it is not <code>NULL</code> + * or empty. This string serves only as a comment in the binary + * file. It will not be accessible by any API. + * @param pErrorCode An ICU UErrorCode parameter. It must not be <code>NULL</code>. + */ +U_CAPI UNewDataMemory * U_EXPORT2 +udata_create(const char *dir, const char *type, const char *name, + const UDataInfo *pInfo, + const char *comment, + UErrorCode *pErrorCode); + +/** @memo Close a newly written binary file. */ +U_CAPI uint32_t U_EXPORT2 +udata_finish(UNewDataMemory *pData, UErrorCode *pErrorCode); + +/** @memo Write an 8-bit byte to the file. */ +U_CAPI void U_EXPORT2 +udata_write8(UNewDataMemory *pData, uint8_t byte); + +/** @memo Write a 16-bit word to the file. */ +U_CAPI void U_EXPORT2 +udata_write16(UNewDataMemory *pData, uint16_t word); + +/** @memo Write a 32-bit word to the file. */ +U_CAPI void U_EXPORT2 +udata_write32(UNewDataMemory *pData, uint32_t wyde); + +/** @memo Write a block of bytes to the file. */ +U_CAPI void U_EXPORT2 +udata_writeBlock(UNewDataMemory *pData, const void *s, int32_t length); + +/** @memo Write a block of arbitrary padding bytes to the file. */ +U_CAPI void U_EXPORT2 +udata_writePadding(UNewDataMemory *pData, int32_t length); + +/** @memo Write a <code>char*</code> string of platform "invariant characters" to the file. */ +U_CAPI void U_EXPORT2 +udata_writeString(UNewDataMemory *pData, const char *s, int32_t length); + +/** @memo Write a <code>UChar*</code> string of Unicode character code units to the file. */ +U_CAPI void U_EXPORT2 +udata_writeUString(UNewDataMemory *pData, const UChar *s, int32_t length); + + +/* + * Hey, Emacs, please set the following: + * + * Local Variables: + * indent-tabs-mode: nil + * End: + * + */ + +#endif diff --git a/Build/source/libs/icu/icu-xetex/tools/toolutil/uoptions.c b/Build/source/libs/icu/icu-xetex/tools/toolutil/uoptions.c new file mode 100644 index 00000000000..05efc170cbd --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/tools/toolutil/uoptions.c @@ -0,0 +1,122 @@ +/* +******************************************************************************* +* +* Copyright (C) 2000, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: uoptions.c +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2000apr17 +* created by: Markus W. Scherer +* +* This file provides a command line argument parser. +*/ + +#include "unicode/utypes.h" +#include "cstring.h" +#include "uoptions.h" + +U_CAPI int U_EXPORT2 +u_parseArgs(int argc, char* argv[], + int optionCount, UOption options[]) { + char *arg; + int i=1, remaining=1; + char c, stopOptions=0; + + while(i<argc) { + arg=argv[i]; + if(!stopOptions && *arg=='-' && (c=arg[1])!=0) { + /* process an option */ + UOption *option=NULL; + arg+=2; + if(c=='-') { + /* process a long option */ + if(*arg==0) { + /* stop processing options after "--" */ + stopOptions=1; + } else { + /* search for the option string */ + int j; + for(j=0; j<optionCount; ++j) { + if(options[j].longName && uprv_strcmp(arg, options[j].longName)==0) { + option=options+j; + break; + } + } + if(option==NULL) { + /* no option matches */ + return -i; + } + option->doesOccur=1; + + if(option->hasArg!=UOPT_NO_ARG) { + /* parse the argument for the option, if any */ + if(i+1<argc && !(argv[i+1][0]=='-' && argv[i+1][1]!=0)) { + /* argument in the next argv[], and there is not an option in there */ + option->value=argv[++i]; + } else if(option->hasArg==UOPT_REQUIRES_ARG) { + /* there is no argument, but one is required: return with error */ + return -i; + } + } + } + } else { + /* process one or more short options */ + do { + /* search for the option letter */ + int j; + for(j=0; j<optionCount; ++j) { + if(c==options[j].shortName) { + option=options+j; + break; + } + } + if(option==NULL) { + /* no option matches */ + return -i; + } + option->doesOccur=1; + + if(option->hasArg!=UOPT_NO_ARG) { + /* parse the argument for the option, if any */ + if(*arg!=0) { + /* argument following in the same argv[] */ + option->value=arg; + /* do not process the rest of this arg as option letters */ + break; + } else if(i+1<argc && !(argv[i+1][0]=='-' && argv[i+1][1]!=0)) { + /* argument in the next argv[], and there is not an option in there */ + option->value=argv[++i]; + /* this break is redundant because we know that *arg==0 */ + break; + } else if(option->hasArg==UOPT_REQUIRES_ARG) { + /* there is no argument, but one is required: return with error */ + return -i; + } + } + + /* get the next option letter */ + option=NULL; + c=*arg++; + } while(c!=0); + } + + if(option!=0 && option->optionFn!=0 && option->optionFn(option->context, option)<0) { + /* the option function was called and returned an error */ + return -i; + } + + /* go to next argv[] */ + ++i; + } else { + /* move a non-option up in argv[] */ + argv[remaining++]=arg; + ++i; + } + } + return remaining; +} diff --git a/Build/source/libs/icu/icu-xetex/tools/toolutil/uoptions.h b/Build/source/libs/icu/icu-xetex/tools/toolutil/uoptions.h new file mode 100644 index 00000000000..55e30f681b2 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/tools/toolutil/uoptions.h @@ -0,0 +1,141 @@ +/* +******************************************************************************* +* +* Copyright (C) 2000, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: uoptions.h +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2000apr17 +* created by: Markus W. Scherer +* +* This file provides a command line argument parser. +*/ + +#ifndef __UOPTIONS_H__ +#define __UOPTIONS_H__ + +#include "unicode/utypes.h" + +/* This should usually be called before calling u_parseArgs */ +/*#if defined(OS390) && (U_CHARSET_FAMILY == U_ASCII_FAMILY)*/ + /* translate args from EBCDIC to ASCII */ +/*# define U_MAIN_INIT_ARGS(argc, argv) __argvtoascii_a(argc, argv)*/ +/*#elif defined(XP_MAC_CONSOLE)*/ +#if defined(XP_MAC_CONSOLE) +# include <console.h> + /* Get the arguments from the GUI, since old Macs don't have a console Window. */ +# define U_MAIN_INIT_ARGS(argc, argv) argc = ccommand((char***)&argv) +#else + /* Normally we do nothing. */ +# define U_MAIN_INIT_ARGS(argc, argv) +#endif + + + +/* forward declarations for the function declaration */ +struct UOption; +typedef struct UOption UOption; + +/* function to be called for a command line option */ +typedef int UOptionFn(void *context, UOption *option); + +/* values of UOption.hasArg */ +enum { UOPT_NO_ARG, UOPT_REQUIRES_ARG, UOPT_OPTIONAL_ARG }; + +/* structure describing a command line option */ +struct UOption { + const char *longName; /* "foo" for --foo */ + const char *value; /* output placeholder, will point to the argument string, if any */ + UOptionFn *optionFn; /* function to be called when this option occurs */ + void *context; /* parameter for the function */ + char shortName; /* 'f' for -f */ + char hasArg; /* enum value: option takes no/requires/may have argument */ + char doesOccur; /* boolean for "this one occured" */ +}; + +/* macro for an entry in a declaration of UOption[] */ +#define UOPTION_DEF(longName, shortName, hasArg) \ + { longName, NULL, NULL, NULL, shortName, hasArg, 0 } + +/* ICU Tools option definitions */ +#define UOPTION_HELP_H UOPTION_DEF("help", 'h', UOPT_NO_ARG) +#define UOPTION_HELP_QUESTION_MARK UOPTION_DEF("help", '?', UOPT_NO_ARG) +#define UOPTION_VERBOSE UOPTION_DEF("verbose", 'v', UOPT_NO_ARG) +#define UOPTION_QUIET UOPTION_DEF("quiet", 'q', UOPT_NO_ARG) +#define UOPTION_VERSION UOPTION_DEF("version", 'V', UOPT_NO_ARG) +#define UOPTION_COPYRIGHT UOPTION_DEF("copyright", 'c', UOPT_NO_ARG) + +#define UOPTION_DESTDIR UOPTION_DEF("destdir", 'd', UOPT_REQUIRES_ARG) +#define UOPTION_SOURCEDIR UOPTION_DEF("sourcedir", 's', UOPT_REQUIRES_ARG) +#define UOPTION_ENCODING UOPTION_DEF("encoding", 'e', UOPT_REQUIRES_ARG) +#define UOPTION_ICUDATADIR UOPTION_DEF("icudatadir", 'i', UOPT_REQUIRES_ARG) +#define UOPTION_WRITE_JAVA UOPTION_DEF("write-java", 'j', UOPT_OPTIONAL_ARG) +#define UOPTION_PACKAGE_NAME UOPTION_DEF("package-name", 'p', UOPT_REQUIRES_ARG) +#define UOPTION_BUNDLE_NAME UOPTION_DEF("bundle-name", 'b', UOPT_REQUIRES_ARG) + +/** + * C Command line argument parser. + * + * This function takes the argv[argc] command line and a description of + * the program's options in form of an array of UOption structures. + * Each UOption defines a long and a short name (a string and a character) + * for options like "--foo" and "-f". + * + * Each option is marked with whether it does not take an argument, + * requires one, or optionally takes one. The argument may follow in + * the same argv[] entry for short options, or it may always follow + * in the next argv[] entry. + * + * An argument is in the next argv[] entry for both long and short name + * options, except it is taken from directly behind the short name in + * its own argv[] entry if there are characters following the option letter. + * An argument in its own argv[] entry must not begin with a '-' + * unless it is only the '-' itself. There is no restriction of the + * argument format if it is part of the short name options's argv[] entry. + * + * The argument is stored in the value field of the corresponding + * UOption entry, and the doesOccur field is set to 1 if the option + * is found at all. + * + * Short name options without arguments can be collapsed into a single + * argv[] entry. After an option letter takes an argument, following + * letters will be taken as its argument. + * + * If the same option is found several times, then the last + * argument value will be stored in the value field. + * + * For each option, a function can be called. This could be used + * for options that occur multiple times and all arguments are to + * be collected. + * + * All options are removed from the argv[] array itself. If the parser + * is successful, then it returns the number of remaining non-option + * strings (including argv[0]). + * argv[0], the program name, is never read or modified. + * + * An option "--" ends option processing; everything after this + * remains in the argv[] array. + * + * An option string "-" alone is treated as a non-option. + * + * If an option is not recognized or an argument missing, then + * the parser returns with the negative index of the argv[] entry + * where the error was detected. + * + * The OS/400 compiler requires that argv either be "char* argv[]", + * or "const char* const argv[]", and it will not accept, + * "const char* argv[]" as a definition for main(). + * + * @param argv This parameter is modified + * @param options This parameter is modified + */ +U_CAPI int U_EXPORT2 +u_parseArgs(int argc, char* argv[], + int optionCount, UOption options[]); + +#endif diff --git a/Build/source/libs/icu/icu-xetex/tools/toolutil/uparse.c b/Build/source/libs/icu/icu-xetex/tools/toolutil/uparse.c new file mode 100644 index 00000000000..ed2f1e90005 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/tools/toolutil/uparse.c @@ -0,0 +1,370 @@ +/* +******************************************************************************* +* +* Copyright (C) 2000-2007, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: uparse.c +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2000apr18 +* created by: Markus W. Scherer +* +* This file provides a parser for files that are delimited by one single +* character like ';' or TAB. Example: the Unicode Character Properties files +* like UnicodeData.txt are semicolon-delimited. +*/ + +#include "unicode/utypes.h" +#include "cstring.h" +#include "filestrm.h" +#include "uparse.h" +#include "unicode/uchar.h" +#include "unicode/ustring.h" +#include "ustr_imp.h" + +#include <stdio.h> + +U_CAPI const char * U_EXPORT2 +u_skipWhitespace(const char *s) { + while(*s==' ' || *s=='\t') { + ++s; + } + return s; +} + +/* + * If the string starts with # @missing: then return the pointer to the + * following non-whitespace character. + * Otherwise return the original pointer. + * Unicode 5.0 adds such lines in some data files to document + * default property values. + * Poor man's regex for variable amounts of white space. + */ +static const char * +getMissingLimit(const char *s) { + const char *s0=s; + if( + *(s=u_skipWhitespace(s))=='#' && + *(s=u_skipWhitespace(s+1))=='@' && + 0==strncmp((s=u_skipWhitespace(s+1)), "missing", 7) && + *(s=u_skipWhitespace(s+7))==':' + ) { + return u_skipWhitespace(s+1); + } else { + return s0; + } +} + +U_CAPI void U_EXPORT2 +u_parseDelimitedFile(const char *filename, char delimiter, + char *fields[][2], int32_t fieldCount, + UParseLineFn *lineFn, void *context, + UErrorCode *pErrorCode) { + FileStream *file; + char line[300]; + char *start, *limit; + int32_t i, length; + + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return; + } + + if(fields==NULL || lineFn==NULL || fieldCount<=0) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return; + } + + if(filename==NULL || *filename==0 || (*filename=='-' && filename[1]==0)) { + filename=NULL; + file=T_FileStream_stdin(); + } else { + file=T_FileStream_open(filename, "r"); + } + if(file==NULL) { + *pErrorCode=U_FILE_ACCESS_ERROR; + return; + } + + while(T_FileStream_readLine(file, line, sizeof(line))!=NULL) { + length=(int32_t)uprv_strlen(line); + + /* remove trailing newline characters */ + while(length>0 && (line[length-1]=='\r' || line[length-1]=='\n')) { + line[--length]=0; + } + + /* + * detect a line with # @missing: + * start parsing after that, or else from the beginning of the line + * set the default warning for @missing lines + */ + start=(char *)getMissingLimit(line); + if(start==line) { + *pErrorCode=U_ZERO_ERROR; + } else { + *pErrorCode=U_USING_DEFAULT_WARNING; + } + + /* skip this line if it is empty or a comment */ + if(*start==0 || *start=='#') { + continue; + } + + /* remove in-line comments */ + limit=uprv_strchr(start, '#'); + if(limit!=NULL) { + /* get white space before the pound sign */ + while(limit>start && (*(limit-1)==' ' || *(limit-1)=='\t')) { + --limit; + } + + /* truncate the line */ + *limit=0; + } + + /* skip lines with only whitespace */ + if(u_skipWhitespace(start)[0]==0) { + continue; + } + + /* for each field, call the corresponding field function */ + for(i=0; i<fieldCount; ++i) { + /* set the limit pointer of this field */ + limit=start; + while(*limit!=delimiter && *limit!=0) { + ++limit; + } + + /* set the field start and limit in the fields array */ + fields[i][0]=start; + fields[i][1]=limit; + + /* set start to the beginning of the next field, if any */ + start=limit; + if(*start!=0) { + ++start; + } else if(i+1<fieldCount) { + *pErrorCode=U_PARSE_ERROR; + limit=line+length; + i=fieldCount; + break; + } + } + + /* error in a field function? */ + if(U_FAILURE(*pErrorCode)) { + break; + } + + /* call the field function */ + lineFn(context, fields, fieldCount, pErrorCode); + if(U_FAILURE(*pErrorCode)) { + break; + } + } + + if(filename!=NULL) { + T_FileStream_close(file); + } +} + +/* + * parse a list of code points + * store them as a UTF-32 string in dest[destCapacity] + * return the number of code points + */ +U_CAPI int32_t U_EXPORT2 +u_parseCodePoints(const char *s, + uint32_t *dest, int32_t destCapacity, + UErrorCode *pErrorCode) { + char *end; + uint32_t value; + int32_t count; + + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return 0; + } + if(s==NULL || destCapacity<0 || (destCapacity>0 && dest==NULL)) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + count=0; + for(;;) { + s=u_skipWhitespace(s); + if(*s==';' || *s==0) { + return count; + } + + /* read one code point */ + value=(uint32_t)uprv_strtoul(s, &end, 16); + if(end<=s || (*end!=' ' && *end!='\t' && *end!=';' && *end!=0) || value>=0x110000) { + *pErrorCode=U_PARSE_ERROR; + return 0; + } + + /* append it to the destination array */ + if(count<destCapacity) { + dest[count++]=value; + } else { + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + } + + /* go to the following characters */ + s=end; + } +} + +/* + * parse a list of code points + * store them as a string in dest[destCapacity] + * set the first code point in *pFirst + * @return The length of the string in numbers of UChars. + */ +U_CAPI int32_t U_EXPORT2 +u_parseString(const char *s, + UChar *dest, int32_t destCapacity, + uint32_t *pFirst, + UErrorCode *pErrorCode) { + char *end; + uint32_t value; + int32_t destLength; + + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return 0; + } + if(s==NULL || destCapacity<0 || (destCapacity>0 && dest==NULL)) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + } + + if(pFirst!=NULL) { + *pFirst=0xffffffff; + } + + destLength=0; + for(;;) { + s=u_skipWhitespace(s); + if(*s==';' || *s==0) { + if(destLength<destCapacity) { + dest[destLength]=0; + } else if(destLength==destCapacity) { + *pErrorCode=U_STRING_NOT_TERMINATED_WARNING; + } else { + *pErrorCode=U_BUFFER_OVERFLOW_ERROR; + } + return destLength; + } + + /* read one code point */ + value=(uint32_t)uprv_strtoul(s, &end, 16); + if(end<=s || (*end!=' ' && *end!='\t' && *end!=';' && *end!=0) || value>=0x110000) { + *pErrorCode=U_PARSE_ERROR; + return 0; + } + + /* store the first code point */ + if(destLength==0 && pFirst!=NULL) { + *pFirst=value; + } + + /* append it to the destination array */ + if((destLength+UTF_CHAR_LENGTH(value))<=destCapacity) { + UTF_APPEND_CHAR_UNSAFE(dest, destLength, value); + } else { + destLength+=UTF_CHAR_LENGTH(value); + } + + /* go to the following characters */ + s=end; + } +} + +/* read a range like start or start..end */ +U_CAPI int32_t U_EXPORT2 +u_parseCodePointRange(const char *s, + uint32_t *pStart, uint32_t *pEnd, + UErrorCode *pErrorCode) { + char *end; + uint32_t value; + + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return 0; + } + if(s==NULL || pStart==NULL || pEnd==NULL) { + *pErrorCode=U_ILLEGAL_ARGUMENT_ERROR; + return 0; + } + + s=u_skipWhitespace(s); + if(*s==';' || *s==0) { + *pErrorCode=U_PARSE_ERROR; + return 0; + } + + /* read the start code point */ + value=(uint32_t)uprv_strtoul(s, &end, 16); + if(end<=s || (*end!=' ' && *end!='\t' && *end!='.' && *end!=';') || value>=0x110000) { + *pErrorCode=U_PARSE_ERROR; + return 0; + } + *pStart=*pEnd=value; + + /* is there a "..end"? */ + s=u_skipWhitespace(end); + if(*s==';' || *s==0) { + return 1; + } + + if(*s!='.' || s[1]!='.') { + *pErrorCode=U_PARSE_ERROR; + return 0; + } + s+=2; + + /* read the end code point */ + value=(uint32_t)uprv_strtoul(s, &end, 16); + if(end<=s || (*end!=' ' && *end!='\t' && *end!=';') || value>=0x110000) { + *pErrorCode=U_PARSE_ERROR; + return 0; + } + *pEnd=value; + + /* is this a valid range? */ + if(value<*pStart) { + *pErrorCode=U_PARSE_ERROR; + return 0; + } + + /* no garbage after that? */ + s=u_skipWhitespace(end); + if(*s==';' || *s==0) { + return value-*pStart+1; + } else { + *pErrorCode=U_PARSE_ERROR; + return 0; + } +} + +U_CAPI int32_t U_EXPORT2 +u_parseUTF8(const char *source, int32_t sLen, char *dest, int32_t destCapacity, UErrorCode *status) { + const char *read = source; + int32_t i = 0; + unsigned int value = 0; + if(sLen == -1) { + sLen = (int32_t)strlen(source); + } + + while(read < source+sLen) { + sscanf(read, "%2x", &value); + if(i < destCapacity) { + dest[i] = (char)value; + } + i++; + read += 2; + } + return u_terminateChars(dest, destCapacity, i, status); +} diff --git a/Build/source/libs/icu/icu-xetex/tools/toolutil/uparse.h b/Build/source/libs/icu/icu-xetex/tools/toolutil/uparse.h new file mode 100644 index 00000000000..dc3e0b8cbbd --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/tools/toolutil/uparse.h @@ -0,0 +1,126 @@ +/* +******************************************************************************* +* +* Copyright (C) 2000-2004, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: uparse.h +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2000apr18 +* created by: Markus W. Scherer +* +* This file provides a parser for files that are delimited by one single +* character like ';' or TAB. Example: the Unicode Character Properties files +* like UnicodeData.txt are semicolon-delimited. +*/ + +#ifndef __UPARSE_H__ +#define __UPARSE_H__ + +#include "unicode/utypes.h" + +U_CDECL_BEGIN + +/** + * Skip space ' ' and TAB '\t' characters. + * + * @param s Pointer to characters. + * @return Pointer to first character at or after s that is not a space or TAB. + */ +U_CAPI const char * U_EXPORT2 +u_skipWhitespace(const char *s); + +/** Function type for u_parseDelimitedFile(). */ +typedef void U_CALLCONV +UParseLineFn(void *context, + char *fields[][2], + int32_t fieldCount, + UErrorCode *pErrorCode); + +/** + * Parser for files that are similar to UnicodeData.txt: + * This function opens the file and reads it line by line. It skips empty lines + * and comment lines that start with a '#'. + * All other lines are separated into fields with one delimiter character + * (semicolon for Unicode Properties files) between two fields. The last field in + * a line does not need to be terminated with a delimiter. + * + * For each line, after segmenting it, a line function is called. + * It gets passed the array of field start and limit pointers that is + * passed into this parser and filled by it for each line. + * For each field i of the line, the start pointer in fields[i][0] + * points to the beginning of the field, while the limit pointer in fields[i][1] + * points behind the field, i.e., to the delimiter or the line end. + * + * The context parameter of the line function is + * the same as the one for the parse function. + * + * The line function may modify the contents of the fields including the + * limit characters. + * + * If the file cannot be opened, or there is a parsing error or a field function + * sets *pErrorCode, then the parser returns with *pErrorCode set to an error code. + */ +U_CAPI void U_EXPORT2 +u_parseDelimitedFile(const char *filename, char delimiter, + char *fields[][2], int32_t fieldCount, + UParseLineFn *lineFn, void *context, + UErrorCode *pErrorCode); + +/** + * Parse a string of code points like 0061 0308 0300. + * s must end with either ';' or NUL. + * + * @return Number of code points. + */ +U_CAPI int32_t U_EXPORT2 +u_parseCodePoints(const char *s, + uint32_t *dest, int32_t destCapacity, + UErrorCode *pErrorCode); + +/** + * Parse a list of code points like 0061 0308 0300 + * into a UChar * string. + * s must end with either ';' or NUL. + * + * Set the first code point in *pFirst. + * + * @param s Input char * string. + * @param dest Output string buffer. + * @param destCapacity Capacity of dest in numbers of UChars. + * @param pFirst If pFirst!=NULL the *pFirst will be set to the first + * code point in the string. + * @param pErrorCode ICU error code. + * @return The length of the string in numbers of UChars. + */ +U_CAPI int32_t U_EXPORT2 +u_parseString(const char *s, + UChar *dest, int32_t destCapacity, + uint32_t *pFirst, + UErrorCode *pErrorCode); + +/** + * Parse a code point range like + * 0085 or + * 4E00..9FA5. + * + * s must contain such a range and end with either ';' or NUL. + * + * @return Length of code point range, end-start+1 + */ +U_CAPI int32_t U_EXPORT2 +u_parseCodePointRange(const char *s, + uint32_t *pStart, uint32_t *pEnd, + UErrorCode *pErrorCode); + + +U_CAPI int32_t U_EXPORT2 +u_parseUTF8(const char *source, int32_t sLen, char *dest, int32_t destCapacity, UErrorCode *status); + +U_CDECL_END + +#endif diff --git a/Build/source/libs/icu/icu-xetex/tools/toolutil/writesrc.c b/Build/source/libs/icu/icu-xetex/tools/toolutil/writesrc.c new file mode 100644 index 00000000000..28303c733e5 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/tools/toolutil/writesrc.c @@ -0,0 +1,190 @@ +/* +******************************************************************************* +* +* Copyright (C) 2005-2007, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: writesrc.c +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2005apr23 +* created by: Markus W. Scherer +* +* Helper functions for writing source code for data. +*/ + +#include <stdio.h> +#include <time.h> +#include "unicode/utypes.h" +#include "unicode/putil.h" +#include "utrie.h" +#include "cstring.h" +#include "writesrc.h" + +U_CAPI FILE * U_EXPORT2 +usrc_create(const char *path, const char *filename) { + char buffer[1024]; + const char *p; + char *q; + FILE *f; + char c; + + if(path==NULL) { + p=filename; + } else { + /* concatenate path and filename, with U_FILE_SEP_CHAR in between if necessary */ + uprv_strcpy(buffer, path); + q=buffer+uprv_strlen(buffer); + if(q>buffer && (c=*(q-1))!=U_FILE_SEP_CHAR && c!=U_FILE_ALT_SEP_CHAR) { + *q++=U_FILE_SEP_CHAR; + } + uprv_strcpy(q, filename); + p=buffer; + } + + f=fopen(p, "w"); + if(f!=NULL) { + char year[8]; + const struct tm *lt; + time_t t; + + time(&t); + lt=localtime(&t); + strftime(year, sizeof(year), "%Y", lt); + strftime(buffer, sizeof(buffer), "%Y-%m-%d", lt); + fprintf( + f, + "/*\n" + " * Copyright (C) 1999-%s, International Business Machines\n" + " * Corporation and others. All Rights Reserved.\n" + " *\n" + " * file name: %s\n" + " *\n" + " * machine-generated on: %s\n" + " */\n\n", + year, + filename, + buffer); + } else { + fprintf( + stderr, + "usrc_create(%s, %s): unable to create file\n", + path!=NULL ? path : "", filename); + } + return f; +} + +U_CAPI void U_EXPORT2 +usrc_writeArray(FILE *f, + const char *prefix, + const void *p, int32_t width, int32_t length, + const char *postfix) { + const uint8_t *p8; + const uint16_t *p16; + const uint32_t *p32; + uint32_t value; + int32_t i, col; + + p8=NULL; + p16=NULL; + p32=NULL; + switch(width) { + case 8: + p8=(const uint8_t *)p; + break; + case 16: + p16=(const uint16_t *)p; + break; + case 32: + p32=(const uint32_t *)p; + break; + default: + fprintf(stderr, "usrc_writeArray(width=%ld) unrecognized width\n", (long)width); + return; + } + if(prefix!=NULL) { + fprintf(f, prefix, (long)length); + } + for(i=col=0; i<length; ++i, ++col) { + if(i>0) { + if(col<16) { + fputc(',', f); + } else { + fputs(",\n", f); + col=0; + } + } + switch(width) { + case 8: + value=p8[i]; + break; + case 16: + value=p16[i]; + break; + case 32: + value=p32[i]; + break; + default: + value=0; /* unreachable */ + break; + } + fprintf(f, value<=9 ? "%lu" : "0x%lx", (unsigned long)value); + } + if(postfix!=NULL) { + fputs(postfix, f); + } +} + +U_CAPI void U_EXPORT2 +usrc_writeUTrieArrays(FILE *f, + const char *indexPrefix, const char *dataPrefix, + const UTrie *pTrie, + const char *postfix) { + if(pTrie->data32==NULL) { + /* 16-bit trie */ + usrc_writeArray(f, indexPrefix, pTrie->index, 16, pTrie->indexLength+pTrie->dataLength, postfix); + } else { + /* 32-bit trie */ + usrc_writeArray(f, indexPrefix, pTrie->index, 16, pTrie->indexLength, postfix); + usrc_writeArray(f, dataPrefix, pTrie->data32, 32, pTrie->dataLength, postfix); + } +} + +U_CAPI void U_EXPORT2 +usrc_writeUTrieStruct(FILE *f, + const char *prefix, + const UTrie *pTrie, + const char *indexName, const char *dataName, + const char *getFoldingOffsetName, + const char *postfix) { + if(prefix!=NULL) { + fputs(prefix, f); + } + if(dataName==NULL) { + dataName="NULL"; + } + if(getFoldingOffsetName==NULL) { + getFoldingOffsetName="utrie_defaultGetFoldingOffset"; + } + fprintf( + f, + " %s,\n" + " %s,\n" + " %s,\n" + " %ld,\n" + " %ld,\n" + " %lu,\n" + " %s\n", + indexName, + dataName, + getFoldingOffsetName, + (long)pTrie->indexLength, (long)pTrie->dataLength, + (unsigned long)pTrie->initialValue, + pTrie->isLatin1Linear ? "TRUE" : "FALSE"); + if(postfix!=NULL) { + fputs(postfix, f); + } +} diff --git a/Build/source/libs/icu/icu-xetex/tools/toolutil/writesrc.h b/Build/source/libs/icu/icu-xetex/tools/toolutil/writesrc.h new file mode 100644 index 00000000000..613ee9d7074 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/tools/toolutil/writesrc.h @@ -0,0 +1,70 @@ +/* +******************************************************************************* +* +* Copyright (C) 2005, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: writesrc.h +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2005apr23 +* created by: Markus W. Scherer +* +* Helper functions for writing source code for data. +*/ + +#ifndef __WRITESRC_H__ +#define __WRITESRC_H__ + +#include <stdio.h> +#include "unicode/utypes.h" +#include "utrie.h" + +/** + * Create a source text file and write a header comment with the ICU copyright. + */ +U_CAPI FILE * U_EXPORT2 +usrc_create(const char *path, const char *filename); + +/** + * Write the contents of an array of 8/16/32-bit words. + * The prefix and postfix are optional (can be NULL) and are written first/last. + * The prefix may contain a %ld or similar field for the array length. + * The {} and declaration etc. need to be included in prefix/postfix or + * printed before and after the array contents. + */ +U_CAPI void U_EXPORT2 +usrc_writeArray(FILE *f, + const char *prefix, + const void *p, int32_t width, int32_t length, + const char *postfix); + +/** + * Calls usrc_writeArray() for the index and data arrays of a runtime UTrie. + * Only the index array is written for a 16-bit UTrie. In this case, dataPrefix + * is ignored and can be NULL. + */ +U_CAPI void U_EXPORT2 +usrc_writeUTrieArrays(FILE *f, + const char *indexPrefix, const char *dataPrefix, + const UTrie *pTrie, + const char *postfix); + +/** + * Writes the UTrie struct values. + * The {} and declaration etc. need to be included in prefix/postfix or + * printed before and after the array contents. + * If getFoldingOffsetName==NULL then "utrie_defaultGetFoldingOffset" is printed. + */ +U_CAPI void U_EXPORT2 +usrc_writeUTrieStruct(FILE *f, + const char *prefix, + const UTrie *pTrie, + const char *indexName, const char *dataName, + const char *getFoldingOffsetName, + const char *postfix); + +#endif diff --git a/Build/source/libs/icu/icu-xetex/tools/toolutil/xmlparser.cpp b/Build/source/libs/icu/icu-xetex/tools/toolutil/xmlparser.cpp new file mode 100644 index 00000000000..195fa30385c --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/tools/toolutil/xmlparser.cpp @@ -0,0 +1,820 @@ +/* +******************************************************************************* +* +* Copyright (C) 2004-2006, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: xmlparser.cpp +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2004jul21 +* created by: Andy Heninger +*/ + +#include <stdio.h> +#include "unicode/uchar.h" +#include "unicode/ucnv.h" +#include "unicode/regex.h" +#include "filestrm.h" +#include "xmlparser.h" + +#if !UCONFIG_NO_REGULAR_EXPRESSIONS && !UCONFIG_NO_CONVERSION + +// character constants +enum { + x_QUOT=0x22, + x_AMP=0x26, + x_APOS=0x27, + x_LT=0x3c, + x_GT=0x3e, + x_l=0x6c +}; + +#define XML_SPACES "[ \\u0009\\u000d\\u000a]" + +// XML #4 +#define XML_NAMESTARTCHAR "[[A-Z]:_[a-z][\\u00c0-\\u00d6][\\u00d8-\\u00f6]" \ + "[\\u00f8-\\u02ff][\\u0370-\\u037d][\\u037F-\\u1FFF][\\u200C-\\u200D]" \ + "[\\u2070-\\u218F][\\u2C00-\\u2FEF][\\u3001-\\uD7FF][\\uF900-\\uFDCF]" \ + "[\\uFDF0-\\uFFFD][\\U00010000-\\U000EFFFF]]" + +// XML #5 +#define XML_NAMECHAR "[" XML_NAMESTARTCHAR "\\-.[0-9]\\u00b7[\\u0300-\\u036f][\\u203f-\\u2040]]" + +// XML #6 +#define XML_NAME XML_NAMESTARTCHAR "(?:" XML_NAMECHAR ")*" + +U_NAMESPACE_BEGIN + +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(UXMLParser) +UOBJECT_DEFINE_RTTI_IMPLEMENTATION(UXMLElement) + +// +// UXMLParser constructor. Mostly just initializes the ICU regexes that are +// used for parsing. +// +UXMLParser::UXMLParser(UErrorCode &status) : + // XML Declaration. XML Production #23. + // example: "<?xml version=1.0 encoding="utf-16" ?> + // This is a sloppy implementation - just look for the leading <?xml and the closing ?> + // allow for a possible leading BOM. + mXMLDecl(UnicodeString("(?s)\\uFEFF?<\\?xml.+?\\?>"), 0, status), + + // XML Comment production #15 + // example: "<!-- whatever --> + // note, does not detect an illegal "--" within comments + mXMLComment(UnicodeString("(?s)<!--.+?-->"), 0, status), + + // XML Spaces + // production [3] + mXMLSP(UnicodeString(XML_SPACES "+"), 0, status), + + // XML Doctype decl production #28 + // example "<!DOCTYPE foo SYSTEM "somewhere" > + // TODO: we don't actually parse the DOCTYPE or internal subsets. + // Some internal dtd subsets could confuse this simple-minded + // attempt at skipping over them. + mXMLDoctype(UnicodeString("(?s)<!DOCTYPE.+?>"), 0, status), + + // XML PI production #16 + // example "<?target stuff?> + mXMLPI(UnicodeString("(?s)<\\?.+?\\?>"), 0, status), + + // XML Element Start Productions #40, #41 + // example <foo att1='abc' att2="d e f" > + // capture #1: the tag name + // + mXMLElemStart (UnicodeString("(?s)<(" XML_NAME ")" // match "<tag_name" + "(?:" + XML_SPACES "+" XML_NAME XML_SPACES "*=" XML_SPACES "*" // match "ATTR_NAME = " + "(?:(?:\\\'[^<\\\']*?\\\')|(?:\\\"[^<\\\"]*?\\\"))" // match '"attribute value"' + ")*" // * for zero or more attributes. + XML_SPACES "*?>"), 0, status), // match " >" + + // XML Element End production #42 + // example </foo> + mXMLElemEnd (UnicodeString("</(" XML_NAME ")" XML_SPACES "*>"), 0, status), + + // XML Element Empty production #44 + // example <foo att1="abc" att2="d e f" /> + mXMLElemEmpty (UnicodeString("(?s)<(" XML_NAME ")" // match "<tag_name" + "(?:" + XML_SPACES "+" XML_NAME XML_SPACES "*=" XML_SPACES "*" // match "ATTR_NAME = " + "(?:(?:\\\'[^<\\\']*?\\\')|(?:\\\"[^<\\\"]*?\\\"))" // match '"attribute value"' + ")*" // * for zero or more attributes. + XML_SPACES "*?/>"), 0, status), // match " />" + + + // XMLCharData. Everything but '<'. Note that & will be dealt with later. + mXMLCharData(UnicodeString("(?s)[^<]*"), 0, status), + + // Attribute name = "value". XML Productions 10, 40/41 + // Capture group 1 is name, + // 2 is the attribute value, including the quotes. + // + // Note that attributes are scanned twice. The first time is with + // the regex for an entire element start. There, the attributes + // are checked syntactically, but not separted out one by one. + // Here, we match a single attribute, and make its name and + // attribute value available to the parser code. + mAttrValue(UnicodeString(XML_SPACES "+(" XML_NAME ")" XML_SPACES "*=" XML_SPACES "*" + "((?:\\\'[^<\\\']*?\\\')|(?:\\\"[^<\\\"]*?\\\"))"), 0, status), + + + mAttrNormalizer(UnicodeString(XML_SPACES), 0, status), + + // Match any of the new-line sequences in content. + // All are changed to \u000a. + mNewLineNormalizer(UnicodeString("\\u000d\\u000a|\\u000d\\u0085|\\u000a|\\u000d|\\u0085|\\u2028"), 0, status), + + // & char references + // We will figure out what we've got based on which capture group has content. + // The last one is a catchall for unrecognized entity references.. + // 1 2 3 4 5 6 7 8 + mAmps(UnicodeString("&(?:(amp;)|(lt;)|(gt;)|(apos;)|(quot;)|#x([0-9A-Fa-f]{1,8});|#([0-9]{1,8});|(.))"), + 0, status), + + fNames(status), + fElementStack(status), + fOneLF((UChar)0x0a) // Plain new-line string, used in new line normalization. + { + } + +UXMLParser * +UXMLParser::createParser(UErrorCode &errorCode) { + if (U_FAILURE(errorCode)) { + return NULL; + } else { + return new UXMLParser(errorCode); + } +} + +UXMLParser::~UXMLParser() {} + +UXMLElement * +UXMLParser::parseFile(const char *filename, UErrorCode &errorCode) { + char bytes[4096], charsetBuffer[100]; + FileStream *f; + const char *charset, *pb; + UnicodeString src; + UConverter *cnv; + UChar *buffer, *pu; + int32_t fileLength, bytesLength, length, capacity; + UBool flush; + + if(U_FAILURE(errorCode)) { + return NULL; + } + + f=T_FileStream_open(filename, "rb"); + if(f==NULL) { + errorCode=U_FILE_ACCESS_ERROR; + return NULL; + } + + bytesLength=T_FileStream_read(f, bytes, (int32_t)sizeof(bytes)); + if(bytesLength<(int32_t)sizeof(bytes)) { + // we have already read the entire file + fileLength=bytesLength; + } else { + // get the file length + fileLength=T_FileStream_size(f); + } + + /* + * get the charset: + * 1. Unicode signature + * 2. treat as ISO-8859-1 and read XML encoding="charser" + * 3. default to UTF-8 + */ + charset=ucnv_detectUnicodeSignature(bytes, bytesLength, NULL, &errorCode); + if(U_SUCCESS(errorCode) && charset!=NULL) { + // open converter according to Unicode signature + cnv=ucnv_open(charset, &errorCode); + } else { + // read as Latin-1 and parse the XML declaration and encoding + cnv=ucnv_open("ISO-8859-1", &errorCode); + if(U_FAILURE(errorCode)) { + // unexpected error opening Latin-1 converter + goto exit; + } + + buffer=src.getBuffer(bytesLength); + if(buffer==NULL) { + // unexpected failure to reserve some string capacity + errorCode=U_MEMORY_ALLOCATION_ERROR; + goto exit; + } + pb=bytes; + pu=buffer; + ucnv_toUnicode( + cnv, + &pu, buffer+src.getCapacity(), + &pb, bytes+bytesLength, + NULL, TRUE, &errorCode); + src.releaseBuffer(U_SUCCESS(errorCode) ? (int32_t)(pu-buffer) : 0); + ucnv_close(cnv); + cnv=NULL; + if(U_FAILURE(errorCode)) { + // unexpected error in conversion from Latin-1 + src.remove(); + goto exit; + } + + // parse XML declaration + if(mXMLDecl.reset(src).lookingAt(0, errorCode)) { + int32_t declEnd=mXMLDecl.end(errorCode); + // go beyond <?xml + int32_t pos=src.indexOf((UChar)x_l)+1; + + mAttrValue.reset(src); + while(pos<declEnd && mAttrValue.lookingAt(pos, errorCode)) { // loop runs once per attribute on this element. + UnicodeString attName = mAttrValue.group(1, errorCode); + UnicodeString attValue = mAttrValue.group(2, errorCode); + + // Trim the quotes from the att value. These are left over from the original regex + // that parsed the attribue, which couldn't conveniently strip them. + attValue.remove(0,1); // one char from the beginning + attValue.truncate(attValue.length()-1); // and one from the end. + + if(attName==UNICODE_STRING("encoding", 8)) { + length=attValue.extract(0, 0x7fffffff, charsetBuffer, (int32_t)sizeof(charsetBuffer)); + charset=charsetBuffer; + break; + } + pos = mAttrValue.end(2, errorCode); + } + + if(charset==NULL) { + // default to UTF-8 + charset="UTF-8"; + } + cnv=ucnv_open(charset, &errorCode); + } + } + + if(U_FAILURE(errorCode)) { + // unable to open the converter + goto exit; + } + + // convert the file contents + capacity=fileLength; // estimated capacity + src.getBuffer(capacity); + src.releaseBuffer(0); // zero length + flush=FALSE; + for(;;) { + // convert contents of bytes[bytesLength] + pb=bytes; + for(;;) { + length=src.length(); + buffer=src.getBuffer(capacity); + if(buffer==NULL) { + // unexpected failure to reserve some string capacity + errorCode=U_MEMORY_ALLOCATION_ERROR; + goto exit; + } + + pu=buffer+length; + ucnv_toUnicode( + cnv, &pu, buffer+src.getCapacity(), + &pb, bytes+bytesLength, + NULL, FALSE, &errorCode); + src.releaseBuffer(U_SUCCESS(errorCode) ? (int32_t)(pu-buffer) : 0); + if(errorCode==U_BUFFER_OVERFLOW_ERROR) { + errorCode=U_ZERO_ERROR; + capacity=(3*src.getCapacity())/2; // increase capacity by 50% + } else { + break; + } + } + + if(U_FAILURE(errorCode)) { + break; // conversion error + } + + if(flush) { + break; // completely converted the file + } + + // read next block + bytesLength=T_FileStream_read(f, bytes, (int32_t)sizeof(bytes)); + if(bytesLength==0) { + // reached end of file, convert once more to flush the converter + flush=TRUE; + } + }; + +exit: + ucnv_close(cnv); + T_FileStream_close(f); + + if(U_SUCCESS(errorCode)) { + return parse(src, errorCode); + } else { + return NULL; + } +} + +UXMLElement * +UXMLParser::parse(const UnicodeString &src, UErrorCode &status) { + if(U_FAILURE(status)) { + return NULL; + } + + UXMLElement *root = NULL; + fPos = 0; // TODO use just a local pos variable and pass it into functions + // where necessary? + + // set all matchers to work on the input string + mXMLDecl.reset(src); + mXMLComment.reset(src); + mXMLSP.reset(src); + mXMLDoctype.reset(src); + mXMLPI.reset(src); + mXMLElemStart.reset(src); + mXMLElemEnd.reset(src); + mXMLElemEmpty.reset(src); + mXMLCharData.reset(src); + mAttrValue.reset(src); + mAttrNormalizer.reset(src); + mNewLineNormalizer.reset(src); + mAmps.reset(src); + + // Consume the XML Declaration, if present. + if (mXMLDecl.lookingAt(fPos, status)) { + fPos = mXMLDecl.end(status); + } + + // Consume "misc" [XML production 27] appearing before DocType + parseMisc(status); + + // Consume a DocType declaration, if present. + if (mXMLDoctype.lookingAt(fPos, status)) { + fPos = mXMLDoctype.end(status); + } + + // Consume additional "misc" [XML production 27] appearing after the DocType + parseMisc(status); + + // Get the root element + if (mXMLElemEmpty.lookingAt(fPos, status)) { + // Root is an empty element (no nested elements or content) + root = createElement(mXMLElemEmpty, status); + fPos = mXMLElemEmpty.end(status); + } else { + if (mXMLElemStart.lookingAt(fPos, status) == FALSE) { + error("Root Element expected", status); + goto errorExit; + } + root = createElement(mXMLElemStart, status); + UXMLElement *el = root; + + // + // This is the loop that consumes the root element of the document, + // including all nested content. Nested elements are handled by + // explicit pushes/pops of the element stack; there is no recursion + // in the control flow of this code. + // "el" always refers to the current element, the one to which content + // is being added. It is above the top of the element stack. + for (;;) { + // Nested Element Start + if (mXMLElemStart.lookingAt(fPos, status)) { + UXMLElement *t = createElement(mXMLElemStart, status); + el->fChildren.addElement(t, status); + t->fParent = el; + fElementStack.push(el, status); + el = t; + continue; + } + + // Text Content. String is concatenated onto the current node's content, + // but only if it contains something other than spaces. + UnicodeString s = scanContent(status); + if (s.length() > 0) { + mXMLSP.reset(s); + if (mXMLSP.matches(status) == FALSE) { + // This chunk of text contains something other than just + // white space. Make a child node for it. + replaceCharRefs(s, status); + el->fChildren.addElement(s.clone(), status); + } + mXMLSP.reset(src); // The matchers need to stay set to the main input string. + continue; + } + + // Comments. Discard. + if (mXMLComment.lookingAt(fPos, status)) { + fPos = mXMLComment.end(status); + continue; + } + + // PIs. Discard. + if (mXMLPI.lookingAt(fPos, status)) { + fPos = mXMLPI.end(status); + continue; + } + + // Element End + if (mXMLElemEnd.lookingAt(fPos, status)) { + fPos = mXMLElemEnd.end(0, status); + const UnicodeString name = mXMLElemEnd.group(1, status); + if (name != *el->fName) { + error("Element start / end tag mismatch", status); + goto errorExit; + } + if (fElementStack.empty()) { + // Close of the root element. We're done with the doc. + el = NULL; + break; + } + el = (UXMLElement *)fElementStack.pop(); + continue; + } + + // Empty Element. Stored as a child of the current element, but not stacked. + if (mXMLElemEmpty.lookingAt(fPos, status)) { + UXMLElement *t = createElement(mXMLElemEmpty, status); + el->fChildren.addElement(t, status); + continue; + } + + // Hit something within the document that doesn't match anything. + // It's an error. + error("Unrecognized markup", status); + break; + } + + if (el != NULL || !fElementStack.empty()) { + // We bailed out early, for some reason. + error("Root element not closed.", status); + goto errorExit; + } + } + + // Root Element parse is complete. + // Consume the annoying xml "Misc" that can appear at the end of the doc. + parseMisc(status); + + // We should have reached the end of the input + if (fPos != src.length()) { + error("Extra content at the end of the document", status); + goto errorExit; + } + + // Success! + return root; + +errorExit: + delete root; + return NULL; +} + +// +// createElement +// We've just matched an element start tag. Create and fill in a UXMLElement object +// for it. +// +UXMLElement * +UXMLParser::createElement(RegexMatcher &mEl, UErrorCode &status) { + // First capture group is the element's name. + UXMLElement *el = new UXMLElement(this, intern(mEl.group(1, status), status), status); + + // Scan for attributes. + int32_t pos = mEl.end(1, status); // The position after the end of the tag name + + while (mAttrValue.lookingAt(pos, status)) { // loop runs once per attribute on this element. + UnicodeString attName = mAttrValue.group(1, status); + UnicodeString attValue = mAttrValue.group(2, status); + + // Trim the quotes from the att value. These are left over from the original regex + // that parsed the attribue, which couldn't conveniently strip them. + attValue.remove(0,1); // one char from the beginning + attValue.truncate(attValue.length()-1); // and one from the end. + + // XML Attribue value normalization. + // This is one of the really screwy parts of the XML spec. + // See http://www.w3.org/TR/2004/REC-xml11-20040204/#AVNormalize + // Note that non-validating parsers must treat all entities as type CDATA + // which simplifies things some. + + // Att normalization step 1: normalize any newlines in the attribute value + mNewLineNormalizer.reset(attValue); + attValue = mNewLineNormalizer.replaceAll(fOneLF, status); + + // Next change all xml white space chars to plain \u0020 spaces. + mAttrNormalizer.reset(attValue); + UnicodeString oneSpace((UChar)0x0020); + attValue = mAttrNormalizer.replaceAll(oneSpace, status); + + // Replace character entities. + replaceCharRefs(attValue, status); + + // Save the attribute name and value in our document structure. + el->fAttNames.addElement((void *)intern(attName, status), status); + el->fAttValues.addElement(attValue.clone(), status); + pos = mAttrValue.end(2, status); + } + fPos = mEl.end(0, status); + return el; +} + +// +// parseMisc +// Consume XML "Misc" [production #27] +// which is any combination of space, PI and comments +// Need to watch end-of-input because xml MISC stuff is allowed after +// the document element, so we WILL scan off the end in this function +// +void +UXMLParser::parseMisc(UErrorCode &status) { + for (;;) { + if (fPos >= mXMLPI.input().length()) { + break; + } + if (mXMLPI.lookingAt(fPos, status)) { + fPos = mXMLPI.end(status); + continue; + } + if (mXMLSP.lookingAt(fPos, status)) { + fPos = mXMLSP.end(status); + continue; + } + if (mXMLComment.lookingAt(fPos, status)) { + fPos = mXMLComment.end(status); + continue; + } + break; + } +} + +// +// Scan for document content. +// +UnicodeString +UXMLParser::scanContent(UErrorCode &status) { + UnicodeString result; + if (mXMLCharData.lookingAt(fPos, status)) { + result = mXMLCharData.group(0, status); + // Normalize the new-lines. (Before char ref substitution) + mNewLineNormalizer.reset(result); + result = mNewLineNormalizer.replaceAll(fOneLF, status); + + // TODO: handle CDATA + fPos = mXMLCharData.end(0, status); + } + + return result; +} + +// +// replaceCharRefs +// +// replace the char entities < & { ካ etc. in a string +// with the corresponding actual character. +// +void +UXMLParser::replaceCharRefs(UnicodeString &s, UErrorCode &status) { + UnicodeString result; + UnicodeString replacement; + int i; + + mAmps.reset(s); + // See the initialization for the regex matcher mAmps. + // Which entity we've matched is determined by which capture group has content, + // which is flaged by start() of that group not being -1. + while (mAmps.find()) { + if (mAmps.start(1, status) != -1) { + replacement.setTo((UChar)x_AMP); + } else if (mAmps.start(2, status) != -1) { + replacement.setTo((UChar)x_LT); + } else if (mAmps.start(3, status) != -1) { + replacement.setTo((UChar)x_GT); + } else if (mAmps.start(4, status) != -1) { + replacement.setTo((UChar)x_APOS); + } else if (mAmps.start(5, status) != -1) { + replacement.setTo((UChar)x_QUOT); + } else if (mAmps.start(6, status) != -1) { + UnicodeString hexString = mAmps.group(6, status); + UChar32 val = 0; + for (i=0; i<hexString.length(); i++) { + val = (val << 4) + u_digit(hexString.charAt(i), 16); + } + // TODO: some verification that the character is valid + replacement.setTo(val); + } else if (mAmps.start(7, status) != -1) { + UnicodeString decimalString = mAmps.group(7, status); + UChar32 val = 0; + for (i=0; i<decimalString.length(); i++) { + val = val*10 + u_digit(decimalString.charAt(i), 10); + } + // TODO: some verification that the character is valid + replacement.setTo(val); + } else { + // An unrecognized &entity; Leave it alone. + // TODO: check that it really looks like an entity, and is not some + // random & in the text. + replacement = mAmps.group(0, status); + } + mAmps.appendReplacement(result, replacement, status); + } + mAmps.appendTail(result); + s = result; +} + +void +UXMLParser::error(const char *message, UErrorCode &status) { + // TODO: something better here... + const UnicodeString &src=mXMLDecl.input(); + int line = 0; + int ci = 0; + while (ci < fPos && ci>=0) { + ci = src.indexOf((UChar)0x0a, ci+1); + line++; + } + fprintf(stderr, "Error: %s at line %d\n", message, line); + if (U_SUCCESS(status)) { + status = U_PARSE_ERROR; + } +} + +// intern strings like in Java + +const UnicodeString * +UXMLParser::intern(const UnicodeString &s, UErrorCode &errorCode) { + const UHashElement *he=fNames.find(s); + if(he!=NULL) { + // already a known name, return its hashed key pointer + return (const UnicodeString *)he->key.pointer; + } else { + // add this new name and return its hashed key pointer + fNames.puti(s, 0, errorCode); + he=fNames.find(s); + return (const UnicodeString *)he->key.pointer; + } +} + +const UnicodeString * +UXMLParser::findName(const UnicodeString &s) const { + const UHashElement *he=fNames.find(s); + if(he!=NULL) { + // a known name, return its hashed key pointer + return (const UnicodeString *)he->key.pointer; + } else { + // unknown name + return NULL; + } +} + +// UXMLElement ------------------------------------------------------------- *** + +UXMLElement::UXMLElement(const UXMLParser *parser, const UnicodeString *name, UErrorCode &errorCode) : + fParser(parser), + fName(name), + fAttNames(errorCode), + fAttValues(errorCode), + fChildren(errorCode), + fParent(NULL) +{ +} + +UXMLElement::~UXMLElement() { + int i; + // attribute names are owned by the UXMLParser, don't delete them here + for (i=fAttValues.size()-1; i>=0; i--) { + delete (UObject *)fAttValues.elementAt(i); + } + for (i=fChildren.size()-1; i>=0; i--) { + delete (UObject *)fChildren.elementAt(i); + } +} + +const UnicodeString & +UXMLElement::getTagName() const { + return *fName; +} + +UnicodeString +UXMLElement::getText(UBool recurse) const { + UnicodeString text; + appendText(text, recurse); + return text; +} + +void +UXMLElement::appendText(UnicodeString &text, UBool recurse) const { + const UObject *node; + int32_t i, count=fChildren.size(); + for(i=0; i<count; ++i) { + node=(const UObject *)fChildren.elementAt(i); + if(node->getDynamicClassID()==UnicodeString::getStaticClassID()) { + text.append(*(const UnicodeString *)node); + } else if(recurse) /* must be a UXMLElement */ { + ((const UXMLElement *)node)->appendText(text, recurse); + } + } +} + +int32_t +UXMLElement::countAttributes() const { + return fAttNames.size(); +} + +const UnicodeString * +UXMLElement::getAttribute(int32_t i, UnicodeString &name, UnicodeString &value) const { + if(0<=i && i<fAttNames.size()) { + name.setTo(*(const UnicodeString *)fAttNames.elementAt(i)); + value.setTo(*(const UnicodeString *)fAttValues.elementAt(i)); + return &value; // or return (UnicodeString *)fAttValues.elementAt(i); + } else { + return NULL; + } +} + +const UnicodeString * +UXMLElement::getAttribute(const UnicodeString &name) const { + // search for the attribute name by comparing the interned pointer, + // not the string contents + const UnicodeString *p=fParser->findName(name); + if(p==NULL) { + return NULL; // no such attribute seen by the parser at all + } + + int32_t i, count=fAttNames.size(); + for(i=0; i<count; ++i) { + if(p==(const UnicodeString *)fAttNames.elementAt(i)) { + return (const UnicodeString *)fAttValues.elementAt(i); + } + } + return NULL; +} + +int32_t +UXMLElement::countChildren() const { + return fChildren.size(); +} + +const UObject * +UXMLElement::getChild(int32_t i, UXMLNodeType &type) const { + if(0<=i && i<fChildren.size()) { + const UObject *node=(const UObject *)fChildren.elementAt(i); + if(node->getDynamicClassID()==UXMLElement::getStaticClassID()) { + type=UXML_NODE_TYPE_ELEMENT; + } else { + type=UXML_NODE_TYPE_STRING; + } + return node; + } else { + return NULL; + } +} + +const UXMLElement * +UXMLElement::nextChildElement(int32_t &i) const { + if(i<0) { + return NULL; + } + + const UObject *node; + int32_t count=fChildren.size(); + while(i<count) { + node=(const UObject *)fChildren.elementAt(i++); + // TODO: see if ICU can use C++ instanceof instead of its own poor man's RTTI + // if(node instanceof UXMLElement) { + if(node->getDynamicClassID()==UXMLElement::getStaticClassID()) { + return (const UXMLElement *)node; + } + } + return NULL; +} + +const UXMLElement * +UXMLElement::getChildElement(const UnicodeString &name) const { + // search for the element name by comparing the interned pointer, + // not the string contents + const UnicodeString *p=fParser->findName(name); + if(p==NULL) { + return NULL; // no such element seen by the parser at all + } + + const UObject *node; + int32_t i, count=fChildren.size(); + for(i=0; i<count; ++i) { + node=(const UObject *)fChildren.elementAt(i); + if(node->getDynamicClassID()==UXMLElement::getStaticClassID()) { + const UXMLElement *elem=(const UXMLElement *)node; + if(p==elem->fName) { + return elem; + } + } + } + return NULL; +} + +U_NAMESPACE_END + +#endif /* !UCONFIG_NO_REGULAR_EXPRESSIONS */ + diff --git a/Build/source/libs/icu/icu-xetex/tools/toolutil/xmlparser.h b/Build/source/libs/icu/icu-xetex/tools/toolutil/xmlparser.h new file mode 100644 index 00000000000..a619769be49 --- /dev/null +++ b/Build/source/libs/icu/icu-xetex/tools/toolutil/xmlparser.h @@ -0,0 +1,245 @@ +/* +******************************************************************************* +* +* Copyright (C) 2004-2005, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: xmlparser.h +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2004jul21 +* created by: Andy Heninger +* +* Tiny XML parser using ICU and intended for use in ICU tests and in build tools. +* Not suitable for production use. Not supported. +* Not conformant. Not efficient. +* But very small. +*/ + +#ifndef __XMLPARSER_H__ +#define __XMLPARSER_H__ + +#include "unicode/uobject.h" +#include "unicode/unistr.h" +#include "unicode/regex.h" +#include "uvector.h" +#include "hash.h" + +#if !UCONFIG_NO_REGULAR_EXPRESSIONS && !UCONFIG_NO_CONVERSION + +enum UXMLNodeType { + /** Node type string (text contents), stored as a UnicodeString. */ + UXML_NODE_TYPE_STRING, + /** Node type element, stored as a UXMLElement. */ + UXML_NODE_TYPE_ELEMENT, + UXML_NODE_TYPE_COUNT +}; + +U_NAMESPACE_BEGIN + +class UXMLParser; + +/** + * This class represents an element node in a parsed XML tree. + */ +class U_TOOLUTIL_API UXMLElement : public UObject { +public: + /** + * Destructor. + */ + virtual ~UXMLElement(); + + /** + * Get the tag name of this element. + */ + const UnicodeString &getTagName() const; + /** + * Get the text contents of the element. + * Append the contents of all text child nodes. + * @param recurse If TRUE, also recursively appends the contents of all + * text child nodes of element children. + * @return The text contents. + */ + UnicodeString getText(UBool recurse) const; + /** + * Get the number of attributes. + */ + int32_t countAttributes() const; + /** + * Get the i-th attribute. + * @param i Index of the attribute. + * @param name Output parameter, receives the attribute name. + * @param value Output parameter, receives the attribute value. + * @return A pointer to the attribute value (may be &value or a pointer to an + * internal string object), or NULL if i is out of bounds. + */ + const UnicodeString *getAttribute(int32_t i, UnicodeString &name, UnicodeString &value) const; + /** + * Get the value of the attribute with the given name. + * @param name Attribute name to be looked up. + * @return A pointer to the attribute value, or NULL if this element + * does not have this attribute. + */ + const UnicodeString *getAttribute(const UnicodeString &name) const; + /** + * Get the number of child nodes. + */ + int32_t countChildren() const; + /** + * Get the i-th child node. + * @param i Index of the child node. + * @param type The child node type. + * @return A pointer to the child node object, or NULL if i is out of bounds. + */ + const UObject *getChild(int32_t i, UXMLNodeType &type) const; + /** + * Get the next child element node, skipping non-element child nodes. + * @param i Enumeration index; initialize to 0 before getting the first child element. + * @return A pointer to the next child element, or NULL if there is none. + */ + const UXMLElement *nextChildElement(int32_t &i) const; + /** + * Get the immediate child element with the given name. + * If there are multiple child elements with this name, then return + * the first one. + * @param name Element name to be looked up. + * @return A pointer to the element node, or NULL if this element + * does not have this immediate child element. + */ + const UXMLElement *getChildElement(const UnicodeString &name) const; + + /** + * ICU "poor man's RTTI", returns a UClassID for the actual class. + */ + virtual UClassID getDynamicClassID() const; + + /** + * ICU "poor man's RTTI", returns a UClassID for this class. + */ + static UClassID U_EXPORT2 getStaticClassID(); + +private: + // prevent default construction etc. + UXMLElement(); + UXMLElement(const UXMLElement &other); + UXMLElement &operator=(const UXMLElement &other); + + void appendText(UnicodeString &text, UBool recurse) const; + + friend class UXMLParser; + + UXMLElement(const UXMLParser *parser, const UnicodeString *name, UErrorCode &errorCode); + + const UXMLParser *fParser; + const UnicodeString *fName; // The tag name of this element (owned by the UXMLParser) + UnicodeString fContent; // The text content of this node. All element content is + // concatenated even when there are intervening nested elements + // (which doesn't happen with most xml files we care about) + // Sections of content containing only white space are dropped, + // which gets rid the bogus white space content from + // elements which are primarily containers for nested elements. + UVector fAttNames; // A vector containing the names of this element's attributes + // The names are UnicodeString objects, owned by the UXMLParser. + UVector fAttValues; // A vector containing the attribute values for + // this element's attributes. The order is the same + // as that of the attribute name vector. + + UVector fChildren; // The child nodes of this element (a Vector) + + UXMLElement *fParent; // A pointer to the parent element of this element. +}; + +/** + * A simple XML parser; it is neither efficient nor conformant and only useful for + * restricted types of XML documents. + * + * The parse methods parse whole documents and return the parse trees via their + * root elements. + */ +class U_TOOLUTIL_API UXMLParser : public UObject { +public: + /** + * Create an XML parser. + */ + static UXMLParser *createParser(UErrorCode &errorCode); + /** + * Destructor. + */ + virtual ~UXMLParser(); + + /** + * Parse an XML document, create the entire document tree, and + * return a pointer to the root element of the parsed tree. + * The caller must delete the element. + */ + UXMLElement *parse(const UnicodeString &src, UErrorCode &errorCode); + /** + * Parse an XML file, create the entire document tree, and + * return a pointer to the root element of the parsed tree. + * The caller must delete the element. + */ + UXMLElement *parseFile(const char *filename, UErrorCode &errorCode); + + /** + * ICU "poor man's RTTI", returns a UClassID for the actual class. + */ + virtual UClassID getDynamicClassID() const; + + /** + * ICU "poor man's RTTI", returns a UClassID for this class. + */ + static UClassID U_EXPORT2 getStaticClassID(); + +private: + // prevent default construction etc. + UXMLParser(); + UXMLParser(const UXMLParser &other); + UXMLParser &operator=(const UXMLParser &other); + + // constructor + UXMLParser(UErrorCode &status); + + void parseMisc(UErrorCode &status); + UXMLElement *createElement(RegexMatcher &mEl, UErrorCode &status); + void error(const char *message, UErrorCode &status); + UnicodeString scanContent(UErrorCode &status); + void replaceCharRefs(UnicodeString &s, UErrorCode &status); + + const UnicodeString *intern(const UnicodeString &s, UErrorCode &errorCode); +public: + // public for UXMLElement only + const UnicodeString *findName(const UnicodeString &s) const; +private: + + // There is one ICU regex matcher for each of the major XML syntax items + // that are recognized. + RegexMatcher mXMLDecl; + RegexMatcher mXMLComment; + RegexMatcher mXMLSP; + RegexMatcher mXMLDoctype; + RegexMatcher mXMLPI; + RegexMatcher mXMLElemStart; + RegexMatcher mXMLElemEnd; + RegexMatcher mXMLElemEmpty; + RegexMatcher mXMLCharData; + RegexMatcher mAttrValue; + RegexMatcher mAttrNormalizer; + RegexMatcher mNewLineNormalizer; + RegexMatcher mAmps; + + Hashtable fNames; // interned element/attribute name strings + UStack fElementStack; // Stack holds the parent elements when nested + // elements are being parsed. All items on this + // stack are of type UXMLElement. + int32_t fPos; // String index of the current scan position in + // xml source (in fSrc). + UnicodeString fOneLF; +}; + +U_NAMESPACE_END +#endif /* !UCONFIG_NO_REGULAR_EXPRESSIONS */ + +#endif |