diff options
author | Jonathan Kew <jfkthame@googlemail.com> | 2006-07-25 12:37:43 +0000 |
---|---|---|
committer | Jonathan Kew <jfkthame@googlemail.com> | 2006-07-25 12:37:43 +0000 |
commit | 10c60368cd42904bd991453f02b684ebe02ef915 (patch) | |
tree | c6fb06ccf22a81c80af08f79df1b436120703de4 /Build/source/libs/icu-xetex/common/brkdict.cpp | |
parent | 4d8b2aac6036acbb6878236c27e2fb110dad8643 (diff) |
adding ICU library sources used by xetex
git-svn-id: svn://tug.org/texlive/trunk@1915 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Build/source/libs/icu-xetex/common/brkdict.cpp')
-rw-r--r-- | Build/source/libs/icu-xetex/common/brkdict.cpp | 242 |
1 files changed, 242 insertions, 0 deletions
diff --git a/Build/source/libs/icu-xetex/common/brkdict.cpp b/Build/source/libs/icu-xetex/common/brkdict.cpp new file mode 100644 index 00000000000..68875b97708 --- /dev/null +++ b/Build/source/libs/icu-xetex/common/brkdict.cpp @@ -0,0 +1,242 @@ +/* +********************************************************************** +* Copyright (C) 1999-2004 IBM and others. All rights reserved. +********************************************************************** +* Date Name Description +* 12/1/99 rtg Ported from Java +* 01/13/2000 helena Added UErrorCode to ctors. +********************************************************************** +*/ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_BREAK_ITERATION + +#include "unicode/ures.h" +#include "brkdict.h" +#include "cmemory.h" + +U_NAMESPACE_BEGIN + +//================================================================================= +// deserialization +//================================================================================= + +BreakDictionary::BreakDictionary(const char* /*dictionaryFilename*/, UErrorCode& status) + : columnMap(NULL), + table(NULL), + rowIndex(NULL), + rowIndexFlags(NULL), + rowIndexFlagsIndex(NULL), + rowIndexShifts(NULL) +{ + if (U_FAILURE(status)) return; + + UResourceBundle *th_dict = ures_open(NULL, "th", &status); + th_dict = ures_getByKey(th_dict, "BreakDictionaryData", th_dict, &status); + if (U_FAILURE(status)) return; + + int32_t len; + const uint8_t * data = ures_getBinary(th_dict, &len, &status); + ures_close(th_dict); + if (U_FAILURE(status)) return; + + readDictionaryFile(data); +} + +BreakDictionary::~BreakDictionary() +{ + ucmp8_close(columnMap); + uprv_free(table); + uprv_free(rowIndex); + uprv_free(rowIndexFlags); + uprv_free(rowIndexFlagsIndex); + uprv_free(rowIndexShifts); +} + +// macros to support readDictionaryFile. The data files originated from a Java +// program, and Java always writes data out in big-endian format. These macros will +// byte-swap the data for appropriate use on Windows. + +#if U_IS_BIG_ENDIAN +#define SWAP32(x) +#define SWAP16(x) +#else +#define SWAP32(x) x = (uint32_t)((x >> 24 & 0xff) | (x >> 8 & 0xff00) | (x << 8 & 0xff0000) | (x << 24 & 0xff000000)) +#define SWAP16(x) x = (uint16_t)((x << 8 & 0xff00) | (x >> 8 & 0xff)) +#endif + +#define DICTIONARY_READ(source, destAddr, len) \ + uprv_memcpy(destAddr, source, len);\ + source+=(len) + + +void +BreakDictionary::readDictionaryFile(const uint8_t * in) +{ + int32_t l; + int32_t version; + + int i; + + // read in the version number (right now we just ignore it) + DICTIONARY_READ(in, &version, 4); + + // read in the column map (this is serialized in its internal form: + // an index array followed by a data array) + DICTIONARY_READ(in, &l, 4); + SWAP32(l); + uint16_t* temp = (uint16_t*) uprv_malloc(sizeof(uint16_t)*l); + DICTIONARY_READ(in, temp, l * sizeof (int16_t) ); + for (i = 0; i < l; i++) { + SWAP16(temp[i]); + } + DICTIONARY_READ(in, &l, 4); + SWAP32(l); + int8_t* temp2 = (int8_t*) uprv_malloc(sizeof(int8_t)*l); + DICTIONARY_READ(in, temp2, l); + columnMap = ucmp8_openAdopt(temp, temp2, l); + + // read in numCols and numColGroups + DICTIONARY_READ(in, &numCols, 4); + SWAP32(numCols); + DICTIONARY_READ(in, &numColGroups, 4); + SWAP32(numColGroups); + + // read in the row-number index + DICTIONARY_READ(in, &l, 4); + SWAP32(l); + rowIndex = (int16_t *)uprv_malloc(l*2); + DICTIONARY_READ(in, rowIndex, l * sizeof (int16_t) ); + for (i = 0; i < l; i++) { + SWAP16(rowIndex[i]); + } + + // load in the populated-cells bitmap: index first, then bitmap list + DICTIONARY_READ(in, &l, 4); + SWAP32(l); + rowIndexFlagsIndex = (int16_t *)uprv_malloc(l*2); + DICTIONARY_READ(in, rowIndexFlagsIndex, l * sizeof(int16_t) ); + for (i = 0; i < l; i++) { + SWAP16(rowIndexFlagsIndex[i]); + } + DICTIONARY_READ(in, &l, 4); + SWAP32(l); + rowIndexFlags = (int32_t *)uprv_malloc(l*4); + DICTIONARY_READ(in, rowIndexFlags, l * sizeof(int32_t)); + for (i = 0; i < l; i++) { + SWAP32(rowIndexFlags[i]); + } + + // load in the row-shift index + DICTIONARY_READ(in, &l, 4); + SWAP32(l); + rowIndexShifts = (int8_t *)uprv_malloc(l); + DICTIONARY_READ(in, rowIndexShifts, l); + + // finally, load in the actual state table + DICTIONARY_READ(in, &l, 4); + SWAP32(l); + table = (int16_t *)uprv_malloc(l*2); + DICTIONARY_READ(in, table, l * sizeof(int16_t) ); + for (i = 0; i < l; i++) { + SWAP16(table[i]); + } + + // the reverse column map occurs next in the file. In the C/C++ code, for the + // time being, we're not going to worry about that. +} + +//================================================================================= +// access to the words +//================================================================================= + +/** + * Uses the column map to map the character to a column number, then + * passes the row and column number to the other version of at() + * @param row The current state + * @param ch The character whose column we're interested in + * @return The new state to transition to + */ +int16_t +BreakDictionary::at(int32_t row, UChar ch) const +{ + int16_t col = ucmp8_get(columnMap, ch); + return at(row, (int32_t)col); +} + +/** + * Returns the value in the cell with the specified (logical) row and + * column numbers. In DictionaryBasedBreakIterator, the row number is + * a state number, the column number is an input, and the return value + * is the row number of the new state to transition to. (0 is the + * "error" state, and -1 is the "end of word" state in a dictionary) + * @param row The row number of the current state + * @param col The column number of the input character (0 means "not a + * dictionary character") + * @return The row number of the new state to transition to + */ +int16_t +BreakDictionary::at(int32_t row, int32_t col) const +{ + if (cellIsPopulated(row, col)) { + // we map from logical to physical row number by looking up the + // mapping in rowIndex; we map from logical column number to + // physical column number by looking up a shift value for this + // logical row and offsetting the logical column number by + // the shift amount. Then we can use internalAt() to actually + // get the value out of the table. + return internalAt(rowIndex[row], col + rowIndexShifts[row]); + } + else { + return 0; + } +} + +//================================================================================= +// implementation +//================================================================================= +/** + * Given (logical) row and column numbers, returns true if the + * cell in that position is populated + */ +UBool +BreakDictionary::cellIsPopulated(int32_t row, int32_t col) const +{ + // look up the entry in the bitmap index for the specified row. + // If it's a negative number, it's the column number of the only + // populated cell in the row + if (rowIndexFlagsIndex[row] < 0) { + return col == -rowIndexFlagsIndex[row]; + } + + // if it's a positive number, it's the offset of an entry in the bitmap + // list. If the table is more than 32 columns wide, the bitmap is stored + // successive entries in the bitmap list, so we have to divide the column + // number by 32 and offset the number we got out of the index by the result. + // Once we have the appropriate piece of the bitmap, test the appropriate + // bit and return the result. + else { + int32_t flags = rowIndexFlags[rowIndexFlagsIndex[row] + (col >> 5)]; + return (flags & (1 << (col & 0x1f))) != 0; + } +} + +/** + * Implementation of at() when we know the specified cell is populated. + * @param row The PHYSICAL row number of the cell + * @param col The PHYSICAL column number of the cell + * @return The value stored in the cell + */ +int16_t +BreakDictionary::internalAt(int32_t row, int32_t col) const +{ + // the table is a one-dimensional array, so this just does the math necessary + // to treat it as a two-dimensional array (we don't just use a two-dimensional + // array because two-dimensional arrays are inefficient in Java) + return table[row * numCols + col]; +} + +U_NAMESPACE_END + +#endif /* #if !UCONFIG_NO_BREAK_ITERATION */ |