summaryrefslogtreecommitdiff
path: root/Build/source/libs/icu-xetex/common/brkdict.cpp
diff options
context:
space:
mode:
authorJonathan Kew <jfkthame@googlemail.com>2006-07-25 12:37:43 +0000
committerJonathan Kew <jfkthame@googlemail.com>2006-07-25 12:37:43 +0000
commit10c60368cd42904bd991453f02b684ebe02ef915 (patch)
treec6fb06ccf22a81c80af08f79df1b436120703de4 /Build/source/libs/icu-xetex/common/brkdict.cpp
parent4d8b2aac6036acbb6878236c27e2fb110dad8643 (diff)
adding ICU library sources used by xetex
git-svn-id: svn://tug.org/texlive/trunk@1915 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Build/source/libs/icu-xetex/common/brkdict.cpp')
-rw-r--r--Build/source/libs/icu-xetex/common/brkdict.cpp242
1 files changed, 242 insertions, 0 deletions
diff --git a/Build/source/libs/icu-xetex/common/brkdict.cpp b/Build/source/libs/icu-xetex/common/brkdict.cpp
new file mode 100644
index 00000000000..68875b97708
--- /dev/null
+++ b/Build/source/libs/icu-xetex/common/brkdict.cpp
@@ -0,0 +1,242 @@
+/*
+**********************************************************************
+* Copyright (C) 1999-2004 IBM and others. All rights reserved.
+**********************************************************************
+* Date Name Description
+* 12/1/99 rtg Ported from Java
+* 01/13/2000 helena Added UErrorCode to ctors.
+**********************************************************************
+*/
+
+#include "unicode/utypes.h"
+
+#if !UCONFIG_NO_BREAK_ITERATION
+
+#include "unicode/ures.h"
+#include "brkdict.h"
+#include "cmemory.h"
+
+U_NAMESPACE_BEGIN
+
+//=================================================================================
+// deserialization
+//=================================================================================
+
+BreakDictionary::BreakDictionary(const char* /*dictionaryFilename*/, UErrorCode& status)
+ : columnMap(NULL),
+ table(NULL),
+ rowIndex(NULL),
+ rowIndexFlags(NULL),
+ rowIndexFlagsIndex(NULL),
+ rowIndexShifts(NULL)
+{
+ if (U_FAILURE(status)) return;
+
+ UResourceBundle *th_dict = ures_open(NULL, "th", &status);
+ th_dict = ures_getByKey(th_dict, "BreakDictionaryData", th_dict, &status);
+ if (U_FAILURE(status)) return;
+
+ int32_t len;
+ const uint8_t * data = ures_getBinary(th_dict, &len, &status);
+ ures_close(th_dict);
+ if (U_FAILURE(status)) return;
+
+ readDictionaryFile(data);
+}
+
+BreakDictionary::~BreakDictionary()
+{
+ ucmp8_close(columnMap);
+ uprv_free(table);
+ uprv_free(rowIndex);
+ uprv_free(rowIndexFlags);
+ uprv_free(rowIndexFlagsIndex);
+ uprv_free(rowIndexShifts);
+}
+
+// macros to support readDictionaryFile. The data files originated from a Java
+// program, and Java always writes data out in big-endian format. These macros will
+// byte-swap the data for appropriate use on Windows.
+
+#if U_IS_BIG_ENDIAN
+#define SWAP32(x)
+#define SWAP16(x)
+#else
+#define SWAP32(x) x = (uint32_t)((x >> 24 & 0xff) | (x >> 8 & 0xff00) | (x << 8 & 0xff0000) | (x << 24 & 0xff000000))
+#define SWAP16(x) x = (uint16_t)((x << 8 & 0xff00) | (x >> 8 & 0xff))
+#endif
+
+#define DICTIONARY_READ(source, destAddr, len) \
+ uprv_memcpy(destAddr, source, len);\
+ source+=(len)
+
+
+void
+BreakDictionary::readDictionaryFile(const uint8_t * in)
+{
+ int32_t l;
+ int32_t version;
+
+ int i;
+
+ // read in the version number (right now we just ignore it)
+ DICTIONARY_READ(in, &version, 4);
+
+ // read in the column map (this is serialized in its internal form:
+ // an index array followed by a data array)
+ DICTIONARY_READ(in, &l, 4);
+ SWAP32(l);
+ uint16_t* temp = (uint16_t*) uprv_malloc(sizeof(uint16_t)*l);
+ DICTIONARY_READ(in, temp, l * sizeof (int16_t) );
+ for (i = 0; i < l; i++) {
+ SWAP16(temp[i]);
+ }
+ DICTIONARY_READ(in, &l, 4);
+ SWAP32(l);
+ int8_t* temp2 = (int8_t*) uprv_malloc(sizeof(int8_t)*l);
+ DICTIONARY_READ(in, temp2, l);
+ columnMap = ucmp8_openAdopt(temp, temp2, l);
+
+ // read in numCols and numColGroups
+ DICTIONARY_READ(in, &numCols, 4);
+ SWAP32(numCols);
+ DICTIONARY_READ(in, &numColGroups, 4);
+ SWAP32(numColGroups);
+
+ // read in the row-number index
+ DICTIONARY_READ(in, &l, 4);
+ SWAP32(l);
+ rowIndex = (int16_t *)uprv_malloc(l*2);
+ DICTIONARY_READ(in, rowIndex, l * sizeof (int16_t) );
+ for (i = 0; i < l; i++) {
+ SWAP16(rowIndex[i]);
+ }
+
+ // load in the populated-cells bitmap: index first, then bitmap list
+ DICTIONARY_READ(in, &l, 4);
+ SWAP32(l);
+ rowIndexFlagsIndex = (int16_t *)uprv_malloc(l*2);
+ DICTIONARY_READ(in, rowIndexFlagsIndex, l * sizeof(int16_t) );
+ for (i = 0; i < l; i++) {
+ SWAP16(rowIndexFlagsIndex[i]);
+ }
+ DICTIONARY_READ(in, &l, 4);
+ SWAP32(l);
+ rowIndexFlags = (int32_t *)uprv_malloc(l*4);
+ DICTIONARY_READ(in, rowIndexFlags, l * sizeof(int32_t));
+ for (i = 0; i < l; i++) {
+ SWAP32(rowIndexFlags[i]);
+ }
+
+ // load in the row-shift index
+ DICTIONARY_READ(in, &l, 4);
+ SWAP32(l);
+ rowIndexShifts = (int8_t *)uprv_malloc(l);
+ DICTIONARY_READ(in, rowIndexShifts, l);
+
+ // finally, load in the actual state table
+ DICTIONARY_READ(in, &l, 4);
+ SWAP32(l);
+ table = (int16_t *)uprv_malloc(l*2);
+ DICTIONARY_READ(in, table, l * sizeof(int16_t) );
+ for (i = 0; i < l; i++) {
+ SWAP16(table[i]);
+ }
+
+ // the reverse column map occurs next in the file. In the C/C++ code, for the
+ // time being, we're not going to worry about that.
+}
+
+//=================================================================================
+// access to the words
+//=================================================================================
+
+/**
+ * Uses the column map to map the character to a column number, then
+ * passes the row and column number to the other version of at()
+ * @param row The current state
+ * @param ch The character whose column we're interested in
+ * @return The new state to transition to
+ */
+int16_t
+BreakDictionary::at(int32_t row, UChar ch) const
+{
+ int16_t col = ucmp8_get(columnMap, ch);
+ return at(row, (int32_t)col);
+}
+
+/**
+ * Returns the value in the cell with the specified (logical) row and
+ * column numbers. In DictionaryBasedBreakIterator, the row number is
+ * a state number, the column number is an input, and the return value
+ * is the row number of the new state to transition to. (0 is the
+ * "error" state, and -1 is the "end of word" state in a dictionary)
+ * @param row The row number of the current state
+ * @param col The column number of the input character (0 means "not a
+ * dictionary character")
+ * @return The row number of the new state to transition to
+ */
+int16_t
+BreakDictionary::at(int32_t row, int32_t col) const
+{
+ if (cellIsPopulated(row, col)) {
+ // we map from logical to physical row number by looking up the
+ // mapping in rowIndex; we map from logical column number to
+ // physical column number by looking up a shift value for this
+ // logical row and offsetting the logical column number by
+ // the shift amount. Then we can use internalAt() to actually
+ // get the value out of the table.
+ return internalAt(rowIndex[row], col + rowIndexShifts[row]);
+ }
+ else {
+ return 0;
+ }
+}
+
+//=================================================================================
+// implementation
+//=================================================================================
+/**
+ * Given (logical) row and column numbers, returns true if the
+ * cell in that position is populated
+ */
+UBool
+BreakDictionary::cellIsPopulated(int32_t row, int32_t col) const
+{
+ // look up the entry in the bitmap index for the specified row.
+ // If it's a negative number, it's the column number of the only
+ // populated cell in the row
+ if (rowIndexFlagsIndex[row] < 0) {
+ return col == -rowIndexFlagsIndex[row];
+ }
+
+ // if it's a positive number, it's the offset of an entry in the bitmap
+ // list. If the table is more than 32 columns wide, the bitmap is stored
+ // successive entries in the bitmap list, so we have to divide the column
+ // number by 32 and offset the number we got out of the index by the result.
+ // Once we have the appropriate piece of the bitmap, test the appropriate
+ // bit and return the result.
+ else {
+ int32_t flags = rowIndexFlags[rowIndexFlagsIndex[row] + (col >> 5)];
+ return (flags & (1 << (col & 0x1f))) != 0;
+ }
+}
+
+/**
+ * Implementation of at() when we know the specified cell is populated.
+ * @param row The PHYSICAL row number of the cell
+ * @param col The PHYSICAL column number of the cell
+ * @return The value stored in the cell
+ */
+int16_t
+BreakDictionary::internalAt(int32_t row, int32_t col) const
+{
+ // the table is a one-dimensional array, so this just does the math necessary
+ // to treat it as a two-dimensional array (we don't just use a two-dimensional
+ // array because two-dimensional arrays are inefficient in Java)
+ return table[row * numCols + col];
+}
+
+U_NAMESPACE_END
+
+#endif /* #if !UCONFIG_NO_BREAK_ITERATION */