summaryrefslogtreecommitdiff
path: root/Build/source/libs/icu-xetex/common/brkdict.cpp
blob: 68875b9770862b9139d1126c052f7a2269fd8705 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
/*
**********************************************************************
*   Copyright (C) 1999-2004 IBM and others. All rights reserved.
**********************************************************************
*   Date        Name        Description
*   12/1/99     rtg         Ported from Java
*   01/13/2000  helena      Added UErrorCode to ctors.
**********************************************************************
*/

#include "unicode/utypes.h"

#if !UCONFIG_NO_BREAK_ITERATION

#include "unicode/ures.h"
#include "brkdict.h"
#include "cmemory.h"

U_NAMESPACE_BEGIN

//=================================================================================
// deserialization
//=================================================================================

BreakDictionary::BreakDictionary(const char* /*dictionaryFilename*/, UErrorCode& status)
 :  columnMap(NULL),
    table(NULL),
    rowIndex(NULL),
    rowIndexFlags(NULL),
    rowIndexFlagsIndex(NULL),
    rowIndexShifts(NULL)
{
    if (U_FAILURE(status)) return;
    
    UResourceBundle *th_dict = ures_open(NULL, "th", &status);
    th_dict = ures_getByKey(th_dict, "BreakDictionaryData", th_dict, &status);
    if (U_FAILURE(status)) return;

    int32_t len;
    const uint8_t * data = ures_getBinary(th_dict, &len, &status);
    ures_close(th_dict);
    if (U_FAILURE(status)) return;

    readDictionaryFile(data);
}

BreakDictionary::~BreakDictionary()
{
    ucmp8_close(columnMap);
    uprv_free(table);
    uprv_free(rowIndex);
    uprv_free(rowIndexFlags);
    uprv_free(rowIndexFlagsIndex);
    uprv_free(rowIndexShifts);
}

// macros to support readDictionaryFile.  The data files originated from a Java
// program, and Java always writes data out in big-endian format.  These macros will
// byte-swap the data for appropriate use on Windows.

#if U_IS_BIG_ENDIAN
#define SWAP32(x)
#define SWAP16(x)
#else
#define SWAP32(x) x = (uint32_t)((x >> 24 & 0xff) | (x >> 8 & 0xff00) | (x << 8 & 0xff0000) | (x << 24 & 0xff000000))
#define SWAP16(x) x = (uint16_t)((x << 8 & 0xff00) | (x >> 8 & 0xff))
#endif

#define DICTIONARY_READ(source, destAddr, len) \
        uprv_memcpy(destAddr, source, len);\
        source+=(len)


void
BreakDictionary::readDictionaryFile(const uint8_t * in)
{
    int32_t l;
    int32_t version;

    int i;

    // read in the version number (right now we just ignore it)
    DICTIONARY_READ(in, &version, 4);

    // read in the column map (this is serialized in its internal form:
    // an index array followed by a data array)
    DICTIONARY_READ(in, &l, 4);
    SWAP32(l);
    uint16_t* temp = (uint16_t*) uprv_malloc(sizeof(uint16_t)*l);
    DICTIONARY_READ(in, temp, l * sizeof (int16_t) );
    for (i = 0; i < l; i++) {
        SWAP16(temp[i]);
    }
    DICTIONARY_READ(in, &l, 4);
    SWAP32(l);
    int8_t* temp2 = (int8_t*) uprv_malloc(sizeof(int8_t)*l);
    DICTIONARY_READ(in, temp2, l);
    columnMap = ucmp8_openAdopt(temp, temp2, l);

    // read in numCols and numColGroups
    DICTIONARY_READ(in, &numCols, 4);
    SWAP32(numCols);
    DICTIONARY_READ(in, &numColGroups, 4);
    SWAP32(numColGroups);

    // read in the row-number index
    DICTIONARY_READ(in, &l, 4);
    SWAP32(l);
    rowIndex = (int16_t *)uprv_malloc(l*2);
    DICTIONARY_READ(in, rowIndex, l * sizeof (int16_t) );
    for (i = 0; i < l; i++) {
        SWAP16(rowIndex[i]);
    }

    // load in the populated-cells bitmap: index first, then bitmap list
    DICTIONARY_READ(in, &l, 4);
    SWAP32(l);
    rowIndexFlagsIndex = (int16_t *)uprv_malloc(l*2);
    DICTIONARY_READ(in, rowIndexFlagsIndex, l * sizeof(int16_t) );
    for (i = 0; i < l; i++) {
        SWAP16(rowIndexFlagsIndex[i]);
    }
    DICTIONARY_READ(in, &l, 4);
    SWAP32(l);
    rowIndexFlags = (int32_t *)uprv_malloc(l*4);
    DICTIONARY_READ(in, rowIndexFlags, l * sizeof(int32_t));
    for (i = 0; i < l; i++) {
        SWAP32(rowIndexFlags[i]);
    }

    // load in the row-shift index
    DICTIONARY_READ(in, &l, 4);
    SWAP32(l);
    rowIndexShifts = (int8_t *)uprv_malloc(l);
    DICTIONARY_READ(in, rowIndexShifts, l);

    // finally, load in the actual state table
    DICTIONARY_READ(in, &l, 4);
    SWAP32(l);
    table = (int16_t *)uprv_malloc(l*2);
    DICTIONARY_READ(in, table, l * sizeof(int16_t) );
    for (i = 0; i < l; i++) {
        SWAP16(table[i]);
    }

    // the reverse column map occurs next in the file.  In the C/C++ code, for the
    // time being, we're not going to worry about that.
}

//=================================================================================
// access to the words
//=================================================================================

/**
 * Uses the column map to map the character to a column number, then
 * passes the row and column number to the other version of at()
 * @param row The current state
 * @param ch The character whose column we're interested in
 * @return The new state to transition to
 */
int16_t
BreakDictionary::at(int32_t row, UChar ch) const
{
    int16_t col = ucmp8_get(columnMap, ch);
    return at(row, (int32_t)col);
}

/**
 * Returns the value in the cell with the specified (logical) row and
 * column numbers.  In DictionaryBasedBreakIterator, the row number is
 * a state number, the column number is an input, and the return value
 * is the row number of the new state to transition to.  (0 is the
 * "error" state, and -1 is the "end of word" state in a dictionary)
 * @param row The row number of the current state
 * @param col The column number of the input character (0 means "not a
 * dictionary character")
 * @return The row number of the new state to transition to
 */
int16_t
BreakDictionary::at(int32_t row, int32_t col) const
{
    if (cellIsPopulated(row, col)) {
        // we map from logical to physical row number by looking up the
        // mapping in rowIndex; we map from logical column number to
        // physical column number by looking up a shift value for this
        // logical row and offsetting the logical column number by
        // the shift amount.  Then we can use internalAt() to actually
        // get the value out of the table.
        return internalAt(rowIndex[row], col + rowIndexShifts[row]);
    }
    else {
        return 0;
    }
}

//=================================================================================
// implementation
//=================================================================================
/**
 * Given (logical) row and column numbers, returns true if the
 * cell in that position is populated
 */
UBool
BreakDictionary::cellIsPopulated(int32_t row, int32_t col) const
{
    // look up the entry in the bitmap index for the specified row.
    // If it's a negative number, it's the column number of the only
    // populated cell in the row
    if (rowIndexFlagsIndex[row] < 0) {
        return col == -rowIndexFlagsIndex[row];
    }

    // if it's a positive number, it's the offset of an entry in the bitmap
    // list.  If the table is more than 32 columns wide, the bitmap is stored
    // successive entries in the bitmap list, so we have to divide the column
    // number by 32 and offset the number we got out of the index by the result.
    // Once we have the appropriate piece of the bitmap, test the appropriate
    // bit and return the result.
    else {
        int32_t flags = rowIndexFlags[rowIndexFlagsIndex[row] + (col >> 5)];
        return (flags & (1 << (col & 0x1f))) != 0;
    }
}

/**
 * Implementation of at() when we know the specified cell is populated.
 * @param row The PHYSICAL row number of the cell
 * @param col The PHYSICAL column number of the cell
 * @return The value stored in the cell
 */
int16_t
BreakDictionary::internalAt(int32_t row, int32_t col) const
{
    // the table is a one-dimensional array, so this just does the math necessary
    // to treat it as a two-dimensional array (we don't just use a two-dimensional
    // array because two-dimensional arrays are inefficient in Java)
    return table[row * numCols + col];
}

U_NAMESPACE_END

#endif /* #if !UCONFIG_NO_BREAK_ITERATION */