summaryrefslogtreecommitdiff
path: root/Build/source/libs/icu/icu-src/source/test/intltest/lstmbetst.cpp
blob: 6c5d5d0ad6138c83fc42744407b40d4f85bf7932 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
// © 2021 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html

#include "unicode/utypes.h"

#if !UCONFIG_NO_BREAK_ITERATION

#include "lstmbetst.h"
#include "lstmbe.h"

#include <algorithm>
#include <sstream>
#include <vector>

#include "charstr.h"

//---------------------------------------------
// runIndexedTest
//---------------------------------------------


void LSTMBETest::runIndexedTest( int32_t index, UBool exec, const char* &name, char* params )
{
    fTestParams = params;

    TESTCASE_AUTO_BEGIN;

    TESTCASE_AUTO(TestThaiGraphclust);
    TESTCASE_AUTO(TestThaiCodepoints);
    TESTCASE_AUTO(TestBurmeseGraphclust);
    TESTCASE_AUTO(TestThaiGraphclustWithLargeMemory);
    TESTCASE_AUTO(TestThaiCodepointsWithLargeMemory);

    TESTCASE_AUTO_END;
}


//--------------------------------------------------------------------------------------
//
//    LSTMBETest    constructor and destructor
//
//--------------------------------------------------------------------------------------

LSTMBETest::LSTMBETest() {
    fTestParams = NULL;
}


LSTMBETest::~LSTMBETest() {
}

UScriptCode getScriptFromModelName(const std::string& modelName) {
    if (modelName.find("Thai") == 0) {
        return USCRIPT_THAI;
    } else if (modelName.find("Burmese") == 0) {
        return USCRIPT_MYANMAR;
    }
    // Add for other script codes.
    UPRV_UNREACHABLE_EXIT;
}

// Read file generated by
// https://github.com/unicode-org/lstm_word_segmentation/blob/master/segment_text.py
// as test cases and compare the Output.
// Format of the file
//   Model:\t[Model Name (such as 'Thai_graphclust_model4_heavy')]
//   Embedding:\t[Embedding type (such as 'grapheme_clusters_tf')]
//   Input:\t[source text]
//   Output:\t[expected output separated by | ]
//   Input: ...
//   Output: ...
// The test will ensure the Input contains only the characters can be handled by
// the model. Since by default the LSTM models are not included, all the tested
// models need to be included under source/test/testdata.

void LSTMBETest::runTestFromFile(const char* filename) {
    UErrorCode   status = U_ZERO_ERROR;
    LocalPointer<const LanguageBreakEngine> engine;
    //  Open and read the test data file.
    const char *testDataDirectory = IntlTest::getSourceTestData(status);
    CharString testFileName(testDataDirectory, -1, status);
    testFileName.append(filename, -1, status);

    int len;
    UChar *testFile = ReadAndConvertFile(testFileName.data(), len, "UTF-8", status);
    if (U_FAILURE(status)) {
        errln("%s:%d Error %s opening test file %s", __FILE__, __LINE__, u_errorName(status), filename);
        return;
    }

    //  Put the test data into a UnicodeString
    UnicodeString testString(FALSE, testFile, len);

    int32_t start = 0;

    UnicodeString line;
    int32_t end;
    std::string actual_sep_str;
    int32_t caseNum = 0;
    // Iterate through all the lines in the test file.
    do {
        int32_t cr = testString.indexOf(u'\r', start);
        int32_t lf = testString.indexOf(u'\n', start);
        end = cr >= 0 ? (lf >= 0 ? std::min(cr, lf) : cr) : lf;
        line = testString.tempSubString(start, end < 0 ? INT32_MAX : end - start);
        if (line.length() > 0) {
            // Separate each line to key and value by TAB.
            int32_t tab = line.indexOf(u'\t');
            UnicodeString key = line.tempSubString(0, tab);
            const UnicodeString value = line.tempSubString(tab+1);

            if (key == "Model:") {
                std::string modelName;
                value.toUTF8String<std::string>(modelName);
                engine.adoptInstead(createEngineFromTestData(modelName.c_str(), getScriptFromModelName(modelName), status));
                if (U_FAILURE(status)) {
                    dataerrln("Could not CreateLSTMBreakEngine for " + line + UnicodeString(u_errorName(status)));
                    return;
                }
            } else if (key == "Input:") {
                // First, we ensure all the char in the Input lines are accepted
                // by the engine before we test them.
                caseNum++;
                bool canHandleAllChars = true;
                for (int32_t i = 0; i < value.length(); i++) {
                    if (!engine->handles(value.charAt(i))) {
                        errln(UnicodeString("Test Case#") + caseNum + " contains char '" +
                                  UnicodeString(value.charAt(i)) +
                                  "' cannot be handled by the engine in offset " + i + "\n" + line);
                        canHandleAllChars = false;
                        break;
                    }
                }
                if (! canHandleAllChars) {
                    return;
                }

                // If the engine can handle all the chars in the Input line, we
                // then find the break points by calling the engine.
                std::stringstream ss;

                // Construct the UText which is expected by the the engine as
                // input from the UnicodeString.
                UText ut = UTEXT_INITIALIZER;
                utext_openConstUnicodeString(&ut, &value, &status);
                if (U_FAILURE(status)) {
                    dataerrln("Could not utext_openConstUnicodeString for " + value + UnicodeString(u_errorName(status)));
                    return;
                }

                UVector32 actual(status);
                if (U_FAILURE(status)) {
                    dataerrln("%s:%d Error %s Could not allocate UVextor32", __FILE__, __LINE__, u_errorName(status));
                    return;
                }
                engine->findBreaks(&ut, 0, value.length(), actual, status);
                if (U_FAILURE(status)) {
                    dataerrln("%s:%d Error %s findBreaks failed", __FILE__, __LINE__, u_errorName(status));
                    return;
                }
                utext_close(&ut);
                for (int32_t i = 0; i < actual.size(); i++) {
                    ss << actual.elementAti(i) << ", ";
                }
                ss << value.length();
                // Turn the break points into a string for easy comparison
                // output.
                actual_sep_str = "{" + ss.str() + "}";
            } else if (key == "Output:" && !actual_sep_str.empty()) {
                std::string d;
                int32_t sep;
                int32_t start = 0;
                int32_t curr = 0;
                std::stringstream ss;
                while ((sep = value.indexOf(u'|', start)) >= 0) {
                    int32_t len = sep - start;
                    if (len > 0) {
                        if (curr > 0) {
                            ss << ", ";
                        }
                        curr += len;
                        ss << curr;
                    }
                    start = sep + 1;
                }
                // Turn the break points into a string for easy comparison
                // output.
                std::string expected = "{" + ss.str() + "}";
                std::string utf8;

                assertEquals((value + " Test Case#" + caseNum).toUTF8String<std::string>(utf8).c_str(),
                             expected.c_str(), actual_sep_str.c_str());
                actual_sep_str.clear();
            }
        }
        start = std::max(cr, lf) + 1;
    } while (end >= 0);

    delete [] testFile;
}

void LSTMBETest::TestThaiGraphclust() {
    runTestFromFile("Thai_graphclust_model4_heavy_Test.txt");
}

void LSTMBETest::TestThaiCodepoints() {
    runTestFromFile("Thai_codepoints_exclusive_model5_heavy_Test.txt");
}

void LSTMBETest::TestBurmeseGraphclust() {
    runTestFromFile("Burmese_graphclust_model5_heavy_Test.txt");
}

const LanguageBreakEngine* LSTMBETest::createEngineFromTestData(
        const char* model, UScriptCode script, UErrorCode& status) {
    const char* testdatapath=loadTestData(status);
    if(U_FAILURE(status))
    {
        dataerrln("Could not load testdata.dat " + UnicodeString(testdatapath) +  ", " +
                  UnicodeString(u_errorName(status)));
        return nullptr;
    }

    LocalUResourceBundlePointer rb(
        ures_openDirect(testdatapath, model, &status));
    if (U_FAILURE(status)) {
        dataerrln("Could not open " + UnicodeString(model) + " under " +  UnicodeString(testdatapath) +  ", " +
                  UnicodeString(u_errorName(status)));
        return nullptr;
    }

    const LSTMData* data = CreateLSTMData(rb.orphan(), status);
    if (U_FAILURE(status)) {
        dataerrln("Could not CreateLSTMData " + UnicodeString(model) + " under " +  UnicodeString(testdatapath) +  ", " +
                  UnicodeString(u_errorName(status)));
        return nullptr;
    }
    if (data == nullptr) {
        return nullptr;
    }

    LocalPointer<const LanguageBreakEngine> engine(CreateLSTMBreakEngine(script, data, status));
    if (U_FAILURE(status) || engine.getAlias() == nullptr) {
        dataerrln("Could not CreateLSTMBreakEngine " + UnicodeString(testdatapath) +  ", " +
                  UnicodeString(u_errorName(status)));
        DeleteLSTMData(data);
        return nullptr;
    }
    return engine.orphan();
}


void LSTMBETest::TestThaiGraphclustWithLargeMemory() {
    runTestWithLargeMemory("Thai_graphclust_model4_heavy", USCRIPT_THAI);

}

void LSTMBETest::TestThaiCodepointsWithLargeMemory() {
    runTestWithLargeMemory("Thai_codepoints_exclusive_model5_heavy", USCRIPT_THAI);
}

constexpr int32_t MEMORY_TEST_THESHOLD_SHORT = 2 * 1024; // 2 K Unicode Chars.
constexpr int32_t MEMORY_TEST_THESHOLD = 32 * 1024; // 32 K Unicode Chars.

// Test with very long unicode string.
void LSTMBETest::runTestWithLargeMemory( const char* model, UScriptCode script) {
    UErrorCode   status = U_ZERO_ERROR;
    int32_t test_threshold = quick ? MEMORY_TEST_THESHOLD_SHORT : MEMORY_TEST_THESHOLD;
    LocalPointer<const LanguageBreakEngine> engine(
        createEngineFromTestData(model, script, status));
    if (U_FAILURE(status)) {
        dataerrln("Could not CreateLSTMBreakEngine for " + UnicodeString(model) + UnicodeString(u_errorName(status)));
        return;
    }
    UnicodeString text(u"อ");  // start with a single Thai char.
    UVector32 actual(status);
    if (U_FAILURE(status)) {
        dataerrln("%s:%d Error %s Could not allocate UVextor32", __FILE__, __LINE__, u_errorName(status));
        return;
    }
    while (U_SUCCESS(status) && text.length() <= test_threshold) {
        // Construct the UText which is expected by the the engine as
        // input from the UnicodeString.
        UText ut = UTEXT_INITIALIZER;
        utext_openConstUnicodeString(&ut, &text, &status);
        if (U_FAILURE(status)) {
            dataerrln("Could not utext_openConstUnicodeString for " + text + UnicodeString(u_errorName(status)));
            return;
        }

        engine->findBreaks(&ut, 0, text.length(), actual, status);
        utext_close(&ut);
        text += text;
    }
}
#endif // #if !UCONFIG_NO_BREAK_ITERATION