summaryrefslogtreecommitdiff
path: root/Build/source/libs/icu/icu-xetex/test/intltest/thcoll.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'Build/source/libs/icu/icu-xetex/test/intltest/thcoll.cpp')
-rw-r--r--Build/source/libs/icu/icu-xetex/test/intltest/thcoll.cpp459
1 files changed, 459 insertions, 0 deletions
diff --git a/Build/source/libs/icu/icu-xetex/test/intltest/thcoll.cpp b/Build/source/libs/icu/icu-xetex/test/intltest/thcoll.cpp
new file mode 100644
index 00000000000..1fd99e7fb24
--- /dev/null
+++ b/Build/source/libs/icu/icu-xetex/test/intltest/thcoll.cpp
@@ -0,0 +1,459 @@
+/*
+**********************************************************************
+* Copyright (C) 1999-2006, International Business Machines
+* Corporation and others. All Rights Reserved.
+**********************************************************************
+* Date Name Description
+* 12/09/99 aliu Ported from Java.
+**********************************************************************
+*/
+
+#include "unicode/utypes.h"
+
+#if !UCONFIG_NO_COLLATION
+
+#include "thcoll.h"
+#include "unicode/utypes.h"
+#include "unicode/coll.h"
+#include "unicode/sortkey.h"
+#include "unicode/ustring.h"
+#include "cstring.h"
+#include "filestrm.h"
+#include "textfile.h"
+
+/**
+ * The TestDictionary test expects a file of this name, with this
+ * encoding, to be present in the directory $ICU/source/test/testdata.
+ */
+//#define TEST_FILE "th18057.txt"
+
+/**
+ * This is the most failures we show in TestDictionary. If this number
+ * is < 0, we show all failures.
+ */
+#define MAX_FAILURES_TO_SHOW -1
+
+CollationThaiTest::CollationThaiTest() {
+ UErrorCode status = U_ZERO_ERROR;
+ coll = Collator::createInstance(Locale("th", "TH", ""), status);
+ if (coll && U_SUCCESS(status)) {
+ //coll->setStrength(Collator::TERTIARY);
+ } else {
+ delete coll;
+ coll = 0;
+ }
+}
+
+CollationThaiTest::~CollationThaiTest() {
+ delete coll;
+}
+
+void CollationThaiTest::runIndexedTest(int32_t index, UBool exec, const char* &name,
+ char* /*par*/) {
+
+ if((!coll) && exec) {
+ errln(__FILE__ " cannot test - failed to create collator.");
+ name = "some test";
+ return;
+ }
+
+ switch (index) {
+ TESTCASE(0,TestDictionary);
+ TESTCASE(1,TestCornerCases);
+ TESTCASE(2,TestNamesList);
+ TESTCASE(3,TestInvalidThai);
+ TESTCASE(4,TestReordering);
+ default: name = ""; break;
+ }
+}
+
+/**
+ * Read the external names list, and confirms that the collator
+ * gets the same results when comparing lines one to another
+ * using regular and iterative comparison.
+ */
+void CollationThaiTest::TestNamesList(void) {
+ if (coll == 0) {
+ errln("Error: could not construct Thai collator");
+ return;
+ }
+
+ UErrorCode ec = U_ZERO_ERROR;
+ TextFile names("TestNames_Thai.txt", "UTF16LE", ec);
+ if (U_FAILURE(ec)) {
+ logln("Can't open TestNames_Thai.txt: %s; skipping test",
+ u_errorName(ec));
+ return;
+ }
+
+ //
+ // Loop through each word in the dictionary and compare it to the previous
+ // word. They should be in sorted order.
+ //
+ UnicodeString lastWord, word;
+ //int32_t failed = 0;
+ int32_t wordCount = 0;
+ while (names.readLineSkippingComments(word, ec, FALSE) && U_SUCCESS(ec)) {
+
+ // Show the first 8 words being compared, so we can see what's happening
+ ++wordCount;
+ if (wordCount <= 8) {
+ UnicodeString str;
+ logln((UnicodeString)"Word " + wordCount + ": " + IntlTest::prettify(word, str));
+ }
+
+ if (lastWord.length() > 0) {
+ Collator::EComparisonResult result = coll->compare(lastWord, word);
+ doTest(coll, lastWord, word, result);
+ }
+ lastWord = word;
+ }
+
+ assertSuccess("readLine", ec);
+
+ logln((UnicodeString)"Words checked: " + wordCount);
+}
+
+/**
+ * Read the external dictionary file, which is already in proper
+ * sorted order, and confirm that the collator compares each line as
+ * preceding the following line.
+ */
+void CollationThaiTest::TestDictionary(void) {
+ if (coll == 0) {
+ errln("Error: could not construct Thai collator");
+ return;
+ }
+
+ UErrorCode ec = U_ZERO_ERROR;
+ TextFile riwords("riwords.txt", "UTF8", ec);
+ if (U_FAILURE(ec)) {
+ logln("Can't open riwords.txt: %s; skipping test",
+ u_errorName(ec));
+ return;
+ }
+
+ //
+ // Loop through each word in the dictionary and compare it to the previous
+ // word. They should be in sorted order.
+ //
+ UnicodeString lastWord, word;
+ int32_t failed = 0;
+ int32_t wordCount = 0;
+ while (riwords.readLineSkippingComments(word, ec, FALSE) && U_SUCCESS(ec)) {
+
+ // Show the first 8 words being compared, so we can see what's happening
+ ++wordCount;
+ if (wordCount <= 8) {
+ UnicodeString str;
+ logln((UnicodeString)"Word " + wordCount + ": " + IntlTest::prettify(word, str));
+ }
+
+ if (lastWord.length() > 0) {
+ // line enabled for j2720
+ doTest(coll, lastWord, word, Collator::LESS);
+ int32_t result = coll->compare(lastWord, word);
+
+ if (result >= 0) {
+ failed++;
+ if (MAX_FAILURES_TO_SHOW < 0 || failed <= MAX_FAILURES_TO_SHOW) {
+ UnicodeString str;
+ UnicodeString msg =
+ UnicodeString("--------------------------------------------\n")
+ + riwords.getLineNumber()
+ + " compare(" + IntlTest::prettify(lastWord, str);
+ msg += UnicodeString(", ")
+ + IntlTest::prettify(word, str) + ") returned " + result
+ + ", expected -1\n";
+ UErrorCode status = U_ZERO_ERROR;
+ CollationKey k1, k2;
+ coll->getCollationKey(lastWord, k1, status);
+ coll->getCollationKey(word, k2, status);
+ if (U_FAILURE(status)) {
+ errln((UnicodeString)"Fail: getCollationKey returned " + u_errorName(status));
+ return;
+ }
+ msg.append("key1: ").append(prettify(k1, str)).append("\n");
+ msg.append("key2: ").append(prettify(k2, str));
+ errln(msg);
+ }
+ }
+ }
+ lastWord = word;
+ }
+
+ assertSuccess("readLine", ec);
+
+ if (failed != 0) {
+ if (failed > MAX_FAILURES_TO_SHOW) {
+ errln((UnicodeString)"Too many failures; only the first " +
+ MAX_FAILURES_TO_SHOW + " failures were shown");
+ }
+ errln((UnicodeString)"Summary: " + failed + " of " + (riwords.getLineNumber() - 1) +
+ " comparisons failed");
+ }
+
+ logln((UnicodeString)"Words checked: " + wordCount);
+}
+
+/**
+ * Odd corner conditions taken from "How to Sort Thai Without Rewriting Sort",
+ * by Doug Cooper, http://seasrc.th.net/paper/thaisort.zip
+ */
+void CollationThaiTest::TestCornerCases(void) {
+ const char* TESTS[] = {
+ // Shorter words precede longer
+ "\\u0e01", "<", "\\u0e01\\u0e01",
+
+ // Tone marks are considered after letters (i.e. are primary ignorable)
+ "\\u0e01\\u0e32", "<", "\\u0e01\\u0e49\\u0e32",
+
+ // ditto for other over-marks
+ "\\u0e01\\u0e32", "<", "\\u0e01\\u0e32\\u0e4c",
+
+ // commonly used mark-in-context order.
+ // In effect, marks are sorted after each syllable.
+ "\\u0e01\\u0e32\\u0e01\\u0e49\\u0e32", "<", "\\u0e01\\u0e48\\u0e32\\u0e01\\u0e49\\u0e32",
+
+ // Hyphens and other punctuation follow whitespace but come before letters
+ "\\u0e01\\u0e32", "<", "\\u0e01\\u0e32-",
+ "\\u0e01\\u0e32-", "<", "\\u0e01\\u0e32\\u0e01\\u0e32",
+
+ // Doubler follows an indentical word without the doubler
+ "\\u0e01\\u0e32", "<", "\\u0e01\\u0e32\\u0e46",
+ "\\u0e01\\u0e32\\u0e46", "<", "\\u0e01\\u0e32\\u0e01\\u0e32",
+
+
+ // \\u0e45 after either \\u0e24 or \\u0e26 is treated as a single
+ // combining character, similar to "c < ch" in traditional spanish.
+ // TODO: beef up this case
+ "\\u0e24\\u0e29\\u0e35", "<", "\\u0e24\\u0e45\\u0e29\\u0e35",
+ "\\u0e26\\u0e29\\u0e35", "<", "\\u0e26\\u0e45\\u0e29\\u0e35",
+
+ // Vowels reorder, should compare \\u0e2d and \\u0e34
+ "\\u0e40\\u0e01\\u0e2d", "<", "\\u0e40\\u0e01\\u0e34",
+
+ // Tones are compared after the rest of the word (e.g. primary ignorable)
+ "\\u0e01\\u0e32\\u0e01\\u0e48\\u0e32", "<", "\\u0e01\\u0e49\\u0e32\\u0e01\\u0e32",
+
+ // Periods are ignored entirely
+ "\\u0e01.\\u0e01.", "<", "\\u0e01\\u0e32",
+ };
+ const int32_t TESTS_length = (int32_t)(sizeof(TESTS)/sizeof(TESTS[0]));
+
+ if (coll == 0) {
+ errln("Error: could not construct Thai collator");
+ return;
+ }
+ compareArray(*coll, TESTS, TESTS_length);
+}
+
+//------------------------------------------------------------------------
+// Internal utilities
+//------------------------------------------------------------------------
+
+void CollationThaiTest::compareArray(Collator& c, const char* tests[],
+ int32_t testsLength) {
+ for (int32_t i = 0; i < testsLength; i += 3) {
+
+ Collator::EComparisonResult expect;
+ if (tests[i+1][0] == '<') {
+ expect = Collator::LESS;
+ } else if (tests[i+1][0] == '>') {
+ expect = Collator::GREATER;
+ } else if (tests[i+1][0] == '=') {
+ expect = Collator::EQUAL;
+ } else {
+ // expect = Integer.decode(tests[i+1]).intValue();
+ errln((UnicodeString)"Error: unknown operator " + tests[i+1]);
+ return;
+ }
+
+ UnicodeString s1, s2;
+ parseChars(s1, tests[i]);
+ parseChars(s2, tests[i+2]);
+
+ doTest(&c, s1, s2, expect);
+#if 0
+ UErrorCode status = U_ZERO_ERROR;
+ int32_t result = c.compare(s1, s2);
+ if (sign(result) != sign(expect))
+ {
+ UnicodeString t1, t2;
+ errln(UnicodeString("") +
+ i/3 + ": compare(" + IntlTest::prettify(s1, t1)
+ + " , " + IntlTest::prettify(s2, t2)
+ + ") got " + result + "; expected " + expect);
+
+ CollationKey k1, k2;
+ c.getCollationKey(s1, k1, status);
+ c.getCollationKey(s2, k2, status);
+ if (U_FAILURE(status)) {
+ errln((UnicodeString)"Fail: getCollationKey returned " + u_errorName(status));
+ return;
+ }
+ errln((UnicodeString)" key1: " + prettify(k1, t1) );
+ errln((UnicodeString)" key2: " + prettify(k2, t2) );
+ }
+ else
+ {
+ // Collator.compare worked OK; now try the collation keys
+ CollationKey k1, k2;
+ c.getCollationKey(s1, k1, status);
+ c.getCollationKey(s2, k2, status);
+ if (U_FAILURE(status)) {
+ errln((UnicodeString)"Fail: getCollationKey returned " + u_errorName(status));
+ return;
+ }
+
+ result = k1.compareTo(k2);
+ if (sign(result) != sign(expect)) {
+ UnicodeString t1, t2;
+ errln(UnicodeString("") +
+ i/3 + ": key(" + IntlTest::prettify(s1, t1)
+ + ").compareTo(key(" + IntlTest::prettify(s2, t2)
+ + ")) got " + result + "; expected " + expect);
+
+ errln((UnicodeString)" " + prettify(k1, t1) + " vs. " + prettify(k2, t2));
+ }
+ }
+#endif
+ }
+}
+
+int8_t CollationThaiTest::sign(int32_t i) {
+ if (i < 0) return -1;
+ if (i > 0) return 1;
+ return 0;
+}
+
+/**
+ * Set a UnicodeString corresponding to the given string. Use
+ * UnicodeString and the default converter, unless we see the sequence
+ * "\\u", in which case we interpret the subsequent escape.
+ */
+UnicodeString& CollationThaiTest::parseChars(UnicodeString& result,
+ const char* chars) {
+ return result = CharsToUnicodeString(chars);
+}
+
+UCollator *thaiColl = NULL;
+
+U_CDECL_BEGIN
+static int U_CALLCONV
+StrCmp(const void *p1, const void *p2) {
+ return ucol_strcoll(thaiColl, *(UChar **) p1, -1, *(UChar **)p2, -1);
+}
+U_CDECL_END
+
+
+#define LINES 6
+
+void CollationThaiTest::TestInvalidThai(void) {
+ const char *tests[LINES] = {
+ "\\u0E44\\u0E01\\u0E44\\u0E01",
+ "\\u0E44\\u0E01\\u0E01\\u0E44",
+ "\\u0E01\\u0E44\\u0E01\\u0E44",
+ "\\u0E01\\u0E01\\u0E44\\u0E44",
+ "\\u0E44\\u0E44\\u0E01\\u0E01",
+ "\\u0E01\\u0E44\\u0E44\\u0E01",
+ };
+
+ UChar strings[LINES][20];
+
+ UChar *toSort[LINES];
+
+ int32_t i = 0, j = 0, len = 0;
+
+ UErrorCode coll_status = U_ZERO_ERROR;
+ UnicodeString iteratorText;
+
+ thaiColl = ucol_open ("th_TH", &coll_status);
+ if (U_FAILURE(coll_status)) {
+ errln("Error opening Thai collator: %s", u_errorName(coll_status));
+ return;
+ }
+
+ CollationElementIterator* c = ((RuleBasedCollator *)coll)->createCollationElementIterator( iteratorText );
+
+ for(i = 0; i < (int32_t)(sizeof(tests)/sizeof(tests[0])); i++) {
+ len = u_unescape(tests[i], strings[i], 20);
+ strings[i][len] = 0;
+ toSort[i] = strings[i];
+ }
+
+ qsort (toSort, LINES, sizeof (UChar *), StrCmp);
+
+ for (i=0; i < LINES; i++)
+ {
+ logln("%i", i);
+ for (j=i+1; j < LINES; j++) {
+ if (ucol_strcoll (thaiColl, toSort[i], -1, toSort[j], -1) == UCOL_GREATER)
+ {
+ // inconsistency ordering found!
+ errln("Inconsistent ordering between strings %i and %i", i, j);
+ }
+ }
+ iteratorText.setTo(toSort[i]);
+ c->setText(iteratorText, coll_status);
+ backAndForth(*c);
+ }
+
+
+ ucol_close(thaiColl);
+ delete c;
+}
+
+void CollationThaiTest::TestReordering(void) {
+ const char *tests[] = {
+ "\\u0E41c\\u0301", "=", "\\u0E41\\u0107", // composition
+ "\\u0E41\\uD835\\uDFCE", "<", "\\u0E41\\uD835\\uDFCF", // supplementaries
+ "\\u0E41\\uD834\\uDD5F", "=", "\\u0E41\\uD834\\uDD58\\uD834\\uDD65", // supplementary composition decomps to supplementary
+ "\\u0E41\\uD87E\\uDC02", "=", "\\u0E41\\u4E41", // supplementary composition decomps to BMP
+ "\\u0E41\\u0301", "=", "\\u0E41\\u0301", // unsafe (just checking backwards iteration)
+ "\\u0E41\\u0301\\u0316", "=", "\\u0E41\\u0316\\u0301",
+ // after UCA 4.1, the two lines below are not equal anymore do not have equal sign
+ "\\u0e24\\u0e41", "<", "\\u0e41\\u0e24", // exiting contraction bug
+ "\\u0e3f\\u0e3f\\u0e24\\u0e41", "<", "\\u0e3f\\u0e3f\\u0e41\\u0e24",
+
+ "abc\\u0E41c\\u0301", "=", "abc\\u0E41\\u0107", // composition
+ "abc\\u0E41\\uD834\\uDC00", "<", "abc\\u0E41\\uD834\\uDC01", // supplementaries
+ "abc\\u0E41\\uD834\\uDD5F", "=", "abc\\u0E41\\uD834\\uDD58\\uD834\\uDD65", // supplementary composition decomps to supplementary
+ "abc\\u0E41\\uD87E\\uDC02", "=", "abc\\u0E41\\u4E41", // supplementary composition decomps to BMP
+ "abc\\u0E41\\u0301", "=", "abc\\u0E41\\u0301", // unsafe (just checking backwards iteration)
+ "abc\\u0E41\\u0301\\u0316", "=", "abc\\u0E41\\u0316\\u0301",
+
+ "\\u0E41c\\u0301abc", "=", "\\u0E41\\u0107abc", // composition
+ "\\u0E41\\uD834\\uDC00abc", "<", "\\u0E41\\uD834\\uDC01abc", // supplementaries
+ "\\u0E41\\uD834\\uDD5Fabc", "=", "\\u0E41\\uD834\\uDD58\\uD834\\uDD65abc", // supplementary composition decomps to supplementary
+ "\\u0E41\\uD87E\\uDC02abc", "=", "\\u0E41\\u4E41abc", // supplementary composition decomps to BMP
+ "\\u0E41\\u0301abc", "=", "\\u0E41\\u0301abc", // unsafe (just checking backwards iteration)
+ "\\u0E41\\u0301\\u0316abc", "=", "\\u0E41\\u0316\\u0301abc",
+
+ "abc\\u0E41c\\u0301abc", "=", "abc\\u0E41\\u0107abc", // composition
+ "abc\\u0E41\\uD834\\uDC00abc", "<", "abc\\u0E41\\uD834\\uDC01abc", // supplementaries
+ "abc\\u0E41\\uD834\\uDD5Fabc", "=", "abc\\u0E41\\uD834\\uDD58\\uD834\\uDD65abc", // supplementary composition decomps to supplementary
+ "abc\\u0E41\\uD87E\\uDC02abc", "=", "abc\\u0E41\\u4E41abc", // supplementary composition decomps to BMP
+ "abc\\u0E41\\u0301abc", "=", "abc\\u0E41\\u0301abc", // unsafe (just checking backwards iteration)
+ "abc\\u0E41\\u0301\\u0316abc", "=", "abc\\u0E41\\u0316\\u0301abc",
+ };
+
+ compareArray(*coll, tests, sizeof(tests)/sizeof(tests[0]));
+
+ const char *rule = "& c < ab";
+ const char *testcontraction[] = { "\\u0E41ab", ">", "\\u0E41c"}; // After UCA 4.1 Thai are normal so won't break a contraction
+ UnicodeString rules;
+ UErrorCode status = U_ZERO_ERROR;
+ parseChars(rules, rule);
+ RuleBasedCollator *rcoll = new RuleBasedCollator(rules, status);
+ if(U_SUCCESS(status)) {
+ compareArray(*rcoll, testcontraction, 3);
+ delete rcoll;
+ } else {
+ errln("Couldn't instantiate collator from rules");
+ }
+
+}
+
+
+#endif /* #if !UCONFIG_NO_COLLATION */