summaryrefslogtreecommitdiff
path: root/Build/source/libs/icu/icu-4.6/test/thaitest/thaitest.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'Build/source/libs/icu/icu-4.6/test/thaitest/thaitest.cpp')
-rw-r--r--Build/source/libs/icu/icu-4.6/test/thaitest/thaitest.cpp571
1 files changed, 571 insertions, 0 deletions
diff --git a/Build/source/libs/icu/icu-4.6/test/thaitest/thaitest.cpp b/Build/source/libs/icu/icu-4.6/test/thaitest/thaitest.cpp
new file mode 100644
index 00000000000..657fab433c0
--- /dev/null
+++ b/Build/source/libs/icu/icu-4.6/test/thaitest/thaitest.cpp
@@ -0,0 +1,571 @@
+/*
+ ******************************************************************************
+ * Copyright (C) 1998-2003, 2006, International Business Machines Corporation *
+ * and others. All Rights Reserved. *
+ ******************************************************************************
+ */
+
+#include <errno.h>
+#include <stdio.h>
+#include <string.h>
+
+#include "unicode/utypes.h"
+#include "unicode/uchar.h"
+#include "unicode/uchriter.h"
+#include "unicode/brkiter.h"
+#include "unicode/locid.h"
+#include "unicode/unistr.h"
+#include "unicode/uniset.h"
+#include "unicode/ustring.h"
+
+/*
+ * This program takes a Unicode text file containing Thai text with
+ * spaces inserted where the word breaks are. It computes a copy of
+ * the text without spaces and uses a word instance of a Thai BreakIterator
+ * to compute the word breaks. The program reports any differences in the
+ * breaks.
+ *
+ * NOTE: by it's very nature, Thai word breaking is not exact, so it is
+ * exptected that this program will always report some differences.
+ */
+
+/*
+ * This class is a break iterator that counts words and spaces.
+ */
+class SpaceBreakIterator
+{
+public:
+ // The constructor:
+ // text - pointer to an array of UChars to iterate over
+ // count - the number of UChars in text
+ SpaceBreakIterator(const UChar *text, int32_t count);
+
+ // the destructor
+ ~SpaceBreakIterator();
+
+ // return next break position
+ int32_t next();
+
+ // return current word count
+ int32_t getWordCount();
+
+ // return current space count
+ int32_t getSpaceCount();
+
+private:
+ // No arg constructor: private so clients can't call it.
+ SpaceBreakIterator();
+
+ // The underlying BreakIterator
+ BreakIterator *fBreakIter;
+
+ // address of the UChar array
+ const UChar *fText;
+
+ // number of UChars in fText
+ int32_t fTextCount;
+
+ // current word count
+ int32_t fWordCount;
+
+ // current space count
+ int32_t fSpaceCount;
+
+ // UnicodeSet of SA characters
+ UnicodeSet fComplexContext;
+
+ // true when fBreakIter has returned DONE
+ UBool fDone;
+};
+
+/*
+ * This is the main class. It compares word breaks and reports the differences.
+ */
+class ThaiWordbreakTest
+{
+public:
+ // The main constructor:
+ // spaces - pointer to a UChar array for the text with spaces
+ // spaceCount - the number of characters in the spaces array
+ // noSpaces - pointer to a UChar array for the text without spaces
+ // noSpaceCount - the number of characters in the noSpaces array
+ // verbose - report all breaks if true, otherwise just report differences
+ ThaiWordbreakTest(const UChar *spaces, int32_t spaceCount, const UChar *noSpaces, int32_t noSpaceCount, UBool verbose);
+ ~ThaiWordbreakTest();
+
+ // returns the number of breaks that are in the spaces array
+ // but aren't found in the noSpaces array
+ int32_t getBreaksNotFound();
+
+ // returns the number of breaks which are found in the noSpaces
+ // array but aren't in the spaces array
+ int32_t getInvalidBreaks();
+
+ // returns the number of words found in the spaces array
+ int32_t getWordCount();
+
+ // reads the input Unicode text file:
+ // fileName - the path name of the file
+ // charCount - set to the number of UChars read from the file
+ // returns - the address of the UChar array containing the characters
+ static const UChar *readFile(char *fileName, int32_t &charCount);
+
+ // removes spaces form the input UChar array:
+ // spaces - pointer to the input UChar array
+ // count - number of UChars in the spaces array
+ // nonSpaceCount - the number of UChars in the result array
+ // returns - the address of the UChar array with spaces removed
+ static const UChar *crunchSpaces(const UChar *spaces, int32_t count, int32_t &nonSpaceCount);
+
+private:
+ // The no arg constructor - private so clients can't call it
+ ThaiWordbreakTest();
+
+ // This does the actual comparison:
+ // spaces - the address of the UChar array for the text with spaces
+ // spaceCount - the number of UChars in the spaces array
+ // noSpaces - the address of the UChar array for the text without spaces
+ // noSpaceCount - the number of UChars in the noSpaces array
+ // returns - true if all breaks match, FALSE otherwise
+ UBool compareWordBreaks(const UChar *spaces, int32_t spaceCount,
+ const UChar *noSpaces, int32_t noSpaceCount);
+
+ // helper method to report a break in the spaces
+ // array that's not found in the noSpaces array
+ void breakNotFound(int32_t br);
+
+ // helper method to report a break that's found in
+ // the noSpaces array that's not in the spaces array
+ void foundInvalidBreak(int32_t br);
+
+ // count of breaks in the spaces array that
+ // aren't found in the noSpaces array
+ int32_t fBreaksNotFound;
+
+ // count of breaks found in the noSpaces array
+ // that aren't in the spaces array
+ int32_t fInvalidBreaks;
+
+ // number of words found in the spaces array
+ int32_t fWordCount;
+
+ // report all breaks if true, otherwise just report differences
+ UBool fVerbose;
+};
+
+/*
+ * The main constructor: it calls compareWordBreaks and reports any differences
+ */
+ThaiWordbreakTest::ThaiWordbreakTest(const UChar *spaces, int32_t spaceCount,
+ const UChar *noSpaces, int32_t noSpaceCount, UBool verbose)
+: fBreaksNotFound(0), fInvalidBreaks(0), fWordCount(0), fVerbose(verbose)
+{
+ compareWordBreaks(spaces, spaceCount, noSpaces, noSpaceCount);
+}
+
+/*
+ * The no arg constructor
+ */
+ThaiWordbreakTest::ThaiWordbreakTest()
+{
+ // nothing
+}
+
+/*
+ * The destructor
+ */
+ThaiWordbreakTest::~ThaiWordbreakTest()
+{
+ // nothing?
+}
+
+/*
+ * returns the number of breaks in the spaces array
+ * that aren't found in the noSpaces array
+ */
+inline int32_t ThaiWordbreakTest::getBreaksNotFound()
+{
+ return fBreaksNotFound;
+}
+
+/*
+ * Returns the number of breaks found in the noSpaces
+ * array that aren't in the spaces array
+ */
+inline int32_t ThaiWordbreakTest::getInvalidBreaks()
+{
+ return fInvalidBreaks;
+}
+
+/*
+ * Returns the number of words found in the spaces array
+ */
+inline int32_t ThaiWordbreakTest::getWordCount()
+{
+ return fWordCount;
+}
+
+/*
+ * This method does the acutal break comparison and reports the results.
+ * It uses a SpaceBreakIterator to iterate over the text with spaces,
+ * and a word instance of a Thai BreakIterator to iterate over the text
+ * without spaces.
+ */
+UBool ThaiWordbreakTest::compareWordBreaks(const UChar *spaces, int32_t spaceCount,
+ const UChar *noSpaces, int32_t noSpaceCount)
+{
+ UBool result = TRUE;
+ Locale thai("th");
+ UCharCharacterIterator *noSpaceIter = new UCharCharacterIterator(noSpaces, noSpaceCount);
+ UErrorCode status = U_ZERO_ERROR;
+
+ BreakIterator *breakIter = BreakIterator::createWordInstance(thai, status);
+ breakIter->adoptText(noSpaceIter);
+
+ SpaceBreakIterator spaceIter(spaces, spaceCount);
+
+ int32_t nextBreak = 0;
+ int32_t nextSpaceBreak = 0;
+ int32_t iterCount = 0;
+
+ while (TRUE) {
+ nextSpaceBreak = spaceIter.next();
+ nextBreak = breakIter->next();
+
+ if (nextSpaceBreak == BreakIterator::DONE || nextBreak == BreakIterator::DONE) {
+ if (nextBreak != BreakIterator::DONE) {
+ fprintf(stderr, "break iterator didn't end.\n");
+ } else if (nextSpaceBreak != BreakIterator::DONE) {
+ fprintf(stderr, "premature break iterator end.\n");
+ }
+
+ break;
+ }
+
+ while (nextSpaceBreak != nextBreak &&
+ nextSpaceBreak != BreakIterator::DONE && nextBreak != BreakIterator::DONE) {
+ if (nextSpaceBreak < nextBreak) {
+ breakNotFound(nextSpaceBreak);
+ result = FALSE;
+ nextSpaceBreak = spaceIter.next();
+ } else if (nextSpaceBreak > nextBreak) {
+ foundInvalidBreak(nextBreak);
+ result = FALSE;
+ nextBreak = breakIter->next();
+ }
+ }
+
+ if (fVerbose) {
+ printf("%d %d\n", nextSpaceBreak, nextBreak);
+ }
+ }
+
+
+ fWordCount = spaceIter.getWordCount();
+
+ delete breakIter;
+
+ return result;
+}
+
+/*
+ * Report a break that's in the text with spaces but
+ * not found in the text without spaces.
+ */
+void ThaiWordbreakTest::breakNotFound(int32_t br)
+{
+ if (fVerbose) {
+ printf("%d ****\n", br);
+ } else {
+ fprintf(stderr, "break not found: %d\n", br);
+ }
+
+ fBreaksNotFound += 1;
+}
+
+/*
+ * Report a break that's found in the text without spaces
+ * that isn't in the text with spaces.
+ */
+void ThaiWordbreakTest::foundInvalidBreak(int32_t br)
+{
+ if (fVerbose) {
+ printf("**** %d\n", br);
+ } else {
+ fprintf(stderr, "found invalid break: %d\n", br);
+ }
+
+ fInvalidBreaks += 1;
+}
+
+/*
+ * Read the text from a file. The text must start with a Unicode Byte
+ * Order Mark (BOM) so that we know what order to read the bytes in.
+ */
+const UChar *ThaiWordbreakTest::readFile(char *fileName, int32_t &charCount)
+{
+ FILE *f;
+ int32_t fileSize;
+
+ UChar *buffer;
+ char *bufferChars;
+
+ f = fopen(fileName, "rb");
+
+ if( f == NULL ) {
+ fprintf(stderr,"Couldn't open %s reason: %s \n", fileName, strerror(errno));
+ return 0;
+ }
+
+ fseek(f, 0, SEEK_END);
+ fileSize = ftell(f);
+
+ fseek(f, 0, SEEK_SET);
+ bufferChars = new char[fileSize];
+
+ if(bufferChars == 0) {
+ fprintf(stderr,"Couldn't get memory for reading %s reason: %s \n", fileName, strerror(errno));
+ fclose(f);
+ return 0;
+ }
+
+ fread(bufferChars, sizeof(char), fileSize, f);
+ if( ferror(f) ) {
+ fprintf(stderr,"Couldn't read %s reason: %s \n", fileName, strerror(errno));
+ fclose(f);
+ delete[] bufferChars;
+ return 0;
+ }
+ fclose(f);
+
+ UnicodeString myText(bufferChars, fileSize, "UTF-8");
+
+ delete[] bufferChars;
+
+ charCount = myText.length();
+ buffer = new UChar[charCount];
+ if(buffer == 0) {
+ fprintf(stderr,"Couldn't get memory for reading %s reason: %s \n", fileName, strerror(errno));
+ return 0;
+ }
+
+ myText.extract(1, myText.length(), buffer);
+ charCount--; // skip the BOM
+ buffer[charCount] = 0; // NULL terminate for easier reading in the debugger
+
+ return buffer;
+}
+
+/*
+ * Remove spaces from the input UChar array.
+ *
+ * We check explicitly for a Unicode code value of 0x0020
+ * because Unicode::isSpaceChar returns true for CR, LF, etc.
+ *
+ */
+const UChar *ThaiWordbreakTest::crunchSpaces(const UChar *spaces, int32_t count, int32_t &nonSpaceCount)
+{
+ int32_t i, out, spaceCount;
+
+ spaceCount = 0;
+ for (i = 0; i < count; i += 1) {
+ if (spaces[i] == 0x0020 /*Unicode::isSpaceChar(spaces[i])*/) {
+ spaceCount += 1;
+ }
+ }
+
+ nonSpaceCount = count - spaceCount;
+ UChar *noSpaces = new UChar[nonSpaceCount];
+
+ if (noSpaces == 0) {
+ fprintf(stderr, "Couldn't allocate memory for the space stripped text.\n");
+ return 0;
+ }
+
+ for (out = 0, i = 0; i < count; i += 1) {
+ if (spaces[i] != 0x0020 /*! Unicode::isSpaceChar(spaces[i])*/) {
+ noSpaces[out++] = spaces[i];
+ }
+ }
+
+ return noSpaces;
+}
+
+/*
+ * Generate a text file with spaces in it from a file without.
+ */
+int generateFile(const UChar *chars, int32_t length) {
+ Locale root("");
+ UCharCharacterIterator *noSpaceIter = new UCharCharacterIterator(chars, length);
+ UErrorCode status = U_ZERO_ERROR;
+
+ UnicodeSet complexContext(UNICODE_STRING_SIMPLE("[:LineBreak=SA:]"), status);
+ BreakIterator *breakIter = BreakIterator::createWordInstance(root, status);
+ breakIter->adoptText(noSpaceIter);
+ char outbuf[1024];
+ int32_t strlength;
+ UChar bom = 0xFEFF;
+
+ printf("%s", u_strToUTF8(outbuf, sizeof(outbuf), &strlength, &bom, 1, &status));
+ int32_t prevbreak = 0;
+ while (U_SUCCESS(status)) {
+ int32_t nextbreak = breakIter->next();
+ if (nextbreak == BreakIterator::DONE) {
+ break;
+ }
+ printf("%s", u_strToUTF8(outbuf, sizeof(outbuf), &strlength, &chars[prevbreak],
+ nextbreak-prevbreak, &status));
+ if (nextbreak > 0 && complexContext.contains(chars[nextbreak-1])
+ && complexContext.contains(chars[nextbreak])) {
+ printf(" ");
+ }
+ prevbreak = nextbreak;
+ }
+
+ if (U_FAILURE(status)) {
+ fprintf(stderr, "generate failed: %s\n", u_errorName(status));
+ return status;
+ }
+ else {
+ return 0;
+ }
+}
+
+/*
+ * The main routine. Read the command line arguments, read the text file,
+ * remove the spaces, do the comparison and report the final results
+ */
+int main(int argc, char **argv)
+{
+ char *fileName = "space.txt";
+ int arg = 1;
+ UBool verbose = FALSE;
+ UBool generate = FALSE;
+
+ if (argc >= 2 && strcmp(argv[1], "-generate") == 0) {
+ generate = TRUE;
+ arg += 1;
+ }
+
+ if (argc >= 2 && strcmp(argv[1], "-verbose") == 0) {
+ verbose = TRUE;
+ arg += 1;
+ }
+
+ if (arg == argc - 1) {
+ fileName = argv[arg++];
+ }
+
+ if (arg != argc) {
+ fprintf(stderr, "Usage: %s [-verbose] [<file>]\n", argv[0]);
+ return 1;
+ }
+
+ int32_t spaceCount, nonSpaceCount;
+ const UChar *spaces, *noSpaces;
+
+ spaces = ThaiWordbreakTest::readFile(fileName, spaceCount);
+
+ if (spaces == 0) {
+ return 1;
+ }
+
+ if (generate) {
+ return generateFile(spaces, spaceCount);
+ }
+
+ noSpaces = ThaiWordbreakTest::crunchSpaces(spaces, spaceCount, nonSpaceCount);
+
+ if (noSpaces == 0) {
+ return 1;
+ }
+
+ ThaiWordbreakTest test(spaces, spaceCount, noSpaces, nonSpaceCount, verbose);
+
+ printf("word count: %d\n", test.getWordCount());
+ printf("breaks not found: %d\n", test.getBreaksNotFound());
+ printf("invalid breaks found: %d\n", test.getInvalidBreaks());
+
+ return 0;
+}
+
+/*
+ * The main constructor. Clear all the counts and construct a default
+ * word instance of a BreakIterator.
+ */
+SpaceBreakIterator::SpaceBreakIterator(const UChar *text, int32_t count)
+ : fBreakIter(0), fText(text), fTextCount(count), fWordCount(0), fSpaceCount(0), fDone(FALSE)
+{
+ UCharCharacterIterator *iter = new UCharCharacterIterator(text, count);
+ UErrorCode status = U_ZERO_ERROR;
+ fComplexContext.applyPattern(UNICODE_STRING_SIMPLE("[:LineBreak=SA:]"), status);
+ Locale root("");
+
+ fBreakIter = BreakIterator::createWordInstance(root, status);
+ fBreakIter->adoptText(iter);
+}
+
+SpaceBreakIterator::SpaceBreakIterator()
+{
+ // nothing
+}
+
+/*
+ * The destructor. delete the underlying BreakIterator
+ */
+SpaceBreakIterator::~SpaceBreakIterator()
+{
+ delete fBreakIter;
+}
+
+/*
+ * Return the next break, counting words and spaces.
+ */
+int32_t SpaceBreakIterator::next()
+{
+ if (fDone) {
+ return BreakIterator::DONE;
+ }
+
+ int32_t nextBreak;
+ do {
+ nextBreak = fBreakIter->next();
+
+ if (nextBreak == BreakIterator::DONE) {
+ fDone = TRUE;
+ return BreakIterator::DONE;
+ }
+ }
+ while(nextBreak > 0 && fComplexContext.contains(fText[nextBreak-1])
+ && fComplexContext.contains(fText[nextBreak]));
+
+ int32_t result = nextBreak - fSpaceCount;
+
+ if (nextBreak < fTextCount) {
+ if (fText[nextBreak] == 0x0020 /*Unicode::isSpaceChar(fText[nextBreak])*/) {
+ fSpaceCount += fBreakIter->next() - nextBreak;
+ }
+ }
+
+ fWordCount += 1;
+
+ return result;
+}
+
+/*
+ * Returns the current space count
+ */
+int32_t SpaceBreakIterator::getSpaceCount()
+{
+ return fSpaceCount;
+}
+
+/*
+ * Returns the current word count
+ */
+int32_t SpaceBreakIterator::getWordCount()
+{
+ return fWordCount;
+}
+
+