diff options
author | Akira Kakuto <kakuto@fuk.kindai.ac.jp> | 2016-02-22 09:24:46 +0000 |
---|---|---|
committer | Akira Kakuto <kakuto@fuk.kindai.ac.jp> | 2016-02-22 09:24:46 +0000 |
commit | 517a70c531cc54407192e7aebcfafaf6d8f9c3e2 (patch) | |
tree | 41e8f96128d967cfbef420baa31cb8c22062529e /Build/source/libs/icu/icu-src/source/test/cintltst/ucsdetst.c | |
parent | 553ae4df24322a4fb019c63b81e98093f2b1a6a2 (diff) |
libs/icu: New convention
git-svn-id: svn://tug.org/texlive/trunk@39817 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Build/source/libs/icu/icu-src/source/test/cintltst/ucsdetst.c')
-rw-r--r-- | Build/source/libs/icu/icu-src/source/test/cintltst/ucsdetst.c | 590 |
1 files changed, 590 insertions, 0 deletions
diff --git a/Build/source/libs/icu/icu-src/source/test/cintltst/ucsdetst.c b/Build/source/libs/icu/icu-src/source/test/cintltst/ucsdetst.c new file mode 100644 index 00000000000..d2ad1e04476 --- /dev/null +++ b/Build/source/libs/icu/icu-src/source/test/cintltst/ucsdetst.c @@ -0,0 +1,590 @@ +/* + **************************************************************************** + * Copyright (c) 2005-2009, International Business Machines Corporation and * + * others. All Rights Reserved. * + **************************************************************************** + */ + +#include "unicode/utypes.h" + +#include "unicode/ucsdet.h" +#include "unicode/ucnv.h" +#include "unicode/ustring.h" + +#include "cintltst.h" + +#include <stdlib.h> +#include <string.h> + +#define ARRAY_SIZE(array) (sizeof(array)/sizeof(array[0])) + +#define NEW_ARRAY(type,count) (type *) malloc((count) * sizeof(type)) +#define DELETE_ARRAY(array) free(array) + +static void TestConstruction(void); +static void TestUTF8(void); +static void TestUTF16(void); +static void TestC1Bytes(void); +static void TestInputFilter(void); +static void TestChaining(void); +static void TestBufferOverflow(void); +static void TestIBM424(void); +static void TestIBM420(void); + +void addUCsdetTest(TestNode** root); + +void addUCsdetTest(TestNode** root) +{ + addTest(root, &TestConstruction, "ucsdetst/TestConstruction"); + addTest(root, &TestUTF8, "ucsdetst/TestUTF8"); + addTest(root, &TestUTF16, "ucsdetst/TestUTF16"); + addTest(root, &TestC1Bytes, "ucsdetst/TestC1Bytes"); + addTest(root, &TestInputFilter, "ucsdetst/TestInputFilter"); + addTest(root, &TestChaining, "ucsdetst/TestErrorChaining"); + addTest(root, &TestBufferOverflow, "ucsdetst/TestBufferOverflow"); +#if !UCONFIG_NO_LEGACY_CONVERSION + addTest(root, &TestIBM424, "ucsdetst/TestIBM424"); + addTest(root, &TestIBM420, "ucsdetst/TestIBM420"); +#endif +} + +static int32_t preflight(const UChar *src, int32_t length, UConverter *cnv) +{ + UErrorCode status; + char buffer[1024]; + char *dest, *destLimit = buffer + sizeof(buffer); + const UChar *srcLimit = src + length; + int32_t result = 0; + + do { + dest = buffer; + status = U_ZERO_ERROR; + ucnv_fromUnicode(cnv, &dest, destLimit, &src, srcLimit, 0, TRUE, &status); + result += (int32_t) (dest - buffer); + } while (status == U_BUFFER_OVERFLOW_ERROR); + + return result; +} + +static char *extractBytes(const UChar *src, int32_t length, const char *codepage, int32_t *byteLength) +{ + UErrorCode status = U_ZERO_ERROR; + UConverter *cnv = ucnv_open(codepage, &status); + int32_t byteCount = preflight(src, length, cnv); + const UChar *srcLimit = src + length; + char *bytes = NEW_ARRAY(char, byteCount + 1); + char *dest = bytes, *destLimit = bytes + byteCount + 1; + + ucnv_fromUnicode(cnv, &dest, destLimit, &src, srcLimit, 0, TRUE, &status); + ucnv_close(cnv); + + *byteLength = byteCount; + return bytes; +} + +static void freeBytes(char *bytes) +{ + DELETE_ARRAY(bytes); +} + +static void TestConstruction(void) +{ + UErrorCode status = U_ZERO_ERROR; + UCharsetDetector *csd = ucsdet_open(&status); + UEnumeration *e = ucsdet_getAllDetectableCharsets(csd, &status); + const char *name; + int32_t count = uenum_count(e, &status); + int32_t i, length; + + for(i = 0; i < count; i += 1) { + name = uenum_next(e, &length, &status); + + if(name == NULL || length <= 0) { + log_err("ucsdet_getAllDetectableCharsets() returned a null or empty name!\n"); + } + } + /* one past the list of all names must return NULL */ + name = uenum_next(e, &length, &status); + if(name != NULL || length != 0 || U_FAILURE(status)) { + log_err("ucsdet_getAllDetectableCharsets(past the list) returned a non-null name!\n"); + } + + uenum_close(e); + ucsdet_close(csd); +} + +static void TestUTF8(void) +{ + UErrorCode status = U_ZERO_ERROR; + static const char ss[] = "This is a string with some non-ascii characters that will " + "be converted to UTF-8, then shoved through the detection process. " + "\\u0391\\u0392\\u0393\\u0394\\u0395" + "Sure would be nice if our source could contain Unicode directly!"; + int32_t byteLength = 0, sLength = 0, dLength = 0; + UChar s[sizeof(ss)]; + char *bytes; + UCharsetDetector *csd = ucsdet_open(&status); + const UCharsetMatch *match; + UChar detected[sizeof(ss)]; + + sLength = u_unescape(ss, s, sizeof(ss)); + bytes = extractBytes(s, sLength, "UTF-8", &byteLength); + + ucsdet_setText(csd, bytes, byteLength, &status); + if (U_FAILURE(status)) { + log_err("status is %s\n", u_errorName(status)); + goto bail; + } + + match = ucsdet_detect(csd, &status); + + if (match == NULL) { + log_err("Detection failure for UTF-8: got no matches.\n"); + goto bail; + } + + dLength = ucsdet_getUChars(match, detected, sLength, &status); + + if (u_strCompare(detected, dLength, s, sLength, FALSE) != 0) { + log_err("Round-trip test failed!\n"); + } + + ucsdet_setDeclaredEncoding(csd, "UTF-8", 5, &status); /* for coverage */ + +bail: + freeBytes(bytes); + ucsdet_close(csd); +} + +static void TestUTF16(void) +{ + UErrorCode status = U_ZERO_ERROR; + /* Notice the BOM on the start of this string */ + static const UChar chars[] = { + 0xFEFF, 0x0623, 0x0648, 0x0631, 0x0648, 0x0628, 0x0627, 0x002C, + 0x0020, 0x0628, 0x0631, 0x0645, 0x062c, 0x064a, 0x0627, 0x062a, + 0x0020, 0x0627, 0x0644, 0x062d, 0x0627, 0x0633, 0x0648, 0x0628, + 0x0020, 0x002b, 0x0020, 0x0627, 0x0646, 0x062a, 0x0631, 0x0646, + 0x064a, 0x062a, 0x0000}; + int32_t beLength = 0, leLength = 0, cLength = ARRAY_SIZE(chars); + char *beBytes = extractBytes(chars, cLength, "UTF-16BE", &beLength); + char *leBytes = extractBytes(chars, cLength, "UTF-16LE", &leLength); + UCharsetDetector *csd = ucsdet_open(&status); + const UCharsetMatch *match; + const char *name; + int32_t conf; + + ucsdet_setText(csd, beBytes, beLength, &status); + match = ucsdet_detect(csd, &status); + + if (match == NULL) { + log_err("Encoding detection failure for UTF-16BE: got no matches.\n"); + goto try_le; + } + + name = ucsdet_getName(match, &status); + conf = ucsdet_getConfidence(match, &status); + + if (strcmp(name, "UTF-16BE") != 0) { + log_err("Encoding detection failure for UTF-16BE: got %s\n", name); + } + + if (conf != 100) { + log_err("Did not get 100%% confidence for UTF-16BE: got %d\n", conf); + } + +try_le: + ucsdet_setText(csd, leBytes, leLength, &status); + match = ucsdet_detect(csd, &status); + + if (match == NULL) { + log_err("Encoding detection failure for UTF-16LE: got no matches.\n"); + goto bail; + } + + name = ucsdet_getName(match, &status); + conf = ucsdet_getConfidence(match, &status); + + + if (strcmp(name, "UTF-16LE") != 0) { + log_err("Enconding detection failure for UTF-16LE: got %s\n", name); + } + + if (conf != 100) { + log_err("Did not get 100%% confidence for UTF-16LE: got %d\n", conf); + } + +bail: + freeBytes(leBytes); + freeBytes(beBytes); + ucsdet_close(csd); +} + +static void TestC1Bytes(void) +{ +#if !UCONFIG_NO_LEGACY_CONVERSION + UErrorCode status = U_ZERO_ERROR; + static const char ssISO[] = "This is a small sample of some English text. Just enough to be sure that it detects correctly."; + static const char ssWindows[] = "This is another small sample of some English text. Just enough to be sure that it detects correctly. It also includes some \\u201CC1\\u201D bytes."; + int32_t sISOLength = 0, sWindowsLength = 0; + UChar sISO[sizeof(ssISO)]; + UChar sWindows[sizeof(ssWindows)]; + int32_t lISO = 0, lWindows = 0; + char *bISO; + char *bWindows; + UCharsetDetector *csd = ucsdet_open(&status); + const UCharsetMatch *match; + const char *name; + + sISOLength = u_unescape(ssISO, sISO, sizeof(ssISO)); + sWindowsLength = u_unescape(ssWindows, sWindows, sizeof(ssWindows)); + bISO = extractBytes(sISO, sISOLength, "ISO-8859-1", &lISO); + bWindows = extractBytes(sWindows, sWindowsLength, "windows-1252", &lWindows); + + ucsdet_setText(csd, bWindows, lWindows, &status); + match = ucsdet_detect(csd, &status); + + if (match == NULL) { + log_err("English test with C1 bytes got no matches.\n"); + goto bail; + } + + name = ucsdet_getName(match, &status); + + if (strcmp(name, "windows-1252") != 0) { + log_data_err("English text with C1 bytes does not detect as windows-1252, but as %s. (Are you missing data?)\n", name); + } + + ucsdet_setText(csd, bISO, lISO, &status); + match = ucsdet_detect(csd, &status); + + if (match == NULL) { + log_err("English text without C1 bytes got no matches.\n"); + goto bail; + } + + name = ucsdet_getName(match, &status); + + if (strcmp(name, "ISO-8859-1") != 0) { + log_err("English text without C1 bytes does not detect as ISO-8859-1, but as %s\n", name); + } + +bail: + freeBytes(bWindows); + freeBytes(bISO); + + ucsdet_close(csd); +#endif +} + +static void TestInputFilter(void) +{ + UErrorCode status = U_ZERO_ERROR; + static const char ss[] = "<a> <lot> <of> <English> <inside> <the> <markup> Un tr\\u00E8s petit peu de Fran\\u00E7ais. <to> <confuse> <the> <detector>"; + int32_t sLength = 0; + UChar s[sizeof(ss)]; + int32_t byteLength = 0; + char *bytes; + UCharsetDetector *csd = ucsdet_open(&status); + const UCharsetMatch *match; + const char *lang, *name; + + sLength = u_unescape(ss, s, sizeof(ss)); + bytes = extractBytes(s, sLength, "ISO-8859-1", &byteLength); + + ucsdet_enableInputFilter(csd, TRUE); + + if (!ucsdet_isInputFilterEnabled(csd)) { + log_err("ucsdet_enableInputFilter(csd, TRUE) did not enable input filter!\n"); + } + + + ucsdet_setText(csd, bytes, byteLength, &status); + match = ucsdet_detect(csd, &status); + + if (match == NULL) { + log_err("Turning on the input filter resulted in no matches.\n"); + goto turn_off; + } + + name = ucsdet_getName(match, &status); + + if (name == NULL || strcmp(name, "ISO-8859-1") != 0) { + log_err("Turning on the input filter resulted in %s rather than ISO-8859-1\n", name); + } else { + lang = ucsdet_getLanguage(match, &status); + + if (lang == NULL || strcmp(lang, "fr") != 0) { + log_err("Input filter did not strip markup!\n"); + } + } + +turn_off: + ucsdet_enableInputFilter(csd, FALSE); + ucsdet_setText(csd, bytes, byteLength, &status); + match = ucsdet_detect(csd, &status); + + if (match == NULL) { + log_err("Turning off the input filter resulted in no matches.\n"); + goto bail; + } + + name = ucsdet_getName(match, &status); + + if (name == NULL || strcmp(name, "ISO-8859-1") != 0) { + log_err("Turning off the input filter resulted in %s rather than ISO-8859-1\n", name); + } else { + lang = ucsdet_getLanguage(match, &status); + + if (lang == NULL || strcmp(lang, "en") != 0) { + log_err("Unfiltered input did not detect as English!\n"); + } + } + +bail: + freeBytes(bytes); + ucsdet_close(csd); +} + +static void TestChaining(void) { + UErrorCode status = U_USELESS_COLLATOR_ERROR; + + ucsdet_open(&status); + ucsdet_setText(NULL, NULL, 0, &status); + ucsdet_getName(NULL, &status); + ucsdet_getConfidence(NULL, &status); + ucsdet_getLanguage(NULL, &status); + ucsdet_detect(NULL, &status); + ucsdet_setDeclaredEncoding(NULL, NULL, 0, &status); + ucsdet_detectAll(NULL, NULL, &status); + ucsdet_getUChars(NULL, NULL, 0, &status); + ucsdet_getUChars(NULL, NULL, 0, &status); + ucsdet_close(NULL); + + /* All of this code should have done nothing. */ + if (status != U_USELESS_COLLATOR_ERROR) { + log_err("Status got changed to %s\n", u_errorName(status)); + } +} + +static void TestBufferOverflow(void) { + UErrorCode status = U_ZERO_ERROR; + static const char *testStrings[] = { + "\x80\x20\x54\x68\x69\x73\x20\x69\x73\x20\x45\x6E\x67\x6C\x69\x73\x68\x20\x1b", /* A partial ISO-2022 shift state at the end */ + "\x80\x20\x54\x68\x69\x73\x20\x69\x73\x20\x45\x6E\x67\x6C\x69\x73\x68\x20\x1b\x24", /* A partial ISO-2022 shift state at the end */ + "\x80\x20\x54\x68\x69\x73\x20\x69\x73\x20\x45\x6E\x67\x6C\x69\x73\x68\x20\x1b\x24\x28", /* A partial ISO-2022 shift state at the end */ + "\x80\x20\x54\x68\x69\x73\x20\x69\x73\x20\x45\x6E\x67\x6C\x69\x73\x68\x20\x1b\x24\x28\x44", /* A complete ISO-2022 shift state at the end with a bad one at the start */ + "\x1b\x24\x28\x44", /* A complete ISO-2022 shift state at the end */ + "\xa1", /* Could be a single byte shift-jis at the end */ + "\x74\x68\xa1", /* Could be a single byte shift-jis at the end */ + "\x74\x68\x65\xa1" /* Could be a single byte shift-jis at the end, but now we have English creeping in. */ + }; + static const char *testResults[] = { + "windows-1252", + "windows-1252", + "windows-1252", + "windows-1252", + "ISO-2022-JP", + NULL, + NULL, + "ISO-8859-1" + }; + int32_t idx = 0; + UCharsetDetector *csd = ucsdet_open(&status); + const UCharsetMatch *match; + + ucsdet_setDeclaredEncoding(csd, "ISO-2022-JP", -1, &status); + + if (U_FAILURE(status)) { + log_err("Couldn't open detector. %s\n", u_errorName(status)); + goto bail; + } + + for (idx = 0; idx < ARRAY_SIZE(testStrings); idx++) { + ucsdet_setText(csd, testStrings[idx], -1, &status); + match = ucsdet_detect(csd, &status); + + if (match == NULL) { + if (testResults[idx] != NULL) { + log_err("Unexpectedly got no results at index %d.\n", idx); + } + else { + log_verbose("Got no result as expected at index %d.\n", idx); + } + continue; + } + + if (testResults[idx] == NULL || strcmp(ucsdet_getName(match, &status), testResults[idx]) != 0) { + log_err("Unexpectedly got %s instead of %s at index %d with confidence %d.\n", + ucsdet_getName(match, &status), testResults[idx], idx, ucsdet_getConfidence(match, &status)); + goto bail; + } + } + +bail: + ucsdet_close(csd); +} + +static void TestIBM424(void) +{ + UErrorCode status = U_ZERO_ERROR; + + static const UChar chars[] = { + 0x05D4, 0x05E4, 0x05E8, 0x05E7, 0x05DC, 0x05D9, 0x05D8, 0x0020, 0x05D4, 0x05E6, 0x05D1, 0x05D0, 0x05D9, 0x0020, 0x05D4, 0x05E8, + 0x05D0, 0x05E9, 0x05D9, 0x002C, 0x0020, 0x05EA, 0x05EA, 0x0020, 0x05D0, 0x05DC, 0x05D5, 0x05E3, 0x0020, 0x05D0, 0x05D1, 0x05D9, + 0x05D7, 0x05D9, 0x0020, 0x05DE, 0x05E0, 0x05D3, 0x05DC, 0x05D1, 0x05DC, 0x05D9, 0x05D8, 0x002C, 0x0020, 0x05D4, 0x05D5, 0x05E8, + 0x05D4, 0x0020, 0x05E2, 0x05DC, 0x0020, 0x05E4, 0x05EA, 0x05D9, 0x05D7, 0x05EA, 0x0020, 0x05D7, 0x05E7, 0x05D9, 0x05E8, 0x05EA, + 0x0020, 0x05DE, 0x05E6, 0x0022, 0x05D7, 0x0020, 0x05D1, 0x05E2, 0x05E7, 0x05D1, 0x05D5, 0x05EA, 0x0020, 0x05E2, 0x05D3, 0x05D5, + 0x05D9, 0x05D5, 0x05EA, 0x0020, 0x05D7, 0x05D9, 0x05D9, 0x05DC, 0x05D9, 0x0020, 0x05E6, 0x05D4, 0x0022, 0x05DC, 0x0020, 0x05DE, + 0x05DE, 0x05D1, 0x05E6, 0x05E2, 0x0020, 0x05E2, 0x05D5, 0x05E4, 0x05E8, 0x05EA, 0x0020, 0x05D9, 0x05E6, 0x05D5, 0x05E7, 0x05D4, + 0x0020, 0x05D1, 0x002B, 0x0020, 0x05E8, 0x05E6, 0x05D5, 0x05E2, 0x05EA, 0x0020, 0x05E2, 0x05D6, 0x05D4, 0x002E, 0x0020, 0x05DC, + 0x05D3, 0x05D1, 0x05E8, 0x05D9, 0x0020, 0x05D4, 0x05E4, 0x05E6, 0x0022, 0x05E8, 0x002C, 0x0020, 0x05DE, 0x05D4, 0x05E2, 0x05D3, + 0x05D5, 0x05D9, 0x05D5, 0x05EA, 0x0020, 0x05E2, 0x05D5, 0x05DC, 0x05D4, 0x0020, 0x05EA, 0x05DE, 0x05D5, 0x05E0, 0x05D4, 0x0020, + 0x05E9, 0x05DC, 0x0020, 0x0022, 0x05D4, 0x05EA, 0x05E0, 0x05D4, 0x05D2, 0x05D5, 0x05EA, 0x0020, 0x05E4, 0x05E1, 0x05D5, 0x05DC, + 0x05D4, 0x0020, 0x05DC, 0x05DB, 0x05D0, 0x05D5, 0x05E8, 0x05D4, 0x0020, 0x05E9, 0x05DC, 0x0020, 0x05D7, 0x05D9, 0x05D9, 0x05DC, + 0x05D9, 0x05DD, 0x0020, 0x05D1, 0x05DE, 0x05D4, 0x05DC, 0x05DA, 0x0020, 0x05DE, 0x05D1, 0x05E6, 0x05E2, 0x0020, 0x05E2, 0x05D5, + 0x05E4, 0x05E8, 0x05EA, 0x0020, 0x05D9, 0x05E6, 0x05D5, 0x05E7, 0x05D4, 0x0022, 0x002E, 0x0020, 0x05DE, 0x05E0, 0x05D3, 0x05DC, + 0x05D1, 0x05DC, 0x05D9, 0x05D8, 0x0020, 0x05E7, 0x05D9, 0x05D1, 0x05DC, 0x0020, 0x05D0, 0x05EA, 0x0020, 0x05D4, 0x05D7, 0x05DC, + 0x05D8, 0x05EA, 0x05D5, 0x0020, 0x05DC, 0x05D0, 0x05D7, 0x05E8, 0x0020, 0x05E9, 0x05E2, 0x05D9, 0x05D9, 0x05DF, 0x0020, 0x05D1, + 0x05EA, 0x05DE, 0x05DC, 0x05D9, 0x05DC, 0x0020, 0x05D4, 0x05E2, 0x05D3, 0x05D5, 0x05D9, 0x05D5, 0x05EA, 0x0000 + }; + + static const UChar chars_reverse[] = { + 0x05EA, 0x05D5, 0x05D9, 0x05D5, 0x05D3, 0x05E2, 0x05D4, 0x0020, 0x05DC, 0x05D9, 0x05DC, 0x05DE, 0x05EA, + 0x05D1, 0x0020, 0x05DF, 0x05D9, 0x05D9, 0x05E2, 0x05E9, 0x0020, 0x05E8, 0x05D7, 0x05D0, 0x05DC, 0x0020, 0x05D5, 0x05EA, 0x05D8, + 0x05DC, 0x05D7, 0x05D4, 0x0020, 0x05EA, 0x05D0, 0x0020, 0x05DC, 0x05D1, 0x05D9, 0x05E7, 0x0020, 0x05D8, 0x05D9, 0x05DC, 0x05D1, + 0x05DC, 0x05D3, 0x05E0, 0x05DE, 0x0020, 0x002E, 0x0022, 0x05D4, 0x05E7, 0x05D5, 0x05E6, 0x05D9, 0x0020, 0x05EA, 0x05E8, 0x05E4, + 0x05D5, 0x05E2, 0x0020, 0x05E2, 0x05E6, 0x05D1, 0x05DE, 0x0020, 0x05DA, 0x05DC, 0x05D4, 0x05DE, 0x05D1, 0x0020, 0x05DD, 0x05D9, + 0x05DC, 0x05D9, 0x05D9, 0x05D7, 0x0020, 0x05DC, 0x05E9, 0x0020, 0x05D4, 0x05E8, 0x05D5, 0x05D0, 0x05DB, 0x05DC, 0x0020, 0x05D4, + 0x05DC, 0x05D5, 0x05E1, 0x05E4, 0x0020, 0x05EA, 0x05D5, 0x05D2, 0x05D4, 0x05E0, 0x05EA, 0x05D4, 0x0022, 0x0020, 0x05DC, 0x05E9, + 0x0020, 0x05D4, 0x05E0, 0x05D5, 0x05DE, 0x05EA, 0x0020, 0x05D4, 0x05DC, 0x05D5, 0x05E2, 0x0020, 0x05EA, 0x05D5, 0x05D9, 0x05D5, + 0x05D3, 0x05E2, 0x05D4, 0x05DE, 0x0020, 0x002C, 0x05E8, 0x0022, 0x05E6, 0x05E4, 0x05D4, 0x0020, 0x05D9, 0x05E8, 0x05D1, 0x05D3, + 0x05DC, 0x0020, 0x002E, 0x05D4, 0x05D6, 0x05E2, 0x0020, 0x05EA, 0x05E2, 0x05D5, 0x05E6, 0x05E8, 0x0020, 0x002B, 0x05D1, 0x0020, + 0x05D4, 0x05E7, 0x05D5, 0x05E6, 0x05D9, 0x0020, 0x05EA, 0x05E8, 0x05E4, 0x05D5, 0x05E2, 0x0020, 0x05E2, 0x05E6, 0x05D1, 0x05DE, + 0x05DE, 0x0020, 0x05DC, 0x0022, 0x05D4, 0x05E6, 0x0020, 0x05D9, 0x05DC, 0x05D9, 0x05D9, 0x05D7, 0x0020, 0x05EA, 0x05D5, 0x05D9, + 0x05D5, 0x05D3, 0x05E2, 0x0020, 0x05EA, 0x05D5, 0x05D1, 0x05E7, 0x05E2, 0x05D1, 0x0020, 0x05D7, 0x0022, 0x05E6, 0x05DE, 0x0020, + 0x05EA, 0x05E8, 0x05D9, 0x05E7, 0x05D7, 0x0020, 0x05EA, 0x05D7, 0x05D9, 0x05EA, 0x05E4, 0x0020, 0x05DC, 0x05E2, 0x0020, 0x05D4, + 0x05E8, 0x05D5, 0x05D4, 0x0020, 0x002C, 0x05D8, 0x05D9, 0x05DC, 0x05D1, 0x05DC, 0x05D3, 0x05E0, 0x05DE, 0x0020, 0x05D9, 0x05D7, + 0x05D9, 0x05D1, 0x05D0, 0x0020, 0x05E3, 0x05D5, 0x05DC, 0x05D0, 0x0020, 0x05EA, 0x05EA, 0x0020, 0x002C, 0x05D9, 0x05E9, 0x05D0, + 0x05E8, 0x05D4, 0x0020, 0x05D9, 0x05D0, 0x05D1, 0x05E6, 0x05D4, 0x0020, 0x05D8, 0x05D9, 0x05DC, 0x05E7, 0x05E8, 0x05E4, 0x05D4, + 0x0000 + }; + + int32_t bLength = 0, brLength = 0, cLength = ARRAY_SIZE(chars), crLength = ARRAY_SIZE(chars_reverse); + + char *bytes = extractBytes(chars, cLength, "IBM424", &bLength); + char *bytes_r = extractBytes(chars_reverse, crLength, "IBM424", &brLength); + + UCharsetDetector *csd = ucsdet_open(&status); + const UCharsetMatch *match; + const char *name; + + ucsdet_setText(csd, bytes, bLength, &status); + match = ucsdet_detect(csd, &status); + + if (match == NULL) { + log_err("Encoding detection failure for IBM424_rtl: got no matches.\n"); + goto bail; + } + + name = ucsdet_getName(match, &status); + if (strcmp(name, "IBM424_rtl") != 0) { + log_data_err("Encoding detection failure for IBM424_rtl: got %s. (Are you missing data?)\n", name); + } + + ucsdet_setText(csd, bytes_r, brLength, &status); + match = ucsdet_detect(csd, &status); + + if (match == NULL) { + log_err("Encoding detection failure for IBM424_ltr: got no matches.\n"); + goto bail; + } + + name = ucsdet_getName(match, &status); + if (strcmp(name, "IBM424_ltr") != 0) { + log_data_err("Encoding detection failure for IBM424_ltr: got %s. (Are you missing data?)\n", name); + } + +bail: + freeBytes(bytes); + freeBytes(bytes_r); + ucsdet_close(csd); +} + +static void TestIBM420(void) +{ + UErrorCode status = U_ZERO_ERROR; + + static const UChar chars[] = { + 0x0648, 0x064F, 0x0636, 0x0639, 0x062A, 0x0020, 0x0648, 0x0646, 0x064F, 0x0641, 0x0630, 0x062A, 0x0020, 0x0628, 0x0631, 0x0627, + 0x0645, 0x062C, 0x0020, 0x062A, 0x0623, 0x0645, 0x064A, 0x0646, 0x0020, 0x0639, 0x062F, 0x064A, 0x062F, 0x0629, 0x0020, 0x0641, + 0x064A, 0x0020, 0x0645, 0x0624, 0x0633, 0x0633, 0x0629, 0x0020, 0x0627, 0x0644, 0x062A, 0x0623, 0x0645, 0x064A, 0x0646, 0x0020, + 0x0627, 0x0644, 0x0648, 0x0637, 0x0646, 0x064A, 0x002C, 0x0020, 0x0645, 0x0639, 0x0020, 0x0645, 0x0644, 0x0627, 0x0626, 0x0645, + 0x062A, 0x0647, 0x0627, 0x0020, 0x062F, 0x0627, 0x0626, 0x0645, 0x0627, 0x064B, 0x0020, 0x0644, 0x0644, 0x0627, 0x062D, 0x062A, + 0x064A, 0x0627, 0x062C, 0x0627, 0x062A, 0x0020, 0x0627, 0x0644, 0x0645, 0x062A, 0x063A, 0x064A, 0x0631, 0x0629, 0x0020, 0x0644, + 0x0644, 0x0645, 0x062C, 0x062A, 0x0645, 0x0639, 0x0020, 0x0648, 0x0644, 0x0644, 0x062F, 0x0648, 0x0644, 0x0629, 0x002E, 0x0020, + 0x062A, 0x0648, 0x0633, 0x0639, 0x062A, 0x0020, 0x0648, 0x062A, 0x0637, 0x0648, 0x0631, 0x062A, 0x0020, 0x0627, 0x0644, 0x0645, + 0x0624, 0x0633, 0x0633, 0x0629, 0x0020, 0x0628, 0x0647, 0x062F, 0x0641, 0x0020, 0x0636, 0x0645, 0x0627, 0x0646, 0x0020, 0x0634, + 0x0628, 0x0643, 0x0629, 0x0020, 0x0623, 0x0645, 0x0627, 0x0646, 0x0020, 0x0644, 0x0633, 0x0643, 0x0627, 0x0646, 0x0020, 0x062F, + 0x0648, 0x0644, 0x0629, 0x0020, 0x0627, 0x0633, 0x0631, 0x0627, 0x0626, 0x064A, 0x0644, 0x0020, 0x0628, 0x0648, 0x062C, 0x0647, + 0x0020, 0x0627, 0x0644, 0x0645, 0x062E, 0x0627, 0x0637, 0x0631, 0x0020, 0x0627, 0x0644, 0x0627, 0x0642, 0x062A, 0x0635, 0x0627, + 0x062F, 0x064A, 0x0629, 0x0020, 0x0648, 0x0627, 0x0644, 0x0627, 0x062C, 0x062A, 0x0645, 0x0627, 0x0639, 0x064A, 0x0629, 0x002E, + 0x0000 + }; + static const UChar chars_reverse[] = { + 0x002E, 0x0629, 0x064A, 0x0639, 0x0627, 0x0645, 0x062A, 0x062C, 0x0627, 0x0644, 0x0627, 0x0648, 0x0020, 0x0629, 0x064A, 0x062F, + 0x0627, 0x0635, 0x062A, 0x0642, 0x0627, 0x0644, 0x0627, 0x0020, 0x0631, 0x0637, 0x0627, 0x062E, 0x0645, 0x0644, 0x0627, 0x0020, + 0x0647, 0x062C, 0x0648, 0x0628, 0x0020, 0x0644, 0x064A, 0x0626, 0x0627, 0x0631, 0x0633, 0x0627, 0x0020, 0x0629, 0x0644, 0x0648, + 0x062F, 0x0020, 0x0646, 0x0627, 0x0643, 0x0633, 0x0644, 0x0020, 0x0646, 0x0627, 0x0645, 0x0623, 0x0020, 0x0629, 0x0643, 0x0628, + 0x0634, 0x0020, 0x0646, 0x0627, 0x0645, 0x0636, 0x0020, 0x0641, 0x062F, 0x0647, 0x0628, 0x0020, 0x0629, 0x0633, 0x0633, 0x0624, + 0x0645, 0x0644, 0x0627, 0x0020, 0x062A, 0x0631, 0x0648, 0x0637, 0x062A, 0x0648, 0x0020, 0x062A, 0x0639, 0x0633, 0x0648, 0x062A, + 0x0020, 0x002E, 0x0629, 0x0644, 0x0648, 0x062F, 0x0644, 0x0644, 0x0648, 0x0020, 0x0639, 0x0645, 0x062A, 0x062C, 0x0645, 0x0644, + 0x0644, 0x0020, 0x0629, 0x0631, 0x064A, 0x063A, 0x062A, 0x0645, 0x0644, 0x0627, 0x0020, 0x062A, 0x0627, 0x062C, 0x0627, 0x064A, + 0x062A, 0x062D, 0x0627, 0x0644, 0x0644, 0x0020, 0x064B, 0x0627, 0x0645, 0x0626, 0x0627, 0x062F, 0x0020, 0x0627, 0x0647, 0x062A, + 0x0645, 0x0626, 0x0627, 0x0644, 0x0645, 0x0020, 0x0639, 0x0645, 0x0020, 0x002C, 0x064A, 0x0646, 0x0637, 0x0648, 0x0644, 0x0627, + 0x0020, 0x0646, 0x064A, 0x0645, 0x0623, 0x062A, 0x0644, 0x0627, 0x0020, 0x0629, 0x0633, 0x0633, 0x0624, 0x0645, 0x0020, 0x064A, + 0x0641, 0x0020, 0x0629, 0x062F, 0x064A, 0x062F, 0x0639, 0x0020, 0x0646, 0x064A, 0x0645, 0x0623, 0x062A, 0x0020, 0x062C, 0x0645, + 0x0627, 0x0631, 0x0628, 0x0020, 0x062A, 0x0630, 0x0641, 0x064F, 0x0646, 0x0648, 0x0020, 0x062A, 0x0639, 0x0636, 0x064F, 0x0648, + 0x0000, + }; + + int32_t bLength = 0, brLength = 0, cLength = ARRAY_SIZE(chars), crLength = ARRAY_SIZE(chars_reverse); + + char *bytes = extractBytes(chars, cLength, "IBM420", &bLength); + char *bytes_r = extractBytes(chars_reverse, crLength, "IBM420", &brLength); + + UCharsetDetector *csd = ucsdet_open(&status); + const UCharsetMatch *match; + const char *name; + + ucsdet_setText(csd, bytes, bLength, &status); + match = ucsdet_detect(csd, &status); + + if (match == NULL) { + log_err("Encoding detection failure for IBM420_rtl: got no matches.\n"); + goto bail; + } + + name = ucsdet_getName(match, &status); + if (strcmp(name, "IBM420_rtl") != 0) { + log_data_err("Encoding detection failure for IBM420_rtl: got %s. (Are you missing data?)\n", name); + } + + ucsdet_setText(csd, bytes_r, brLength, &status); + match = ucsdet_detect(csd, &status); + + if (match == NULL) { + log_err("Encoding detection failure for IBM420_ltr: got no matches.\n"); + goto bail; + } + + name = ucsdet_getName(match, &status); + if (strcmp(name, "IBM420_ltr") != 0) { + log_data_err("Encoding detection failure for IBM420_ltr: got %s. (Are you missing data?)\n", name); + } + +bail: + freeBytes(bytes); + freeBytes(bytes_r); + ucsdet_close(csd); +} |