diff options
Diffstat (limited to 'Build/source/libs/icu/icu-50.1/i18n/csrucode.cpp')
-rw-r--r-- | Build/source/libs/icu/icu-50.1/i18n/csrucode.cpp | 152 |
1 files changed, 152 insertions, 0 deletions
diff --git a/Build/source/libs/icu/icu-50.1/i18n/csrucode.cpp b/Build/source/libs/icu/icu-50.1/i18n/csrucode.cpp new file mode 100644 index 00000000000..d286cd3a504 --- /dev/null +++ b/Build/source/libs/icu/icu-50.1/i18n/csrucode.cpp @@ -0,0 +1,152 @@ +/* + ********************************************************************** + * Copyright (C) 2005-2012, International Business Machines + * Corporation and others. All Rights Reserved. + ********************************************************************** + */ + +#include "unicode/utypes.h" + +#if !UCONFIG_NO_CONVERSION + +#include "csrucode.h" +#include "csmatch.h" + +U_NAMESPACE_BEGIN + +CharsetRecog_Unicode::~CharsetRecog_Unicode() +{ + // nothing to do +} + +CharsetRecog_UTF_16_BE::~CharsetRecog_UTF_16_BE() +{ + // nothing to do +} + +const char *CharsetRecog_UTF_16_BE::getName() const +{ + return "UTF-16BE"; +} + +UBool CharsetRecog_UTF_16_BE::match(InputText* textIn, CharsetMatch *results) const +{ + const uint8_t *input = textIn->fRawInput; + int32_t confidence = 0; + + if (input[0] == 0xFE && input[1] == 0xFF) { + confidence = 100; + } + + // TODO: Do some statastics to check for unsigned UTF-16BE + results->set(textIn, this, confidence); + return (confidence > 0); +} + +CharsetRecog_UTF_16_LE::~CharsetRecog_UTF_16_LE() +{ + // nothing to do +} + +const char *CharsetRecog_UTF_16_LE::getName() const +{ + return "UTF-16LE"; +} + +UBool CharsetRecog_UTF_16_LE::match(InputText* textIn, CharsetMatch *results) const +{ + const uint8_t *input = textIn->fRawInput; + int32_t confidence = 0; + + if (input[0] == 0xFF && input[1] == 0xFE && (input[2] != 0x00 || input[3] != 0x00)) { + confidence = 100; + } + + // TODO: Do some statastics to check for unsigned UTF-16LE + results->set(textIn, this, confidence); + return (confidence > 0); +} + +CharsetRecog_UTF_32::~CharsetRecog_UTF_32() +{ + // nothing to do +} + +UBool CharsetRecog_UTF_32::match(InputText* textIn, CharsetMatch *results) const +{ + const uint8_t *input = textIn->fRawInput; + int32_t limit = (textIn->fRawLength / 4) * 4; + int32_t numValid = 0; + int32_t numInvalid = 0; + bool hasBOM = FALSE; + int32_t confidence = 0; + + if (getChar(input, 0) == 0x0000FEFFUL) { + hasBOM = TRUE; + } + + for(int32_t i = 0; i < limit; i += 4) { + int32_t ch = getChar(input, i); + + if (ch < 0 || ch >= 0x10FFFF || (ch >= 0xD800 && ch <= 0xDFFF)) { + numInvalid += 1; + } else { + numValid += 1; + } + } + + + // Cook up some sort of confidence score, based on presense of a BOM + // and the existence of valid and/or invalid multi-byte sequences. + if (hasBOM && numInvalid==0) { + confidence = 100; + } else if (hasBOM && numValid > numInvalid*10) { + confidence = 80; + } else if (numValid > 3 && numInvalid == 0) { + confidence = 100; + } else if (numValid > 0 && numInvalid == 0) { + confidence = 80; + } else if (numValid > numInvalid*10) { + // Probably corruput UTF-32BE data. Valid sequences aren't likely by chance. + confidence = 25; + } + + results->set(textIn, this, confidence); + return (confidence > 0); +} + +CharsetRecog_UTF_32_BE::~CharsetRecog_UTF_32_BE() +{ + // nothing to do +} + +const char *CharsetRecog_UTF_32_BE::getName() const +{ + return "UTF-32BE"; +} + +int32_t CharsetRecog_UTF_32_BE::getChar(const uint8_t *input, int32_t index) const +{ + return input[index + 0] << 24 | input[index + 1] << 16 | + input[index + 2] << 8 | input[index + 3]; +} + +CharsetRecog_UTF_32_LE::~CharsetRecog_UTF_32_LE() +{ + // nothing to do +} + +const char *CharsetRecog_UTF_32_LE::getName() const +{ + return "UTF-32LE"; +} + +int32_t CharsetRecog_UTF_32_LE::getChar(const uint8_t *input, int32_t index) const +{ + return input[index + 3] << 24 | input[index + 2] << 16 | + input[index + 1] << 8 | input[index + 0]; +} + +U_NAMESPACE_END +#endif + |