diff options
Diffstat (limited to 'Build/source/libs/poppler/poppler-src/poppler/UTF.cc')
-rw-r--r-- | Build/source/libs/poppler/poppler-src/poppler/UTF.cc | 289 |
1 files changed, 287 insertions, 2 deletions
diff --git a/Build/source/libs/poppler/poppler-src/poppler/UTF.cc b/Build/source/libs/poppler/poppler-src/poppler/UTF.cc index f7b02d14795..878837a8831 100644 --- a/Build/source/libs/poppler/poppler-src/poppler/UTF.cc +++ b/Build/source/libs/poppler/poppler-src/poppler/UTF.cc @@ -1,6 +1,6 @@ //======================================================================== // -// UTF.h +// UTF.cc // // Copyright 2001-2003 Glyph & Cog, LLC // @@ -14,7 +14,7 @@ // under GPL version 2 or later // // Copyright (C) 2008 Koji Otani <sho@bbr.jp> -// Copyright (C) 2012 Adrian Johnson <ajohnson@redneon.com> +// Copyright (C) 2012, 2017 Adrian Johnson <ajohnson@redneon.com> // Copyright (C) 2012 Hib Eris <hib@hiberis.nl> // Copyright (C) 2016 Albert Astals Cid <aacid@kde.org> // Copyright (C) 2016 Jason Crain <jason@aquaticape.us> @@ -27,6 +27,7 @@ #include "goo/gmem.h" #include "PDFDocEncoding.h" #include "UTF.h" +#include "UnicodeMapFuncs.h" #include <algorithm> bool UnicodeIsValid(Unicode ucs4) @@ -130,3 +131,287 @@ bool UnicodeIsWhitespace(Unicode ucs4) Unicode const *i = std::lower_bound(spaces, end, ucs4); return (i != end && *i == ucs4); } + +// +// decodeUtf8() and decodeUtf8Table are: +// +// Copyright (c) 2008-2009 Bjoern Hoehrmann <bjoern@hoehrmann.de> +// +// Permission is hereby granted, free of charge, to any person +// obtaining a copy of this software and associated documentation +// files (the "Software"), to deal in the Software without +// restriction, including without limitation the rights to use, copy, +// modify, merge, publish, distribute, sublicense, and/or sell copies +// of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be +// included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS +// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for details. +// +static const uint32_t UTF8_ACCEPT = 0; +static const uint32_t UTF8_REJECT = 12; +static const uint32_t UCS4_MAX = 0x10FFFF; +static const Unicode REPLACEMENT_CHAR = 0xFFFD; + +static const uint8_t decodeUtf8Table[] = { + // The first part of the table maps bytes to character classes + // to reduce the size of the transition table and create bitmasks. + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 00..1f + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 20..3f + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 40..5f + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 60..7f + 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, // 80..9f + 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, // a0..bf + 8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, // c0..df + 10,3,3,3,3,3,3,3,3,3,3,3,3,4,3,3, 11,6,6,6,5,8,8,8,8,8,8,8,8,8,8,8, // e0..ff + + // The second part is a transition table that maps a combination + // of a state of the automaton and a character class to a state. + 0,12,24,36,60,96,84,12,12,12,48,72, 12,12,12,12,12,12,12,12,12,12,12,12, + 12, 0,12,12,12,12,12, 0,12, 0,12,12, 12,24,12,12,12,12,12,24,12,24,12,12, + 12,12,12,12,12,12,12,24,12,12,12,12, 12,24,12,12,12,12,12,12,12,24,12,12, + 12,12,12,12,12,12,12,36,12,36,12,12, 12,36,12,12,12,12,12,36,12,36,12,12, + 12,36,12,12,12,12,12,12,12,12,12,12, +}; + +// Decode utf8 state machine for fast UTF-8 decoding. Initialise state +// to 0 and call decodeUtf8() for each byte of UTF-8. Return value +// (and state) is UTF8_ACCEPT when it has found a valid codepoint +// (codepoint returned in codep), UTF8_REJECT when the byte is not +// allowed to occur at its position, and some other positive value if +// more bytes have to be read. Reset state to 0 to recover from +// errors. +inline uint32_t decodeUtf8(uint32_t* state, uint32_t* codep, char byte) +{ + uint32_t b = (unsigned char)byte; + uint32_t type = decodeUtf8Table[b]; + + *codep = (*state != UTF8_ACCEPT) ? + (b & 0x3fu) | (*codep << 6) : + (0xff >> type) & (b); + + *state = decodeUtf8Table[256 + *state + type]; + return *state; +} + +// Count number of UTF-16 code units required to convert a UTF-8 string +// (excluding terminating NULL). Each invalid byte is counted as a +// code point since the UTF-8 conversion functions will replace it with +// REPLACEMENT_CHAR. +int utf8CountUtf16CodeUnits(const char *utf8) +{ + uint32_t codepoint; + uint32_t state = 0; + int count = 0; + + while (*utf8) { + decodeUtf8(&state, &codepoint, *utf8); + if (state == UTF8_ACCEPT) { + if (codepoint < 0x10000) + count++; + else if (codepoint <= UCS4_MAX) + count += 2; + else + count++; // replace with REPLACEMENT_CHAR + } else if (state == UTF8_REJECT) { + count++; // replace with REPLACEMENT_CHAR + state = 0; + } + utf8++; + } + if (state != UTF8_ACCEPT && state != UTF8_REJECT) + count++; // replace with REPLACEMENT_CHAR + + return count; +} + + +// Convert UTF-8 to UTF-16 +// utf8- UTF-8 string to convert. If not null terminated, set maxUtf8 to num +// bytes to convert +// utf16 - output buffer to write UTF-16 to. Output will always be null terminated. +// maxUtf16 - maximum size of output buffer including space for null. +// maxUtf8 - maximum number of UTF-8 bytes to convert. Conversion stops when +// either this count is reached or a null is encountered. +// Returns number of UTF-16 code units written (excluding NULL). +int utf8ToUtf16(const char *utf8, uint16_t *utf16, int maxUtf16, int maxUtf8) +{ + uint16_t *p = utf16; + uint32_t codepoint; + uint32_t state = 0; + int nIn = 0; + int nOut = 0; + while (*utf8 && nIn < maxUtf8 && nOut < maxUtf16 - 1) { + decodeUtf8(&state, &codepoint, *utf8); + if (state == UTF8_ACCEPT) { + if (codepoint < 0x10000) { + *p++ = (uint16_t)codepoint; + nOut++; + } else if (codepoint <= UCS4_MAX) { + *p++ = (uint16_t)(0xD7C0 + (codepoint >> 10)); + *p++ = (uint16_t)(0xDC00 + (codepoint & 0x3FF)); + nOut += 2; + } else { + *p++ = REPLACEMENT_CHAR; + nOut++; + state = 0; + } + } else if (state == UTF8_REJECT) { + *p++ = REPLACEMENT_CHAR; // invalid byte for this position + nOut++; + } + utf8++; + nIn++; + } + // replace any trailing bytes too short for a valid UTF-8 with a replacement char + if (state != UTF8_ACCEPT && state != UTF8_REJECT && nOut < maxUtf16 - 1) { + *p++ = REPLACEMENT_CHAR; + nOut++; + } + if (nOut > maxUtf16 - 1) + nOut = maxUtf16 - 1; + utf16[nOut] = 0; + return nOut; +} + +// Allocate utf16 string and convert utf8 into it. +uint16_t *utf8ToUtf16(const char *utf8, int *len) +{ + int n = utf8CountUtf16CodeUnits(utf8); + if (len) + *len = n; + uint16_t *utf16 = (uint16_t*)gmallocn(n + 1, sizeof(uint16_t)); + utf8ToUtf16(utf8, utf16); + return utf16; +} + +static const uint32_t UTF16_ACCEPT = 0; +static const uint32_t UTF16_REJECT = -1; + +// Initialise state to 0. Returns UTF16_ACCEPT when a valid code point +// has been found, UTF16_REJECT when invalid code unit for this state, +// some other valid if another code unit needs to be read. +inline uint32_t decodeUtf16(uint32_t* state, uint32_t* codePoint, uint16_t codeUnit) +{ + if (*state == 0) { + if (codeUnit >= 0xd800 && codeUnit < 0xdc00) { /* surrogate pair */ + *state = codeUnit; + return *state; + } else if (codeUnit >= 0xdc00 && codeUnit < 0xe000) { + /* invalid low surrogate */ + return UTF16_REJECT; + } else { + *codePoint = codeUnit; + return UTF16_ACCEPT; + } + } else { + if (codeUnit >= 0xdc00 && codeUnit < 0xe000) { + *codePoint = (((*state & 0x3ff) << 10) | (codeUnit & 0x3ff)) + 0x10000; + *state = 0; + return UTF16_ACCEPT; + } else { + /* invalid high surrogate */ + return UTF16_REJECT; + } + } +} + +// Count number of UTF-8 bytes required to convert a UTF-16 string to +// UTF-8 (excluding terminating NULL). +int utf16CountUtf8Bytes(const uint16_t *utf16) +{ + uint32_t codepoint; + uint32_t state = 0; + int count = 0; + + while (*utf16) { + decodeUtf16(&state, &codepoint, *utf16); + if (state == UTF16_ACCEPT) { + if (codepoint < 0x80) + count++; + else if (codepoint < 0x800) + count += 2; + else if (codepoint < 0x10000) + count += 3; + else if (codepoint <= UCS4_MAX) + count += 4; + else + count += 3; // replace with REPLACEMENT_CHAR + } else if (state == UTF16_REJECT) { + count += 3; // replace with REPLACEMENT_CHAR + state = 0; + } + utf16++; + } + if (state != UTF8_ACCEPT && state != UTF8_REJECT) + count++; // replace with REPLACEMENT_CHAR + + return count; +} + +// Convert UTF-16 to UTF-8 +// utf16- UTF-16 string to convert. If not null terminated, set maxUtf16 to num +// code units to convert +// utf8 - output buffer to write UTF-8 to. Output will always be null terminated. +// maxUtf8 - maximum size of output buffer including space for null. +// maxUtf16 - maximum number of UTF-16 code units to convert. Conversion stops when +// either this count is reached or a null is encountered. +// Returns number of UTF-8 bytes written (excluding NULL). +int utf16ToUtf8(const uint16_t *utf16, char *utf8, int maxUtf8, int maxUtf16) +{ + uint32_t codepoint = 0; + uint32_t state = 0; + int nIn = 0; + int nOut = 0; + char *p = utf8; + while (*utf16 && nIn < maxUtf16 && nOut < maxUtf8 - 1) { + decodeUtf16(&state, &codepoint, *utf16); + if (state == UTF16_ACCEPT || state == UTF16_REJECT) { + if (state == UTF16_REJECT || codepoint > UCS4_MAX) { + codepoint = REPLACEMENT_CHAR; + state = 0; + } + + int bufSize = maxUtf8 - nOut; + int count = mapUTF8(codepoint, p, bufSize); + p += count; + nOut += count; + } + utf16++; + nIn++; + } + // replace any trailing bytes too short for a valid UTF-8 with a replacement char + if (state != UTF16_ACCEPT && state != UTF16_REJECT && nOut < maxUtf8 - 1) { + int bufSize = maxUtf8 - nOut; + int count = mapUTF8(REPLACEMENT_CHAR, p, bufSize); + p += count; + nOut += count; + nOut++; + } + if (nOut > maxUtf8 - 1) + nOut = maxUtf8 - 1; + utf8[nOut] = 0; + return nOut; +} + +// Allocate utf8 string and convert utf16 into it. +char *utf16ToUtf8(const uint16_t *utf16, int *len) +{ + int n = utf16CountUtf8Bytes(utf16); + if (len) + *len = n; + char *utf8 = (char*)gmalloc(n + 1); + utf16ToUtf8(utf16, utf8); + return utf8; +} |