summaryrefslogtreecommitdiff
path: root/Build/source/libs/poppler/poppler-src/poppler/UTF.cc
diff options
context:
space:
mode:
Diffstat (limited to 'Build/source/libs/poppler/poppler-src/poppler/UTF.cc')
-rw-r--r--Build/source/libs/poppler/poppler-src/poppler/UTF.cc289
1 files changed, 287 insertions, 2 deletions
diff --git a/Build/source/libs/poppler/poppler-src/poppler/UTF.cc b/Build/source/libs/poppler/poppler-src/poppler/UTF.cc
index f7b02d14795..878837a8831 100644
--- a/Build/source/libs/poppler/poppler-src/poppler/UTF.cc
+++ b/Build/source/libs/poppler/poppler-src/poppler/UTF.cc
@@ -1,6 +1,6 @@
//========================================================================
//
-// UTF.h
+// UTF.cc
//
// Copyright 2001-2003 Glyph & Cog, LLC
//
@@ -14,7 +14,7 @@
// under GPL version 2 or later
//
// Copyright (C) 2008 Koji Otani <sho@bbr.jp>
-// Copyright (C) 2012 Adrian Johnson <ajohnson@redneon.com>
+// Copyright (C) 2012, 2017 Adrian Johnson <ajohnson@redneon.com>
// Copyright (C) 2012 Hib Eris <hib@hiberis.nl>
// Copyright (C) 2016 Albert Astals Cid <aacid@kde.org>
// Copyright (C) 2016 Jason Crain <jason@aquaticape.us>
@@ -27,6 +27,7 @@
#include "goo/gmem.h"
#include "PDFDocEncoding.h"
#include "UTF.h"
+#include "UnicodeMapFuncs.h"
#include <algorithm>
bool UnicodeIsValid(Unicode ucs4)
@@ -130,3 +131,287 @@ bool UnicodeIsWhitespace(Unicode ucs4)
Unicode const *i = std::lower_bound(spaces, end, ucs4);
return (i != end && *i == ucs4);
}
+
+//
+// decodeUtf8() and decodeUtf8Table are:
+//
+// Copyright (c) 2008-2009 Bjoern Hoehrmann <bjoern@hoehrmann.de>
+//
+// Permission is hereby granted, free of charge, to any person
+// obtaining a copy of this software and associated documentation
+// files (the "Software"), to deal in the Software without
+// restriction, including without limitation the rights to use, copy,
+// modify, merge, publish, distribute, sublicense, and/or sell copies
+// of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+// BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+// ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+//
+// See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for details.
+//
+static const uint32_t UTF8_ACCEPT = 0;
+static const uint32_t UTF8_REJECT = 12;
+static const uint32_t UCS4_MAX = 0x10FFFF;
+static const Unicode REPLACEMENT_CHAR = 0xFFFD;
+
+static const uint8_t decodeUtf8Table[] = {
+ // The first part of the table maps bytes to character classes
+ // to reduce the size of the transition table and create bitmasks.
+ 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 00..1f
+ 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 20..3f
+ 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 40..5f
+ 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 60..7f
+ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, // 80..9f
+ 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, // a0..bf
+ 8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, // c0..df
+ 10,3,3,3,3,3,3,3,3,3,3,3,3,4,3,3, 11,6,6,6,5,8,8,8,8,8,8,8,8,8,8,8, // e0..ff
+
+ // The second part is a transition table that maps a combination
+ // of a state of the automaton and a character class to a state.
+ 0,12,24,36,60,96,84,12,12,12,48,72, 12,12,12,12,12,12,12,12,12,12,12,12,
+ 12, 0,12,12,12,12,12, 0,12, 0,12,12, 12,24,12,12,12,12,12,24,12,24,12,12,
+ 12,12,12,12,12,12,12,24,12,12,12,12, 12,24,12,12,12,12,12,12,12,24,12,12,
+ 12,12,12,12,12,12,12,36,12,36,12,12, 12,36,12,12,12,12,12,36,12,36,12,12,
+ 12,36,12,12,12,12,12,12,12,12,12,12,
+};
+
+// Decode utf8 state machine for fast UTF-8 decoding. Initialise state
+// to 0 and call decodeUtf8() for each byte of UTF-8. Return value
+// (and state) is UTF8_ACCEPT when it has found a valid codepoint
+// (codepoint returned in codep), UTF8_REJECT when the byte is not
+// allowed to occur at its position, and some other positive value if
+// more bytes have to be read. Reset state to 0 to recover from
+// errors.
+inline uint32_t decodeUtf8(uint32_t* state, uint32_t* codep, char byte)
+{
+ uint32_t b = (unsigned char)byte;
+ uint32_t type = decodeUtf8Table[b];
+
+ *codep = (*state != UTF8_ACCEPT) ?
+ (b & 0x3fu) | (*codep << 6) :
+ (0xff >> type) & (b);
+
+ *state = decodeUtf8Table[256 + *state + type];
+ return *state;
+}
+
+// Count number of UTF-16 code units required to convert a UTF-8 string
+// (excluding terminating NULL). Each invalid byte is counted as a
+// code point since the UTF-8 conversion functions will replace it with
+// REPLACEMENT_CHAR.
+int utf8CountUtf16CodeUnits(const char *utf8)
+{
+ uint32_t codepoint;
+ uint32_t state = 0;
+ int count = 0;
+
+ while (*utf8) {
+ decodeUtf8(&state, &codepoint, *utf8);
+ if (state == UTF8_ACCEPT) {
+ if (codepoint < 0x10000)
+ count++;
+ else if (codepoint <= UCS4_MAX)
+ count += 2;
+ else
+ count++; // replace with REPLACEMENT_CHAR
+ } else if (state == UTF8_REJECT) {
+ count++; // replace with REPLACEMENT_CHAR
+ state = 0;
+ }
+ utf8++;
+ }
+ if (state != UTF8_ACCEPT && state != UTF8_REJECT)
+ count++; // replace with REPLACEMENT_CHAR
+
+ return count;
+}
+
+
+// Convert UTF-8 to UTF-16
+// utf8- UTF-8 string to convert. If not null terminated, set maxUtf8 to num
+// bytes to convert
+// utf16 - output buffer to write UTF-16 to. Output will always be null terminated.
+// maxUtf16 - maximum size of output buffer including space for null.
+// maxUtf8 - maximum number of UTF-8 bytes to convert. Conversion stops when
+// either this count is reached or a null is encountered.
+// Returns number of UTF-16 code units written (excluding NULL).
+int utf8ToUtf16(const char *utf8, uint16_t *utf16, int maxUtf16, int maxUtf8)
+{
+ uint16_t *p = utf16;
+ uint32_t codepoint;
+ uint32_t state = 0;
+ int nIn = 0;
+ int nOut = 0;
+ while (*utf8 && nIn < maxUtf8 && nOut < maxUtf16 - 1) {
+ decodeUtf8(&state, &codepoint, *utf8);
+ if (state == UTF8_ACCEPT) {
+ if (codepoint < 0x10000) {
+ *p++ = (uint16_t)codepoint;
+ nOut++;
+ } else if (codepoint <= UCS4_MAX) {
+ *p++ = (uint16_t)(0xD7C0 + (codepoint >> 10));
+ *p++ = (uint16_t)(0xDC00 + (codepoint & 0x3FF));
+ nOut += 2;
+ } else {
+ *p++ = REPLACEMENT_CHAR;
+ nOut++;
+ state = 0;
+ }
+ } else if (state == UTF8_REJECT) {
+ *p++ = REPLACEMENT_CHAR; // invalid byte for this position
+ nOut++;
+ }
+ utf8++;
+ nIn++;
+ }
+ // replace any trailing bytes too short for a valid UTF-8 with a replacement char
+ if (state != UTF8_ACCEPT && state != UTF8_REJECT && nOut < maxUtf16 - 1) {
+ *p++ = REPLACEMENT_CHAR;
+ nOut++;
+ }
+ if (nOut > maxUtf16 - 1)
+ nOut = maxUtf16 - 1;
+ utf16[nOut] = 0;
+ return nOut;
+}
+
+// Allocate utf16 string and convert utf8 into it.
+uint16_t *utf8ToUtf16(const char *utf8, int *len)
+{
+ int n = utf8CountUtf16CodeUnits(utf8);
+ if (len)
+ *len = n;
+ uint16_t *utf16 = (uint16_t*)gmallocn(n + 1, sizeof(uint16_t));
+ utf8ToUtf16(utf8, utf16);
+ return utf16;
+}
+
+static const uint32_t UTF16_ACCEPT = 0;
+static const uint32_t UTF16_REJECT = -1;
+
+// Initialise state to 0. Returns UTF16_ACCEPT when a valid code point
+// has been found, UTF16_REJECT when invalid code unit for this state,
+// some other valid if another code unit needs to be read.
+inline uint32_t decodeUtf16(uint32_t* state, uint32_t* codePoint, uint16_t codeUnit)
+{
+ if (*state == 0) {
+ if (codeUnit >= 0xd800 && codeUnit < 0xdc00) { /* surrogate pair */
+ *state = codeUnit;
+ return *state;
+ } else if (codeUnit >= 0xdc00 && codeUnit < 0xe000) {
+ /* invalid low surrogate */
+ return UTF16_REJECT;
+ } else {
+ *codePoint = codeUnit;
+ return UTF16_ACCEPT;
+ }
+ } else {
+ if (codeUnit >= 0xdc00 && codeUnit < 0xe000) {
+ *codePoint = (((*state & 0x3ff) << 10) | (codeUnit & 0x3ff)) + 0x10000;
+ *state = 0;
+ return UTF16_ACCEPT;
+ } else {
+ /* invalid high surrogate */
+ return UTF16_REJECT;
+ }
+ }
+}
+
+// Count number of UTF-8 bytes required to convert a UTF-16 string to
+// UTF-8 (excluding terminating NULL).
+int utf16CountUtf8Bytes(const uint16_t *utf16)
+{
+ uint32_t codepoint;
+ uint32_t state = 0;
+ int count = 0;
+
+ while (*utf16) {
+ decodeUtf16(&state, &codepoint, *utf16);
+ if (state == UTF16_ACCEPT) {
+ if (codepoint < 0x80)
+ count++;
+ else if (codepoint < 0x800)
+ count += 2;
+ else if (codepoint < 0x10000)
+ count += 3;
+ else if (codepoint <= UCS4_MAX)
+ count += 4;
+ else
+ count += 3; // replace with REPLACEMENT_CHAR
+ } else if (state == UTF16_REJECT) {
+ count += 3; // replace with REPLACEMENT_CHAR
+ state = 0;
+ }
+ utf16++;
+ }
+ if (state != UTF8_ACCEPT && state != UTF8_REJECT)
+ count++; // replace with REPLACEMENT_CHAR
+
+ return count;
+}
+
+// Convert UTF-16 to UTF-8
+// utf16- UTF-16 string to convert. If not null terminated, set maxUtf16 to num
+// code units to convert
+// utf8 - output buffer to write UTF-8 to. Output will always be null terminated.
+// maxUtf8 - maximum size of output buffer including space for null.
+// maxUtf16 - maximum number of UTF-16 code units to convert. Conversion stops when
+// either this count is reached or a null is encountered.
+// Returns number of UTF-8 bytes written (excluding NULL).
+int utf16ToUtf8(const uint16_t *utf16, char *utf8, int maxUtf8, int maxUtf16)
+{
+ uint32_t codepoint = 0;
+ uint32_t state = 0;
+ int nIn = 0;
+ int nOut = 0;
+ char *p = utf8;
+ while (*utf16 && nIn < maxUtf16 && nOut < maxUtf8 - 1) {
+ decodeUtf16(&state, &codepoint, *utf16);
+ if (state == UTF16_ACCEPT || state == UTF16_REJECT) {
+ if (state == UTF16_REJECT || codepoint > UCS4_MAX) {
+ codepoint = REPLACEMENT_CHAR;
+ state = 0;
+ }
+
+ int bufSize = maxUtf8 - nOut;
+ int count = mapUTF8(codepoint, p, bufSize);
+ p += count;
+ nOut += count;
+ }
+ utf16++;
+ nIn++;
+ }
+ // replace any trailing bytes too short for a valid UTF-8 with a replacement char
+ if (state != UTF16_ACCEPT && state != UTF16_REJECT && nOut < maxUtf8 - 1) {
+ int bufSize = maxUtf8 - nOut;
+ int count = mapUTF8(REPLACEMENT_CHAR, p, bufSize);
+ p += count;
+ nOut += count;
+ nOut++;
+ }
+ if (nOut > maxUtf8 - 1)
+ nOut = maxUtf8 - 1;
+ utf8[nOut] = 0;
+ return nOut;
+}
+
+// Allocate utf8 string and convert utf16 into it.
+char *utf16ToUtf8(const uint16_t *utf16, int *len)
+{
+ int n = utf16CountUtf8Bytes(utf16);
+ if (len)
+ *len = n;
+ char *utf8 = (char*)gmalloc(n + 1);
+ utf16ToUtf8(utf16, utf8);
+ return utf8;
+}