summaryrefslogtreecommitdiff
path: root/Build/source/libs/teckit/TECkit-2.5.1/source/Sample-tools/TxtConv.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'Build/source/libs/teckit/TECkit-2.5.1/source/Sample-tools/TxtConv.cpp')
-rw-r--r--Build/source/libs/teckit/TECkit-2.5.1/source/Sample-tools/TxtConv.cpp530
1 files changed, 530 insertions, 0 deletions
diff --git a/Build/source/libs/teckit/TECkit-2.5.1/source/Sample-tools/TxtConv.cpp b/Build/source/libs/teckit/TECkit-2.5.1/source/Sample-tools/TxtConv.cpp
new file mode 100644
index 00000000000..629915a4139
--- /dev/null
+++ b/Build/source/libs/teckit/TECkit-2.5.1/source/Sample-tools/TxtConv.cpp
@@ -0,0 +1,530 @@
+/*
+ TxtConv.c
+ Copyright (c) 2002-2004 SIL International.
+
+ 2008-01-23 jk revised endian-ness stuff to allow Universal build
+ 5-May-2005 jk added include <stdlib.h> and <string.h> to keep gcc happy
+ 10-Mar-2004 jk added -u option to control handling of unmappable input
+*/
+
+#include "TECkit_Engine.h"
+
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+
+#if __MWERKS__
+#if __dest_os == __mac_os
+#include "SIOUX.h"
+#include "console.h"
+#endif
+#endif
+
+#ifdef HAVE_CONFIG_H
+# include "config.h" /* a Unix-ish setup where we have config.h available */
+#endif
+
+#if (defined(__dest_os) && (__dest_os == __win32_os)) || defined(WIN32) /* Windows target: little-endian */
+# undef WORDS_BIGENDIAN
+#endif
+
+#ifdef __APPLE__
+#include <TargetConditionals.h>
+#endif
+
+#if defined(TARGET_RT_BIG_ENDIAN) /* the CodeWarrior prefix files or Apple TargetConditionals.h sets this */
+# if TARGET_RT_BIG_ENDIAN
+# undef WORDS_BIGENDIAN
+# define WORDS_BIGENDIAN 1
+# else
+# undef WORDS_BIGENDIAN
+# endif
+#endif
+
+#define kInBufLen 4096
+#define kOutBufLen (kInBufLen * 4)
+
+static int
+doConversion(TECkit_Converter cnv, FILE* inFile, FILE* outFile, UInt32 opts)
+{
+ UInt32 inBase;
+ UInt32 inLength;
+ char inBuffer[kInBufLen];
+ char outBuffer[kOutBufLen];
+ UInt32 savedLen = 0;
+ UInt32 offset = 0;
+ TECkit_Status status;
+
+ inBase = ftell(inFile);
+ fseek(inFile, 0, SEEK_END);
+ inLength = ftell(inFile) - inBase;
+ fseek(inFile, inBase, SEEK_SET);
+
+ while (1) {
+ UInt32 inUsed, outUsed, lookahead;
+ UInt32 amountToRead = kInBufLen - savedLen;
+ char* inPtr = inBuffer;
+ UInt32 inAvail;
+
+ if (offset + amountToRead > inLength)
+ amountToRead = inLength - offset;
+
+ inAvail = savedLen + amountToRead;
+ if (inAvail > 0) {
+ fread(inBuffer + savedLen, 1, amountToRead, inFile);
+ offset += amountToRead;
+
+ status = TECkit_ConvertBufferOpt(cnv, (Byte*)inPtr, inAvail, &inUsed,
+ (Byte*)outBuffer, kOutBufLen, &outUsed, opts, &lookahead);
+ fwrite(outBuffer, 1, outUsed, outFile);
+
+ switch (status & kStatusMask_Basic) {
+ case kStatus_OutputBufferFull:
+ case kStatus_NeedMoreInput:
+ if (inUsed < inAvail) {
+ savedLen = inAvail - inUsed;
+ memcpy(inBuffer, inBuffer + inAvail - savedLen, savedLen);
+ }
+ else
+ savedLen = 0;
+ continue;
+
+ case kStatus_UnmappedChar:
+ fprintf(stderr, "processing aborted at unmappable character, within %lu characters before file offset %lu\n",
+ lookahead, offset - amountToRead + inUsed);
+ break;
+
+ default:
+ fprintf(stderr, "bad returned status from TECkit_ConvertBuffer: %ld\n", status);
+ break;
+ }
+ }
+
+ if ((status & kStatusMask_Basic) != kStatus_UnmappedChar) {
+ // flush the converter
+ do {
+ status = TECkit_FlushOpt(cnv, (Byte*)outBuffer, kOutBufLen, &outUsed, opts, &lookahead);
+ fwrite(outBuffer, 1, outUsed, outFile);
+ savedLen -= inUsed;
+ inPtr += inUsed;
+ } while ((status & kStatusMask_Basic) == kStatus_OutputBufferFull);
+
+ if ((status & kStatusMask_Basic) == kStatus_UnmappedChar)
+ fprintf(stderr, "processing aborted at unmappable character, within %lu characters before end of input\n", lookahead);
+ else if ((status & kStatusMask_Basic) != kStatus_NoError)
+ fprintf(stderr, "bad returned status from TECkit_Flush: %ld\n", status);
+ }
+
+ if ((status & kStatusMask_Warning) == kStatus_UsedReplacement)
+ fprintf(stderr, "warning: unmapped input replaced by default replacement character\n");
+
+ break;
+ }
+
+ return ((status & kStatusMask_Basic) != 0)
+ ? 2
+ : (
+ ((status & kStatusMask_Warning) != 0)
+ ? 1
+ : 0
+ );
+}
+
+static int
+stringArg(int* pargc, char*** pargv, char** pstr)
+{
+ int err = 0;
+ if (*pargc == 1) {
+ fprintf(stderr, "missing value after %s\n", **pargv);
+ err = 1;
+ }
+ if (*pstr != 0) {
+ fprintf(stderr, "repeated argument %s\n", **pargv);
+ err = 1;
+ }
+ *pstr = *++*pargv;
+ --*pargc;
+ return err;
+}
+
+int
+main(int argc, char** argv)
+{
+#if __MWERKS__ && __dest_os == __mac_os
+ SIOUXSettings.asktosaveonclose = 0;
+ argc = ccommand(&argv);
+#endif
+ char* progName = argv[0];
+ char* tecFileName = 0;
+ char* inFileName = 0;
+ char* outFileName = 0;
+ char* nameID = 0;
+ char forward = 1;
+ char* unmappedOption = 0;
+ UInt16 inForm = kForm_Unspecified;
+ UInt16 outForm = kForm_Unspecified;
+ char cmdLineErr = 0;
+ UInt16 normForm = 0;
+ char noBOM = 0;
+ UInt32 opts = kOptionsUnmapped_UseReplacementCharSilently;
+ int rval;
+
+ UInt32 len = 0;
+ char* table = 0;
+ FILE* tecFile;
+ FILE* inFile;
+ FILE* outFile;
+ UInt32 sourceFlags, targetFlags;
+ TECkit_Status status;
+ TECkit_Converter cnv;
+
+ if (TECkit_GetVersion() != kCurrentTECkitVersion) {
+ fprintf(stderr, "incorrect TECkit library version\n");
+ return 1;
+ }
+
+ while (--argc) {
+ char* arg = *++argv;
+ if (arg[0] == '-') {
+ if (strlen(arg + 1) == 1) {
+ switch (arg[1]) {
+ case 'n':
+ cmdLineErr += stringArg(&argc, &argv, &nameID);
+ break;
+ case 't':
+ cmdLineErr += stringArg(&argc, &argv, &tecFileName);
+ break;
+ case 'i':
+ cmdLineErr += stringArg(&argc, &argv, &inFileName);
+ break;
+ case 'o':
+ cmdLineErr += stringArg(&argc, &argv, &outFileName);
+ break;
+ case 'u':
+ cmdLineErr += stringArg(&argc, &argv, &unmappedOption);
+ break;
+ case 'r':
+ forward = 0;
+ break;
+ }
+ }
+ else if (strcmp(arg + 1, "if") == 0) {
+ ++argv;
+ --argc;
+ if (strcmp(*argv, "bytes") == 0)
+ inForm = kForm_Bytes;
+ else if (strcmp(*argv, "utf8") == 0)
+ inForm = kForm_UTF8;
+ else if (strcmp(*argv, "utf16") == 0)
+#ifdef WORDS_BIGENDIAN
+ inForm = kForm_UTF16BE;
+#else
+ inForm = kForm_UTF16LE;
+#endif
+ else if (strcmp(*argv, "utf16be") == 0)
+ inForm = kForm_UTF16BE;
+ else if (strcmp(*argv, "utf16le") == 0)
+ inForm = kForm_UTF16LE;
+ else if (strcmp(*argv, "utf32") == 0)
+#ifdef WORDS_BIGENDIAN
+ inForm = kForm_UTF32BE;
+#else
+ inForm = kForm_UTF32LE;
+#endif
+ else if (strcmp(*argv, "utf32be") == 0)
+ inForm = kForm_UTF32BE;
+ else if (strcmp(*argv, "utf32le") == 0)
+ inForm = kForm_UTF32LE;
+ }
+ else if (strcmp(arg + 1, "of") == 0) {
+ ++argv;
+ --argc;
+ if (strcmp(*argv, "bytes") == 0)
+ outForm = kForm_Bytes;
+ else if (strcmp(*argv, "utf8") == 0)
+ outForm = kForm_UTF8;
+ else if (strcmp(*argv, "utf16") == 0)
+#ifdef WORDS_BIGENDIAN
+ outForm = kForm_UTF16BE;
+#else
+ outForm = kForm_UTF16LE;
+#endif
+ else if (strcmp(*argv, "utf16be") == 0)
+ outForm = kForm_UTF16BE;
+ else if (strcmp(*argv, "utf16le") == 0)
+ outForm = kForm_UTF16LE;
+ else if (strcmp(*argv, "utf32") == 0)
+#ifdef WORDS_BIGENDIAN
+ outForm = kForm_UTF32BE;
+#else
+ outForm = kForm_UTF32LE;
+#endif
+ else if (strcmp(*argv, "utf32be") == 0)
+ outForm = kForm_UTF32BE;
+ else if (strcmp(*argv, "utf32le") == 0)
+ outForm = kForm_UTF32LE;
+ }
+ else if (strcmp(arg + 1, "nfc") == 0)
+ normForm = kForm_NFC;
+ else if (strcmp(arg + 1, "nfd") == 0)
+ normForm = kForm_NFD;
+ else if (strcmp(arg + 1, "nobom") == 0)
+ noBOM = 1;
+ else {
+ fprintf(stderr, "unknown option %s\n", arg);
+ ++cmdLineErr;
+ }
+ }
+ else {
+ fprintf(stderr, "missing option flag at %s\n", arg);
+ ++cmdLineErr;
+ }
+ }
+
+ if (unmappedOption != 0) {
+ if (strlen(unmappedOption) != 1 || *unmappedOption < '0' || *unmappedOption > '2') {
+ fprintf(stderr, "unknown value '%s' for -u option (expected 0-2)\n", unmappedOption);
+ ++cmdLineErr;
+ }
+ else {
+ switch (*unmappedOption) {
+ case '0':
+ opts = kOptionsUnmapped_UseReplacementCharSilently;
+ break;
+ case '1':
+ opts = kOptionsUnmapped_UseReplacementCharWithWarning;
+ break;
+ case '2':
+ opts = kOptionsUnmapped_DontUseReplacementChar;
+ break;
+ }
+ }
+ }
+
+ if (cmdLineErr != 0 || inFileName == 0 || outFileName == 0) {
+ fprintf(stderr, "\
+Usage: %s -i inFile -o outFile [-t tecFile] [-r] [-if inForm] [-of outForm] [-nobom] [-nf[cd]] [-u n]\n\
+ Required arguments:\n\
+ -i <file> input file\n\
+ -o <file> output file\n\
+ Optional arguments:\n\
+ -t <file> compiled TECkit mapping (.tec) file\n\
+ -r reverse (RHS->LHS, or Unicode->Byte) mapping\n\
+ -if <form> input encoding form\n\
+ -of <form> output encoding form\n\
+ -nobom don't write a BOM to Unicode output\n\
+ -nf[cd] apply NFC or NFD normalization to Unicode output\n\
+ -u <n> handling of unmappable input:\n\
+ 0 = use replacement character\n\
+ 1 = use replacement but generate warning\n\
+ 2 = stop conversion\n\
+ Encoding forms:\n\
+ bytes utf8 utf16be utf16le utf16 utf32be utf32le utf32\n\
+", progName);
+ exit(cmdLineErr != 0);
+ }
+
+ // examine the mapping (if any) to determine the input and output code spaces
+ if (tecFileName != 0) {
+ tecFile = fopen(tecFileName, "rb");
+
+ if (tecFile == 0) {
+ fprintf(stderr, "unable to load mapping table file %s\n", tecFileName);
+ return 1;
+ }
+
+ fseek(tecFile, 0, SEEK_END);
+ len = ftell(tecFile);
+ fseek(tecFile, 0, SEEK_SET);
+
+ table = (char*)malloc(len);
+ if (table == 0) {
+ fprintf(stderr, "out of memory!\n");
+ fclose(tecFile);
+ return 1;
+ }
+
+ fread(table, 1, len, tecFile);
+ fclose(tecFile);
+
+ status = forward
+ ? TECkit_GetMappingFlags((Byte*)table, len, &sourceFlags, &targetFlags)
+ : TECkit_GetMappingFlags((Byte*)table, len, &targetFlags, &sourceFlags);
+ if (status != kStatus_NoError) {
+ fprintf(stderr, "couldn't get encoding flags from mapping\n");
+ return 1;
+ }
+ }
+ else {
+ sourceFlags = kFlags_Unicode;
+ targetFlags = kFlags_Unicode;
+ }
+
+ inFile = fopen(inFileName, "rb");
+ if (inFile == 0) {
+ fprintf(stderr, "unable to open input file \"%s\"\n", inFileName);
+ return 1;
+ }
+
+ if (sourceFlags & kFlags_Unicode) {
+ // if the input is supposed to be Unicode, and the inForm is unspecified, try to guess it
+ // or skip over the BOM if one is found that matches the specified inForm
+ unsigned char sig[4];
+ if (inForm == kForm_Bytes) {
+ fprintf(stderr, "improper input encoding form for this mapping");
+ return 1;
+ }
+
+ switch (inForm) {
+ case kForm_Unspecified:
+ fread(sig, 1, 4, inFile);
+ if (sig[0] == 0xef && sig[1] == 0xbb && sig[2] == 0xbf) {
+ inForm = kForm_UTF8;
+ fseek(inFile, 3, SEEK_SET);
+ }
+ else if (sig[0] == 0xfe && sig[1] == 0xff) {
+ inForm = kForm_UTF16BE;
+ fseek(inFile, 2, SEEK_SET);
+ }
+ else if (sig[0] == 0xff && sig[1] == 0xfe && sig[2] == 0x00 && sig[3] == 0x00) {
+ inForm = kForm_UTF32LE;
+ /* fseek(inFile, 4, SEEK_SET); */
+ }
+ else if (sig[0] == 0xff && sig[1] == 0xfe) {
+ inForm = kForm_UTF16LE;
+ fseek(inFile, 2, SEEK_SET);
+ }
+ else if (sig[0] == 0x00 && sig[1] == 0x00 && sig[2] == 0xfe && sig[3] == 0xff) {
+ inForm = kForm_UTF32BE;
+ /* fseek(inFile, 4, SEEK_SET); */
+ }
+ else {
+ inForm = kForm_UTF8;
+ fseek(inFile, 0, SEEK_SET);
+ }
+ break;
+
+ case kForm_UTF8:
+ fread(sig, 1, 3, inFile);
+ if (!(sig[0] == 0xef && sig[1] == 0xbb && sig[2] == 0xbf))
+ fseek(inFile, 0, SEEK_SET);
+ break;
+
+ case kForm_UTF16BE:
+ fread(sig, 1, 2, inFile);
+ if (!(sig[0] == 0xfe && sig[1] == 0xff))
+ fseek(inFile, 0, SEEK_SET);
+ break;
+
+ case kForm_UTF16LE:
+ fread(sig, 1, 2, inFile);
+ if (!(sig[0] == 0xff && sig[1] == 0xfe))
+ fseek(inFile, 0, SEEK_SET);
+ break;
+
+ case kForm_UTF32BE:
+ fread(sig, 1, 4, inFile);
+ if (!(sig[0] == 0x00 && sig[1] == 0x00 && sig[2] == 0xfe && sig[3] == 0xff))
+ fseek(inFile, 0, SEEK_SET);
+ break;
+
+ case kForm_UTF32LE:
+ fread(sig, 1, 4, inFile);
+ if (!(sig[0] == 0xff && sig[1] == 0xfe && sig[2] == 0x00 && sig[3] == 0x00))
+ fseek(inFile, 0, SEEK_SET);
+ break;
+ }
+ }
+ else {
+ // if the input is not Unicode, the inputForm had better be Bytes
+ switch (inForm) {
+ case kForm_Unspecified:
+ inForm = kForm_Bytes;
+ break;
+ case kForm_Bytes:
+ break;
+ default:
+ fprintf(stderr, "improper input encoding form for this mapping");
+ return 1;
+ }
+ }
+
+ // choose an output encoding form (if not specified) and write BOM to Unicode output file
+ outFile = fopen(outFileName, "wb");
+ if (outFile == 0) {
+ fprintf(stderr, "unable to open output file \"%s\"\n", outFileName);
+ return 1;
+ }
+
+ if (targetFlags & kFlags_Unicode) {
+ if (outForm == kForm_Bytes) {
+ fprintf(stderr, "improper output encoding form for this mapping");
+ return 1;
+ }
+ if (outForm == kForm_Unspecified) {
+ if (inForm == kForm_Bytes)
+ outForm = kForm_UTF8;
+ else
+ outForm = inForm;
+ }
+ if (!noBOM) {
+ switch (outForm) {
+ case kForm_UTF8:
+ fwrite("\xef\xbb\xbf", 1, 3, outFile);
+ break;
+
+ case kForm_UTF16BE:
+ fwrite("\xfe\xff", 1, 2, outFile);
+ break;
+
+ case kForm_UTF16LE:
+ fwrite("\xff\xfe", 1, 2, outFile);
+ break;
+
+ case kForm_UTF32BE:
+ fwrite("\x00\x00\xfe\xff", 1, 4, outFile);
+ break;
+
+ case kForm_UTF32LE:
+ fwrite("\xff\xfe\x00\x00", 1, 4, outFile);
+ break;
+ }
+ }
+ }
+ else {
+ switch (outForm) {
+ case kForm_Unspecified:
+ outForm = kForm_Bytes;
+ break;
+ case kForm_Bytes:
+ break;
+ default:
+ fprintf(stderr, "improper output encoding form for this mapping");
+ return 1;
+ }
+ }
+
+ // OK, we have figured out the input and output encoding forms we want to use;
+ // now at last we can instantiate a converter
+ status = TECkit_CreateConverter((Byte*)table, len, forward, inForm, outForm | normForm, &cnv);
+ if (table != 0)
+ free(table);
+ if (status != kStatus_NoError) {
+ fprintf(stderr, "bad returned status from TECkit_CreateConverter: %ld\n", status);
+ return status;
+ }
+
+ // do the actual conversion
+ rval = doConversion(cnv, inFile, outFile, opts);
+
+ fclose(inFile);
+ fclose(outFile);
+
+ status = TECkit_DisposeConverter(cnv);
+ if (status != kStatus_NoError) {
+ fprintf(stderr, "bad returned status from TECkit_DisposeConverter: %ld\n", status);
+ return status;
+ }
+
+ return rval;
+}