diff options
Diffstat (limited to 'Build/source/libs/icu-xetex/test/intltest/testidn.cpp')
| -rw-r--r-- | Build/source/libs/icu-xetex/test/intltest/testidn.cpp | 446 |
1 files changed, 446 insertions, 0 deletions
diff --git a/Build/source/libs/icu-xetex/test/intltest/testidn.cpp b/Build/source/libs/icu-xetex/test/intltest/testidn.cpp new file mode 100644 index 00000000000..b706f2bf42c --- /dev/null +++ b/Build/source/libs/icu-xetex/test/intltest/testidn.cpp @@ -0,0 +1,446 @@ +/* +******************************************************************************* +* +* Copyright (C) 2003, International Business Machines +* Corporation and others. All Rights Reserved. +* +******************************************************************************* +* file name: testidn.cpp +* encoding: US-ASCII +* tab size: 8 (not used) +* indentation:4 +* +* created on: 2003-02-06 +* created by: Ram Viswanadha +* +* This program reads the rfc3454_*.txt files, +* parses them, and extracts the data for Nameprep conformance. +* It then preprocesses it and writes a binary file for efficient use +* in various IDNA conversion processes. +*/ + +#include <stdio.h> +#include <stdlib.h> +#include "unicode/utypes.h" + +#if !UCONFIG_NO_IDNA && !UCONFIG_NO_TRANSLITERATION + +#define USPREP_TYPE_NAMES_ARRAY + +#include "unicode/uchar.h" +#include "unicode/putil.h" +#include "cmemory.h" +#include "cstring.h" +#include "unicode/udata.h" +#include "unewdata.h" +#include "uoptions.h" +#include "uparse.h" +#include "utrie.h" +#include "umutex.h" +#include "sprpimpl.h" +#include "testidna.h" +#include "punyref.h" + +UBool beVerbose=FALSE, haveCopyright=TRUE; + +/* prototypes --------------------------------------------------------------- */ + + +static void +parseMappings(const char *filename, UBool reportError,TestIDNA& test, UErrorCode *pErrorCode); + +static void +compareMapping(uint32_t codepoint, uint32_t* mapping, int32_t mapLength, + UStringPrepType option); + +static void +compareFlagsForRange(uint32_t start, uint32_t end,UStringPrepType option); + +static void +testAllCodepoints(TestIDNA& test); + +static TestIDNA* pTestIDNA =NULL; + +static const char* fileNames[] = { + "NamePrepProfile.txt" + }; +static UStringPrepProfile *profile = NULL; +static const UTrie *idnTrie = NULL; +static const int32_t *indexes = NULL; +static const uint16_t *mappingData = NULL; +/* -------------------------------------------------------------------------- */ + +/* file definitions */ +#define DATA_NAME "uidna" +#define DATA_TYPE "icu" + +#define MISC_DIR "misc" + +extern int +testData(TestIDNA& test) { + char* filename = (char*) malloc(strlen(IntlTest::pathToDataDirectory())*5555); + //TODO get the srcDir dynamically + const char *srcDir=IntlTest::pathToDataDirectory(); + char *basename=NULL; + UErrorCode errorCode=U_ZERO_ERROR; + char *saveBasename =NULL; + + profile = usprep_open(NULL, DATA_NAME, &errorCode); + if(U_FAILURE(errorCode)){ + test.errln("Failed to load IDNA data file. " + UnicodeString(u_errorName(errorCode))); + return errorCode; + } + + idnTrie = &profile->sprepTrie; + indexes = profile->indexes; + mappingData = profile->mappingData; + + //initialize + pTestIDNA = &test; + + /* prepare the filename beginning with the source dir */ + if(uprv_strchr(srcDir,U_FILE_SEP_CHAR) == NULL){ + filename[0] = 0x2E; + filename[1] = U_FILE_SEP_CHAR; + uprv_strcpy(filename+2,srcDir); + }else{ + uprv_strcpy(filename, srcDir); + } + basename=filename+uprv_strlen(filename); + if(basename>filename && *(basename-1)!=U_FILE_SEP_CHAR) { + *basename++=U_FILE_SEP_CHAR; + } + + /* process unassigned */ + basename=filename+uprv_strlen(filename); + if(basename>filename && *(basename-1)!=U_FILE_SEP_CHAR) { + *basename++=U_FILE_SEP_CHAR; + } + + /* first copy misc directory */ + saveBasename = basename; + uprv_strcpy(basename,MISC_DIR); + basename = basename + uprv_strlen(MISC_DIR); + *basename++=U_FILE_SEP_CHAR; + + /* process unassigned */ + uprv_strcpy(basename,fileNames[0]); + parseMappings(filename,TRUE, test,&errorCode); + if(U_FAILURE(errorCode)) { + test.errln( "Could not open file %s for reading \n", filename); + return errorCode; + } + + testAllCodepoints(test); + + usprep_close(profile); + pTestIDNA = NULL; + free(filename); + return errorCode; +} +U_CDECL_BEGIN + +static void U_CALLCONV +strprepProfileLineFn(void * /*context*/, + char *fields[][2], int32_t fieldCount, + UErrorCode *pErrorCode) { + uint32_t mapping[40]; + char *end, *map; + uint32_t code; + int32_t length; + /*UBool* mapWithNorm = (UBool*) context;*/ + const char* typeName; + uint32_t rangeStart=0,rangeEnd =0; + + if(fieldCount != 3){ + *pErrorCode = U_INVALID_FORMAT_ERROR; + return; + } + + typeName = fields[2][0]; + map = fields[1][0]; + + if(uprv_strstr(typeName, usprepTypeNames[USPREP_UNASSIGNED])!=NULL){ + + u_parseCodePointRange(fields[0][0], &rangeStart,&rangeEnd, pErrorCode); + + /* store the range */ + compareFlagsForRange(rangeStart,rangeEnd,USPREP_UNASSIGNED); + + }else if(uprv_strstr(typeName, usprepTypeNames[USPREP_PROHIBITED])!=NULL){ + + u_parseCodePointRange(fields[0][0], &rangeStart,&rangeEnd, pErrorCode); + + /* store the range */ + compareFlagsForRange(rangeStart,rangeEnd,USPREP_PROHIBITED); + + }else if(uprv_strstr(typeName, usprepTypeNames[USPREP_MAP])!=NULL){ + /* get the character code, field 0 */ + code=(uint32_t)uprv_strtoul(fields[0][0], &end, 16); + + /* parse the mapping string */ + length=u_parseCodePoints(map, mapping, sizeof(mapping)/4, pErrorCode); + + /* store the mapping */ + compareMapping(code,mapping, length,USPREP_MAP); + + }else{ + *pErrorCode = U_INVALID_FORMAT_ERROR; + } + +} + +U_CDECL_END + +static void +parseMappings(const char *filename,UBool reportError, TestIDNA& test, UErrorCode *pErrorCode) { + char *fields[3][2]; + + if(pErrorCode==NULL || U_FAILURE(*pErrorCode)) { + return; + } + + u_parseDelimitedFile(filename, ';', fields, 3, strprepProfileLineFn, (void*)filename, pErrorCode); + + //fprintf(stdout,"Number of code points that have mappings with length >1 : %i\n",len); + + if(U_FAILURE(*pErrorCode) && (reportError || *pErrorCode!=U_FILE_ACCESS_ERROR)) { + test.errln( "testidn error: u_parseDelimitedFile(\"%s\") failed - %s\n", filename, u_errorName(*pErrorCode)); + } +} + + +static inline UStringPrepType +getValues(uint32_t result, int32_t& value, UBool& isIndex){ + + UStringPrepType type; + + if(result == 0){ + /* + * Initial value stored in the mapping table + * just return USPREP_TYPE_LIMIT .. so that + * the source codepoint is copied to the destination + */ + type = USPREP_TYPE_LIMIT; + }else if(result >= _SPREP_TYPE_THRESHOLD){ + type = (UStringPrepType) (result - _SPREP_TYPE_THRESHOLD); + }else{ + /* get the state */ + type = USPREP_MAP; + /* ascertain if the value is index or delta */ + if(result & 0x02){ + isIndex = TRUE; + value = result >> 2; //mask off the lower 2 bits and shift + + }else{ + isIndex = FALSE; + value = (int16_t)result; + value = (value >> 2); + + } + if((result>>2) == _SPREP_MAX_INDEX_VALUE){ + type = USPREP_DELETE; + isIndex =FALSE; + value = 0; + } + } + return type; +} + + + +static void +testAllCodepoints(TestIDNA& test){ + /* + { + UChar str[19] = { + 0xC138, 0xACC4, 0xC758, 0xBAA8, 0xB4E0, 0xC0AC, 0xB78C, 0xB4E4, 0xC774, + 0x070F,//prohibited + 0xD55C, 0xAD6D, 0xC5B4, 0xB97C, 0xC774, 0xD574, 0xD55C, 0xB2E4, 0xBA74 + }; + uint32_t in[19] = {0}; + UErrorCode status = U_ZERO_ERROR; + int32_t inLength=0, outLength=100; + char output[100] = {0}; + punycode_status error; + u_strToUTF32((UChar32*)in,19,&inLength,str,19,&status); + + error= punycode_encode(inLength, in, NULL, (uint32_t*)&outLength, output); + printf(output); + + } + */ + + uint32_t i = 0; + int32_t unassigned = 0; + int32_t prohibited = 0; + int32_t mappedWithNorm = 0; + int32_t mapped = 0; + int32_t noValueInTrie = 0; + + UStringPrepType type; + int32_t value; + UBool isIndex = FALSE; + + for(i=0;i<=0x10FFFF;i++){ + uint32_t result = 0; + UTRIE_GET16(idnTrie,i, result); + type = getValues(result,value, isIndex); + if(type != USPREP_TYPE_LIMIT ){ + if(type == USPREP_UNASSIGNED){ + unassigned++; + } + if(type == USPREP_PROHIBITED){ + prohibited++; + } + if(type == USPREP_MAP){ + mapped++; + } + }else{ + noValueInTrie++; + if(result > 0){ + test.errln("The return value for 0x%06X is wrong. %i\n",i,result); + } + } + } + + test.logln("Number of Unassinged code points : %i \n",unassigned); + test.logln("Number of Prohibited code points : %i \n",prohibited); + test.logln("Number of Mapped code points : %i \n",mapped); + test.logln("Number of Mapped with NFKC code points : %i \n",mappedWithNorm); + test.logln("Number of code points that have no value in Trie: %i \n",noValueInTrie); + + +} + +static void +compareMapping(uint32_t codepoint, uint32_t* mapping,int32_t mapLength, + UStringPrepType type){ + uint32_t result = 0; + UTRIE_GET16(idnTrie,codepoint, result); + + int32_t length=0; + UBool isIndex; + UStringPrepType retType; + int32_t value, index=0, delta=0; + + retType = getValues(result,value,isIndex); + + + if(type != retType && retType != USPREP_DELETE){ + + pTestIDNA->errln( "Did not get the assigned type for codepoint 0x%08X. Expected: %i Got: %i\n",codepoint, USPREP_MAP, type); + + } + + if(isIndex){ + index = value; + if(index >= indexes[_SPREP_ONE_UCHAR_MAPPING_INDEX_START] && + index < indexes[_SPREP_TWO_UCHARS_MAPPING_INDEX_START]){ + length = 1; + }else if(index >= indexes[_SPREP_TWO_UCHARS_MAPPING_INDEX_START] && + index < indexes[_SPREP_THREE_UCHARS_MAPPING_INDEX_START]){ + length = 2; + }else if(index >= indexes[_SPREP_THREE_UCHARS_MAPPING_INDEX_START] && + index < indexes[_SPREP_FOUR_UCHARS_MAPPING_INDEX_START]){ + length = 3; + }else{ + length = mappingData[index++]; + } + }else{ + delta = value; + length = (retType == USPREP_DELETE)? 0 : 1; + } + + int32_t realLength =0; + /* figure out the real length */ + for(int32_t j=0; j<mapLength; j++){ + if(mapping[j] > 0xFFFF){ + realLength +=2; + }else{ + realLength++; + } + } + + if(realLength != length){ + pTestIDNA->errln( "Did not get the expected length. Expected: %i Got: %i\n", mapLength, length); + } + + if(isIndex){ + for(int8_t i =0; i< mapLength; i++){ + if(mapping[i] <= 0xFFFF){ + if(mappingData[index+i] != (uint16_t)mapping[i]){ + pTestIDNA->errln("Did not get the expected result. Expected: 0x%04X Got: 0x%04X \n", mapping[i], mappingData[index+i]); + } + }else{ + UChar lead = UTF16_LEAD(mapping[i]); + UChar trail = UTF16_TRAIL(mapping[i]); + if(mappingData[index+i] != lead || + mappingData[index+i+1] != trail){ + pTestIDNA->errln( "Did not get the expected result. Expected: 0x%04X 0x%04X Got: 0x%04X 0x%04X", lead, trail, mappingData[index+i], mappingData[index+i+1]); + } + } + } + }else{ + if(retType!=USPREP_DELETE && (codepoint-delta) != (uint16_t)mapping[0]){ + pTestIDNA->errln("Did not get the expected result. Expected: 0x%04X Got: 0x%04X \n", mapping[0],(codepoint-delta)); + } + } + +} + +static void +compareFlagsForRange(uint32_t start, uint32_t end, + UStringPrepType type){ + + uint32_t result =0 ; + UStringPrepType retType; + UBool isIndex=FALSE; + int32_t value=0; +/* + // supplementary code point + UChar __lead16=UTF16_LEAD(0x2323E); + int32_t __offset; + + // get data for lead surrogate + (result)=_UTRIE_GET_RAW((&idnTrie), index, 0, (__lead16)); + __offset=(&idnTrie)->getFoldingOffset(result); + + // get the real data from the folded lead/trail units + if(__offset>0) { + (result)=_UTRIE_GET_RAW((&idnTrie), index, __offset, (0x2323E)&0x3ff); + } else { + (result)=(uint32_t)((&idnTrie)->initialValue); + } + + UTRIE_GET16(&idnTrie,0x2323E, result); +*/ + while(start < end+1){ + UTRIE_GET16(idnTrie,start, result); + retType = getValues(result,value,isIndex); + if(result > _SPREP_TYPE_THRESHOLD){ + if(retType != type){ + pTestIDNA->errln( "FAIL: Did not get the expected type for 0x%06X. Expected: %s Got: %s\n",start,usprepTypeNames[type], usprepTypeNames[retType]); + } + }else{ + if(type == USPREP_PROHIBITED && ((result & 0x01) != 0x01)){ + pTestIDNA->errln( "FAIL: Did not get the expected type for 0x%06X. Expected: %s Got: %s\n",start,usprepTypeNames[type], usprepTypeNames[retType]); + } + } + + start++; + } + +} + + +#endif /* #if !UCONFIG_NO_IDNA */ + +/* + * Hey, Emacs, please set the following: + * + * Local Variables: + * indent-tabs-mode: nil + * End: + * + */ |