diff options
author | Karl Berry <karl@freefriends.org> | 2019-01-08 19:24:46 +0000 |
---|---|---|
committer | Karl Berry <karl@freefriends.org> | 2019-01-08 19:24:46 +0000 |
commit | 3eec52e2f37fb5ad2e6de4ec20cb26ff280777a1 (patch) | |
tree | 206a748c51d12870ca8426c9d7a59a11639d87b2 /Build/source/libs/icu/icu-src/source/tools | |
parent | 730f4601e0de27c5eccd405e393f80425fa6cdf2 (diff) |
icu 63.1
git-svn-id: svn://tug.org/texlive/trunk@49641 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Build/source/libs/icu/icu-src/source/tools')
27 files changed, 587 insertions, 434 deletions
diff --git a/Build/source/libs/icu/icu-src/source/tools/ctestfw/ctestfw.vcxproj b/Build/source/libs/icu/icu-src/source/tools/ctestfw/ctestfw.vcxproj index de4818c3237..baabf4f3814 100644 --- a/Build/source/libs/icu/icu-src/source/tools/ctestfw/ctestfw.vcxproj +++ b/Build/source/libs/icu/icu-src/source/tools/ctestfw/ctestfw.vcxproj @@ -75,7 +75,7 @@ <CompileAs>Default</CompileAs> </ClCompile> <Link> - <OutputFile>..\..\..\bin\icutest61.dll</OutputFile> + <OutputFile>..\..\..\bin\icutest63.dll</OutputFile> <AdditionalDependencies>icuuc.lib;icutu.lib;%(AdditionalDependencies)</AdditionalDependencies> <AdditionalLibraryDirectories>..\..\..\lib;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories> <ProgramDatabaseFile>.\..\..\..\lib\icutest.pdb</ProgramDatabaseFile> @@ -101,7 +101,7 @@ <CompileAs>Default</CompileAs> </ClCompile> <Link> - <OutputFile>..\..\..\bin\icutest61d.dll</OutputFile> + <OutputFile>..\..\..\bin\icutest63d.dll</OutputFile> <AdditionalDependencies>icuucd.lib;icutud.lib;%(AdditionalDependencies)</AdditionalDependencies> <AdditionalLibraryDirectories>..\..\..\lib;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories> <GenerateDebugInformation>true</GenerateDebugInformation> @@ -127,7 +127,7 @@ <CompileAs>Default</CompileAs> </ClCompile> <Link> - <OutputFile>..\..\..\bin64\icutest61.dll</OutputFile> + <OutputFile>..\..\..\bin64\icutest63.dll</OutputFile> <AdditionalDependencies>icuuc.lib;icutu.lib;%(AdditionalDependencies)</AdditionalDependencies> <AdditionalLibraryDirectories>..\..\..\lib64;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories> <ProgramDatabaseFile>.\..\..\..\lib64\icutest.pdb</ProgramDatabaseFile> @@ -150,7 +150,7 @@ <CompileAs>Default</CompileAs> </ClCompile> <Link> - <OutputFile>..\..\..\bin64\icutest61d.dll</OutputFile> + <OutputFile>..\..\..\bin64\icutest63d.dll</OutputFile> <AdditionalDependencies>icuucd.lib;icutud.lib;%(AdditionalDependencies)</AdditionalDependencies> <AdditionalLibraryDirectories>..\..\..\lib64;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories> <GenerateDebugInformation>true</GenerateDebugInformation> @@ -187,4 +187,4 @@ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> <ImportGroup Label="ExtensionTargets"> </ImportGroup> -</Project>
\ No newline at end of file +</Project> diff --git a/Build/source/libs/icu/icu-src/source/tools/ctestfw/tstdtmod.cpp b/Build/source/libs/icu/icu-src/source/tools/ctestfw/tstdtmod.cpp index bb1008da6a6..f4580b165da 100644 --- a/Build/source/libs/icu/icu-src/source/tools/ctestfw/tstdtmod.cpp +++ b/Build/source/libs/icu/icu-src/source/tools/ctestfw/tstdtmod.cpp @@ -13,25 +13,37 @@ #include "unicode/tstdtmod.h" #include "cmemory.h" #include <stdio.h> +#include "cstr.h" +#include "cstring.h" TestLog::~TestLog() {} IcuTestErrorCode::~IcuTestErrorCode() { - // Safe because our handleFailure() does not throw exceptions. - if(isFailure()) { handleFailure(); } + // Safe because our errlog() does not throw exceptions. + if(isFailure()) { + errlog(FALSE, u"destructor: expected success", nullptr); + } +} + +UBool IcuTestErrorCode::errIfFailureAndReset() { + if(isFailure()) { + errlog(FALSE, u"expected success", nullptr); + reset(); + return TRUE; + } else { + reset(); + return FALSE; + } } -UBool IcuTestErrorCode::logIfFailureAndReset(const char *fmt, ...) { +UBool IcuTestErrorCode::errIfFailureAndReset(const char *fmt, ...) { if(isFailure()) { char buffer[4000]; va_list ap; va_start(ap, fmt); vsprintf(buffer, fmt, ap); va_end(ap); - UnicodeString msg(testName, -1, US_INV); - msg.append(UNICODE_STRING_SIMPLE(" failure: ")).append(UnicodeString(errorName(), -1, US_INV)); - msg.append(UNICODE_STRING_SIMPLE(" - ")).append(UnicodeString(buffer, -1, US_INV)); - testClass.errln(msg); + errlog(FALSE, u"expected success", buffer); reset(); return TRUE; } else { @@ -40,17 +52,25 @@ UBool IcuTestErrorCode::logIfFailureAndReset(const char *fmt, ...) { } } -UBool IcuTestErrorCode::logDataIfFailureAndReset(const char *fmt, ...) { +UBool IcuTestErrorCode::errDataIfFailureAndReset() { + if(isFailure()) { + errlog(TRUE, u"data: expected success", nullptr); + reset(); + return TRUE; + } else { + reset(); + return FALSE; + } +} + +UBool IcuTestErrorCode::errDataIfFailureAndReset(const char *fmt, ...) { if(isFailure()) { char buffer[4000]; va_list ap; va_start(ap, fmt); vsprintf(buffer, fmt, ap); va_end(ap); - UnicodeString msg(testName, -1, US_INV); - msg.append(UNICODE_STRING_SIMPLE(" failure: ")).append(UnicodeString(errorName(), -1, US_INV)); - msg.append(UNICODE_STRING_SIMPLE(" - ")).append(UnicodeString(buffer, -1, US_INV)); - testClass.dataerrln(msg); + errlog(TRUE, u"data: expected success", buffer); reset(); return TRUE; } else { @@ -59,12 +79,55 @@ UBool IcuTestErrorCode::logDataIfFailureAndReset(const char *fmt, ...) { } } +UBool IcuTestErrorCode::expectErrorAndReset(UErrorCode expectedError) { + if(get() != expectedError) { + errlog(FALSE, UnicodeString(u"expected: ") + u_errorName(expectedError), nullptr); + } + UBool retval = isFailure(); + reset(); + return retval; +} + +UBool IcuTestErrorCode::expectErrorAndReset(UErrorCode expectedError, const char *fmt, ...) { + if(get() != expectedError) { + char buffer[4000]; + va_list ap; + va_start(ap, fmt); + vsprintf(buffer, fmt, ap); + va_end(ap); + errlog(FALSE, UnicodeString(u"expected: ") + u_errorName(expectedError), buffer); + } + UBool retval = isFailure(); + reset(); + return retval; +} + +void IcuTestErrorCode::setScope(const char* message) { + scopeMessage.remove().append({ message, -1, US_INV }); +} + +void IcuTestErrorCode::setScope(const UnicodeString& message) { + scopeMessage = message; +} + void IcuTestErrorCode::handleFailure() const { - // testClass.errln("%s failure - %s", testName, errorName()); + errlog(FALSE, u"(handleFailure)", nullptr); +} + +void IcuTestErrorCode::errlog(UBool dataErr, const UnicodeString& mainMessage, const char* extraMessage) const { UnicodeString msg(testName, -1, US_INV); - msg.append(UNICODE_STRING_SIMPLE(" failure: ")).append(UnicodeString(errorName(), -1, US_INV)); + msg.append(u' ').append(mainMessage); + msg.append(u" but got error: ").append(UnicodeString(errorName(), -1, US_INV)); + + if (!scopeMessage.isEmpty()) { + msg.append(u" scope: ").append(scopeMessage); + } + + if (extraMessage != nullptr) { + msg.append(u" - ").append(UnicodeString(extraMessage, -1, US_INV)); + } - if (get() == U_MISSING_RESOURCE_ERROR || get() == U_FILE_ACCESS_ERROR) { + if (dataErr || get() == U_MISSING_RESOURCE_ERROR || get() == U_FILE_ACCESS_ERROR) { testClass.dataerrln(msg); } else { testClass.errln(msg); diff --git a/Build/source/libs/icu/icu-src/source/tools/ctestfw/unicode/testlog.h b/Build/source/libs/icu/icu-src/source/tools/ctestfw/unicode/testlog.h index 811f62fba14..9db35b9ca85 100644 --- a/Build/source/libs/icu/icu-src/source/tools/ctestfw/unicode/testlog.h +++ b/Build/source/libs/icu/icu-src/source/tools/ctestfw/unicode/testlog.h @@ -32,17 +32,31 @@ public: class T_CTEST_EXPORT_API IcuTestErrorCode : public ErrorCode { public: - IcuTestErrorCode(TestLog &callingTestClass, const char *callingTestName) : - testClass(callingTestClass), testName(callingTestName) {} + IcuTestErrorCode(TestLog &callingTestClass, const char *callingTestName) + : testClass(callingTestClass), testName(callingTestName), scopeMessage() {} virtual ~IcuTestErrorCode(); + // Returns TRUE if isFailure(). - UBool logIfFailureAndReset(const char *fmt, ...); - UBool logDataIfFailureAndReset(const char *fmt, ...); + UBool errIfFailureAndReset(); + UBool errIfFailureAndReset(const char *fmt, ...); + UBool errDataIfFailureAndReset(); + UBool errDataIfFailureAndReset(const char *fmt, ...); + UBool expectErrorAndReset(UErrorCode expectedError); + UBool expectErrorAndReset(UErrorCode expectedError, const char *fmt, ...); + + /** Sets an additional message string to be appended to failure output. */ + void setScope(const char* message); + void setScope(const UnicodeString& message); + protected: virtual void handleFailure() const; + private: TestLog &testClass; const char *const testName; + UnicodeString scopeMessage; + + void errlog(UBool dataErr, const UnicodeString& mainMessage, const char* extraMessage) const; }; #endif diff --git a/Build/source/libs/icu/icu-src/source/tools/gencfu/gencfu.cpp b/Build/source/libs/icu/icu-src/source/tools/gencfu/gencfu.cpp index c4098c8cac3..1ac9ad3fd8a 100644 --- a/Build/source/libs/icu/icu-src/source/tools/gencfu/gencfu.cpp +++ b/Build/source/libs/icu/icu-src/source/tools/gencfu/gencfu.cpp @@ -319,7 +319,7 @@ int main(int argc, char **argv) { return NULL; } - long t = fread(result, 1, fileSize, file); + long t = static_cast<long>(fread(result, 1, fileSize, file)); if (t != fileSize) { delete [] result; fclose(file); diff --git a/Build/source/libs/icu/icu-src/source/tools/gencnval/gencnval.c b/Build/source/libs/icu/icu-src/source/tools/gencnval/gencnval.c index 8d2a04c1ac7..04ad747e077 100644 --- a/Build/source/libs/icu/icu-src/source/tools/gencnval/gencnval.c +++ b/Build/source/libs/icu/icu-src/source/tools/gencnval/gencnval.c @@ -965,7 +965,7 @@ createNormalizedAliasStrings(char *normalizedStrings, const char *origStringBloc if (currStrLen > 0) { int32_t normStrLen; ucnv_io_stripForCompare(normalizedStrings, origStringBlock); - normStrLen = uprv_strlen(normalizedStrings); + normStrLen = (int32_t)uprv_strlen(normalizedStrings); if (normStrLen > 0) { uprv_memset(normalizedStrings + normStrLen, 0, currStrSize - normStrLen); } diff --git a/Build/source/libs/icu/icu-src/source/tools/gennorm2/gennorm2.cpp b/Build/source/libs/icu/icu-src/source/tools/gennorm2/gennorm2.cpp index 2d24d61071b..bce5336be62 100644 --- a/Build/source/libs/icu/icu-src/source/tools/gennorm2/gennorm2.cpp +++ b/Build/source/libs/icu/icu-src/source/tools/gennorm2/gennorm2.cpp @@ -266,6 +266,11 @@ void parseFile(std::ifstream &f, Normalizer2DataBuilder &builder) { fprintf(stderr, "gennorm2 error: parsing code point range from %s\n", line); exit(errorCode.reset()); } + if (endCP >= 0xd800 && startCP <= 0xdfff) { + fprintf(stderr, "gennorm2 error: value or mapping for surrogate code points: %s\n", + line); + exit(U_ILLEGAL_ARGUMENT_ERROR); + } delimiter=u_skipWhitespace(delimiter); if(*delimiter==':') { const char *s=u_skipWhitespace(delimiter+1); diff --git a/Build/source/libs/icu/icu-src/source/tools/gennorm2/n2builder.cpp b/Build/source/libs/icu/icu-src/source/tools/gennorm2/n2builder.cpp index 7b8920d0d3a..1d388852312 100644 --- a/Build/source/libs/icu/icu-src/source/tools/gennorm2/n2builder.cpp +++ b/Build/source/libs/icu/icu-src/source/tools/gennorm2/n2builder.cpp @@ -29,7 +29,9 @@ #include "unicode/errorcode.h" #include "unicode/localpointer.h" #include "unicode/putil.h" +#include "unicode/ucptrie.h" #include "unicode/udata.h" +#include "unicode/umutablecptrie.h" #include "unicode/uniset.h" #include "unicode/unistr.h" #include "unicode/usetiter.h" @@ -41,7 +43,6 @@ #include "norms.h" #include "toolutil.h" #include "unewdata.h" -#include "utrie2.h" #include "uvectr32.h" #include "writesrc.h" @@ -58,8 +59,8 @@ static UDataInfo dataInfo={ 0, { 0x4e, 0x72, 0x6d, 0x32 }, /* dataFormat="Nrm2" */ - { 3, 0, 0, 0 }, /* formatVersion */ - { 10, 0, 0, 0 } /* dataVersion (Unicode version) */ + { 4, 0, 0, 0 }, /* formatVersion */ + { 11, 0, 0, 0 } /* dataVersion (Unicode version) */ }; U_NAMESPACE_BEGIN @@ -94,14 +95,14 @@ const HangulIterator::Range HangulIterator::ranges[4]={ Normalizer2DataBuilder::Normalizer2DataBuilder(UErrorCode &errorCode) : norms(errorCode), phase(0), overrideHandling(OVERRIDE_PREVIOUS), optimization(OPTIMIZE_NORMAL), - norm16Trie(nullptr), norm16TrieLength(0) { + norm16TrieBytes(nullptr), norm16TrieLength(0) { memset(unicodeVersion, 0, sizeof(unicodeVersion)); memset(indexes, 0, sizeof(indexes)); memset(smallFCD, 0, sizeof(smallFCD)); } Normalizer2DataBuilder::~Normalizer2DataBuilder() { - utrie2_close(norm16Trie); + delete[] norm16TrieBytes; } void @@ -407,11 +408,13 @@ void Normalizer2DataBuilder::postProcess(Norm &norm) { class Norm16Writer : public Norms::Enumerator { public: - Norm16Writer(Norms &n, Normalizer2DataBuilder &b) : Norms::Enumerator(n), builder(b) {} + Norm16Writer(UMutableCPTrie *trie, Norms &n, Normalizer2DataBuilder &b) : + Norms::Enumerator(n), builder(b), norm16Trie(trie) {} void rangeHandler(UChar32 start, UChar32 end, Norm &norm) U_OVERRIDE { - builder.writeNorm16(start, end, norm); + builder.writeNorm16(norm16Trie, start, end, norm); } Normalizer2DataBuilder &builder; + UMutableCPTrie *norm16Trie; }; void Normalizer2DataBuilder::setSmallFCD(UChar32 c) { @@ -419,7 +422,7 @@ void Normalizer2DataBuilder::setSmallFCD(UChar32 c) { smallFCD[lead>>8]|=(uint8_t)1<<((lead>>5)&7); } -void Normalizer2DataBuilder::writeNorm16(UChar32 start, UChar32 end, Norm &norm) { +void Normalizer2DataBuilder::writeNorm16(UMutableCPTrie *norm16Trie, UChar32 start, UChar32 end, Norm &norm) { if((norm.leadCC|norm.trailCC)!=0) { for(UChar32 c=start; c<=end; ++c) { setSmallFCD(c); @@ -484,7 +487,7 @@ void Normalizer2DataBuilder::writeNorm16(UChar32 start, UChar32 end, Norm &norm) norm16|=Normalizer2Impl::HAS_COMP_BOUNDARY_AFTER; } IcuToolErrorCode errorCode("gennorm2/writeNorm16()"); - utrie2_setRange32(norm16Trie, start, end, (uint32_t)norm16, TRUE, errorCode); + umutablecptrie_setRange(norm16Trie, start, end, (uint32_t)norm16, errorCode); // Set the minimum code points for real data lookups in the quick check loops. UBool isDecompNo= @@ -502,13 +505,13 @@ void Normalizer2DataBuilder::writeNorm16(UChar32 start, UChar32 end, Norm &norm) } } -void Normalizer2DataBuilder::setHangulData() { +void Normalizer2DataBuilder::setHangulData(UMutableCPTrie *norm16Trie) { HangulIterator hi; const HangulIterator::Range *range; // Check that none of the Hangul/Jamo code points have data. while((range=hi.nextRange())!=NULL) { for(UChar32 c=range->start; c<=range->end; ++c) { - if(utrie2_get32(norm16Trie, c)>Normalizer2Impl::INERT) { + if(umutablecptrie_get(norm16Trie, c)>Normalizer2Impl::INERT) { fprintf(stderr, "gennorm2 error: " "illegal mapping/composition/ccc data for Hangul or Jamo U+%04lX\n", @@ -524,13 +527,13 @@ void Normalizer2DataBuilder::setHangulData() { if(Hangul::JAMO_V_BASE<indexes[Normalizer2Impl::IX_MIN_COMP_NO_MAYBE_CP]) { indexes[Normalizer2Impl::IX_MIN_COMP_NO_MAYBE_CP]=Hangul::JAMO_V_BASE; } - utrie2_setRange32(norm16Trie, Hangul::JAMO_L_BASE, Hangul::JAMO_L_END, - Normalizer2Impl::JAMO_L, TRUE, errorCode); - utrie2_setRange32(norm16Trie, Hangul::JAMO_V_BASE, Hangul::JAMO_V_END, - Normalizer2Impl::JAMO_VT, TRUE, errorCode); + umutablecptrie_setRange(norm16Trie, Hangul::JAMO_L_BASE, Hangul::JAMO_L_END, + Normalizer2Impl::JAMO_L, errorCode); + umutablecptrie_setRange(norm16Trie, Hangul::JAMO_V_BASE, Hangul::JAMO_V_END, + Normalizer2Impl::JAMO_VT, errorCode); // JAMO_T_BASE+1: not U+11A7 - utrie2_setRange32(norm16Trie, Hangul::JAMO_T_BASE+1, Hangul::JAMO_T_END, - Normalizer2Impl::JAMO_VT, TRUE, errorCode); + umutablecptrie_setRange(norm16Trie, Hangul::JAMO_T_BASE+1, Hangul::JAMO_T_END, + Normalizer2Impl::JAMO_VT, errorCode); // Hangul LV encoded as minYesNo uint32_t lv=indexes[Normalizer2Impl::IX_MIN_YES_NO]; @@ -542,49 +545,16 @@ void Normalizer2DataBuilder::setHangulData() { } // Set the first LV, then write all other Hangul syllables as LVT, // then overwrite the remaining LV. - // The UTrie2 should be able to compact this into 7 32-item blocks - // because JAMO_T_COUNT is 28 and the UTrie2 granularity is 4. - // (7*32=8*28 smallest common multiple) - utrie2_set32(norm16Trie, Hangul::HANGUL_BASE, lv, errorCode); - utrie2_setRange32(norm16Trie, Hangul::HANGUL_BASE+1, Hangul::HANGUL_END, - lvt, TRUE, errorCode); + umutablecptrie_set(norm16Trie, Hangul::HANGUL_BASE, lv, errorCode); + umutablecptrie_setRange(norm16Trie, Hangul::HANGUL_BASE+1, Hangul::HANGUL_END, lvt, errorCode); UChar32 c=Hangul::HANGUL_BASE; while((c+=Hangul::JAMO_T_COUNT)<=Hangul::HANGUL_END) { - utrie2_set32(norm16Trie, c, lv, errorCode); + umutablecptrie_set(norm16Trie, c, lv, errorCode); } errorCode.assertSuccess(); } -namespace { - -struct Norm16Summary { - uint32_t maxNorm16; - // ANDing values yields 0 bits where any value has a 0. - // Used for worst-case HAS_COMP_BOUNDARY_AFTER. - uint32_t andedNorm16; -}; - -} // namespace - -U_CDECL_BEGIN - -static UBool U_CALLCONV -enumRangeMaxValue(const void *context, UChar32 /*start*/, UChar32 /*end*/, uint32_t value) { - Norm16Summary *p=(Norm16Summary *)context; - if(value>p->maxNorm16) { - p->maxNorm16=value; - } - p->andedNorm16&=value; - return TRUE; -} - -U_CDECL_END - -void Normalizer2DataBuilder::processData() { - IcuToolErrorCode errorCode("gennorm2/processData()"); - norm16Trie=utrie2_open(Normalizer2Impl::INERT, Normalizer2Impl::INERT, errorCode); - errorCode.assertSuccess(); - +LocalUCPTriePointer Normalizer2DataBuilder::processData() { // Build composition lists before recursive decomposition, // so that we still have the raw, pair-wise mappings. CompositionBuilder compBuilder(norms); @@ -652,13 +622,19 @@ void Normalizer2DataBuilder::processData() { indexes[Normalizer2Impl::IX_MIN_COMP_NO_MAYBE_CP]=0x110000; indexes[Normalizer2Impl::IX_MIN_LCCC_CP]=0x110000; + IcuToolErrorCode errorCode("gennorm2/processData()"); + UMutableCPTrie *norm16Trie = umutablecptrie_open( + Normalizer2Impl::INERT, Normalizer2Impl::INERT, errorCode); + errorCode.assertSuccess(); + // Map each code point to its norm16 value, // including the properties that fit directly, // and the offset to the "extra data" if necessary. - Norm16Writer norm16Writer(norms, *this); + Norm16Writer norm16Writer(norm16Trie, norms, *this); norms.enumRanges(norm16Writer); + // TODO: iterate via getRange() instead of callback? - setHangulData(); + setHangulData(norm16Trie); // Look for the "worst" norm16 value of any supplementary code point // corresponding to a lead surrogate, and set it as that surrogate's value. @@ -670,22 +646,63 @@ void Normalizer2DataBuilder::processData() { // and select the best value that only breaks the composition and/or decomposition // inner loops if necessary. // However, that seems like overkill for an optimization for supplementary characters. - for(UChar lead=0xd800; lead<0xdc00; ++lead) { - uint32_t surrogateCPNorm16=utrie2_get32(norm16Trie, lead); - Norm16Summary summary={ surrogateCPNorm16, surrogateCPNorm16 }; - utrie2_enumForLeadSurrogate(norm16Trie, lead, NULL, enumRangeMaxValue, &summary); - uint32_t norm16=summary.maxNorm16; - if(norm16>=(uint32_t)indexes[Normalizer2Impl::IX_LIMIT_NO_NO] && - norm16>(uint32_t)indexes[Normalizer2Impl::IX_MIN_NO_NO]) { - // Set noNo ("worst" value) if it got into "less-bad" maybeYes or ccc!=0. - // Otherwise it might end up at something like JAMO_VT which stays in - // the inner decomposition quick check loop. - norm16=(uint32_t)indexes[Normalizer2Impl::IX_LIMIT_NO_NO]-1; + // + // First check that surrogate code *points* are inert. + // The parser should have rejected values/mappings for them. + uint32_t value; + UChar32 end = umutablecptrie_getRange(norm16Trie, 0xd800, UCPMAP_RANGE_NORMAL, 0, + nullptr, nullptr, &value); + if (value != Normalizer2Impl::INERT || end < 0xdfff) { + fprintf(stderr, + "gennorm2 error: not all surrogate code points are inert: U+d800..U+%04x=%lx\n", + (int)end, (long)value); + exit(U_INTERNAL_PROGRAM_ERROR); + } + uint32_t maxNorm16 = 0; + // ANDing values yields 0 bits where any value has a 0. + // Used for worst-case HAS_COMP_BOUNDARY_AFTER. + uint32_t andedNorm16 = 0; + end = 0; + for (UChar32 start = 0x10000;;) { + if (start > end) { + end = umutablecptrie_getRange(norm16Trie, start, UCPMAP_RANGE_NORMAL, 0, + nullptr, nullptr, &value); + if (end < 0) { break; } + } + if ((start & 0x3ff) == 0) { + // Data for a new lead surrogate. + maxNorm16 = andedNorm16 = value; + } else { + if (value > maxNorm16) { + maxNorm16 = value; + } + andedNorm16 &= value; + } + // Intersect each range with the code points for one lead surrogate. + UChar32 leadEnd = start | 0x3ff; + if (leadEnd <= end) { + // End of the supplementary block for a lead surrogate. + if (maxNorm16 >= (uint32_t)indexes[Normalizer2Impl::IX_LIMIT_NO_NO]) { + // Set noNo ("worst" value) if it got into "less-bad" maybeYes or ccc!=0. + // Otherwise it might end up at something like JAMO_VT which stays in + // the inner decomposition quick check loop. + maxNorm16 = (uint32_t)indexes[Normalizer2Impl::IX_LIMIT_NO_NO]; + } + maxNorm16 = + (maxNorm16 & ~Normalizer2Impl::HAS_COMP_BOUNDARY_AFTER)| + (andedNorm16 & Normalizer2Impl::HAS_COMP_BOUNDARY_AFTER); + if (maxNorm16 != Normalizer2Impl::INERT) { + umutablecptrie_set(norm16Trie, U16_LEAD(start), maxNorm16, errorCode); + } + if (value == Normalizer2Impl::INERT) { + // Potentially skip inert supplementary blocks for several lead surrogates. + start = (end + 1) & ~0x3ff; + } else { + start = leadEnd + 1; + } + } else { + start = end + 1; } - norm16= - (norm16&~Normalizer2Impl::HAS_COMP_BOUNDARY_AFTER)| - (summary.andedNorm16&Normalizer2Impl::HAS_COMP_BOUNDARY_AFTER); - utrie2_set32ForLeadSurrogateCodeUnit(norm16Trie, lead, norm16, errorCode); } // Adjust supplementary minimum code points to break quick check loops at their lead surrogates. @@ -705,14 +722,19 @@ void Normalizer2DataBuilder::processData() { indexes[Normalizer2Impl::IX_MIN_LCCC_CP]=U16_LEAD(minCP); } - utrie2_freeze(norm16Trie, UTRIE2_16_VALUE_BITS, errorCode); - norm16TrieLength=utrie2_serialize(norm16Trie, NULL, 0, errorCode); + LocalUCPTriePointer builtTrie( + umutablecptrie_buildImmutable(norm16Trie, UCPTRIE_TYPE_FAST, UCPTRIE_VALUE_BITS_16, errorCode)); + norm16TrieLength=ucptrie_toBinary(builtTrie.getAlias(), nullptr, 0, errorCode); if(errorCode.get()!=U_BUFFER_OVERFLOW_ERROR) { - fprintf(stderr, "gennorm2 error: unable to freeze/serialize the normalization trie - %s\n", + fprintf(stderr, "gennorm2 error: unable to build/serialize the normalization trie - %s\n", errorCode.errorName()); exit(errorCode.reset()); } + umutablecptrie_close(norm16Trie); errorCode.reset(); + norm16TrieBytes=new uint8_t[norm16TrieLength]; + ucptrie_toBinary(builtTrie.getAlias(), norm16TrieBytes, norm16TrieLength, errorCode); + errorCode.assertSuccess(); int32_t offset=(int32_t)sizeof(indexes); indexes[Normalizer2Impl::IX_NORM_TRIE_OFFSET]=offset; @@ -750,16 +772,13 @@ void Normalizer2DataBuilder::processData() { u_versionFromString(unicodeVersion, U_UNICODE_VERSION); } memcpy(dataInfo.dataVersion, unicodeVersion, 4); + return builtTrie; } void Normalizer2DataBuilder::writeBinaryFile(const char *filename) { processData(); IcuToolErrorCode errorCode("gennorm2/writeBinaryFile()"); - LocalArray<uint8_t> norm16TrieBytes(new uint8_t[norm16TrieLength]); - utrie2_serialize(norm16Trie, norm16TrieBytes.getAlias(), norm16TrieLength, errorCode); - errorCode.assertSuccess(); - UNewDataMemory *pData= udata_create(NULL, NULL, filename, &dataInfo, haveCopyright ? U_COPYRIGHT_STRING : NULL, errorCode); @@ -769,7 +788,7 @@ void Normalizer2DataBuilder::writeBinaryFile(const char *filename) { exit(errorCode.reset()); } udata_writeBlock(pData, indexes, sizeof(indexes)); - udata_writeBlock(pData, norm16TrieBytes.getAlias(), norm16TrieLength); + udata_writeBlock(pData, norm16TrieBytes, norm16TrieLength); udata_writeUString(pData, toUCharPtr(extraData.getBuffer()), extraData.length()); udata_writeBlock(pData, smallFCD, sizeof(smallFCD)); int32_t writtenSize=udata_finish(pData, errorCode); @@ -787,7 +806,7 @@ void Normalizer2DataBuilder::writeBinaryFile(const char *filename) { void Normalizer2DataBuilder::writeCSourceFile(const char *filename) { - processData(); + LocalUCPTriePointer norm16Trie = processData(); IcuToolErrorCode errorCode("gennorm2/writeCSourceFile()"); const char *basename=findBasename(filename); @@ -797,54 +816,33 @@ Normalizer2DataBuilder::writeCSourceFile(const char *filename) { if(extension!=NULL) { dataName.truncate((int32_t)(extension-basename)); } + const char *name=dataName.data(); errorCode.assertSuccess(); - LocalArray<uint8_t> norm16TrieBytes(new uint8_t[norm16TrieLength]); - utrie2_serialize(norm16Trie, norm16TrieBytes.getAlias(), norm16TrieLength, errorCode); - errorCode.assertSuccess(); - - FILE *f=usrc_create(path.data(), basename, "icu/source/tools/gennorm2/n2builder.cpp"); + FILE *f=usrc_create(path.data(), basename, 2016, "icu/source/tools/gennorm2/n2builder.cpp"); if(f==NULL) { fprintf(stderr, "gennorm2/writeCSourceFile() error: unable to create the output file %s\n", filename); exit(U_FILE_ACCESS_ERROR); - return; } fputs("#ifdef INCLUDED_FROM_NORMALIZER2_CPP\n\n", f); + char line[100]; - sprintf(line, "static const UVersionInfo %s_formatVersion={", dataName.data()); + sprintf(line, "static const UVersionInfo %s_formatVersion={", name); usrc_writeArray(f, line, dataInfo.formatVersion, 8, 4, "};\n"); - sprintf(line, "static const UVersionInfo %s_dataVersion={", dataName.data()); + sprintf(line, "static const UVersionInfo %s_dataVersion={", name); usrc_writeArray(f, line, dataInfo.dataVersion, 8, 4, "};\n\n"); - sprintf(line, "static const int32_t %s_indexes[Normalizer2Impl::IX_COUNT]={\n", - dataName.data()); - usrc_writeArray(f, - line, - indexes, 32, Normalizer2Impl::IX_COUNT, - "\n};\n\n"); - sprintf(line, "static const uint16_t %s_trieIndex[%%ld]={\n", dataName.data()); - usrc_writeUTrie2Arrays(f, - line, NULL, - norm16Trie, - "\n};\n\n"); - sprintf(line, "static const uint16_t %s_extraData[%%ld]={\n", dataName.data()); - usrc_writeArray(f, - line, - extraData.getBuffer(), 16, extraData.length(), - "\n};\n\n"); - sprintf(line, "static const uint8_t %s_smallFCD[%%ld]={\n", dataName.data()); - usrc_writeArray(f, - line, - smallFCD, 8, sizeof(smallFCD), - "\n};\n\n"); - sprintf(line, "static const UTrie2 %s_trie={\n", dataName.data()); - char line2[100]; - sprintf(line2, "%s_trieIndex", dataName.data()); - usrc_writeUTrie2Struct(f, - line, - norm16Trie, line2, NULL, - "};\n"); - fputs("\n#endif // INCLUDED_FROM_NORMALIZER2_CPP\n", f); + sprintf(line, "static const int32_t %s_indexes[Normalizer2Impl::IX_COUNT]={\n", name); + usrc_writeArray(f, line, indexes, 32, Normalizer2Impl::IX_COUNT, "\n};\n\n"); + + usrc_writeUCPTrie(f, name, norm16Trie.getAlias()); + + sprintf(line, "static const uint16_t %s_extraData[%%ld]={\n", name); + usrc_writeArray(f, line, extraData.getBuffer(), 16, extraData.length(), "\n};\n\n"); + sprintf(line, "static const uint8_t %s_smallFCD[%%ld]={\n", name); + usrc_writeArray(f, line, smallFCD, 8, sizeof(smallFCD), "\n};\n\n"); + + fputs("#endif // INCLUDED_FROM_NORMALIZER2_CPP\n", f); fclose(f); } diff --git a/Build/source/libs/icu/icu-src/source/tools/gennorm2/n2builder.h b/Build/source/libs/icu/icu-src/source/tools/gennorm2/n2builder.h index 61b36be0044..77b4fef533a 100644 --- a/Build/source/libs/icu/icu-src/source/tools/gennorm2/n2builder.h +++ b/Build/source/libs/icu/icu-src/source/tools/gennorm2/n2builder.h @@ -24,10 +24,10 @@ #if !UCONFIG_NO_NORMALIZATION #include "unicode/errorcode.h" +#include "unicode/umutablecptrie.h" #include "unicode/unistr.h" #include "normalizer2impl.h" // for IX_COUNT #include "toolutil.h" -#include "utrie2.h" #include "norms.h" U_NAMESPACE_BEGIN @@ -95,9 +95,9 @@ private: return indexes[Normalizer2Impl::IX_MIN_MAYBE_YES]- ((2*Normalizer2Impl::MAX_DELTA+1)<<Normalizer2Impl::DELTA_SHIFT); } - void writeNorm16(UChar32 start, UChar32 end, Norm &norm); - void setHangulData(); - void processData(); + void writeNorm16(UMutableCPTrie *norm16Trie, UChar32 start, UChar32 end, Norm &norm); + void setHangulData(UMutableCPTrie *norm16Trie); + LocalUCPTriePointer processData(); Norms norms; @@ -107,7 +107,7 @@ private: Optimization optimization; int32_t indexes[Normalizer2Impl::IX_COUNT]; - UTrie2 *norm16Trie; + uint8_t *norm16TrieBytes; int32_t norm16TrieLength; UnicodeString extraData; uint8_t smallFCD[0x100]; diff --git a/Build/source/libs/icu/icu-src/source/tools/gennorm2/norms.cpp b/Build/source/libs/icu/icu-src/source/tools/gennorm2/norms.cpp index f0fb291aac3..96692f233cc 100644 --- a/Build/source/libs/icu/icu-src/source/tools/gennorm2/norms.cpp +++ b/Build/source/libs/icu/icu-src/source/tools/gennorm2/norms.cpp @@ -12,12 +12,12 @@ #include <stdio.h> #include <stdlib.h> #include "unicode/errorcode.h" +#include "unicode/umutablecptrie.h" #include "unicode/unistr.h" #include "unicode/utf16.h" #include "normalizer2impl.h" #include "norms.h" #include "toolutil.h" -#include "utrie2.h" #include "uvectr32.h" U_NAMESPACE_BEGIN @@ -67,7 +67,7 @@ UChar32 Norm::combine(UChar32 trail) const { } Norms::Norms(UErrorCode &errorCode) { - normTrie=utrie2_open(0, 0, &errorCode); + normTrie = umutablecptrie_open(0, 0, &errorCode); normMem=utm_open("gennorm2 normalization structs", 10000, 0x110100, sizeof(Norm)); // Default "inert" Norm struct at index 0. Practically immutable. norms=allocNorm(); @@ -75,7 +75,7 @@ Norms::Norms(UErrorCode &errorCode) { } Norms::~Norms() { - utrie2_close(normTrie); + umutablecptrie_close(normTrie); int32_t normsLength=utm_countItems(normMem); for(int32_t i=1; i<normsLength; ++i) { delete norms[i].mapping; @@ -92,7 +92,7 @@ Norm *Norms::allocNorm() { } Norm *Norms::getNorm(UChar32 c) { - uint32_t i=utrie2_get32(normTrie, c); + uint32_t i = umutablecptrie_get(normTrie, c); if(i==0) { return nullptr; } @@ -100,7 +100,7 @@ Norm *Norms::getNorm(UChar32 c) { } const Norm *Norms::getNorm(UChar32 c) const { - uint32_t i=utrie2_get32(normTrie, c); + uint32_t i = umutablecptrie_get(normTrie, c); if(i==0) { return nullptr; } @@ -108,18 +108,18 @@ const Norm *Norms::getNorm(UChar32 c) const { } const Norm &Norms::getNormRef(UChar32 c) const { - return norms[utrie2_get32(normTrie, c)]; + return norms[umutablecptrie_get(normTrie, c)]; } Norm *Norms::createNorm(UChar32 c) { - uint32_t i=utrie2_get32(normTrie, c); + uint32_t i=umutablecptrie_get(normTrie, c); if(i!=0) { return norms+i; } else { /* allocate Norm */ Norm *p=allocNorm(); IcuToolErrorCode errorCode("gennorm2/createNorm()"); - utrie2_set32(normTrie, c, (uint32_t)(p-norms), errorCode); + umutablecptrie_set(normTrie, c, (uint32_t)(p - norms), errorCode); return p; } } @@ -153,28 +153,20 @@ UBool Norms::combinesWithCCBetween(const Norm &norm, uint8_t lowCC, int32_t high return FALSE; } -U_CDECL_BEGIN - -static UBool U_CALLCONV -enumRangeHandler(const void *context, UChar32 start, UChar32 end, uint32_t value) { - return ((Norms::Enumerator *)context)->rangeHandler(start, end, value); -} - -U_CDECL_END - void Norms::enumRanges(Enumerator &e) { - utrie2_enum(normTrie, nullptr, enumRangeHandler, &e); + UChar32 start = 0, end; + uint32_t i; + while ((end = umutablecptrie_getRange(normTrie, start, UCPMAP_RANGE_NORMAL, 0, + nullptr, nullptr, &i)) >= 0) { + if (i > 0) { + e.rangeHandler(start, end, norms[i]); + } + start = end + 1; + } } Norms::Enumerator::~Enumerator() {} -UBool Norms::Enumerator::rangeHandler(UChar32 start, UChar32 end, uint32_t value) { - if(value!=0) { - rangeHandler(start, end, norms.getNormRefByIndex(value)); - } - return TRUE; -} - void CompositionBuilder::rangeHandler(UChar32 start, UChar32 end, Norm &norm) { if(norm.mappingType!=Norm::ROUND_TRIP) { return; } if(start!=end) { diff --git a/Build/source/libs/icu/icu-src/source/tools/gennorm2/norms.h b/Build/source/libs/icu/icu-src/source/tools/gennorm2/norms.h index 4bf6e760e20..5684e65e672 100644 --- a/Build/source/libs/icu/icu-src/source/tools/gennorm2/norms.h +++ b/Build/source/libs/icu/icu-src/source/tools/gennorm2/norms.h @@ -15,12 +15,12 @@ #if !UCONFIG_NO_NORMALIZATION #include "unicode/errorcode.h" +#include "unicode/umutablecptrie.h" #include "unicode/uniset.h" #include "unicode/unistr.h" #include "unicode/utf16.h" #include "normalizer2impl.h" #include "toolutil.h" -#include "utrie2.h" #include "uvectr32.h" U_NAMESPACE_BEGIN @@ -176,8 +176,6 @@ public: virtual ~Enumerator(); /** Called for enumerated value!=0. */ virtual void rangeHandler(UChar32 start, UChar32 end, Norm &norm) = 0; - /** @internal Public only for C callback. */ - UBool rangeHandler(UChar32 start, UChar32 end, uint32_t value); protected: Norms &norms; }; @@ -190,7 +188,7 @@ private: Norms(const Norms &other) = delete; Norms &operator=(const Norms &other) = delete; - UTrie2 *normTrie; + UMutableCPTrie *normTrie; UToolMemory *normMem; Norm *norms; }; diff --git a/Build/source/libs/icu/icu-src/source/tools/genrb/parse.cpp b/Build/source/libs/icu/icu-src/source/tools/genrb/parse.cpp index 465c099baba..44eb0c082bc 100644 --- a/Build/source/libs/icu/icu-src/source/tools/genrb/parse.cpp +++ b/Build/source/libs/icu/icu-src/source/tools/genrb/parse.cpp @@ -1018,6 +1018,11 @@ addCollation(ParseState* state, TableResource *result, const char *collationTyp icu::CollationInfo::printReorderRanges( *t->data, t->settings->reorderCodes, t->settings->reorderCodesLength); } +#if 0 // debugging output + } else { + printf("%s~%s collation tailoring part sizes:\n", state->filename, collationType); + icu::CollationInfo::printSizes(totalSize, indexes); +#endif } struct SResource *collationBin = bin_open(state->bundle, "%%CollationBin", totalSize, dest, NULL, NULL, status); result->add(collationBin, line, *status); diff --git a/Build/source/libs/icu/icu-src/source/tools/genrb/reslist.cpp b/Build/source/libs/icu/icu-src/source/tools/genrb/reslist.cpp index 2e04bbce21e..0493347ebe5 100644 --- a/Build/source/libs/icu/icu-src/source/tools/genrb/reslist.cpp +++ b/Build/source/libs/icu/icu-src/source/tools/genrb/reslist.cpp @@ -1395,7 +1395,7 @@ SRBRoot::compactKeys(UErrorCode &errorCode) { int32_t offset; suffix = keys + map[j].oldpos; for (suffixLimit = suffix; *suffixLimit != 0; ++suffixLimit) {} - offset = (int32_t)(keyLimit - key) - (suffixLimit - suffix); + offset = static_cast<int32_t>((keyLimit - key) - (suffixLimit - suffix)); if (offset < 0) { break; /* suffix cannot be longer than the original */ } diff --git a/Build/source/libs/icu/icu-src/source/tools/genren/genren.pl b/Build/source/libs/icu/icu-src/source/tools/genren/genren.pl index 8859a27ef57..f85b96ac607 100755 --- a/Build/source/libs/icu/icu-src/source/tools/genren/genren.pl +++ b/Build/source/libs/icu/icu-src/source/tools/genren/genren.pl @@ -106,6 +106,9 @@ print HEADER <<"EndOfHeaderComment"; #if !U_DISABLE_RENAMING +// Disable Renaming for Visual Studio's IntelliSense feature, so that 'Go-to-Definition' (F12) will work. +#if !(defined(_MSC_VER) && defined(__INTELLISENSE__)) + /* We need the U_ICU_ENTRY_POINT_RENAME definition. There's a default one in unicode/uvernum.h we can use, but we will give the platform a chance to define it first. Normally (if utypes.h or umachine.h was included first) this will not be necessary as it will already be defined. @@ -236,8 +239,15 @@ foreach(sort keys(%CFuncs)) { # print HEADER "#define $_ $_$U_ICU_VERSION_SUFFIX\n"; } -print HEADER "\n#endif\n"; -print HEADER "\n#endif\n"; + +print HEADER <<"EndOfHeaderFooter"; + +#endif /* !(defined(_MSC_VER) && defined(__INTELLISENSE__)) */ +#endif /* U_DISABLE_RENAMING */ +#endif /* URENAME_H */ + +EndOfHeaderFooter + close HEADER; diff --git a/Build/source/libs/icu/icu-src/source/tools/gensprep/gensprep.c b/Build/source/libs/icu/icu-src/source/tools/gensprep/gensprep.c index e3466379e12..a9c9efac95b 100644 --- a/Build/source/libs/icu/icu-src/source/tools/gensprep/gensprep.c +++ b/Build/source/libs/icu/icu-src/source/tools/gensprep/gensprep.c @@ -355,7 +355,7 @@ strprepProfileLineFn(void *context, if (*s == '@') { /* special directive */ s++; - length = fields[0][1] - s; + length = (int32_t)(fields[0][1] - s); if (length >= NORMALIZE_DIRECTIVE_LEN && uprv_strncmp(s, NORMALIZE_DIRECTIVE, NORMALIZE_DIRECTIVE_LEN) == 0) { options[NORMALIZE].doesOccur = TRUE; diff --git a/Build/source/libs/icu/icu-src/source/tools/icu-file-utf8-check.py b/Build/source/libs/icu/icu-src/source/tools/icu-file-utf8-check.py new file mode 100755 index 00000000000..41b5a7b3f89 --- /dev/null +++ b/Build/source/libs/icu/icu-src/source/tools/icu-file-utf8-check.py @@ -0,0 +1,104 @@ +#! /usr/bin/python -B + +# Copyright (C) 2016 and later: Unicode, Inc. and others. +# License & terms of use: http://www.unicode.org/copyright.html + +# Copyright (C) 2009-2011, International Business Machines Corporation, Google and Others. +# All rights reserved. + +# +# Script to check that ICU source files contain only valid UTF-8 encoded text, +# and that all files except '.txt' files do not contain a Byte Order Mark (BOM). +# +# THIS SCRIPT DOES NOT WORK ON WINDOWS +# It only works correctly on platforms where the native line ending is a plain \n +# +# usage: +# icu-file-utf8-check.py [options] +# +# options: +# -h | --help Print a usage line and exit. +# +# The tool operates recursively on the directory from which it is run. +# Only files from the ICU github repository are checked. +# No changes are made to the repository; only the working copy will be altered. + +import sys +import os +import os.path +import re +import getopt + + +def runCommand(cmd): + output_file = os.popen(cmd); + output_text = output_file.read(); + exit_status = output_file.close(); + if exit_status: + print >>sys.stderr, '"', cmd, '" failed. Exiting.' + sys.exit(exit_status) + return output_text + + +def usage(): + print "usage: " + sys.argv[0] + " [-h | --help]" + + +# +# File check. Check source code files for UTF-8 and all except text files for not containing a BOM +# file_name: name of a text file. +# is_source: Flag, set to True if file is a source code file (.c, .cpp, .h, .java). +# +def check_file(file_name, is_source): + f = open(file_name, 'r') + bytes = f.read() + f.close() + + if is_source: + try: + bytes.decode("UTF-8") + except UnicodeDecodeError: + print "Error: %s is a source code file but contains non-utf-8 bytes." % file_name + + if ord(bytes[0]) == 0xef: + if not (file_name.endswith(".txt") or file_name.endswith(".sln") + or file_name.endswith(".targets") + or ".vcxproj" in file_name): + print "Warning: file %s contains a UTF-8 BOM: " % file_name + + return + +def main(argv): + try: + opts, args = getopt.getopt(argv, "h", ("help")) + except getopt.GetoptError: + print "unrecognized option: " + argv[0] + usage() + sys.exit(2) + for opt, arg in opts: + if opt in ("-h", "--help"): + usage() + sys.exit() + if args: + print "unexpected command line argument" + usage() + sys.exit() + + output = runCommand("git ls-files "); + file_list = output.splitlines() + + source_file_re = re.compile(".*((?:\\.c$)|(?:\\.cpp$)|(?:\\.h$)|(?:\\.java$))") + + for f in file_list: + if os.path.isdir(f): + print "Skipping dir " + f + continue + if not os.path.isfile(f): + print "Repository file not in working copy: " + f + continue; + + source_file = source_file_re.match(f) + check_file(f, source_file) + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/Build/source/libs/icu/icu-src/source/tools/icu-svnprops-check.py b/Build/source/libs/icu/icu-src/source/tools/icu-svnprops-check.py deleted file mode 100644 index b0e86b3e88c..00000000000 --- a/Build/source/libs/icu/icu-src/source/tools/icu-svnprops-check.py +++ /dev/null @@ -1,189 +0,0 @@ -#! /usr/bin/python - -# Copyright (C) 2016 and later: Unicode, Inc. and others. -# License & terms of use: http://www.unicode.org/copyright.html - -# Copyright (C) 2009-2011, International Business Machines Corporation, Google and Others. -# All rights reserved. - -# -# Script to check and fix svn property settings for ICU source files. -# Also check for the correct line endings on files with svn:eol-style = native -# -# THIS SCRIPT DOES NOT WORK ON WINDOWS -# It only works correctly on platforms where the native line ending is a plain \n -# -# usage: -# icu-svnprops-check.py [options] -# -# options: -# -f | --fix Fix any problems that are found -# -h | --help Print a usage line and exit. -# -# The tool operates recursively on the directory from which it is run. -# Only files from the svn repository are checked. -# No changes are made to the repository; only the working copy will be altered. - -import sys -import os -import os.path -import re -import getopt - - -# file_types: The parsed form of the svn auto-props specification. -# A list of file types - .cc, .cpp, .txt, etc. -# each element is a [type, proplist] -# "type" is a regular expression string that will match a file name -# prop list is another list, one element per property. -# Each property item is a two element list, [prop name, prop value] -file_types = list() - -def parse_auto_props(): - aprops = svn_auto_props.splitlines() - for propline in aprops: - if re.match("\s*(#.*)?$", propline): # Match comment and blank lines - continue - if re.match("\s*\[auto-props\]", propline): # Match the [auto-props] line. - continue - if not re.match("\s*[^\s]+\s*=", propline): # minimal syntax check for <file-type> = - print "Bad line from autoprops definitions: " + propline - continue - file_type, string_proplist = propline.split("=", 1) - - #transform the file type expression from autoprops into a normal regular expression. - # e.g. "*.cpp" ==> ".*\.cpp$" - file_type = file_type.strip() - file_type = file_type.replace(".", "\.") - file_type = file_type.replace("*", ".*") - file_type = file_type + "$" - - # example string_proplist at this point: " svn:eol-style=native;svn:executable" - # split on ';' into a list of properties. The negative lookahead and lookbehind - # in the split regexp are to prevent matching on ';;', which is an escaped ';' - # within a property value. - string_proplist = re.split("(?<!;);(?!;)", string_proplist) - proplist = list() - for prop in string_proplist: - if prop.find("=") >= 0: - prop_name, prop_val = prop.split("=", 1) - else: - # properties with no explicit value, e.g. svn:executable - prop_name, prop_val = prop, "" - prop_name = prop_name.strip() - prop_val = prop_val.strip() - # unescape any ";;" in a property value, e.g. the mime-type from - # *.java = svn:eol-style=native;svn:mime-type=text/plain;;charset=utf-8 - prop_val = prop_val.replace(";;", ";"); - # If the prop value "is quoted", remove the quotes. - # See svn:keywords for an example of a quoted prop value. - match = re.match('^"(.+)"$', prop_val) - if match: - prop_val = match.group(1) - proplist.append((prop_name, prop_val)) - - file_types.append((file_type, proplist)) - # print file_types - - -def runCommand(cmd): - output_file = os.popen(cmd); - output_text = output_file.read(); - exit_status = output_file.close(); - if exit_status: - print >>sys.stderr, '"', cmd, '" failed. Exiting.' - sys.exit(exit_status) - return output_text - -svn_auto_props = runCommand("svn propget svn:auto-props http://source.icu-project.org/repos/icu") - -def usage(): - print "usage: " + sys.argv[0] + " [-f | --fix] [-h | --help]" - - -# -# UTF-8 file check. For text files with svn:mime-type=text/anything, check the specified charset -# file_name: name of a text file. -# base_mime_type: svn:mime-type property from the auto-props settings for this file type. -# actual_mime_type: existing svn:mime-type property value for the file. -# return: The correct svn:mime-type property value, -# either the original, if it looks OK, otherwise the value from auto-props -# -def check_utf8(file_name, base_mime_type, actual_mime_type): - - f = open(file_name, 'r') - bytes = f.read() - f.close() - file_is_utf8 = True - try: - bytes.decode("UTF-8") - except UnicodeDecodeError: - file_is_utf8 = False - - if not file_is_utf8 and actual_mime_type.find("utf-8") >= 0: - print "Error: %s is not valid utf-8, but has a utf-8 mime type." % file_name - return actual_mime_type - - if file_is_utf8 and actual_mime_type.find("charset") >=0 and actual_mime_type.find("utf-8") < 0: - print "Warning: %s is valid utf-8, but has a mime-type of %s." % (file_name, actual_mime_type) - - if ord(bytes[0]) == 0xef: - if not file_name.endswith(".txt"): - print "Warning: file %s contains a UTF-8 BOM: " % file_name - - # If the file already has a charset in its mime-type, don't make any change. - - if actual_mime_type.find("charset=") >= 0: - return actual_mime_type; - - return base_mime_type - - -def main(argv): - fix_problems = False; - try: - opts, args = getopt.getopt(argv, "fh", ("fix", "help")) - except getopt.GetoptError: - print "unrecognized option: " + argv[0] - usage() - sys.exit(2) - for opt, arg in opts: - if opt in ("-h", "--help"): - usage() - sys.exit() - if opt in ("-f", "--fix"): - fix_problems = True - if args: - print "unexpected command line argument" - usage() - sys.exit() - - parse_auto_props() - output = runCommand("svn ls -R "); - file_list = output.splitlines() - - for f in file_list: - if os.path.isdir(f): - # print "Skipping dir " + f - continue - if not os.path.isfile(f): - print "Repository file not in working copy: " + f - continue; - - for file_pattern, props in file_types: - if re.match(file_pattern, f): - # print "doing " + f - for propname, propval in props: - actual_propval = runCommand("svn propget --strict " + propname + " " + f) - #print propname + ": " + actual_propval - if propname == "svn:mime-type" and propval.find("text/") == 0: - # check for UTF-8 text files, should have svn:mime-type=text/something; charset=utf8 - propval = check_utf8(f, propval, actual_propval) - if not (propval == actual_propval or (propval == "" and actual_propval == "*")): - print "svn propset %s '%s' %s" % (propname, propval, f) - if fix_problems: - os.system("svn propset %s '%s' %s" % (propname, propval, f)) - - -if __name__ == "__main__": - main(sys.argv[1:]) diff --git a/Build/source/libs/icu/icu-src/source/tools/pkgdata/pkgdata.cpp b/Build/source/libs/icu/icu-src/source/tools/pkgdata/pkgdata.cpp index ffcf459c349..b2cd0ab5eb9 100644 --- a/Build/source/libs/icu/icu-src/source/tools/pkgdata/pkgdata.cpp +++ b/Build/source/libs/icu/icu-src/source/tools/pkgdata/pkgdata.cpp @@ -511,7 +511,7 @@ main(int argc, char* argv[]) { static int runCommand(const char* command, UBool specialHandling) { char *cmd = NULL; char cmdBuffer[SMALL_BUFFER_MAX_SIZE]; - int32_t len = strlen(command); + int32_t len = static_cast<int32_t>(strlen(command)); if (len == 0) { return 0; @@ -904,7 +904,8 @@ static void createFileNames(UPKGOptions *o, const char mode, const char *version if (IN_DLL_MODE(mode)) { sprintf(libFileNames[LIB_FILE], "%s", libName); } else { - sprintf(libFileNames[LIB_FILE], "%s%s", + sprintf(libFileNames[LIB_FILE], "%s%s%s", + (strstr(libName, "icudt") ? "lib" : ""), pkgDataFlags[LIBPREFIX], libName); } @@ -1020,7 +1021,7 @@ static int32_t pkg_createSymLinks(const char *targetDir, UBool specialHandling) char name2[SMALL_BUFFER_MAX_SIZE]; /* file name to symlink */ const char* FILE_EXTENSION_SEP = uprv_strlen(pkgDataFlags[SO_EXT]) == 0 ? "" : "."; -#if !defined(USING_CYGWIN) && U_PLATFORM != U_PF_MINGW +#if U_PLATFORM != U_PF_CYGWIN /* No symbolic link to make. */ if (uprv_strlen(libFileNames[LIB_FILE_VERSION]) == 0 || uprv_strlen(libFileNames[LIB_FILE_VERSION_MAJOR]) == 0 || uprv_strcmp(libFileNames[LIB_FILE_VERSION], libFileNames[LIB_FILE_VERSION_MAJOR]) == 0) { @@ -1225,7 +1226,7 @@ static int32_t pkg_installFileMode(const char *installDir, const char *srcDir, c if (f != NULL) { for(;;) { if (T_FileStream_readLine(f, buffer, SMALL_BUFFER_MAX_SIZE) != NULL) { - bufferLength = uprv_strlen(buffer); + bufferLength = static_cast<int32_t>(uprv_strlen(buffer)); /* Remove new line character. */ if (bufferLength > 0) { buffer[bufferLength-1] = 0; diff --git a/Build/source/libs/icu/icu-src/source/tools/toolutil/filestrm.cpp b/Build/source/libs/icu/icu-src/source/tools/toolutil/filestrm.cpp index 2653747727f..a926848985a 100644 --- a/Build/source/libs/icu/icu-src/source/tools/toolutil/filestrm.cpp +++ b/Build/source/libs/icu/icu-src/source/tools/toolutil/filestrm.cpp @@ -104,14 +104,14 @@ T_FileStream_tmpfile() U_CAPI int32_t U_EXPORT2 T_FileStream_read(FileStream* fileStream, void* addr, int32_t len) { - return fread(addr, 1, len, (FILE*)fileStream); + return static_cast<int32_t>(fread(addr, 1, len, (FILE*)fileStream)); } U_CAPI int32_t U_EXPORT2 T_FileStream_write(FileStream* fileStream, const void* addr, int32_t len) { - return fwrite(addr, 1, len, (FILE*)fileStream); + return static_cast<int32_t>(fwrite(addr, 1, len, (FILE*)fileStream)); } U_CAPI void U_EXPORT2 diff --git a/Build/source/libs/icu/icu-src/source/tools/toolutil/filetools.cpp b/Build/source/libs/icu/icu-src/source/tools/toolutil/filetools.cpp index 176a791b0df..6e88c94b520 100644 --- a/Build/source/libs/icu/icu-src/source/tools/toolutil/filetools.cpp +++ b/Build/source/libs/icu/icu-src/source/tools/toolutil/filetools.cpp @@ -134,7 +134,7 @@ static int32_t whichFileModTimeIsLater(const char *file1, const char *file2) { /* Swap the file separater character given with the new one in the file path. */ U_CAPI void U_EXPORT2 swapFileSepChar(char *filePath, const char oldFileSepChar, const char newFileSepChar) { - for (int32_t i = 0, length = uprv_strlen(filePath); i < length; i++) { + for (int32_t i = 0, length = static_cast<int32_t>(uprv_strlen(filePath)); i < length; i++) { filePath[i] = (filePath[i] == oldFileSepChar ) ? newFileSepChar : filePath[i]; } } diff --git a/Build/source/libs/icu/icu-src/source/tools/toolutil/package.cpp b/Build/source/libs/icu/icu-src/source/tools/toolutil/package.cpp index d96c6dd36dd..f4e428a37e7 100644 --- a/Build/source/libs/icu/icu-src/source/tools/toolutil/package.cpp +++ b/Build/source/libs/icu/icu-src/source/tools/toolutil/package.cpp @@ -610,7 +610,7 @@ Package::readPackage(const char *filename) { memcpy(prefix, s, ++prefixLength); // include the / } else { // Use the package basename as prefix. - int32_t inPkgNameLength=strlen(inPkgName); + int32_t inPkgNameLength= static_cast<int32_t>(strlen(inPkgName)); memcpy(prefix, inPkgName, inPkgNameLength); prefixLength=inPkgNameLength; @@ -1043,7 +1043,7 @@ Package::addItem(const char *name, uint8_t *data, int32_t length, UBool isDataOw memset(items+idx, 0, sizeof(Item)); // copy the item's name - items[idx].name=allocString(TRUE, strlen(name)); + items[idx].name=allocString(TRUE, static_cast<int32_t>(strlen(name))); strcpy(items[idx].name, name); pathToTree(items[idx].name); } else { diff --git a/Build/source/libs/icu/icu-src/source/tools/toolutil/swapimpl.cpp b/Build/source/libs/icu/icu-src/source/tools/toolutil/swapimpl.cpp index a64a6a1703f..e8850cb9864 100644 --- a/Build/source/libs/icu/icu-src/source/tools/toolutil/swapimpl.cpp +++ b/Build/source/libs/icu/icu-src/source/tools/toolutil/swapimpl.cpp @@ -243,7 +243,7 @@ uprops_swap(const UDataSwapper *ds, * swap the main properties UTrie * PT serialized properties trie, see utrie.h (byte size: 4*(i0-16)) */ - utrie2_swapAnyVersion(ds, + utrie_swapAnyVersion(ds, inData32+UPROPS_INDEX_COUNT, 4*(dataIndexes[UPROPS_PROPS32_INDEX]-UPROPS_INDEX_COUNT), outData32+UPROPS_INDEX_COUNT, @@ -274,7 +274,7 @@ uprops_swap(const UDataSwapper *ds, * swap the additional UTrie * i3 additionalTrieIndex; -- 32-bit unit index to the additional trie for more properties */ - utrie2_swapAnyVersion(ds, + utrie_swapAnyVersion(ds, inData32+dataIndexes[UPROPS_ADDITIONAL_TRIE_INDEX], 4*(dataIndexes[UPROPS_ADDITIONAL_VECTORS_INDEX]-dataIndexes[UPROPS_ADDITIONAL_TRIE_INDEX]), outData32+dataIndexes[UPROPS_ADDITIONAL_TRIE_INDEX], @@ -336,7 +336,7 @@ ucase_swap(const UDataSwapper *ds, ((pInfo->formatVersion[0]==1 && pInfo->formatVersion[2]==UTRIE_SHIFT && pInfo->formatVersion[3]==UTRIE_INDEX_SHIFT) || - pInfo->formatVersion[0]==2 || pInfo->formatVersion[0]==3) + (2<=pInfo->formatVersion[0] && pInfo->formatVersion[0]<=4)) )) { udata_printError(ds, "ucase_swap(): data format %02x.%02x.%02x.%02x (format version %02x) is not recognized as case mapping data\n", pInfo->dataFormat[0], pInfo->dataFormat[1], @@ -391,7 +391,7 @@ ucase_swap(const UDataSwapper *ds, /* swap the UTrie */ count=indexes[UCASE_IX_TRIE_SIZE]; - utrie2_swapAnyVersion(ds, inBytes+offset, count, outBytes+offset, pErrorCode); + utrie_swapAnyVersion(ds, inBytes+offset, count, outBytes+offset, pErrorCode); offset+=count; /* swap the uint16_t exceptions[] and unfold[] */ @@ -493,7 +493,7 @@ ubidi_swap(const UDataSwapper *ds, /* swap the UTrie */ count=indexes[UBIDI_IX_TRIE_SIZE]; - utrie2_swapAnyVersion(ds, inBytes+offset, count, outBytes+offset, pErrorCode); + utrie_swapAnyVersion(ds, inBytes+offset, count, outBytes+offset, pErrorCode); offset+=count; /* swap the uint32_t mirrors[] */ diff --git a/Build/source/libs/icu/icu-src/source/tools/toolutil/toolutil.vcxproj b/Build/source/libs/icu/icu-src/source/tools/toolutil/toolutil.vcxproj index 4df76dd4559..bf379f20789 100644 --- a/Build/source/libs/icu/icu-src/source/tools/toolutil/toolutil.vcxproj +++ b/Build/source/libs/icu/icu-src/source/tools/toolutil/toolutil.vcxproj @@ -71,7 +71,7 @@ <CompileAs>Default</CompileAs> </ClCompile> <Link> - <OutputFile>..\..\..\bin\icutu61.dll</OutputFile> + <OutputFile>..\..\..\bin\icutu63.dll</OutputFile> <AdditionalDependencies>icuuc.lib;icuin.lib;%(AdditionalDependencies)</AdditionalDependencies> <AdditionalLibraryDirectories>..\..\..\lib;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories> <ProgramDatabaseFile>.\..\..\..\lib\icutu.pdb</ProgramDatabaseFile> @@ -97,7 +97,7 @@ <CompileAs>Default</CompileAs> </ClCompile> <Link> - <OutputFile>..\..\..\bin\icutu61d.dll</OutputFile> + <OutputFile>..\..\..\bin\icutu63d.dll</OutputFile> <AdditionalDependencies>icuucd.lib;icuind.lib;%(AdditionalDependencies)</AdditionalDependencies> <AdditionalLibraryDirectories>..\..\..\lib;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories> <ProgramDatabaseFile>.\..\..\..\lib\icutud.pdb</ProgramDatabaseFile> @@ -123,7 +123,7 @@ <CompileAs>Default</CompileAs> </ClCompile> <Link> - <OutputFile>..\..\..\bin64\icutu61.dll</OutputFile> + <OutputFile>..\..\..\bin64\icutu63.dll</OutputFile> <AdditionalDependencies>icuuc.lib;icuin.lib;%(AdditionalDependencies)</AdditionalDependencies> <AdditionalLibraryDirectories>..\..\..\lib64;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories> <ProgramDatabaseFile>.\..\..\..\lib64\icutu.pdb</ProgramDatabaseFile> @@ -146,7 +146,7 @@ <CompileAs>Default</CompileAs> </ClCompile> <Link> - <OutputFile>..\..\..\bin64\icutu61d.dll</OutputFile> + <OutputFile>..\..\..\bin64\icutu63d.dll</OutputFile> <AdditionalDependencies>icuucd.lib;icuind.lib;%(AdditionalDependencies)</AdditionalDependencies> <AdditionalLibraryDirectories>..\..\..\lib64;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories> <ProgramDatabaseFile>.\..\..\..\lib64\icutud.pdb</ProgramDatabaseFile> @@ -227,4 +227,4 @@ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> <ImportGroup Label="ExtensionTargets"> </ImportGroup> -</Project>
\ No newline at end of file +</Project> diff --git a/Build/source/libs/icu/icu-src/source/tools/toolutil/udbgutil.cpp b/Build/source/libs/icu/icu-src/source/tools/toolutil/udbgutil.cpp index 446e11aaf90..285f68a0ec6 100644 --- a/Build/source/libs/icu/icu-src/source/tools/toolutil/udbgutil.cpp +++ b/Build/source/libs/icu/icu-src/source/tools/toolutil/udbgutil.cpp @@ -400,7 +400,7 @@ U_CAPI int32_t paramStatic(const USystemParams *param, char *target, int32_t targetCapacity, UErrorCode *status) { if(param->paramStr==NULL) return paramEmpty(param,target,targetCapacity,status); if(U_FAILURE(*status))return 0; - int32_t len = uprv_strlen(param->paramStr); + int32_t len = static_cast<int32_t>(uprv_strlen(param->paramStr)); if(target!=NULL) { uprv_strncpy(target,param->paramStr,uprv_min(len,targetCapacity)); } @@ -412,14 +412,14 @@ static const char *nullString = "(null)"; static int32_t stringToStringBuffer(char *target, int32_t targetCapacity, const char *str, UErrorCode *status) { if(str==NULL) str=nullString; - int32_t len = uprv_strlen(str); + int32_t len = static_cast<int32_t>(uprv_strlen(str)); if (U_SUCCESS(*status)) { if(target!=NULL) { uprv_strncpy(target,str,uprv_min(len,targetCapacity)); } } else { const char *s = u_errorName(*status); - len = uprv_strlen(s); + len = static_cast<int32_t>(uprv_strlen(s)); if(target!=NULL) { uprv_strncpy(target,s,uprv_min(len,targetCapacity)); } @@ -554,7 +554,6 @@ static const USystemParams systemParams[] = { #endif { "uconfig.internal_digitlist", paramInteger, "b", 1}, /* always 1 */ { "uconfig.have_parseallinput", paramInteger, "b", UCONFIG_HAVE_PARSEALLINPUT}, - { "uconfig.format_fastpaths_49",paramInteger, "b", UCONFIG_FORMAT_FASTPATHS_49}, }; diff --git a/Build/source/libs/icu/icu-src/source/tools/toolutil/unewdata.cpp b/Build/source/libs/icu/icu-src/source/tools/toolutil/unewdata.cpp index 84c9d526ba2..32b615c39b8 100644 --- a/Build/source/libs/icu/icu-src/source/tools/toolutil/unewdata.cpp +++ b/Build/source/libs/icu/icu-src/source/tools/toolutil/unewdata.cpp @@ -61,17 +61,17 @@ udata_create(const char *dir, const char *type, const char *name, length = 0; /* Start with nothing */ if(dir != NULL && *dir !=0) /* Add directory length if one was given */ { - length += strlen(dir); + length += static_cast<int32_t>(strlen(dir)); /* Add 1 if dir doesn't end with path sep */ if (dir[strlen(dir) - 1]!= U_FILE_SEP_CHAR) { length++; } } - length += strlen(name); /* Add the filename length */ + length += static_cast<int32_t>(strlen(name)); /* Add the filename length */ if(type != NULL && *type !=0) { /* Add directory length if given */ - length += strlen(type); + length += static_cast<int32_t>(strlen(type)); } diff --git a/Build/source/libs/icu/icu-src/source/tools/toolutil/writesrc.cpp b/Build/source/libs/icu/icu-src/source/tools/toolutil/writesrc.cpp index edff1f9e544..10b4ad246f6 100644 --- a/Build/source/libs/icu/icu-src/source/tools/toolutil/writesrc.cpp +++ b/Build/source/libs/icu/icu-src/source/tools/toolutil/writesrc.cpp @@ -22,13 +22,14 @@ #include <time.h> #include "unicode/utypes.h" #include "unicode/putil.h" +#include "unicode/ucptrie.h" #include "utrie2.h" #include "cstring.h" #include "writesrc.h" static FILE * usrc_createWithHeader(const char *path, const char *filename, - const char *generator, const char *header) { + const char *header, const char *generator) { char buffer[1024]; const char *p; char *q; @@ -71,20 +72,34 @@ usrc_createWithHeader(const char *path, const char *filename, } U_CAPI FILE * U_EXPORT2 -usrc_create(const char *path, const char *filename, const char *generator) { - // TODO: Add parameter for the first year this file was generated, not before 2016. - static const char *header= - "// © 2016 and later: Unicode, Inc. and others.\n" - "// License & terms of use: http://www.unicode.org/copyright.html\n" - "//\n" - "// Copyright (C) 1999-2016, International Business Machines\n" - "// Corporation and others. All Rights Reserved.\n" - "//\n" - "// file name: %s\n" - "//\n" - "// machine-generated by: %s\n" - "\n\n"; - return usrc_createWithHeader(path, filename, generator, header); +usrc_create(const char *path, const char *filename, int32_t copyrightYear, const char *generator) { + const char *header; + char buffer[200]; + if(copyrightYear<=2016) { + header= + "// © 2016 and later: Unicode, Inc. and others.\n" + "// License & terms of use: http://www.unicode.org/copyright.html\n" + "//\n" + "// Copyright (C) 1999-2016, International Business Machines\n" + "// Corporation and others. All Rights Reserved.\n" + "//\n" + "// file name: %s\n" + "//\n" + "// machine-generated by: %s\n" + "\n\n"; + } else { + sprintf(buffer, + "// © %d and later: Unicode, Inc. and others.\n" + "// License & terms of use: http://www.unicode.org/copyright.html\n" + "//\n" + "// file name: %%s\n" + "//\n" + "// machine-generated by: %%s\n" + "\n\n", + (int)copyrightYear); + header=buffer; + } + return usrc_createWithHeader(path, filename, header, generator); } U_CAPI FILE * U_EXPORT2 @@ -100,7 +115,7 @@ usrc_createTextData(const char *path, const char *filename, const char *generato "#\n" "# machine-generated by: %s\n" "\n\n"; - return usrc_createWithHeader(path, filename, generator, header); + return usrc_createWithHeader(path, filename, header, generator); } U_CAPI void U_EXPORT2 @@ -229,6 +244,68 @@ usrc_writeUTrie2Struct(FILE *f, } U_CAPI void U_EXPORT2 +usrc_writeUCPTrieArrays(FILE *f, + const char *indexPrefix, const char *dataPrefix, + const UCPTrie *pTrie, + const char *postfix) { + usrc_writeArray(f, indexPrefix, pTrie->index, 16, pTrie->indexLength, postfix); + int32_t width= + pTrie->valueWidth==UCPTRIE_VALUE_BITS_16 ? 16 : + pTrie->valueWidth==UCPTRIE_VALUE_BITS_32 ? 32 : + pTrie->valueWidth==UCPTRIE_VALUE_BITS_8 ? 8 : 0; + usrc_writeArray(f, dataPrefix, pTrie->data.ptr0, width, pTrie->dataLength, postfix); +} + +U_CAPI void U_EXPORT2 +usrc_writeUCPTrieStruct(FILE *f, + const char *prefix, + const UCPTrie *pTrie, + const char *indexName, const char *dataName, + const char *postfix) { + if(prefix!=NULL) { + fputs(prefix, f); + } + fprintf( + f, + " %s,\n" // index + " { %s },\n", // data (union) + indexName, + dataName); + fprintf( + f, + " %ld, %ld,\n" // indexLength, dataLength + " 0x%lx, 0x%x,\n" // highStart, shifted12HighStart + " %d, %d,\n" // type, valueWidth + " 0, 0,\n" // reserved32, reserved16 + " 0x%x, 0x%lx,\n" // index3NullOffset, dataNullOffset + " 0x%lx,\n", // nullValue + (long)pTrie->indexLength, (long)pTrie->dataLength, + (long)pTrie->highStart, pTrie->shifted12HighStart, + pTrie->type, pTrie->valueWidth, + pTrie->index3NullOffset, (long)pTrie->dataNullOffset, + (long)pTrie->nullValue); + if(postfix!=NULL) { + fputs(postfix, f); + } +} + +U_CAPI void U_EXPORT2 +usrc_writeUCPTrie(FILE *f, const char *name, const UCPTrie *pTrie) { + int32_t width= + pTrie->valueWidth==UCPTRIE_VALUE_BITS_16 ? 16 : + pTrie->valueWidth==UCPTRIE_VALUE_BITS_32 ? 32 : + pTrie->valueWidth==UCPTRIE_VALUE_BITS_8 ? 8 : 0; + char line[100], line2[100], line3[100]; + sprintf(line, "static const uint16_t %s_trieIndex[%%ld]={\n", name); + sprintf(line2, "static const uint%d_t %s_trieData[%%ld]={\n", (int)width, name); + usrc_writeUCPTrieArrays(f, line, line2, pTrie, "\n};\n\n"); + sprintf(line, "static const UCPTrie %s_trie={\n", name); + sprintf(line2, "%s_trieIndex", name); + sprintf(line3, "%s_trieData", name); + usrc_writeUCPTrieStruct(f, line, pTrie, line2, line3, "};\n\n"); +} + +U_CAPI void U_EXPORT2 usrc_writeArrayOfMostlyInvChars(FILE *f, const char *prefix, const char *p, int32_t length, diff --git a/Build/source/libs/icu/icu-src/source/tools/toolutil/writesrc.h b/Build/source/libs/icu/icu-src/source/tools/toolutil/writesrc.h index fdcf1f9a6b4..35ba256793c 100644 --- a/Build/source/libs/icu/icu-src/source/tools/toolutil/writesrc.h +++ b/Build/source/libs/icu/icu-src/source/tools/toolutil/writesrc.h @@ -23,6 +23,7 @@ #include <stdio.h> #include "unicode/utypes.h" +#include "unicode/ucptrie.h" #include "utrie2.h" /** @@ -30,7 +31,7 @@ * Writes a C/Java-style comment with the generator name. */ U_CAPI FILE * U_EXPORT2 -usrc_create(const char *path, const char *filename, const char *generator); +usrc_create(const char *path, const char *filename, int32_t copyrightYear, const char *generator); /** * Creates a source text file and writes a header comment with the ICU copyright. @@ -76,6 +77,33 @@ usrc_writeUTrie2Struct(FILE *f, const char *postfix); /** + * Calls usrc_writeArray() for the index and data arrays of a UCPTrie. + */ +U_CAPI void U_EXPORT2 +usrc_writeUCPTrieArrays(FILE *f, + const char *indexPrefix, const char *dataPrefix, + const UCPTrie *pTrie, + const char *postfix); + +/** + * Writes the UCPTrie struct values. + * The {} and declaration etc. need to be included in prefix/postfix or + * printed before and after the array contents. + */ +U_CAPI void U_EXPORT2 +usrc_writeUCPTrieStruct(FILE *f, + const char *prefix, + const UCPTrie *pTrie, + const char *indexName, const char *dataName, + const char *postfix); + +/** + * Writes the UCPTrie arrays and struct values. + */ +U_CAPI void U_EXPORT2 +usrc_writeUCPTrie(FILE *f, const char *name, const UCPTrie *pTrie); + +/** * Writes the contents of an array of mostly invariant characters. * Characters 0..0x1f are printed as numbers, * others as characters with single quotes: '%c'. diff --git a/Build/source/libs/icu/icu-src/source/tools/tzcode/icuzones b/Build/source/libs/icu/icu-src/source/tools/tzcode/icuzones index 168a86f9181..c1ded7ba3ed 100644 --- a/Build/source/libs/icu/icu-src/source/tools/tzcode/icuzones +++ b/Build/source/libs/icu/icu-src/source/tools/tzcode/icuzones @@ -103,27 +103,75 @@ Link America/Regina Canada/East-Saskatchewan # removed from backward in 2017c # Europe/Dublin data from the tz database. # -# Rule NAME FROM TO TYPE IN ON AT SAVE LETTER/S -# Rule Eire--ICU 1971 only - Oct 31 2:00u 0 GMT -# Rule Eire--ICU 1972 1980 - Mar Sun>=16 2:00u 1:00 IST -# Rule Eire--ICU 1972 1980 - Oct Sun>=23 2:00u 0 GMT -# Rule Eire--ICU 1981 max - Mar lastSun 1:00u 1:00 IST -# Rule Eire--ICU 1981 1989 - Oct Sun>=23 1:00u 0 GMT -# Rule Eire--ICU 1990 1995 - Oct Sun>=22 1:00u 0 GMT -# Rule Eire--ICU 1996 max - Oct lastSun 1:00u 0 GMT +# +# tzdata2018e suggests to use alternative zone definitions for use GB-Eire +# and EU rule after Oct 27, 1968. (Yoshito 2018-05-04) +# + +# Zone NAME GMTOFF RULES FORMAT [UNTIL] +Zone Europe/Dublin--ICU -0:25:00 - LMT 1880 Aug 2 + -0:25:21 - DMT 1916 May 21 2:00s + -0:25:21 1:00 IST 1916 Oct 1 2:00s + 0:00 GB-Eire %s 1921 Dec 6 # independence + 0:00 GB-Eire GMT/IST 1940 Feb 25 2:00s + 0:00 1:00 IST 1946 Oct 6 2:00s + 0:00 - GMT 1947 Mar 16 2:00s + 0:00 1:00 IST 1947 Nov 2 2:00s + 0:00 - GMT 1948 Apr 18 2:00s + 0:00 GB-Eire GMT/IST 1968 Oct 27 +# The next line is for when negative SAVE values are used. +# 1:00 Eire IST/GMT +# These three lines are for when SAVE values are always nonnegative. + 1:00 - IST 1971 Oct 31 2:00u + 0:00 GB-Eire GMT/IST 1996 + 0:00 EU GMT/IST + +Link Europe/Dublin--ICU Eire--ICU + +# +# tzdata2018e also added another instance of negative DST to +# Europe/Prague and Africa/Windhoek +# # Zone NAME GMTOFF RULES FORMAT [UNTIL] -# Zone Europe/Dublin--ICU -0:25:00 - LMT 1880 Aug 2 -# -0:25:21 - DMT 1916 May 21 2:00s -# -0:25:21 1:00 IST 1916 Oct 1 2:00s -# 0:00 GB-Eire %s 1921 Dec 6 # independence -# 0:00 GB-Eire GMT/IST 1940 Feb 25 2:00s -# 0:00 1:00 IST 1946 Oct 6 2:00s -# 0:00 - GMT 1947 Mar 16 2:00s -# 0:00 1:00 IST 1947 Nov 2 2:00s -# 0:00 - GMT 1948 Apr 18 2:00s -# 0:00 GB-Eire GMT/IST 1968 Oct 27 -# 0:00 Eire--ICU GMT/IST - -# Link Europe/Dublin--ICU Eire--ICU +Zone Europe/Prague--ICU 0:57:44 - LMT 1850 + 0:57:44 - PMT 1891 Oct # Prague Mean Time + 1:00 C-Eur CE%sT 1945 May 9 + 1:00 Czech CE%sT 1946 Dec 1 3:00 +# Vanguard section, for zic and other parsers that support negative DST. +# 1:00 -1:00 GMT 1947 Feb 23 2:00 +# Rearguard section, for parsers that do not support negative DST. + 0:00 - GMT 1947 Feb 23 2:00 +# End of rearguard section. + 1:00 Czech CE%sT 1979 + 1:00 EU CE%sT + +# Slovakia +Link Europe/Prague--ICU Europe/Bratislava--ICU + +# RULE NAME FROM TO TYPE IN ON AT SAVE LETTER/S +# Vanguard section, for zic and other parsers that support negative DST. +#Rule Namibia 1994 only - Mar 21 0:00 -1:00 WAT +#Rule Namibia 1994 2017 - Sep Sun>=1 2:00 0 CAT +#Rule Namibia 1995 2017 - Apr Sun>=1 2:00 -1:00 WAT +# Rearguard section, for parsers that do not support negative DST. +Rule Namibia--ICU 1994 only - Mar 21 0:00 0 WAT +Rule Namibia--ICU 1994 2017 - Sep Sun>=1 2:00 1:00 CAT +Rule Namibia--ICU 1995 2017 - Apr Sun>=1 2:00 0 WAT +# Zone NAME GMTOFF RULES FORMAT [UNTIL] +Zone Africa/Windhoek--ICU 1:08:24 - LMT 1892 Feb 8 + 1:30 - +0130 1903 Mar + 2:00 - SAST 1942 Sep 20 2:00 + 2:00 1:00 SAST 1943 Mar 21 2:00 + 2:00 - SAST 1990 Mar 21 # independence +# Vanguard section, for zic and other parsers that support negative DST. +# 2:00 Namibia %s +# Rearguard section, for parsers that do not support negative DST. + 2:00 - CAT 1994 Mar 21 0:00 +# From Paul Eggert (2017-04-07): +# The official date of the 2017 rule change was 2017-10-24. See: +# http://www.lac.org.na/laws/annoSTAT/Namibian%20Time%20Act%209%20of%202017.pdf + 1:00 Namibia--ICU %s 2017 Oct 24 + 2:00 - CAT +# End of rearguard section. |